diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..4cf7a6b58ce2050e1d0127ba5d95968c3cb4363b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +8B-Final/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/8B-Final/added_tokens.json b/8B-Final/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..7617541fda819214aeba044908f69a9dcce2d287 --- /dev/null +++ b/8B-Final/added_tokens.json @@ -0,0 +1,34 @@ +{ + "": 151674, + "": 151672, + "": 151668, + "": 151658, + "": 151666, + "": 151673, + "": 151671, + "": 151670, + "": 151667, + "": 151669, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/8B-Final/config.json b/8B-Final/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29acd26a09cb23172a4860fea4727244f9d51852 --- /dev/null +++ b/8B-Final/config.json @@ -0,0 +1,30 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.1", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151675 +} diff --git a/8B-Final/generation_config.json b/8B-Final/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e58a45f48b34e768f767bba26fb351540059f0fe --- /dev/null +++ b/8B-Final/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "4.51.1" +} diff --git a/8B-Final/merges.txt b/8B-Final/merges.txt new file mode 100644 index 0000000000000000000000000000000000000000..31349551d90c7606f325fe0f11bbb8bd5fa0d7c7 --- /dev/null +++ b/8B-Final/merges.txt @@ -0,0 +1,151388 @@ +#version: 0.2 +Ġ Ġ +ĠĠ ĠĠ +i n +Ġ t +ĠĠĠĠ ĠĠĠĠ +e r +ĠĠ Ġ +o n +Ġ a +r e +a t +s t +e n +o r +Ġt h +Ċ Ċ +Ġ c +l e +Ġ s +i t +a n +a r +a l +Ġth e +; Ċ +Ġ p +Ġ f +o u +Ġ = +i s +ĠĠĠĠ ĠĠĠ +in g +e s +Ġ w +i on +e d +i c +Ġ b +Ġ d +e t +Ġ m +Ġ o +ĉ ĉ +r o +a s +e l +c t +n d +Ġ in +Ġ h +en t +i d +Ġ n +a m +ĠĠĠĠĠĠĠĠ ĠĠĠ +Ġt o +Ġ re +- - +Ġ { +Ġo f +o m +) ;Ċ +i m +č Ċ +Ġ ( +i l +/ / +Ġa nd +u r +s e +Ġ l +e x +Ġ S +a d +Ġ " +c h +u t +i f +* * +Ġ } +e m +o l +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ +t h +) Ċ +Ġ{ Ċ +Ġ g +i g +i v +, Ċ +c e +o d +Ġ v +at e +Ġ T +a g +a y +Ġ * +o t +u s +Ġ C +Ġ st +Ġ I +u n +u l +u e +Ġ A +o w +Ġ ' +e w +Ġ < +at ion +( ) +Ġf or +a b +or t +u m +am e +Ġ is +p e +t r +c k +â Ģ +Ġ y +i st +-- -- +. ĊĊ +h e +Ġ e +l o +Ġ M +Ġb e +er s +Ġ on +Ġc on +a p +u b +Ġ P +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ +as s +in t +> Ċ +l y +ur n +Ġ $ +; ĊĊ +a v +p ort +i r +- > +n t +ct ion +en d +Ġd e +it h +ou t +t urn +ou r +ĠĠĠĠ Ġ +l ic +re s +p t += = +Ġth is +Ġw h +Ġ if +Ġ D +v er +ag e +Ġ B +h t +ex t += " +Ġth at +** ** +Ġ R +Ġ it +es s +Ġ F +Ġ r +o s +an d +Ġa s +e ct +k e +ro m +Ġ // +c on +Ġ L +( " +q u +l ass +Ġw ith +i z +d e +Ġ N +Ġa l +o p +u p +g et +Ġ} Ċ +i le +Ġa n +at a +o re +r i +Ġp ro +; čĊ +ĉĉ ĉĉ +t er +a in +Ġ W +Ġ E +Ġc om +Ġre turn +ar t +Ġ H +a ck +im port +ub lic +Ġ or +e st +m ent +Ġ G +ab le +Ġ - +in e +il l +in d +er e +: : +it y +Ġ + +Ġt r +el f +ig ht +( ' +or m +ul t +st r +. . +" , +Ġy ou +y pe +p l +Ġn ew +Ġ j +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +Ġf rom +Ġ ex +Ġ O +l d +Ġ [ +o c +: Ċ +Ġs e +Ġ le +---- ---- +. s +{ Ċ +' , +an t +Ġa t +as e +. c +Ġc h +< / +av e +an g +Ġa re +Ġin t +âĢ Ļ +_ t +er t +i al +a ct +} Ċ +iv e +od e +o st +Ġc lass +Ġn ot +o g +or d +al ue +al l +f f +( );Ċ +on t +im e +a re +Ġ U +Ġp r +Ġ : +i es +iz e +u re +Ġb y +i re +Ġ} ĊĊ +. p +Ġs h +ic e +a st +pt ion +tr ing +o k +_ _ +c l +# # +Ġh e +ar d +) . +Ġ @ +i ew +ĉĉ ĉ +Ġw as +i p +th is +Ġ u +ĠT he +id e +a ce +i b +a c +r ou +Ġw e +j ect +Ġp ublic +a k +v e +at h +o id +Ġ= > +u st +q ue +Ġre s +) ) +' s +Ġ k +an s +y st +un ction +**** **** +Ġ i +Ġ us +p p +on e +a il +== == +n ame +Ġst r +Ġ / +Ġ & +a ch +d iv +yst em +el l +Ġh ave +er r +ou ld +ul l +p on +Ġ J +_ p +Ġ= = +ig n +S t +. Ċ +Ġp l +) ;ĊĊ +f orm +p ut +ou nt +} ĊĊ +d d +it e +Ġg et +r r +om e +Ġ âĢ +ar am +c c +Ġ* / +E R +I n +le s +_ s +on g +i e +Ġc an +Ġ V +er v +p r +Ġ un +ro w +b er +Ġd o +l l +Ġ el +Ġs elf +at ed +ar y +Ġ . +' ] +u d +Ġ en +ĠT h +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ +t e +_ c +u ct +Ġa b +or k +. get +Ġ # +a w +res s +o b +N ame +ap p +[ ' +Ġal l +or y +it ion +an ce +e ar +Ġcon t +v ent +i a +Ġw ill +I N +ĠĠĠĠĠĠĠĠ Ġ +re turn +Ġ< / +d ata +) ĊĊ +R e +p le +il d +th er +Ġy our +" Ċ +( $ +Ġ out +) , +Ġh as +S tring +s o +Ġ up +a x +Ġde f +Ġb o +g e +al se +O N +p er +ic h +Ġb ut +Ġ Ċ +Ġ _ +_ m +ad d +que st +od el +s elf +er y +f t +en s +// // +a ke +. C +Ġg o +Ġf unction +Ġ K +iv ate +Ġ im +Ġcon st +. t +Ġ*/ Ċ +) ;čĊ +Ġv oid +Ġs et +ĠS ystem +c ri +( )Ċ +l i +ĉ if +. m +al ly +s et +e p +âĢĻ s +b o +de f +' ,Ċ +Ġm e +Ġ ! +at ch +" > +" ,Ċ +e c +ĠI n +p h +Ġ | +_ f +Ġv ar +en ce +I d +re e +in k +le ct +u g +et h +Ġel se +-------- -------- +con t +Ġs o +at ic +Ġl o +p ro +t on +s s +ow n +ab el +o int +ou s +el d +S T +T he +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +R E +" : +ol or +t p +e g +ke y +u de +ĠS t +ou nd +Ġa r +" );Ċ +en er +s er +b ject +ess age +f er +Ġm ore +ation s +ent s +Ġh is +Ġthe y +. S +Ġ Y +u se +n e +is h +ol d +_ d +i o +i eld +Ġp er +C ont +ing s +## ## +Ġd ata +Ġs a +e f +f o +Ġon e +en g +Ġd is +A T +Ġn ame +Ġtr ue +v al +le d +. f +Ġn e +Ġ end +. T +c re +ar k +lo g +E x +err or +_ id +ur re +ang e +Ġn ull +rr ay +Ġm y +p an +ic t +at or +V iew +L ist +ĉ return +âĢ Ŀ +Ġp re +Ġ x +cl ude +ar g +o v +. h +Ġ > +Ġthe ir +' ) +ir st +ic k +g h +L E +O R +Ġpr ivate +t em +čĊ čĊ +us er +Ġ ) +c om +. A +" ;Ċ +Ġ id +re ad +Ġwh o +_ b +" >Ċ +Ġt ime +Ġm an +r y +==== ==== +rou p +ro p +p ublic +v el +um ber +b le +Ġwh ich +******** ******** +Ġan y +Ġf alse +w e +Ġv alue +Ġl i +" ) +nd er +g r +Ġn o +p aram +f ig +.c om +Ġa pp +_ l +ion s +. D +ĠC h +Ġab out +Ġa dd +Ġs u +Ġstr ing +I D +Ġo ver +str ing +. l +our ce +_ C +] Ċ +Ġ qu +ĠS tring +c a +S E +Ġ ro +s h +u al +T ype +s on +n ew +er n +Ġa g +A R +] ;Ċ +] . +Ġ ? +ic al +Ġd es +ut h +i x +ay s +Ġt ype +' t +a ult +Ġin ter +v ar +. b +Ġp art +. d +urre nt +I T +E N +en c +( f +r a +v alue +ch o +ut ton +o se +Ġ! = +at er +à © +re ate +ol l +p os +y le +n g +A L +us ing +am es +Ġ{ čĊ +at es +el y +Ġw ork +Ġ em +in al +Ġs p +Ġwh en +.s et +ĠĠĠĠ ĠĠ +) :Ċ +t o +qu ire +ind ow +le ment +pe ct +as h +[ i +Ġu se +. F +pe c +Ġa d +o ve +ce ption +eng th +in clude +ad er +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +at us +T h +it le +r it +v oid +() . +( Ċ +Ġof f +Ġo ther +Ġ& & +' ;Ċ +m s +Ġbe en +Ġt e +m l +c o +n c +erv ice +Ġ % +** Ċ +an n +ad e +ĊĊ ĊĊ +lo ck +con st +pon se +Ġs up ++ + +d ate +Ġa cc +Ġh ad +Ġb u +ĠR e +Ġw ere +Ġf ile +Ġw ould +ĠâĢ ľ +v en +is s +Ġ our +c lass +r aw +Ġy ear +D ata +Ġv al +Ġs ome +f ter +y s +Ġ// / +rou nd +v iew +Ġp e +Ġth ere +Ġsa id +d u +o f +l ine +/ * +d uct +Ġh er +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +R es +Ġc o +Ġcom m +is e +m in +ĠĠĠĠ Ċ +# include +eth od +. P +ut e +Ġas s +I nt +as k +lo c +Ġli ke +od y +Ġle t +lo ad +Ġa m +ro l +Ġg r +y p +Ġal so +ĠI t +ur l +if ic +or s +_ P +_ n +ig h +Ġth an +C om +A N +U L +at ing +ĠTh is +re f +_ S +Ġst atic +ro ll +Ġj ust +Ġres ult +i an +id th +Ġthe m +) );Ċ +d er +re ak +C on +: // +u le +.. . +ar ch +em ent +Ġ< < +us h +en se +ar r +Ġint o +c ess +am p +i ed +um ent +Ġ \ +] , +w o +al s +Ġwh at +an c +V alue += ' +ol um +Ġp os +ag es +ay er +Ġs c +u es +" )Ċ +_ T +Ġl ist +( s +Ġc ase +C h +ĉĉĉĉ ĉ +//// //// +pon ent +Ġ z +Ġk n +le t +D E +re d +Ġf e +Ġ} ,Ċ +Ġ , +( t +Ġf irst +' );Ċ +w ord +Ġ import +Ġa ct +Ġch ar +C T +ĠT r +op le += { +ĉ f +i ent +c ent +. j +le ction +) )Ċ +Ġon ly +Ġpr int +m er +. W +o ck +Ġ -- +T ext +Ġo p +an k +Ġit s +Ġb ack +[ " +Ġne ed +Ġc l +Ġs ub +Ġl a +( ( +. " +O bject +Ġst art +f ile +( self +n er +e y +Ġus er +Ġ ent +ĠC om +it s +ĠC on +ou ble +ow er +it em +ver y +ĠW e +lic k +Ġ Q +ph p +t tp +' : +ic s +Ġu nder +Ġ* Ċ +. L +) ; +ic es +Ġre g +) čĊ +ĉ public +S S +Ġth en +re at +i ous +. G +e k +ire ct +he ck +cri pt +n ing +ĠU n +Ġm ay +ĠW h +B o +I tem +str uct +. st +re am +ib le +lo at +Ġor g +u nd +s um +_ in +.. / +_ M +Ġh ow +r ite +' Ċ +T o +w w +Ġpe ople +ind ex +. n +ht tp +( m +ect or +Ġin d +Ġj av +] ,Ċ +ĠH e +_ st +f ul +o le +) {Ċ +Ġsh ould +op y +el p +i er +_ name +ers on +I ON +ot e +Ġt est +Ġb et +rr or +ul ar +ã Ģ +Ġ Ð +b s +t ing +Ġm ake +T r +Ġa fter +ar get +R O +olum n +r c +_ re +def ine +Ġr ight +r ight +d ay +Ġl ong +[ ] +( p +t d +con d +ĠP ro +Ġre m +ption s +v id +. g +Ġ ext +Ġ __ +' )Ċ +p ace +m p +Ġm in +st ance +a ir +a ction +w h +t ype +ut il +a it +< ? +I C +t ext +Ġp h +Ġf l +. M +cc ess +b r +f ore +ers ion +) ,Ċ +. re +ate g +Ġl oc +in s +- s +tr ib +ĠI nt +Ġa rray +, " +P ro +( c +ess ion +> ĊĊ +Ġs he +" ] +ap h +Ġex p +ert y +ĠS e +Ġp ar +un c +E T +Ġre ad +pr int +Ġre l +Ġfor m +Ġd r +Ex ception +in put +Ġtr ans +#### #### +ord er +B y +Ġa w +it ies +u ff +pl ay +. add +ĠâĢ ĵ +Ġw ant +Ġcom p +ment s +Ġ| | +a z +b e +Ġn umber +Ġre quire +ĠE x +Ġc ol +Ġ key +em ber +Ġt wo +Ġs ize +Ġwh ere +U T +res ult +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ou gh +or ld +o od +u ch +at ive +g er +are nt +Ġ/ * +Ġar g +Ġwh ile +( this +Ġre c +Ġd if +St ate +Ġs pec +r ide +_ F +Ġlo ok +A M +il ity +et er +âĢĻ t +ĊĊ Ċ +ay out +---------------- ---------------- +ag er +Ġc ould +Ġb r +end s +u res +Ġkn ow +et s +ĠI f +ĠS h +. w +b ack +Ġs er +Ġ+ = +Ġf r +() );Ċ +Ġh and +I nd +UL L +I m +() ;ĊĊ +Ġm ost +Ġtr y +Ġn ow +rou gh +> čĊ +ack age +Ġh im +. _ +if y +Ġb reak +Ġ );Ċ +re n +# define +it t +Ġa p +ĉ c +( n +ĠY ou +: ĊĊ +- m +Ġe very +ust om +li ent +oc ument +cri ption +E rror +- b +Ð ¾ +] [ +tr ans +Ġp oint +Ġst d +Ġf il +T ime +Ġm od +Ġ -> +Ġ error +a h +Ġt ext +roll er +lo se +q l +Ġp ol +> < +. B +- c +Ġop en +Ġe st +ĠĠĠĠĠĠĠĠ Ċ +Ġn ext +I M +Ñ Ĥ +O T +à ³ +Ġf ollow +cont ent +ĠĠĠĠĠĠĠĠ ĠĠĠĠ +Ġin clud +H E +ĠR es +Ġh ref +Ð ¸ +Ġc ar +yp es +im age +U n +Ġbo ol +A D +Ġg ame +.F orm +row s +* / +vel op +.D rawing +Ġp ath +is ion +Ġe ach +ĠP l +_t ype +P ath +ne ction +Ġa v +' ). +Ġsup port +EN T +re m +" ). +Ġo wn +Ġc or +c ount +m iss +u ally +Ġm em +st d +i ence +se arch +" ĊĊ +F orm +Ġs ex +en ame +Ġs ign +Ġ et +ĠĠĠĠĠĠĠĠ ĠĠ +', ' +ĠA pp +Ġth ose +o ff +Ġ err +Ġs ystem +Ġbe st +c ode +Ġs ame +Ġd i +us s +Ġc reate +ath er +A rray +. in +f e +S ervice +U N +at s +Ġ Z +al th +Ġm ade +tr ue +A B +Ġm ark +r id +if ied +, čĊ +y n +p ress +Ġg roup +Ġf in +ĠL icense +F ield +eg er +Ġw orld +in ess +t y +Ġpro cess +( b +Ġc re +ar n +iv es +Ġm ain +ide o +_ g +A G +val id +im g +P I +Ġc olor +Ġre port +Ġt ake +ri b +O M +Ġd ay +Re quest +Ġs k +b ers +ĉ s +.A dd +o ot +Im age +Ġcom ple +ol lection +Ġto p +Ġf ree +A S +D e +ĠO n +I G +et a +D ate +Ġa ction +O ver +it or +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +n ot +Ġind ex +h er +ic on +O n +;čĊ čĊ +iv ity +m and +.W indows +O L +Ġre al +Ġm ax +l and +.. .. +r aph +Ġbu ild +le g +ass word +? ĊĊ +âĢ ¦ +o ok +u ck +Ġm essage +t est +iv ers +Ġin put +Ġar t +Ġbet ween +G et +ent er +g round +en e +à ¡ +.l ength +N ode +( i +C lass +f or +ĠâĢ Ķ +t en +o in +Ġ ke +u i +ĠI N +Ġt able +s ub +ĠL e +Ġhe ad +Ġm ust +//////// //////// +. util +Cont ext +Ġor der +Ġm ov +o ver +Ġcont in +Ġs ay +st atic +.T ext +Ġclass Name +pan y +Ġt er +he ad +r g +Ġpro duct +Th is +. âĢĿ +ĠB ut +lo y +Ġd ouble +s g +Ġpl ace +. x +m essage +Ġin formation +pr ivate +Ġo per +c ed +d b +"> +ater ial +ile d +Ġp ut +Q u +Ñ Ģ +un g +m ap +ĉĉĉĉ ĉĉĉĉ +Ġle vel +Com ponent +bo ok +cre en +_ RE +Ġcon fig +ã ģ +O r +. data +Ġd ocument +", " +trib ute +u x +L og +fer ence +p ost +_ e +Ġloc al +and om +ass ert +V al +lect ed +in a +atab ase +A dd +Ġcont ent +.p rint +s igned +r ic +." ĊĊ +Ġf a +! ĊĊ +- f +iv ed +Ġ quest +. ex +Ġf loat +Ġde velop +о Ð +M ap +ad ing +Ġpos s +U E +n amespace +_ O +ĉ b +.G et +> ( +j son +etail s +Ġto o +Ġext ends +ĠN one +Ġf ore +( String +form at +Ġg reat +int er +ca le +Ñ ģ +r on +iv ing +E nt +enc y +x t +o y +Ġmon th +Ġh app +Ġsup er +b ar +def ault +_ de +ord s +l n +( {Ċ +ĠI nd +as es +Ġt itle +Ġcont ext +o h +- p +E m +Ġm et +T est +Ġl ife +_ v +ĠU S +U I +oc ation +m d +Ġ[ Ċ +Ġ ] +s w +Ġin cre +s cript +ent ial +w ays +. de +Ġs rc +Ġc atch +ĠA meric +// Ċ +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ġp ay +pl it +âĢ Ķ +Ġc oun +ob j +.ph p +Ġch ange +eth ing +' re +ast er +lo s +l ation +ĠĠ Ċ +L e +à ¤ +( { +read y +ĠN o +Ġpos ition +Ġo ld +Ġbo ok +able d +b ug +H and +} ;ĊĊ +is play +av ing +Ġgo ver +Ġv ersion +S ystem +n ect +res ponse +St yle +U p +ang u +Ġth ree +in it +er o +Ġl aw +end if +Ġb ase +em ail +( l +_ V +Ġcon f +AT E +Ġd uring +t es +Ġcon sole +ĠP r +Ġs pe +v es +p ath +ial og +d ition +_t o +ard s +Ġagain st +et work +ĠP h +_ L +c ur +im it +W ith +Ġp ower +i um +' ;ĊĊ +Ġw om +le ft +our ces +at ri +ĠI m +ĠM an +or th +$ { +qu als +es e +_s ize +Ġis s +ot al +- g +i que +r ame +Ġw idth +er g +) ( +itt le +T R +ĠThe y +enc es +r l +on s +Ġl abel +. y +- t +up date +an el +s c +.t o +Ġpro ject +à ¼ +Ġe lement +Ġsu ccess +ĉĉ Ċ +.s h +r am +ch ed +() )Ċ +Ġ( Ċ +Ġd ate +Ġto t +_ ST +A ll +ific ation +ĉ var +Ġt ri +ch em +m y +Ġb ig +ĠA d +ĠA t +ot s +n um +A ct +Ġm ap +er a +co pe +. $ +, âĢĿ +Ġp op +Ġf ew +Ġl en +u id +et ers +u les +Ã Ń +s ource +http s +Ġd em +Ġe ar +######## ######## +Ġm atch +or ies +ac es +ĠC l +Ġn ode +ir c +loc al +un ity +} ;Ċ +Ġan other +< < +og le +Ġs it +ew ork +T E +. I +N S +olog y +ou ght +.C ont +> > +Ġc are +st ate +ĉ private +Ġe ffect +++ ) +_f ile +end ing +L ine +F or +i or +ĠS c +Ġf un +.S ize +ĉ else +] ) +st art +v ious +Ġ} , +our s +Ġle g +Ġs ervice +Ġs ince +ir on +L abel +Ġn on +Ġl os +ict ion +Ġf ull +act er +bo ard +g ress +Ġt urn +ith er +.s ize +Ġb ody +res h +et urn +( _ +y les +orm al +p i +Ġsom ething +! -- +u int +Ġpro du +Ġst and +Ġpro ble +Ġav ailable +m t +ĠB l +Ġ ... +Ġb lock +In put +Ġke ep +C ount +op en +Ġ[ ' +Ġth row +uild er +A ction +Ġth ings +Tr ue +Ġ url +ĠB o +print f +Ġre d +j s +.c reate +ĠO r +St atus +In stance +Ġcont rol +Ġcom e +Ġc ustom +loc ation +m odel +Ġ čĊ +Ġs ource +Ġe as +. out +] ĊĊ +one y +Ġaw ait +Ġpart ic +A P +ub lish +od es +_p ro +p ly +rit er +Ġpro v +Ġm ill +H T +] )Ċ +Ġch ang +Ġas k +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +Ġout put +Ġem ail +.p ush +Ġ} čĊčĊ +in ation +atri x +T able +u ccess +] );Ċ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġdis c +( [ +Ġb usiness +he ight +. html +t a +f ield +Ġrequire d +_ R +Ġgover n +} čĊčĊ +le x +. , +ĠS et +ur ch +// / +t s +a f +Ġm ight +ist ory +S tr +Ġne ver +Res ponse +ar se +ad a +ĠH ow +Ġ* ) +Ġ ; +Ġh ard +A d +Ġinter n +us ed +( data +m od +ann el +Ġn p +ug g +Ġ/ >Ċ +Ġcal led +b ody +Ġch o +( r +_s et +ir d +Ġ> = +Ġ} ;Ċ +Ġo ptions +ĠG ener +Ġhe ight +P oint +Y ou +et y +C lick +Ġsm all +Ġ ide +Ġacc ess +angu age +Ġprot ected +Ġj ob +ĠTh ere +D ef +Ġadd ress +Ġu int +N ot +o o +ap s +< div +ain ed +at ur +Ġs um +- w +ĠD ate +Ġl ittle +Ġf ri +Y PE +Ġp ort +e h +pr ing +_p ath +Ġst atus +a im +bo ol +Ġap pe +Ġo s +. name +ens ion +_ G +Ġup date +Con fig +a ff +ER R +Ġ< = +at ely +# if +u ction +ĠT e +Ġl ink +ĠU ser +.f ind +. org +m e +Ġg iven +O ut +# endif +Ġbet ter +P age +Ġfe el +en n +M L +Ġal ready +Ġinclud ing +o ogle +r u +ic ally +pro p +le an +out er +Ġal ways +ord ing +I f +or age +Ġp arent +v is +ĉĉĉĉ ĉĉĉ +Ġg ot +st and +Ġle ss +/ s +ĠA ss +ap t +ire d +ĠA dd +Ġacc ount +p loy +Ġd er +res ent +Ġl ot +Ġval id +ĉ d +Ġb it +pon ents +Ġfollow ing +_ ex +S ON +Ġs ure +oc ial +Ġp rom +ert ies +he ader +.p ro +Ġbo olean +Ġse arch +k en +Ġor ig +Ġ er +E d +E M +a ut +l ing +al ity +By Id +b ed +ĉc ase +eth er +pos it +Ġinv est +ĠO R +Ġs ays +miss ion +AM E +Ġtem p +o ad +Ġre st +in fo +Ġinter est +A rg +Ġper form +pon s +ĠV iew +Ġv er +l ib +( const +U til +List ener +ar ge +Ġm ult +Ġd ie +Ġs ite +../ ../ +E L +Ġval ues +Ġ} )Ċ +p en +N o +ic ro +Ġbe h +Ġ' ./ +ac y +re c +() -> +ĉ ĠĠĠ +" )) +Cont ent +_ W +ple ment +Ġw on +Ġv ideo +ad i +p oint +% % +Ġg l +erv ed +v iron +I F +ut ed +ã ĥ +' m +Ġc ert +Ġpro f +Ġc ell +ar i +Ġpl ayer +a is +Ġc ost +Ġh um +( R +Ġoff ic +k s +.t ext +at ures +Ġtot al +Ġ*/ ĊĊ +o pe +Ġst at +U M +Ġlo ad +ight s +Ġc lear +u ro +Ġte chn +up port +I R +Ġ row +Ġse em +Ġ q +Ġsh ort +ĠN ot +ip p +G roup +se ction +m ax +ir l +Ġover ride +Ġcom pany +Ġd one +" );čĊ +Ġg re +. Re +Ġbel ie +r ist +Ġhe alth +AN T +() ĊĊ +ĠB e +. value +ĠG r +ott om +Ġarg s +P T +st atus +f unc +um ents +- h +N umber +: čĊ +ĠL og +er ver +Ġ) ,Ċ +am ent +Ġob j +in c +Ġchild ren +ic y +I Z +and s +ab ly +Ġdist rib +Ġc ur +er ial +Ġd ays +re ated +re ct +- l +ir m +idd en +om b +Ġin itial +.j s +Ġ â +Qu ery +Ġon line +im al +. con +a u +U rl +cont rol +ire ction +Ġin stance +OR T +ĠF r +wh ere +Ġjav ax +Ġorg an +ap ter +Ġre ason +o ptions +ĠM ar +( a +Ġwith in +.âĢĿ ĊĊ +O DE +_ DE +ad min +end ed +Ġdes ign +ĠD ata +un e +ĠF ile +ro ot +Ġc ent +Ġa rr +_ add +l en +p age +, ' +_ str +Ġb ro +ab ility +ou th +/ c +p ose +irt ual +ear ch +_ url +arg in +H ttp +Ġs chool +av a +Ġcons ider +.l abel +ĠA rray +we b +o pt +.print ln +ul ation +Ġf unc +P L +Ġ" \ +ĠT ext +act ory +(f unction +n ull +Ġen g +d own +Ġin clude +ĠE n +ĠD r +Ġd b +! ! +s ide +Ġin it +quire d +ĠS he +C olumn +re act +Ġan n +Ġst op +Ġl ater +ĠTh at +ent ion +d f +U G +I LE +Ġc lient +ra ft +ff er +PO ST +el per +Ġlo ve +qu ote +ou d +Ġj son +Ġab le +Ġm en +A X +ĠC opyright +à ¶ +av ig +re q +C lient +} );Ċ +.C om +er c +il t +pec ial +_c om +ro om +. Name +Ġg ive +am b +i ke +Ġcon dition +cl ient +ator s +: " +Ġc opy +ut ure +ivers ity +ern al +{ { +ĠC an +ou nc +d o +Ġo cc +Ġapp ro +th ers +z e +Ġe ither +ĠF l +Ġimport ant +Ġle ad +at tr +AR T +E qual +Ġd a +et ch +ent ity +Ġfam ily +add ing +Ġo ption +Ġex ist +ic a +ĠO bject +' ve +v ers +ition al +out put +ĠTr ue +ĠO F +_t ime +Ġof fer +Ġ} );ĊĊ +H ER +eg in +" " +Ġw ater +Ġc he +ĠM y +ore d +Ġst ep +anc es +C K +A Y +à ¸ +str uction +( C +ou ch +St ream +act ive +am a +Ent ity +pro duct +() {Ċ +Ġgovern ment +ĠI D +aj or +A nd +Ġdis play +Ð » +Ġt imes +Ġf our +Ġf ar +Ġpres ent +ĠN S +Ġ\ Ċ +ue st +Ġb as +e cho +ch ild +if ier +Hand ler +Ġl ib +Prop erty +trans lation +Ġro om +Ġon ce +Ġ[ ] +cent er +================ ================ +Ġresult s +Ġcontin ue +Ġt alk +_ get +Ġg row +.s w +e b +ĠP ublic +O P +ec ute +ol s +Ġ ** +" );ĊĊ +Ġm ass +ure d +.c lass +om ic +Ġme an +ip s +Ġa ut +);čĊ čĊ +Ġun til +Ġmark et +Ġare a +u it +Ġl ength +ĠW ith +struct or +e vent +"> < +ĠS p +I V +Ġm us +if f +Ġk ind +a uthor +ound s +m b +_ key +w idth +posit ory +Ġl ight +u k +R ow +oh n +al f +viron ment +app er +ollection s +Ġs ide +_in fo +Ġex ample +im ary +Ġw r +Ġc amp +cri be +" / +Ġm iss +w ay +Ġb ased +Ġpl an +V is +om ain +un k +Ġaw ay +U P +< T +O S +i od +ĠM on +âĢĻ re +Ġli k +à § +iv ely +. v +im er +iz er +S ub +Ġbut ton +ĠU p +Ġexper ience +C L +Ġre nder +_ value +Ġn ear +UR L +al t +Ġcoun try +ib ility +() ,Ċ +e ad +Ġa uthor +Ġspec ific +b ase +( name +on es +ĠD o +Ġal ong +y ear +Ġexp ress +. ' +en v +Ġbeg in +Ġso ftware +Ġim p +Ġw in +ó n +Ġth ing +Tr ans +ĠT HE +Ġ< ? +Ġwh y +Ġdoes n +i j +g ing +ĉ g +Ġs ingle +off set +ar ning +og raph +le y +_c ount +Ġan al +cre ate +/ m +ĠR eg +un ch += $ +is k +Ġright s +( M +Ġ"" "Ċ +ap er +.m odel +Ġp o +em pty +art ment +Ġa nt +ĠWh en +Ġwom en +ĠE d +Ġse ason +Ġde st +à £ +( h +Ġposs ible +Ġse ver +Ġb tn +Ġdid n +Ġs ent +Ġen c +Ġcomm and +Ġ ],Ċ +_ x +Ġre cent +ol ution +v ector +ĠB y +ĠM ay +ĠA ct +» ¿ +Ġm oney +IN T +bs ite +ĉ p +. čĊ +ï »¿ +s l +atter n +ĠC lass +Ġto ld +ud io +c urrent +Ġe qu +Ġa uto +ĠSt ate +d a +ms g +)) ;ĊĊ +Ġwork ing +Ġqu ery +ĠB r +Ġw indow +a uth +on ly +ĉ t +Ġle ast +ag n +Ġex pl +it ter +ar ing +Ġc olumn +ĠGener al +": " +er al +ri or +Ġrec ord +I B +E X +Ġd at +Ġm aking +u ed +ĠC ar +em p +" . +ĠM ed +Ġc lose +Ġper cent +Ġp ast +( g +: ( +Ġw rite +Ġm ove +Ġp at +Cont rol +.T o +Ġv i +*/ Ċ +in ate +' ll +ag ed +N ull +Ġspec ial +IZ E +Ġc ity +/* Ċ +ĠE ng +ix ed +in ary +p y +Ġe ff +ar io +Ġt ell +av or +Ġse lect +le vel +im um +op er +B uilder +I P +') ,Ċ +es c +Ġf ont +" ;ĊĊ +ĠA m +ish ed +ill s +Int er +O W +Ġcour se +Ġl ate +idd le +Ġam ount +Ġas ync +in o +c ul +Ġ ì +and le +_ user +Ġb en +ĠC al +Ġ$ _ +ĠR ep +Ġen ough +T oken +. user +( j +S c +W idth +n ow +at form +Ġlook ing +Ġh old +M odule +IT Y +v o +is on +.D ata +y c +Ġp ot +ĠTr ump +id ual +id es +r t +Ġprop erty +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +am ework +g o +Ġl ow +Ġpar a +Ġpr ice +ur y +Ġto day +ro y +Ġ' / +Ġpol it +Ġ' ' +ym b +P h +Ġad v +Ġatt ack +ĠS te +RO M +an a +Ġme ans +Ġst ory +id s +ak en +Ġme et +Ġm om +ĠâĢ ĺ +Ġ? > +Ġd en +ob ile +ch ange +ĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +ic i +n a +ĠF orm +Ġs ort +Se lect +p are +Ġth ought +_ con +Ġt ask +oc us +ĠD E +ĠM in +Ġo pt +ĉb reak +um er +K E +th en +Ġd et +ĠT est +port s +Ġre view +(' / +m ove +Ġsw itch +ER T +p atch +ann ot +ã Ĥ +Ġab ove +it ive +Ġquest ion +ĠQ u +ãĢĤ ĊĊ +g le +Ġw ord +Ġprov ide +ĠR eturn +Ġre search +ã o +u str +Ġp ublish +chem a +} } +ĠC ON +- in +all back +Ġco ver +\ \ +c olor +ĠI S +Ġwh ether +im ate +is c +B ar +Ġd iv +B e +our n +Ġh aving +le m +pl ayer +ab s +am era +ne y +Ġex c +get her +pl ied +a o +[ $ +Ġ+ + +i pe +sh ow +/ d +[ : +ag ement +le v +_ ID +r ary +ad es +_ se +a use +Ġem ploy +Ġ*/ čĊ +Ġf re +Ġ' @ +Ġcomple t +Ġl arge +r al +\ x +Ġf ac +< String +Ġcre ated +up er +.st ate +Ġh ost +ener ic +/ b +( ! +wh ile +i as +B UG +Ġ );ĊĊ +Ġro le +Re g +ĠC olor +St art +Ġp orn +t op +Ġwe b +Ġde v +Ġde al +++ )Ċ +Int eger +pos ition +. on +Ġ( " +ä ¸ +Ġproble m +s v +Ġp ress +AB LE +AT ION +ĠSe e +an ch +Ġth ough +le ep +Ġ< !-- +Ġpoint s +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠ +. J +Ġ :: +p tr +D B +++ ;Ċ +.p ng +n ode +so ft +pon d +Ġe ver +-------------------------------- -------------------------------- +M enu +(' # +Ġs ervices +p g +} )Ċ +param s +Ġact ually +Ġ" / +Em pty +M ethod +Ġid ent +un ic +Ġmill ion +Ġa ff +st yle +Ġcon c +i os +ign ment +UL T +P r +" ;čĊ +Ġunder stand +u ary +Ġhapp en +Ġser ver +ĠC o +S C +Ġle s +Ġfile s +G rid +s ql +Ġof ten +Ġin fo +_ tr +s rc +on y +Ġsp ace +um b +Ġpass word +Ġst ore +, ĊĊ +ĠWh at +g ed +ĠF alse +U s +sw er +_ index +Ġform at +m ost +s m +N ew +Ġd etails +Ġpro b +ĠAN D +() čĊ +il ar +Ġ$ { +ry pt +.C ollections +$ this +ĠF ree +_ of +(f alse +d ated +Ġ> > +Ġf ace +CT ION +Ġs ave +Ġt yp +de v +(" # +AG E +cont ainer +ed it +Q L +Ġitem s +Ġs ocial +i en +ĠRe act +) .ĊĊ +Ġm ar +Ġre du +ĠR E +.p ut +Ġm ajor +C ell +n ext +Ġexpect ed +Ġy et +Ġin div +trib utes +at is +am ed +Ġf ood +S ource +( string +Ġ+ Ċ +it es +d r +Ġmem bers +Ġcom b +item s +ĠP er +T H += True +Ġb ar +_ SE +com m +( w +)ĊĊ Ċ +Ġs end +Ġin c +un signed +F A +Ġparam s +app ing +ro s +ug in +f a +Ġcon nection +Ġ} ;ĊĊ +Ġbe come +M ode +Ġe v +Ġdif f +ĠUn ited +He ight +ful ly +im ages +Ġm akes +Ġg lobal +Ġcont act +' :Ċ +Ġab s +а Ð +f loat +Ġex cept +ĠP ol +Ch ild +t yp +Ġcert ain +i ón +O UT +Ġim pro +ile s +Ġ-- >Ċ +ĠP art +val ues +os s +/ ** +il it +ĠE vent +cur ity +st er +Ġchar acter +Ġnew s +Ġ" , +Ġde vice +c el +log in +he et +Def ault +@ " +ĉ Ġ +c lick +( value +ĠA b +Ġpre vious +ERR OR +oc al +Ġm aterial +Ġbel ow +ĠCh rist +Ġmed ia +co ver +ĠU I +Ġf ail +Ġbl ack +Ġcom ponent +ĠAmeric an +Ġadd ed +Ġbu y +st it +Ġc ame +Ġde lete +prop erty +od ing +Ġc ard +rop s +Ġhttp s +Ġro ot +Ġhand le +C C +B ack +em plate +Ġget ting +_b y +m ail +_s h +. assert +ĠD ec +( true +Ġcom put +Ġcl aim +' => +ĠS ub +Ġa ir +op s +n av +em ents +( id +Ġent er +ang ed +E nd +Ġloc ation +Ġn ight +Ġdo ing +ĠR ed +l in +}ĊĊ Ċ +vid er +Ġp ick +Ġw atch +ess ages +Ġhum an +Ġd am +p end +d ir +Ġt ax +Ġg irl +re et +Ġbo x +Ġstr ong +( v +re l +Ġinter face +Ġm sg +f ect +_ at +Ġh ouse +Ġtr ack +' );ĊĊ +j e +ĠJ ohn +ist r +( S +ub e +Ġc e +itt ed +V ER +* ) +p arent +Ġapp lication +an y +.sw ing +Ġp ack +\ u +Ġpr act +Ġse ction +ct x +Ġun signed +.P oint +ĠO ne +Ä ± +ip le +a id +Ñ ĥ +V ector +by te +Ġw ait +Ġà ł +à ¥ +Ġto gether +Ġth rows +F O +' )) +h ost +is ing +. view +Ġter ms +fr amework +- r +Ġapp ly +Ġs ession +O ptions +ugg est +Ġo thers +w itter +Ġf und +In it +__ ( +ens or +G ET +Ġsever al +i i +[ j +I O +Ġtem plate +P osition +Ġe con +ach ine +Ġ il +.s pring +m ain +el t +im ent +Re c +m m +ĠUn iversity +urs or +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠ +G L +ict ure +ith ub +c er +c ast +F rom +a les +Ġsub ject +p assword +n y +Ġes c +.w rite +ï¼ Į +Wh at +. H +Ġh istory +ĠF e +Ġindiv idual +un it +Ġ-- > +Ġd u +I ST +Ġus ers +f s +f alse +un t +T itle +Ġm ot +Ġf uture +ach ed +Ġstart ed +Ġm ode +Ġ' < +_ array +Ġa x +'] ;Ċ +i res +Th ere +ug ht +t ml +pos ed +ic ult +Ġto ok +Ġg ames +Ġ} } +Ġ? >Ċ +Ġproduct s +I s +Ġb ad +ĠD es +.p ath +' ĊĊ +ĠP ost +av el +( : +Ġneed s +Ġkn own +F l +Ġex ec +Ġse en +um e +Ġb order +Ġl ive +tem p +P er +Ġvar iable +i et +ĠD ef +Ġg e +em e +_b ack +f irst +Ġprovid ed +//////////////// //////////////// +Ġfil ename +Ġh ope +ul y +a uto +f ind +_ string +b tn +it ude +At tribute +Ġyou ng +.t xt +Ġwe bsite +ĠP rop +Ġe y +> ();Ċ +ion al +AR R +iction ary +ur ther +. +t x +Ġp ur +u el +ymb ol +u ation +ang er +Ġback ground +ec ess +ef ined +.... .... +Ġdes cription +Ġrep resent +") );Ċ +press ion +row ser +Ġser ies +ward s +($ _ +a ise +Ġh ot +ac ity +ri es +action s +C reate +ad io +amp les +Ġorig inal +ens ive +f ont +st ream + using +.spring framework +ser ver +Ġb ill +AC K +il ename +Ġfr ame +Ġ= Ċ +Ed it +adi us +Ġd raw +ank s +Ġd eter +Ġcom es +_ int +Ġfore ach +ang le +Ġe lect +pect ed +He ader +ist ration +F alse +ĠG ame +Ġfil ter +Act ivity +Ġl arg +in ition +Ġ" < +is ed +Ġrem ove +ĠTr ans +m et +se e +Form at +Com mand +ĠE X +N one +Ġfr ont +A SE +ĠR ec +ound ation +Ġv o += \" +( * +Ch ange +.W rite +g roup +i ents +u y +******************************** ******************************** +Ġd ig +h r +( - +Ġg en +n umber +ve c +uro pe +ent ry +L L +Ġst e +Val id +'] , +_p aram +Ġse lected +Ġacc ording +ĠD is +Ġ util +B uffer +_ error +Ġass oci +_S IZE +Ġw or +Ġprint f +r ag + ł +D D +ĠV al +Ġact iv +E ng +et ime +Ġv irtual +a ign +a ur +ĠP res +ĠEx ception +Ġany thing +ĠO ff +Ġh ours +Ġw ar +Arg s +ag ing +Ġmodel s +ĠT ime +O b +am s +j oy +Ġear ly +. read +Ġc enter +ĠIn itial +Ġl anguage +l ength +x y +Ġs n +Ġin f +P ost +Ġag o +Ġeas y +_c ode +ĠAN Y +_ ch +Ġdown load +( T +av ed +âĢ ĵ +Ġstud ents +Ġf ig +l ight +x x +Ġbu ffer +ĠD ep +ĠM ath +IT H +Ġvar i +Ġd ue +F actory +Ġp or +Ġe p +ot ype +Ġcan not +Ġwh ite +< int +ter n +Ġreg ister +Ġpre d +cl us +_d ate +Ġ/ ** +Ġa uth +Ġ[ ]Ċ +Ġper iod +n own +Ġv ot +Ġs creen +' d +T ypes +Ġt mp +е Ð +ur al +Ġben ef +_ y +Ġn et +ĠSt ates +'] [' +ĠN e +ĠN OT +Ġn eg +Ġcomm on +s cope +Ġc red +g es +_T YPE +Ġs uggest +o om +.ĊĊ Ċ +Ġac cept +Ġr andom +er m +ĠV ector +w ith +T ER +( str +Ġres pons +Ġh it +.S et +gr id +ri a +Ġc lick +und le +C ase +ins ert +Util s +Ġ"" " +Ġim plement +at al +tem pt +tem plate +oc r +return s +Ġplay ers +us ers +ed ef +ĠTh ese +Ġam ong +Ġde b +h a +.get Element +Ġc irc +Ġan swer +Ġw alk +Ġt reat +ĠG e +ĠC reate +Ġa ge +Ġre q +O ST +ang ular +Ñ ı +Ġf ive +Ġdistrib uted +Ġfri end +T P +Ġc lean +ow s +.Control s +d is +Ġw ords +. io +z y +Ġhe ader +ĠC heck +âĢĻ m +j ust +h older +=" čĊ +. annot +Ġcol lection +' . +Ġsim ilar +Ġt aken +(" % +Or der +'] Ċ +-m d +ĠT H +ac ed +Ġis n +/ j +Ġs on +gr aph +ĠInt eger +Ġn ecess +re en +Ġ um +Ġ\ < +Ġmom ent +Ġbr ing +Ġind ic +ys is +Le vel +ver se +urre nc +_t est +Ġent ire +D own +Ġ}ĊĊ Ċ +( result +ĠRe ad +à ¨ +M od +Ġtry ing +") ,Ċ +Ġm ember +ĠC or +OD O +- control +un time +ĠS im +D ialog +pl ot +_ on +Ġph ys +} / +Ġn amespace +ĉ čĊ +ac c +Pl ayer +A RE +Ġf oot +Ġbo ard +p art +Ġs us +w ise +ĠM c +Ġp ush +AT A +Ġp lease +ri ed +we et +b it +id ed +V E +ĠS w +U B +Ġt ypes +ed ia +Ġc los +ace book +Wh en +Ġed it +ig ger +Ġen erg +Cont ainer +Ġph ot +ĠC ount +ĠE urope +.I s +ĠR uss +pe ed +ĠS tr +Ġp y +Ġc ult +Ġdef ined +cc ount +Ġob t +.L ocation +Ġth read +il le +Ġinst ead +str ong +ĠS ec +U RE +Ġide a +. se +em y +select ed +Con nection +ac ing +th read +.n ext +Ġc oll +Ġfil m +ist ic +Ġcomp et +Ġcon n +th ough +Ġcom pan +ock et +Ġte ach += ( +Ġph one +Ġact ive +de lete +tr ies +Ġm o +Ġde ath +} );ĊĊ +oc ol +W idget +Ġart icle +ro du +and id +Ñ ĭ +ĠC r +k a +() : +lo od +ĉĉĉ Ċ +Ġal most +Ġs ell +erv let +ri p +Un it +Ġapp lic +Ġcon nect +Ġfe ature +Ġv ia +' ), +Ġl im +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠG u +Eng ine +Ġen s +Ġen vironment +b lock +HER E +N ULL +g y +t ag +) ). +ex p +Ġcom pl +Ġinst all +Ġcomple te +que ue +atur al +Ġgener al +th on +Ġask ed +o res +( res +Ġres erved +S P +ĠâĢ ¦ +Å Ĥ +Ġsign ific +O ff +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠA g +ĠJ ust +ĠE rror +Ġin fl +ad ata +Ġ icon +ask s +' ' +_ LO +? . +ac count +Ġ( * +' )ĊĊ +r ap +_ var +ĠF OR +Ġpart y +ĠY our +c at +str y +. new +bo ot +ĠN ov +Ġv ector +Ġn ormal +Ġf urther +Re pository +Ġd atabase +att le +Ġmus ic +Ġspe ed +Ġd oc +pro cess +IG HT +.p arse +Ġt aking +Ġvi ol +ce ed +ĠA fter +Ġfor ward +Ġc rit +"/ >Ċ +ro t +Ġfa iled +ef ore +Ġconc ern +o e +b a +Ġs ender +Ġter m +h as +=" # +Ġpot ential +N um +Ġpublish ed +.c lose +ĠIm age +str aint +U D +ĠO b +Ġprob ably +l im +" :Ċ +olum e +Ġcon sum +ag ue +ens ions +Ġinvest ig +- year +') ; +-s m +Ġen joy +or ig +er ing +c p +le ased +ple ments +Ġreturn s +p at +B O +ĠH ouse +.L abel +Ġwe ight +igh b +Ġcondition s +Ġex ception +d escription +Ġtr ad +- to +Ġ{ } +Ġmod ule +EN D +. ap +.p rops +Ġcon structor +av es +Ġf avor +ĠN ow +; i +ĠM ain +_ k +er ies +âĢĻ ll +trans form +imest amp +P re +Ġm er +. res +st ant +L ocation +_N AME +Ġlos s +Ġ ĊĊ +n et +Ġeng ine +B lock +Ġiss ues +Ġpar se +ĠB ar +Ġst ay +ĠJ SON +Ġd om +air s +w ner +Ġl ower +", čĊ +ĠD em +uf act +Ġp s +Ġper fect +R L +Ġed uc +l s +em ory +ARR ANT +u ge +Ġex act +. key +al led +e ch +ie f +\ / +o ke +Ġfor mer +al loc +Ġs ix +id a +Ġm argin +Ġhe art +al d +p ack +.getElement ById +ĠW ARRANT +Ġr ather +Ġbuild ing +er man +lic e +Ġquest ions +iz es +le ge +irect ory +Ġj e +Ġc as +pro ps +ut f +Ġse curity +Ġhow ever +we ight +Ġins ide +Ġpres ident +Ch ar +ĠW ITH +.m ap +Ġgr aph +Ġt ag +_st atus +Ġat tempt +op p +us es +ĉ const +Ġr ound +, $ +Ġfri ends +Em ail +? > +Res ource +KE Y +os p +. query +ĠN orth +able s +ist rib +_c lass +el lo +Th at +Ð º +pecial ly +ĠPres ident +Ġcamp aign +Ġal t +are a +Ġch all +Ġop port +.C on +Ġenerg y +li ke +. string +ing ton +) * +y y +Ġprof ession +ir th +Ġse g +æ ľ +Ġh or +i ers +c an +Ġbeh ind +Pro duct +f g +ĠS k +.j pg +? : +] ;ĊĊ +Ġcall back +ĠH ttp +Ñ Į +l ong +M S +AT H +Ġr aise +Ġwant ed +row n +ut or +l t +] = +el ine +M A +Ġse par +c s +se mb +D is +bs erv +ĠW ill +Ġpol icy +Ġth ird +ph one +Ġb ed +/ g +. __ +ĠIn c +iz ing +.re move +in stance +.t ype +Ġs erv +E ach +Ġh ar +ĠM essage +( key +SE LECT +P os +)) ;čĊ +Ġre comm +Ġtr aining +ĠE nt +ĠCh ar +ic ht +(f ile +Ġp rior +G ame +Ġex it +Param s +.c ore +P C +n es +anc ed +( request +P assword +} >Ċ +Ġm ag +Ġre lease +Ġsh all +ud ent +ĠS outh +and o +: ' +.Tab Index +s k +ann er +is set +Ġout side +led ge +Ġ å +ĠR ob +Ġim m +! Ċ +ĠWe b +D es +B C +anc ial +R oute +D ec +fer ences +Ġp urch +ĠM odel +ct or +g n +_st art +_ un +. * +is es +Ġg round +Ġun ique +Ġbe aut +{ " +Ġp our +ĠO ct +Ġt ree +set s +_ res +') -> +_re g +(" \ +Ġby te +B l +Ġd ating +Ġm atter +ĠR em +Ġ' ../ +ĠA ug +ĠL a +Ġ$ ( +ourn al +i am +Ġshow s +w rite +Ġb all +Ġsim ply +Ġf ast +Ġmem ory +A SS +ĠO f +ov ed +ant e +a ul +ist ry +)) );Ċ +Ġf it +< string +Ġpolit ical +anc el +_ . +c ard +.c urrent +o ch +_ image +\ t +# Ċ +( L +Ġindu stry +com ing +Ġex tra +Ġreport ed +.st art +Ġres ources +Ġim g +fl ow +_E X +(n ull +ĠP re +Ġwr ong +inter face +Param eter +n ers +á » +t ure +ers ist +oun try +Ġseem s +al ance +de st +ĉ String +Ġm aint +Ġun it +act ers +ĠT R +if ul +export s +pro ject +App lication +leg ate +Ġt akes +ter m +Ġet c +ust er +Ġappe ar +add ress +Ġf em +h s +Ġh om +, - +Ġdiff icult +Ġcom ing +O pen +Ġset tings +ĠW ar +ĠTh en +Ġaut om +ĠF oundation +Ġqu ite +D escription +Ġb log +i qu +P S +_f ield +J son +SS ION +ĠS ch +ĠL O +Ġdes cri +Ġevery one +Ġpret ty +Ġlong er +Ġm enu +Ġcurrent ly +se c +Ġrelations hip +################ ################ +ĠM ap +as et +Ġparam eters +Ġcr ush +" čĊ +IL ITY +ig ration +Ġc out +t otal +Ġn ames +nd ef +") ; +ri end +yn amic +Ġeff ort +Ġact ual +Ġfield s +O UN +t ers +Ġf ix +_m odel +Ġc ases +C A +M y +Inter face +ĠS E +] ] +al le +ĠN ational +ĠArray List +in line +. V +ar a +ref ix +as c +Re ader +ĠÐ ¿ +ast ic +( () +C l +.annot ation +Ġperform ance +ail y +.to String +.n et +view s +. end +ay ers +l ate +ĠA pr +ed eral +'] ) +.b ody +Ġhigh er +_f l +c r +al ert +_n ode +ĠG oogle +Ġit self +A uth +urrenc y +Ġsignific ant +app end +Ġres pect +str ap +Ġun a +riter ia +P ORT +.ap ache +Out put +Ġpro gress +Ġm id +ĠM icrosoft +Ġres ource +ab lish +Ġd im +. load +.A pp +Ġd irection +Ġadd itional +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ +Ġnum bers +Ġcompan ies +.T h +Ġs ound +user name +Ġstat ement +Ġal ert +Ġcon tract +h ome +_l ength +.Com ponent +e v +. Ex +ï¼ ļ +" ; +ĠH igh +Ġ )ĊĊ +ĠP oint +op h +Ġl ines +-> _ +" )ĊĊ +o x +app lication +Ġ ]Ċ +ĊĊĊĊ ĊĊ +Ġso on +ction s +ing er +Ġj oin +ĠP e +Ġ ë +Ġl as +. E +c ss +/ or +ĠSt art +ĠT O +Ġsub s +con n +com ponents +DE BUG +qu are +F unction +end ar +. index +Ġf ill +Ä Ļ +Ġcho ose +h ow +ĠAmeric a +ass ets +-------- ---- +ĠV alue +Ġoff ice +Ġv eh +Ġtrans form +ĠAr t +Ġin de +Ġf n +Ġim plements +ang o +ple te ++ " +t mp +am ily +Ġhas h +miss ions +E ST +g t +Pro vider +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ġfl ag +Ġpartic ip +d en +ĠReturn s +Ġnot e +ü r +p m +ide os +Ġspec ified +ĠE N +est er +ol id +Ġup on +( std +ĉ v +Ġ' \ +u z +Ġv ert +Ġv ict +ĉ self +Ġ" $ +. k +Ġgroup s +g ithub +l ang +Ġm ut +T O +Ġv e +ĠP lease +;ĊĊ Ċ +ac cess +Ġ{ " +re a +Ġr isk +ick er +og gle +ĉ while +AN G +.s end +Ġwom an +Ġget s +Ġ ign +ĠI d +_ log +ON E +Ġe vid +ĠH ar +_s ub +Ġend l +Ġinclud ed +() );ĊĊ +ĠA p +ig r +Ġs em +ĠBl ack +d oc +_t able +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +- up +Ġca use +Ġ .. +Ġv an +_d ict +Ġf ocus +IN D +CE SS +.L og +Ġmult iple +id o +Ġreg ard +- M +and ler +our se +Ġde g +. U +Ġadd ition +Ġvar ious +Ġrece ive +е н +ĠH T +Ob j +D F +Ġincre ase +ĠO pen +] ; +Ġcomm it +? Ċ +ateg ories +at ory +sh ip +ĠM ich +Ġh tml +rom ise +Ġle ave +Ġstr ateg +av en +ĠCon sole +k nown +- n +_ LE +.com ponent +Ġb re +S ession +i ance +Ġal ign +typ edef +_ result +ĠW HERE +.s plit +Ġread ing +FA ULT +Ġc lo +Ġnot ice +_p r +ar ter +Ġlo ck +Ġstand ard +et ic +ell ow +Ġp adding +ĠH is +Ġst ates +_c ast +( P +a a +Ġintern al +e an +ĠP RO +ĠK ey +Ġes pecially +m ing +Ġc ross +Ġn ational +_ object +f ilter +Ġs cript +. update +_ i +ĠAss ert +/ core +%% %% +Ġproble ms +ist or +Ġ. = +Ġar ch +Ġwrit ten +Ġm ilit +M ENT +. ch +ca pe +ĠM us +_ config +ĠA PI +fo ot +Ġim ages +end l +. In +F irst +Ġpl atform +.pro t +O ption +st e +ĠT ODO +Ġfor ce +. cont +ĉ echo +ĠD av +P tr +( B +R T +ĠB ase +] [' +Ġann ounc +con sole +ĠP y +d s +. as +Ġpre vent +ap an +Ġ{ ' +} ' +Ġde ad +V AL +Q UE +**************************************************************** ******** +Ġch arg +R eturn +Ġf ul +d om +Ġr ules +Ġmod ify +Ġe val +h am +at ement +\ < +ul a += False +R A +Ġcont ains +Ġst ack +m ar +Ġ{ }Ċ +Ġund efined +A ss +ĠCh ina +ve y +* Ċ +Ġplay ing +) / +act or +Ġb ottom +li er +ĠN umber +Ġcou ple +D C +ĠS O +g or +.set Text +s uccess +com mand +F ilter +ĠO ur +_ item +Ġc tx +Ġro ad +V ersion +c ase +ur t +av ior +y ch +semb ly +ĠPro duct +Ġh eld +a fe +Ġinclud es +< quote +Ġa void +ĠF in +ĠM od +Ġt ab +an o +à ± +ipp ing +- e +Ġins ert +t arget +ch an +.M odel +IM E +\ Ċ +Ġm achine +av y +ĠN O +ĠInt er +Ġoper ation +mod al +T ag +] : +Ġprodu ction +Ġare as +Ġre n +_f rom +n bsp +Ġoper ator +m en +app ed +_p er +z en +(" . +.s ave +=" {{ +Ġt or +( response +Ġc andid +Ġcon v +a iled +ĠL ib +com p +ur a +ï¿ ½ +ĠH ere +Ġarg ument +h ood +Ġest ablish +ograph y +Ġon Click +amb da +Ġs ch +Ġmov ie +Ġse c +Ġact ivity +Ø § +Ġs ql +_ all +inc ip +Ġprovid es +Ġs ys +ack et +Ġwas n +Ġus es +ĠF unction +.g oogle +ĠRes ult +Vis ible +ag ma +el come +ĠS y +ĠC ent +AL SE +ac ión +EX T +Ġl icense +ĠL ong +Ġacc om +Ġab ility +. height +Act ive +olog ical +ol y +)) , +.S e +Ġparam eter +pr ite +AB ILITY +.s ervice +ĠG roup +_ query +ĠI tem +in ing +Ġj ud +im s +f ix +ind er +ag ram +Ġfunction s +Ġexper i +ĠE m +Ġro t +Ġp en +.b tn +ĠA S +#if def +Ġcho ice +ĠP age +_P RO +Q U +å ı +ant ity +Â Ń +word s +Ġread only +Ġf lex +prot ected +ĠAn y +Ġchar acters +enc ed +ĠJ uly +il er +C ard +ur ance +Ġre v +.e vent +al y +Ġwon der +ĠP ort +Ġleg al +ro le +Ġt en +Ġgo es +M P +wh ite +): čĊ +)) čĊ +Ġre ference +Ġm is +ĠPro ject +ick s +> & +C ON +Ġre pl +Ġreg ular +St orage +ram ework +Ġgo al +Ġt ouch +.w idget +Ġbu ilt +d es +P art +( re +Ġw orth +h ib +g ame +ĠÐ ² +ac ion +ĠWh ite +(t ype +( ` +Ġn atural +Ġin j +Ġcal cul +ĠApr il +. List +Ġassoci ated +ĉ System +~ ~ += [ +Ġst orage +Ġby tes +Ġtr avel +Ġs ou +Ġpass ed +! = +as cript +. open +Ġgr id +Ġb us +Ġrec ogn +A b +Ġh on +ĠC enter +Ġpre c +b uild +HT ML +ĠS an +Ġcoun tries +a led +t oken +k t +Ġqu al +L ast +ad ow +Ġman ufact +id ad +j ango +N ext +x f +. a +Ġporn o +ĠP M +er ve +it ing +_ th +c i += None +g s +Ġlog in +at ives +'] );Ċ +Ä ħ +Ġ ill +I A +child ren +D O +Ġlevel s +Ġ{ { +Ġlook s +Ġ" # +To String +Ġnecess ary +ĠĠĠ Ċ +c ell +En try +Ġ' # +Ġext rem +Select or +Ġplace holder +L oad +Ġre leased +O RE +En umer +ĠT V +SE T +in q +P ress +ĠDep artment +Ġprop erties +Ġres pond +S earch +a el +Ġre qu +ĠB ook +/ Ċ +( st +Ġfin ancial +ick et +_in put +Ġth reat +( in +Str ip +ì Ŀ +ç ão +Ġevid ence +)) ; +ĠB ro +Ġ[ ];Ċ +Ġ ou +b uf +S cript +d at +Ġr ule +# import +=" / +S erial +Ġstart ing +[ index +a e +Ġcon trib +s ession +_ new +ut able +o ber +Ġ" ./ +Ġlog ger +Ġrecent ly +Ġreturn ed +č čĊ +)) )Ċ +ition s +Ġse ek +Ġcomm unic +Ġ" . +Ġuser name +E CT +D S +Ġother wise +ĠG erman +. aw +Ad apter +ix el +Ġsystem s +Ġd rop +Ġstruct ure +Ġ$ ("# +enc ies +ann ing +ĠL ink +ĠRes ponse +Ġst ri +Å ¼ +ĠD B +æ Ĺ +and roid +sub mit +ot ion +( @ +.t est +ĊĊĊĊ ĊĊĊĊ +] ;čĊ +Ġdirect ly +Ġ" % +r is +el ta +A IL +) {čĊ +m ine +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +( k +b on +as ic +p ite +__ _ +M ax +Ġerror s +ĠWh ile +Ġarg uments +Ġens ure +R ight +-b ased +We b +Ġ- = +Ġint rodu +ĠIn st +ĠW ash +ord in +j oin +D atabase +Ġgr ad +Ġus ually +IT E +Prop s +? >Ċ +ĠG o +@ Override +RE F +Ġ ip +ĠA ustral +Ġ ist +View ById +Ġser ious +Ġcustom er +.prot otype +od o +c or +Ġdo or +ĠWITH OUT +Ġpl ant +Ġbeg an +Ġdist ance +() ). +Ġch ance +Ġor d +c ame +pr agma +Ġprot ect +rag ment +ĠN ode +en ing +Ñ ĩ +Ġr oute +ĠS chool +h i +Ġne ighb +A fter +lic it +Ġcon tr +Ġpr imary +A A +.Write Line +util s +Ġb i +R ed +.L inq +. object +Ġlead ers +un ities +Ġg un +on th +ĠDe v +F ILE +Ġcom ments +_l en +ar row +am ount +R ange +s ert +Grid View +Ġup dated +ĠM o +Ġin form +oci ety +al a +A ccess +Ġh ab +Ġc reat +_ arg +ĠJan uary +ĠD ay +") čĊ +up le +d ocument +gor ith +m enu +ĠO ver +b b +.t itle +_ out +Ġle d +ur i +Ġ? >Ċ +r un +Ġsc ene +( array +de vice +_t itle +ag on +] čĊ +ab y +Ġbe came +bo olean +Ġp ark +ĠC ode +up load +rid ay +ĠSept ember +F e +Ġs en +c ing +F L +C ol +ut s +_p age +in n +Ġim plied +al ing +Ġyour self +.C ount +con f +Ġa ud +_in it +. ) +Ġw rote +N G +. Error +ä » +.f or +Ġe qual +ĠRe quest +Ġser ial +Ġallow s +X X +Ġm iddle +ch or +à ¸ +erv al +.C olumn +read ing +Ġesc ort +ĠAug ust +Ġquick ly +Ġwe ap +ĠC G +rop ri +h o +Ġc op +( struct +ĠB ig +Ġv s +Ġfre qu +. Value +Ġaction s +Ġpro per +Ġin n +Ġobject s +Ġm atrix +av ascript +Ġon es +.g roup +Ġgre en +Ġp aint +ool s +y cl +enc ode +ol t +com ment +. api +D ir +Ġun e +iz ont +.p osition +Ġdes igned +_ val +av i +ir ing +t ab +Ġl ayer +Ġview s +Ġre ve +ra el +ĠO N +r ics +n p +Ġc ore +() );čĊ +M ain +Ġexp ert +ĉĉ čĊ +_ en +Ġ/ > +ut ter +I AL +ail s +ĠK ing +*/ ĊĊ +ĠM et +_ end +add r +or a +Ġ ir +M in +Ġsur pr +Ġre pe +Ġdirect ory +P UT +- S +Ġe lection +h aps +.p re +c m +Val ues +Ġ" Ċ +c olumn +iv il +Log in +in ue +Ġbeaut iful +Ġse cret +(e vent +Ġch at +um s +Ġorig in +Ġeffect s +Ġman agement +ill a +t k +Ġset ting +ĠC our +Ġmass age +ĉ end +Ġhapp y +Ġfin ish +Ġc amera +ĠV er +ĠDem ocr +ĠH er +( Q +con s +it a +Ġ' . +{ } +ĉ C +Ġst uff +Ġ :Ċ +ĠA R +T ask +h idden +er os +IG N +at io +ĠHe alth +ol ute +Ent er +' > +ĠT witter +ĠCount y +s cribe +Ġ= >Ċ +Ġh y +f it +Ġmilit ary +Ġsa le +re quired +n on +boot strap +h old +r im +- old +ĠD own +Ġm ention +cont act +_g roup +od ay +Ġto wn +Ġsol ution +u ate +ell ing +] -> +ot es +ent al +om en +osp ital +ĠS up +_ EN +Ġsl ow +SE SSION +Ġbl ue +ag o +Ġl ives +Ġ ^ +. un +in st +en ge +Ġcustom ers +Ġc ast +ud get +ï¼ ģ +ic ens +Ġdeter min +Se lected +_ pl +ue ue +Ġd ark +// ĊĊ +s i +ther n +ĠJ apan +/ w +P U +ĠE ast +ov ie +Ġp ackage +Ġn or +Ġap i +b ot +" ];Ċ +_p ost +ul ate +Ġcl ub +') );Ċ +Ġlo op +PI O +ion e +sh ot +In itial +Ġplay ed +reg ister +rou ght +_m ax +ac ement +m atch +raph ics +A ST +Ġexist ing +Ġcomple x +D A +.C h +.com mon +m o +Ġ' ../../ +it o +Ġanal ysis +Ġdel iver +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +id x +à ł +ong o +ĠEng lish +< !-- +Ġcomput er +EN SE +Ġp as +Ġr ais +H ash +Ġm obile +Ġo wner +F IG +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +th es +Ġat tr +w d +.t ime +aw n +Ġtreat ment +ĠA c +. View +im pl +m ore +p ass +Ġh a +.f rom +Ġle ading +FF FF +( error +. ui +at ar +ad ers +d ates +Ġz u +Ġfl ow +T arget +Ġinvol ved +Ġi o +par se +$ _ +he st +. int +- item +as y +S p +Ġsh ift +N T +Ġt f +_T R +. web +C S +Ġ} ) +Ġey es +_ z +' );čĊ +if orn +Ġ{ @ +Ġn ice +.l ist +ĠĠĠĠ čĊ +Ġf loor +Ġred irect +ĠU K +( [' +Ġw ish +Ġcap t +leg al +ĠI O +Ġst age +. String +ĠA fr +ig en +ĠS H +De lete +ell s +Ġsol id +Ġmeet ing +Ġwork ed +Ġed itor +in y +Ð ¼ +_ read +. Id +e ff +Off set +ch a +US ER +ĉĉ ĠĠĠ +ipp ed +Ġd ict +ĠR un +.h pp +Ġan g +x ml +im ple +Ġmed ical +_t oken +con nect +Ġh our +Ġcont roller +_m essage +U ID +G r +and ed +_C H +Ġbook s +Ġspe ak +am ing +Ġm ount +Rec ord +ĉ struct +.W eb +ond on +Ġ// Ċ +Ġf elt +.A uto +id ge +_p os +P R +Ġmod ern +C ollection +_m sg +C D +ĠL o +Ġsecond s +ib ly +.e quals +Ġintern ational +# pragma +oo th +W riter +i ate +Ġce le +ĠB it +iv o +iv ery +r d +HE CK +Ġc ache +.c ount +Ġro ll +.Re ad +RE D +Ġset up +izont al +model s +arg v +Ġconsider ed +=" ../ +set tings +ĠR el +Ġgrow th +Ġm ix +ĠWash ington +Ġpl t +ĠI M +á º +Ġturn ed +ĠDate Time +ĠW ed +( url +Ġ" - +Ġlet ter +As ync +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +ĠOct ober +_l ine +Ġatt ention +Ġcol lect +ĠH ash +Ġim ag +T ree +Ġsit uation +et te +_n o +IV E +Ġv on +.t arget +Ġknow ledge +Ġdr ive +.p ost +Ġb lood +Ġc it +pr imary +Ġconfig uration +te e +Ġph oto +is ode +Tr ace +Ġg ave +Ġsh ot +ĠA ir +Ġm other +pr ice +Ġmor ning +)) {Ċ +- x +Ġtr ade +Ġdes c +Ġ&& Ċ +Ġparent s +A pi +å Ī +t ed +w er +Ġ æ +Ġs y +ĠK e +Par ser +å ħ +anc y +Ġpie ce +iforn ia +to String +r an +id ing +PT ION +com es +/ lic +.c lient +E l +L ong +Ġprofession al +ru pt +v a +Ġcomplet ely +Ġpract ice +Ġse lection +R em +in i +Ġc am +RE E +Ġsit es +p a +AT US +Ñģ ÑĤ +arr ant +* ( +_ KEY +ĠB utton +ĠF riday +se qu +Ġre ader +Ġm essages +è ¯ +Ġbu f +K e +Ġn ov +H P +M sg +al ign +ar ily +Ġ' , +_w ith +Ġd as +Ġhe ard +at omic +ri al +) [ +Ġdis e +@ end +Ġg old +Ġf air +Ġsa les +. Button +str ict +s ave +Ġme asure +Ġ" + +ec ause +View Controller +ĠT able +.p aram +Ġdec ided +(( ( +IN FO +Ġopport unity +T e +IC ENSE +cc ording +k i +ĠU N +Ġcont ain +Ġman ager +Ġp ain +ĠF ire +rom e +Ġpl ans +F ound +l ay +ĠDec ember +Ġinfl u +à º +ren ch +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ġ +az ing +b rief +c all +wo od +Ġload ed +Ġgr and +/ f +im p +_ U +ST R +âĢ ¢ +Ġcred it +.C olor +or ge +QUE ST +Ġdiffer ence +ĠP C +w args +Ġp ub +und ay +Ġf ra +.m ax +Ġtri ed +ann els +s end +Ġreport s +Ġad ult +ä º +Ġcons ist +ĠSt reet +ĠPro gram +S QL +M atrix +ounc il +- A +ĉ w +Ġwho se +Ġrel ig +ĠS ex +Ġg ives +n one +.m essage +( G +.aw t +- right +ĠNov ember +ell ig +ut ive +Ä ĥ +over n +Ġeas ily +Ġide as +ĠÐ ½ +/c ss +ly ing +el le +C an +_c olor +оР² +Ġp air +ng th +Ġs plit +d rop +art y +on a +Ġcap ital +Ġhe ar +Ġex ists +ĉ log +em o +R un +o i +Ġpar ser +ĠM ethod +Ġeduc ation +[ k +Ġlib rary +> ";Ċ +_ UN +ĉ std +od ed +Ġcall s +h ere +R el +Ġbr and +back ground +g a +_add ress +_param s +C ategory +ĠInd ia +_e vent +Ġ ing +R ender +.c l +ump y +Ġp et +F C +ĠA nt +Ex t +Ġchar ge +en ed +gr ad +E O +Ġdep end +Ġ .ĊĊ +fr ame +Ġd f +Ġh uge +ĠP ART +ed s +; ; +ĠA M +Ġbas ic +ĠL et +lic h +Ġar m +Ġst ar +Ġf ederal +W ork +Ġcar ry +ĠIs rael +( obj +={ { +Ġs aved +Ġs yn +Ġconst ant +V ENT +Ġpos itive +Ġcon duct +Ġsk in +Ġear lier +Ġl ayout +ĠI P +O UR +Ġt im +styles heet +_ cl +ĠC ard +++ ){Ċ +Ġtem per +ĠDav id +ĉ try +.d art +Ġwant s +Ġp icture +Ġv ideos +ĠCom m +is ions +_M AX +M apping +- content +ĠE ar +- de +Ġpre m +br uary +Ġcom ponents +Ġthrough out +Ġp ull +Ġp ages +ent e +res pond +Ġg as +cript or +Ġed ge +Ġb ound +A CT +**** ** +Ġcre ating +ĠC H +Ġnull ptr +B r ++ ' +.c o +> :: +Ġle arning +.L ength +_S H +Ġpat ients +A IN +Ġk ids +Ġcom fort +Ġsh own +ug ins +ĠB ack +ell a +_C L +Ġl at +Ġdis patch +Ġclass es +. at +.b egin +Ġsuccess ful +b an +Ġobt ain +ĠS l +Ġl ack +iter ator +Th read +(s ize +Ġn one +.h as +_ X +s ort +n ap +p et +b in +ĠCan ada +The y +Ġd ans +ĠM at +< td +Ġh air +Ġ' ',Ċ +Ġc u +Ġlaw s +let ed +p ed +Ġp ow +Ġk new +_C OM +_ , +ĠM ag +id ents +( req +Ġ ), +- center +Ġw ide +ĠA uthor +st ants +Ġjob s +Ġm ath +et imes +Bo olean +Ġs cope +_ is +Ġme as +Ġkey s +el ay +Ġexact ly +'=> ' +ĠP aul +m as +ĉ print +(l en +f d +Ġ) ; +. Event +q li +ir it +ield s +om an +ĠT op +Ġv ote +Ġm ask +Ġthem e +- Ċ +Ġpro ps +Ġf ine +Ġwrit er +_ offset +c ar +Ġal tern +Ġc opyright +Ġdest roy +pp er +Ġgener ate +pp ed +âĢĻ d +ĠĠĠĠĠĠ Ċ +m ake +ĠSh ow +Ġb rowser +Ġfavor ite +Ġcare er +Ġhappen ed +( char +Ġrecomm end +Ġl iter +.f ilter +gr ade +Ġ £ +Ph one +om s +Ġn amed +- label +ip o +ĠO ther +Ġp anel +Ġro ck +S cale +ĉ assert +Ð ´ +Ġtr ust +fr ont +Ġdem on +A r +N et +Ġecon omic +foot er +Ġr ace +(n ode +ĠO ption +s plit +Ġphys ical +if est +Ġrem oved +. http +)) ,Ċ +Ġlook ed +' ; +d ing +g est +atur day +/lic enses +Pr ice +Ġd ro +Ġto wards +Ġun s +ĠC L +ĉ static +Ġ rows +Ġdef ine +.re place +Ġf ather +ĠDes ign +ass ign +m ut +De vice +D id +') )Ċ +omet ry +ay load +Ġh istor +ĠP aram +ĠBo olean +Ġn ature +Ġj s +Ġn ation +i h +Ġdis cover +se m +Hand le +ĉ r +ĠTe chn +Ġw all +{ $ +@ property +Ġ" ../ +Ġex am +.d raw +opp ing +Ġnear ly +Ġco ol +Ġinde pend +RE S +Ġhand ler +ĠMon day +Ġs un +St yles +ous ly +Ġ ĉ +v est +D isplay +( y +atic ally +Ġpred ict +y ing +Ġsom etimes +" ]Ċ +Ġdr ink +Ġb ul +ific ations +. insert +.re g +Ġtest s +Al ignment +Ġal leg +Ġat tribute +ĠN ote +Ġmy self +art s +N ow +Ġinterest ing +li ents +Ġpop ulation +ĠCal ifornia +" I +å ¹ +Ġgre ater +ues day +Ġth ous +Ġcost s +Ġla unch +\ Http +k er +b and +ĠPl ay +Ġb and +.sh ape +es ome +art icle +.r f +Ġw er +á s +em bers +us r +B A +ic an +et t +valid ate +ult i +Ġimmedi ately +z er +Ġfig ure +o es +ell er +irc le +ĠS ign +.d b +Ġr ank +By tes +Ġproject s +_re c +UL AR +A PI +ĠL ine +P ort +Ġp oll +Ġg iving +id ence +-- Ċ +Ġpl ot +ic ial +Ġw arrant +IT ION +ĠD ouble +Ġbill ion +gorith m +Ġequ ipment +D ATE +Ġ@ " +E E +Ġp le +i ation +Ġhead ers +Ġpro ced +.Component Model +ĠOb ama +Ġp a +ĠB est +im ately +.get String +. \ +mp loy +Ġr aw +_b lock +und red +" },Ċ +.Group Layout +Ġb rought +NS String +th row +cre ated +.N ew +_ view +C P +ep s +O p +Ġgr atis +Ġ' " +Ġinter view +"" "Ċ +Ġpart ial +Ġa ria +b ing +A uthor +Bo ok +ĠP at +um an +Us ers +pl us +ĠD irect +ven ue +al pha +UC CESS +ĠC all +Ġ );čĊ +im ated +Ġrem ain +Ġant i +ĠL ondon +Ġsaf ety +PO SE +o les +cont roller +By te +ĠCour t +ĠPh il +ĠAss oci +en a +å IJ +_ST R +co in +resh old +Ġb atch +_C lick +entic ation +> ';Ċ +ent y +Ġbegin ning +Ġz ero +ĠCon vert +Ġt err +Ġp aid +Ġincre ased +c atch +-s ize +act ivity +e quals +Ġque ue +Ġ" ' +ĠIntern ational +Ġf ür +urs day +Ġsc ient +all ow +ax is +Ġapp ropri +ed ge +Ġid x +S uccess +ent ifier +: \ +x is +Ġmax imum +ark s +Ġb irth +( index +Ġmay be +.p y +file s +Ġlim ited +_ check +lo ok +pl ies +Ġmov ement +'] . +Ġbro ad +ĠB E +ĠUn ityEngine +.c pp +ĠE very +Ad min +Ġf ans +p ared +Ċ ĠĠĠĠĊ +Ġfore ign +Ġp an +Ġt our +ĠOr der +Ġmov ing +Ġa uf +C all +c b +Å Ł +vent ory +ĠS ql +Ġful ly +Click Listener +W ORD +Ġannounc ed +) čĊčĊ +Ġagre ed +ri e +Ġe arn +_l ink +. array +(t ext +Ġmaterial s +, p +ff ff +v g +Ġ © +Ġun less +aj ax +LO G +Ġsex ual +Ġ\ " +- time +Ġco ach +Ġsupport ed +Ġphot os +if orm +.C reate +) ] +ri er +Ġd ialog +av er +ig e +) + +_id x +: [ +_m in +ĠC ong +Ġpress ure +Ġteam s +S ign +b egin +ri an +NE SS +L S +Ġimpro ve +ĠS unday +Ġdef inition +ig er +roll ers +Ġthink ing +T emplate +- F +Ġem erg +pl ates +ĠUS A +.set State +ĠAl so +re v +Ġen able +ĠC O +PE CT +Ġcon cept +) - +ĠâĢ ¢ +Ġset s +Ġmean ing +em on +ĠCon s +c mp +ed er +ann ed +icens ed +ĠS uper +Ġd aily +Ġmult i +_ u +Ġchall eng +_m ode +ĠP romise +Ġstr ict +j o +int on +( list +On ly +> { +Ġveh icle +í ķ +ĠPl ayer +ĠD el +Ġp ool +. url +nes day +();čĊ čĊ +Ġ" );Ċ +L ocal +. ");Ċ +Ġorgan ization +re nder +ĠApp lication +Ġsum mer +ex pected +N A +Ġr ap +_ obj +Ġsur face +ĠP UR +Ġ}, ĊĊ +Ġvariable s +(m essage +Ġop in +.b ack +а н +Ġwork ers +v m +C o +ught er +Ġm aster +Ġ" ", +Ġst ories +. User +Ġcele br +ines e +B S +ĠCom mand +ash board +Ġo g +k g +. image +.st yle +Ġstep s +ĠB en +( args +ĠP erson +, y +Ġofficial s +| Ċ +Ġsk ills +v c +Ġbuild er +Ġg ar +A ccount +ĠA uth +ç Ķ +'] )Ċ +ĠA T +n n +. Int +SS ERT +Ġeffect ive +LE TE +Ġto ols +AR D +Ġdig ital +D ouble +ĠF ind +R C +Ġin line +/ r +AR AM +AS K +Ġint ent +a ight +_add r +Ġrequest s +.f irst +Ġde bug +Ġsp ent +() ));Ċ +Å Ľ +Ġpr incip +Log ger +clud es +. use +Ġsur v +med ia +ĠFe bruary +ĠM ac +Ġmiss ing +Ġw ife +Ġtalk ing +ĠM ake +Ġc art +Ġloc ated +E nc +- a +ch ron +Ġc ards +Ġgu y +Ġp ers +ĠY es +ate ver +ĠA ng +ol ar +ĠE ven +Ġacc ur +ĠP ower +ĠG old +c lear +Pro cess +Ġrec ords +Ġk illed +.c lear +ĠWARRANT IES +Ġpur pose +pan el +J ECT +ÃŃ a +Ġex erc +W S +/ L +. exports +Ġ__ _ +Ġs in +S ervlet +Ġd é +.de lete +ro ke +S l +ug h +ear s +Ġpoint er +Ġh op +all ery +Ġo bs +co very +ĉ char +ĉĉĉĉ ĉĉĉĉĉĉ +ĉ def +oc ity +itch en +ul ations +ĠF IT +Ġ ). +straint s +vent ion +Ġrequ ires +ĠO per +M E +OUN T +al let +Ġn orm +I RE +ex as +Ġprogram s +Ġwe ak +' .$ +u ing +ĉ ĠĠĠĠĠĠĠ +Ġm il +Ġf irm +init ely +_VAL UE +ap se +atis f +Ġdem and +_m od +Ġdescri bed +Ġpl aces +V ID +Ġal one +Ġex port +Ġv ec +ĠM ax +Ġactiv ities +ict ures +g ener +Ġm a +Ĥ ¬ +Ġexpress ion +C allback +_ content +ĠM ost +Ġtest ing +E C +CH ANT +Ġad just +.Th reading +( ctx +Ġag ree +ig hest +Ġu i +ĠL aw +. Y +> ĊĊ +.ex ample +ber g +Ġmov ed +ĉ e +ĠS aturday +Ġpay load +Ä ĩ +) :ĊĊ +Ġbe y +ur er +< script +Ġs ymbol +Ġass um +Ġp ul +E ffect +Ġh undred +To ol +ak ed +con nection +Ġvo ice +Ġp d +Ġtrans action +Ġlink s +E rr +ĠInd ian +T C +atal og +n i +s ign +<< " +j i +y a +Ġdemon str +ul ated +. St +Ġinst it +Ġbo ost +Ġcell s +ol ic +.P ro +: , +"> \ +Ġth us +ĠReg ister +h ol +ĠCh inese +Ġpost ed +Ġm agn +ab ilities +Ġdise ase +Ġrem ains +ĠPro f +- form +Ġc in +org an +ic ate +Ġst ress +] * +Ġ ---------------------------------------------------------------- +_ context +or ry +Ġd ied +m at +Ġstart s +.M essage +Ġrun s +Ġgu ide +Ġwarrant y +ential s +d ict +ĠS ize +ul er +Ġrespons ible +_SE T +Ġcont aining +ĠPr ice +| | +F S +Ġem p +_b utton +( uint +Ġsu ff +p th +Ġdef initely +put e +Ġmarket ing +ĠW H +ĠS ie ++ = +OL OR +Ġcons ult +Ġs igned +Ġse quence +le e +Ġrequire ments +h y +Ex press +M T +se y +Ġ ult +å ® +ellig ence +Ġanal y +Ġd ress +eng ine +ĠG reat +ĠAnd roid +ĠA lex +m ode +D ictionary +.D ate +ä ½ +V ICE +Ġfam ilies +ĠRuss ian +ĠT imes +.c all +$ ( +Pro file +Ġf older +ch es +Ġleg is +_ row +un es +Ù Ħ +Ġ} ). +Ass ert +ag en +ĠH and +I ter +Ġbig gest +ore ach +Ġpol ic +Ġper missions +Ġshow ed +ĠE lement +Ġtop ic +âĢĶ âĢĶ +ro ad +ĠB ank +rec ord +Ġpart ners +ĠR ef +ess ions +Ġass ess +U ST +ĠPart y +pro du +L C +Ġ ul +. form +h ide +c opy +UT F +ĠSO FTWARE +čĊčĊ čĊ +ĠL in +un a +ug ar +Ġadmin istration +Ġopen ing +Ġsc an +Ġcontin ued +com ponent +.s p +Ġhapp ens +um my +ĠP R +.F ile +ĠDown load +Lo ading +d i +Ġwait ing +_A DD +T ab +.query Selector +Ġecon omy +ĠF rench +t xt +Ġf ant +_ ;Ċ +H older +S H +Ġn umpy +Ġst reet +Ġm ale +\ Model +ang ing +ĠB ill +Ġprevious ly +B I +ĠSec ret +Ġm ist +ĠF ield +up s +ĠPro cess +Ġke pt +ĠO T +Ġtrad itional +. i +am in +Ġhelp s +An y +orig in +ilt ers +j u +d esc +ĠA ccount +Ġ) čĊ +k top +ol ly +Ġf s +Ġ ê +Ġ ut +Ġcent ral +(t est +.A n +Ġs atisf +G R +ĠF ull +Ġhe at +ib er +Ġon to +m os +S chema +Ġfact ory +" .$ +aw s +St atement +(t arget +ĉ new +.b e +Ġg uest +Ġm al +AR Y +Ġre ached +Ġm ouse +Ġchall enge +ĉd ouble +ĠT em +Ġt error +Ġex tract +_T O +Ġsepar ate +Ġm ir +h elp +Ġcap acity +ĠProp erty +k an +_c reate +ĠL ight +.p arent +Ġunderstand ing +Ġeas ier +Ġ| = +Ġen h +Ġf at +Ġprot est +am m +_ AT +- of +il s +ĠO h +Ġps ych +Ġ$ . +ind s +Ġrel ative +sh op +sh ort +ĠS and +uest ion +Ġf ear +/ ĊĊ +. context +Ġschool s +Ġser ve +z one +_d b +Ġmajor ity +ex ample +Ġl ang +ĉ ĠĠ +Reg ister +end o +Ġprocess ing +_t emplate +- user +Ġe g +C OM +ĠBl ue +i ro +Ġrem ote +ĠI T +#! / +Ġred istrib +ra z +ĠS ince +ĠT ur +Back ground +== = +Ġref lect +Ġpro s +c md +Ġwh om +Com pat +ĠA re +Id entifier +ĠTh om +_ port +g u +Ġmon itor +r m +Ġpat ient +ver ter +Ġg ain +- ui +In st +Ġd ies +A rea +_f ilter +Ġgr at +Ġreal ity +ord inate +ol ved +Cont act +Ġcompl iance +_ or +ĠV ar +d l +Ġapp end +G ER +(m ax +.re nder +Ġd ynamic +ordin ates +_ options +_c olumn +Ġb atter +s pace +L a +ĠS ource +/b in +Ġd os +ĠBo ard +ĠTh read +ĠA L +( config +ĠM er +Ġm iles +_ header +ETH OD +iz z +Ġbenef it +Ġinteg r +(c urrent +ul o +. default +ĠD iv +Ġt on +o th +erv ation +ed om +Ġb aby +ce ived +.t op +rior ity +ĠL ocal +ri age +Ġattack s +Ġh ospital +Ġfem ale +ĠLog in +ĠFl or +Ġch ain +ash ion +Text ure +S ave +Ġf arm +.cont ains +.T est +Ġknow s +Ġgener ally +ip eline +Ġme ant +enc ia +Ġn icht +Ġcont ents +P M +ched ule +( line +C G +j ob +ĠRe al +u er +f irm +Ġ Ø +et ro +" `Ċ +Ġspe ech +Ġth r +fore ach +Ġw arn +ĉ l +Ġhe avy +< li +N e +Ġinvestig ation +M ath +- title +Ġch urch +Ġdes pite +ch ain +Ġwh atever +ar ian +f n +Ġm eta +} )ĊĊ +U FF +Ġregard ing +_S UCCESS +m es +ĠInt ent +Ġres olve +pos s +ir a +for ce +o ice +à ¢ +Ġp m +Ġup dates +A rr +Ġ Ñ +test ing +Ġto ward +nt ax +ë ĭ +Ġlist en +Ġgo als +Instance State +D r +Ġr are +Ġtr ail +Ke ys +C al +C ar +ĠPe ople +ĉ local +class es +Re ference +.for Each +em b +act iv +Ġpr im +red ict +Ġr ad +æķ ° +.B ack +Ġsp read +Ġc lock +Ġv ir +ed itor +Ġeffort s +Ġbr anch +Ġind ust +Ġmot or +Ġam b +Ġdat etime +Ġren cont +ĠChrist ian +ĠAmeric ans +f ull +Ġf mt +.m ain +Ġca used +_ update +ĠCont ent +AT CH +Ġb ath +ĠE ach +Ġr adio +ach ment +uz z +Sub mit +Ġre strict +ab in +ĠL oad +Ġext ension +Ġess ay +Ġh at +avi our +to Be +": [ +Ġoffer ed +Ġv ill +(d ouble +æĹ ¥ +b c +_f ree +ĠM iss +ĠB er +Ġ è +ĠL ike +Ġhelp ed +.get Name +_ AL +Ġsp irit +ĠAp ache +w s +Ġthere fore +( params +_ img +Ġpe ace +Ġinc or +ĠEX PECT +Ġmin or +ip es +ĉ data +select or +c ity +tr ie +.b ase +_f rame +Ġopen ed +/ json +L Y +n u +.D e +t f +m argin +.P arse +Ġp i +Ġe q +b d +Field s +ĠT ree +Ġb an +ist an +Ċ ĠĠĠĠĠĠĠĠĊ +ĉg l +Ġprodu ced +s ystem +M ark +_h ash +Ġb g +Ġconst it +ĠLe ague +Ġmiss ion +_ format +([ Ċ +clus ion +! " +Ð · +b reak +ĉs witch +Ġth er +Trans form +Ġfoot ball +- link +r oute +. auth +Ġb ag +ov ers +Ġen abled +Ġr ac +( I +C R +anc ing +Ġman aged +_ q +NG TH +Ġm ac +ĠA uto +ament e +Ġ' ', +.App end +Ġp in +. item +ack ing +Ġocc as +p erson +Ġt i +.Re g +Ġh aven +Ġg lass +Ġ" ) +_ char +res ource +Ġep isode +Ġ' _ +ĠE s +ĠEar th +Âł Âł +UP DATE +ĠS ou +u is +t ypes +Ġm as +Ġf av +Ġcon struct +_r ate +er as +Ġ| Ċ +rop erties +Ġext ernal +Ġap plied +Ġpre fix +ot ed +l ers +Ġc old +ĠS P +ĠCh urch +ĠOut put +los ed +ç ļ +ific ate +oper ation +her it +x FF +. env +_ err +os h +D irection +C ancel +ĠFr ank +Ġfind ing +. )ĊĊ +Ġr outer +ãĥ » +s es +Ġc row +== ' +Ġs and +Ġr id +it ure +Ġent re +Ġo bserv +Ġv ac +ð Ł +- T +A rt +n ight +. search +Ġex change +Ġdistr ict +. os +Ġdep artment +Ġdoc uments +Ġcent ury +ĠN ext +H ost +ĠK IND +Ġsus p +- P +re nd +. em +u ite +ist ers +( json +ĠAn n +w t +at i +ĠHT ML +wh en +D irectory +Ġsh ut +< a +ed y +Ġhealth y +Ġtemper ature +ĠG en +Ġmet al +Ġsub mit +ĠD O +Ġat tract +Ġ{ };Ċ +ĠW ord +Ġl l +Ġseem ed +k o +I ED +Ġl abor +.Cont ext +Ġas set +y ou +Ġc ars +ĠC olumn +Ġr é +Ġs quare +ĠNS String +âĢĿ , +ap es +.. .Ċ +Ġthan ks +( props +Ġt ick +Ġexper iment +Ġpr ison +t ree +- text +ĠIO Exception +-w idth +_ST ATUS +f ast +-b ody +- header +Ġgu ar +cre te +ĠT im +Ġclear ly +ĠRepublic an +Ġjust ify +и ÑĤ +ĉ ĠĠĠĠ +c ache +; // +Ġpres ence +Ġfact ors +Ġemploy ee +] )) +M ember +Ġselect or +b or +ĠM ex +çļ Ħ +ut ex +_t ag +ail ure +ĠN et +Ġre li +E G +Ġf printf +Ġte en +lo ss +Ġle aving +De legate +Ġbe at +Ġmin ute +sub scribe +Ġredistrib ute +Con stants +Ġcan cer +/ { +B L +Ġs pan +ĠCh ild +C enter +Ġear th +Y S +ĠLe vel +Ġse a +.s upport +.in ner +. Item +ill ing +ĠĠĠĠĊ ĠĠĠĠĊ +ĠL abel +ĠE st +( arg +bo Box +ĉf oreach +c os +F ailed +sw ers +Ed itor +r ont +ĠM P +ex pr +ĠL ife +Ġ? ? +ö r +Ġatt end +ĠQ ue +Ġspec ies +- D +Ġa us +Str uct +Ġadvant age +ost on +-b lock +in itial +C RE +Ġtr uly +Ġcomp are +or ney +Ġs pect +F ull +b es +Ġvis ible +Ġm ess +st ances +Ġcl oud +_v ersion +Ġf urn +ic ago +LO W +Ġtraff ic +Ġf ol +rypt o +Ġdecl ar +Ġsl ot +ĠEx t +ĠEng land +ĠU nder +Ġt a +let ter +Ġoffic er +ĠDon ald +Y es +_ json +IT ableView +ĠU SE +mploy ee +Ġopin ion +ĠA ut +b order +Ġad vice +Ġautom atically +is co +Ġm m +. vis +am l +Ġinitial ize +Ġ( { +Ġ ;ĊĊ +Ġgener ation +Ġb its +clip se +Ġun f +ut ors +pl t +Ġdel ta +est roy +is is +< br +Ġlimit ations +Ġend ed +ĠM ad +il m +Th ese +ĠMin ister +Ġch art +F ragment +Ġindepend ent +Y ear +Ġin str +Ġt ags +A VE +ĠAr ch +st op +Pro gress +Ġm i +Ġlearn ed +G e +Ġhot el +S M +T YPE +Ġc y +ERS ION +un ately +l imit +s el +Ġmov ies +Ġste el +o z +g b +ĠC amp +s ite +ĠLog ger +P LE +оР´ +. right +ĠC ore +Ġm ixed +st ep +Ġput s +s uper +R outer +. Http +ly ph +ĠColor s +Ġandroid x +. str +Ġinn ov +Ġde ck +' >Ċ +ap ers +] ( +cont inue +s pec +ĠR oad +AS H +ili ar +Ġcontin ues +Ġapp oint +Ġ# Ċ +ĠV ir +Ġ?> " +Ġb in +} ", +go ing +e ach +B D +ĠA ccess +D oc +ĠMan agement +B ER +ask et +.get Instance +Ġestablish ed +so cket +IN S +ĉv irtual +ĉ result +RE AD +_ height +ĠF ont +Ġ( );Ċ +_ html +Ġneighb or +l or +Ġg ather +Ġ} )ĊĊ +Ġid entity +Ġf ab +p adding +ĠR oute +Enumer able +à ´ +Ġfor ced +/j query +.ĊĊ ĊĊĊĊ +res ents +_ left +.P aram +ĉ throw +ĠH am +Ġevent ually +ac er +p ub +Ġtr a +un ique +d el +ĠFlor ida +ĠC lean +x a +Ġ · +Ġvalid ate +Vis ual +Ex pression +_f unc +m ember +ĉ h +tr l +ĉ G +nap shot +ĠProp Types +v in +] )ĊĊ +ow l +if ies +Ġ$ ('. +ĠCont ext +ĠTo ast +. Key +Ġoffic ers +/ n +s n +und efined +. items +ut ow +am age +Ġaccount s +ook ie +Se ction +ici ans +Ġad vis +( is +[: , +ĠFr ance +F unc +ic ious +Ġto k +Ch annel +ĠA D +_N UM +Ġtime out +lem ma +rem e +u j +.A l +uc lear +( os +(" < +[ Ċ +f etch +Ġb al +Ġgu id +- align +ĠW rite +ĠOn ce +utow ired +OD ULE +Ġp itch +C F +by tes +ĠCom mission +Ġincre d +P ER +_ response +ĠL os +par ser +Ġass ume +. Request +ĠT oken +_p osition +Ġn om +- term +Ġrem aining +i ostream +Ġpie ces +ap y +ĠL ess +r ange +umb n +pr ise +_ option +Im pl +k wargs +Ġbusiness es +Al ert +Ġpart ies +ĠCont ainer +ĠPr ivate +ĠPl an +Ġregister ed +Ġj our +ack er +ен и +/ > +ch at +se ct +Ġcre ation +olut ely +Ġinst ant +Ġdel ivery +ick en +y es +ĠFr anc +bl ing +end a +[ ( +_r ange +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠ +Ġsched ule +Con n +Ġthan k +x d +Ġh ook +Ġdocument ation +Param eters +H ello +v t +Ġart icles +Ġw est +def ined +. select +ok ens +ĠV AL +.f ile +res et +Ġmy s +ĠM A +] ), +Ġc ities +rel ated +å Ľ +Ġappe ared +Ġw id +.p anel +ĠIn s +. entity +Ġde cre +ĠL ou +(t ime +ĠTh ank +.create Element +Ġmention ed +oun ce +ĠT ry +ĠW all +/ images +ĠM enu +' čĊ +ĠE r +Ġcrit ic +ĠY ear +( param +Ġf lo +N N +oot er +Ġ ];Ċ +ĠA ff +" github +room s +Ġh yp +g lobal +Ġa vec +æľ Ī +Ġcomplet ion +Ġcon d +onym ous +( temp +Ġst ars +Ġre levant +Ġcover ed +Ġel im +_t ypes +( bool +Ġt u +_ex ists +Ġsec ure +Ġst ored +] / +x F +ĠCont roller +Ġm igr +M I +ĠD en +Ġann ual +U IL +- and +Ġcr ime +b el +Ġk itchen +@ g +_p h +ourn ament +ĠS ocial +ĠS pecial +log ger +Ġt ail +Ġun known +d ed +Ġapp rec +(d b +c f +Ġass ign +- out +ĠM ont +d p +w idget +Ġst one +- primary +. grid +Result s +az z +Ġda ughter +Ġcur r +Ġl in +Ġs outh +form s +ĠO UT +let te +ak s +ig ure +ĠE U +var iable +Ġb rief +ĠSc ott +Ġcon ference +and a +_ lock +or al +Ġe ine +OR S +//////////////////////////////// //////////////////////////////// +ess o +Ġr is +Ġg ender +est ic +L icense +( out +Ġm s +Se e +Ġwill ing +az e +Ġs ports +Ġy es +l u +Ġp urs +/j avascript +- pro +nav bar +_pro duct +/ bootstrap +Ġdr iving +Ġ Ä +Ġpro pos +ult ip +up lic +. email +Ġappro x +( cl +Ġwe ar +Ġrep ly +ass et +Ġ ice +Ġt x +k r +ĠGerman y +ĠGe orge +Ġc b +ĉ err +M ove +Ġpol y +vo ice +} " +Ġan imal +A v +ĠL ocation +Ġn ative +] [" +< double +Ġm ais +, int +Ġpre par +Ġinter val +plement ation +_ ERR +Ġb ug +> " +st at +Ġ} ,čĊ +< span +Ġfa ith +Ġ rom +pre v +ĠE lect +F ind +Ġg od +ot or +// ---------------------------------------------------------------- +orig inal +C pp +ĠSen ate +Ġposition s +Ġweap ons +Ġco ff +Ġpur poses +p ol +Ġim press +Ġanim als +. Entity +(n p +Ġmur der +Ġ` ` +fl ag +Ġsol utions +ĠAct ive +Ġb right +.d ate +Ġsit u +ï¼ Ī +. ID +Ġs ie +), čĊ +ak t +S pace +.d at +.index Of +h an +az ine +ĠZ e +Ġcr ash +( / +> = +Ð ± +iv a +.Auto Size +ĠL at +_ ext +Initial ize +.reg ister +OP Y +Ġre verse +_d is +'] [ +Ġprom pt +ont o +ĠJ ournal +r outer +Ġmys qli +# else +) " +-x s +let s +ph an +. LE +W ill +Ġaff ord +Ġsk ill +-t oggle +N C +B ind +T S +J ust +iter al +Y P +ĉ unsigned +Ġw ind +)) :Ċ +Ġw arning +ĠW ater +Ġd raft +Ġc m +Ġs am +Ġhold ing +z ip +ĠSc ience +Ġsup posed +G en +Ġdi et +< h +ĠP ass +v i +Ġhus band +� � +n ote +ĠAb out +ĠIn stitute +Ġcl imate +.Form at +Ġn ut +est ed +Ġapp arent +Ġhold s +f i +new s +C M +v ideo +': ' +D ITION +p ing +Ġsen ior +w a +-- >Ċ +_ default +ĠD atabase +re p +E SS +ner gy +.F ind +_m ask +Ġr ise +Ġk ernel +:: $ +. Q +Ġoffer ing +de cl +ĠC S +Ġlist ed +Ġmost ly +eng er +Ġblock s +ol o +Ġgover ning +\ F +Ġcon cent +.get Text +Ġm b +Ġocc urred +Ġchang ing +Sc ene +_C ODE +B eh +" The +Ġt ile +ĠAssoci ation +ĉ P +al ty +_ ad +od ies +i ated +Ġpre pared +poss ible +Ġm ort +TE ST +Ġign ore +Ġcal c +Ġr s +Ġassert Equals +Ġs z +ĠTH IS +. "Ċ +Ġcan vas +j ava +Ġd ut +VAL ID +.s ql +. input +Ġa ux +S up +Ġart ist +V ec +_T IME +.string ify +et ween +ĠC ategory +Ġ[ - +ĠDev Express +ĠJ ul +Ġr ing +. ed +Y Y +L et +Text Field +Ġfl at +_p rint +ĠOT HER +ad ian +Ġcheck ed +e le +Al ign +stand ing +Ġ[ ], +Ġl ab +uck y +ĠChrist mas +( image +.m odule +Ġl ots +Ġslight ly +(f inal +er ge +è ¿ +ĠPol ice +ĠR ight +Ġaw ard +ĠO S +Ġ{ }ĊĊ +Ġp tr +ov es +ic ated +еР¼ +Ġman age +olid ay +Am ount +ool Strip +t body +N av +w rap +B B +Ġwatch ing +ari os +Ġoption al +_ K +ĠL icensed +.M ap +T imer +ĠA P +ĠRe v +( o +, c +um in +eta iled +ĠH y +Ġbl ank +ag ger +ĠS elf +() [ +.m ake +ear n +ch annel +< pre +ble m +_p assword +_s p +ic ing +e z +Ġthe ory +ĠT er +, n +log o +ĠHT TP +() )) +.h andle +> ;Ċ +W orld +Ġpy thon +Ġl if +Ġtr av +Ġcon ven +com pany +ĠCl ub +V er +B tn +Ġz one +product s +ĠE duc +Ġver ify +ĠM il +on o +] );ĊĊ +EN CE +Ġpack et +Ġc er +Ġen umer +Ġpar s +form ed +Ġocc up +t re +Ġexerc ise +D ay +_s um +Ġask ing +apt ion +Ġord ers +Ġsp ending +ĠE RR +.D is +ĠU til +âĢľ I +\ ' +? ) +/ >Ċ +Ġem ot +Ġinflu ence +ĠAfr ica +att ers +Ù ħ +.s ession +Ġch ief +ĉĉĉĉĉĉĉĉ ĉĉĉ +Ġto m +clud ed +ser ial +_h andler +.T ype +ap ed +Ġpolic ies +- ex +- tr +bl ank +mer ce +Ġcover age +Ġr c +_m atrix +_ box +Ġcharg es +ĠB oston +P e +Ġcirc um +Ġfil led +Ġn orth +icture Box +ĉ res +è ® +Ġter min +Ġ[ â̦ +IRE CT +Ġb er +Ġ" ../../ +ret ch +.c ode +_c ol +ĠGovern ment +Ġarg v +ĠL ord +as i +Ex ec +ĉ let +vert is +Ġdiscuss ion +en ance +out ube +type of +Ġs erved +ĠP ut +ĉ x +Ġs weet +B efore +ateg y +. of +ĠM aterial +S ort +ON T +ig ital +Wh y +Ġs ust +Ġ ç +ab et +Ġseg ment +Ġ[ ],Ċ +ĠMus lim +Ġfind ViewById +c ut +_T EXT +ĠM ary +Ġlo ved +Ġl ie +ĠJ O +Ġis set +mon th +Ġpr ime +t i +ĠCar ol +U se +ĠP op +ĠS ave +Int erval +ex ecute +d y +ĠI ran +_ cont +ĉ T +Ġph ase +check box +we ek +Ġh ide +Ġt il +Ġj u +C ustom +b urg +/ M +T ON +Ġqu ant +Ġr ub +ix els +Ġinst alled +Ġd ump +Ġproper ly +( List +Ġdec ide +app ly +H as +Ġkeep ing +Ġcitiz ens +Ġj oint +p ool +S ocket +_ op +Ġweap on +gn ore +ĠEx ec +ott en +ĠM S +Ġ( - +ĠRe view +Ġex amples +Ġt ight +! ( +D P +ĠMessage Box +Ġphot ograph +UR I +é t +l ow +ĠGr and +.p ersistence +Ġmaint ain +Ġnum s +Ġz ip +ial s +ĠG ets +pe g +ĠB uffer +~~ ~~ +ra structure +ĠP L +u en +ob by +size of +Ġp ic +Ġse ed +Ġexperi enced +Ġo dd +Ġk ick +Ġproced ure +avig ator +- on +, j +ĠAl though +Ġuser Id +ac cept +Bl ue +IC olor +l ayer +av ailable +Ġend s +.t able +Ġdat aset +b us +Ġexpl ain +( pro +ĠCommit tee +Ġnot ed +] :Ċ +D im +std io +. ",Ċ +_s ource +ĠWe ek +ĠEd ge +Ġoper ating +Ġest e +i pl +ag ination +Ġpro ceed +Ġanim ation +.Model s +ĠW atch +i at +Ġopp on +/ A +Re port +Ġs ounds +_b uf +IEL D +Ġbu nd +ĉ get +.p r +(t mp +Ġk id +>ĊĊ Ċ +Ġy ang +Not Found +Ñ Ĩ +m ath +@g mail +ĠL IMIT +red ients +Ġv ent +avig ate +L ook +Ġrelig ious +Ġr and +ri o +( GL +_ ip +u an +ici ency +ĠCh ange +> čĊčĊ +ĠEnt ity +Ġrencont re +ĠR et +pl an +é n +BO OL +ur ies +tr ain +Def inition +======== ==== +z z +An imation +ĠO K +_m enu +.b l +_s core +Ġac ad +( System +Ġref resh +'=> $ +.G raphics +ament o +p id +t c +Ġt ips +Ġhom es +Ġf uel +â ĸ +_h elper +ĠĠ čĊ +ĠR oom +.C lose +_ attr +ĠM ount +ĠE v +ar ser +_t op +e ah +ĠDe lete +ãĢ į +u ke +Ġus age +ar ia +_de v +Ġtext ure +Ġconvers ation +e per +Be an +d one +non atomic +ĠSe cond +Ġshoot ing +_p re +Com ponents +Ġ] ĊĊ +__ , +stit ution +.Ch ar +> ();ĊĊ +Ġpresent ed +Ġw a +ok er +- ĊĊ +in er +Ġbe coming +Ġinc ident +At t +Ġreve aled +for c +Ġbo ot +.p age +Enumer ator +_ -> +Ph oto +Ġs pring +. ", +ĠD ictionary +B JECT +Ġloc ations +Ġs amples +Input Stream +ĠB rown +Ġst ats +qual ity +Ñ ħ +-d is +Ġhelp ing +Ġp ed +( se +ĠWh o +al ian +int ernal +Ġf t +> (). +-> { +Ġm ine +Ġs ector +Ġg ro +Ġopport unities +Ġà ¼ +Ġm p +Ġalleg ed +Ġdoub t +M ouse +Ab out +_p art +Ġch air +Ġstop ped +lo op +ent ities +Ġapp s +ans ion +Ġm ental +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠ +F R +Ġdef end +c are +Ġide al +/ api +ur face +Ġe le +ul ator +ĠR ights +angu ages +Ġfund s +Ġad apt +At tributes +Ġdep loy +opt s +Ġvalid ation +Ġconcern s +u ce +.n um +ult ure +il a +Ġc up +Ġp ure +.F ore +ĠHash Map +.value Of +as m +M O +Ġc s +Ġst ores +Ġ ************************************************************************ +Ġcommunic ation +m em +.Event Handler +. Status +_ right +.set On +S heet +Ġident ify +ener ated +order ed +Ġ" [ +Ġs we +Con dition +ĠA ccording +Ġpre pare +Ġro b +P ool +Ġs port +r v +ĠR outer +Ġaltern ative +( [] +ĠCh icago +ip her +is che +ĠDirect or +k l +ĠW il +key s +Ġmy sql +Ġw elcome +k ing +ĠMan ager +Ġca ught +) }Ċ +S core +_P R +Ġsur vey +h ab +He aders +AD ER +Ġdec or +Ġturn s +Ġr adius +err upt +C or +Ġm el +Ġin tr +( q +ĠA C +am os +M AX +ĠG rid +ĠJes us +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠ +.D E +Ġt s +Ġlink ed +f ree +ĠQ t +Ġ/** čĊ +Ġf aster +ct r +_ J +D T +.C heck +Ġcomb ination +Ġint ended +- the +- type +ect ors +am i +ut ing +Ġum a +X ML +U CT +A p +ĠR andom +Ġr an +.s ort +Ġsort ed +. Un +_P ER +it ory +Ġprior ity +ĠG al +ĠO ld +h ot +ĠD isplay +(s ub +_T H +_ Y +ĠC are +load ing +K ind +_h andle +, , +r ase +_re place +.add EventListener +ĠR T +Ġenter ed +g ers +Ġ ich +( start +/ app +Ġbro ther +M emory +Out let +Ġ utf +pre c +Ġn avigation +OR K +Ġd st +D etail +Ġaud ience +Ġd ur +Ġcl uster +un ched +Ġ ], +Ġcomfort able +. values +ĠT otal +Ġsn ap +Ġstand ards +Ġperform ed +h and +(" @ +å Ń +Ġph il +ib r +tr im +Ġfor get +Ġdo ctor +.Text Box +icon s +, s +ĠO p +S m +St op +ĉ List +ĉ u +Com ment +_V ERSION +.X tra +P erson +r b +LO B +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +ĠCent ral +IC K +ra q +Ġput ting +Ġm d +ĠL ove +Pro gram +B order +o or +Ġallow ing +a fter +Ġent ries +ĠMay be +] ). +ĠSh ort +) \ +.n ow +f riend +Ġpre fer +ĠG PIO +os is +ĠGame Object +Ġsk ip +Ġcompet ition +_m atch +lic ations +_CON T +.group Box +Ġal s +" We +_e q +l an +_ search +ĠMus ic +as is +Ġb ind +ĠIs land +r um +( E +Ġse at +V ideo +Ġa ck +ree k +={ () +Ġr ating +Ġrestaur ant +DE X +(b uf +pp ing +ual ity +Ġle ague +Ġfoc used +ap on +$ data +CL UD +CLUD ING +Ġabs olute +( query +Ġtell s +A ng +Ġcomm unities +Ġhon est +ok ing +Ġap art +ar ity +/ $ +_m odule +ĠE nc +. an +.Con fig +C re +Ġsh ock +ĠAr ab +I ENT +/ re +Ġre trie +ycl er +is a +ĠO rgan +. graph +Ġ í +ĠB AS +En um +Ġposs ibly +ÑĢ Ð°Ð +ĠJapan ese +Ġc raft +ĠPl ace +Ġtal ent +Ġfund ing +Ġconf irmed +Ġc ycle +/ x +G E +Ġhe aring +Ġpl ants +Ġm outh +p ages +or ia +ĠRem ove +_t otal +Ġo d +oll apse +do or +Ġb ought +Ġadd r +AR CH +_d im +dd en +Ġdec ades +RE QUEST +Ġvers ions +f ire +Ġmov es +f b +Ġcoff ee +.con nect +ĠR ow +Ġs chema +S cope +- Type +Ġfight ing +Ġret ail +Ġmod ified +T F +File s +n ie +_com mand +st one +Ġ ÑĤ +_ thread +Ġb ond +ĠDevelop ment +Ġp t +F ORM +ple t +Ġident ified +c pp +Ġc oding +ok ed +ĠM aster +ID TH +Ġres idents +red it +ĠPh oto += - +un te +ate ur +_ST ATE +ĠS ing +Ġshe et +. val +or se +Ġh ers +Ġdetermin ed +Com mon +Ġw ed +_ queue +P H +ĠAt l +cre d +/L ICENSE +Ġm es +Ġadv anced +.j ava +.S h +G o +k ill +f p +_set tings +Ġp al +Ġtr uck +Ġcomb ined +Ġ" ${ +ĠCor por +Ġjo ined +ĠJ ose +ĠC up +un s +est ival +lev ision +Ġbro ken +Ġmar riage +ĠWest ern +Ġrep resents +ĠT itle +Ġs s +.A ss +ongo ose +ient o +< >();Ċ +Ġabs olutely +Ġsm ooth +TER N +ĠUn less +W ord +Ġmer ge +ig an +ĠV ol +Ġn n +.get Id +ĠÐ · +Ġsex y +Ġseek ing +S ingle +. this +Ġk om +b ound +; " +Ġfont Size +_d f +Ġinj ury +( H +Ġiss ued +_ END +: self +Ġp atch +Ġle aves +Ġad opt +File Name +ãĢ IJ +Ġexec utive +ĠBy te +] ))Ċ +Ġn u +out ing +clud ing +- R +. options +Ġsub stant +av ax +ĠB UT +Ġtechn ical +Ġtw ice +Ġm ás +Ġun ivers +y r +Ġdr ag +ĠD C +Ġs ed +Ġb ot +ĠP al +ĠH all +forc ement +Ġa uch +.m od +not ation +_file s +.l ine +_fl ag +[ name +Ġres olution +Ġb ott +(" [ +end e +( arr +F ree +( @" +ĠD istrict +PE C +: - +P icker +ĠJ o +ĠĠĠĠĠ Ċ +ĠR iver +_ rows +Ġhelp ful +Ġmass ive +--- Ċ +Ġmeas ures +ĠR untime +Ġwor ry +ĠS pec +ĉ D +ãĢ ij +Ġ) {Ċ +Ġwor se +(f ilename +Ġl ay +Ġmag ic +ĠThe ir +ou l +st roy +ĠWh ere +Ġsu dden +Ġdef e +Ġb inding +Ġfl ight +ĠOn Init +ĠW omen +ĠPol icy +Ġdrug s +ish ing +(' ../ +ĠM el +pe at +t or +Ġpro posed +Ġst ated +_RE S +Ġe ast +ĠCON DITION +_d esc +Ġwin ning +fol io +M apper +ĠP an +ĠAn ge +.s ervlet +Ġcop ies +L M +Ġv m +å į +Ġd ictionary +S eg +el ines +ĠS end +Ġ iron +ĠF ort +.d omain +Ġdeb ate +Not Null +e q +ach er +l f +ĉf mt +Ġlaw y +Ä Ł +ĠM en +Ġtr im +( NULL +Ġ! ! +Ġp ad +Ġfollow s +"] [" +re qu +ĠE p +.g ithub +( img +et o +(' \ +S ervices +umbn ail +_m ain +ple ted +fort unately +Ġw indows +Ġpl ane +ĠCon nection +. local +u ard +} \ +== " +and on +ĠR oy +w est +ig inal +em ies +it z +') :Ċ +ĠP eter +Ġt ough +Ġredu ced +Ġcalcul ate +Ġrap id +c ustomer +Ġeff icient +Ġmed ium +Ġf ell +. ref +ĠC as +Ġfeed back +S peed +( output +aj e +Ġc ategories +Ġfe e +} ; +Ġde leted +re h +Ġpro of +D esc +B uild +Ġs ides +.Array List +- % +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ø ± +.m atch +л и +Ġfe els +Ġachie ve +Ġcl im +_ ON +ĠC D +Ġteach er +_c urrent +b n +_P L +ist ing +En able +G EN +Ġt v +Ġso ck +Ġpl ays +Ġdis count +ĠK E +ĠDe bug +F ore +ĠI raq +Ġappear ance +M on +Ġst yled +ĠH uman +i ot +ĠH istory +Ġs ac +ĠC ollection +Ġrecomm ended +.Se lected +Ġorgan izations +Ġdiscover ed +co hol +ad as +ĠThom as +M ay +Ġcons erv +Ġdom in +ĠF ollow +ĠSe ction +ĠTh anks +User name +Ġrec ipe +Ġwonder ful +.s leep +_ if +ĉĊ ĉĊ +orn o +Ġr u +_t arget +." " +à ¦ +Event Args +Ġinput s +Ġf if +Ġv ision +c y +ĠS eries +) ((( +Ġtr ading +Ġmark er +B egin +Ġtyp ically +Ġca uses +drop down +_DE BUG +Ġdet ect +c ountry +! ");Ċ +ĉ R +app y +Ġc ref +(' < +" => +ĠL E +read er +Ġadmin istr +à µ +uck et +Ġf ashion +. char +iz ar +Ġdis able +Ġsu c +ĠL ive +iss ue +Ġmet adata +fl ags +Ġ ðŁ +Ġcomm itted +Ġv a +Ġr ough +Ġ'' 'Ċ +Ġhigh light +_var s +V O +Ġenc oding +- Z +_s ign +$ ("# +Ġr ain +reate st +ĠEN D +Se lection +Ġcandid ates +Ġs av +. Empty +Ġdec isions +Ġcoll abor +rid ge +fe ed +ress ion +Ġperson s +V M +eg a +_B IT +A ccording +ack ed +Ġdoll ars +_lo ss +ĠC ost +} "Ċ +Not ification +Ġpro stit +Ġauthor ity +.re c +Ġsp okes +ĠT oday +ist ant +ĠHe ad +âĢĿ . +ertain ment +ce an +cul ate +Ġv en +How ever +_ arr +Ġtok ens +G raph +ĠJ ud +ĠVir gin +ĠS erial +un ning +M utable +ag ers +.c sv +Ġdevelop ing +Ġinstruction s +Ġprom ise +Ġrequest ed +_ encode +/ " +ĠI con +u ilt +- day +Ġint elligence +. IS +ĠO bservable +ĠH ard +Bo ol +ident ial +.An chor +Ġsell ing +C I +AG ES +t le +b ur +UFF ER +R Y +Ġbig ger +Ġr at +Ġfam ous +Ġtyp ename +Ġexpl ained +} }Ċ +Ġn uclear +- N +Ġcr isis +ĠEnt er +Ġan swers +/ ${ +/ pl +Ġse qu +_n ext +m ask +Ġstand ing +Ġpl enty +ĠC ross +ĉ ret +d ro +ĠC ast += true +ĠCh ris +ic io +ĠM ike +Dec imal +add Component +L en +Ġco ck +Ġ# { +UR N +< tr +Ġauthor ities +Res ources +- H +B ottom +_ qu +put er +ester day +Dis patch +s ince +Ġfam iliar +, i +V C +Ġm ent +, C +Ġfre edom +Ġr outes +ĠB uy +Ġcomm ands +Ġm esh +/ C +ĠSet tings +- style +Ġw itness +Ġc le +Ġun ion +ef ault +are t +Ġthought s +Ġ ---- +_pro cess +_ us +ing ly +U ES +T ouch +ĠÐ ¼ +_ open +ĠV ec +Ġre ward +.C lick +/ : +Ġn ie +Ch anges +M onth +ï¼ Ł +Ġexec ution +Ġbe ach +( Integer +ĉ a +/ ' +.Font Style +Ġab ort +ĠS ingle +( isset +Ġd p +Ġ}} +Ġ* = +ĠP S +Ġdanger ous +[ p +OM E +O ther +ĠString Builder +Point s +head ing +Ġc urrency +Ġpercent age +_A PI +Ġclass ic +the ad +ĠM O +F E +Id x +aw ait +Ġà ¨ +Ġacc ident +Ġvari ant +Ġm yst +ĠL and +ĠB re +Ġh arm +ĠA cc +Ġcharg ed +ion es +Vis ibility +ar ry +ĠL anguage +Ġwalk ing +" .ĊĊ +if er +Ġleaders hip +.F rom +yn am +Ġt imestamp +i pt +ĠH as +REF ER +ĠIt s +Ġlist ener +UT E +_d escription +Ġexperi ences +Ġcre ates +R S +c art +bl ack +Ġcho ices +w ar +Ġ'' ' +Ġorder ed +Ġeven ing +Ġp il +Ġt un +ĠB ad +( app +r andom +Ġexp licit +Ġarr ived +Ġf ly +Ġecon om +-m ail +Ġlist s +Ġarch itect +ĠP ay +Ġd s +ĠS ol +Ġveh icles +H z +- com +Ġk ing +_e qual +ĠH elp +Ġab use +-- ;Ċ +Ġex tr +Ġchem ical +ä ¿ +Ġor ient +Ġbre ath +ĠS pace +(e lement +w ait +DE D +ig ma +Ġent r +Ġs ob +- name +Ġaff ected +ik a +Ġco al +_w ork +Ġhundred s +Ġpolit ics +sub ject +Ġconsum er +ANG E +Ġrepe ated +S end +Ġ# [ +Ġprot ocol +Ġlead s +use um +E very +Im port +(c ount +Ġchalleng es +Ġnov el +Ġdep art +b its +.C urrent +Ġ` ${ +ot ing +( \ +Ġcreat ive +Ġbu ff +Ġintrodu ced +us ic +mod ules +A re +-d oc +l anguage +_c ache +Ġto d +? > {{ +ĠRes ource +ĠSt andard +ĠP rem +up dated +ival ent +Ġas sets +_t emp +Ġinterest s +Ġhard ware +ĠR om +ĠSh are +Ġ' 'Ċ +Ġ* , +ĠT ake +ĠIm ages +_C HECK +(type of +ĠJ un +\< ^ +Ġli qu +Ġwor st +ymb ols +ĉĉĉ ĠĠĠ +Ġdr ivers +ĠD ocument +en o +ĠTechn ology +Ġappro ved +ump s +Ġs now +form ance +_A SSERT +u its +Ù Ĩ +Ġdiffer ences +. Visible +ĉĉĉ čĊ +ĠP s +_f etch +Ġto do +. ',Ċ +Ġs el +ur ers +in valid +Ġt weet +V EL +Ġresearch ers +Ġs printf +ĠR O +Ġp el +.Tr ans +Ġil legal +d ialog +sm arty +l g +_M IN +Ġher o +f inal +Ġp p +.L e +Ġc i +ĉ RT +Ġsuggest ed +p df +ach ing +ĠR o +ĠProp erties +ĠS i +Ġbuy ing +Ġm u +Ġl ands +if iers +ĠF ILE +RO UP +Ġh older +ĠS on +Ġsym pt +.r oute +) ? +Ġarg c +Ġfor t +Ġcas ino +_c ategory +Ġfor um +p refix +apt ure +T ube +em s +im ize +Ġn ue +a us +c ourse +AT OR +() ), +Ad vertis +ING S +Ġack now +ĠKore a +pl ing +Ġwork er +PL IED +h al +ĠRich ard +Element s +ĉĉĉ Ġ +st ar +Ġrelationship s +Ġche ap +AC H +ĠX ML +, & +ĠLou is +Ġr ide +_F AIL +Ġch unk +[ s +_O UT +Ġch osen +_ [ +/ ( +ĠJ eff +_s l +pr iv +ĠCan adian +Ġun able +_F LAG +Ġn os +h igh +Ġl ift +f un +() { +el ly +ycler View +_ as +_L IST +Ġr adi +.get Value +ĠAnge les +ĠS pan +_in stance +it ors +Ġm igration +A K +O h + ® +. selected +ĠG T +Ġadv ance +ĠSt yle +.Data GridView +e ction +Ñ İ +p io +ro g +Ġsh opping +ĠR ect +I lluminate +O U +ĉ array +Ġsubstant ial +Ġpre gn +Ġprom ote +IE W +.L ayout +Ġsign s +/ . +Ġlet ters +Bo ard +ct rl +" \ +ĠJ ones +Ġvert ex +Ġj a +Ġaff ili +Ġwe alth +ĉ default +Ġsignificant ly +Ġe c +Ġx s +act ual +.p er +_st ep +an vas +m ac +Ġtrans l +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Iter ator +Ġo ch +agnost ic +ĠD uring +ĠDE FAULT +Ġt ill +Ġsign ature +Ġb ird +ĠO l +ĠI r +H S +av atar +ESS AGE +Ġe lev +Ġm t +ĠN av +Ġrel ax +Ġpl ate +IT EM +( date +.n ot +Ġgr ade +Ġ} ),Ċ +? "ĊĊ +i ences +H igh +ĠD IS +dis abled +Q UI +Ġno ise +a ux +ĠU P +os a +Ġv oc +Ġ )) +oc om +_O FF +ĠD b +L ock +.e clipse +, d +ĠD raw +Ġ" ( +Ġvis ited +Ġâ Ī +Ġsuc ceed +Ġim possible +a ire +ĠT urn +Ġd ish +F G +Ġs ensor +AN N +ab a +Ġsur g +] );čĊ +Ġf p +_ an +- J +- G +ĠJ ob +Con vert +ĠKE Y +Ġauth ors +_s erver +\ r +Ġ-* - +f lex +Ġs oc +R et +Ġs alt +Ġâ̦ ĊĊ +ĠC lear +(p age +-d anger +Ġroom s +con v +# { +. op +ĠA rea +_S C +h en +Ġbeg ins +- y +Ġexc ited +Ġign ored +Ġbon us +st udent +ĠM ember +Ġrel atively +ĠL ow +ĠPro du +ate way +pos ure +Ġth ick +ani el +( view +ĠCr ush +Ext ension +I l +e ed +LO C +. im +. Items +Ġconflic t +.pre vent +Ġon Create +u v +is er +Ġw ave +M ar +ĠComm unity +ic he +ĠNo thing +[ m +ĠLe e +ri ends +è re +!! ! +an z +. result +ĠS K +_P ARAM +Ġdem ocr +Back Color +.ex ists +" It +( options +ra zy +as er +\ Database +al endar +_ ass +; }Ċ +vert ex +ine craft +W arning +arg o +Ġact or +ĠInst ead +ĠUs ing +S elf +@ interface +Ġspe aking +ĠPar is +ĠL ICENSE +.n ode +ĠF ood +E IF +ĠB i +. Start +ĠI B +Ġun iversity +ĠHe ader +.pro duct +C opy +et c +r ical +Ġ> >> +book s +Ġal gorithm +Ġ' __ +(j avax +Ġnumer ous +Sh are +H ave +Ġrec ru +Ġpro ve +.sub string +he alth +е л +Ġdec imal +Ġcomm ission +s cription +x C +Ġsum mary +att ed +Ġclo ser +fin ished +() ){Ċ +ĠW ood +_field s +k u +_ items +Fl ag +Ġconf idence +ĠF ederal +du x +Ġcomp at +Ġvert ical +Ð ¹ +è s +; ">Ċ +_m anager +() ))Ċ +ID E +: ", +__ Ċ +ĠW ay +Ñ Ī +T emp +ĠS TR +rit ten +S ync +ĠA V +ĠC EO +ĠG uid +Ġenvironment al +Ġcorrespond ing +ĉ console +Ġjust ice +ĠJ S +Ġl ived +g ar +ĠG raph +ĠSt at +Ġi Phone +. al +ĠH D +Ġocc ur +Ġth reshold +Ġon click +RE G +.Graphics Unit +M eta +Å ¾ +Ġc um +.g nu +à « +Ġobt ained +Ġcompl aint +Ġe ating +Ġt ar +_t ask +Ġopt s +( to +P ass +Ġpl astic +t ility +ĠW in +.prevent Default +p ile +ĠG ar +Ġqu antity +_l ast +Ġg reatest +D ao +_D IS +ĠUs ed +ĠH P +rit ing +S ION +bl ue +d omain +Ġs cores +N ormal +_ admin +ĠA SSERT +Th en +** * +d ist +l on +Ġh ate +sh al +Image View +d atabase +Ġp and +Ġlog ic += false +b g +ĠConfig uration +Ġn ur +O G +Ġmar ried +: + +Ġdro pped +Ġreg istration +оР¼ +ult iple +iz ers +sh ape +.c opy +Ġwe aring +ĠC ath +Ġded icated +Ġ.. .Ċ +Ġadv oc +ĠF amily +Ġstat ements +em atic +ampions hip +Ġmot iv +ĠH ave +Ġbl ow +J ob +c ert +_v ector +inst all +ĠC OPY +em bed +D IR +ĠS pring +Ġex hib +cd n +ĠCom ment +ĠOption al +. player +ĠD ark +( pos +ĠSh ould +Ġcent re +ĠGu ard +ó w +Ġtr ouble +EN ER +( unsigned +_s ervice +Ġn s +ul ing +ĠMex ico +ĠN Y +mys ql +Ġl ic +å ľ +M r +- fl +ĠC ustomer +id i +Ġ? >ĊĊ +ri ble +Ġп ÑĢ +Ġs izes +_STR ING +valid ation +ĠJ on +( Http +add Class +N odes +Ġfrag ment +Ġsp oke +Ġw aste +J oin +Ġill ustr +el i +c ient +Ġa id +Ġpro sec +') {Ċ +Ġpass ing +Ġf aces +Sh ape +_ Z +it i +Ġal le +Ġro bot +ĠĠĠĠĠĠĠ Ċ +ĠS pe +Ġrece iving +ĠD etails +Ġ" ) +m g +_RE F +Ġcompar ison +* , +ĠF ound +_s ession +( U +/ F +Ġx xx +N etwork +d ers +Ġcap ture +Ġcor re +ĠL td +ĠAd v +[ @ +Ġcl ip +M ill +ĠPro file +Ġend if +Ġob lig +des cribe +.e lement +riter ion +L D +er ed +Ġfav our +s core +ĠF ilter +at tributes +Ġcheck s +In flater +ĠPl us +Ġscient ific +Ġpriv acy +He ad +Ġfe at +Ġdeg rees +ĠP ale +; "> +Ġfil ms +ĠA udio +ĠT ag +ĠE nergy +it ar +par ator +Ġf ellow +Ġev t +ĠT ri +ĠD AM +cl oud +ĠP assword +ĠDemocr ats +ĠAc ad +$ lang +Ġre b +() )ĊĊ +н Ñĭ +ĠB ur +read cr +Ġh ex +Con sole +ct l +ous el +ĠWill iam +Ġa z +_P ORT +Ġpract ices +Ġany where +ĠP osition +Ġ- >Ċ +i ams +.user name +place holder +Ġo der +ĠSecret ary +Ġi T +mon d +event s +? âĢĿ +.S ub +Ġatt ached +Ġn ão +Ġest ate +. action +Ġfig ures +Ġ} );čĊ +Ġsubs cri +.t ag +n am +. plot +no on +li ament +Char acter +.t ab +Ġw inter +ĠVar iable +Ġtre es +Ġpr oud +( V +_ load +Ġh ier +ĠE con +Ġf d +Ġvict ims +R est +ian a +Ġf ake +.Print ln +Ġstr len +Ġs ad +Ġb le +Pro t +Ġbutton s +Ġte levision +Ġlog o +ext ension +ĉ j +ste in +acion es +Ġ"" "ĊĊ +Ġsim p +Ġrecord ed +Ġbr ings +Ġprincip al +Ġfe es +(s ource +k dir +Ġutil s +Ġcorrect ly +f il +Ġw el +P air +-b utton +s cale +ver ify +[ c +Ġ-- - +Ġes cape +ik es +Lower Case +ic ian +Ġch apter +ĠT YPE +Ġsh adow +Ġaw esome +W E +el if +Ġl ambda +Ġdist inct +Ġb are +- off +Ġcol our +.append Child +ole c +ag a +.f ill +ĉs uper +Ġad j +( position +.get Item +Sh ort +Ġtot ally +V D +ĠT re +_ ep +v ements +ĠS olution +Ġfund ament +F ollow +Ġfac ility +Ġhappen ing +O F +.text Box +S pan +Ġ « +id en +Ġex ceed +(p arent +Ġc p +ç » +Ġhas n +Ġp ri +Ġcon sequ +n en +ĠIN TO +I gnore +ĠF uture +Ġcar bon +ĠSte el +f mt +ok ie +Ġs pl +(t itle +- info +Ġde als +Ġfix ture +e a +D iv +Ġtest ed +_ return +)ĊĊ ĊĊ +upport ed +ĠC ook +Ġpay ing +ĠI ll +Ġarrest ed +ĠPr ime +_c allback +> ,Ċ +dr iver +On ce +ab b +_by tes +ĠS ets +( Object +Ġc c +Ġsh ell +al o +); // +( log +ct ors +) +Ġ$ (". +.p os +Ġbo ys +Ġwed ding +Ġag ents +=" _ +ĠAr my +Ġh int +v ision +Ġte ch +ĠCon nect +Ġleg end +ĠB et +.B ase +Sub ject +Ġl it +Rem ove +Ġ" : +ĠF inal +pear ance +ĠiT unes +Ġparticip ants +ĠPy thon +Ġbus y +i el +vert ices +Ġtemplate Url +ĠC lose +Im g +ĠCorpor ation +t imestamp +Ġext end +Ġwe bsites +Ġposs ibility +о ÑĤ +Ġk ö +Ġme at +Ġrepresent ation +Ġ ĉĉ +_ST ART +.app ly +ĠVal ley +ĠS uccess +H i +Ġn ob +ĠI Enumerable +_ select +ge o +. ")Ċ +Ġturn ing +Ġfab ric +(" ");Ċ +Ġpers pective +é Ĺ +ĠS n +Th ank +; j +.Param eters +ĉ ĠĠĠĠĠĠĠĠĠĠĠ +Ġfact s +Ġun t +.in stance +################################ ################################ +- end +ĠJO IN +ĠH en +Ġur i +åIJ į +Ġн а +ĠIn fo +Ġconduct ed +Ġà ¥ +OUR CE +Ġw ine +J ohn +.Error f +ĠA ge +ound ed +Ġreal ize +Ġ] ; +Ġsub sequ +, m +( User +ian o +Ġaccom pl +is p +.st d +é ĩ +ĠB ed +.set Attribute +B R +ke ep +ĠA LL +Ġis ol +am ma +P ackage +Ġoccas ion +-s uccess +еР´ +ĠLIMIT ED +st rip +() ĊĊĊ +istrib ution +Color s +Ġ+ :+ +Did Load +al er +Ġt id +ĠL ED +ĠLink ed +ĠC art +() )čĊ +_RE AD +Ġkill ing +ĠP HP +fe ction +Ġinst ances +c v +"/ > +Ġs f +Ġtax es +_ location +ĠBit coin +u able +r ank +ign ore +tr ack +к а +Ġshould n +ĠO P +=> {Ċ +Ġk m +Ġh elper +_ head +ĠWh ether +oc o +_b l +Ġstat istics +Ġbeaut y +Ġto g +t ip +ëĭ ¤ +Ġc sv +(s ql +std lib +we ak +Ġlik es +Ä į +Ġrepe at +Ġap artment +Ġem ph +_ edit +Ġv it +ĉ type +E ven +ut en +Ġcircum stances +b ian +Ġs ugar +W indows +ì ŀ +Ġobs erved +/ data +Ġcal endar +Ġstri ke +ĠR ES +_s c +f ony +ore m +( z +p ower +et ect +ĠS at +.d escription +Ġg ang +ĠS ports +ong s +ĠB undle +.s um +on ce +Ġacc used +Ġexplo re +Ġapprox imately +Ġlos ing +thes is +ĠF und +Ġdi agn +A utowired +prop erties +Ġ_ . +Ġc nt +ced ure +Ġy y +Ġgr ant +so ck +.inner HTML +Ġ] );Ċ +ĠCON FIG +=' $ +] ];Ċ +UN D +Ġg lob +Ġd ire +uff le +_M EM +Ġauth entic +> (" +Ġdec ade +ĠIm port +Ġorigin ally +Ġj Query +Ġindic ate +Ġours elves +S w +.l bl +ener ate +Ġbas ically +ĠH om +Ġ+ #+ +ĠBrit ain +ĠK ar +to Equal +.st op +Ġmod al +is i +Ġsuggest s +Ġd type +Ġt ur +b f +Ġconnection s +ĠB efore +ist ed +m ouse +Ġpul led +.b uild +Ġlegis lation +Ġfor th +p ad +eg o +.N ow +Ġexc iting +}ĊĊ ĊĊ +Ġcom pr +Ġsh ares +Ġr ig +g reen +_ vec +Ġenumer ate +A uto +ic ator +ĠR ay +as se +Ġh oliday +Ġnull able +g un +_d etails +Ġwr apper +se q +ĠYou ng +ju ana +Ġ" __ +lic ense +ser ve +^ ( +id ers +.Rem ove +rop down +' S +p in +(t oken +.D efault +Ġreason able +amp ion +ĠS ociety +Ġbe i +erv es +r ad +ĠF ox +_ images +Ġw heel +') [ +Ġc fg +( By +Con structor +Ġv ary +.sw ift +Ġpro xy +ĉ H +ĠAn other +ĠP en +Ġcheck ing +Ġj est +man ager +Or igin +ug s +o ir +>< !-- +Ġexpress ed +Ġmod er +Ġag encies +Ġi h +-h idden +ious ly +ĠR od +Ġso le +M ed +.A ny +Ġp c +b al +Ex ample +ĠS ale +Ġst rip +ĠCom p +Ġpresident ial +M ost +put ation +( ref +ĠF our +_f ilename +Ġen forcement +Ø ¯ +ĠGe org +we ights +/ l +Ġag gress +Ġd rawing +and y +< I +- j +ak a +h ref +Ġteach ers +_ Q +( it +ĠM B +Ġtemp orary +ire base +str a +æĹ ¶ +è ´ +( label +ou p +Ġtop ics +Ġport ion +id os +ĠJew ish +Ġre covery +Ġstand s +# [ +Ġafter noon +ĠArt icle +_ att +Ġexpl an +ĠP ak +.setOn ClickListener +. children +Ġi k ++ ( +l ag +Ġdis k +Ġcont rovers +"> & +as p +Ġw ie +ĠAustral ian +ĠYou Tube +At tr +cont ains +du ce +ĠM att +at ern +Ġvol unte +Ġnew sp +V P +olt ip +Ġde legate +_m eta +Ġaccur ate +ĠEx ample +% , +ĠD aily +Ġc abin +ĠS W +Ġlim its +k ip +Ġar my +Ġend ing +Ġb oss +ĠD ialog +Al so +="# " +ord an +row se +- min +Ġ" & +_ loc +U X +Ġdevelop ers +Ġaccur acy +Ġmaint enance +Ġhe av +Ġfil ters +.T oolStrip +Ġn arr +ĠE mp +ORD ER +ĠM obile +.S erial +.out put +.c ol +M aterial +um a +Ġconsum ers +sh ift +Ġp ued +Ġmin i +c ollection +Ġk an +.c enter +H istory +Ġben ch +() ); +itor ies +Ġcrow d +_c all +Ġpow ers +- E +Ġdis miss +Ġtalk s +ĠCh annel +for ward +_ control +/s rc +i est +**************** ******** +Ġbet a +(c olor +_O BJECT +ĠA pi +Ġeffect ively +C amera +s d +uss y +D ict +ĠE ffect +ib ilities +Ġreturn ing +ĠF ar +Ġ' ') +Ġmod ules +il ation +Ġ( % +TR GL +Ġst orm +on na +ĠEX P +Ġs pons +Ġdis pl +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +f all +å Į +ign Key +_ US +et rics +Ġhand les +T L +_ amount +ow a +br and +ĠT ool +Ġus ual +. Z +cre ment +ad ium +st ock +Ġserv ing +ĠB on +Ġline ar +ĠT arget +ĠR adio +H L +Sh ader +om atic +ag ues +in ity +d iff +_ iterator +qu ot +Ġ ,Ċ +c allback +Ġsympt oms +[ _ +ĠB ul +ĠF eb +und o +_ account +Ġtyp edef +и Ñģ +tr as +User Id +ĠP enn +ĠSup reme +} > +user Id +ĠK im +Ġg a +Ġart ists +å ¸ +ĠAb stract +ok emon +Ġh am +o val +Ġch a +at en +å Ĩ +F ixed +Ġvul ner +ĠParam eters +qu antity +.C lear +Servlet Request +Ġy a +Ġsou l +trans action +Ġsol o +Ġp airs +æ Ķ +ĠG re +_ word +ĠC C +Ġg i +z ie +Ġsched uled +rot ation +gy pt +ul ous +:: _ +ĠE ll +< ! +ĉĉ ĠĠ +l p +ah a +C opyright +Ġdr am +Ġdi agram +ĠM em +Ġg arden +Com p +Ġattempt s +uff ix +> () +Ġphil osoph +_re l +å ¼ +Ġs v +.se cond +ant o +.J son +ĠTe le +_ local +_s end +Ġas pects +ì Ĺ +IB LE +Ġr ail +Ġwid ely +ash ed +i ar +in f +up per +d jango +_result s +iss ing +Ġequ ivalent +OUN D +Ġt y +Ġpotential ly +Advertis ement +ĠRec ord +resent ation +_w idget +ound ing +Ġrelig ion +Ġcons c +ĠL im +. am +H tml +Ġ' : +P ATH +_s pec +ort ed +id ades +_sh ape +Ġkeep s +.S ave +ĠL oc +or i +ĠT EST +unic ip +Ġreg ions +Ġbelie ves +/ en +pos ite +{ ' +pre pare +_ const +s ample +ĠWill iams +Ġstr t +_ Get +ĠAnd rew +. active +Ġl ayers +Visual Style +az y +ĠK n +Ġac id +ĠAs ia +Ġex cess +ĉm y +Ġkey board +ens us +Ġcre w +Ġmiss ed +m aster +ĠW ild +Ġnew ly +Ġwin ner +Ġst ub +ic ode +.m ove +D omain +ĠS ar +Ġfore st +LE D +claim er +.ex it +ĠW indow +Ġres istance +ĠC HECK +(" - +ĠR yan +Ġp ipe +Ġco ast +DE F +// ! +_ off +ex it +Ġult imately +imit ive +ĠKe ep +Ġhistor ical +Ġany way +ĠJack son +ock er +ER N +ĠU INT +y ntax +ER Y +is ms +Ġc n +Ġocc urs +Ġ; ; +Text View +A E +/ img +Ġy esterday +- default +Ġt iny +Ġpro c +Ġal ive +ĠRE G +. th +ear ing +.get Logger +< link +_ login +F older +ab c +lyph icon +н о +Ġnot iced +od igo +Ġed ition +im ator +. Enabled +.parse Int +Ġy ards +ĉĉĉĉĉĉĉĉ ĉĉĉĉ +Ġver bose +л Ñı +_B Y +.log in +.* ;Ċ +ĠM id +é es +Ġg lo +Ġbuild ings +Ġz e +ĠI ter +Ġt ube +ĠP ot +\ M +< th +br idge +ĠS cript +ĠM odule +Ġv acc +Ġinstall ation +v y +VisualStyle BackColor +ĠS M +.t otal +b at +Ġfind s +Ġat mos +Sub view +iz ard +Ġrepl acement +lic ated +ap is +Ġlog ged +ĠLe ft +G ui +_ Type +t m +P ad +Ġhouse hold +Ġre le +Ġpropos al +_CL ASS +:: :: +Ġinf rastructure +In ject +/ html +Ġad s +iz za +Ġm g +ctr ine +% Ċ +< html +- image +Ġatt orney +< m +(' , +Ġcan n +Ġprint ln +o ose +Ġy ellow +.ex p +p ayment +Ġtable View +aw ay +Ġopp osition +ĠAg ain +ĠH andle +Ġex clusive +in ar +é r +оР± +ĠC ODE +emp orary +Ġre act +pi pe +c z +. activity +Ġlarg ely +Ġdis s +ax y +es is +ĠR en +Ġc orn +.Use VisualStyleBackColor +d ays +Ġfr uit +In sert +_ enc +E st +_de c +ĠL uc +Ġü ber +param eters +P ERT +ex press +_pro file +Un known +Ġrev olution +.add ress +_re quire +Ġun iform +ĠP ack +l ar +ĠU ITableView +Ġdep ends +Valid ation +conf irm +O wner +Ġt rib +h et +ĠI de +ans as +L anguage +u et +ĠP o +ĠSte ve +Ġcont est +_DE FAULT +Ġapparent ly +RE EN +Ġfrequ ently +Ġtrad ition +ocol ate +S I +ĠArg ument +F ocus +ert e +ĠL ayout +Ġd x +Ġgener ator +ĠW ait +P olicy +l ights +.Ex ecute +P y +Ġbed room +ed a +ra id +ĉs ize +Ġan cient +Ġp ump +Ġd w +Ġ(! ( +Ġspec ify +( status +ĠF BI +.ex ception +Ġrem ark +ly mp +ant ee +Up load +ern et +é ¡ +in ent +ĠR ender +d m +ĠM emory +r ich +ĠT ools +Ġk ne +Ġper m +b ad +Ġd inner +.res et +Ġj Label +Fe ature +.S ervice +Ġ( {Ċ +Ġre ferred +.class List +Ġinit With +ĠText View +Ġne ither +Ġcount y +Ġ" { +ç § +Ġt ack +class Name +ĠUS ER +Ġre new +` ` +get Name +Ġb rown +Err ors +ert o +Ġsust ain +S O +let es +ĠIn valid +Ġen emies +un ge +Ġexist ence +err a +Ċ ĠĠĊ +utor ial +# a +p ay +char ge +ĠI re +ate st +Ġexp los +Ġf ired +N ER +ĠT y +ic ion +U ri +Ġobvious ly +ĠC olum +Ġ' + +ĠDe vice +- related +_ ARG +Ġv or +ĠLess er +_O P +Serial izer +Ġup grade +L ight +Ġc odes +++ ;čĊ +Ġwrit es +fo od +Ġé t +@ section +Ġtrack s +Ġserious ly +ch t +(size of +Ġimmedi ate +Ġscient ists +Ġ{ $ +_ ne +.Anchor Styles +Ġaccom mod +ĠHar ry +Ġs ight +ĠPale st +ersist ent +Ġ Ñĥ +- input +Ġco ordinates + · +W elcome +.con f +Ġgre w +Ġb old +ĠC PU +(m y +Ġperfect ly +Ġmom ents +ĠM ovie +- data +yst al +_W IDTH +ĠS creen +æ Ŀ +Ġdis ap +Ġredu ction +.Get Component +_M ODULE +Ġgener ic +Ġd y +all er +Ġc url +ĠB ody +Ġb anks +, t +av g +Ġev il +Ġmanufact urer +Ġrece iver +Column s +Ġing redients +ĉ out +qu es +.L oad +Ġslow ly +ĠT own +ĠC ell +_n ormal +_p refix +ĠAl ert +(" { +ä r +âĢľ The +ĠM D +Ġcour ses +ath an +é Ļ +oc c +ĠS ER +es ign +Add r += [' +(" ./ +] } +.f ont +ĠInst agram +ĠB order +od a +Ġh all +Ġr um +_b it +Ġs aving +_d own +R andom +_reg ister +( Context +Ġoppos ite +R oom +Y ES +ан и +Ġenjoy ed +_r un +C lear +âĢ ĺ +ĠF ord +on ic +ost en +"] ) +_ auth +// čĊ +Ġsuff icient +LE S +Ġph en +Ġo h +_c sv +Ġrout ine +.Are Equal +ay lor +Ġb asket +_COM M +rypt ed +S im +ĠSh op +Ġstud io +at os +( W +[ string +ä t +og a +Ġsh r +Ġs ick +An other +Ġdo ors +_N E +ĠTH REE +. order +raz il +Ġmap s +_TR UE +trans late +Ġnear by +Ġn ach +LO AT +b atch +Ġl ux +ash es +ang ers +â̦ â̦ +_E VENT +_ UP +Ġact s +in v +_M ETHOD +cc ion +Ġret ain +ut ch +ĠÐ ± +Ġknow ing +Ġrepresent ing +N OT +p ng +Con tract +Ġtr ick +ĠE dition +uplic ate +Ġcontrol led +c fg +j avascript +Ġmil k +Wh ite +Se quence +aw a +Ġdiscuss ed +ĠB ush +ĠY ES +.f actory +t ags +Ġt act +Ġs id +$ $ +ĠE num +Ġfr ames +} ); +Ġreg ul +'] ;čĊ +Reg ion +ff f +Ġc ro +( com +=" + +St udent +Ġdis appoint +RES ULT +Count er +Ġbut ter +ĠH a +ĠD igital +Ġb id +"> {{ +ing ers +ĠC ountry +_t pl +"] )Ċ +/ k +d ating +: # +ĠD ATA +yn chron +_b ody +olly wood +Ġval or +ip ient +o ft +UB L +doc s +Ġsyn chron +Ġform ed +ru ption +Ġlist a +Request Mapping +Ġvill age +Ġkn ock +oc s +" { +_fl ags +Ġtrans actions +Ġhab it +ĠJ e +ed en +Ġa ircraft +ir k +ĠA B +Ġfair ly +. inter +.A ct +Ġinstr ument +remove Class +.com mand +Ñ ī +ĉm em +( min +Ġo t +Ġcol le += s +time out +Ġid s +ĠM atch +ij n +z ero +Ġnetwork s +.g ov +Ġint el +Ġsection s +out ine +(c md +(d ir +ĠLI ABILITY +ĠB log +Ġbr idge +ĠC V +con vert +Ġ" )Ċ +ĠB ern +_P O +e val +( set +to ol +Ġpay ments +Beh aviour +Ġcon crete +Ġel ig +Ġacc eler +Ġh ole +_ o +TE GER +Ġgraph ics +O wn +Form atter +on der +Ġpack ages +/ a +ĠK now +Or Default +Ġdut y +W ait +н а +_rec ord +[ t +M esh +Ġon going +.be ans +Ġt an +Ġinter pret +ast ers +QU AL +Ġleg s +\ Request +- file +_m utex +ĠS aint +// # +Ġpro hib +( info +: = +lin ux +Ġb lo +ot ic +ĉf inal +_ex p +ĠSt op +ap ing +(s aved +_p ush +Ġe ase +_F R +pons ive +str cmp +: ĊĊĊĊ +ä» ¶ +ol i +Ġextrem e +Ġprof essor +Im ages +.IO Exception +Ġaddress es +plement ed +Ġincor por +Ġuse Effect +_O F +ĠD a +n ombre +IR ST +Ġdisc rim +Ġcomp ens +greg ate +anc ell +ach es +ĠC riteria +$ result +D estroy +Ġsecond ary +W atch +ĠS em +ĠMc C +Ġacad emic +U pper +:: ~ +ut ral +ĠD og +ad ed +Valid ator +Ġder ived +Ġset Timeout +ĠK en +Ġtyp ical +ĠB ob +Ġb ounds +ĠSe ason +Ġc razy +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +-r outer +itt est +ĠM ir +Ġemot ional +, v +c n +/ st +å ½ +on om +Ġdecl ared +> . +ail ing +Ġ/* <<< +Ġnorm ally +(M e +ev in +lik ely +Ġpoint ed +ĠSt ack +Ġw alls +. Vector +me an +] ]Ċ +Ġlist ening +ad v +Ġsw ap +IF T +Ø ª +. argv +ul s +< option +not ations +Ġemail s +ĠU kr +ast a +ĠTh us +ĠSt one +Ġappe al +. âĢĻ +Ġreg ulations +Pre ferences +ĠPh one +ul f +ĠD R +Ġtechn ologies +Ġpar agraph +Ġnecess arily +.e ach +< float +res a +Ġunder st +Ġf inger +press ed +-b y +if fer +w atch +ĠB a +A IM +Ġwe ights +ĠR on +') }} +[ self +-------- --Ċ +per iment +Ġto String +x ic +ĠC amera +! ĊĊĊĊ +aur ant +P refix +Ġinstit utions +: int +Ġex posure +p attern +ĠLin ux +.n umber +red ient +Argument Exception +ĠCh ief +" }, +Ġelect ronic +r ong +er d +sp Net +ra it +/ ', +ĠOh io +Cont rollers +Ġcontin uing +ĠT emplate +ĠE th +s z +/ env +En v +% . +art ers +) (( +ĠT ABLE +Ġà ® +per ature +pro gress +P res +ê ° +im plementation +Ġb ien +Ġstre ets +_M SG +New s +## # +: / +Ġcut ting +x B +ress ed +_EN ABLE +l ab +Ġca using +] ));Ċ +b ra +x FFFF +il ly +plet ion +w ill +_b ar +Ġstruct ures +ĠI mp +Û Į +Ġ< > +Ġ ---------------- +_B UFFER +.d ir +Ġpl ain +Ġpe er +g g +oint s +Ġsomew hat +Ġw et +Ġemploy ment +Ġtick ets +ir ms +Ġt uple +s is +$ sql +r ig +Ġcon version +Ġg es +Ġconfig ure +eg r +ĠC a +Ġ__ (' +ou ston +.t oken +Bl ack +Ġmag azine +A W +. IN +os ing +Ġbro ke +ĠC ru +DE LETE +Ġdestroy ed +(M ath +Ġappro val +-d om +ĠI II +table View +Ġdesign s +Ġcrush ing +Ġcons ent +dir name +om p +Ġc rypt +? ( +or ough +. o +ĉ list +ams ung +."" "Ċ +err ing +G oogle +_p air +_IN IT +rem arks +Ġg ear +F ill +l ife +} ")Ċ +Ġsuit able +Ġsurpr ised +_RE QUEST +Ġman ifest +att en +Ġfr ustr +ov ement +.c lick +Ġi i +Ġexp ansion +ig s +P arse +.Reg ular +R ob +_l ayout +ì ł +Ġtrans lation +ĠBe aut +B est +_C OLOR +< label +Ġliqu id +IT S +Ġpro d +Ġoper ate +UI Kit +Ġn atur +arg ument +_d etail +ĠCent re +Ġ" -- +Ġ}} " +lo cale +.t v +_se q +Ġup coming +Ch art +ĠDiv ision +Ġclin ical +Com pany +S epar +l as +ĠH un +: s +Ġhead ing +оР³ +Ġ" ");Ċ +[ id +b ia +Ġst retch +ic ide +Ġre produ +.pro ject +leg end +end ers +Ġrespons es +Ġon t +rit ical +Ġref uge +ĠL i +Ġ: ĊĊ +ĠTh ree +.cont roller +_IN DEX +_F OR +\Model s +j ax +ĉex it +Ġâ ĸ +Ġc overs +ĉ y +- . +IND OW +Ġfail s +in cludes +Ġf ault +Ġl y +ñ o +.s lice +ILE D +ĠP ur +ĠAs ian +_b atch +.M ax +v l +ĠCOPY RIGHT +Ġg iant +ĠMan ual +ĠC opy +Class Name +He alth +C ursor +IB Outlet +Ġt we +æ ³ +_label s +Ġcol lected +Ġfurn iture +Ġdeal ing +Control s +ĠHot el +ck s +Ġch ose +âĶ Ģ +od d +S R +Ù Ĭ +ì Ħ +Ġacc ord +ĠM ove +ĠM ode +ĠM ock +Ġthread s +++ ++ +ĠO ptions +Ref resh +ĠD id +'] -> +u cc +_ch annel +. abs +Ġ{ },Ċ +ĠW al +er ior +Ġmain ly +ĠDr iver +NotFound Exception +Ġcount s +e am +Ġ& = +Q uestion +ĠA li +Ġany more +d etail +t ail +Ġm ile +ĠF air +Ġs orry +Ġsurround ing +Ġad m +De v +Ġmari juana +ĠS ound +ĠA sh +F D +Te am +. port +Ġ[ ]ĊĊ +ub ble +Ġas c +Ġint ention +A cc +ch i +ust ers +Ġins pired +se g +CL U +Ġman ip +M etadata +Con nect +ĠB eh +Ġfind ings +Ġas sembly +w orld +Ġrem ained +Ġu id +( . +Ġm x +Lo op +ĊĊĊĊ Ċ +Ġfant astic +wh o +ak i +ĠB asic +ĠY et +ĠUs ers +ik ip +Ġhead s +ĠMich igan +_ it +ĠTor onto +Ġrec ording +Ġsub mitted +_var iable +medi ate +.graph ics +Ġst ood +Ġre ar +vel ocity +_M ESSAGE +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ro les +ĠT our +_ year +end ment +amp s +ĠIre land +m al +Ġyoung er +Ġstrugg le +Ġc able +ĠSD L +(' - +an es +ĠNe ed +.R ow +P ol +ĠP H +_s cript +ag em +ĠB as +_s pace +. loc +: i +ad r +Ġengine ering +it en +) & +Ġu k +ĠL ittle +_C OUNT +x A +Array List +æ į +Ġ" ")Ċ +An chor +Ġh ang +t witter +Ġcompet itive +.s rc +ãģ Ĺ +Ġtrans late +ĠCre ates +ook s +ĠR oll +'' 'Ċ +/ sh +s ome +Enc oding +.res olve +Ġdesign er +ĠSt orage +Ġz a +ĠN ever +Ġsomew here +Ġbox es +.s ource +Ġpy game +Ġgrow n +.t w +() ),Ċ +', [' +Ġoppon ent +(s rc +.l ayer +AP P +ĠAct iv +Ġguest s +ĠVAL UES +};ĊĊ Ċ +.n ative +Ġamount s +. RE +Ġcl one +Ġwer en +Ġ" << +_ ac +Ġbreak ing +Ġreli able +.P OST +ĠSk y +Ġ' & +Ġsaved InstanceState +ast ing +ill ion +com ments +ult y +.m enu +/ config +Ġ ĊĊĊ +T ODO +Ġpurch ased +_c or +ĉ auto +Compat Activity +com plete +_ graph +is odes +Ġsitu ations +ĠH or +Re ceive +âĢľ We +Ġent ities +.assert Equals +оРº +ĠS ans +v ince +rom pt += Ċ +Ġ/ . +.Se lect +yl v +Ġb att +A udio +Ġincreasing ly +.B undle +Ġexpl ains +the ast +. offset +Ġh al +Ġtechn ique +_l imit +Ġdraw n +AY ER +Ġfeature d +yy yy +at in +ph en +ach el +! \ +l ower +ĠG R +Ġp ag +ĠP arse +Ġt ou +ä¸ Ģ +D istance +Index Path +Ġh ell +s im +UT TON +Us age +elen ium +ĠF all +Ġ" .$ +ĠM u +Ġcr uc +Ġs ont +REF IX +Ġinter ior +ĠO lymp +.Auto Scale +par a +Axis Alignment +Ġr iver +D to +Ġwith draw +Re act +- class +b efore +_ alloc +Cont ents +ĠW as +I CT +Ġform ula +Ġindic ates +ĠĠĠĠ ĊĊ +_st ore +it ting +ĠIt alian +_S et +_re port +Ġp id +_V ER +Ġw ins +ĠCl oud +") {Ċ +ch ester +Ġden ied +Ġw ird +ĠSte p +Ġinvest ors +b old +_d isplay +ou ver +or er +Res et +Ġsurg ery +Ġstrateg ies +/m aterial +_ unit +Ġc ouncil +.P er +ĠâĢ ŀ +Ġre form +F ramework +Ġlist ing +_b tn +Ġb is +% d +eg as +Ġsudden ly +_S ER +Ġa o +_d irectory +f as +Ġprem ium +Ġtrack ing +ĠB L +Ġm ature +Ġbath room +Ġ'/ ' +ĠÄ ij +Per formed +Ġsold iers +arn ings +Ġwalk ed +- con +b ottom +Ġsurpr ising +Ġg ene +Us uario +.DE FAULT +ĠM IT +C ODE +ĠE gypt +p icker +ys ql +AT URE +d etails +ĠCon ference +In formation +ĠM ail +-d own +r aries +b ro +Ġsubject s +Ġ' * +è¯ · +or ient +: @ +ver bose +E F +Ġto ler +eng ers +Ġend point +Ġstr ange +Ġcol on +Ġpre ferred +de p +ĠE V +ARR AY +Ġw he +Ġp up +_n odes +Ġtalk ed +Ġinstit ution +db c +Ġex posed +te en +ĠFr ont +T T +_N ONE +\/ \/ +pro gram +Ġencour age +. ` +sh ire +ĠIsl am +e en +N I +' " +.W idth +Ġlik ed +Ġ{ ... +ĠSystem s +Ġvot re +Ġmanufact uring +Con verter +ĠIn f +ì ļ +D TO +Ġin ches +Ġ ठ+à ¹ +ĠChar les +B U +")) ;ĊĊ +ĠL abor +un n +Ġest im +m obile +ĠL earn +_C ALL +â Ħ +Ġind ices +Ġt ub +ikip edia +C ost +row able +ë ¡ +g age +Ġfunction ality +uzz le +em os +.l ib +Ġd ass +еРº +enn a +Ġsh ots +Ġrest ore +/ D +For Key +], [ +al ias +l int +.st ream +æ ł +_FORM AT +Ġsil ver +.re pository +Ġlegis l +.B order +_fe atures +Per mission +Ġhous es +ĠW ars +_COM P +Ġinj uries +Ġconstant ly +fl utter +EN U +ĠCon f +Ġrecogn ized +Ġpract ical +Ġde cent +B J +] ); +ast y +ĠAct ivity +-m ode +Ġsl ide +.IsNullOr Empty +ĠY OU +P ower +ind ices +Ġqual ified +Ġthrow n +h ello +ĠN ick +l ah +as sembly +ĠSm all +old ing +Sh ould +ĠSil ver +(saved InstanceState +Ġtog gle +.N ot +C trl +: nil +ĠCont inue +ĠB oot +æ ī +ĠM ur +d on +ĠF A +S napshot +Ġassoci ation +fo x +, a +az ione +] )čĊ +CT YPE +Ġf ade +ĠD ar +.n avigation +Ġl uck +SC RI +ĠDe ad +Ġterm inal +_LE NGTH +Ġeff iciency +Ġun w +Ġn arrow +iment o +( Color +ĠSe a +_ area +, A +_ opt +ĠHill ary +.t ask +ĠJ ac +ast ed +ĠAd am +ĠIl legal +Ġsearch ing +Instance Of +J ava +ĠForm at +Ġreal ized +ĠChild ren +Ġk il +(f rame +âĢĿ .ĊĊ +Ġscen ario +"] );Ċ +Ġincred ible +li x +IO Exception +ĠQ uest +il ty +Ġun lock +â Ĥ¬ +Ġre ferences +ĠV ert +B inding +eg ative +Ġwr ap +.d atabase +( content +B uf +ĠTr ad +ĠA ud +tr ace +.m ock +Ġther apy +ĉ L +.To Int +ĠKing dom +B us +ha ust +"" "ĊĊ +( end +.draw able +[ ];Ċ +ĠH ospital +Ġph arm +---- - +ĠA G +é d +> ");Ċ +Ġw allet +at able +) $ +Ġmonth ly +Ġdi agnostic +S ymbol +Ġiter ator +un finished +Ġimm igration +s r +RO W +(g ame +Ġclo thes +ĠU nt +Ġactiv ation +_C on +.h ash +Ġinitial ly +.H ash +Ġcut s +f ound +ĠSt ory +ÑĨ и +ac ao +_T YP +pro to +est r +-p age +ah r +Ġincor rect +ĠJose ph +TextBox Column +_st yle +ĠD aniel +s heet +Ġl iv +l ined +Ġr a +R untime +_ empty +sl ug +_ struct +ë Ĭ +m u +Ġper mitted +Ġreg ional +Ġsob re +ĠS uch +Ġ[ _ +Ġro of +.Al ignment +t imes +.m sg +Ġche st +ĠT ab +Ġest a +ä n +Ġsubs cription +( command +s pecial +Ġme al +") :Ċ +_ ctx +Ġclos ely +et ry +- be +ad el +ĠR am +ig est +ĠSpan ish +Ġcommit ment +Ġw ake +* >( +P HP +_ { +ck er +< List +_n ull +ĠRes erved +Ġin her +.Column s +.A spNet +_IN VALID +ĠParam eter +Ġex pr +} { +Cell Style +Ġval uable +Ġfun ny +In v +Ġst able +* t +Ġp ill +pl iers +ĠC SS +ĠCon dition +ĠS peed +ublish er +Ġoff ensive +ce st +ic as +Ġsp ark +ĠPro te +set up +IF Y +ĠT ax +Wh o +F amily +- for +. uk +Ġf asc +sv g +") ). +Ġbirth day +âĸ Ī +ve h +el led +Ġimport s +ĠIsl amic +T A +ĠSt an +we ather +Ġsus pect +e ature +enn es +W M +.m inecraft +av id +è ½ +.se curity +in os +G ood +Ġm arch +Ġposs ess +us uario +Con s +am ber +ched uler +Ġhor se +ç ½ +(b ody +ĠTrans form +_de code +.s vg +Ġf oo +Ġd ella +ext ends +am er +Ġprocess ed +ĠH arr +ĠA I +Ġk o +CH AR +( % +Ġt ap +({ ' +c roll +D OM +Ġte a +Ġre in +Ġworld wide +_f n +sh a +Ġb ir +ç ões +="# "> +Ġrepresent ed +ill er +(ex pected +Ġd ance +Ġvisit ors +.con cat +-b it +UR RE +ĠR og +v p +ip h +ĠL LC +it led +iam i +C oll +_re al +_sh ow +_f older +Ġd ar +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġl atter +arch y +Ġb ow +Ġout come +ĠPost ed +Ġris ks +ĠThere fore +Ġowners hip +Ġpar allel +Ġp ending +ge ometry +Ġrecogn ize +ST EM +ĠC P +Ġimm igr +IT LE +ĠĠĠĠ ĉĉ +conn ected +Ġsm ile +(d ocument +\ Component +vert ical +Ġconsum ption +Ġsh oes +. impl +un ks +. ";Ċ +Ġfood s +_ );Ċ +.assert True +Ġp ipeline +Ġcollection s +Ġearn ed +ĠC ert +Ġpartners hip +( action +Ġc d +ĠV ery +Option al +Ġscre ens +Ġtit les +ener ator +Ġab andon +k ind +IL TER +Ġclos ing +lic a +_ inter +Ġcamp us +set ting +S prite +ãģ ¯ +_re ply +To List +: \/\/ +ed e +Ġfol ks +Ġbo at +( argv +Ġperman ent +Ġcarry ing +Ġconserv ative +import ant +. img +ĠIm m +Ġdim ensions +al and +s ingle +Ex it +-------- -- +ari ant +tern al +Se conds +ĠIt aly +ot lin +.Res ume +=' " +) == +cept or +Ġs ca +/m ain +Sec urity +_d at +Ġlet s +Ġa qu +Ġwhen ever +b erry +Ġact ing +ant i +p d +& gt +æ Ń +Z one +T oday +! . +To Props +ab is +it able +Ġg al +] { +iz ona +Ġin contri +N ET +/// Ċ +[ in +_s ave +Ġex em +ĠK enn +Ġev olution +var s +_st ats +- only +ĠColor ado +Ġwatch ed +b our +Ġsever e +Ġprofession als +port ion +Ġguar ante +Ð ³ +Ġpush ed +ĠG i +ï ½ +Ġt um +ĠA z +ĠEdge Insets +")) ;čĊ +is se +. ac +Set ting +Ġapprec iate +ĠValue Error +Ġsur ve +ĠR ole +. Inter +plot lib +j et +d am +Ġplatform s +te le +UT O +ĠInt ernal ++ : +} ;čĊ +Gener al +\ Entity +Ġlawy er +qu iv +ĠPost s +is o +Ġacc um +ob e +Ġmark s +Ġ] ;ĊĊ +ĉ text +.s uccess +cur r +as a +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +Ġth in +_ over +are st +ĠO s +( address +Ġvel ocity +Ġ[] ;ĊĊ +=" ../../ +ĠPr iv +b ow +Ġguar antee +% ĊĊ +Ġeval uate +.LE NGTH +Ġin ventory +q a +_de bug +.On ClickListener +Ġl ies +Ġassess ment +dat etime +.background Color +Ġ*/ čĊčĊ +ra f +un wrap +ĠF oot +Ġnot ify +Ġlow est +DO CTYPE +Ġl anguages +ex tra +- back +Ġein en +tem plates +_p ass +ĠM ust +Ġest á +_c ore +ĠSc ot +A I +Ġb ias +ations hip +Con stant +Ġprogram ming +In s +uspend Layout +ĠPRO VID +ant es +Ġsh irt +in ated +. OK +[ a +Ġthink s +? ĊĊĊĊ +Ġregard less +ĠMag ic +ul ating +ĉ class +add Group +RE ATE +ĠS U +Ġsim pl +c opyright +Ġb unch +Ġun iverse +ĠE rr +Ġpresent ation +c ategories +Ġatt ach +.s ign +_A C +Ġdisc ipl +Ġregular ly +Ġprim arily +ink s +[ [ +.r and +.sh ould +ownt own +=" ' +Ġs ans +Ġsupport ers +se quence +G O +. .ĊĊ +ĠS pr +Ġcare fully +U IColor +dest roy +Ġtod os +ĠOR DER +ott ed +Ġd ont +aud i +_ player +g re +ĠO il +< body +_st ack +.P adding +ĠProduct s +Ġpriv ile +Ġinj ured +ĠF urther +Ġal ias +.Resume Layout +_LE N +Ġs es +'] ;ĊĊ +cre ens +Ġdirect ed +.S uspendLayout +od ge +.A t +mark s +ĠUn ivers +ert s +ĠE sc +Ġnav bar +Ġutil ity +agnost ics +Ġin ject +ĠD NA +Ġ" ," +am ar +Ġe u +Ġrestaur ants +_p ut +ut ers +Tool Strip +t w +ist ro +Ġz oom +Ġleg it +pec ific +ĠC ome +Ġlocal Storage +Ġabs or +.P anel +ĠDesign er +Ġo w +IC AL +_ uri +(f ield +Ġsup erv +Ex ists +Ġrespect ively +ĠSt and +Con f +uss ian +Ġar c +Ġ nd +uck s +Ġre str +Ġseason s +ĠCh apter +ĠSw itch +p ic +Ġh i +load ed +Ġfl uid +-b tn +Ġrun time +. it +B N +Op acity +as ant +ry ption +-n ative +Ġta ught +å ¯ +ag ment +Ġm ul +Reg istry +_ grid +ĠBro ok +: Set +Ġm ongoose +AM ES +inner HTML +Ġs oci +ĠInt el +get Id +C md +Ġaccess ible +r ames +le ton +Ġ__ ( +ĉ delete +ĠS quare +" ĊĊĊ +Ġbu cket +avor ite +ĠB reak +++ ] +Ġbr ush +Ġt ensor +/ http +T ile +Ġfunction al +Ġ" * +wh el +Ġt ent +ĠChar acter +Ġse es +. ST +B ig +Ġext ern +Url s +)) )), +ĠJ r +.B uilder +. ; +n l +_ Init +ĠH ER +ż e +mys qli +_ icon +v an +Ġfeel ings +Ġle an +Ġhop ing +T V +="čĊ +b est +all as +ent ed +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĊ +_con nection +Ġrep o +en abled +аРº +Ġsh a +Ġmembers hip +Status Code +in ating +_s m +_c ustom +_ weight +Ġc ss +St at +_ env +link s +TR L +ĠH it +, r +up id +Ġop ens +Ġg ent +_v is +Ġj oy +< w +_c ost +ĠPy Object +ren ce +ĠGeorg ia +ĠBro ad +m ma +â Ĥ +p f +Ġ" \" +Ġ( & +om o +Ġliter ally +Ī ĺ +met ric +Ġb ars +z ed +(w indow +ĠIsrael i +Ġform al +ident ifier +.d ao +ĠDe ath +% ;Ċ +Ġdecl are +ar ms +RE AM +PERT Y +Ġconsequ ences +to ols +Pe ople +ĠWh ich +> ();čĊ +.de code +_A CT +Button s +.f loat +.F irst +ë ¥ +ĠPol it +ĠX CT +T ags +ĠCG Float += str +Ġle af +- check +ĠI ss +.s ystem +log out +ach t +Ang le +s in +ch art +INT ER +ĠN UM +B asic +.P roperties +ä¸ Ń +_ change +ĠB razil +Ab stract +Ġ: +: +_ use +а л +ĠL y +IB UT +Ġout er +Ġ-- >čĊ +Ġrel ief +l ap +qu er +_p arent +he ap +LO SE +Ġcomb ine +ĠR ose +ow ers +Ġproced ures +ĠS ort +an im +var iant +eh icle +Ġsign ing +Pr imary +c urrency +Ġsex e +o en +th eta +em an +Ġimpress ive +(' _ +ĉ U +ĠText Style +_c nt +Ġs lice +(' : +Ġunderst ood +H is +Ġinform ed +Ġn ick +(T AG +h d +Ġelection s +est ure +ĠS anta +ĠCo ast +.p df +inc iple +.cl one +b orn +ut a +Ġl icensed +C r +Ġb read +ĠH ouston +Ġn od +Ġhop es +ĠCG Rect +Ġgu ilty +.g if +Ġro se +.Com mon +T ip +AN K +ĠF C +D uring +ĠSym fony +Ġdef ensive +k m +) > +arch ive +ĠU RI +ycl ing +- o +ĠWe bsite +AM P +ish ment +Ġdo ctors +D irect +AR I +ĠRed irect +ier en +_d ist +y o +ĠPro gress +Ġz um +Ġmem or +ĠE D +Ġj ur +æį ® +_T ABLE +Ġu uid +Ex pr +. head +(' % +point er +Ġest imate +ĠG reg +Ġlo ader +Ġi OS +Ġm ens +[ y +Ġref used +Ġprec ision +is ch +ĠA CTION +Cl oud +s With +( ret +_ADD R +_con f +(d f +Ġlock ed +Ġr ising +ãĥ» ãĥ» +ĠM s +Ġscen es +_EX T +_ raw +_ the +pe ople +Ġre con +ĠF un +Ġb less +ĠUp dated +ü n +ĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +pe ction +Re lease +.log ger +ĠS Y +Ġcoun sel +ur d +_ true +Ġevery body +iv ot +Ġh ence +ĠN AS +Ġoppos ed +unk nown +ĠDES C +ĠCh air +fa iled +ĠIN CLUDING +Ġwrit ers +{ }Ċ +ÃŃ t +_c opy +} : +ĠB at +Ġconvert ed +ed ing +pl acement +ĠH ost +S ound +и м +Ġs ought +m id +Ġsal ary +og g +âĦ ¢ +b ul +Ġw ir +valid ator +_ST AT +.st ore +ĠB attle +ı n +Ġ-- >ĊĊ +Tr ump +d ot +ĠCON T +.f etch +Ġcontin u +w as +Ġfra ud +_t mp +mit ter +.p ictureBox +G A +Ġt ournament +. Input +[ r +ex ion +cent age +ĠKore an +und ef +ĠAv ailable +resh ape +Ġk it +ĠStr uct +ĠS UB +An swer +_l ib +.t witter +Ġo re +ĠDr agon +.Ex t +, k +Ġexplan ation +ref s +ĠDr ive +ĠTr aining +.H as +int age +b ig +olog ist +enn is +Ù ĩ +Ġch icken +ĠĠĠĠĠĠĠĠĠĠ Ċ +ç Ľ +ãģ § +Ġpe ak +Ġdrink ing +Ġen code +ĠNE W +m alloc +ĉf printf +Ġ= ================================================================ +in cluding +Ġprincip les +ĠM ah +st orage +- key +Ġkey word +% ; +Ġtr ained +.con trib +Ġk v +__ ':Ċ +ĠB oy +param eter +Ġsu ite +Ġthous and +Ġco ordinate +-g enerated +íķ ĺ +gener ated +Ġad mitted +Ġp ussy +# w +Ġsw im +un ion +N a +ĠRoy al +.ch annel +Up dated +_RO OT +Ġv ital +ra ction +ĠCrush er +Ġpre ced +Ġhor izontal +Blue print +Ġattr s +Ġsm oke +Ð Ĵ +. Equals +F B +ĠRes ources +roll ing +Ġpass es +ĠN um +rot ate +et ype +\ ", +Ġsens itive +Ġt all +? âĢĿĊĊ +Pro xy +i y +_ section +âĢĶâĢĶ âĢĶâĢĶ +br id +Ġcirc uit +at an +EN C +Ġdr iven +Ġvot ed +Ġeduc ational +Ġinter action +abet es +Ġt one +ĠInitialize Component +Ġmer ely +Ġì ŀ +co okie +_ div +ĠUIL abel +vel y +} );čĊ +_ ENT +#+ #+ +art icles +ĠSou thern +Ġstrong er +ĠG iven +ĠE ric +ĠI R +ab stract +U nder +n able +Ġincre ment +ov en +Ġco in +_t imer +Ġsuffer ed +ĠF REE +'] ." +ĠQue en +st ats +Ġmeet ings +Ġenter ing +Ġalong side +(s ession +it als +Ġfound ation +ĠC redit +. div +_ ALL +pc ion +_st at +ick ing +Default s +_s rc +Ġoutput s +/ B +Ġent hus +-b l +.Fore Color +ĉ temp +F ace +Ġinter act +Ġwe ird +M ount +re ll +ud ents +Ġrequire ment +ĠS us +I ER +Ġe lected +re ference +ĠM E +Ġserv ers +.w ait +Ġsnap shot +il ton +Ġtri es +Ġt ipo +.T ime +> w +Ġmount ain +Ġp ounds +Ġ[ ... +ex ists +Ġng On +_M AP +Ġf lying +xi ety +ĉ value +_D B +un o +Ġse ats +T URN +. author +! ) +or ce +Ġindic ated +.s in +Ġass ignment +im iento +ĠF rame +_g en +in ery +_ ) +m essages +.set tings +ĠMe an +ĠM useum +ir q +att ach +ĠPalest in +_ QU +_t ags +Ġcas ual +em en +ASS WORD +$ s +ĠC irc +оР¹ +et ric +/ P +Ġep och +< head +_C MD +Ġg it +Ġpen alty +or ph +_ users +ours es +.Date Time +atern ion +_pro ject +Ġsuper ior +ĠD am +ĠSe attle +X Y +> The +ĠA k +Ġgr ass +/* čĊ +(d is +Ġgun s +Ġt b +ĠK evin +. args +ĠA h +op ed +( J +column s +arg uments +ĠWith Events +_f ull +ĠDef ense +S imple +Ġdeath s +Ġext ensive +ĠSt ill +ĠEx pression +ĠAg ency +Ġperform ing +F X +Ġus uario +U AL +S ide +od os +apt op +Ġcred entials +_c ap +at ient +ĠDis ney +Ġa i +Ġch ip +Ġvol t +.make Text +%%%%%%%% %%%%%%%% +Ġbelie f +_LO C +ĠC ivil +N avigation +Ġreve al +Ġviol ent +ĠF il +Ġc atalog +em ed +sc an +. control +Ġconstit ution +C ountry +Separ ator +_A PP +top ic +uet ooth +M IN +Ġdes criptor +y t +ET HER +Ġdistrib ute +' }Ċ +.tr im +.L ine +Ġl bl +assert Equals +ĠD et +omb ok +( width +Ġt ort +ĠEXP RESS +ac o +Us ing +ĠBr and +w all +EM ENT +ĠComm unic +< uint +ĠG UI +EG IN +ĠR ange +/ i +ĠT aylor +c ost +Ġrespond ed +ĠTh eme +n ce +IS H +Ġfeat uring +Return s +ĠK r +Ġ .Ċ +Ġn am +_c b +Test ing +Ġ{ }, +y al +.f ield +Ġ/ = +_SH ORT +m ates +Test Case +ain less +Ġeval uation +_ ITEM +ĠPac ific +ĉ k +Ġc ant +ĠR os +) s +Ġf et +STR ING +ĠDis pose +g al +ĠJ oin +ĠP orn +ĠCath olic +AR GET +cp u +ç łģ +.sc roll +IS ING +ifest yle +anc ement +Ġm erc +ĠB rowser +eter min +Ġover flow +Av ailable +Ġbott le +: UI +ific ial +Ġco ord +clar ation +Ġcon j +G LOBAL +ok u +Ġk wargs +cond itions +ul um +Ġg enu +ĠH ero +å İ +Ġun expected +ĠDAM AGES +Ġk a +ĠC ould +UP PORT +ĠPh otos +Ġconf ident +Ġdet ected +de g +rg b +Ġstrong ly +Ġ} ;čĊ +Ġ) : +Ġle ct +urs ive +RO L +ĠWe ight +Ġent ertainment +Ġ) );Ċ +Ġg onna +Ġb b +.d o +G S +Ġmist ake +D L +ĠPROVID ED +ear ning +L imit +iss ions +[ v +ä¸ į +ir ty +D el +Ġunder lying +pre ne +Ġj aw +ĠD I +pe er +Ġobject ive +Ġde posit +Ġk on +Ġes p +.set Visibility +/ login +< typename +Ġfr anch +/ e +Par allel +Ġsc ored +ĠH on +ĠV ill +ig a +Ġant icip +_ assert +ĠO pt +Ġdescri bes +w an +m ount +Ġmonitor ing +Ġt out +ëĬ Ķ +}, { +................ ................ += int +Ġc ust +---- -- +Ġatmos phere +P AR +ort e +IS IBLE +ĠI ron +ĠNot ification +.log ging +ĠBO OL +-p oint +Ġaf raid +ent a +Ġtom orrow +@ implementation +Ġeng age +ĠAn th +ĠF loor +ĠU l +To ols +Ġb ab +Ġcare ful +ãģ Ħ +Ġcruc ial +Ġcalcul ated +ĠS A +Ġw y +D X +_T AG +ind ed +Ġj et +ĠEngine ering +.M AX +en z +v d +Ġpublic ation +Ġ## # +Ġfac ed +ra ham +ĠC apt +As set +ĠCon stants +Ġlo ans +_ IP +ĠF ish +Red uc +_m at +Date Format +_m e +[] [] +Ġintegr ity +ĠC ourse +lob als +Ġfac ilit +Ġem br +ĠN g +.S ystem +Ġmanufact urers +Ġpro ven +.on Create +Ġal arm +Ġ § +Ġcomm only +ic os +æĸ ° +ĠSt ation +} ). +ĠF ilm +w i +ç ī +Ġeng aged +St ats +Ġgovern ments +Ġafford able +_p roperty +Ġag es +(' -- +Ġf ör +ĠProf essor +Ġhy dro +P ush +Ġorgan ized +Ac cept +é m +_c ell +Ġn b +p b +Art icle +Ġrem oval +Ġauth entication +ĠF R +l ide +Ġple asure +ap ol +Ġpart ition +ĠS ide +Ġcr imes +Ġdem o +hold ers +ĠPak istan +In struction +Ġexpect ations +.sc ene +Ġ' ) +h es +ino is +_P ro +Ġm olec +and al +_sh ort +Ġdefault s +Ġn ations +in en +Ġr t +O CK +P acket +S B +ĠSH ALL +_cont ents +ise conds +vert y +á t +G uid +n om +Ġcon clusion +. Update +Ġlo vely +Ġem it +b ec +ĉĉĉĉ Ġ +Ġintel lect +Ġb rew +ec ycle +F ire +Ġad mit +Ġar bit +Ġarr ang +ĠM IN +M ail +ĠN ative +C ur +Ġcon vent +.R untime +" }Ċ +.R un +Ġprint ed +Ġconven ient +. ar +m ock +ĠAdmin istration +ãģ ¾ +Ġelect ron +fl ate +Ġl ombok +Ġjava fx +n h +Ġsup plies +Ġvisit ing +ah l +Ġpow der +Ġult imate +Ġorient ation +ut as +_s cale +Con firm +ph ones +ĠOper ation +/ T +_IN TER +Ġair port +Ġmet rics +Ġphen omen +a udio +Ġm ai +( K +h u +all ing +rodu ction +ĠTrans port +ĠNOT E +æĸ ĩ +Ġfew er +_T IM +ì § +к и +A ge +F IN +Ġì Ŀ +ĠAt tribute +group s +er k +at to +. define +.AspNet Core +ategor ia +ĠS ir +( form +< User +. round +_d ay +.A ll +Servlet Response +.N o +l arge +IG H +qu ent +Ġvir us +Ġret ro +Ġim per +Bit map +Ġv ice +Ġoff ense +ist e +ĠA UTH +Ġê ° +ToolStrip MenuItem +G u +Ġr ape +ĠDav is +Ġover whel +: flutter +- table +ĠCon structor +Pr ivate +e ven +ch r +Ġap plies +_at tribute +Ġcon tribute +E VER +L ines +ĠAf ghan +Vis itor +ĠS L +se ason +C U +Ġintrodu ction +Ġmat plotlib +Å ij +Ġnewsp aper +âĢĶ and +< tag +Ġin i +Ġd iverse +Ignore Case +ĠU r +Ag ent +Ġb ull +.em it +( Exception +ar Layout +Ġincred ibly +ĠTr ust +={ ( +- nav +Ġe quals +Ġl ady +ĠP od +d isc +al am +ĠI V +â Ļ +iv idual +ph i +add ed +Ġdifficult y +Ġcomp act +ĠAction Result +c ers +_class es +Non Null +Ġqu it +Ġp ou +S witch +ir s +- test +ĠK ind +ĠCal endar +Ġstream ing +} ', +S W +Ġst ead +oc a +Ġprov ince +Ġcol span +Ġperson nel +ĠE mployee +Ġprodu cer +Ġevery where +od b +Ð Ł +bs olute +act ivate +Ġgr inding +ĠBuild ing +ĠSand ers +(s c +ĠOff set +//////// //// +} ;čĊčĊ +({ " +Ġscan f +ĠY Y +ĉdef er +Ġj ew +Ġrestrict ions +.m p +[ l +ä¸ ĭ +label s +red icate +aw esome +Ġw aves +Ġcon front +Ġmeas ured +Ġdat as +_ex it +ot ton +Ġshould er +ask a ++ # +ĠĠĠĠĠĠĠĠĊ ĠĠĠĠĠĠĠĠĊ +Ġtro ops +ĠU nd +_c ard +w ich +Ġn ous +Ġ"/ " +s b +Ġcommunic ations +Ex port +Ġdec ode +th s +inter pret +By Name +ĠSp irit +ed ges +O LE +ĠE M +t it +ĠTh rough +Ġb io +ĠP ackage +or ne +Ġ} . +` ;Ċ +Ġok ay +ĠZe aland +ident ity +(n ext +ĠB ang +Lib rary +Ġheav ily +il on +Ġdi pl +Ġrot ate +put s +) ',Ċ +ĠData Table +Ġmay or +.to LowerCase +Ġsome how +ĠNor thern +al c +Ġcap abilities +Ġv ibr ++ Ċ +ĠS u +ĠRes et +_m ean +Ġc ig +.cl oud +ĠB and +ĠF actory +ĠAr izona +_ io +op her +Ġconsc ious +Ġà ¶ +\ Controllers +_s peed +ĠF ac +_C om +ĠB ible +w en +ED IT +Ġun n +ĠSt aff +ĠIn n +Ġmechan ism +ĠM embers +Ġmigration Builder +'] .' +.get Int +< void +ĉf ree +oid s +\ Support +Ġautom atic +Ġch ances +Ð ¶ +Ġcomp licated +[ row +ah oo +Ġ}ĊĊ ĊĊ +Model s +W in +Ġt ape +ir us +iz on +on omy +(" _ +: . +.st ereotype +( env +_re ct +(w ith +Ġassert That +Ġcon straints +put y +E mployee +T D +Ġgu itar +ĠJew s +.pro cess +Ġf iction +ĠSh ared +âĶĢ âĶĢ +Ġprop ag +.N et +Ġachie ved +ĉ Q +Ġn urs +Sh ared +_FAIL URE +Ġbeh aviour +Ġcol s +ism o +Ġfem in +Ġchalleng ing +Ġpost ing +enc il +Ġcapt ured +ĠD ou +( word +ĠTur key +pan ies +Ġre putation +ORM AL +Ġelig ible +prot ocol +id as +(f rom +Ġfin ance +- per +Ġg otten +H A +d uration +ĠP arent +Ġin vent +Ġre start +ол ÑĮ +r ition +(r s +< bool +i ert +Ġmod ification +ĠT X +readcr umb +b ank +$ / +ĠMill er +] ),Ċ +.Check ed +Ġsac r +se curity +Ġp ose +ĠBr ad +Ġfit ness +Ġannounc ement +ation Token +Ġserv es +ne ed +Ġge ometry +AR S +æ Ģ +andid ate +Ġs prite +_s plit +We ek +ad ies +> (Ċ +?> " +Ġ/// Ċ +Ġein er +Ġweek ly +ĉlog ger +_p op +_m an +Ġmigr ations +Ġask s +Ġb s +Ġfall s +.W here +- height +_fe ature +.M in +Ġhy per +Ġvol atile +Ġtw enty +Typ ography +Un able +D et +, f +-m od +Ġsett lement +Ġcontract s +n ome +B ad +ĠB rian +(user name +!! !! +Ġh ack +.F ield +H R +ĠJ ordan +iz a +Ġ ł +ĠSh er +. header +( other +ĠD ub +( op +ĠR ound +Ġv ie +Ġap pl +ĉ J +ĠIn sert +ĠL P +reg on +ĠM PI +Ġan chor +ac a +ø r +Ġa de +anch or +que e +ĠTree Node +Ġtarget ed +Ġla id +AB EL +v et +ĠOr igin +A nt +. ');Ċ +ex pect +ed Reader +ĠM ajor +Ġin ch +Com par +Ġpre view +Ġill ness +ĠCONTR ACT +ĠInd epend +u uid +Ġn ome +Ġt c +ĠA venue +is an +Ġph rase +_m ove +") [ +Ġprov ision +Ġconcent r +_ IR +ĠU t +() + +Ġn as +! , +ĠRob in +i ations +at itude +Ġp x +ĠWith out +/b ash +ek t +re ement +Ob server +ĠReg ion +UBL IC +Ġ{ // +K N +å · +Game Object +å ¾ +enc oding +Ġ** * +project s +Ġt k +Ġche ese +EM PL +ar o +Ġا ÙĦ +Ġcons ists +ref resh +ure au +ĠSc anner +Ġso il +Ġfl avor +Data Source +Ex ecute +ени е +Ġsh it +åĪ Ĩ +< any +Ġretrie ve +Ġbelong s +.st rip +abs olute +Ġexp anded +bo y +): - +Ġresc ue +.J Label +Ġre ly +Ġal ignment +-f amily +Ġre nd +OLUM N +Ġb orrow +Ġqu otes +ĠL ew +Ġsh ower +ĠDE LETE +_lo op +! "ĊĊ +ĉ re +Ġattempt ed +aver age +ĠP aint +quis ition +ol en +Ġliter ature +ĠRe ference +_TEXT URE +ĠS eg +ĠInd ust +ct ype +D UCT +_H OST +ĠTr ade +Ġpl ugins +Ġbre ast +ul se +Ġcreat ure +ãģ Ļ +ĠW i +Ġsup plied +c oll +! (" +Ġfuck ing +ĠCh rome +ĠU ri +ĠN ation +Ġvert ices +T HE +ĠOr iginal +on de +Ġsh arp +Ġcook ing +Ġ{ /* +ĠPs ych +ĠH ollywood +=$ _ +.D ock +Ġg er +Ġb one +_con n +_se c +ys ics +Ġ= " +S al +s f +Ġdeep ly +ang les +T erm +b ell +ĠQu ick +ener ation +adio Button +åħ ¥ +}čĊčĊ čĊ +Ġcapt ion +l c +ĠE L +, [ +ĠĠĠĠĠĠ čĊ +ret t +(m ethod +ĠFl ash +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +W ISE +.s cale +Ġrough ly +_ child +m emory +ay ing +Ġinitial ized +in ator +а ÑĢ +Ġsc alar +ĠH o +ai res +(c olumn +.de stroy +P ACK +Ġh em +ang el +_S UB +. qu +Ġ × +DE FAULT +pos itories +ĠL ength +ĠF ast +Ġsign als +Ġ// $ +ri ers +Ġd ummy +AN Y +Ġperson ality +Ġa gricult +Pl atform +ER O +ĠT ra +Ġen orm +ĉ W +Action Result +Ġa ver +[ str +Ġ' -- +.S printf +Ġdeb ut +Ġ Ñĩ +h ex +_ utils +Ġp b +U ITableView +Ġz ur +. encode +Ġv ag +.error s +о н +Ġm r +ĠA ward +Ġc pu +Ġpress ed +' est +ĠF estival +' T +Ġa k +res olve +.m e +Ġn ic +Ġgen re +Ġat trib +ĠMo on +Ġarr ive +ĠD ating +Ġt m +.Config uration +. red +Ġgl m +Ġst ations +sw itch +Ġt ied +äº º +Ġ/ >Ċ +Ġsubsequ ent +pos able +-fl uid +Ġth orough +Ġpublic ly +apt ers +ĠWil son +_P RE +y ard +ä ¼ +ĉ in +Ġre vers +Ġbul let +cri bed +nes ota +Ġ($ _ +ann on +c ursor +Ġclo thing +ĠM ulti +: ', +Ġv ess +ordin ator +Ġein em +C annot +Ġar med +ĉ V +ä¸ Ĭ +.F lat +ĠS ep +ĠSub ject +_f ont +Ġcharacter istics +D one +el n +######## #### +PO S +Ġd ensity +ĠPl atform +- items +Ġo vers +Ġpush ing +ç ¤ +.Con nection +_ term +Ġinitial ization +________________ ________________ +ç ¬ +.d ocument +les h +ĉd ocument +ĠP in +ç a +Ġdefinition s +.P ath +_W RITE +Ġ ĉĊ +? >ĊĊ +Ġter rible +be an +ick ets +ĠS V +B uy +(t ask +Ġreg ime +g oogle +Ġcr ack +.vis it +N UM +ener gy +Ġstr uck +_s ample +.p ayload +Ġre vis +ĠSc ene +Ġp g +Ġbreak fast +URRE NT +.char At +_ex ception +ĠAnt on +Ġguid elines +Ġex haust +ĠFin ancial +Ġind ent +Ġdes ktop +H idden +F ailure +Ġpr inciple +Ġ iv +Ġse ks +n etwork +Ġnumber Of +ĠAl bert +ĉ long +, . +Ġz eros +f ade +ĠT yp +ĠT erm +ĠAr ts +.App lication +Ġbeh alf +æĪ · +Ġm ere +(` ${ +Ġaware ness +elp ers +f lix +Ġwe igh +Ġestim ates +. child +/ O +ĠBit map +.b ottom +Ġ************************************************************************ ** +Ex pect +ent o +ĠFor um +ver al +Ġj ail +Ġab ilities +ĠH OLD +ĠC it +Ġd ynam +Ġgr ay +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉ +.next Int +ant ly +ĠAR ISING +( private +Ġreject ed +ĠN ic +Ġle ather += {Ċ +aly tics +th etic +.T op +.P age +={ ` +Ġ ;čĊ +de pth +m ann +W D +ĠS om +.R ight +Ġ) }Ċ +Ġtr ait +Ã Ĺ +i ac +Ġr v +S ample +.X ml +opp ed +ĠÑ Ħ +list s +Ġt ear +ivers ary +.c ollection +ĠCon stitution +ĠHttp Response +Ġbr ill +ĠP rom +h over +ĠM iami +Ġarg ue +_f loat +Ġ ãĤ +Ġn at +ĠT al +Ġinteg ration +(c ur +Ġrem oving +Ġco eff +ĠTh ough +Ġfore cast +ĠV egas +S ite +Ġtr ab +ĠHen ry +- i +Ġinvol ves +B T +Ġs lo +In voke +Ġl ucky +r at +Ġ? Ċ +Ġhand led +(f d +cont ents +ĠO FF +R F +Ġst y +ĠM otor +ter y +t ax +M AP +ĠMr s +Ġph ones +ĠUI View +")) );Ċ +( dev +ĠIr ish +Ġw s +D I +_OFF SET +ĠEvent s +Ġst ages +Ġ} // +Ġhab en +ST ANCE +ĠS in +ĠM oney +(t op +Ġappoint ment +VER SION +met adata +_com ment +Ġcolle agues +map s +â ĺ +Ċ ĉĊ +( al +_re q +Ġf ut +Ġarchitect ure +ĠWH ETHER +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +_s creen +Ġstyle Urls +Ġmon ster +. up +ph ia +Ġprocess or +ĠT err += ', +ĠMan ufact +ĠN T +k el +ib ern +ĉf ile +A li +rient ation +Ġ// ! +ap ore +ane ous +ĠC reat +f older +Ġh ay +Sup press +( left +Ġe uro +Ġdis claimer +ustr y +sh ips +_f d +ĠF a +_in sert +Ġro l +if ting +ĠCom ments +_b r +Ġloss es +ĠAdd ed +ch arg +Ġп о +_s ystem +ĠS ometimes +ĠSp ain +(g roup +ial is +Ġdoll ar +ĠAr gs +qu ires +ĠT en +.s css +Ġsurv ive +us age +Ġj un +im iter +ï¼ģ ĊĊ +Ġfif th +t oggle +Ġdecl ine +($ " +(L ong +ing e +Ġpil ot +-l ight +-r adius +Ġpod cast +Ġnatur ally +P ages +ä¸ º +ĠDes pite +Ġlight ing +Ġcr ate +ĠB inary +Ġredu cing +Ġe leg +ĠM ouse +ĠTest Bed +Ġbefore Each +_ ARRAY +Red irect +Ġf lood +Ġsh ips +Ġelectric ity +)* ( +ê ¸ +ĠV iet +her o +Ġd ia +ĠK ent +he art +Ġthreat s +_ acc +Ġs ymbols +is chen +_in st +C riterion +ĠT IM +. Height +Ġ âĢĻ +();ĊĊ Ċ +Product s +_S P +ĠC y +Ġdepend ent +est e +Ġdat os +d it +аР² +IGN AL +Ġless on +"> ' +ĠC over +ĠH ope +ĠT imer +Ġd ad +vid ers +ĠPh ot +/ ? +rop y +om ing +as ion +Ġ\ ( +ĠE T +ĠRe ading +Ġep isodes +l m +ech a +Ġne uro +Ġhar mon +Ġlib eral +- ind +D ATA +Ġevery day +Ġdiv ided +ĠActive Record +fig ure +U A +ä ¹ +riend ly +te ch +.game Object +иÑĤ ÑĮ +Ġmo on +ft ime +Ġno ch +ĠT ORT +ĠV M +.in itial +( child +Ġmus ical +Ġo c +b as +ĠH ay +_l ong +Ġmem set +ile y +adel phia +S V +ro at +_t x +Ġl on +ĠngOn Init +b p +ĠGold en +AC HE +Ġwor ried +az i +E ar +T ake +(f p +bur gh +_ Data +g res +ĠO nt +p us +Ġtrans parent +Ġp ocket +Ġr am +igr ations +. čĊčĊ +Ġ[ ( +Ġadopt ed +Ġreported ly +ĠD ream +Ġ} ));Ċ +los ing +Ġte eth +ĠBook s +", & +enn y +LE MENT +Ġg el +ĠPl ant +! âĢĿ +.h ost +ĠRep ly +re ngth +Ġrecogn ition +Ġ}} >Ċ +L A +Ġmir ror +Ġassist ant +( device +Ġspirit ual +b uilder + § +Ġou tr +Ġt t +ĠP ER +Ġrad ical +Method s +Ġp ace +ud y +Ġg ut +ĠG reek +Ġnon atomic +ĠP aper +_G PIO +Ġob st +.A d +viron ments +ĠS ov +( con +ĠTrans action +. assign +ĉc atch +el ter +Ġbit coin +_G R +ĠčĊ +met ic +Ġtrans formation +åı · +Ġr gb +istrib utions +Ġimp licit +/ in +dest ination +аÑĤ ÑĮ +Z ero +Ġun set +. where +.g o +Ġform ation +Ġdeclar ation +() čĊčĊ +ĠEx pl +ĉĉĉ ĠĠ +/ pro +.J SON +Ġdes k +.sub str +//---------------------------------------------------------------- ------------ +ly n +p son +dis able +ĠF unc +ĉ Assert +ĠM ARK +Ġdefe at +Ġbl ind +Ġconst ants +. headers +UIL D +Ġexp enses +P ixel +Ġh r +Ġf el +ĠEast ern +_d el +ĠC ub +Ġs q +ĉc ount +ĠD irectory +Ġex clus +Ġhistor ic +Ġ ------------------------------------------------ +Ġcom position +Ġdata GridView +ĠB urn +ĠB C +M aster +Ġsp awn +Ġbe aring +.Set Active +il o +Ġg allery +Ġfound ed +Ġav ailability +.s qrt +Ġp es +ĠD OM +m ate +O ct +Ġmatch ed +it ivity +Ġan xiety +.pr ice +ĠIn stant +ì Ĭ +Ġt ut +IC ollection +.sh ared +_s ql +t bl +lib rary +_de stroy +erm al +ĠNot es +ĠE in +Ġsou thern +ĠOTHER WISE +Ġmac ro +.l ower +cl s +Content View +.l ink +const ant +ĠB es +Ġsome body +n b +"> { +( local +.. ... +ĠN ull +m x +Ġà § +Ġp ause +-------- --- +_M O +ĠC M +Ġfor Key +ĠD VD +Ġclose st +_DE VICE +ĠSte phen +ĠB BC +ĠTr avel +P aint +ĠResult s +ĠR ule +Ġt p +Ġrat ings +c in +c sv +> / +ĠG OP +l ad +Ġ ÑĢ +Ġindex Path +m atrix += f +ars ed +Ġ} ); +ĠC os +ĠS core +Ġt ak +ĠE SP +ĠIN C +_N ULL +-f lex +"] [ +int o +el and +Author ization +_F ALSE +Ġg ate +Ġv id +ist ent +T IME +Ġre write +Ġt ie +Ġarch ive +.event s +.get Parameter +ĠPer mission +Ġprogram me +Ġ é +j ud +Ġcam eras +(s ys +ĠSy rian +Ġimpro vements +Ġh ip +Ġsu icide +Ġsch olar +Ġcompat ible +rem ote +.d own +F UNCTION +Ġman aging +ĠUI Kit +. raw +>> >> +Ġdem ands +ell ite +Ġd ent +ĠM icro +åı ĸ +'] [$ +ĠI E +im ension +Ġt rem +Ġg ained +.w ith +. ok +h ou +Ġb om +amp aign +Ġjoin ing +f ish +Ġadd Subview +Ġnor thern +.c or +ore t +D ie +in ish +_com p +Ġatt ended +Ġcoll apse +ĠS S +ac ent +_E QUAL +ĠDe ep +R GB +ĉ test +ol ves +us et +Un ityEngine +w riter +Res olver +, % +if ference +_re move +ond a +Ġfem me +de code +Br anch +Ġfl ush +Ġinnov ative +Test s +Ġ[' ./ +Ġcover ing +. admin +ultip art +(l ambda + namespace +ĠS port +Ġ! ( +ac les +Ġde pression +ĠK ong +Ġp ert +ĠCon n +ĠOther wise +/ home +s upported +Ġp ink +Ġinv ited +ñ os +_en abled +Ġ- Ċ +F W +en ers +ĠM Y +Ġsuggest ions +Can vas +Ġf er +ĠMarket ing +@ Test +unt u +ĠV en +ĠC ou +iv als +Don ald +lim ited +ĉĉĉĉĉĉ Ċ +Ġanal yst +( entry +Ġrepresent ative +_at tributes +Ġf ur +.h ide +res p +ado res +rid es +ĠJ osh +ro bot +ĠN AT +Ġs esso +Ġintegr ated +: true +part s +Ġst upid +: event +@end section +Ġp u +.T able +ĠY ii +` ;ĊĊ +Ġcl ang +=" "> +eng an +_param eters +.int ernal +ĠMod ern +Ġmet ric +Ġsem i +={ {Ċ +.am azon +ĠB B +aint y +view port +Ġstart Activity +dis patch +**** * +Ġfl av +iffer ent +[ this +Ġst ake +Ġarg ued +vious ly +.w ork +ĠO ak +O ld +( async +not es +Ġfl ip +Ġdis ag +ĠT E +ĉ error +< ' +Ġ» ĊĊ +Ġfilter ed +ĠM ach +Ġh ung +_d ump +_s amples +-dis miss +Ġr ay +Im plemented +D K +Ġj ed +Ġbreak s +Ġf its +. gr +ĠZ ero +or o +Ġequ ally +Ġ' [ +Ġconcern ing +< meta +play ers +_P OS +_s im +J an +Ġyour s +ĉ N +Ġsp ir +Ġch ampion +ĠAn alysis +ap a +ĠNS Log +_l ines +ñ a +ĉĉ ĠĠĠĠĠĠĠ +.S c +Re p +etro it +ur able +M IT +com pat +own ed +_ind ices +], čĊ +Ġdis covery +ĠDie go +ob i +. Index +Ġtrend s +PL AY +.n o +Ġl ens +_c fg +Ġan no +ag an +Ġperiod s +ter ms +y z +Ġattack ed +ib ration +PEC IAL +_ grad +Ġaccord ance +.Read Line +.de vice +ri x +. container +m ay +erc ise +ĠL u +Ġr g +ĠÑģ ÑĤ +ĉĉĊ ĉĉĊ +( un +TERN AL +Ġless ons +Ġalleg ations +Ġtrans mission +.Re f +M obile +ĠT ournament +ĠN ut +ĠG a +ĠCap ital +def inition +- exp +c lean +Ġfant asy +Ġenh ance +ent ence +'] :Ċ +ack ets +Ġcelebr ate +@ ", +Serialize Field +Ġarray s +t b +ĉ st +[ assembly +( reg +.c ategory +Ġimpro ving +Ġsal ope +Byte Array +Or iginal +Ġ[ {Ċ +åĽ ŀ +ĠCl in +oen ix +ĠS amsung +Ġmaint ained +Ġag enda +f ail +Ġpres ents +Ġtim ing +.m ark +' >< +Ġprom ot +Ġin cl +_ only +ë¥ ¼ +ĠAtt orney +- date +Ġlands cape +Ġf u +S Y +.p rop +ĠA rr +p ag +Parallel Group +': čĊ +Ġlog s +a unch +unc i +n ama +Table Cell +iss ues +. { +ec urity +_ex ec +old s +Ġhost s +Ġpro to +_ import +_s ort +ĠB ow +ĠN ormal +ĠF arm +.create ParallelGroup +R otation +. err +Ġp leased +it age +.W h +ĉĉ ĠĠĠĠ +M R +ĠM ORE +ĠN atural +_ transform +B ASE +ener al +ut down +.common s +W T +Ġa an +. Result +d og +Ġclick ing +), ĊĊ +# line +Oper ator +Ġc iv +Ġm erg +ob uf +ng then +Ġ[ { +Ġcan cell +tr igger +. : +W ORK +decl are +Ġdecre ase +ÅĽ ci +lo om +.N one +ĠM I +ĠJ ason +Ġhealth care +iam ond +s ylvania +* x +ĠR a +[ b +Ġprint ing +ph abet +ĠLab our +op per +Ġz ijn +-t arget +_F UNCTION +Ġo ct +ени Ñı +åľ ¨ +Ġwest ern +Ġcomput ers +ĠR ET +Hash Map +[ String +get Value +_D ATE +.N ext +ĠF if +é l +ick ed +æ İ +-M M +Ġ{ ĊĊĊ +Ġcontact s +Ġdig its +Pro du +Ġunus ual +Ġrapid ly +t ures +Ġang ry +c ancel +xx xx +_p arser +id ity +_P REFIX +Ġme hr +Ġrare ly +et he +op es +Ġ% . +work s +Ġthe ta +Ġcontrib ution +ĠT ony +Ġsqu ad +аР¹ +Ġî n +th ere +out ed +ĉ q +Ļ Ĥ +g ood +L I +é¡ µ +ĠL iving +iz abeth +Ġk t +ĠD allas +] ],Ċ +Ġ/ >ĊĊ +Ġrais ing +/r outer +_g ame +ĠC UR +z ens +. es +Ġfont Weight +(f unc +not ification +Ġ'../../ ../ +Ġbl ame +ãĢĤ ĊĊĊĊ +an co +Id entity +f ollow +Ġart s +x s +Ġofficial ly +ĠSt udio +Ġrecommend ations +Ġloc ale +Ġam ateur +ĠEn able +Ġcap s +. End +- add +_g shared +ĠC T +For ce +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĊ +Ġor ange +Ġl p +Ġanswer ed +.G rid +Ġd ual +Ġstrateg ic +Ġnob ody +Ġf atal +_ est +( el +Ġì ł +ĠB udd +A IT +_f actor +- one +ĠH AVE +" čĊčĊ +Pro f +Ġä r +str ings +Ġdir ty +ĠF ace +ĠB egin +ĠB us +Ġw is +åŃ Ĺ +Ġspe aker +Ġcar rier +ĠO m +Ġhad n +All ow +:: __ +Ġver b +ĠCom plete +ĠE asy +Ġb ills +ĠĠ ĊĊ +Vert ical +Ġpr on +ĠDef ine +Ġlook up +variable s +Ġpand as +um es +Ġinn oc +Ġset Up +ĠCh ampionship +art ist +ĠC Type +F oundation +à¹ Ī +ĠSet up +Ġrec ipes +ĠU IColor +ĠF ight +Ġauthor ized +_c lick +_s uccess +ang an +ĠMount ain +ĠDo ctor +Ġeg g +ĠMedic ine +c les +` .Ċ +[ int +d ashboard +ĠApp ro +-d r +Ġprodu ces +Ġrent al +Ġre load +Ġarr ival +sp ot +Ġund ert +Ġequ ipped +Ġpro ved +Ġcent ers +Ġdef ines +al so +Ġop acity +ĠUn fortunately +ĠIll inois +Ġн е +ĠTem ple +ĠTr ail +ĠK elly +Ġmeasure ment +Ġsepar ated +-c ircle +H ey +ĠRE AD +ig its +Ġ ib +ĠM OD +atter y +аР· +Ġv end +ен ÑĤ +ĠHttp Client +s afe +_A SS +ic it +ĠCon struct +ĠC lo +ĠS ix +_T OKEN +(b lock +Ġwarn ed +/* ! +! Ċ +Ġinnov ation +_ " +Ġ );čĊčĊ +Ġsp ots +Ġcho osing +.c s +Ġflex ible +U Int +Ġscr atch +- al +Ġf estival +Ġout standing +================================ ================ +M ean +ĠO regon +s ymbol +. account +d ney +'' ' +! ", +Ġpart icle +à ĥ +[ MAX +IV ER +ER ENCE +NS Mutable +ĠColum bia +_ ĊĊ +.f r +Ġc ogn +V R +ĠMethod s +ĠM ade +ĠB R +ĠEl se +Ġeg gs +Ġsw ing +ĠIn v +Ġdise ases +Ġf irms +Ġle mma +}` );Ċ +l ings +Ġg ym +umin um +.T rim +M em +Ġcritic ism +ibern ate +_T X +ion i +Ġguid ance +Ġrepeated ly +Ġsup plier +Ġpaint ing +.F ragment +ed Exception +Ġw iring +Ġcour ts +W EB +æľ ī +\ . +ill ance +Ġb rows +ĠP attern +PL ICATION +ĠSum mer +Ch ain +Ġc ute +mer cial +Ġd il +ĠFrank lin +ĉg lobal +IN CLUDING +h istory +Ġl st +Q t +SD L +al ia +i ere +( ... +ĉc in +iff s +vel ope +ĠR oot +cl uster +User Name +ign e +< S +Ġf est +Ġindic ating +ke eper +Ġc ada +é g +cons in +ĠG B +Ġl b +em ony +-icon s +_d oc +Act or +e lem +.De lete +Ġin fection +ĠPriv acy +Ġgreat ly +ĠP os +ĠT reat +Fl ow +Ġattract ive +ĠMar c +s udo +tes y +- an +ab ama +ĠW ould +Ġsu ck +index Path +ĠE t +T imes +Ġclub s +_ass oc +Ġac quired +(" : +Ġint ense +.m aps +Ex pected +T oggle +Ġa y +Ġl ifestyle +-c alled +ĠS now +V olume +Ġcann abis +ĠD irection +ĠLim ited +-s pecific +Ġd owntown +/ icons +Ġre ven +L eg += null +Key board +') ). +Ġ"" ;čĊ +Ġatt itude +.n avigate +- error +AM PLE +ĠJ ay +v r +c ow +.com pile +Ġmem ories +_m ark +ĠMin nesota +Ġk osten +Ġprob ability +w arning +Ġgen etic +F ixture +ĠHash Set +N ombre +_m onth +Æ ° +- start +xy gen +ĉ ft +i agnostics +ĠMat thew +Ġconcept s +Ġcon str +. State +и н +N ov +Î ± +ĠP anel +ä¸ ª +com pare +> ()Ċ +Ġapply ing +Ġprom ised +Ġo x +nc ia +ĠValid ation +ort s +_c ur +e lect +ey e +( Data +Ġreport er +ĠB uff +Ġs r +Ġ" ; +ick y +Ġtemp or +S N +Ġres ident +pi res +ys ical +Ġend orse +ĠS ong +is Empty +le et +_ util +Ġdist ingu +ĠT alk +ĠM ot +( default +.A rg +gorith ms +_ words +im mer +_res et +f amily +W W +Ġsav ings +ĠâĢ Ŀ +_en able +side bar +Run ning +Ġal i +Ġtest im +Ġwarn ings +ĠCh em +ĠEx it +Ġfound er +pect or +Ġr m +_d ataset +ĠD as +Ġh an +Get ty +á l +Ġn y +Ġpo verty +Ġresult ed +.b y +ĠVis it +Ġobt aining +/ '.$ +ĠĠĠĠĠĠĠĠĠĠĠ Ċ +sh all +_LE FT +UI Image +_ Name +h ave +ĠN ob +l r +- footer +Ġn aked +ĠG arden +\F acades +Ġgrad uate +Ġfranch ise +pl ane +Ġcontrib utions +Ġstring With +Ġc rypto +Ġmov ements +ath ers +Ġlif etime +Ġcommunic ate +j ar +ĠFr agment +_ IF +ĠN avy +ĠF igure +Ġsim ulation +_st op +Ġreport ers +Ġvers us +aj a +ĠÎ ± +Ġgovern or +List Item +Ġse aled +.Back ground +ed i +ash ing +Ġl ip +ĠI h +mer ge +Ġn ec +el ocity +ATE G +Ġse eds +Ġflo ating +_F A +w alk +ĉ user +_de pth +Ġw age +@ app +N il +( [" +( vector +Ġsecret ary +Ġj Panel +ve z +³³ ³³ +d irection +ĠE P +Ġh unt +Json Property +ĠP ORT +] ", +аР¿ +ĠFore ign +pan ic +Ġtri als +ĠA le +Ġr ural +- value +author ized +ĠScot land +.d rop +ĠM T +ç ± +row th +File Path +Ġrec all +if le +Ġc el +ĠSE LECT +k n +_c ase +Ġc rop +s ure +p ot +IC S +Ġst em +Ġindust ries +P ut +Ġa ber +road cast +Icon s +) ")Ċ +æĪIJ åĬŁ +g ui +Ġassum ed +Ġr x +E A +è § +EL L +Ġdo se +Ġin e +Ġde eper +l ider +Ġord inary +Ġg olf +_IM AGE +ĠN AME +(m odule +Ġat om +Ġbel t +Ġoff ices +b eta +Ġphilosoph y +( JSON +-f ield +Ġintrodu ce +Ġconven ience +opt im +> "Ċ +ath y +Ġemploy er +qu ate +Ġed ited +Arg uments +ĠN ations +__ ) +Ġno se +ĠS ample +' )ĊĊĊ +Ġc ake +.get Attribute +H D +Mod ified +Ġpredict ed +Å Ħ +an ie +S orry +(d oc +w ind +ie ve +Ġprov isions +AT ER +OT E +M Y +.A utowired +ĠB ath +. Boolean +Ġback end +.M ouse +ater al +p aper +Con st +ĠV R +_ entity +_C TRL +ĠProte ction +ĠG M +ĠStud y +Ġsou p +ot ime +' use +] " +/ users +a ug +ĠH ong +_n orm +ãģ ¨ +Ġse cre +(B uild +ĠCon tract +ol as +Ġsa uce +Ġaggress ive +Ġrac ial +char acter +@ @ +Ġcomp ile +ĠV oid +_re m +_m emory +k k +Ġm ic +S ame +U tility +ĠH tml +ĠX ml +Read y +Ġg all +Ġalleged ly +ĉĉĉĉ ĠĠĠ +ĠMet al +ĠPerson al +Ġborder Radius +rx js +object s +Ġwant ing +Ġb owl +v endor +offset of +ĠR s +ĠR ating +Ġr ally +_N ODE +ĠM ix +Ġadvert is +Ġnarr ative +s al +Ġm c +SE rror +Ġf ingers +Ġaccom pany +Ġt ired +Ġstr ide +Ġgu i +el ist +Loc ale +Ġrele ases +ik ing +Ġan ger +)) )ĊĊ +alle st +Sum mary +( O +(f or +Ġbasket ball +Ġroad s +ĠInst all +ĠF ab +it map +Ġ) )Ċ +Ġinter section +ighb or +ĠB ry +ĠHER E +So ftware +elf are +ac s +Ġtrail er +.get Class +ch ars +Ġreg ulation +Ġref ers +Ġde struction +Ġcontin uous +ĠAust in +é ¢ +ak an +.w indow +ĠTem plates +Ġabs ence +: n +Ġdis order +fl ash +Ġde let +bo ards +ĠĠ ĉ +RO P +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġac qu +Ġlaws uit +ĠRe views +Ġgar age +t imer +Ġe j +ĠRect angle +Ġflow ers +il st +ĠIn stance +S uper +d et +dis posing +ĠE S +ĠI C +ver e +S k +_ch annels +put ed +/ null +nn en +ĠG allery +_g lobal +Auth entication +ĠR ank +Ġblock ed +Ġcal m +mark et +ĉ val +Ġa ug +per iod +ĠCon stant +Ġ?> ">Ċ +Ġl obby +p al +Ġs ink +ia h +Ð ¡ +urn ame +Ġcon ver +Ġinvestig ate +Ch rist +H ub +ĠIN D +ĠP ed +ur as +ĉ url +ĠT ro +Ġpre ferences +Ġguarante ed +` ĊĊ +Ġport ions +Ġeval u +' > ;ĊĊ +.AutoScale Mode +Ġc ats +Ġreg istry +ul us +F I +p ayload +- search +Ġstay ing +ac ious +Dec oration +Re view +In f +Ke ep +it is +, String +Co ord +Ġper o +S ex +ĠAtl anta +uest a +Arg b +> * +} _ +F ooter +Ġemploy ed +_b ound +v ide +.f unc +$ scope +Ġsp o +ĠAn al +ounc ed +ar ound +Ġrestr iction +Ġsh ops +å Ģ +ĠLat in +-c ol +Ġbare ly +ĠE uro +E r +Ġfa ire +_d istance +_un lock +Qu ote +IV ATE +Ġå Ī +Ġaim ed +ĠRet rie +. iter +Ġwr apped +Ġagre ements +str ument +( product +Ġstud ied +.set Value +Ġy e +ĠC ache +MB OL +Ġquarter back +Ġsy ntax +.getElements By +.v ersion +we bsite +Run ner +_s ingle +at iv +ĠAl tern +ĠBeaut iful +right arrow +Ġd iversity +pl ash +( co +.F ill +Ġtyp ing +Ġcl ar +H it +O O +ac co +w orth +Ġscript s +ĠMuslim s +ĠL L +erv ing +( boolean +Ġbase ball +ĠC AN +MA IL +de pend +Ġrespect ive +Ġconst expr +.* ;ĊĊ +'] ))Ċ +Ġy ard +Ġident ical +if ecycle +US H +up iter +. validate +cl i +IST ER +Ind icator +F ail +Ġdemocr acy +. var +Ġsatisf ied +------------ - +enc er +h or +Ġr ounds +DA O +o a +Ġfl ask += c +[ ]Ċ +/d ist +Ġpart e +Ġconfirm ation +er on +aw are + +Ġdepend encies +ĠV ideos +- row +Ġ** /Ċ +Ġn ou +Ġh over +æ ŀ +Ġn in +ĠUS D +M ac +_L oad +Ġout comes +_s ocket +Ġqu eries +w m +Ġhit ting +in ux +M ich +ud ge +AT AB +Ġvulner able +ä ¾ +Ġport folio +: YES +ĉm ap +B ound +Ġiter ation +in cess +Ġact ors +ĠQ ual +_c lean +ãĢij ãĢIJ +MS G +G reen +ĠOff icer +Ġsm oking +> ', +ĠF lo +++ ; +oly gon +Ġbul k +Ġdr ama +Ġexception s +os ed +Ġ+ čĊ +Ġleg acy +C V +Ġcontrib uted +ĠTer ms +Ġb t +Ġunt uk +Ġal ien +=== Ċ +ĉ Vector +Ġl s +On line +.f acebook +num eric +ock ets +A ut +b ury +-re dux +ĠRed istributions +GLOBAL S +urrenc ies +Ġt ons +âĢĻ , +Ġà ª +(c ol +ĠS ymbol +Ġstay ed +ĠM L +Ġm unicip +Ġsex o +S en +n r +Ġg ains +Ġshort ly +.M enu +à ½ +KN OWN +Ġoper ators +- V +ĠPat rick +/ add +_C O +ir ation +(p ost +Post s +/ _ +Ġpl ug +Ġintellect ual +Ġmet ab +Ġpregn ancy +ĠPrem ier +n m +Ġpred iction +ĠMin istry +Th ree +val uate +ĠMin i +b u +оР· +< ul +Ġd d +ol ving +ĠC ut +Ġs chem +.tr ain +it ate +Ġr ice +Ġbird s +ãģ « +m iddle +struction s +Ġn erv +a que +Ġfl u +Ġsurv ival +ĠGal axy +ĠF ant +. Order +At trib +irt s +é c +M ovie +Ġcon ce +qu arters +Ġm ood +.Add Range +Ġres olved +ãĥ Ī +Ġburn ing +ĉĉĉĉ čĊ +ĠW E +Ġhost ing +L AB +Ġman agers +Ġstre ngthen +< const +ĠFire base +on ed +ĠJ ean +' ";čĊ +ĠS av +.B old +Ġen ables +ĉt mp +Ġman ually +ĠS qu +user id +.f unction +.c ache +LO PT +.S ervices +dd it +t im +< img +ĠTh ings +ĠEvery thing +Ġa pt +em and +Ġroll ing +ë ¦ +. level +Ġst om +ĠW inter +Ġview ing +( values +ocom plete +v ia +up o +Ġabort ion +i ère +ï¼ ij +_B UTTON +_d omain +Ġb ra +ĠA st +in as +Ġstat ist +c od +L R +Ġdr ives +Ġfollow ers +Ġall ies +ĉc urrent +ecess ary +Ġdam aged +_ pt +and les +oun tries +Ġsim ult +e u +Ġcontrovers ial +_G ROUP +Ġr ib +. Info +: mm +.n ormal +_ADD RESS +Ġ íķ +add le +ĠD ur +. Element +W arnings +Ġcred its +Ġin hib +Ġem issions +Ġh az +.y outube +ugg ed +Ġbo ther +ĠK ansas +ĠF ixed +ĠTest s +ĠF IX +Un iform +Ġk ont +>> > +st ation +lo re +at ype +ish op +/ **************************************************************** +Com boBox +Ġvac ation +Ġiniti ative +Ġdefault Value +con cat +ĠK h +ĠW elcome +ized Name +M igration +Ġgrad ient +H ot +Ġhard ly +el o +ĠStud ents +Ġlo ose +at z +.S end +' / +Ġunivers al +Ġenter prise +Ġreg ex +Ġvis itor +ĠF ly +Se q +à¸ Ļ +ĠVis ual +Ġlib raries +ato es +P ayment +Ġp ent +Ġgather ed +VRT X +ĠD M +S plit +Ġlet ting +Ð Ŀ +_error s +ep och +P ARAM +c u +ÑģÑĤ в +ol utions +Edit ing +font s +Ġalloc ated +ĠB ased +( Y +ĠJud ge +Ġbro thers +FILE S +ç o +w b +_P I +' ^ +Ġs word +.s ervices +Ġn l +T im +ig g +ĠMo ore +Ġcrypt oc +åĩ º +_post s +ot ate +? ' +... .ĊĊ +Ġk l +=" $ +Ġdec oration +Ạ¡ +ĠD IRECT +G UI +) =>{Ċ +Ġnews letter +Ġprec is +(p oint +ĠEqu ipment +ut y +ĠD ave +Ġparticip ation +u arios +x it +.A s +ET ER +or ous +Ġsh ield +[] > +ilit ary +. origin +Ġprom otion +U nt +Ġc t +TR A +View Holder +Ġsig ma +d elta +are house +con tract +( Vector +Ġcompet e +/ form +/ components +Ġn r +ĠInd ones +Ġо ÑĤ +ĠV olume +.f iles +(res p +/ models +Ġsur f +stand ard +/ o +ĠXCT Assert +V ICES +.C ode +SE D +Ġact ivate +D elta +Ġlimit ation +ri j +Ġpregn ant +: ^( +Ġs our +p ie +Ġexp ense +ic ation +ĠL arge +Ġ ± +ĠB owl +(model s +/ N +P a +.re load +Ġwonder ing +Exec ution +ĉ ĠĠĠĠĠĠ +ĠG raphics +ĠCont in +_j ob +Ġget Name +ĠM agn +ĠD WORD +m ad +Ġn h +fe atures +} ");Ċ +he ets +(tr ain +z n +Ġrecru it +.con nection +Ġbar rel +Ġste am +_set ting +Ġang ular +ane ously +Ġb il +ĠN orm +(! $ +ib t +% ( +Ġpos it +ĠF ather +int endo +L ive +Ġport s +Ġme j +Ġland ing +pon der +Ġc od +_HE ADER +.M argin +Ġball s +Ġdiscuss ions +Ġbl end +H ex +Ġfarm ers +Ġmaint aining +ĠĠĠ čĊ +s yn +[ T +r us +uff ers +Ġcontrib utors +_s ys +.De bug +Ġconstruct ed +om es +? id +sl ider +Ġsup pliers +scri ber +p es +Ð ŀ +": čĊ +\ Controller +)) ĊĊĊ +Ġl ua +M ulti +EN S +S rc +Ġpet ition +Ġsl ave +look ing +V ERT +ĉ vector +S pecial +h h +an ne +ĠN iger +/ views +z ing +end ant +< C +s peed +Ġ{ };ĊĊ +Begin Init +Ġf open +@ RequestMapping +End Init +Ġp unch +S ender +é Ķ +get Message +/t ypes +.P I +(' ');Ċ +oc used +( all +Ġdrop down +). __ +ĠV in +.Fore ignKey +can f +ou red +ĠOrgan ization +ĠÐ ° +ĠC ulture +(cl s +, _ +rg ba +ìĿ ĺ +.data GridView +Ġdo zen +ĠG es +_sh ared +n ick +Ġh osp +om eter +Ġclaim ing +ib les +ri k +æĺ ¯ +en ario +Ġd engan +ob b +m ont +_r ank +('/ ', +Ġap olog +P s +_p ower +ĠG ree +Ġful fill +Ġfire base +Ġf are +ĠH im +Ġbe an +â̦ . +ĠS PI +_R X +Ġper ception +rel ative +comp ile +u um +ut os +a uc +ĠAs k +Ġindic ator +/ th +.set String +ĠWis consin +.D omain +Ġart ificial +De velop +ĠSar ah +Ġl ying +( search +ĠEmp ire +urr ing +æĹ¶ éĹ´ +=" ${ +Ġget Id +ĠP ayment +trans ition +Ġ ]. +ix in +V T +- select +Ġdemonstr ated +Ġlast Name +employ ment +.get Property +Ġf ought +file Name +ĠP ers +-c ard +a str +attr s +Ġprom inent +Des ign +anc ouver +ãģĹ ãģ +ard o +se cret +Ġr ag +Ġpo ison +-m an +, omitempty +ĉ un +it zer +ĠCas ino +ĠR oss +- foot +(result s +Pl an +Ġlas er +ê¸ ° +_D R +F acebook +Ġbo ards +st a +] ], +Ġt iles +S IZE +Ġ= ~ +Ġprem ier +oc ab +Ġenc oded +Ġres erve +ĠAfghan istan +ĠList Node +url s +Ġsub mission +Ġne u +Ġ# +# +_P OST +Ġmo ist +ell i +ellig ent +. alert +ó d +b re +ĠCol lect +Ġgraph ic +Ġlong itude +ĠPro vid +ĠCal culate +x ffff +c riteria +Ġw aters +ro ck +lo quent +ĠT rib +Ġbur st +Ġsuff ix +.Ext ensions +ish es +iv el +ĠLI KE +ĠGet ty +.Action Event +.s lf +ĠH AL +up al +E AR +ud i +_time out +U F +ĠSing apore +ĠAd vent +_int erval +cha ft +ĠE mer +Ġtele phone +ĠTur k +_ interface +ĠO wn +Ġencour aged +< Object +_T ext +ĠOnt ario +ĠApp ly +.f irebase +Ġant ib +P riority +ene z +D ays +c id +urre nce +; / +inn ed +Ñģ Ñı +Ġve z +f w +// $ +att ack +Ġstart up +ain ers +.f ragment +op acity +( conn +he im +.n etwork +( stream +ĠN ON +t ol +ĠX box +ĠD S +Ġc ached +Ġprostit utas +ĠB alt +(' [ +Ġno except +" ' +Ġs d +. valid +_ ag +Ġr aces +Ġro d +itud es +< >( +.Pro duct +Form s +NE W +P ay +ĉ boolean +_ contact +ĠElect ric +sk ip +Ġw ur +Ġch ronic +_d river +ĠS ab +ĠU lt +ĠR ad +ST ATUS +ĠLew is +O B +Ġgift s +.Re c +TR UE +Ġint ensity +Mark er +.com pare +ff ic +C ookie +ĠB aby +ĠBig Decimal +ile t +ĠHOLD ERS +ĠL ady +Ġl ung +ĠAl abama +Ġd ess +` );Ċ +ĠB uilder +_reg ion +Ġne utral +Bo th +Ġh p +Ġh orn +Ġseg ments +ĠE C +"=> " +( rec +ĠP i +G M +Ġl aptop +Sc alar +is d +-d ialog +ĠAnd erson +Ġmist akes +ĠH an +j es +est ination +Ġprom ises +b id +ĠSc ient +G IN +ĠPer formance +b age +. users +le ading +Ġor al +G raphics +_P TR +h ang +Ġin ev +process ing +F actor +ĠN A +$ string +Ġground s +.Save Changes +c lock +cri pcion +ĠNew ton +g c +.in cludes +Ġbl ast +Ġ'- ' +Ġpued e +.S ession +Ġgre p +_f inal +ĠG ay +ĠG ive +ir i +-st ar +ĠUI Image +_ep och +ub b +ent h +Ġel ite +Ġcampaign s +ĠP orno +_ assign +Prot ocol +ĠBe ing +ĠAir port +Ġconvent ional +ĠW at +ĠC I +ET A +ĠAnth ony +Ġtable t +( format +Ġconsist ently +ĠI owa +Ġav atar +.c ursor +! [ +Ġh anging +H er +S uch +';ĊĊ Ċ +orge ous +() == +Ġview Model +Ġ ãĥ +Ġel s +ĠAg ent +F etch +ap or +Ġc x +p read +ĠP ier +oe ff +S n +ĠV irtual +A pr +.Wh ite +_M OD +ĠPoint s +å¤ ± +Ġgen es +Ġv endor +Ġmain stream +< src +ĠEl izabeth +Dec oder +- state +ĠG lass +nc y +adi ans +_m on +ĠRem ote +Ġwire less +ĠM i +å ī +è¡ ¨ +st age +ĠT ile +ll ib +V ariant +== Ċ +Ġgold en +(Q String +.put Extra +ĠD om +ĠAn imation +Ġinter active +if act +éĻ ¤ +LE T +Ġfrequ ent +Ġ< >Ċ +F ilename +Ġs ne +ĠFoot ball +Ġr ival +Ġdis aster +ion ic +ĠD amage +. Resource +- en +ĠT ypes +get String +( board +Ġb ol +pl ain +z ym +ภ² +Ġsc anner +ild er +_msg s +æ ı +(int ent +Ġde struct +Ġb ust +ĠE mploy +on i +ĠUI ViewController +Ġodd s +ear er +Ge ometry +Ġy ii +_EX PORT +ĠAtt ack +Ġn iet +Ġim pression +ĠG il +_pro b +ĠC F +ĠEx perience +/pl ugins +.M ethod +Ġbelie fs +N ative +_b uild +Ġv ig +Ġr anks +cover ed +s uch +G uard +.p ack +add er +iv ia +l ng +Ġв Ñĭ +T imestamp +_n ow +Ġp oker +Ġun c +Ġsh apes +-t ypes +_per iod +p k +Ġveter an +Ġson o +Ġappoint ed +over flow +.d river +_c at +ut t +pl ant +im b +ĠAc cept +Ġconc ert +ĉ node +ĉ z +? >čĊ +Ġb anned +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġto xic +Ġdisap pe +È Ľ +Ġgr ace +ate ful +Re ply +ĠCru z +Ġsc rap +Ġkey words +s imp +Ġmort gage +Ġcy ber +ĠEx ecute +Ġlat itude +if u +.C OM +d bo +Ġsort s +ĠG as +om ial +.L ocal +Cell s +.Re place +String s +.f it +ĠTh ird +% ",Ċ +Ġ{} ". +ĠS ony +Ġ[ : +Ġfall en +. ')Ċ +in h +ĠM C +Ġred is +C odes +Ġprofile s +h ook +Reduc er +_F UNC +Ġn avigate +str len +Ġh orm +á ŀ +ĠS R +. boot +Ġdig est +ĉ header +.find One +æ ģ +Db Type +n ia +_m erge +Ġdon ne +/ Getty +_CH AR +Ġb ands +. URL +art ial +Ġf req +Ġs ist +N g +Ġrender ing +\ Core +Widget s +ĠV A +Ġactiv ists +St e += _ +all a +St amp +Ġload s +Ġx x +ĠL earning +.M vc +u ir +(" $ +Ġconnect ing +Read Only +ur u +ĠE ag +B IT +_DE L +å § +arr ass +ext ernal +ĠY OUR +ĠB rew +ĠF ive +Ġres ize +ig id +er ation +ĠÑ į +åĬ ł +ĠC atch +Ù ģ +ĠLe on +am il +.B ody +Cl ip +/ list +.b r +Edit Text +ĉ db +.G ame +(Build Context +back end +.R ed +face book +.url s +m r +rol led +---- --- +Ġinter vention +Ġretire ment +ĠK it +ĠP RE +Upper Case +ĠS ocket +Ġ: - +Ġstudy ing +ĠMet ro +ard ed +Ġconvers ations +C alled +Ġexam ine +ert ificate +.g z +-res ponsive +Ġref und +_n etwork +allow ed +em pt +Ġme als +C ategories +Ġtravel ing +Ġk g +Ġsh ame +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġexplicit ly +Ġmath ematic +ĠS uite +ĠR GB +****** / +Ġmix ture +lear ning +.t emplate +att s +w x +ĉ ctx +.p roperties +Ġdrink s +ĠE ither +set Text +.get Data +.z ip +Ġreve als +< table +.Hash Map +ĠH ur +) ");Ċ +.f ramework +ĠST ART +feed back +Ġsaf ely +. icon +config ure +. lock +.l ayers +/> .Ċ +Ġrank ed +_ impl +ĠHand les +Ġhost ed +Ġup dating +al bum +é Ŀ +Ġsh ader +Edit ors +- round +[] { +Ġse p +ĠH i +TE M +look up +.m an +_IN PUT +Ġthreat ened +_IM PORT +Ġd rops +ru it +s id +bo th +ĠEx cel +Ġj er +ord inary +еР¹ +V IEW +re ply +Ġ) :Ċ +color s +ver ified +_T r +_p arse +Ġcon gress +P romise +int s +ĠM other +.A pi +ĠD uration +Ġfirst Name +inherit doc +ĠM ars +Ġa pr +OD Y +Ġvis its +Ġhe aling +let ters +)) );čĊ +f uture +.F ramework +Ġk iss +Ġinv olve +Ġsil ent +ad ows +Ġany body +s ch +Ġsole ly +- img +Ġprop ri +Ġin struct +Ġlic enses +Ġm eth +Ġcond em +ĠD omain +ĠHarr is +Ġs Ã¥ +CE PT +B atch +@ extends +ĠCONTR IBUT +.Data Frame +_p acket +rec ision +Ġfoc using +. ht +__ ":Ċ +: Get +ĠK C +Ġpass age +Seg ment +_c enter +-z A +_B L +Ġconv in +Ġclass ified +ĠNS Mutable +_ ap +t ile +Rect angle +(n ums +v ens +ĠUI Button +ĠF eder +am o +Ġout line +ĠPar ser +Ġâ ī +ĠWork s +.S chema +Ġeng ines +_com mon +_ old +Ġset ContentView +Ġ/// < +ĠB T +f m +Ġd ivers +_ weights +em ark +ĠA CT +Ġpro portion +over lay +.dir name +ĠG it +_REF ERENCE +< > +l b +_r ule +è´ ¥ +ĠPut in +Ġsleep ing +() :čĊ +Ġpres erve +Ġpar liament +ĠLook ing +Ġpick ing +ĠDis patch +Ġsl ip +ë ĵ +ĠL yn +_sign al +config uration +ĠP itt +ad en +pro cedure +Ġenthus i +f ight +ĠCons ider +Ġt orn +Conn ected +.c os +_group s +ĠTh ink +Ġdel iber +Ġres id +work ing +.column s +ĠCal led +Ġes lint +> ", +_D OWN +h ist +ĠAdv anced +Ġre wards +act ors +Ġsil ence +Ġmy th +Ġne ur +Ġa uction +.Get String +ek s +( project +ĉ msg +ĉ output +Ġcomplaint s +, S +Ġt bl +Ġ, ĊĊ +ri ors +ah ren +Ġlawy ers +re dux +_s ymbol +off ee +_RES ULT +( Name +UT C +.current Time +Ġorgan is +. arg +Ġmin im +w ick +Ġrece ives +B alance +Ġspeak s +ĠD ays +ĠBel ow +t ipo +P resent +Ġres erv +h p +Ġr it +_R IGHT +-- ) +Ġchair man +D IS +ĠBO OST +Ġexper iments +__ );Ċ +Ġst amp +Ġf ert +Ġf ond +T er +el ve +ure n ++ i +end ency +Ġvirt ually +... " +ï½ ŀ +- cent +_un ique +Ġpr icing +m ic +RES H +Ġ:: : +Ġan notation +ĠC ircle +ong odb +it as +Ġ% ( +( component +Ġо б +( port +-h our +. obj +L BL +Ġj ury +GB T +Ġsp y +ĠProf essional +Ġ"" ;ĊĊ +Ġstri king +Ġdiscrim ination +Ġp ays +lic t +ent es +Ġthrow ing +ĠPl ugin +( def +ĠRuntime Exception +ĠM igration +Ġd ic +b ag +on ia +Ġcor ruption +( Map +Ġpr z +.d to +Ġac quire +State ToProps +Ġlo ving +оР¶ +_p attern +Ġemot ions +Ġpublish er +_b e +Ġcoup les +o j +ĠCh art +Ġt rop +.t ool +Ġestablish ment +Ġd ol +Ġto wer +Ġl ane +ĠSy dney +Ġfill ing +claim ed +Ġdialog ue +Ġcon vention +book ing +pare ncy +æ ± +ĠGener ic +\ Schema +Ġr anges +/ ch +Ġpan els +Ġr uled +çĶ Ł +.t s +_s ets +Ġclean up +Pre vious +ĠAn imal +($ ( +ĠA ve +oll ar +_e val +ĉ Name +(t ree +Ġ" ] +Ġdut ies +=' / +Click ed +Ġdifferent ly +ĠCl ark +Ġd it +olog ists +Ġsy nd +Ġs ends +- known +k b +ĠMod al +it ative +Ġr acing +Ġhigh lights +ĠSim on +ĠCapt ain +ä¿ ¡ +ĠC B +cont in +ar an +Ġphys ics +ret ty +et al +.m d +ax ios +Ġspeak ers +Ġpre p +Ġaward ed +ì§ Ģ +ĠC orn +ĠN ature +UD IO +Ġpro j +- pre +[ u +Fe atures +Ġis Equal +B inary +s ig +Ġconf usion +ĠH at +Ġkt ó +.config ure +M ON +/ edit +_A dd +, true +Ġc li +Error Message +- loader +Dim ensions +ultip ly +Ġ{ !! +ĠSql Command +Ġsp oken +Ġp ics +Ġto y +( Key +ĠLo op +Ø ¨ +E ATURE +in ction +_set up +w rapper +Ġt ong +c ular +O pt +.P l +=" , +(l ength +um n +Ġch rom +Ġse vent +ĠIllegal ArgumentException +ĉ start +Ġbeg un +CE PTION +dat aset +ĠF ailed +col s +Ġkne e +im ore +.sp lice +sh ell +ig gers +Ġthem es +ĠD J +ĠAss istant +- $ +May be +Ġorder ing +ĠInt elligence +ĠMass achusetts +Ġfail ing +el son +G reat += i +.re st +Ġinv ite +-dis able +.Group Box +âĢĻ est +Ġtack le +g v +et ter +Ġ), čĊ +_r ules +.w arn +function s +ĠChrist ians +Ġback ed +Ġsl ider +Ġenjoy ing +n est +Ġh ij +_m s +// * +An notations +ĠVariable s +< V +( server +ĠOr acle +element s +Ġorgan isation +_point er +ĠHe aders +[ d +Ġdead line +iss a +Ġkn ife +ĠNAS A +ĠHe ight +ĠAs ync +Ġven ue +.d om +bour ne +ĠHaw ai +Ġmem o +ict ions +Ġsurve illance +om i +/ assets +Ġed u +Ä Ľ +Ġro ster +Ġh ired +ĠT ok +Ġpl acement +ur ations +Ġset State +ĠMag azine +Ġhor ror +T ry +Ġl ag +ĠEvery one +th ur +)) ;čĊčĊ +. return +Ġsy mp +âĸĪ âĸĪ +Ġn ights +work er +Ġa le +ennes see +.st ep +Ġsynchron ized +our i +Do es +. change +f on +.set Background +irc ular ++ - +ĠC IA +ĠJ ane +ĠSim ilar +- I +level and +Ġpros pect +_f ound +ĉc olor +.D iagnostics +Ġann ounce +Ġassum es +/ tr +Ġb d +ĠCar bon +Ġanal ys +.de st +n ik +ĠL ie +- index +Draw able +ĠT AG +Ġtri angle +_F LOAT +ĉĉ ĠĠĠĠĠ +.bl ack +v ue +cur acy +Ġaffect s +Ġsure ly +Sl ider +uk i +c ery +Ġun ter +.pro file +ord on +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +le ave +Ġsmart phone +g ie +Ġcons pir +Ġt utorial +ç± » +Ġc ab +ĠSum mary +* ĊĊ +ä h +" This +Ġsl ides +" +c ycle +ĠB ull +path s +Ġun p +Ġview DidLoad +_M odel +Ġassert True +Ġr ated +De cl +vert ed +ĠD at +b rew +Ġpoint ing +M s +ĠPoint er +) ' +_n on +ĠSE C +Ġy eah +g ency +initial ize +f ly +[ pos +, g +Te le +Ġj oke +Ġcl ause +.find ById +en es +( instance + £ +Ġs lic +_h ome +Ġ*/ }Ċ +_p ages +(s ervice +R P +ĠAm ong +.get Current +ãĤ ¹ +Ġs lee += [Ċ +ol er +Ġlib ert +Ġ` Ċ +Ġw enn +l ated +Ġimm une +( Node +ĠPro blem +ĠA bs +log s +Ġ ../ +ĠA DC +Ġ}} ">Ċ +> ');Ċ += b +ĠW ind +lah oma +Ġalloc ate +or ian +Ġpres cription +- quality +ĠMay or +in ely +end foreach +ĠCom plex +k om +T Y +] ]. +. Style +_m any +',' $ +Ġbar rier +ĠF etch +ĠMar vel +Ġres ist +ог о +b idden +ĠRun nable +: false +Ġbuild s +ĠSt age +Ġd ub +emp o +.s ite +;ĊĊ ĊĊ +ĠDen ver +Ġre vel +Ġtrigger ed +Ġd ice +_f ail +Ġg c +ĉ X +ĠTh rowable +.r outer +ĠRev olution +ÑĢ Ð° +_N ON +Ł ¥ +Ġel der +Ġab road +ĠÐ µ +ĠAd ult +bl r +g lyphicon +Ġprom oting +Ġ iz +ĠS olid +_lo ader +ear ly +.en abled +- edit +ĠU L +_ play +ĠInt errupt +Ġadvant ages +uc le +Ġmechan ical +.table LayoutPanel +ĠWork ing +Ġan onymous +R ating +ig ious +_ph one +.addAction Listener +Ġfr an +und en +Ġ*) & +_ bool +ul ative +Ġcon e +ĠM ult +Ġm ö +ĠFor ward +] ):Ċ +Ġconvin ced +act ed +ãģ ĵ +ĠConfig ure +Ġce iling +D er +Ġpass engers +Group s +Ġsoc cer +/ W +avi ors +sw ith +ĠZ one +. Options +ĠM om +ied er +Array s +Ġtreat ments +Ġprotect ing +f ac +Ġpick le +Button Item +Ġblock ing +str ar +à ² +ĠEx port +Ġth rew +ott a +ĠB ASE +.w s +.LE ADING +order By +_d elay +ĠP u +.d ll +ĠCh oose +Pol ice +ĠBE GIN +box es +Ġdiam ond +, l +Ġ ĉĉĉ +Ġcur ious +t v +Ġerot ische +ack ages +ĉ Set +T ick +.b order +static method +Ġch er +in voice +Ġcr u +Ġdef ect +_m etadata +re lation +ik an +[ N +(Q t +( Base +æģ ¯ +be at +ĠEm pty +ĉ o +_sh ift +Ġreg ret +Th ose +C ent +ĠPort ug +ĠIs lands +ĠT IME +Man agement +-s p +ê me +Ġnot ion +un ifu +P K +è¡ Į +ĠCUR LOPT +\" \ +U V +ç º +d ra +c ou += ` +ĠD estroy +r p +.c ancel +G G +r untime +ĠV ue +Ġprogress ive +/s ervices +Ġrun ner +_FR AME +.ToolStrip MenuItem +Ġ' ,' +d elay += utf +Ġscreen ing +Ġpull ing +om as +Ġan th +- new +/ local +Ġi Pad +Ġt witter +Ġd ying +Ġhe aven +ĠU Int +ĠSen ator +Ġpres um +ĠWalk er +Ġover come +ete ction +Ġemb arrass +Ch ina +In clude +RO LL +Ġdata Type +D avid +ภ£ +lo p +-m onth +Ġsc ar +ĠS afe +Ġ **************************************************************** +Ġaccess ories +Ġr amp +_U SE +Ġcontr ad +)) ]Ċ +Ġpre st +ĠH R +ĠR ap +Ġus ize +Ġcap ability +Ġc ort +- next +Ġbur den +_read er +Ġ@ @ +reg ular +ĠK a +M AN +Ġa str +Ġ' ')Ċ +Ġf ed +Ġpars ing +ĠY ears +Ġbro ker +": {" +Ġa kt +In ventory +abe led +Ġarg parse +****** *Ċ +vers ation +Ġc ord +ĠT i +Ġhope fully +Ġa h +ver b +Ġst olen +. Entry +Ġexpect ing +O rientation +Ġpower ed +Ġp ersist +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +'] ); +')) ,Ċ +ĠC ash +ĉ item +gr ades +rop ol +b asic +Ġ" );čĊ +Ġaw ards +(r ange +- all +ĠIB Outlet +ĠInd eed +---------------------------------------------------------------- ------------ +Ġstom ach +Ġfl ower +Ġs ew +_t imes +av is +Q String +ĠR outes +_pro t +Ġcom edy +Ġlog out +Ġwood en +Ġpost er +p iece +.J oin +ĠP ok +cel ona +mut ex +;čĊ čĊčĊ +Ġstri kes +Load ed +) arg +es a +Un ited +E p +PE LL +ĠAtl antic +ul let +app le +Ġsett led +a con +Ġprint er +ĠG C +å® ļ +Ġrender ed +, âĢĻ +he it +s ocial +. ge +ĠR ick +ĠUt ah +g ot +on ical +ĠSc roll +ĠSc iences +Ġj ug +Ġam pl +ent i +LE FT +Ġt abs +Ġenorm ous +.get Key +loc ate +. EX +.st orage +.W e +Ġto ast +ĠAdd itionally +ĠN OW +_ UPDATE +Ġtrans ferred +th a +.D isplay +_ ui +ID EO +Ġmeaning ful +ĠMos cow +, this +ĠVict oria +æĶ ¹ +ĠÐ Ł +.st ack +ĠB arn +pared Statement +: string +Ġb ij +ĠST ATE +Ġemploy ers +ĉ input +( | +Ġle x +in voke +ĉ num +++ , +at ial +ors es +Ġfor k +_t xt +ĠAnton io +Ġ( < +aver se +Ġdev ast +ãĢ Ģ +.D ec +ĠG ard +/ ui +. % +tr i +Ġrol led +Value Pair +itt en +ĠTh er +Ġv rou +ĠFl ow +ĠFin ance +ĠCom b +H C +.set Visible +is l +Ġp k +Ġup set +( raw +ĠV ice +e atures +ĠL ang +Look ing +ĠA ST +Ġtri ps +ĠJust in +b rowser +=" '.$ +. vertices +- co +}/ { +Ġ? , +ĠD omin +ĠBel g +" < +Ġsup pose +add y +Ġwalk s +ERR U +_f ilters +Pre ferred +sc ene +е Ñģ +ĠAff airs +Ġ"# { +Ġon Submit +Ġstock s +/ view +g ree +- get +h it +J o +.get C +Initial ized +ÑĤ и +c uts +( Type +ĠAg reement +ĠViet nam +Ġ/* ! +Ġp izza +- view +_ em +Ġl hs +Ġm uy +ĠId ent +ĠF riends +Ġab und +_A D +.t imestamp +- ' +Ġd uplicate +Ġhun ting +Ġregul atory +ia o +am ous +ĠEnt ertainment +[ A +iat ric +_CL IENT +ĠK ids +/p kg +B reak +)) );ĊĊ +ĠSh ape +Ġrel ating +Int errupt +able Opacity +emb re +Ġmyst ery +Ġjournal ists +rit able +.L ink +Ġstop ping +CRE T +.D B +Ġpopular ity +Ġg ew +Ġim pr +set Value +FL AG +ĉm ax +Ġb ake +w y +ĠEcon omic +Ġen contr +Ġf name +/ de +R ank +Ġbug s +.s m +Ġmed ian +D OWN +ĠS ure +At Index +ĠD ick +Ġ( __ +.d elta +F r +Ġsuggest ing +ĠRec yclerView +, e +ST ART +/************************************************************************ **** +xf ord +Ġrece ipt +CL AIM +read only +Ġeng aging +C a +as ma +Ġens uring +Eng lish +ĠV ancouver +hy th +Ġpurch asing +ĠP I +. word +(s p +.h ome +: def +Ġg ig +ĠV e +for um +ĠM itch +B ay +_F L +Ġs oll +_column s +Ġminor ity +b ird +Ġhand ed +SS L +ST AT +Ġnerv ous +ĥ ½ +Ġfile Path +CRE ATE +A w +Ġp ens +se ed +ĠCom pute +ol k +ĠAs set +re ach +'), čĊ +n avigation +L F +/ util +ĠP ub +Ġâ Ķ +c ion +## Ċ +II I +Tag Name +Ġam id +per mission +if iable +xFFFF FFFF +н и +.B uffer +_ irq +d ark +Ġret val +.f ire +produ ction +.list en +ĠWe ather +Ġbuy ers +. ne +er p +ĠP ent +Ġw elfare +Ġpage Size +ĠSt adium +ert a +Ġle v +amp a +P ager +Ġcharg ing +ĠNet flix +| null +_r andom +.x path +Ġst ere +ĠIS IS +pons es +( loc +ey ond +ĠOff icial +ĠMary land +Data Type +_p ar +{ }, +ĠEn joy +_SH IFT +ĠA wards +_ENT RY +Ġseem ingly +entic ate +Ġheart s +_ ;ĊĊ +ĠH IV +Ġindiv id +ĠFl ag +_ ctrl +ĠC allback +, z +ĠG PU +ĉ obj +ĠPh oenix +ĠB US +Ġrub ber +_A UTH +ĠSol utions +( location +Variable s +.set Enabled +_h igh +W O +G esture +Ġre try +Ġobject ForKey +allow een +Ġm os +ĠC ele +Ġik ke +(c ell +ĠM ODE +ren a +Ġdescri bing +Ġph i +Ġr d +Ġdes erve +Ġwhe els +å¸ Ĥ +Ġcrit ics +N amespace +ĠF ra +Ġ ĊĊĊĊ +Ġall a +Ġrequ iring +æľ Ł +ut ation +Ġdelay ed +Ġadministr ative +Ġb ay +.h idden +T ex +Ġbound aries +Ġ] );ĊĊ +ĠFollow ing +~ / +F i +_con v +_T ITLE +Ġdes de +ICollection View +Ali as +Ġb ite +pat ient +_COMM AND +Com pleted +ĉ elif +( < +B usiness +ĠP ool +Ġpurs ue +ĠB an +_st eps +_DE CL +um ble +Ġcom bo +ĠL ayer +.x r +Ġd up +-------- - +Ġmod ifier +ro b +re z +Ġath letes +Us ed +w ear +Ġlegit imate +Ġ" ĊĊ +Ġh v +St d +ĠH old +Ġsurv iv +ĠAll iance +ĠEar ly +Beh avior +(f ont +/lib s +Ġrect angle +Ġs inger +Ġam p +Equal To +Ġ" ." +Ġgirl friend +å ± +line ar +obs erv +Ġpi ù +Ġcomple ment +With Value +(p assword +t ake +Bl ank +ĠCom par +' ", +_p olicy +m ongoose +_FA ILED +.re port +R atio +.Perform Layout +us able +m ers +_re nder +PE ED +Ġles b +ĉ E +_t ool +Ġl adies +о Ñģ +)) ))Ċ +;; ;; +.d ot +Ġn est +pe ak +uk kit +ec a +_S W +Ġ& ( +ĠOk lahoma +Ġbank ing +ĠN intendo +Ġreprodu ce +_element s +_m ac +pro xy +Ġremark able +}/ ${ +Ġout s +.has Next +M ODE +Ġan ime +.con n +Un ique +D om +Ġimportant ly +itt y +Ġju ice +T w +ĠPart ners +Ġattack ing +Ġport able +am iento +.P ictureBox +.g en +Ġopt imal +Ġre cre +Ġjournal ist +ĠEx tract +ĠMore over +Ġmargin Top +.A p +Ġf iring +Na N +ĉ template +аР´ +. En +Ġdef ence +ĠT el +il en +j an += data +ĠU rl +ĠRe uters +(t otal +ĠFif th +Ġess ays +Ġinterpret ation +Ġchar ity +ĠR ules +Ġsub section +st yled +az er +l ags +L IST +Ġupload ed +Ġtr ash +Ġreg istr +Ġsell er +>' ;čĊ +Ġstart Time +ç Ļ +s y +(Http ServletRequest +Ġtr ap +G C +Ġembed ded +Ġsurround ed +im its +T X +yl inder +ĠF al +Ġsent ences +ĠJ a +IF ICATION +we apon +ov ation +Ġco at +Ġinter pol +Ġl ips +ĠK y +Ġv ectors +_ am +Ġint ake +.w orld +Ġin box +ĠM AC +_ ab +(name of +Ġent ert +Ġgather ing +ĠS IM +++ . +ny a +' }} +ĠUP DATE +Ġp ac +( html +ĠS ant +i ating +ĠIde as +Ġspr ay +ĠH art +Ġver ification +ades h +/ modules +ĠM ind +ĠSized Box +Ġsh elter +Ġher oes +att y +Ġcert ified +s j +Ġê tre +ÅĤ o +Ġpublish ing +ĠMal ays +.get User +ĠPro vider +ĠLinked List +ĠB or +RO UND +d id +t ain +p ire +ĠJ enn +t el +and e +_f ront +ĠMc G +Test Method +à¸ Ń +Ġoccasion ally +ĠW ales +Ġexerc ises +ĠÐ Ĵ +- plus +Ġvalid ator +Ġpr ayer +L ATED +_ author +Ġlab our +++ Ċ +-e quiv +ĠG PL +Ġface book +s imple +g ly +Process or +ip y +Ġ* > +Ġcle ared +ĠP ush +Ġpen is +Struct ure +li j +ĠM organ +Ġhand ful +" .Ċ +| \ +Ġ ******************************** +ĠA qu +_ IC +.load s +Ġm eter +ĠMar ine +:: { +ĠT S +ĠArray s +.T itle +GR AM +ter min +Ġco inc +El se +_st ates +-r un +m embers +ast ro +Ġon Press +Ġbe ings +Ġabandon ed +Ġtax p +own ers +.m ode +Ġdiagn osis +Ġ_ Ċ +ĠK night +ĉ A +Ġob serve +), ' +! ")Ċ +ĠPar a +Ġvari ation +( False +ĠAnt i +Ġg ri +Ġhome less +? v +Ġbe z +.S erver +re lease +ĠP atri +Ġchar s +Ġrank ing +activ ation +Ġw ides +q r +.S ql +ac ular +ĠB ot +_s ync +Ġhapp iness +Ġvolunte ers +Ġs its +/ < +[ e +(file Name +Ġcap ac +ĠMar ia +f ather +Ġgr am +* i +Ġcas o +_d raw +ĠR aw +ĠIter ator +ĠP adding +P D +BO X +ĠS PECIAL +Ġfe cha +Ġv ide +ĠLe ader +ä» ¥ +$ (". +Ġdiam eter +Ġm ild +Ġrock s +app ings +d irectory +.fl ush +ĠJ ess +UN IT +ĠP ear +Ġmand atory +S ur +q t +Ġstream s +Ġco operation +ĠS ac +Ġche aper +ĉ ch +an imation +f are +( height +( True +N Y +Ġw rest +Ġpoll s +Ġencounter ed +ĠMarket able +_P ASSWORD +_SE LECT +ĠArab ia +_c lock +Ġv oy +Ġи з +Ġst ir +is ible +-e ffect +.c reated +Ġto ys +ĠTrad able +Ġr ust +Ġstr cpy +_t imestamp +Ġtalent ed +, null +ĠJ obs +ĠPort land +Ġweak ness +Th row +ĠAng el +ä¿ ® +Ġun cert +ï¼ī Ċ +ĠìĿ ´ +Wh ich +Ġ[- ]: +S omething +Ġconv icted +k le +ed ium +Ġbranch es +Ġb ases +ç ® +Ġcomplex ity +ĠF ig +. reshape +$ db +_CON ST +ĠT es +.r untime +Ġden y +ĠB SD +Ġk r +h att +ĠSt atic +Ġunivers ities +Re place +Ġdro ve +Ġad oles +_pl ugin +ĠL GBT +Ġt ex +du ction +ED I +ĠT ed +_ URI +Ġre ception +art en +.S ingle +r ice +sc ious +_b g +Ġw ages +ĠS ervlet +UIL ayout +Ġform atted +.M od +< class +is en +Ġrepresent atives +"] = +Ġport al +ĠHun ter +Ġh iring +__ )Ċ +ric ulum +u o +li est +Ġt ears +L at +Ġliter al +.In sert +Ġc urs +ĠCom put +Ġterror ism +Ġswe ep +Ġ[] čĊ +Ġpass enger +Ġeast ern +Ġtwe ets +Ġoper ated +w nd +ĠS yn +.t ools +ĠW M +ul ates +Ġbacter ia +( bytes +.set Data +Ġvis ibility +// ================================================================ +el m +Ġgener ating +Ġm v +Ġk h +j en +/ search +Ġaccount ing +se gment +act ic +. ip +Ġdeploy ment +Ġfoot er +> ',Ċ +Ġexpand ing +ĠHam ilton +ĠCon trib +.T ables +Act iv +H H +ocom merce +_ ; +Ġamong st +ow ing +ĠC old +AP H +Ġpsych ological +_t ensor +Ġpack aging +ĠSw eden +Ġp are +Ġag gregate +Ġmoder ate +_h and +Ġdesign ated +Ġdr um +Ġget User +ĠC reek +_s cope +ĠTrans fer +ĠM arg +Ġfight ers +W nd +ĠS el +ĠLa unch +Ġemerg ing +if rame +ĠAdd itional +Ġf ears +Ġsat ellite +_ : +Ġdis posing +Get Value +Http Post +AT IVE +ul ary +View s +Ġatt ending +ĠT ennessee +ĠM ission +Ġmedic ation +ĠW y +ĠAn na +Ø ¹ +ĠVert ex +.t ypes +O rgan +.DataGridView TextBoxColumn +ĠR S +Ġtemp o +( App +Version UID +.p oint +ĠD utch +H ours +L U +Ġqu oted +.b uilder +ĠPer fect +ĠAl ways +_t wo +Ġexclus ively +ĠC ra +ific ar +ĠA WS +ing ham +com plex +k ernel +Ġgr avity +Ġw i +Ġover view +ĠW ant +ĠW P +( sh +. rotation +St ates +ĠTe en +_com ponents +ì Īĺ +Re ceived +Ġly rics +rit es +ĉĉĉĉĉ Ġ +-A merican +[ num +/ python +ĠU ART +Ġapp le +ĠJon athan +Ġmoment um +ภ± +Ĥ ¹ +Ġm ich +and ra +Ġbi ological +ĠM ens +Ġ% % +else a +ĠMex ican +.rand int +Ġt ale +ĠValid ate +Ġdefe ated +.ht m +Ġcop per += / +cos ystem +Ġr ip +dec imal +.V ISIBLE +ĠT a +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉ +Ġdownload ed +en vironment +Ġnom ine +build ing +ĠSp ot +ipher al +Ġal to +qu et +ĠF T +/ get +/m aster +W IN +åħ ĥ +W est +arg c +Ġprodu cers +ĠM uch +_st orage +cred it +CON T +Ġv et +Ġvo ices +(' ', +Ġinstr uments +ĠM SG +es se +re pository +om ics +Ġdeal er +St ill +Ġb anner +asc ii +Ġrem arks +[ js +Ġshort er +g ulp +Ġmyst er +Ġk un +ĠB ird +Ġti ene +n ut +ĠU m +Ġw ise +Y eah +INE SS +_b egin +- heading +C ourse +Ġ čĊčĊ +omb ie +grad ed +ĠG PS +Ġ że +F it +c aption +ö n +/ image +l ia +(m od +Ġle ak +en za +/ H +ĠH appy +D ist +n x +ĠGovern or +(l ast +te acher +ĠS ent +s upport +ject ory +Ġ Ùħ +Reg istration +ĠGr ay +, false +Ġadjust ed +( settings +< R +ĠM age +Ġpl aint +_ )Ċ +ĉ it +omet ric +. bootstrap +Ġcar ries +I p +Ġ! $ +Ġswim ming +ĠMar io +ĠQuest ions +P ACE +æĸ ¹ +e or +}} " +Ġo ven +ĠK on +Ġwis dom +Ġac quisition +ess ment +ag ine +Ġexpress ions +Sequential Group +F ront +ul pt +aw k +'] )ĊĊ +_ AR +Ġanal og +ul in +_PR INT +ĠL G +Ġb lob +ĠFurther more +_com ponent +ĠC ole +L AN +SCRI PTION +Ġl ap +icens ing +_TIME OUT +ĠF ro +Ġli ability +Ġcom posed +.create SequentialGroup +_p erson +Ġbe am +ĉ ĠĠĠĠĠĠĠĠ +ĠNot Found +. 'Ċ +ÃŃ s +.Text View +P DF +Ġk ar +__ (' +Ġ" :" +_m essages +Ġhar vest +.h istory +> 'Ċ +-f old +æ Ĭ +ĠBet ter +Ġ"\ < +sp acing +Ġfurn ished +os er +] }Ċ +Ġ$ " +p ull +.P ost +( ip +Ĺ ı +.f ront +nt e +ĠF M +g uid +Ġnegot iations +agon al +Ġtrem end +unge on +Ad v +car ousel +ÃŁ e +_DE SC +Ġham mer +áº Ń +ĠĠĠĠĠĠĠĠ ĊĊ +-c ore +-s ervice +Ġcorn ers +ĠS F +p red +> A +ĠJ Label +Ġrom antic +Ġtestim ony +os c +ĠGener ation +as ures +_int ernal +Ġprint s +Ġ] )Ċ +ĠC leveland +re po +D isc +Ġ" >Ċ +�� �� +Ġne arest +_t b +( require +EO F +- child +Ġbu dd +.Xtra Editors +alt ies +\": \" +W ords +Ġloc ally +Ġpurch ases +Draw er +ex tract +Ġexec ut +} '. +user data +Ġfocus es +-min ute +ĠP ublish +og o +Ġmount ains +B ot +} >{ +Ġt ension +ro d +m esh +Ġtransform ed +, R +() }Ċ +.l ong +Ġg orgeous +ĠS chedule +Ġol dest +Ġsub process +( IN +y ect +ĠCo oper +arn ess +ĠMon itor +.p art +ĠN BC +Ġc otton +Ġh ol +Ġrg ba +ĠB io +Cont inue +P od +Ġparticip ating +clus ions +(By Val +à ¬ +ĠH OW +_set opt +Ġaccompany ing +at on +Ġ/ \ +ĠAuth entication +i én +ĠBar ack +/* . +Ġe ager +ĠC ancel +< lemma +ep h +ĉ window +Ġinc idents +), ( +.D es +ib e +ĠFunction s +Ġhosp itals +Ġo xygen +root Scope +Ġd rew +ĉ request +not ice +ak u +am ents +f ar +Ġprec ise +_w rapper +Ġlisten ers +A Z +.b ounds +ĠA verage +field set +_ axis +Ġexam ination +' .Ċ +mon s +++) {čĊ +ĠForm s +íķ ľ +Cpp Method +_tr ace +Ġengine er +ĠFl at +Ġrev ision +Ġhe ating +/ profile +.r u +p riority +Ġin fer +_ST REAM +Ġ* )( +> $ +OLE AN +OK IE +IB ILITY +U AGE +ĠSur vey +Ġres ign +w ing +Ġsecre ts +Ġch ips +JSON Object +Des ktop +_SY MBOL +(res ource +ĠĊ +Ġnew est +ul i +Ġdes ert +Ġd ip +ĠP ow +Ġequ ation +Ġposs ibilities +ĠF ed +os ph +Ġ[ % +Ġb ubble +ether lands +Ġc ement +. auto +_ AN +âĢĻ . +se lection +ĠB ond +D en +- O +.get Type +.W indow +p res +Ġsw inger +" })Ċ +Ġp ip +Ġm ice +Ġcomp ound +- plugin +ik o +Ġcent uries +ic ular +-in line +ĉ key +> \< +EN SION +Ġ[ čĊ +Ġprecis ely +Ġét é +ĠP ast +ĠCam bridge +-f ull +Ġanaly ze +ĠSte ven +Ġn em +d ue +ore n +Ġmus cles +ij ing +/ - +ĠKenn edy +R M +oss ible +Ġact ress +Ġd olor +å½ ķ +Ne ed +.t oggle +ĠR ace +w ers +.m aterial +ĠD ue +ĠP el +# print +Ġindepend ence +ex us +Sh adow +Ġenc oder +( level +ĠSw ift +.d oc +_se lection +Ġserial VersionUID +Label s +Ġperform ances +.T ag +ĠN HL +iz en +/ UIKit +_CONT ROL +Ġearn ings +ĠAl t +_H ANDLE +C tx +Ġpers u +Ġtr an +ç ¨ +_CH ANNEL +Ġsatisf action +ĠG P +io x +m itt +land o +Ġp ig +inal s +ê ncia +S urface +ĠU UID +Ġbenef icial +Ġsequ ences +ĉmem set +Ġmag ical + « +Ġw orn +AS C +pop up +COM P +_b efore +en ess +U i +L es +.re quire +.Serial izable +add Gap +Ġauthor ization +.py plot +urr ay +lat itude +fr ames +aj s +Ġcomp ass +Ġobserv ations +_s up +.en viron +Ġtri ple +ĠRub y +Ġdr ain +_F ILTER +S an +UM P +Null Exception +ĠG ab +ow e +ĠTurk ish +_se quence +ĠGr ant +uel a +Ġw o +Ġc ube +i q +Ġdis orders +Ġextra ordinary +Ġc trl +ĠSe q +ent r +Ġsan ctions +uts ch +Re ports +Ġin herit +Per iod +Ġphot ography +ĠF ramework +Ġspecial ist +Ġ? ĊĊ +_ selected +.P layer +Ġal location +( account +Ġstruct ural +v able +- offset +.App CompatActivity +аР¼ +.Add WithValue +Ġicon s +Ġshut down +_l ow +ĠCom pare +ĠC e += head +l am +.p redict +_DE C +ĠS leep +ĠGr atis +Ġsuggest ion +ĠD EL +ca ff +av irus +No thing +ŀ ĭ +Ġwides pread +Ġmechan isms +Ġtext Align +occ up +ĠR ail +: NS +Ġf iber +Ġm k +Ġv intage +-l ong +.re duce +. Entities +( record +Ġple asant +FR ING +.C ells +OT T +ĉelse if +_con firm +ĠView Group +s ym +Ġpr ay +Ġsus pected +Cont ains +Ġb orders +Ġcomponent Did +ASS ERT +Ġinf inite +- order +Ġh ello +ĠGr ade +.currentTime Millis +apol is +z h +ĉ Object +: \\ +H O +val uation +Ġvoc ab +Ġcou pon +atab ases +.Get Type +L earn +] =" +ĠG ary +ot ive +Ġas h +Ġb ib +XX XX +Ġbal anced +VAL UE +ĠN at +_A d +< E +åĮ º +ĠMethod Info +L IB +Ġconsider able +ĠInd ustry +test s +.set Title +ĠBl uetooth +Ġm apped +ĠBru ce +ĠMain Window +ĉ status +Ġr az +ĠM and +Ġclass ification +Per missions +Ġ---------------------------------------------------------------- ------------ +Ġcontain ers +: set +_x ml +Ġwh ilst +Th rough +Ġval ign +Ġworld s +C ORD +ED IA +ÑĢ Ð¾Ð² +Ġsp are +ĠH ad +ĠDE F +(p tr +Ġwarm ing +ठ¾ +Ġcons ensus +ag ne +CT L +Ġì ķ +.M ain +web Element +Ġp ist +Fl ash +App end +.tw img +T ap +Ġveget ables +al g +.s ample +Ġcoach ing +( ind +Cell Value +Check Box +ĠH ell +RO OT +Ġst adium +Ġinvestig ating +) % +st ed +ĠW riting +Ġê ² +Ġun o +Ġ{{ -- +Ġco ords +Ġun ser +organ ization +ĠCr ime +ĠDemocr at +Ġv in +/ file +- api +ĠA y +Ġfund ed +ĠBre xit +ĠG h +ent ina +c ases +Ġd ash +Ġ!! }Ċ +H I +Off ice +Ġcapt ain +Ġwor ship +\ C +Ġglo be +_ board +Ġbab ies +Ġconsec utive +Ġenh anced +ere um +ĠAd vis +Ġgr ain +Ġc raw +ancell ationToken +. alpha +_W ITH +ĠO tt +ĠC ool +.b atch +Ġver ified +(c allback +Ġreg ards +ĠInt Ptr +ouch er +Ġk in +Ġtou ched +it Ãł +ath on +Ġadj acent +Ġaccom panied +LE AR +Ġim plies +Ġh ill +ĠBalt imore +=" - +Fin ally +S am +ic opt +Ġs od +Ġm aj +ĠSh ipping +Ġget All +Ġcoach es +Ġdon ations +il ot +ĠT ar +c err +Ġbad ge +Ġmark ers +ĠR and +ais ed +iss ance +Ġexpl oring +uc ed +ĠIndones ia +Ġbene ath +Ġmagn etic +Ġm useum +match Condition +Ġdis rupt +Ġrem ind +ĠT M +Ġ/ >< +Ġf ool +Ġes k +.N ull +ĠD ies +_OUT PUT +_TYP ED +Ġpaint ed +Ġsoph istic +ĠB ear +* n +_P ACK +Ġdeliver ing +ĠC OUNT +åį ķ +Ġj eg +-c ar +f name +Ġr anging +ĠN eg +/ ******/ +ĠCH AR +Ġul tra +Gr ad += t +Ġjud ges +ĠD ise +ann ers +Ġsc al +_c al +ĠCON NECTION +_ embed +(f n +ĠC raft +ĠP as +") -> +.con vert +.res ource +ĠST ATUS +ô ng +ĠT it +Ġclass room +ĠArch itect +ĠK ings +Ġstead y +/* !Ċ +ĠG ene +) ";Ċ +ic ia +st an +ĠCon struction +um per +w c +ĠC BS +ing ing +-p arty +(d river +M ARK +Ġn ested +ew ard +Ġdepend ency +Ġm ales +ĠO NE +ĠProdu ction +][ $ +ãĥ¼ ãĥ +_LO AD +ĠB ol +el ry +ł éϤ +ĠRe quire +Ġpl acing +xx x +CA LE +Ġth umb +Ch oose +Ġprot otype +VO ID +Ġles bian +Ġtra its +Sh arp +Ġconsum e +Tr uth +Ġaction Performed +ĠEnvironment al +ĠDe an +Ġest ado +s ame +Ġnumer ic +Ġtrans it +. Email +-s ide +_R UN +ĠVill age +_OP EN +è ¦ +.re m +-w arning +any a +Property Changed +Ġ(! _ +( check +il ia +ĠSo ft +st eps +ĠMad rid +Memory Warning +Ġhand lers +Ġexperi encing +Ġins pect +button s +Receive MemoryWarning +chem y +Link s +Ġur llib +.System Colors +ĠE igen +Ġpun ishment +:UI Control +bar a +- set +Ġ}čĊčĊ čĊ +Ġtoler ance +Ġinter faces +. redirect +ighb ors +cs rf +_back ground +. Utils +_H T +ĠInter est +im os +Ġgr ants +Ġexam ined +Ð Ķ +Ġc f +for ge +back s +ĠObject s +_s ent +. entry +ĠTH EN +ell ido +c ia +, res +/std c +. nd +( Int +ĠAuth ors +ĠApp CompatActivity +' { +Ġmed i +M usic +ig m +ce ipt +Ġa uss +Ġtarget ing +ĠKe ys +h n +: ]Ċ +Ġmin eral +à ® +.c a +om ed +Ġshe ets +Ġc amb +Ġdead ly +.in ject +( unit +ĠSe lection +.g ms +( connection +Ġ$ (" +é mon +ĠCurrent ly +pt e +_path s +le af +Ġimp lications +pos al +ä½ į +[ / +anc ia +é Ľ +m ul +c ie +Ġge ile +im als +UI View +Ġs urre +serial ize +IS O +Ġarbit rary +Ġsock addr +.f n +ĠM erc +Ġcast ing +Key Down +Ġnew Value +op ens +T odo +Ġflex ibility +ĉĉĉĉ ĠĠ +V elocity +ú n +row ing +Ġcomput ed +` )Ċ +st atement +Ġr i +_c art +L ow +trans fer +.n av +Ġgr ave +ĠDo or +ĉ alert +.sub scribe +- profile +ĉb ase +ĠâĪ Ĵ +__ ĊĊ +Ġengine ers +Ġexplos ion +Ġd ari +ĉ Log +on al +Ġisol ated +{ i +ĠM sg +F uture +Ġrac ist +-w rap +ĠV ers +b org +IS ION +Ġ ÑĢаР+ĠY an +init With +Ġn omin +( empty +ÃŃ n +ãĤ ¤ +ĉ width +Ġch amber +/ ajax +EM P +Ġnec es +iv os +log ic +*) & +cript s +Row At +ib lings +Ġe ars +Ġcomput ing +Ġm aker +ĠNe ither +b readcrumb +Ġserial ize +ĠWith in +Ġd ell +_TR ACE += a +Ġwish es +-in ch +ĠD or +Ġinnoc ent +ĠD ol +Ġint ens +for ced +ĠB IT +Ġphotograph s +Ġcas a +ĠL en +\F ramework +.S imple +Ġde ar +)/ ( +ip pi +Ġown s +Pl ayers +Ġpropos als +.p i +us alem +D amage +Ġcal ories +ĠCreat ive +Ġ[ $ +Ġ// čĊ +And View +è me +.c ustom +_f actory +command s +_lo ok +Ġstr cmp +Y N +a ired +Ġaud it +о ÑģÑĤ +ĠRe verse +ropri ate +et ics +< vector +.s elenium +. or +Ġpred icate +Ġfinish ing +Ġk le +ĠRep os +ĠK han +ĠM aking +ĠF S +Ġp ute +ĉ state +_S UPPORT +' - +orient ation +Ġexist ed +atur a +Ġexpect s +ĠSh adow +Ġorgan iz +å ŀĭ +Ġsusp ension +Ġu it +Ġsimult aneously +ĠAff ero +: ");Ċ +Ġro cket +c as +eter mine +ace ut +x l +ĠA MD +( graph +ass oci +_C R +.ar ange +(j Label +Ġbe ef +Qu ick +.c ard +] ): +- gr +.G ONE +_C LOSE +ĠNe v +ÃŃ as +Ġste pped +ĠFre edom +ĠW R +NS Array +_r x +_d ialog +Ġhot els +Ġ( \< +ĠD iamond +Ġassum ption +um i +( items +č ččĊ +æ³ ķ +Ġn el +Book s +åİ ¿ +us b +ĠF IN +æ ¬ +Ġcorpor ations +US A +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +.p roperty +ew ise +_ plot +"> ';Ċ +Ġpe pper +Ġsh ed +ĠMed ium +ĠC ookie +Ġoverse as +ed or +asure ment +åŃ ĺ +Ġ' .' +Ġph p +ĠPRO C +Ġexception al +( th +ĠJ et +Ġoccup ied +.set Image +ĠRel ated +uck er +M embers +PR INT +ĠG lo +_V IEW +} ",Ċ +Ġad option +[] )Ċ +ĠMiss ouri +ĠLin coln +eral d +Pop up +Ġf ate +- bootstrap +fe ctions +ĠP oll +_ARG S +in ance +-h ome +. ), +_d one +: ĊĊĊ +Ġdiscuss ing +ĠSQL Exception +Ġelect ro +ĉ req +Ġz w +Ġl ui +Ġover night +$ user +ĠW AY +Ġall erg +Ġdisappoint ed +Ġradi ation +Ġimpress ed +ific ates +Ġto b +CL ASS +Ġc uda +_d et +- post +ul u +Trans lation +-h and +.y ear +ĠM ongo +Ġun clear +. engine +WEB PACK +r ices +_AC CESS +Ġh olidays +per cent +.Id entity +ĠG ov +Ġpassion ate +!! . +ĠGree ce +plus plus +')) ; +G P +Ġexc it +.tab Page +_ cond +Ġspons or +M ODULE +_pro c +Ġ$ Ċ +Ġr ational +.T ool +Ġi hr +cc a +åĵ ģ +ĠE state +IB UTE +Action Performed +ĠS olar +¦ Ĥ +Ġequ ity +t id +Ġrec ip +.s imple +m k +ĠL uke +ĠGuard ian +Ġenc rypted +Ġdomin ant +. place +ĠN V +Ġtong ue +( Get +Ġst ainless +.P lay +Ġe b +ac i +.b uffer +readcr umbs +Ġvacc ine +p rom +Ġuser Info +Ġsl ug +Serial izedName +-w ide +Ġre actions +ĠY ang +ĠAdd s +(user Id +Ġpl ates +ĠM EM +Ġb ail +In side +et ed +Ġels if +Ġs ake +Ġc ycles +Ġì Ĺ +ĉ I +-c ollapse +ĠG MT +De claration +Ġg ros +Ġreach es +Ġcust ody +Unt il +t u +ĠCh en +Ġn x +( addr +ĠO ffer +Ġcol leg +ass ador +Ġm apper +ĠS IGNAL +ĠB loom +ĠH oll +ĠIm per +-d es +_s ite +Pro c +E qu +Ġat omic +ĠW oman +s ent +sc ar +Ġint elligent +ĠGet ting +ĠReg istration +ĠPh ill +Ġkill er +unic ode +Ċ ĉĉĊ +ĠJac ob +ĠCon st +Ġloc ate +Ġca us +ĠSch olar +Ġconstitution al +Ġinfl ation +ĠG ot += array +end um +Ġtransl ated +Ġdiv orce +En tries +Ġs or +ĠQu ote +irl ines +U K +Ġexc el +( opt +ĠAD V +,: , +Ġcontact ed +ĠD A +Ġr ings +ĠIndust rial +.get Context +Ġforg otten +ĠT an +Ġp ants +Ġo v +Ġdec oder +ĠPart ial +Ġv c +Ġbatt les +A rial +FRING EMENT +ir ates +, w +aint enance +ĠO d +ĠTechn ologies +åī į +ĠCar ter +.find All +N ome +B en +ĠUs age +ĠP icture +Ġbad ly +_p anel +Ġpat ent +ĠProt ocol +lot te +ĉ player +je ctions +Ġd ou +_re lease +urn iture +_t ax +ĠF ields +.d ataset +_m aster +CLU DE +ĠPh arm +b st +Ġoper ational +.c ell +Ġident ifying +Ġj wt +t uple +ĠT C +ĠC ro +ix map +- components +gener al +Ġo z +_D e +_d ouble +ĠTo o +.View Group +g ate +d ings +ph otos +Ġgrand e +ol lect +_l in +Ġaw ful +f ilters +Ġaltern ate +es p +Ġcomp ress +e o +ĠS cale +Ġind irect +Ġinv oice +ĊĊĊĊĊĊĊĊ ĊĊĊĊĊĊĊĊ +Start ing +ĠPl ayers +ie le +. then +Or d +ĠT uple +Ġb out +ĠStat istics +Pre view +Ġp uzzle +ĠW idth +ST ATE +Ġover lay +ĉ on +Ġin fr +Ġsm allest +lock ed +ÑĤ о +ss l +Ġde emed +Ġs co +re ck +Ġj Button +Ġmiss ions +ç§ ° +.Selected Index +T ABLE +Se pt +Ġacknow ledge +Ġstrt otime +ĠT ell +ĠD ak +Ġal uminum +Ġf ence +ĠSt ars +CON FIG +Ġretro fit +Ġemph asis +/ header +ĠS omething +in ished +=' ".$ +ĠValid ators +Ġpol ar +section s +.as px +Ġas pir +.M ock +Code Gen +Ġpe ut +Ġaccept ing +Ġback ing +P icture +/ ap +еР³ +_SE C +- use +annot ation +Ġcogn itive +Ġg rip +h our +ĠLeg al +Ġep ic +.t oolStrip +.not ify +.L ast +OR IZ +M iddleware +cri ptions +l ash +_F OUND +ĠLiver pool +Ġ{} ", +Inst all +Ġn it +Ġfig ured +[ len +.W in +.pl atform +Ġgam bling +(d t +av ery +ĉ include +Wh ether +R outing +Ġther ap +Rem ote +ĠL oss +y ll +Ġappro ached +ĠV ehicle +ĠAl pha +Ġvoc ê +ans wers +NS Dictionary +cons ider +un used +ĠF an +or able +f re +ĠDIS CLAIM +ĠAct or +. ] +to Have +.user Id +Ġspeed s +ew ay +Ġrec urs +ĠÐ ³ +_pr iv +! âĢĿĊĊ +Ch oice +Ġsett le +Ġplan es +' }, +T om +IT ER +! "Ċ +å » +achel or +Ġsepar ation +Ġd al +ad j +Ġreg isters +r iz +ĠNot ice +Ġl u +Ġcour age +Ġax es +cell ent +.as ync +Ġcompat ibility +ç « +Ġ! ĊĊ +ĉ title +Y LE +ĉ message +U UID +OLD ER +ĠH H +ĠStyle Sheet +Ġaccess ed +. validation +t asks +Ġpoll ution +.c anvas +Ġing redient +ĠC abin +A h +old own +ĠNO I +ĠÃ Ĺ +[ f +ed uc +y alty +(n ot +_ State +am en +Ġda o +ud ad +ell ers +} & +lic ity +_W INDOW +Ġt atto +val or +.R ange +Ġrefer enced +ĠRes erve +M oney +SCRI PT +/ product +cho ices +Ġt in +ãĤ ĵ +Ġsepar ator +Ġp kg +am med +ĠM AT +! !ĊĊ +Ġr aid +Ġmotiv ation +ĠX P +ĠBack ground +ĠQu aternion +.define Property +ik er +ĉp arent +ĠOrigin ally +ant age +ĠH ans +Ġtim eline +.c ur +op ic +ĠSe qu +m ust +ĠCo al +Ġform atter +_R GB +Ġ_ (" +'} ),Ċ +Ġ= ================ +ĠF UNCTION +Ġl ng +ic ates +l ive +_ engine +Ġtown s +')) ĊĊ +ĠP K +( api +ĉs canf +pack et +.ph one +á Ģ +ĠAnd y +_N AMES +PL Y +Ġmin s +im i +Ġbr ick +Ġbl ade +.std out +}` ;Ċ +Sh ift +ĉs b +ĠCheck s +Ġphenomen on +Av atar +Ġmin istry +ro se +ĉ File +Ġtit led +( LOG +Ġg an +des ign +(), čĊ +Ġb ones +st m +ÅĽ Äĩ +ĠInput Stream +Ġvol unt +ĠSerial izable +Ġfight er +ĠDr ag +T witter +Ġsubs id +ç ¼ +Ġfor ums +.load ing +log ged +_ this +Ġterr ain +Ġir re +ĠIn g +ĠC N +_object s +. uid +Ġconscious ness +T INGS +ĠG all +Ġport ray +ĠDevelop er +Ġparticip ant +Ġ" ;čĊ +/ model +ĠOper ations +^ \ +ĠL ater +Ġrais es +-n one +.m eta +=' .$ +Fin ished +Ġrepl acing +Ġsam pling +ĠJ en +" There +RE AL +A LE +ìĬ ¤ +Or ders +_param eter +ĠOlymp ic +Ġtr ès +Ġare na +i ol +; ?> +Ġimpact s +ĠW S +: get +Ġfl ights +ĠRuss ell +c amera +F n +s igma +Ġfor cing +Ġloc als +Ġdepart ure +Ġcelebr ation +ĠS ay +ï¼ Ĵ +ĠH ills +.has OwnProperty +Ġtyp ings +.A PI +Ġdon ation +Operation Exception +.Act ivity +c plusplus +ĠChar lie +Ġimport ed +Ġd ann +Ġoccas ions +Ġimplement ing +Ġpur ple +.d ialog +SQL Exception +ern o +Ġw ars +Ġpast e +Ġdecre ased +Ġhar sh +Ġel abor +input s +ĠView s +Ġerror Message +_m ul +ĉ write +ĠC op +ĠAnn ual +(b utton +Ġv ida +b ars +ĠHar vard +ĉex pect +Ġindex es +Ġdocument ary +Ġf lesh +OR LD +ĠD elta +M AND +Br ush +-c olumn +Ġdevelop ments +method Visitor +s lice +ĠP DO +Ġinvest ing +ir able +Ġxml ns +ï¼ Ľ +art a +Ġthe ories +_c ity +Ġ$ __ +Cre ating +( pr +D ropdown +ism atch +ĠN ET +'] )){Ċ +ĠVal ues +ĠSE O +ĠST AT +Ġe cosystem +Ġtem pt +Ġ\ \ +Ġ// {Ċ +ĠChrist opher +ĠKent ucky +ĠHttp ServletResponse +Ġhy brid +y on +Ġfeed ing +ĠEx tra +N orm +IT CH +ĠSe an +ĠUp load +m un +p ur +Ġp ersistent +ĠID C +ĠPer form +.m erge +_ room +Mean while +! =' +ĠW el +Args Constructor +.D atabase +Ġcount ing +() * +Ķ åĽŀ +ĠT OP +m ill +ĠD T +IGN ED +ĠK B +Ġcomp ly +S outh +_c ollection +Ch apter +Ġexpl aining +_ AM +_t s +c ards +Ġqu el +Ġp ole +Ġtouch down +ĠO thers +Ġpe ers +ĠType Error +Ġsix th +Ġche er +Ġdis pute +us c +) ], +th umb +Ġh iding +ĠS IG +lik es +ĠP AGE +.Ref lection +Ġhead quarters +T ING +ĠG host +M LE +$ Ċ +Ġcontr ary +ext end +'] ). +FF ECT +ĠP interest +úmer o +ric ane +ĉs ession +Ġcr ystal +- Control +overn ment +og raf +- action +v olume +ft en +Ġun con +Ġan imate +Ġle ase +sc r +Ġref use +ãĢ ĭ +ft p +in formation +Ġeval uated +Ġin jection +Ġj ack +Ġwork shop +æ³ ¨ +PT H +ĠT s +off er +ĉ os +Ġking dom +M issing +Ġlaw makers +ext Field +Ġsing ing +ab i +/ client +.m edia +ATEG ORY +Sign ature +% ',Ċ +ĠF uck +][ : +Ġsens ors +/ com +ĠPr imary +.S QL +_pro gram +Ġp ills +Ġinteg ral +Ġfle et +Ġdro pping +.s l +Be en +Ġp ets +Ġadvis ed +Ġdr agon +_ EDIT +( im +F ER +ĠDr ug +(r andom +Ġcomp ression +ou st +[ % +Ġbuy er +h op +R oles +man age +Ġpain ful +ĠBr anch +-mod al +en ant +ĠM esh +/ font +ĠG raham +Ġâ ĺ +Ġn c +ĠFranc is +Ġspec ification +Ġdam ages +- config +Ġthe oret +sec ure +_m ulti +aceut ical +Ġdemand ing +en ne +IST S +() ));ĊĊ +Re ason +Re cent +ph ase +Ġps y +_M AN +Ġvolunte er +å ¿ +istrib uted +li o +Ġproduct ivity +_com m +S pring +n is +. weight +ĠC ancer +Al loc +ĠT weet +Ġsepar ately +ĉ check +_p roperties +. Unit +_CL K +Ġg t +Ġ( );ĊĊ +Ġhand y +ĠThom pson +Ġunn ecessary +ĠRe ader +G N += request +ĠU tility +.Re pository +ĠA x +hy dr +ie u +Ġth y +Ġl t +_m ail +ä¿® æĶ¹ +ail and +ĠPhil ip +Ġbit ter +Ġbet ting +Ġtim ed +ock s +' a +Ġal gorithms +Ġre interpret +Ġto ss +ro gen +Ġhop ed +( selected +Ġvent ure +TE X +ĠLe ave +.Sub string +Ġgr ateful +uk a +ĠCon sumer +Ġag greg +C ircle +ภģ +_block s +Ġleg ally +Ġ" | +ãĥ ĥ +. board +.A b +Function s +rec ipe +è ĩ +ĠO xford +Ġwho les +.B uild +_ch anged +h ai +Ġdepart ments +I mp +Ġcoal ition +IN FRINGEMENT +Ġemp ower +itch es +N orth +Ġinfl amm +ON SE +Ġmiss ile +ĠR aj +ĠIss ue +Ġat oi +ca led +.Cont rollers +ĠW olf +Ġcrush ers +á» ĩ +.A uth +.add Attribute +h is +Ġbo ots +.c lean +c amp +Ġten ant +Ġt une +Ġ{} '. +Ġwork out +Re po +Ġpartial ly +MI SSION +j amin +ĠS B +Ġdetermin ation +Ġ' ');Ċ +ĠB eng +Ġv os +Ġin hab +/ lang +s burgh +Exec utor +h one +ĠCh allenge +_link s +.Le vel +Ġunder ground +-c ode +Ġoptim ization +log ging +_de st +Ġsn ake +Ġchemical s +_IMPORT ED +ado op +ĠTH AT +man aged +Ġredu ces +ĠRE AL +ĠG uy +_GENER IC +/ ******************************** +. amount +Ġd ere +get Time +Ġp ant +an onymous +Ġharmon y +ĠAl an +Ġscen arios +Ġd irt +ht ags +M c +Sh ell +r in +{ čĊčĊ +.p ow +ĉ client +Ġconspir acy +Ġad mission +ĠReg ional +ĠView Controller +ĠPhilipp ines +Ġde pos +Ġp ap +ĠP ad +P aul +.Com boBox +Ġt utor +ĠRec ipe +w riting +Ġcontrib utor +OT H +Sm all +V I +Ġh acer +e qu +ĠEx amples +h uman +.m essages +ĉt yp +Ġ( čĊ +ĠS SL +LE N +ĠRom ney +( grid +ĉ min +Ġ> ĊĊ +Ġfr uits +Ġvot er +In line +pan e +ĠC ollections +char set +Ġsp am +z b +item ap +Ġsucceed ed +_C OL +Ġel apsed +im eter +Ġrecover ed +T ensor +hatt an +.set up +ist o +( head +ĠS IZE +Ġtact ics +Ġdist ur +Ġpre val +ici os +( Value +_c ols +ĠF at +Ġse al +Ġs ons +Ġens ures +Ġpress ing += & +igen ous +Ġharass ment +_ JSON +Ġign or +yn omial +om er +_st atic +Ġsignific ance +Ġcirc les +_S ystem +Ġdiscipl ine +Ġdress ed +Ġs phere +Ġclim b +_ actions +ĠB ab +Ġ' =', +_s chema +" use +Ġund ers +Ġc ups +.s creen +/ new +Ġappe aring +T OP +vis ed +cl ang +Ġinvestig ators +Ġmyster ious +Ġprom ising +Ġqual ify +Ġc ave +Ġequ ip += x +G T +( link +. velocity +. erase +ot er +++++ ++++ +pro fit +Ġz ones +_ uid +- ser +Ġobject ives +Ġmil f +web kit +(m atch +ne h +ĠAssoci ated +ĠT odo += d +C am +Ġv ocal +Ġs udo +( EX +Ġtr ou +AB C +.b ean +ĠG round +ĠRE ST +we ets +In g +im on +_b us +ĠC OLOR +un to +Ġf oss +ĠLink s +ä ng +/ forms +pr ises +Ġachie vement +C ALL +ел ÑĮ +ĠVer ify +_S OURCE +apt cha +ID D +_re ference +G old +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĊ +Re ceiver +Ġa j +_d irection +} ] +ĠCom pet +Ġb ang +ĠC ass +- url +te chn +ĠJer usalem +long itude +' );čĊčĊ +Ġwin ners +T asks +ĠD MA +Ġtool tip +İ · +ĠB ra +_d uration +cur y +parent s +---- >( +ĠK ir +Ġint ros +Ġsk etch +Ġsk illed +Ġim mer +Ġade quate +_re p +( header +_ like +Ġper ceived +ss h +Ġassum ing +Ġf f +_u uid +ul as +Ġdemocr atic +. entities +S eries +aph ore +Ġnew er +} ( +SE C +ai ro +Ġcomm od +Ġprivile ge +Ġde ux +ĠH op +.' / +ct ic +. ';Ċ + C +ĠWar ren +Ġoptim izer +ĠSER VICES +_ oper +get Attribute +ĠMc K +_s elf +.r s +" )ĊĊĊ +Get Component +er ce +Ġt ous +un its +'] );čĊ +Z oom +/ E +Ġobs c +Ġfast est +on line +Ġpeace ful +ff en +Ġc argo +ĉ pr +Ġseek s +z u +Tr im +Ġw ard +Ġver d +Ġblog s +.exception s +ĠPrem ium +ĠN etherlands +S afe +Fin ish +ĠAl bum +_A CC += this +v irtual +] > +_L ABEL +ĠN ich +_w in +ĠA aron +W P +; $ +aim s +ĠImage View +Ġend less +ER A +_DIS ABLE +Ġcancel led +- us +Ġins pection +em in +ĠG rey +- open +Ġiter ations +. owner +Ġk eras +.P assword +ĠR y +ĠIN S +A ir +ĠSe veral +.Tab Stop +ING LE +ĠH air +ĠCan vas +AA AA +Ġfl aw +ced es +.Re port +í Ĭ +ĠT ips +cript ors +.trans action +.S pring +Ġview er +Ġins ights +è¾ ĵ +ord ion +U INT +se ek +ĠA uf +ìŀ IJ +Ġstr ain +To oltip +Ġd z +ign al +ad t +Ġu c +fin ite +Ġn m +.c md +ĠMy Sql +[ data +.j ackson +.t ree +Request Param +_ agent +") ]čĊ +Ġass ass +( Constants +: ss +ĠM AN ++- +- +ĠB ottom +print s +ĠS ame +@ Autowired +sw ap +ici ón +Ġprotest ers +Ġh oney +ĠV eter +(C alendar +- ad +ĠBrook lyn +L ife +_V AR +ze ch +ĠC ALL +_C AST +ĠE lection +Ġthick ness +V ery +_IN TEGER +- dev +)) )) +ap at +oo oo +d emo +Ġparse Float +ĠR ather +ST IT +m aker +[ current +chron o +Ġch rist +ãģ ª +ĠD etail +ư á» +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġs ul +id ency +Q ue +Ġeleg ant +ap ons +Ġdish es +Ġinteg ers +( read +find ViewById +ĠAm ount +ĠSk ip +Ġhab its +* )( +Ġmon sters +M AC +: end +Ġfr ank +As sembly +Ġd fs +Ġne ut +_TYP ES +e qual +loy d +( uri +Ġch i +Ġdefend ant +Ġconflic ts +Ġv il +- js +ĠPe ace +Ġmut able +) sender +ĠF ocus +å» º +Ġapprec iated +s leep +ĠR ED +C ulture +Ġdesign ers +_g enerator +c odes +/ ex +.Get Value +umb led +.scal ajs +per or +Ġveter ans +Ġ} )čĊ +Ġun fortunately +_C REATE +M ass +ĠCL AIM +ĠMe et +_s upport +B ank +() .Ċ +D ark +_LO W +ĠMin ing +ĠO wner +ier a +Client e +Ġencour aging +> S +Ġboy friend +ĠH alf +ĠA CC +A ff +_ ar +-l ife +c x +.J Button +iz ado +.z ero +.open qa +ot on +.text Content +Ġto ll +at ie +Ġball ot +- number +. Exception +ĉ params +c ircle +-m ap +Ġn ap +ĠRob ot +ĠI ch +reg istration +Am azon +roll ment +( exp +Ġt anks +ĠG ordon +Ġmach inery +Ġbas eline +æ ĭ +Ø © +ĠCon vention +ĉ config +ook ies +m ult +Rec ords +ĠE ST +Ġgar bage +Ġcon form +id al +Ġb arg +Ġsurv ived +Ġinvestig ations +.contains Key +---------------------------------------------------------------- ----------Ċ +ort ion +Ġhor r +_ http +Ġm ant +] ;čĊčĊ +b inary +em pl +Ġin quiry +ĠMean while +Ġcollect ing +.Entity Framework +", ĊĊ +ĠP ic +@ Inject +ick ness +ĠB inding +Ġcont rolling +re verse +Ġch airs +semb led +( add +Dis abled +an as +.trans late +-------- ---Ċ +Ġref lected +"] ĊĊ +Ex ternal +Ar row +Single ton +% x +Ġ Å +Ġan cest +ĠOr leans +ĉc md +Ġprohib ited +ith metic +(ch annel +_c ss +For ward +.s ocket +Ġl uc +â Ĩ +ĠFire fox +ĠM ovies +) _ +. ends +( shape +Ġde alt +Ġs aves +Ġgl ory +Ġmej or +Ġbreath ing +Ġ eller +get Data +Ġang les +Ġtool bar +Ġsp acing +IP S +Ġflo ors +_ACT IVE +Ġsh uffle +/ shared +ĠE le +ed ish +Ġweb cam +.ex pect +il oc +ĠIn cludes +Ġtweet ed +Ġ: ) +ĠEss ay +F ix +-b etween +_ web +.con v +Ġrac ism +Ġreflect s +um m +иÑĤ е +_f ooter +/d ocs +ĠP our +Ng Module +.initial ize +pattern s +_ In +ĠAb b +* čĊ +Ġsent iment +b uff +_count s +Ġre use +ch unk +Ġim posed +Primary Key +Fore ground +Ġconsum ed +? ! +Ġd ick +Ġch ron +ĠF ern +Ġrespons ive +Ġin sect +icult y +Ġr w +Ġal ike +Ġsub set +ĠCook ies +ĠP air +Ġt ier +IF O +av our +ĠQ U +, sizeof +Ġmerg ed +m v +it ol +yl on +Ġjump ed +. role +ens aje +R ules +Ġb rowse +An imator +Ġy oga +Ġvari ants +Ġcour tesy +ur an +p bs +else if +Al t +ĠL ane +CL K +IM ARY +_PRO PERTY +ï¼ IJ +Ġch an +Ġgrad ually +Ġsh ake +Ġbl onde +... ");Ċ +-se x +Ġgame play +ac ies +.ref resh +US B +ĠPl ot +W as +iss ippi +ĠT ensor +Ġcryptoc urrency +Ġdifficult ies +De leted +With out +_ append +_ ver +")) čĊ +Ġhonest ly +Ġp ivot +Ġtem ps +_p s +ĠUn like +[: - +V S +_in f +Ġjun ior +Ġanim ations +Ġfile path +? {{ $ +Ġun icode +pl aces +ĠC offee +.S E +ĠP AR +(t xt +ge bra +Ġf ires +Main Window +med ium +Ġ( âĢľ +Ġl g +Ġc mp +/ base +_l ayers +_ entries +Ġadmin ister +ĠSU CH +B P +ĠScott ish +ĉčĊ ĉčĊ +gu ard +ĠStr ong +In sn +ĠC AP +as ury +ĠSE E +C lock +er ie +\ models +Ġ$ $ +ĠC ab +Ġwur de +Ġsold ier +Ġcl ips +Ġarrang ement +ĠW onder +ĠH orn +Ġsc ared +Ġc ure +m kdir +Ġal igned +ĠP ink +Ġland ed +Dim ension +Scroll Pane +.ch at +.W ith +ĠTr ain +] .Ċ +Ġth irty +Ġdur able +Ġl d +Ġlate init +Ġch arts +Ġins ult +.F atal +_ ct +Ġm asks +CLU DED +Pres ident +Ġcol ours +g ments +.at tributes +ĠF lex +ĠC lock +ÃŃ cul +im en +J O +ĠReg ex +_L INK +Ġc ouch +ĠIN PUT +Ġbe ating +b usiness +pre ced +. unit +ĠF el +N ever +osp el +.start swith +ĠE PA +. only +Ġprevent ing +y er +Column Name +Ġelev ation +fl u +icy cle +Ġoff line +Tool bar +Ġcompet ing +) ]. +Ġm og +Ġis Valid +As k +_ av +_l at +AN C +ĠJ oh +k ers +Ġgu ards +Ġch ains +ĠSimple DateFormat +.st atic +Ġvess el +Ġm ud +Ġst abil +Ġst ret +g m +am ation +ç ľ +-w ith +Ġro s +_P A +Ġresult ado +Ġconf idential +ĠTok yo +ĉ using +ĠMath f +omb ine +ĠESP N +Ġdeal ers +Ġdismiss ed +TR Y +Ġte ens +rec ords +Ġw ings +g allery +account s +_L IB +Ġj acket +ĠNS Object +Ġst ones +ĠDel ivery +ĠD iet +/w atch +Ġto ilet +ĠG uest +.d ay +Ġint val +Vis it +Ġinvestig ated +Ġpent ru +ĠThe atre +andid ates +L ang +ĠS erv +Ġcont rollers +Ġset Title +N P +am y +fl at +( ui +_d ocument +è ĥ½ +ĠC oin +ĠAd ams +pt ic +Ġproduct ive +Ġaccompl ished +čĊčĊ čĊčĊ +Ġdefer red +ient es +Ġs inc +ol ars +Right arrow +Ġvari ations +( offset +.Layout Inflater +Ġsus pend +Ġprevent ion +_pr ivate +_ js +âĺ ħ +Ġw ieder +at um +Ĵ Į +Ġappear ances +.D ocument +Ġvalid ates +cal endar +} ";Ċ +.d emo +con ut +Ġcorre ction +ĠDe al +Ġbatter ies +.d uration +, \ +_m arker +m ulti +Ġh alt +Ġc ms +Ġsh aped +B ro +re duce +Ġ #### +CT OR +ĠBen ef +Ġicon ic +Ġp iano +Ġeffect iveness +| .Ċ +Ġa jax +Ġv olumes +ภ¡ +Ġcl js +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ath s +ra its +å¤ § +Ñ ĸ +_m ult +Ġfasc inating +A verage +Ġpr é +ĠChair man +.find Element +_p in +Ġcomp aring +Ġdark ness +-F i +- server +Ġselect ing +ster dam +ĠPart s +FORM ATION +Ġnot ing +Ġp ile +og s +Ġpa lette +_d o +it ize +() ( +Ġdef ining +Ġremain der +Un its +_T ASK +Http Client +S ocial +Ġfund ra +N R +ch est +C urrency +.ad apter +Ġd op +un ting +ANG UAGE +" He +ĉ index +_p ackage +.I con +Ġrep et +m ass +=" .$ +ĠS ud +Ġl id +pro vince +ì ľ +G PIO +Ð ļ +ĠMy SQL +Ġdoc s +ĠG A +Ġip sum +K ernel +Ġaccept s +Ġfit ting +Ġcu ando +Ġd uplic +ĠBro ther +ĠK le +num s +Ġmor ph +Ġ ######## +ĠCG Point +< unsigned +ä¾ ĭ +ĠD uke +.set Bounds +q s +or ic +j er +Ġregard ed +Http Request +Ġbond s +Ġthorough ly +enc ent +Ġhighlight ed +Ġac res +Ġwork place +ĠL ux +Ġqu ot +.in flate +Ġdocument ed +Ġadd iction +Ġmut ation +.c ity +Ġbott les +ĠRepos itory +on n +err no +ARI ABLE +åº ¦ +_B EGIN +gl as +' })Ċ +ĠMass age +ĠWh it +reg ex +W A +Ġout let +- head +Ġexp ired +ĠTh ai +/ include +grad ient +scan f +Ġse am +w al +ĉb uf +B earer +Ġprec ious +if acts +co ord +Ġexpl oration +.get Y +(h andle +Top ic +ĠV ent +r hs +---- --Ċ +ĠB right +Ġg uild +m other +st orm +Ġmunicip al +Ġin k +.T YPE +w l +... manual +ĠTechn ical +Ġcorpor ation +ĠH W +ank a +T AIL +ist as +Ġperform s +ĠBeh avior +.F or +_ ORDER +ĠK ick +Ġcallback s +_d r +ue go +h ub +uff icient +sk y +Ġb p +ht able +ĠON LY +ĠAUTH ORS +.Arg ument +" };Ċ +ĠTh under +ĠK om +.Sh ould +A UTH +ah u +_p ayment +Ġst arter +ìĦ ľ +ìļ © +B log +.p atch +Ġgovern ed +ass y +-f ound +Ġthe ater +ĠFont Weight +ĠBat man +" If +.R andom +_d elta +ĠC E +Auth enticated +Ġdr one +Ġc ous +r adius +M er +( None +ĠN J +_ headers +Ġam er +py test +ĠA ctions +ĉĉĉ ĠĠĠĠ +Ġet t +Ġh oly +Ġun comfort +ĠN in +ĠDec imal +ĠM essages +.s ender +] ])Ċ +Ġembr ace +Th ough +/ sp +Ġcult ures +Ġhigh way +t ar +.f ail +_h idden +ĠcomponentDid Mount +ĠW right +Ġj ag +_ il +../../ ../ +ig u +F ood +Ġa ce +Ġa ños +US D +Ġmut ual +Log ic +Ġtem ple +Ġbrief ly +ĠT rip +class method +default s +Ġch unks +,, ,, +ĠRe ason +$ id +-up s +Ġdam n +Ġtruck s +Ġun limited +Ġsc ulpt +ĠC ards +Ġaut or +ĠTest ing +Ġdies e +sh ops +ç ´ +(p ayload +ĠP ATH +ĠMem orial +Ġridic ulous +eg ree +-w inning +Ġre hab +Ġsophistic ated +wp db +ĉ path +! ";Ċ +_S YS +.s peed +Ġso ap +s uffix +W rap +Ġenh ancement +à ī +ú b +Ġplay list +Ġmix ing +ant idad +=" ";Ċ +ĠRev ision +ĠBe at +.in c +-w ay +enc ias +ul ers +C at +id el +ĠSh ip +.set Color +Ġthreat ening +.mod ules +Ġafter wards +ĠD ashboard +Ċ ĠĊ +Sign al +Ġpr imer +orne ys +ici ary +Ġl igne +_p redict +Ġa est +_ https +> : +ĠL ex +Ġrencont res +eg ral +sc ala +_f amily +ÃŁ en +_s ym +Ġuncert ainty +ĠVAL UE +Ġ} ;čĊčĊ +Ġbro ader +Ġh orses +ãģ Ŀ +ĠK al +ob a +_IN ET +ĠK ill +j query +am ination +[ @" +Ġm uj +## #Ċ +First OrDefault +then Return +C he +/ footer +Ġpark s +as je +ĠG ulf +Ġmod est +. Init +ï¼Ł ĊĊ +Ġpros pects +Ġs vg +Ġå ı +.D ialog +_N ET +Ġ( ($ +Ġe k +ĠW arning +ĠM K +< LM +Ġ' čĊ +i em +h etic +Ġi x +th ink +-sh adow +ĠE ld +ĠNev ada +ĠLe af +ĠG ROUP +Ġprom o +ent ine +ĉ Map +ĠModel s +ĠK rist +_k ernel +-m ade +Ġc err +As sets +ell ar +Ġinv oked +.v ue +Ġcult iv +C losed +Ġgener ates +ffff ff +thes ize +s qrt +ĠCast le +.c ar +Ġke en +und a +ĠC row +ĠSing h +y thon +Ġbe ans +l arg +æĸĩ ä»¶ +Aw esome +unc ate +Path s +o ji +(c urr +CON DS +Ġm im +Ġshould ers +H ard +ast es +а еÑĤ +Ġconv ince +de cess +m ade +ĠC MD +. Im +Ġcha os +ens ively +Ġcool ing +Ġbur ied +(' @ +_S e +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉ +.com pany +.sub mit +ph ant +Ġboot strap +_h elp +à § +.d ump +Ġdif er +_m apping +Ġcirc ular +Ġescort s +Ġb ere +Ġgrad u +ĠLeg end +im edia +ĠBar celona +Ġbed s +åĪ ° +ãĢ Ĭ +_v olume +Ġtremend ous +Ġsc aling +Ġp ins +en as +type param +D ashboard +render er +Ġsp i +Ġ& $ +ĠSk in +alm art +Ġh ockey +Ġ'" .$ +Ġerr no +Ġb ew +Follow ing +.M odule +er able +ĠM ilitary +ĠR io +_ available +ĠSur face +Ġst ab +IF IER +ĠL IST +Ġd ashboard +Ġcl usters +.pl ugin +Ġj ou +ĠDec or +F our +Ġdel le +****** /Ċ +ia z +in de +ch ing +Ġget Item +.Add ress +ment ed +A meric +Pl ain +Ġus b +ĠPract ice +_ ment +.bl ue +H int +ÑĢаР² +Ġconn ector +Ġinher ited +и в +Ġinterval s +Ġc ere +Ġu d +Ġin con +.Ex ists +ĠM ic +F K +(c ard +.Set tings +Ġexhib ition +Ġon Pressed +Ġrest ored +eng u +. def +Ġrec v +." );čĊ +enc oder +ather ine +( dest +az ed +# endregion +sem bl +, M +ob y +Ġп еÑĢ +.C all +Ġattend ance +-b order +Ġaddress ing +ê n +ĠLe v +Ġb ash +ben ch +C redentials +Sp acing +( of +_RE SET +ig uous +Ġcr uel +Ġcross ed +Ġle ur +ĠG olf +or rect +Ġpack ets +ĠData Set +Ġpart ly +SEQU ENTIAL +Ġindic ation +ĠS alt +ac ia +Ġ* );Ċ +ĉ info +ĠView Bag +on z +Ġeditor ial +ĠA rena +Ġs ir +_ Static +( socket +s u +cho ose +.m onth +.M y +é ri +; font +do es +Ġcon verter +Ġsal v +Ġl r +Ġinflu enced +(f eature +ĠQue ens +let t +_M ON +& amp +Touch ableOpacity +O FF +Ġmetab ol +( iter +Ġvit amin +ĠIND IRECT +aut om +_p ublic +Ġadjust ment +Ġspecial ized +w indows +.add All +Ġaccording ly +ĠJ OptionPane +Ġcell spacing +Ġqu ad +Ġcre ep +Ġout lets +}` )Ċ +Ġpri est +_TH READ +ĠMar x +ĠBy Val +Ġc ual +éĿ ¢ +Ġtempor arily +An n +ke leton +å ¥ +ĠLO C +au er +der ive +Ġbeh aviors +as ename +ĠCent ury +Ġhor rible +ME SS +_ List +we i +P at +ĠCh oice +_F ROM +ĉ line +.in voke +.B ottom +Ġnow here +." ĊĊĊĊ +_ export +Ġstrugg led +.Ap pearance +ĠJ Button +ĠJer emy +([ [ +Ġkick ed +mar shal +st aff +es ity +Ġqu iz +_e ffect +Ġ} ));ĊĊ +m el +b anner +ĠP IN +Ġin vention +Ġcons olid +Ġop s +ĠB etween +j ack +ern ational +Ġsacr ifice +ag ation +ĠJ oy +Ġam endment +ĠS old +Ġprison ers +ан нÑĭ +Doc uments +) ])Ċ +ust ed +ĠLine arLayout +os o +_E M +.s elf +.M iddle +) // +Ġ\ ' +Ġfuck ed +ĠM urray +Ġprof ound +_E LEMENT +ult a +il ers +port folio +J une +t cp +mod ified +ĠTr ace +ĠK el +aly zer +) => +ĠRep air +_B E +Br and +u art +pre view +Ġiniti atives +run ning +b ang +ĉ update +ĠCo ach +R ich +Ġy outube +Ġrit ual +app a +ĠRobin son +prec ision +//////////////////////////////////////////////////////////////// //////////// +=[ ]Ċ +Ġcelebr ated +OT O +Ġin clusion +J P +' ;čĊčĊ +Ġnot able +(_ . +Man aged +Ġgu ides +& nbsp +ated Route +ĠAd just +Ġcol ored +_s cores +ĠTes la +_pro gress +.in st +[' _ +.fl ags +Ġf close +_O PER +ż y +_n ote +Ġtrans gender +å ķ +RI PT +Ġabs ent +Ġam et +Ġoper and +ë © +Ġh ood +to LowerCase +av o +ĠCirc uit +ĠL ind +-- }}Ċ += m +Ġsup press +ĠM AP +i ang +- admin +Ġside bar +ĠB u +ĠH ex +, F +ĠSign al +Ġtrans parency +ĠFeder ation +/ V +Re q +Ġpul se +Ġt ends +Num bers +% ' +Ġde port +dat as +_U INT +_ tra +ok o +Ġ" ? +comp et +sole te +und ry +Ġover lap +}` ,Ċ +. ly +_sum mary +ĠL ost +.C enter +Ġdis ability +.Serial ization +Ġge om +Ġ? : +ĠW o +Ġsh ipped +Ĥ æķ° +Ġu gly +Ġexcit ement +Ġext erior +Ġcheck out +Ġk ur +, D +ĠAl aska +Ġsyn thetic +ĠB udget +ĠSub scribe +Ġ& Ċ +ÈĻ i +ĠY u +ĉ query +} .Ċ +Ġtr aged +ass en +Ġaccommod ation +Ġphys ician +Ġren amed +Ġtid ak +z Äħ +Ġmin us +ny ch +_EX CEPTION +thread s +Ġt ire +_c reated +ens ure +Ġworth y +Ġexc use +Ġclo th +.parent Node +/pl atform +ĠU FC +ĠG tk +un ny +Ġg ibt +ke ley +h um +(t x +ĉ dev +Ġout fit +do ors +Ġf on +ic ut +vol atile +Ġhom osex +Max imum +Ġexp end +Ġ});ĊĊ Ċ +E q +ond ers +dep artment +ĠPhys ics +" });Ċ +Ġpar ad +.S tr +Ġse le +IF IED +Ġdel ivers +iv an +Ġrespons ibilities +Ġadvoc ates +è µ +ĠR ID +.param eters +M etrics +ron ics +ĠUITableView Cell +A bsolute +ip se +yl um +MLE lement +_VAL ID +< title +D lg +p aces +Ġsynd rome +be ans +_d atabase +oz illa +ĠM eg +DB G +Ġl ub +Bag Constraints +ab ad +Ġproject ed +_BY TE +.Size F +st reet +ĊĊĊĊ ĊĊĊĊĊĊ +ĠLO SS +Ġdirect ors +/ news +Ġnurs ing +ĠD one +. HTTP +dis count +ĠR ot +To Many +Ġen abling +Ġauss i +ost a +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +è½ ½ +Ġhel icopt +ĠIn side +ä¿¡ æģ¯ +is per +ĠAll ah +ARCH AR +Ġroll s +Com pare +X P +Index Of +S UM +Ġass ured +ĠPhys ical +End point +.G lobal +.d etail +Ġthe ft +.j upiter +Ġhum or +.R ender +A lex +.c ap +Ġbuff ers +Ġdis pose +t ion +.p resent +z el +, P +Ġdesper ate +.get Column +Ġtw in +ì ĸ +.c an +Ġf lee +ĠIran ian +Ġstick y +ĠU TC +L T +//////////////////////////////// //////////////// +Ġl icensing +_PO INT +ĠM aps +Ġl ol += models +-t ab +ĠN ash +_log ger +tor ch +ĠCON SEQUENTIAL +Not Empty +/ react +Ġp f +Ġassert ion +Ġsubsequ ently +_c an +Ġpand emic +og ue +"+ Ċ +_ ent +_P aram +.ĊĊ ĊĊĊĊĊĊ +Res earch +C apture +Ġbel oved +d em +Ġextract ed +Ġf ights +ER C +(a uth +position s +Ġrevers ed +(st ack +Ġ_ ) +uto ff +_fl ow +ç Ĥ¹ +( Game +Ġex cluded +ĠCS V +c g +ĠT itan +p ause +Ġcer ca +Ġdump ster +L ess +Ġkotlin x +aster xml +Ġpoint ers +Ġfl ows +ĠT un +ĠMain Activity +Ġdis cret +Ġcomb inations +vis it +_b ind +oot ing +d ater +_look up +.n io +Ġswe at +ĠR d +Ġscient ist +ĠP ixel +@ NgModule +Play ing +Ġunf old +Trans late +ĠLaw rence +ĠFIX ME +B ill +ĠR IGHT +Ġwhere ver +Ġo ok +vid ence +Ġ] ]; +ĠSk ill +unist d +ĠðŁ ĻĤ +Ġfem ales +-- )Ċ +İ· åıĸ +ĠF red +Over all +Ù Ĥ +Ġess ence +Ġthere by +Ġw ounded +ĠD OWN +les son +text ure +R ound +Ġautom ated +ĠÐ ¡ +ĠUp dates +Ġsh ade +p ublish +ĠG ear += lambda +Ġle ver +) +" +h ill +Ġrad ar +ry ing +Ġ" ). +f illed +Ġline up +Ġd l +Ġworks pace +V o +_d t +ë ² +_ Item +NS URL +. verify +ĠHawai i +G od +M arch +Ġ[â̦ ] +Ġpel o +ur ious +ĠPitt sburgh +. It +C lean +> \<^ +Ġi os +s ound +"] ; +Ġfre ed +rot tle +ĠL ower +[ count +å Ŀ +Ġp ale +ĠWay ne +ear th +_c ategories +U CK +.m etadata +Ġsum mon +H OME +олÑĮ з +Ġmanufact ured +Ġdo ck +Ġcompet itors +_MODE L +ok ia +ĠH ey +Î ¿ +Ġback ward +ĠPO SS +rop a +Ġc ri +_O BJ +Trans port +-h igh +Ġerot ik +_s lot +Ġart ic +_f ramework +-ser if +ĠSql DbType +') ( ++ "/ +Ġw ore +S il +Ġst oring +ĠPh ase +u ant +Ġb ump +in ho +Ġd ign +Ġback s +q q +(h ash +Ġge o +Ġt ender +Log o +! )Ċ +ĠM X +ĠAr thur +esso a +_C h +Ġbed rooms +="# ">< +Ġth roat +ins ic +.int eger +Ġpr imitive +Truth y +Ġfacilit ate +Ġcreat ivity +ĠD NS +Ġg ra +ue z +Ġcount less +ĠPol and +' M +ĠD ist +Ġv est +Ġcert ification +á» ij +h eld +ext ensions +( static +Ġgr ades +ĠU ber +ãģ Ł +Ġ[ ])Ċ +dat os +Ġget Data +ĠCh arg +ĠB S +.m icrosoft +.v ideo +.d irection +->{ ' +l ua +ape st +Ġbo iler +ere k +Ġdec ides +.j ar +IS C +ĠW ords +(C ON +EMPL ATE +ree ze +sh ots +app s +unt ed +.set Name +:: < +-b old +ê ² +å¯ Ĩ +Long rightarrow +Ġunf air +Ġear ning +Ġsh elf +URE MENT +Ġid le +_M ENU +.C ustom +AG ER +- " +_s witch +b ecause +) view +m are +_ condition +ĠStart ing +M vc +(p re +d ump +_LO CK +at etime +.c allback +ĠC er +op ol +ib rary +Ġres ervation +ĉĉĉĉĉĉĉ Ċ +lect or +grad uate +Ġgener ous +Ġ ion +ric ao +m q +_com plete +(c ursor +ĠForm Control +: center +Ġsub stitute +ĠPl anning +Ġp ension +Ġrecommend ation +ĠT ags +Ġg ef +Ġalbum s +Ġwash ing +ro c +Ġtr ains +at ings +Ġex ponent +ack bar +- ln +á g +.Data Annotations +ĠE IF +ĠMalays ia +ĉ PORT +on us +Ġcle ver +Ġpe u +> ĊĊĊĊ +ĠArg uments +Ġdebug ging +( right +' D +com pute +Ġfin est +OR AGE +Ġspect acular +ph rase +Ġind ia +Ġlegend ary +b irth +Ġcom posite +Ġg rows +ĠT D +Ġep id +Ġlaunch ing +] ][ +Min utes +ĠCh a +Ġclean ed +Ġwitness es +uk an +ĉ Type +Ġhab e +par agraph +ĠJ Panel +ĠH ann +Ġvar ied +ĠP okemon +ĠM UST +åĬ ¨ +.vis ibility +op up +^ [ +.exp and +Ġ" ', +.f asterxml +_ auto +ĠShe et +mark er +Par cel +ew s +ĠStr ategy +-m aking +Ġun ve +Ġtrail ing +Ġclick s +ĠGet Component +ĉ content +IG ENCE +ERN EL +NSMutable Array +Ġb reat +Ġharm ful +¶ Ī +Ġbes ides +Ġb oring +Ġbrut al +v ang +(p arse +qu ick +Ġpy test +Ġswitch ing +() ]Ċ +Ġì Ħ +L ER +ĉf ont +Ġnet t +) ]ĊĊ +(/ \ +æŀ ľ +to Array +Ġbre ed +ĠC AR +ĠWe apon +A bs +t ot +Ġset Name +apt ive +Ġ: , +Ġesc aped +ord en +ĠP ri +th umbnail +Ġdescri ptions +/ styles +ĠPC I +Ġal phabet +astic search +NOT E +Ġc ialis +ĠGr iff +Ġpor que +Ġprote ins +pl ays +Ġst ating +Ġimag ination +Ġfac ial +ĠMe chan +Ġarr anged +_ used +Ġarrang ements +ĠP ipe +host name +Ġprov inc +T it +.Flat Style +ĠS plit +ĠLo ader +.c c +Ġclin ic +---------------- ------------ +Ġb aking +ĠEN T +ne ath +ãĢģ ĊĊ +AN E +.EntityFramework Core +app ers +. ic +ĠNg Module +ĠF ORM +Ġ' ; +-pro fit +h w +en emy +ĠE ye +Ġca ution +t own +Ġur ged +ĠJim my +ynchron ous +-s ized +m aking +, { +] ', +_ Object +ah oma +Ġactiv ist +IN VAL +ĠCom mercial +ĠOr lando +(t ab +ĠØ ¨ +Al gorithm +Ġher itage +Get Mapping +Ġfail ures +ri os +at iva +Ġt et +Ġcar pet +( Z +th ree +Ġdisc losure +. ERROR +_c alled +Ġd ial +Ġoccas ional +.E rr +Ġfunc ion +caff old +Ġrele asing +ï¼ī ĊĊ +_ Value +ĠV ari +y ellow +Ġstrugg les +.c al +ĠDak ota +ĉc lose +Ġsand wich +Ġanaly tics +Ġ** ) +& # +ĠJ os +Ġpass ive +AT TR +Th rowable +ĠM un +ĠU int +(dis posing +ar ak +ĠLe aders +Ġaffect ing +Ġitem View +Ġeconom ics +f v +à¹ Ģ +.r b +ĠOver all +Ġwealth y +Ġev olved +nd a +ĠH us +re strict +um en +ĠA gricult +! ĊĊĊ +Ġexp ires +Ġspokes person +int erval +Ġà ¢ +Ġque en +(n il +ing o +He ap +Ù İ +Ġcompl ain +S ym +ĠCl one +ĠR u +ĠW ILL +ĠCr ystal +/ content +ing en +oint ment +Last Name +av icon +ĠIB M +ĠDim ension +an h +icip ants +ĠAn ne +.pro gress +Ġal go +ob il +ĠV oice +ĠF E +Ġg li +Ġv ed +Ġprevent s +\ Column +Ġfol k +ett i +Ġm n +ĠCL ASS +Ġdisplay ing +ĠK l +ĠF err +d uto +. ib +Ġd ados +' name +-s pace +Ġit alian +Ġin verse +Ġd ense +ut er +ĠI Enumerator +-s ign +Ġnation wide +Ġperson a +Ġsol ved +Ġdram atically +Log out +Ġgr av +Ġanalys es +ol lo +Ġl amp +. team +ĠE rot += [" +Ġd ancing +Ġ?> / +Ġc ater +ff e +ĠSh a +ĠB os +ĠRE QUIRE +ĠMon ster +ĠR B +ĠI DE +Ġsu its +Ġform Data +( theta +Ġsp atial += NULL +ĠSql Connection +Ġ à +ĠV enez +ĠMor ning +Ġpublic ations +ĠNON INFRINGEMENT +first Name +ud s +W ould +_HE AD +Ġinvest ed +st able +f red +Ġcommand er +SE S +âĢĶ a +an che +ĠM ovement +ë ³ +S uite +Ġjur isdiction +ë¦ ¬ +ĠB eth +j Query +ĠIs a +Ġd ental +, * +ĠL imit +ili ation +=" { +b ast +Ġt urb +is y +O OK +Ġadvoc ate +im ag +LE CTION +л ÑĮ +(c ategory +.de c +Ġun iqu +_s n +Ġattract ed +Ġà ī +ĠRun ning +_ edges +ĠDis able +_A S +åĽ ¾ +Ġnetwork ing +_br anch +H aving +toBe Truthy +G I +Ġcamp s +se p +-p art +Ġ)ĊĊ ĊĊĊĊĊĊ +ustral ia +ĠRe ports +rit o +Ġwa ist +_pl us +ĠW W +-p erson +Apr il +Ġs ar +.t ar +Ġagricult ural +t ic +Ġt cp +Ġset Value +agent o +ĠAp pe +p iler +CA DE +Ġan che +atch er +Ġcom ics +Ġl bs +_se gment +'] =$ +itt ers +ich er +G INE +Ġutil ize +ĠC ursor +_ex pression +Ġd ag +< long +Ġr hyth +æı IJ +Ġconsult ation +Y et +")) ĊĊ +_M AC +c ould +Ġ' \\ +ĠV o +ĉ http +Ġg s +ph er +- grid +J ames +J ul +Ġsch on +Ġtensor flow +ĠLOG GER +am as +Ġsc ipy +Ġconv iction +. ag +Ġadministr ator +)) {čĊ +Ġn un +" group +P or +Ġnur se +ex pression +ak y +ĠHe avy +. opt +.get All +Ġover l +/ ", +_c ountry +ç İ +ĠG ENER +_r oute +ĠD al + ´ +ol oad +Ġuncomfort able +(m enu +Ġhost name +' ");Ċ +Ġcalcul ations +-c lick +Ġprotect ive +ãĤ ¯ +_F orm +ung s +Act ual +m f +ĠProcess ing +ĠIn ventory +(m atrix +app ropriate +w eg +ij a +Ġch r +Ġr ifle +-w sj +k ar +Ġindepend ently +I OS +Ġconsist ency +v n +/s ystem +ĠCh anges +Ġexp ose +ici ents +Ġrel ate +ĉ next +è ¨ +ud es +Ġglass es +F XML +.... .. +ĠP df +Ġappro ve +Ġ{ \ +Ġexist e +)) ( +ARE NT +оР¿ +ĠL atest +ĠNiger ia +.Inter faces +Ġrem oves +En emy +Ġen force +vert s +ĉ pos +_text ure +W ARD +ĠINC IDENT +( container +Ġdef ending +ĠR X +ĠH ook +br is +ĠFl ask +Gr ay +. )Ċ +vis ibility +ĠRedirectTo Action +err al +_e lem +Ġres on +front end +_variable s +ater ia +Ġ+ " +ave led +RI X +Ġdef icit +_C heck +YY YY +To One +sp y +Ġun ited +end ent +Ġp ode +ãģ Į +C AT +(f mt +ĠBon us +Ġre ck + º +Mod ules +Ġvac uum +R adio +ĠDAM AGE +P en +ĠPark er +; ;Ċ +ĠRe ally +_n eg +p ending +Ġnomine e +ĠC ategories +ĠUl tra +We apon +Ġdef ender +I ss +ĠG ender +ĠD ress +Ġimpr ison +Ġbank rupt +imension al +PH A +ĠStr ateg +ĠPROF ITS +Ġp atri +//////////////////////////////////////////////////////////////// //////////////// +de legate +Ġfor State +Ġdev oted +_m ake +Ġterror ists +ĠS nap +_n av +ĠA A +ĠI an +ĉ app +Pl acement +_h dr +< K +Ġs ang +st roke +- Q +> x +.T ask +m oney +ib aba +' });Ċ +ĠSpec ific +ĠLine ar +_O PT +Hash Code +( Player +.Contains Key +Ġcoll apsed +trans parent +_R ANGE +View er +(c fg +Ġsort ing +Ġinf ected +ĠN ach +Ġaccommod ate +.element s +_P ART +ĠSex y += get +( year +Ġx hr +: ] +ows ki +Ġsum mar +Ġ ¿ +Ġint e +Ġwork flow +ĠTai wan +vers ions +åı ij +Ġsurprising ly +Ġopt ical +Ġpro ces +Ġdisag ree +Ġnue vo +ĠC AM +sort ed +le ases +ist le +Id ent +ĉ event +ject ed +Ch unk +V ars +.pro vider +Ġproceed ings +Ġin clusive +Ġart work +end ants +ï¼ļ Ċ +se en +Ġl ig +Ġm akers +_f un +Ġlength s +Path Variable +[ item +ภµ +De ad +FFFF FF +ĠUr ban +up les +ich en +(null ptr +.s pec +, System +UR ATION +(j ob +å¼ ı +Ġtrack er +Å Ļ +ĠM R +ĠSQL ite +Ġd to +Ġ; ;Ċ +Ġm int +ĠInt roduction +ca o +Ġquestion ed +Ġf itted +rev ision +s q +Ġm ig +_un its +_ async +Ġf lick +});ĊĊ Ċ +Ġnot re +}` , +F ilters +Ġm undo +_d ays +Ġfr m +ut c +Ġval s +ew idth +ĠGener ator +ĠArt ist +ĠID s +ĠArt icles +re ater +ĠComponent Fixture +. = +Ġr ou +- no +.b ukkit +eg g +ĠD iff +atic s +Ñĥ Ñĩ +âĢĶ ĊĊ +ĠChar lotte +by e +Ġ} );čĊčĊ +ĠV ik +ĠB row +Ġl v +ĠG ib +-w ing +GL IGENCE +(I l +ĠEngine er +.W ait +ĠP ictures +Ġr het +Ġth ermal +Ġpr aise +< >();ĊĊ +ĠSp ider +P ause +ĠB aker +Ġsl ower +Ġ} ]Ċ +_en queue +Ġdisappe ared +ĠT icket +IN UX +_LOC AL +аÑģ Ñģ +@Inject able +comm unity +Gesture Recognizer +åĽ ½ +Ġsca les +Ġ- ( +/ '+ +ĠS it +Ġexecut ives +ard ing +Ġad vers +Ġback wards +ĉ context +ĠH amp +ĠP F +ĠDe ck +ĠCra ig +A merican +Ġb ell +Ġpro l +uf en +Ġr ng +ar shal +ĠSim ply +first name +sh ore +J uly +Ġmort ality +ĠâĨĴ ĊĊ +Help ers +Ġbench mark +em ade +Ġorganis ations +.g son +ĠText Field +Ġciv ilians +.Array s +ĠMiss issippi +Ġinter mediate +get User +_cl uster +Rel ative +fore ign +.querySelector All +Fore ignKey +Ġreason ably +-------- -Ċ +C ards +ĠK am +ĠTh or +Ġroll er +-e lement +ĠC urrency +dd ie +ALL Y +ĠR A +Ġper met +aa aa +Ġhom ework +ĠV it +Ġm old +ĠF er +[ start +Ġstatist ical +Ġsc ary +_H OME +.B egin +Con struct +ogen ic +ĠDEAL INGS +Ġtamb ién +ix on +. ind +ac re +Ġtransform s +ĠN ap +.B lock +uss ia +pir ation +ul ent +Ġce il +Cl ause +na ire +T ES +Ġne at +ST D +ĠReg Exp +per form +: ) +Ġun ions +Ġs ublic +Ġw inds +lo ating +g lich +Ġp agination +S kill +App ly +ĠOper ator +ist ogram +Ġqual ities +C ross +Ġde com +], " +ĠJ uan +.mod al +.Ch ild +ĠRog er +STIT UTE +:CGRect Make +a lette +Ġst a +as ide +Ġbl ur +ĠW a +if etime +re ed +control s +Ġb ins +Ġп ол +*/ ,Ċ +U IS +ĠR ou +ĠDem o +- awesome +ĠCh ain +Ġh asta +ĠB art +. KEY +Ġvend ors +nof ollow +ĠD est +_b uilder +Ġarg ues +_ answer +g oto +ĠRES ULT +ĠM ON +Ġp oder +o ons +_C ASE +Ġrep lic +Ġfin ancing +ĠD ATE +c ern +_tr ack +t ies +/ logo +ĠNE GLIGENCE +get Type +> T +b et +g irl +ĠINCIDENT AL +-s ite +.tr igger +ĠL isa +_input s +Ġrel atives +Logged In +Config ure +I K +. accept +Res ume +ĠD raft +Ġ* >( +ĠW A +ed ian +ern ess +ĠLayout Inflater +*/ čĊčĊ +oth y +Ġoblig ation +Sub scribe +Ġth umbnail +ex ist +Ġins isted +ĠU ICollectionView +ĠAng ular +Ġtable ts +ĠImp act +ãĢį ĊĊ +ah o +Ġcharacter istic +g d +Ġ= ================================================ +our t +` . +App ro +Co ordinate +Rem ember +Ġmar ine +] ==' +ĠAdmin istrator +.get Default +Ġforg ot +ĠStruct ure +V ue +ars ing +m oment +k w +_c ursor +Att ack +Ġath letic +Ġdiagn osed +Ġend e +åĪ łéϤ +H ouse +ĠP ARAM +Ġw iki +ĠO pp +Ġcons ervation +Ġs nd +_t em +sub str +ĠC ape +.s im +UT ION +an an +âĢĻ un +Ġg y +- work +Ġcomp elling +=' # +ĉs ub +Ġdirect ories +íĬ ¸ +Ġtouch es +out ines +.C ollection +s chedule +.l at +ĠDo ctrine +CA A +ĠRe fer +Ġshift s +Ġlik elihood +pre ter +ĠF emale +Ġinter cept +Ġl ou +çĻ » +Ġr ug +ĠC rown +Ġ************************************************************************ **** +- product +Ġprompt ed +ung le +d ocker +ĠT u +ĠUn ique +_ Error +ul os +Ġâ Ħ +Ġ( ` +Get ting +_s cal +ĠEn h +ü t +Ġsust ained +Ġp atches +Ġpros per +ĠG aza +_l ight +Ġin cons +-------- Ċ +ĉĉ ĠĠĠĠĠĠ +S F +C N +: ";Ċ +ĠColl ins +( *) +Ġcomp ilation +'] čĊ +Ġcon sequence +, ... +Ġd m +ĠB LOCK +Cl uster +Ġsk i +(arg c +T uple +Ġjo ins +ĠSher iff +W ar +ind i +Ġcomment ed +H OST +Ġinv itation +apan ese +Ġperm its +preced ented +_z one +ĠA my +_R D +Min imum +Ġinv ocation +.en able +icht en +- owned +" id +_PO INTER +F ac +Ġspecific ations +Ġnom ination +Ġg p +< ( +Ġrob ots +ĠJ erry +Ġhold ers +Ġw and +c ms +Ġ} ))Ċ +.To ast +ĠI List +B ased +z oom +/ style +ĠBe ck +M en +Ġcontrib uting +Ġund o +ĠO H +Ġadd Object +Ġe igen +sign up +éĶ Ļ +Ġdist ant +PAR ATOR +ĠM ari +Ġm á +E mp +ó s +Ġì Īĺ +ev t ++ j +p ark +ĠSt ay +ĠD un +Ġso y +> % +az ines +Ġti empo +(m e +p resent +.Th is +Ġedit ors +F IELD +.W ork +ĠUn iverse +Ġdr unk +.t imer +Ġalter ed +ĠN ar +ëł ¥ +.Act ive +id or +ç Ń +.delta Time +Ġawk ward +& quot +ĠSaf ari +Ġtr icks +MENT S +div ision +Ġvary ing +ĠHigh way +Ġphotograph er +ĠSt ewart +Ġlast ing +.P re +.amazon aws +ĠL uck +.D escription +ĠN az +n eg +Ġc ó +<<" \ +ĠSur v +ĠU nc +Rec ipe +.Border Style +Ġmod ifications +- at +AT FORM +h dr +ak o +Ġsublic ense +ĠJ ump +Ġbe im +ĠMan hattan +. bool +_h w +ÑĤ ÑĮ +B in +Ġg ateway +" ": +ĠU IS +:" + +- def +ĠReg ular +/ testing +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +string stream +Ġdis par +Ġmob il +- read +ĠAd apter +ĠCh ampions +Ġsched uler +Ġk ills +ĠM ultiple +ir ror +Ġgod s +AD O +ak te +ĠUs uario +.c ircular +Ġre cept +ĠEx pr +Ġelder ly +Ġnic ely +Ġbest e +W ant +Ġclass ical +.s prite +obj c +ĠM ason +Ġsist ema +.Bl ack +es o +ĠZe it +Ġdiv id +Ġent ers +_sub ject +ĠPlan et +.w arning +ĠG ram +_t okens +Ġhousehold s +_c ustomer +user Name +c ross +Ġp ione +Ġass ists +_S M +ib o +Ġlo yal +Ġuse less +# elif +ĠUlt imate +C ome +g el +Ġd ich +xy z +ik el +ob ra +_s can +ĠInter ior +ĠN ice +Ġpl ac +ĉt arget +Ġvir al +ass o +() / +und e +ĠAd obe +O s +vis ited +ĠO W +ĠFe ed +ĠSe quence +Ġman ages +in son +ĠLouis iana +{ }) +ĠH ab +ĠL D +Ġb ip +pr ites +(e lem +.h ibernate +él é +Ġoh ne +_trans action +Ġann unci +P ublished +ĠH onda +ĠT am +ĠP acket +_ selector +Ġchalleng ed +Process ing +-h over +Ġtr ainer +_c ancel +ĠNS Dictionary +ab ric +ĠM LS +_s ensor +Ġshr ink +ĠF X +th reshold +ĉH X +-m ark +` .` +S cheme +(f ull +_w riter +ĠS ys +Ġf led +ĠC in +-w idget +ĠPre vious +G ender +_ question +Fe ed +Ġscr ut +(p refix +ãĢĤ ãĢĤ +Ġin fections +Part s +Ġhier archy +_DE LETE +ĠPat ient +_p ay +Ġprom oted +Ġì ĭ +Ġcivil ian +Ġagricult ure +ĠP iece +Ġst ance +uts che +Ass ign +.A CTION +F ig +_r adius +ĠS ync +du cer +f ailure +ens ed +pt ime +B M +_dat etime +qu ivo +QUE UE +èĢ ħ +Ap pear +Ġsum mit +: void +Ġv ine +è® ¤ +on ne +_TR ANS +.g reen +_ cc +Ġhung ry +Ġ" > +() );čĊčĊ +Ex tract +iz ens +Ġsol ver +Not ify +Ġeng lish +ĠSh opping +inter faces +RE Q +Ġil leg +ĠUI ImageView +Ġdis connect +ĠUnt il +ĠConserv ative +@ Column +Ġshift ed +Ġ: čĊ +Ġf ich +Ġd la +Ġsh oe +"), čĊ +ular ity +_RE SP +We ather +UI Application +. iterator +Ġag ing +.P arent +ow ie +(e qual +ĠCon v +/ default +Ġmeas uring +.pre v +.Is Valid +.F at +Ġs Äĥ +key words +with out +Ġso vere +Ġex changes +Ġm elt +Ġis lands +ĠInt egr +Ġjump ing +Ġg le +Ġjournal ism +Ġd ated +Local ized +ĠRef resh +Part icle +Ġa a +ĠSTR ICT +Ġb od +.Pro cess +_A UTO +ĠP ublished +e very +Ġtechn ological +ls x +Ġir rit +Add itional +Ġdel imiter +_l anguage +- area +bo ys +ĠT ube +Ġw at +Ġmechan ics +_ owner +Sp ell +ĠSt ories +.Append Line +Table View +h em +st ick +oll ower +I FF +ĠU V +oll ision +S UB +Ġcompar able +Ġdon de +s ales +ll vm +Ġ} ],Ċ +OTT OM +ĠPur pose +L ab +Ġinterview ed +o is +as il +.set Id +ĠIn struction +-- > +ĠMod ified +ation ally +ĠMe eting +è¯ ¯ +# region +Ġrout ing +.f ocus +ĠYou th +< D +ĠN ag +contact s +Ġform ing +Ġm ie +',[' ../ +ĠB P +Ġapp et +ĠTe acher +ĠT P +Ġann ually +outed EventArgs +ĠSpe aker +Ġre name +CF G +(" // +æİ ¥ +/p ages +Ġpr és +ĠSp ell +.All ow +ĠINT ERRU +Ġ( # +âĢĻ ĊĊ +_G eneric +.im show +_t im +- face +(& ( +atin um +Ġrevolution ary +ĠH ours +r ain +Ġany time +Ġab b +.j sp +Scroll View +ĠTr uth +Ġanticip ated +Ġacc ent +. checked +Ġspec ifies +Ġca f +Ġcell padding +Ġcook ed +ĠH ugh +pe ek +_R ATE +Ġd orm +/ čĊ +IV ITY +.Cont roller +(p art +.con straint +Ġinv asion +MO VE +Ġgl uc +l ename +Ġam en +eng lish +ĠSw itzerland +";ĊĊ Ċ +pe st +.col lect +N ib +ĠD ict +ĠE mb +(sub ject +Ġoutr age +Ġdec iding +Ġsent enced +F echa +" A +Ġqu er +Ġfont Family +Ġqu adr +- Y +_C ACHE +Ġanaly zed +Ġg aining +ĠAgain st +ĠSou l +ta u +Ġlight weight +ĠT F +ĠEffect s +.T ypes +.add Class +Ġv egan +é ģ +.' " +ĠExpl orer +.d etect +.sh ift +Ġoblig ations +last Name +Ġassoci ations +ĠTime Span +un ter +ĠF resh +Compat ible +P ub +id ges +. option +var i +.hash Code +Ġg eb +. section +- not +ĠSub mit +T N +reg istry +_m edia +Ġn aj +ff t +Ġm ate +-th ird +Ġp ockets +est a +Ġb ent +ĠN ord +Ġretail ers +ĠMor ris +."" "ĊĊ +W rong +Ġ ÅĽ +R ay +. ec +ĠB ind +_H AND +(n on +is Valid +Ġsimilar ly +_L IMIT +Ġdynam ics +Ġdist inction +ãģ Ĩ +< N +Ġor th +ĠToy ota +ĠK ate +ĠL S +or ie +ĠSpr ings +Ġf reak +last name +_M ULT +-st ep +" ( +AD DR +Ġentert aining +_CON F +Ġdec oded +Ġst reak +Ġwait ed +Ġnot ified +rodu ced +vis ual +.Layout Params +æ ° +es ian +f its +s pring +ĠBern ie +User Defaults +Ġped est +Ap pearance +ĠW iki +ĠNOT ICE +Ġs sh +Ġdur ante +ĠZ ip +ı r +ĠNAT O +Ġtw elve +Ġro yal +ï ¸ +Ġmer chant +ĠF urniture +'] ),Ċ +, X +Ġfold ers +ĠG ate +ĉf unc +p ick +_us uario +ĠV erm +ment ion +ur pose +Ġalert s +x ious +_s ig +ĠF u +Ġ( : +Ġd umb +åħ ³ +Ġaccur ately +éĩ į +R B +-s creen +ĠV ER +j our +Ġrom ance +uc ceed +. choice +Ġad ip +_d ims +Serial izable +ãĤ ĭ +.j ob +Ġpro g +uch ar +Ġg ently +ĠR SS +ict ured +_ENABLE D +ĉ label +aw ks +ĠEn sure +rem ember +ìł ķ +Ġtrans mit +{{ $ +.Trans action +ur se +_rel ative +Ġs ized +ĠX X +ĠPr incess +ĠL arry +Ġpr ó +ĠÑģÑĤ ÑĢ +Ġs isters +estr uct +Ġcheck point +: length +ĠCar los +/ icon +_T ARGET +T okens +Ġpat ience +ĠSe lected +q ty +.show Message +Ġwild life +ĠP rops +b m +- arrow +Ġpar cel +fire base +ĠBen jamin +cess o +.t im +ĠG arc +. any +ĠHOW EVER +ĠK o +Ġgrab bed +_f rames +Ġobject AtIndex +ĠADV ISED +Ġsub ur +ĉ GL +Ġ}) }Ċ +-l ength +ìĭ ľ +ĠPot ter +_b uff +.g ui +ĠEnc oding +E lect +-m essage +Ġ � +Ġ ÈĻi +ĠArgument NullException +а ÑĨи +Ġmin imize +Ġrespond ing +$_ [' +ĠInd ividual +á c +ĠIN TER +Ġmast urb +ĠB in +(' $ +ëĵ ľ +Ġopen ly +Ġ> < +Ġun to +olog ically +ĠM ul +VID IA +Ġsl im +ĠCommission er +( on +Ġunder neath +/ db +v ote +( Message +ĠP ope +Def ined +Ġsw ift +ur f +Ġadapt ed +SE L +Ġreven ues +Ġdiv ine += y +Grad ient +_ act +Ġ/*! < +Ġpoly gon +ĠF DA +ĠC arr +at ables +(std out +Ġrefr iger +Ġco ordin +avor ites +ÑĪ Ð¸ +Ġcompass ion +ĠPOSS IBILITY +- secondary +ur acy +Ġcomp romise +_A V +_ os +Ġbes ide +ĥ Ŀ +Ġl n +.pl ugins +Cap acity +al ah +.b in +ĠC RC +_b alance +Ġflex Direction +Ġam bit +Ġnick name +ĠFor ces +C LE +ĠSh ell +Ġs ail +ĠW riter +ĠA lice +d w +ĠInd ians +ĠMar shall +_S RC +Ġnormal ized +ĠJ ag +ãĤ Ĵ +ze it +r pc +ÃŃ c +.in line +Ġtrav ers +_n umeric +Ġutil ities +Ġev ac +IN PUT +ĉ register +M X +ĠCamp bell +Ġdatas ets +Ġdem anded +Ġinitial State +g an +Ġe i +Un expected +- web +tr ait +, Y +ĠT odd +Ġske leton +Ġoptim ize +ç¬ ¬ +ĠU pon +ĠSt Object +Ġap lic +.' P +v ron +. UN +Ġpaint er +izar re +Ġl av +Ġp om +p reg += function +( serial +ific a +um ing +åľ ° +ãģ Ĥ +- op +U CH +ĠH end +.prop Types +Ġy o +Ġrout ines +Ġcar ing +S em +Ġres erves +Ġprior ities +red its +IST R +Content Type +ĠSch w +/ media +Ġe str +Ġclim bing +- week +cher che +s ensor +To Array +ĠMont real +Ġcloud s +ĠInject able +ĠR ice +Ġpropag anda +_pro vider +Ġind oor +Ġin aug +Ġdipl om +Ġmess aging +_m ut +å ¦Ĥ +Ġk w +ON S +ari ans +R PC +) ]čĊ +-r ay +ĠS or +m all +Ġmarket place +Ġv tk +M a +og an +ig i +Ġspons ored +ĠD ani +.S EVER +>' .$ +m ultipart +ĠW ol +Ġtable Name +ĠUser name +Background Color +Ġf right +_E MAIL +Sept ember +_val s +op ia +Ġsp otted +- Ch +Ġdata Source +/ "Ċ +ек ÑĤ +ĠRequest Method +ĠRe place +-d o +ah n +ĠPh D +] .ĊĊ +N ON +g ement +ĠTh r +Ġquiet ly +Ġtort ure +Ġte as +ĠC Y +Ġa tr +develop ment +-d etail +Ġlight er +Ġarg uing +Ġdes erves +Ġcur riculum +_CON TEXT +ÅĤ y +H ITE +ĉ ID +/ uploads +Ġt its +re o +_d rop +. UTF +Ġpick up +Ġgro cery +ĠP ure +Ġeas iest +Ph il +.f eature +(" * +Ġinvest or +t ok +Ġj ar +L os +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +. queue +-s peed +M al +um blr +ĠCON ST +ĠH RESULT +ĠD ance +(file Path +Ġattrib uted +ॠį +ĠB und +co ins +Ġs ão +Ġp ir +person al +Ġpre lim +Ġprop ose +ĠT L +] ]) +ĠSub scription +ĠK re +, len +.First OrDefault +) -- +_product s +.Get Bytes +Sh ip +Ġenc rypt +ĠS G +ĠM yst +h ir +Ġiter ate +Ġint end +.mock ito +Ġch apters +( angle +ĠV lad +è® ¾ +' .ĊĊ +Response Body +ĠAb d +de al +Ġbar riers +-out line +b ill +ĠF alls +_se cond +. include +. ceil +Ġoccup ation +ph ony +.move To +ĠJenn ifer +AST ER +; ">< +ĠEn abled +Ġtermin ate +ĠI o +l ations +ĠTHE ORY +Ġear liest +Ġr ack +ĠSc ar +sh ake +ch ip +Ġu v +Ġall iance +п иÑģ +ĠGOOD S +z ione +ĠV I +Ġ{ - +Ġfilter ing +Ġmis con +.Dock Style +Ġb ush +Ġj unk +æ Į +ĠQ UE +Ġhook s +Ġfirm ware +Ġmiddle ware +d ic +ĠOak land +Ġarr ives +P ayload +p ixel +] | +Ġstart Date +.P RO +_a udio +Ġmid field +igid body +ĠSw iss +ĠCl ip +ĠD ump +ĠText Box +Ġg eh +y ield +od s +Ġrefer endum +Back end +ĠC ream +Ġdomin ated +ĠArch ive +Ġrid ers +.prepare Statement +Ġqu ando +Ġche f +w iki +in el +am pling +(" \\ +Ġs ag +_pro xy +ãģ ķ +p do +.getElementsBy TagName +Ġdemonstr ation +ĠN PC +Ġarch ivo +end ance +Ġefficient ly +( actual +.t ableView +Ġm ush +Ġbe ars +_thread s +j as +ah un +Ġne ural +Ġdesign ing +ĠG DP +Ġlift ed +çĽ ® +ĠJ oint +ĠIn clude +ĠGi ants +Ġwithdraw al +ĠR ent +n ative +ĠSe ek +gress ion +_C PU +\ S +ĠSh ield +Ġsol ic +Ġbo om +yect o +Ġmanufact ure +ĠâĢ ĭ +Ġb box +Ġearth qu +ollect ors +:@" % +Ġlo ops +J e +alk ing +ĠWh ats +ĠBo ys +. book +ARG E +_p ixel +Ġsus pects +Î ¹ +us p +ĠBM W +ie ces +(p erson +å¼ Ģ +é » +ĠPod cast +Ġb ou +( Item +à » +( Input +Http Get +Ġb urg +) ^ +BO ARD +*/ , +Ġg ulp +ĠB enn +Ġdeck s +.status Code +Ġac ute +Ġh ug +ug u +Ġp led +," % +h ape +Ġз ап +ĠMain e +.re al +Ġd alam +ĠMin or +.F loat +dis p +Ġt l +Ġen count +=> $ +Ġf g +te es +ĠRec omm +ä l +Ġchem istry +Block s +O ID +Ġfore x +ĠApp end +Ġ{ * +ĠSup ply +CG Float +(b l +Ġat e +ador a +Ġg ust +Ass oci +> .Ċ +F ETCH +.s erial +widget s +ard less +ie fs +_F ULL +ernet es +ĠP red +Ø Ń +äº ĭ +ub ernetes +ĠL aura +Ġl abeled +High light +Ġanno ying +/ update +(d escription +Ġintim id +$ c +")) )Ċ +.A P +Ġ[] * +ĠEX IT +.H ost +ĠOP EN +.send Message +_c amera +_t ile +Ġth erm +onom ous +Ġdis adv +Ġna ar +index Of +ĠP P +.prot ocol +AF E +Ġtext ures +################################ ################ +umb ai +.st ats +ĠG E +Ġi e +ĠST D +ĠM ann +.ref lect +K B +Ġd ive +.w av +/* ---------------------------------------------------------------- +/ settings +.l ifecycle +Ġda ughters +or us +ub er +N ING +st ri +ĠT ip +Ġz n +Ġswitch ed +in et +uff y +ĠTransport ation +( conf +fr ica +ĠX L +ĠLe ad +_per cent +< Map +Ġthr ust +or b +ik k +Ġtra uma +Access or +ĠF it +ĠString Buffer +ex pl +(s creen +Ġaud iences +ĠO PTION +_ round +[ node +be h +-> __ +per missions +ĠD etermine +.M an +Ġadv ances +. InputStream +Ġstrong est +Ġe Bay +Ġ# - +Ġdir name +ĠS MS +Ġmedic ations +Ġam ended +Ġchurch es +ĠImper ial +$ row +ĠMad ison +ĠIn sp +Ġaff air +Ġpsych ology +v h +Ġsever ity +âĢ IJ +Ġstri ps +A H +vert ising +Ġcon se +IM AGE +ĠSt ats +ĉs c +.C ursor +Ġfree ze +ss on +(x ml +ĠSus an +.t ile +ed ed +ĠĠĠĠ ĉĉĉ +uel le +ĠMitch ell +b ased +Oper and +½ æķ° +ĠF F +ĉstr cpy +ounc es +ild o +.execute Query +Ġapproach ing +ĠSe ven +Ġn uts +Ġr ic +ass ignment +Ġcalcul ator +ĠMur phy +ĠB ou +í Ħ +Ġbut t +Ġt icks +Project s +il ib +.text Color +m ov +_log o +( template +ĠIN IT +Ġimage View +scri ptions +OR ITY +Con sumer +Ġun precedented +Ġtour ist +Ġbr on +Ġcontract or +Ġlic ence +ĠN am +æ ¯ +( transform +_AT T +P ref +ĠG am +Ġvess els +Ġh av +L ater +.To Lower +Ġurl s +Ġbreak down +Ġpen alties +Ġf oster +ĠU E +Ġcl ue +com ed +åIJį ç§° +-m ain +Ġp ts +Ġcount ed +ict s +/ post +Ġget attr +Ġp ing +ANCE L +Ġp ec +Ñħ од +ant om +ĠBlue print +ĠEvent Emitter +Ġl ä +æ ² +Ġstr aw +( comp +' une +> N +- client +es Module +-b ase +Ġret reat +_s imple +ĉĉĉĉĉĉ Ġ +fe e +') čĊčĊ +Control Item +Ġsubscri bers +ple ase +ĠE ff +Ġp ound +ĠBy tes +ĠTe a +_ activity +Ġmax im +Ġop code +B SD +. constant +; } +omb res +Ġcare ers +) .ĊĊĊĊ +Ġsp reading +-exp anded +ĠOr d +amar in +Ġmob ility +Un fortunately +ak k +N L +_ redirect +ĠP G +ĠS ensor +b ol +t ap +_MEM ORY +ĠUI Alert +plit ude +We bsite +ĠLog o +lo ve +[ ind +Ġalto gether +Ġwonder ed +Ġes per +ĠLib eral +Ġo ss +Ġel it +Ġst iff +od ox +_ment ions +ĠDou glas +_p id +ĠC K +ĠinitWith Frame +.b log +p kg +ang hai +QUI RED +u u +Ġm kdir +AT AL +Ġun h +in ces +st h +Ġhypo thesis +Ġc ata +ĠT B +ĠCl ar +Ġpre decess +Ġsitu ated +-w orld +)) / +Ġhead lines +.st at +Ġout break +sp ath +_FLAG S +ĠServlet Exception +S un +F ROM +ĠD ir +ãĥ»ãĥ» ãĥ» +_co ord +ĠOpt im +Mon itor +.b it +XX X +Ġtod as +f eld +ÑĢ Ð¸ +im ir +Ġpolit ically +Ġmolec ular +Ġtrad ed +Ġ{{ $ +ĠSw edish +Ġ'@ / +_RE AL +Ġw arehouse +t oday +, L +or p +< section +- br +ym e +ĠUser Service +Ġlib erty +Ġmoment o +( Image +< size +S ch +Ġj og +i ology +arent ly +Ġquant um +ĠAb u +Ġr im +Ġman a +Font Size +Build ing +st airs +AIL ABLE +Ġ& ' +Ġs ect +Ġs igh +(b atch +.I Container +p oll +ĠCor ps +Î µ +ar u +ĠK ay +.r ange +_click ed +ĠRobert s +.N etwork +fin ish +- Man +Ġcolleg es +ĠF ine +")) ,Ċ +f ilm +Ġrem inded +Ġgest ure +out il +Ġthread ing +Ġobj et +Ġt ours +activ ated +.m kdir += user +Ġre de +f ü +_SY STEM +p v +Ġcon gr +Ġmass asje +Ġpract ition +Un iversity +Ġtab index +Ð ĺ +S ets +Ġcount ies +g uest +f an +Ġword en +.d i +на Ñĩ + ¿ +ig Decimal +Ġsh ore +Ġg ö +Ġrep airs +Ġhelp ers +Ġcenter ed +OL LOW +Ġmap StateToProps +Ġc ents +< A +Ġexpect ation +Oct ober +Ġbg color +ca les +.C ON +ĠV el +Ġcry ing +-se ason +Ġfunction ing +_LOC ATION +ü ss +ber y +Par a +omin ator +- le +Ġeth ical +has htags +emp lo +Ġn úmero +( activity +.St op +.str ftime +IL D +Ġto e +ĉ Node +") čĊčĊ +ĠPu erto +Ġexec uting +ĠG UID +Ġoppos ing +al ph +Ġexhib it +_fl ash +Ġme ille +Ġjson Object +H ero +aint ed +_D OM +Ġw il +Ġslo pe +Ġm Ã¥ +ĠIraq i +Ġorgan ize +ĉj Query +H UD +sh ine +. we +ĠSk ills +pons or +Ġcon clusions +Ġre forms +Ġrel uct +n amed +ĠOl iver +Ġ// }Ċ +- looking +Ġf og +ĠH O +ĠF ried +Ġinev itable +ĠData GridView +H our +il les +log ical +Ġconnect ivity +.tw ig +ĠK yle +(d st +- Sh +ĠStud ios +( Level +.j et +_PRO TO +-de coration +OT HER +Ġread ily +.Param eter +Ġmultip ly +ĠL IB +ar med +Ġsoon er +æ Ħ +_ ES +Ġfoss il +ĠA nc +âĢľ This +l odash +Py thon +Ġhist ogram +west ern +Ġinf ant +Ġco ordinator +Ġn ib +: m +Ġres pected +Ġdef init +& T +_p ad +ĠTr igger +th al +Ġimage Named +Ġbeat en +ĉ rc +ĠPal ace +Ġhaz ard +Ġisol ation +_ rc +cont re +OUT PUT +Ġre ign +ĠPl ate +AT ES +Ġfl ux +Ġpack s +.get Selected +Ġparticip ated +Ġneed le +-de pth +:::: :: +-l aw +ins pace +on itor += no +ĠAt omic +ĠBr ain +Edit able +-s c +red ential +ĠP erry +k ie +Ġ ----------Ċ +.st roke +( Intent +Ġun ity +um lah +F urther +Ġpr ze +Ġs ø +ãĤ Ĭ +ĠPROC UREMENT +ĠH ousing +Ġatt orneys +Ġcomp ose +atter ing +" What +dra ul +Ġstraight forward +In stant +.J TextField +Ġtr ades +л а +Ġ{ ! +Ġl ately +IM G +ĠA ld +ĠIN NER +Ġcart oon +.S ource +F ALSE +Ġd ough +f en +( rect +Data Table +N ick +ĠBut ter +read s +_com ments +EN V +ĠConnect icut +-F IRST +ĉĉĉ ĠĠĠĠĠ +ach i +.M sg +re ction +Ġrelax ed +Ġsha ft +Ġe f +ĠAdd ing +Ġbre ach +Ġ ï¼ļ +ram a +Ġconduct ing +Ġ( ; +(g l +ĠCA USED +ash i +ĠF LAG +ĠCom merce +ĠIN TEGER +h ours +ĠSchool s +Ġn ucle +Ag ain +pro j +Ġsevent h +EMPL ARY +(m ock +'] ,čĊ +_S PEED +> false +Ġsp a +ĠN ear +ì ķ +Ġintr ig +_m embers +w ave +Ġanalyst s +_O S +ed in +ĠF ri +Ġretrie ved +Reg ular +_ obs +EX PORT +')}} " +" class +__ (( +b ucket +Ġst ro +ĠP atch +yst ick +ful ness +ap os +D a +ĉĉĉĉĉ ĠĠĠ +Ġen rich +un ordered +h ole +C ong +< Product +ĠC urt +( the +_l ower +Ġavoid ing +Ġbu zz +Ġv iable +ub a +- is +are l +Ġact ed +-d etails +ภĩ +ĠThe ory +ĠP un +ĠAn onymous +... "Ċ +è res +åı ¯ +ĠV ision +_se m +ash a +Ġcelebr ity +Ġend Date +Ġpop ulate +Ġcu is +qu ant +f loor +Ġglob ally +Ġcru ise +ĠStan ley +Ġb ikes +.get Connection +Ġpoor ly +_ other +amp ing +." );ĊĊ +od i +_A DMIN +.color s +ĠG aming +> ';ĊĊ +STR UCT +Q R +ID s +(arg uments +_a ux +( Event +_PR IVATE +ĠTre k +Ġdownload s +m utable +_STR UCT +(w x +Ġdom ains +js px +ĠVi agra +Command s +J s +.c fg +Content Pane +ĠEdit Text +à¥į ठ+Att ach +ĠAR M +posit ive +ĠGener ated +Ġse ized += : +Ġelectron ics +ĠApp Component +/ ',Ċ +.equals IgnoreCase +Do ctrine +d isk +ĠPolit ical +CH O +< F +ĉ height +ĠB ug +. le +ik h +Ġmill iseconds +Ġconstit u +m ag +.n l +-r ange +ang gal +', [ +ropol itan +Ġà ľ +ĠU C +.d esc +-L AST +f stream +ib il +Ġf ier +VER Y +Ġë ³ +IR T +_ UI +( abs +Ġkne es +Ġro okie +ĠV ac +are na +comm end +- \ +ĠSUB STITUTE +So ft +Ġpart ir +we alth +è¦ ģ +(d ataset +ĠCl imate +- show +Ġreli ability +_ch unk +ä» £ +_st ock +ĠEX EMPLARY +ï¸ ı +Ġv ÃŃ +Ġsm iled +Ġdr ill +.F unction +ĠS I +Ġreg ression +- X +ĠJ ar +p ref +ĉs uccess +ĠHit ler +Ġinst inct +Ġfem mes +Ġlo ver +< Ċ +Ġmulti plier +r il +Res ize +ĠAuthor ization +ĠK an +Dispatch ToProps +Ġc rops +t okens +ec n +ential ly +ĠINTERRU PTION +f ake +Und efined +ĠA K +ĠTest Case +Ġr ab +Ġtor rent +ĠO t +B ars +Ġlect ure +Ġen jo +Ġrespond s +Ġindex ed +Of Work +_ch ain +)) -> +ĠBeaut y +Ġ` < +Ġtouch ing +Ġ| -- +ĉf lag +normal ize +Ġtr apped +Ġestablish ing +/b uild +A J +f y +- react +av n +RI PTION +Ġk ut +ĠF ashion +ĠIn form +cur ities +< byte +ĠUkr ain +Ġs ug +Ġconsist ing +ood le +. ctx +.To List +Ġcomment ary +Ġtransf ers +Ġn ost +ih ad +ĠU pper +Ġconf using +miss ing +- cl +Ġbound ing +Ġcongress ional +Ġreve aling +d h +r up +Ġt res +re peat +, ĊĊĊĊ +_t ac +Ġexp ed +G irl +h orizontal +Ġ"../../ ../ +( option +Ġwe iter +ĉs ql +Ġ=> {Ċ +Ġgar lic +Ġre pr +Ġrepl ies +( prop +Ġspir its +Ġins pire +Ġbas ement +.re ject +Ġhint s +Ġpoll ing +ĉ ĠĊ +_r ating +Ġc ath +av ier +Ġcomp ressed +ĠV S +] ' +Ġjud icial +ĠT rend +tr aining +EST AMP +ogn ition +Ä ģ +SE NT +vent ions +Ġconsult ant +um ph +Ġuser Service +, NULL +k h +D ear +_B AD +it ations +Ġmet aph +' é +and ise +-f ont +.ch art +Ġs g +_ Controller +.j peg +ĠUL ONG +ĉg ame +( ss +ĠM aj +ĉg o +ĠS ad +ĠB erg +ĠM ine +P ack +Ġres istant +ĠR OM +Ġp eg +ĠStan ford +ĠY ahoo +Ġsca led +Ġl an += [] +"/ > ččĊ +Ġs ud +ĉ background +Ġsch olars +-m uted +ar á +Ġ= ==== +Ġ__ __ +C reat +ene ver +/w p +ĠV PN +Error Code +) ],Ċ +(b uilder +ĠEn emy +S ensor +us a +Ġtr iggers +Ġplayoff s +_RE Q +Ġ( ~ +ĠBar ry +Ġperman ently +ĠR UN +Ġb ure +.Fat alf +Ġch ick +ĉ panic +ps i +ok a +éĢ ī +> [ +Ġunderstand s +ĠJun ior +ĠIN FO += mysqli +ust ain +-s ource +s erv +ĠC REATE +. au +Ġsell s +ĠĠĊ ĠĠĊ +E urope +z w +pre h +ĠNS A +Ġx y +ภ´ +ĠB eyond +Inst ead +Non Query +Ġar ise +Ġavoid ed +.em place +_model s +} ),Ċ +Ġh id +Ġ& _ +.p oints +.get Width +.Ex ec +Ġ// // +ĠS essions +... \ +ĠCol omb +Ġacceler ation +rest ore +Ġ ile +ob ic +< Node +ĠD X +ĠBes ides +. age +ĠCont ains +N ational +ĠIm plementation +Ġeff ic +ĠR M +H y +ĠWed ding +ok ies +Ġrec ursive +Ġprosec utors +.Se lection +ĠForm ula +Been Called +[i i +ĠFr an +Ġtraged y +_F EATURE +Ļ ¨ +comp ass +ĠB h +? ĊĊĊ +.w riter +ĠH our +Db Context +io v +am on +re pr +é ĥ +ĉf i +'] ] +ĠD ry +. ro +ĠO bserv +æł ĩ +Form er +ĠB alance +ĉ json +Ġpr zy +I SS +( sock +ĠL INE +Ġde ce +Ġal ly +Ġtend ency +F un +Ġschem es +Ġinter ven +æĺ İ +Ġad verse +quote lev +Ġsacr ific +_s ide +Ġmut ex +AG IC +Ġocc urring +ĠCommunic ation +um ar +ç¼ ĸ +ĠTreat ment +.p erson +ĠL C +Ġe ch +( (" +ĠDise ase +ä d +ĠA Z +.A ccount +Ġcontinu ously +END ING +ĠRET URN +- string +.f ilename +syn thesize +Res ponder +( opts +reg s +Ġn uest +Pe er +// ------------------------------------------------ +Ġg auge +ĠK in +.s chema +Ġarr ange +ĠBl ake +_Type Info +C over +ĠHamp shire +P aper +-in ner +util ity +Ġcross origin +F OR +Ġign oring +ĠD D +av an +Ġtrad itions +Ġget String +Ġeth ics +ĠMaterial s +DE SC +Ġen zym +io let +ĠCh ip +ĠMc Donald +Ġn erve +ç Ħ +") ] +æ± Ĥ +ĠS ugar +_S IM +j peg +Ġdiscret ion +ĠT N +bo ve +ĠMin imum +ĠForm Group +Ġwork force +ĠExec ution +err er +ĉ ĠĠĠĠĉ +Ġpres cribed +.Text Align +OP EN +ĠP B +im ity +ĠEx ternal +° C +ĠApplication Controller +Ġb arr +imp licit +_d ot +ĠCol on +C OLOR +.Pro ject +* }Ċ +pl aint +get Text +Ġindivid ually +Ġcheck box +U Y +ĠL amb +Ġdys function +ĠL ar +à ° +ĠCre ating +');ĊĊ Ċ +" They +loc ations +_C ORE +Inter action +umbn ails +ĠPart ner +b rit +Ġless er +ĠSl ot +set Attribute +ĠW ave +.p o +/ store +Ġbrows ing +_p d +sum e +s ed +Cur ve +Ġpl asma +Ġsusp icious +ìĿ ¸ +ĠB ah +ĠExp licit +_C C +.Client Size +\ View +Ġsub stit +lo on +ĠG AME +ĠB rid +Ľ 建 +_ User +Ġsqu ares +f one +Ġsac red +ug hs +] interface +ĠTh row +ĠK irk +Ġemp ire +Ġassess ed +T ax +ĠHe aven +-b uffer +_STAT IC +én é +-b ordered +Ġpun ct +(m ode +Ġke ine +S ent +ĠCal cul +ĠE ve +Ġsty lish +Ġoil s +.Test Case +Ġtrad emark +Ġliter ary +Ġconcentr ations +ĠRel ations +( Class +Ġstd in +Ġv æ +back up +. VERSION +.AutoScale Dimensions +st arter +Transaction al +- panel +St udio +k c +ĠCh amber +ĠSpi el +Ġr ho +ا ÙĦ +! ' +.At tributes +Ġmurder ed +apeut ic +Ġint imate +Ġtext Field +ĠBuff alo +d ummy +" % +ĠLib erty +ob ar +ĠT ank +ĠPop ular +erv isor +ĠIn iti +ĠM all +ĠP rior +C AP +ĠCl ay +ĠCert ificate +.L ock +-st rip +-dr iven +/ all +ĠMessageBox Buttons +_SE CRET +_p b +Ġr ats +ा ठ+Ġn t +.R outer +_top ic +Ġt ennis +ĠP UBLIC +ĠActiv atedRoute +Ġ' ,Ċ +Ġcost ume +Ġj okes +. Handle +ĉ byte +Ġflav ors +( cc +Ġperson as +ĉ image +ĠN azi +Ġgram mar +Ġú lt +Ġval ve +Ġv ic +ĠR achel +_in valid +P refs +std int +(r oute +Ġhtml specialchars +Ġpe oples +pl ine +Ġn v +ĠQu ant +opp ers +Ġcurrent User +ĠC atal +Ġrecon c +Ġconj unction +l x +amb urg +Ġinflu ential +d anger +ind ers +Ġ% @", +.config uration +os ome +. identity +Ġpick er +n ost +ĠDI Y +Aug ust +ab lo +Le af +ĠRec o +ck o +DO C +ĠH erm +: any +ĠInt erview +ĠT ex +x fe +( work +Ġle ap +He ading +Ġqu arters +\ Bundle +re b +Per haps +ĠG mbH +B irth +ĉ sum +ĠWat son +.n il +ç ¡ +{ }ĊĊ +ica id +Get ter +" name +Ġ" čĊ +_n one +z m +ac ute +uest o +Ġs ous +Ġre build +Ġnewsp apers +ĠH az +Ġk its +if o +Bl ur +Ġsu ited +- In +à ¯ +ĠKe ith +ĠNor way +IN IT +ire ccion +iet ies +_us age +ĠDou g +r ise +Ġtr illion +im ited +ĠR EL +al ic +Ġcritic ized +the orem +Ġce ase +Ġsid ew +ĠT erry +Ġsubs idi +Ġfirm ly +Ġaw s +Ġh ott +Ġdress ing +bad ge +ĠApp lications +è¿ ĶåĽŀ +Ġlaugh ed +Ġh obby +Ġmus icians +Ġ* . +. placeholder +Ġcount ers +ĠCap itol +SD K +Ġhel met +and box +qu it +Ġcriminal s +Ġteen ager +( update +G l +.se lection +Ġdis charge +Ġpresent ing +ufact urer +_UN KNOWN +Ġstress ed +å ύ +Pro to +_cor rect +ha us +Ġren ov +Ġfire arms +Ġtechn ically +-b rowser +Ġc andy +St roke +Ġexec utor +Ġocc urrence +ĠIP v +_INTER FACE +ĠRetrie ve +.b ad +Ex change +Nav bar +ĠK id +(get ApplicationContext +_ST OP +ĠB oss +List eners +Ġshoot er +ĠAl b +ä ch +Ġp ix +.key Code +al one +Ġabs urd +ĠC um +ĠNewton soft +ik t +Ġlaugh ing +Ġcapital ism +ree Node +T x +_QU ERY +.S leep +( login +Web Element +Ġcelebr ating +Ġde precated +Ġma ar +Ġart istic +_ASS OC +ĠBorder Radius +ĉw p +Ġsurviv ors +In ner +- red +Ġprosec ution +_ pp +(" $ +Ġcomm a +un checked +graph ics +r ors +G ROUND +( public +Ġcustom ized +ĠArk ansas +ĠR ew +Ġexp iration +× ķ +ĠC ul +Ġn ons +.F ilter +Ġsen ator +_def inition +ash ington +ym ph +/ J +Ġf use +ram id +ĠSup plier +Ġaut ocomplete +Ġ} ), +." ĊĊĊ +_function s +ĉ to +.e val +ĠT Object +Re ferences +Ġhe ated +H AL +Ġ)) }Ċ +} $ +ĠB arr +_UN IT ++ $ +Ġget Value +ip ed +ch ied +(v m +c ue +_int eger +_c ourse +th ird +Ġrevis ed +** /Ċ +_D IRECT +Out Of +(" ( +ĠFe el +Ġre ass +Ġsub title +per i +n f +Ġenjo ys +Ġtreat s +) this +-t abs +anc ers +Ġcontin ent +Ġcard io +S er +. question +Ġph rases +Valid ators +Ġpop ul +Ġl ÃŃ +s ong +_IN TERNAL +Ġadvis er +Ġp uzz +Ġambit ious +ĠT ob +ĠD P +Ġpres idency +Ġsurre nder +Ġwatch es +_b inary +ĠSo on +Ġcan ada +(" ")Ċ +] =' +ĠBr andon +eps ilon +r w +.add Child +.C opy +Pr incipal +Ph otos +Ġmarg inal +Ġbas ics +e ing +M ust +_ String +Ġo le +M agento +.c ustomer +(p rev +ภ¥ +Ġlo yalty +C og +Ġprot ocols +ĠCom panies +Ġtheoret ical +Ġaccess ing +ĠZ en +. ones +att ice +_w orld +z es +Ġtatto o +Ġmen os +Ġinter sect +"] ;ĊĊ +bel ie +Ġin active +.read line +-label led +.d one +lick r +ĠW ORK +Ġderiv ative +Ġd atabases +âĤ Ĥ +Ġs x +.is Array +Ġy s +Ġp ada +ĠBul let +(` / +is Active +ĠCG Size +(equal To +ĠColum bus +Ġmar ry +DE V +_l imits +ron es +I AS +Ġt au +min o +_W rite +ĠW ine +Ġ[ [' +ĠP ull +rit ers +ri ents +Ġsh ifting +up p +_TIM ER +ĠCondition s +Ạ¥ +ĠOr ders +ĠSt rength +æī Ģ +Ġvalid ity +Ġf ot +et ur +Ġb olt +åĨ ħ +ĠAl ong +os hi +Ġassum ptions +Ġmag azines +_S PI +Ġp unt +_PRO DUCT +Ġrel ay +ĠJ avascript +. te +- es +Ġwidget s +(f s +< Item +_ex tra +Ġrecru iting +E t +Ġnecess ity +p w +Ġnov els +uss els +Cre ator +ĠM VP +ĠO C +th ood +cl ients +)) * +Ġcharacter ized +_SE ND +ut i +T y +.from Json +@ Service +ãĤ Ĥ +Ch ris +_ Is +ĠJohn ny +Ġclean er +ĠInitial izes +UN K +( axis +еР· +ie val +ĠWar riors +} )( +DM I +âĻ Ģ +ĠTre asury +Ġfe as +Ġsl a +_EN UM +l hs +ĠIn stit +ipp ers +Line ar +Re ading +quir ies +-c ell +ch rome +.S earch +IN A +ç±» åŀĭ +ĠĊ ĠĊ +ĠSam uel +Ġmill s +Ġdon ate +ĠGe o +( rows +Ġshe ep +Ġé l +ä½ ĵ +Ġb em +_UN USED +ĠR CC +Ġintrodu cing +att a +ĠP riority +ĠF B +ĠSer ge +> "; +atch ing +ĠKnow ledge +ĉ The +; margin +less ness +op ard +um atic +() ));čĊ +Ġf als +(c ache +Type Id +éĢ ļ +_ choice +ĠGo th +ĠS ites +M G +_b order +Ind ices +Compar er +ĠRed istribution +Ġclo set +Ġvers atile +Input s +**************** **** +Ġob esity +qu iz +gr a +(g lobal +åĬ ¡ +Ġcollect or +Ġk or +ov able +AD C +ĠEvent Handler +. nc +Ġplay back +ient os +_p erm +_W ARNING +ĠOlymp ics +.n orm +ĠBroad cast +_sm all +dr ive +. iloc +Ġtyp ed +M EM +_con s +DM ETHOD +Ġl un +.d istance +(p ar +po on +Ġb ast +activ ities +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +: čĊčĊ +S ER +) && +_l st +ĠPol ish +Ġknock ed +Ġfrustr ation +au kee +Ġph osph +iqu id +_c oeff +æŃ ¤ +L atest +ĠD ust +T ipo +Ġmaint ains +Ġmar sh +inc inn +l bl +C are +Ġneighborhood s +_g pio +ĠAr senal +D em +ĠW he +_h ook +Ġl dc +ĠHar per +ĠBer keley +Ġgrad uated +Per cent +Ġarr iving +ĠAdvent ure +(s cope +(' * +qu arter +ĠMar ie +Spe aking +_code gen +Ġimm un +c aster +ãĤ Į +åķ Ĩ +ĠDim ensions +.rec ord +Ġtext o +ĠMich elle +P ending +( by +_P AR +uch t +be e +.Th read +amp ire +k now +ĠClin ical +Ġmargin Bottom +Ġdistingu ish +.F ull +. undefined +ĠSequ elize +################################################################ ############ +Ġeduc ated +_O VER +åº ı +ĠÂł ĠÂł +_e ach +Ġur ge +de part +Ġdon ors +ĠA u +Ġbill ions +Ġbelong ing +_ age +_ Int +Ġsub stances +m achine +!! !ĊĊ +Ġjson ify +ib bean +ĠC ad +Ġend Time +Ġc ycling +ĠUIT extField +Ġle verage +Ġvan illa +e at +La unch +( pt +st ates +ĠControl s +ĠRes pons +ĠJ ake +Ġas leep +fort unate +.next Line +Size Mode +ìĿ ¼ +Testing Module +G erman +ĠInvest ig +.re verse +ĠB ACK +( DateTime +Ġnon profit +ĠEx pect +Ġt anto +'] ), +ĉ the +M ultiple +(get Activity +_W AIT +Ġj á +de cor +lev ance +ĠGit Hub +min ation +_qu antity +.Sc anner +ĠL ion +éĶĻ è¯¯ +Ġd re +Ġtan tra +Ġcontent Type +Ġf id +_ alt +NS IndexPath +- pl +åĮ ĸ +Ġantib iot +table s +ac ial +ĠReg istry +Ġol ive +ig ers +Ġsubscri ber +_p res +ĠSy ntax +Ġlo vers +. Byte +old ers +_for ward +al ways +C aption +Pr iv +ĠT ampa +is ateur +-labelled by +ĠTo String +Ġì Ĥ¬ +Ġinit iated +W F +Ġinstitution al +in ject +ĠSc r +Ġdo ctrine +Ġsp acious +is ure +ĠAn a +" time +ess aging +Ġc id +ĠN an +Ġin complete +T AG +-b uild +Dec ember +Ġres idual +(P DO +ĠList en +Ġg lyph +Ġg aps +ne a +.R ect +Ġsa u +ĠPhot ograph +Ġexec utable +ĠExp ert +Cor outine +_s izes +ĠN L +.is Valid +); }Ċ +- reg +Ġc iting +c wd +ĠOtt awa +ĠB att +Ġrenew able +Ġprelim inary +Ġas ylum +Ġw rist +Ġutil iz +Ġdet ention +F ast +Ġan ge +incinn ati +Ġste ering +ĠNa N +ios ity +/ page +Ġè ¿ +ster ol +Ġdis g +( DB +ĠDESC RIPTION +Ġ_ $ +Ġobst acle +Ġb izarre +Ġextr action +_ex pected +Ġlos es +ĠCele br +Ġhtml For +Ġexplo it +олÑĮз ов +XY Z +Ġmagn et +amp ed +Ġat oms +S ources +pect ives +Ñģ ли +Ġ= čĊ +Ġd are +ĠWal ter +Ġbright ness +Ġan notations +ë ı +is ke +S chedule +. images +ros so +Ġ" .. +g amma +Ġin structor +Ġover write +- am +Ġdevast ating +ĠSaint s +Ġh s +Ġbon uses +$ output +ij d +(Action Event +mon itor +Ġmatt ress +Jan uary +.j p +Ġcar acter +Ġim pose +_re st +ĠSign ature +Ġcoron avirus +ãģ Ĭ +_com pare +Me asure +it ated +el ijk +ig os +es ar +Ġrush ed +met ry +_SE PARATOR +_W E +_ATTR IBUTE +Ġy aml +Ġspec s +ĠR ah +ph eric +ĠInvest ment +ä ll +Ġappe aling +Ġview port +ç © +Ġmargin Left +Ġsub tract +ĠED IT +ĉ ArrayList +gr ading +ĠF ailure +as per +EE K +(n ow +< object +ĠAl ignment +ple ado +q tt +( ERROR +ĠIN VALID +Ġuser id +ra ises +ID I +Ġvari ance +ĠN il +/ delete +_M AIN +.T oken +.C ategory +> )Ċ +Coll ision +ĠGre ater +ĠR acing +al an +Ġmon etary +, new +ĠS orry +. Enable +ĠInstant iate +oll en +ë© ´ +ĠCall ing +_h our +AD A +Ġsh y +) ** +Ġ== > +Ġes pecial +Ġinterpre ted +! =" +Ġpharm acy +.s ingle +ĠC ialis +Ġpar as +.to UpperCase +ĠDem on +Pr ime +Ġrank ings +Add ing +_H ASH +ĠEx am +Ú © +ĠVict or +Ok ay +"] ;čĊ +Ġfort une +ĠF ETCH +exp and +.Inter op +Ġb arn +æ ¶Ī +ue vo +Ġspec ulation +âĶĢâĶĢ âĶĢâĶĢ +ĠN u +ĠBl ues +(f name +Ġinhab it +Ġ\" % +C ES +ular io +_c r +Ġvalid ated +Ġmid night +ank ing +Ġincorpor ate +Ġpurs uit +EX P +pr ime +P id +- US +ĠN urs +ĠW heel +é ĺ +Ġin p +Ġsupport ive +.m ember +ĠSh ot +.Check Box +Ġaff irm +T or +Full Year +Ġconsider ably +cred entials +_ opts +R oll +( round +Ġcom ent +_U ART +Ġext ending +R G +result ado +it u +.get Session +Ġattr action +& D +$ html +ĠJess ica +ĠAssoci ate +a ñ +_ ed +ĠL ag +Ġorig ins +()) -> +add EventListener +IAL OG +åIJ ¦ +.Com pare +Al bum +ĠK u +< Q +arg est +Ġpro long +Ġconfig urations +Ġaccident ally +_ph oto +Ġ'' ;čĊ +Ġver se +B ob +Ġfarm ing +del ivery +ĠM ack +Ġuse Selector +.bootstrap cdn +keep ing +en y +. upload +ĠM ETHOD +cre ator +< _ +ĠE aster +. -- +UI Button +ãĤ ī +om eters +Ġsh ine +Ġh ogy +\ s +Ġh arness +.C ell +Ġlif ting +Ġcomb ines +ĠOcc up +ex clude +pat ial +Ġres pir +_f it +Ġfif ty +ĠM ol +Ġtun ed +-d imensional +Ġq s +Ġto ps +> ";ĊĊ +quis ite +ch annels +/ res +ĠAn alytics +.app compat +/ to +Ġon Error +( attr +IR M +Ġrag az +- as +.Se cond +orient ed +Ġdon n +Ġlight ning +f id +ĠP le +ãģ¾ ãģĻ +t ro +.Tr ue +O bservable +× Ļ +umb ing +Ġpros pective +-f ilter +Ġpurs uant +(p oints +.B ind +Ġp alm +clear fix +ö s +ĠG onz +Ġwe aken +Dr ive +en ido +l ld +ob ox +ane an +G ot +ä¿ Ŀ +Reg ex +æ ĥ +Ġsal ad +ass is +" net +inherit Doc +ĠR V +qu ier +Ġcl azz +ı ÅŁ +oster one +Ġair line +.list dir +Ġdownload ing +ĠP alm +w aukee +& lt +.B L +_IN LINE +off s +<< ( +_new s +Ġch ase +/ >< +Ġeuro s +ĠEgypt ian +ĠSt ainless +_BO OL +ĠG uild +ĠD ynam +[index Path +Ġ ï +Ġmemor able +ĠCh ampion +Resource Manager +.Log in +ĠForm er +yp ed +Ġl leg +; ", +D WORD +Ġtax i +Ġbom bs +ra h +.t ags +_test s +st ones +âĢĿ ) +[ g +r type +Ġv u +Ġhost ile +Ch ars +ĠPatri ots +/ status +< B +ĠIn come +ĠD ad +Ġpat rol +_CH ANGE +Ġup graded +Ġch ina +set q +Start ed +.U ndef +Ġcheck sum +Ġfrustr ated +{ o +Ġen f +Ġwood s +ĠAny one +Enc ode +ĠQt Widgets +are as +Ġshe er +sk i +end point +_T est +S oup +~~~~~~~~ ~~~~~~~~ +(f iles +ĉĉĉĉĉ čĊ +.sp ark +Ġval ued +Ġ% Ċ +.control s +ĠXCTAssert Equal +Ġf ame +ĠR ic +D OT +ĠAlbert a +ä½ ¿ +os al +.Web Controls +Ġ ------------ +ĠM is +ĠS YS +Non null += item +Ġexp ire +Dec ode +_ operation +ĠValid ator +.C ENTER +uff s +* m +Ġav ant +æ¬ ¡ +âĢľ You +.per mission +... ) +ĠL ic +_co ords +.n ombre +c lo +.Int ernal +ĠCh o +_s w +ĉ Il +cl k +Ġcast le +(l ayer +p it +Ġgu ided +Ġâĸ Ī +Ġsuper b +Ġsup plements +_c ent +Ġpe ek +IN ARY +.Content Alignment +f alls +")) ; +W all +). čĊ +ĠD anny +irm ingham +IAL IZ +( create +" In +Service Provider +Ġpr iced +mac ro +am ac +. box +---- Ċ +ãĥ « +ĠS uit +ur st +br u +ourn als +num ero +__ ()Ċ +D as +ĠM itt +ud er +? \ +f u +[ B +Ġ: )ĊĊ +(int er +br ains +Ġatt itudes +Ver ify +Ġsign atures +ack Bar +Ġg d +J ack +.c at +Ġz z +war f +FT ER +");ĊĊ Ċ +Al ive +IC LE +ĠWh atever +Ġout lined +s prite +еР² +_A B +_DE PTH +Ġcrush ed +aa a +(e v +æľ º +Ant i +IC O +is EqualTo +.s un +ic ulo +s ale +_h ex +ĠV k +apt or +Un ion +ĠDis count +list a +.Undef Or +Ġautom ation +N or +å¯ ¹ +åı Ĥæķ° +Ġref lex +ĠLa ure +.showMessage Dialog +.t emp +Ġa kan +Ġ__ ____ +.Is True +ARE D +ag le +E nergy +Ġquant ities +âĢĻ Ã© +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġcitizens hip +m outh +Ġin appropriate +ĠOut door +White Space +An onymous +load s +webElement Properties +T en +Ġacc idents +Ġadvertis ement +ĠY emen +(c all +Ġsl avery +Ñģ п +ĠL am +_BIT S +ome ga +ĠO le +Ġkid n +_A n +ĠR aid +Cre ation +s aved +Ġpro port +W ARNING +\ P +Ġp wd +Data Reader +is cher +ade on +ĠP redict +Ġreason ing +Ġdestroy ing +H el +* d +ĠLeg isl +_P r +ĉĉĉ ĠĠĠĠĠĠĠ +Ġsymp ath +Ġch ess +Ġm am +: hover +Ġconvert s +Ġp ela +Ġprogress ion +Ġ"_ " +ĠG ill +ĉ show +Ġsupposed ly +ac curacy +el in +Ġunf olding +ĠHy per +Ġw anna +Ġup s +( # +ĠCr iminal +( Point +at Lng +act ly +Ġcontract ors +'] } +draul ic +ód igo +ĠT T +ĠW ide +ĠAR G +_ ic +FLAG S +S chool +Ġclear ing +-be ing +={ [ +, const +man ent +Over lay +(' " +éĩ ı +ĠT imestamp +Ġmail ing +ĠC ake +.Th at +Ġmed itation +q p +Ġemp resa +ĠL ions +Ġw eld +ĠLinked In +Ġc ush +Ġgen ome +.Index Of +ag ain +Ġf allback +Ġcamp ing +re dd +-strip ed +Ġd v +Fe bruary +ĠPro xy +us k +Ġdies el +W RITE +RE AK +L orem +.In voke +- div +Inter ceptor +ĠD H +ia les +Ġvill ages +Ø ´ +ĠEN V +S ys +.X R +Ġpo em +à Ĥ +c ade +pl ots +Ġ{ ( +.g it +/s vg +nc mp +ĠÄ į +ain es +åĩ ½æķ° +Ġ( )ĊĊ +ops is +ĠRel ationship +_ aut +ĠB omb +ĉ com +* sizeof +off icial +_p ayload +ĉĉĉĉĉ ĠĠ +.m anager +ĠA round +ĉs end +ĠEx ercise +ĠB illy +iv i +Ġneed ing +_url s +_t asks +ĠH em +Ġtear Down +enc rypt +.t ie +Ġas m +IC H +ĠCGRect Make +ìĦ ± +ul ong +Ġit r +ĠG ST +Ġoffer ings +ro be +EE E +oper ators +_PRO P +ind ent +A DE +or f +ë IJ +Ġbless ed +vas cular +Ġcon oc +H appy +B ridge +ilit ation +j oint +ĠAdmin istr +- transform +Ġmeant ime +/ K +ĠBed room +Ġrig id +Ġbrows ers +EM PTY +.S erialize +_ ED +Ġst itch +Ġj an +ell t +Ġbr ace +Ġtr ails +p ublished +å¯Ĩ çłģ +} ')Ċ +Ġac ids +Ġ! !! +_d irect +> ());Ċ +aj Äħ +_O CC +Ġplan ets +æ Ł¥ +ĠDub lin +Ġser ie +.print f +de ep +` ) +Ġ\ $ +ĠÎ ¼ +_V IDEO +end ors +ĠC rypto +F ar +.Trans parent +.T R +ias m +_tr aining +Ġteach es +ĠB elt +Ġlimit ing +ĠK ath +ĠIndex Path +Ġachie vements +Ġser á +interop Require +Ġdis se +.I f +arm ing +uls ion +P o +_DE TAIL +Prot otype +ĠC AL +Ġagre es +.v o +.Execute NonQuery +ĠTop ic +Ġ' {} +Ar m +Ġe cc +M ag +Ġserial ized +ĉ conn +c ached += tf +ĠByte Array +prot obuf +var char +ĉ ASSERT +Ġlist e +_tr igger +· ¸ +Fe el +T ahoma +ĠL ik +Ġstruct ured +erg us +.In itial +_ ge +cl js +.cont act +Ġand ere +$ stmt +_C URRENT +ĠDis cover +$ res +form atter +H a +vang st +Ġem erge +ãĢĤ âĢĿ +ĠCabin et +-s quare +éĥ ¨ +Ġr age +ĠA J +ĠV T +sh adow +ĠFa ith +en ames +pret ty +has il +part y +Ġvar char +Ġf otos +Ġal um +ĠBelg ium +.y label +Ġde j +_num bers +Ġh u +.set Adapter +ĠUs ually +(s ample +.Sh ared +Ġbook ed +Ġ>> = +Ġmin erals +"> +pro g +bo o +_m d +_p ack +(ex press +ut z +\ Auth +, id +ĠCh ile +act ice +Ġrecruit ment +Ġpos es +Ġvulner ability +inst anc +or um +d ess +Ġx l +%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%% +( fig +Ġdelet ing +.d el +) ')Ċ +ĠWeek ly +?? ? +(str cmp +sm ith +Ġpurs uing +- so +ĠApp s +/ 'Ċ +Ġdec is +FO RE +Every one +Ġl anes +V irtual +. attach +( Log +ĠMed icaid +( Path +ĠTurn er +/ application +Ġport rait +Ġopp ose +check out +Ġfinish es +_M E +Bar rier +S ong +V AR +Ear lier +rell a +Ġh ast +az ar +Ġpull s +ng x +Ġinspir ing +Ñĥ Ñİ +-d irection +Ġexplos ive +Ġcreated At +st o +Ġwhe at +ĠB uilt +' ai +Ġtrack ed +ham mad +RowAt IndexPath +_ heap +D ue +Ġconnect s +.p ublish +em u +Ġbul lets +B AR +ol ate +Ġintern ally +Ġcatch ing +-p assword +ou ched +æĢ § +e ous +Ġx range +Q uality +v v +Man age +( ($ +ac ements +ĠBro thers +ĠHE AD +ĠUn supported +s an +es i +** *Ċ +Ġadapt ation +ĠWork er +'] / +.save fig +( trans +Ø ¬ +ne e +Cor rect +... ")Ċ +Ġsubmit ting +-p ath +ĉ last +iss an +.x label +ĠS epar +/ no +_b est +ĠM ills +_s ock +(f lag +Ġdest inations +em ption +ĠF AIL +å ĴĮ +Ġr p +f act +ĉ len +D AY +Ġse iz +_d st +l ip +.Line ar +ĠB asket +$ t +$ i +- brand +ĠNe il +ĠE q +Ġth ou +og ene +Ġscholar ship +æĽ ´ +Ġs wo +ag inator +en i +( book +Ġbl ink +th us +Ġcancell ationToken +ĠPalestin ians +Ġprofit able +Ġback pack +ens on +< Long +Ġp ools +Ġst icks +Ġspokes woman +Be ing +ĠHer itage +ĠN ike +SH A +ĠNotImplemented Exception +$ core +ĠR ico +/ latest +ĠC zech +ner Radius +(l ines +Ġsem ester +Ġw ounds +Pro cedure +.m ail +() ):Ċ +Ġcor rid +ter ed +ĠN CAA +Ġgal axy +_k ind +il k +Ġtr as +_P OL +ĠH et +Ġrefuge e +Ġteen age +.b inding +post al +Ġiç in +ĠData Type +é ĸ +ycl erview +, value +_id entifier +< b +Ġout file +čĊ ĠĠĠĠčĊ +Ġcr é +Ġrespond ents +ĠBe ast +ce led +Ġinter f +-th eme +g if +ĠR angers +IT AL +Ġauthentic ate +Com pletion +urs ors +Ġcin ema +Ġdisc our +ĠJ aw +OCK ET +Ġpr ayers +ĠL uis +fr ag +=[ Ċ +Ġbr ave +_p ose +C ertificate +- fe +ifer ay +ĠFl ags +Container Gap +ĠC rit +Result Set +ĉc ur +Ġcorrespond s +St aff +.Http ServletRequest +Ġneur ons +ĠMain AxisAlignment +ed ar +Ġg ad +_p arts +ĠÎ ² +Ġf x +/ files +ĠB ros +hip s +Ġgluc ose +Ġfar ms +Ġment ally +rest aurant +Table Name +ĠMer cedes +. Visual +Ġan ch +inal g +_r untime +Ġpropri etary +Ġintent ions +iz i +S lice +; "> true +ĠNY C +Ġb ored +ĠD etect +Ġapp ar +Ġje ans +ĠT ak +I OD +ĠH orse +( FILE +( ? +ri que +optim izer +n at +lo ys +ĉ Token +oub ted +u ess +oco a +Data Member +_P OWER +class List +Push Button +ĠWi Fi +. Stream +.g uild +Ġn og +ĠPortug al +ĠUnt er +Pr imitive +b oss +ĠDe utsch +Ġerot ic +Ġstr conv +.Try Parse +Ġgr ams +.S uccess +_p k +ĠHar vey +-m inded +.c ountry +[] " +Ġang el +Ġbe ats +ĠV or +il io +.m aster +s omething +ĠP ACK +( if +Request Body +Ġant es +/w idget +Ġmod o +ĠA W +find er +Ġoptim ized +Ġmiss iles +N B +ĉint ernal +t ex +ĠS ri +Ġdam aging +ĠM ais +- Allow +ĠZ h +- alt +Ġ ));ĊĊ +è ī +Ġinflu ences +Ġc atal +_REG ISTER +ĠAPI s +-cent ury +Ġbi ology +ĠAct ual +Ġhe els +TR ACE +_D IG +D ataset +ĠM atter +Ġclass ifier +.w ikipedia +ĠRog ers +Ġdon ated +raw ler +en en +Ġcas inos +ort al +Ġpr ive +s pe +duc ers +. ep +Ġgr asp +ac ji +Ġd airy +Ġb uses +.com m +. ins +ĠI RS +ĠBe er +ad c +o ard +_M ET +Ġ' +' +r ans +Ġkind a +ĠâĶ Ĥ +ĠM aur +аР³ +Ġband width +ib us +ĠD ifferent +(m at +ĠRes ume +_UN S +est ablish +Ġfon ction +Sub scription +_com pany +Ġlight ly +.con firm +.y aml +ĠBo ost +Com merce +- template +_DEL AY +ĠH I +Ġn avig +(S ender +ĠH S +_ "+ +ĠRE QUEST +Ġw ifi +=" "Ċ +]) -> +Ġro pe +Ġviol ated +Ġgl ance +ĠK urd +Ġè ® +de ck +ĠIS BN +Ġin fect +ĠF oo +Ġget ter +Ġt ener +ap pe +.h h +_h ot +< AM +p oly +! ",Ċ +Ġconver ting +ĠW WE +RO S +(' { +Com mit +) L +ĠO re +Ġsp arse +Ġdis posal +Ġcan celed +åIJ İ +Ġa er +Ġvin yl +á» ĥ +rec ogn +ark ing +Ġtrick y +* s +Ġproceed s +Ġis o +Ġco conut +Ġcraft ed +IEL DS +Ġquest o +Ġcomm un +_CON NECT +Ġtraff icking +De ep +a ções +c odigo +ve au +Ġbet ray +int a +T ED +æ r +m art +_B US +/ sc +ial ly +Ġcigaret tes +è¯ ģ +(n n +Ġmodel ing +/ products +w arn +Ġmet ro +ĠI v +& ) +ĠC able +Î » +Compar ison +g ary +ĠB A +P ART +Ġp v +_up dated +C redit +orth y +observ able +Ġthe atre +B LE +; }ĊĊ +la unch +_str ings +ug o +ĠR PG +- auth +Ð ł +hol m +ĠP and +U id +Ġim ply +ìľ ¼ +'] =' +/ User +Ġstr cat +нÑĭ й +Data Adapter +Ġland sc +Ġdipl omatic +ï¼ ĵ +************************************************************************ **** +ĠCh icken +Ġbc rypt +.In f +[ col +ĠQu antity +- position +Ġdiet ary +Ġfil mm +Is rael +Pre v +ĠMill ion +Ġrem ed +Ġbill ing +Ġout doors +.t m +Ġn ad +F org +Z Z +Ġs sl +], ' +K T +f req += document +bl ur +¬ ¸ +ĠJeff erson +C s +(s ave +Ġstr ap +Ind ia +Ġide ology +BO SE +ĠF P +( ans +Ġfe ver +ĠY am +K ing +à ² +AT ING +bo hydr +roll back +Ġnew Node +ĠN VIDIA +Ġhon our +ĠCon firm +xb d +Ġsuccess or +/ u +l iv +ourn aments +Att achment +Ġgr up +Ġtri be +Ġca res +e ft +_s ame +' label +Ġ ãĢIJ +M otor +Ġin exp +Ġ" (" +_POS ITION +Ġval ley +ĠResult Set +Ġpres erved +Ġmut ations +Ġquestion ing +mun ition +parse Int +ĠS r +ĠMet adata +âĢĿ ï¼Į +timestamp s +Ġtrans itions +í Ļ +Ñ Ĭ +i om +.D o +Ġp ine +Ġf ung +Ġtrans mitted +ct ime +ĠF am +Re vision +B as +UP ER +D estination +toHave BeenCalled +Ġun fortunate +IN ES +_pro f +Am ong +ĠCy ber +ĠB attery +gen re +ĠView Model +- = +Ġutil ized +p aint +.Integer Field +ern ity +comp iler +âĢĭ ĊĊ +ĠM asters +.To Array +Ġstrt ol +ĠUkrain ian +} ));Ċ +Ġsh emale +" That +for all +/ download +Ġrhet oric +.l atitude +ĠWH EN +Ġshock ing +IF IC +.N ormal +_F OLDER +Ġdr ift +Ġmount ing +- book +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ĠWire less +> ".$ +Ġrel ies +( Console +Int ernational +-> {$ +M id +Ġdis sert +dd s +Ġdepos its +ĉd river +# ga +pr ising +print ln +Ġpres enter +Ġmin es +C SS +ĠD ual +(! ( +Ġk am +Ġis Loading +ĠProt ect +. upper +ar ium +]: ĊĊĊ +Y ii +-sh irt +ĠIM AGE +_color s +Ġur gent +.Cont ainer +! (Ċ +S aturday +Ġsoci eties +ĠTh an +ĠC od += @ +Ġattach ments +.m obile +Ġsp ite +Ġb ounce +raw l +instanc etype +ĠTr uck +Ġmanip ulation +( Config +-in st +Ġst or +it ution +Preferred Gap +Ġmain AxisAlignment +Ġlist ened +'' 'ĊĊ +ott age +- project +.AP PLICATION +ĉ root +Ġwh it +Ġb ilder +Ġk er +Ġappl iances +row ave +ìĿ Ģ +ematic s +ĠO rg +op ing +_SE ARCH +Ġch am +add ContainerGap +Ġ( ). +ĠAr row +Il legal +Current ly +Ġus a +Ġpassword s +Ġre nown +av ern +ĠEv il +Ġconc at +Ġdu o +Ġv ale +ĠBe an +Ġindic ators +cm ath +ĠP ump +Nov ember +ific ant +_DOM AIN +reg ar +ĠPort al +" $ +Ġformer ly +"] :Ċ +ĠVis ibility +.getElementsBy ClassName +_RE D +Ġch ampions +à ´ +Val or +_ es +* a +-re peat +B and +.st age +Ġbure auc +C nt +et en +- function +Ġm uito +P ID +_ editor +Ġcrash ed +de ad +k at +ag h +ĠEX T +ass er +-sm all +Ġreal iz +( Entity +ú s +ĠAct ually +ĠEl ite +Ġhel m +(non atomic +ash er +Comm unity +all eng +ir y +ĠG rowth +Ġs ue +Ġfrequ encies +_des criptor +.At tribute +Ġrecip ients +_N S +/ "+ +ib an +Ġath lete +ĠI gn +_D MA +(d s +ĠRequire ments +AD I +ere z +\ Admin +br aska +ĠR ust +Rel ation +C OD +ĠV ERSION +em ma +)) { +.D uration +ĠC amb +- logo +Ġread able +Ġcre ators +() ];Ċ +Up Down +-h alf +.get Month +(s f +P ic +Ġhun ger +.t x +Ġexceed ed +_se ed +( ^ +_s k +.per form +Ġ> :: +Ġm ongo += float +bind Param +Sm art +if a +Ġse curities +Ġpre jud +Ġ, " +Ġcor ps +Ġv ra +amac are +it err +(M edia +uch e +Ġc ob +Ġlib er +. geometry +Loc ator +Ġsl iding +Ġsurg ical +_C UR +Ġcon sect +[ * +ĠRes ort +St ub +_DO UBLE +ĠS oph +Ġelect oral +_dis able +ĠÑģ о +ĠLight ning +Ġment ions +oc y +Ġle aked +Ġrelax ing +Pres enter +v sp +Ġgu ilt +=- =- +.re ply +ĠMir ror +C amp +Ġ+#+ #+#+ +Ġ+#+#+#+ #+#+ +.A uthor +Ġdirect ive +-h ook +íĦ ° +}ĊĊ ĊĊĊ +@ pytest +_r and +m is +Ġcolor ful +u je +lass es +ĠClass es +.h ave +% ), +é¢ ĺ +Ġdistur bing +sub string +ĠK oh +In vest +p urchase +Ġrec ycling +ĠA RT +ier archy +Ġf ps +.check Box +íķ ´ +_m aterial +duc ation +Ġf w +ud it +Ġreview ing +ĠS id +S yntax +ĠW ritten +arg ar +UM E +/ q +Class ifier +Off icial +Ġj azz +Ġom ega +Ph ysics +Ġl ugar +_access or +.command s +Ab ility +ĠB atch +R AM +Ġencount ers +. Qu +BY TE +ĠD istribution +Ġus o +ĠReco very +appro ved +Ġden ial +/sh are +Linked List +)čĊčĊ čĊ +udd y +Ġf ines +Ġr y +Un icode +ĉ render +Ġprem ises +Ġp on +ali ases +/F oundation +c uda +ĠC ock +,: ) +(f older +Ġm éd +dr ag +Ġtal ents +ĠĠĠ ĊĊ +е ÑģÑĤв +m ob +.y ml +Ġa ster +Ġdis cre +go al +ĠGT X +ĠS UCCESS +ĠL ONG +(f ind +Ġsing ular +_s z +ĠEth ereum +.. Ċ +Ġir res +')) {Ċ +Ġmin isters +St eps +ivers al +ĠNever theless +- led +Ġ( %) +ç¡ ® +Ġtime zone +Ġstr anger +(re nder +Ġsh util +Ġm ph +Ġtri o +pp y +Ġpred omin +Ġend ors +ĠRuss ians +ĉ row +Ġw izard +.s erialize +Ġcompl ained +Ġs ido +Ġdelight ed +-m e +ĠR av +H uman +ad ays +rec v +Work ing +J ump +ĠÃ¥ r +ĠAut omatic +_B ase +æł ¼ +aur ants + ¯ +æ ¸ +(C Type +IF I +( amount +Ġbelie ving += mysql +Ġf ir +Ġrest oration +ere co +Ð ¢ +_ '+ +Ġe book +Ġde bris +(input s +AY OUT +Ġscre aming +av ia +land er +Ġdist ress +Ġas sembled +ĠA void +( thread +ĠR PC +_EX IT +( queue +и ÑģÑĤ +D ll +Ġsk ull +_p ub +che z +min ate +ens en +Ġins ane +b ounds +ĠR osen +Ġcondition ing +process ed +v ideos +f our +.Con v +| ;Ċ +Person al +cer pt +:UIControlState Normal +Ġdos es +ĠKar l +ĠFre qu +.B ASE +ĠV ote +Ġcon current +ĠMessageBox Icon +Ġà ĸ +ĠDub ai +ĠR etail +: number +ĠOb server +ĠBig Integer +_ origin +_W ORK +F rames +Ġnot ably +. âĢľ +Ġtrop ical +Ġn iche +am ina +.s ys +(t okens +mod ify +os it +st rom +ĠCom ics +O PTION +T icket +Ġfact ories +Ġdis put +_F ile +ĠFin n +ee e +ĠDisc ord +_m oney +.t pl +_s afe +L B +Ġgl ut +J K +.fl ow +- cont +g os +Ġhor izon +ĠR ush +:: * +P ipe +ull a +bor ough +he imer +(m ove +( Text +} );čĊčĊ +w elcome +ĠCom ponents +Ġgovern ance +c losed +ĉm argin +Ġla undry +ĠTerm inal +iz ards +. âĢĶ +.rem ote +.r adius +ĠQue bec +Ġd h +T ech +ĠM ist +s eller +_l iteral +Ġgen ius +Ġbr ains +g em +ĠMe asure +Ġcata st +r ance +.Text Field +Ġconsum ing +Ġ'\ '' +oubted ly +ĠC ertain +E v +ert i +be ing +Ex perience +Ġ// [ +ĠArab ic +ĠC rist +ĠAz ure +Ġhor a +l adesh +\ Blueprint +d ar +.re l +Ġsup rem +ĠRe agan +ĠAt tributes +-s idebar +Ġuse Styles +ĠA irlines +Ġh ills +/x html +v inc +_m ock +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +ĠP ill +.Layout Style +ĠCommand er +] < +sign ature +Ġ{ }čĊ +Ġhat red +Ġë ĭ +ole sterol +Ġ ******** +ancell or +c rop +T IM +ĉĉ ĊĊ +ys qli +uit ive +ĉun set +_s el +Ġmen us +t ick +Ġconstit ute +ĠElement s +ĠRed is +agg io +_f p +_de pend +em as +CA ST +or ange +j on +ĠEm ily +Ġpot atoes +Ġre ceptor +ĠElect ronic +ĠL ights +Ġcomb ining +ĠSome one +Ġ######## . +ĠT OD +/ show +X d +." ' +af x +Ġtr agic +St yled +ĠMar co +G allery +d ale +.âĢĿ ĊĊĊĊ +é rie +/s ervice +äº Ĩ +Ġamb ient +_SET TINGS +.Ad apter +l ene +Ġtrav els +Not ice +Ġcle ans +ĠF em +ch air +Ñĥ н +/ my +_b ad +ĠEcon omics +IS A +_C NT +(M enu +äº İ +ĠR idge +Ġlength y +D ot +Ġjump s +Ġhe y +$ pdf +Ġw orm +Ġs ut +Ġsh er +iam o +ĠCal c +trie ve +Ġc ops +ĠCh rom +Ġreg ulated +reat ment +ĠHigh er +ok s +Ġde ze +LOC ATION +ongs To +Ġfin ite +Ġvar ies +Ġposition ed +' il +éĩ ij +Ġh ike +(d one +play list +Ġad a +Ġcoast al +ĠN ancy +.DateTime Field +Cpp CodeGen +ĠSimilar ly +re ur +ĠCon tr +ĠH idden +ĠB eta +atch ed +_inst all +. Output +Look up +ĠRich mond +qu ared +Ġm anga +-control s +ĠBern ard +L arge +Ġslic es +Ġoff ence +ĠM ega +Ġest ar +Ġjoint s +Ġsum m +_pl atform +B uff +.add Subview +Ġret ained +Let ter +.d im +Ġess ere +ĠS caffold +EX PECT +ĉ RE +.long itude +ü nd +Ġstat ue +.add Widget +ĠCar ibbean +add PreferredGap +il de +UIL abel +ĠOp port +Ġimper ial +urs ion +Ġmand ate +Ġpromot ional +Ġv k +ia ÅĤ +Ġp yl +ĠCre ation +оз д +Ġsim pler +. what +ĠRec ent +St orm +. quantity +ĠL ov +" - +ubb les +_not ification +(w orld +ur ger +* (- +: "Ċ +h m +ans hip +ĠAl most +Ġmotor cycle +_f ee +Ġabsor b +ĠVin cent +Ġsound ed +ÃŃ st +Ġpharm aceutical +ht ag +ĠKind le +ital ize +ĠEm peror +oust ic +Ġspecial ists +åħ ¬ +Border Style +/ \ +RE LATED +(', ', +(ex pr +Ġh t +åį Ī +_C reate +Ġspecial ly +Ġ[] ;čĊ +Ġhe el +Ġse pt +_ arch +(in itial +% .ĊĊ +\", \" +Ġdiscuss es +Ġu pt +Ġ[ & +Ġman us +.h and +ĠM AIN +ĠDen mark +Ġ], čĊ +Ġcr yst +Ġn ack +Co ords +_in ner +Ġmid st +Ġaw ake +ĠÐ ŀ +-b reak +ÃŃ vel +_P ASS +ĠParam s +Ġdet r +Ġsp ider +ĠCon cept +Ġpre nd +CH ED +.Ex it +Ġpop ulated +Ġvirt ue +_SE SSION +Ġnou vel +o auth +Ġд аннÑĭ +r ink +.Header Text +atur ated +Ġer st +Ġå ħ +ॠĩ +_vis ible +ey er +Ġli able +Ġde be +Ġb w +{- # +_W IN +df s +H over +ĠP UT +- angle +Ġnob le +Ġtr aces +enc v +Ġuser Data +_in s +ĠS uz +Ġnews letters +ĠMod i +Ġentreprene urs +Ġtrib ute +Ġrum ors +Ġr r +ĠQu arter +ê³ ł +Ġfeed s +ó g +Ġen velope +Ġle ar +Ġk ø +develop er +Sim ilar +: ")Ċ +sub scription +Mod ifier +ital ic +Ġn asty +Ġtermin ation +Ġchar ming +Ġâ Ł +ton s +.tr ace +h ots +ĠU R +M ont +Ġjust ified +ĠG ang +ine a +Ġb og +( ap +_ $ +Ġcont amin +.D ot +ĉ Debug +( exports +Ġpa ired +ĠAss ignment +Ġautom obile +ĵ į +Ġph ases +v w +@ SuppressWarnings += \ +r ant +- ed +ĉ await +Ġcert ificates +'> " +Ġint act +CT RL +M ike +greg ation +AT TERN +Ġre public +_up per +ili ary +Ġcomput ation +h ire +ĠSh in +_ ANY +ĠManufact urer +ĠC arm +Ġbear ings +_c omb +c ad +ur istic +Ġwholes ale +Ġdon or +.inter faces +press o +ĠBr un +-c lose +pro ve +_S K +ĉf rame +et ros +ĠP ain +_EX P +ĠL T +_f s +.dat as +ĉ ss +vo ir +ĠA xis +M ajor +=" < +[ h +Ġprof ess +igr ate +(s core +Key word +" os +ĠĠĠĠ ĉĊ +an alysis +Ġre play +.p ass +\ d +t ls +Ġsan ct +.l ight +_m obile +ÑģÑĤ ÑĮ +ĉt otal +u ity +Ġpa used +N AS +Ġen core +lo e +Ġ-* -ĊĊ +.h igh +am pler +ĠSec ure +Ġfrag ments +_ vel +ill ary +ĠSte in +ĠD awn +Ġmax imize +ภ¢ +Ġ/ ^ +Ġcontin ually +Ġsh adows +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠI ActionResult +Ġinform ación +C HECK +.Selected Item +b undle +ol ley +< Int +AIN ER +ĠW ing +tit les +ount ain +C Y +ĠLoc ale +form er +< context +R adioButton +_s chedule +Ġfab ulous +Rob ert +_PRO FILE +Ġg ates +IM P +ĠPent agon +g old +b ach +employ ees +R otate +Ġch amp +Ġsel bst +Al tern +Ġconvert View +/ , +Ġ~ ( +St reet +_ place +Ġpersonal ized +P ublisher +ĠSO CK +_NAMES PACE +ĠStand ards +so ever +_C ENTER +Inter est +ô t +tem perature +View port +get Resource +Ġeat en +Ġsem pre +Ġab normal +Ġc ylinder +Ġtroub les +n od +Ñĭ в +g ames +_g l +Pl ane +g rey +_t bl +.Component Placement +ĠCh ase +Log ging +man y +ì Ĩ +Ġfl ame +="< +Ġtra jectory +_r ing +Ġhydro gen +tr on +Ġstat ute +Ġcondition al +Ġtr ay +-s chool +(w idget +$ config +Ġrequest ing +. uint +et on +brit ies +Of Type +AD MIN +p redict +Ġg egen +ĠH app +OC UMENT +ĠA part +Ġ---- - +ro e +u ide +just ify +ĠSqu ad +Ġprof es +.b ot +_c urrency +inn en +ĠM umbai +ĠNum bers +avana ugh +agn itude +âĢľ There += http +çī ĩ +Ġv b ++' {{ $ +Ġin ode +s il +Ġh ace +Ġsever ely +ĠOver view +Ġspr aw +Ġbeach es +: left +· » +($ { +ĠF IRST +ĠSp a +- ass +Ġb aise +ĠN ODE +ĠP izza +P et +(se q +\ ">Ċ +CppMethod Pointer +Ġv p +Ġi a +_se conds +em et +/b lob +_TH RESH +... čĊ +D est +ĠN H +.data Source +it és +ĠJ ak +s ell +Ġwork shops +< u +Ġr ivals +ĠEX ISTS +h om +-t oken +compat ible +.J Panel +Ġphys icians +art in +Ġdes irable +Ġdistinct ive +.D ep +g id +ili ate +, max +Ġprem iere +Ġq Debug +Ġadvoc acy +Ġwh isper +P t +Ġun changed +_q ty +请 æ±Ĥ +Se ason +avel ength +ĠP ul +Ġd ÃŃa +'] ]],Ċ +al is +(" & +bor o +Ġb m +ĠR adi +w rong +ĠGo ing +ime Type +ij i +- feedback +ĠN ames +ĠB apt +Ġprob able +ĠE ther +ĠPolit ics +_prot ocol +lin ing +S at +Ġcor rel +.Pr imary +(null able +RI ORITY +Ġcolor ing +Ġutil izing +d as +Ġexport ed +Ġcar riers +Con v +. editor +i ó +(h andles +Ġapprec iation +. import +ĠAust ria +ĠStr ip +il ight +Ġappropri ately +ĠP rest +ĠW ir +ĠUI Application +al chemy +ĠM ob +ĠD etermin +ergus on +register ed +_con vert +ĠVlad imir +.Show Dialog +ref lect +Ġsh ook +Ġass ure +ĠO ften +Ġcivil ization +Ġvocab ulary +fore ground +ĠS cope +Ġunw anted +act ing +Ġ( [] +Ġmark ing +. original +ĠMO VE +Ġsport ing +ception s +NS Number +S izes +Ġprovinc ial +_Tr ans +Ġproblem atic +d igit +ĠEm ma +lock s +ĠC rew +ib a +') : +ish a +Ġm amm +Ġocc ured +w cs +(r ule +Ġmerch andise +es pecially +ĠT win +Ġn aming +Ġs log +Ġimpro ves +Ġad her +: text +.h adoop +_HT TP +.to List +.dis abled +Ġl enses +.in i +ĠR are +ĠUb untu +Ġsc ram +ol ation +tit ulo +Every thing +Ġnod ded +icht ig +_const ant +z c +l ift +ĠNot ify +ond o +ĠIN F +(" + +ĠK az +Ġd read +.m apper +le ur +ĠCome y +ĠN B +ic ers +.P ush +ĠH ack +ĠBrazil ian +_pro d +Ġ// ĊĊ +Ġb icycle +Ġun available +Ġadoles cent +bl k +Ġmit ig +_bl ue +ì ĺ +fade In +ĠUtil ities +ĠM N +; k +< style +- status +ind o +Ġinn ings +Ġg j +Ġ|| = +.e u +: Number +Ġcuis ine +ĠURL s +ie k +Ġw ires +ĉ ps +ie g +.m k +so ap +Ġsom etime +Ġst ap +_s eries +.T arget +æ º +.dest ination +OUN TER +R aises +& A +Ġsmart phones +NI Env +.s dk +Ġhelicopt er +Ġim pe +ĠB irth +A U +b readcrumbs +co ords +Ġexplo red +Ġl od +ĠI p +g able +ian e +Ġart ifacts +Box Layout +ا ر +list ener +.c art +ĠH uff +ĠHind u +ĠData Types +ĠDr upal +IGN ORE +Ġoffset s +ĠR TC +- login +æ ® +ĠQ Object +Ġprosec utor +R ock +_ch at +W ay +ì ² +Ġneg lig +Ġd ude +; < +Ġdeleg ates +_f ailed +/ dev +/ work +( New +et able +() " +( Icons +Ġp ork +ĠModel AndView +ĠV IP +ĠK or +m ix +Ġox id +ĠSC REEN +ĠFour th +/ ",Ċ +Ġte e +ĠSte vens +t icks +Ġp ledge +ib bon +ĠLo an +Ġne o +n umpy +ĠShared Preferences +- oriented +ĠLogger Factory +ĠGraph QL +zen ia +" _ +W omen +.c ast +Ġdeliber ately ++ b +ĠAr n +font Size +Ġm aze +Ġbl amed +.m as +} )čĊ +eler ik +Ġsc anning +ĠWork shop +Ġfind en +Ġca ut +UI Font +( return +al in +cast le +//////////////////////////////////////////////////////////////// //////// +Ġincent ive +op ath +b lob +Ġcigaret te +Ġfert il +*/ ĊĊĊ +ĠSh ar +Ċ ĠĠĠĠĠĠĊ +Ġunc ertain +ĠS ton +Oper ations +ĠSp encer +Ġdef in +ĠS olo +on est +·» åĬł +Ġu omo +G ive +Ġdent ro +; padding +ent ai +ĠC ars +Ġenthus iasm +ĠOper ating +S kip +par ation +Ġprotect s +Ġre ver +d g +ĠC incinnati +Ġconsect etur +Ġm uss +employ ed +a uses +ink le +. Values +£ ¼ +lo v +_W ARN +Ġbook mark +ĠAp ollo +. axis +Ġm ét +Ġop ener +Ġtum or +d an +Ġelement ary +Ġsk ipped +ĠK er +as ia +_res p +Ġdem ol +ĠCan adians +Ġt astes +U Integer +Ġ' ${ +.aw s +RO ID +ri ans +M Q +ord able +Ġcous in +Prop agation +(S ession +ph alt +UL D +ĠSc alar +Ġblo ody +Ġ ঠ+.m ask +, q +ĠUn its +Ġcent res +ĠPr im +. ]ĊĊ +ĠSh aw +P rom +ĠTh ought +Check er +_output s +( chan +E INVAL +Ġb ob +_c mp +P ed +Ġmat rices +Ġvrou wen +Ġgenu inely +high light +(d isplay +) != +Ġdel icate +ĠL uther +ĠM iles +Ġuser ID +% = +ate urs +_B UF +---- ---Ċ +imit ives +Ġsh elves +sl ow +_in formation +LE G +W r +.form s +cel and +/ un +: & +.âĢĻ ĊĊ +=" % +Ġpro st +Ġfont size +uc ión +get ic +am t +=" . +Dec or +B rit +Ġ"" ). +Ġfound ing +.File Name +ĠT ier +Ġdisc lose +á m +.s yn +.View Holder +lic ant +_st age +Mon day +Ġdes erialize +t alk +Ġtradition ally +æĢ ģ +Ø ® +LE X +Ġe h +ĉ ROM +Ġ{ })Ċ +Quest ions +nc py +Ġfix ing +к Ñĥ +_ Key +: x +ĠSTR ING +ĠÑĦ ай +ĉ left +ĠBen ch +ell ij +UR RED +ĠDi agram +} catch +/ time +ĠMiss ing +db name +Ġs ore +ĠW alt +ugg ing +rep resent +ĠG S +ne ys +ĉ page +Ġvol can +(b tn +Ġexceed s +Ġ erg +Ġpil ots +ĠS ed +ers ions +Ġpat ron +R V +/ top +. asset +_c ross +. Editor +.t b +Ġwel coming +SC REEN +) findViewById +C oder + ",Ċ +_P in +ues e +Ġover rides +_ ready +Adv anced +Ġop i +-c art +("/ ", +ĠDe b +CR Y +ĠVert ical +ĠO VER +ĠCorpor ate +Ġ"" ; +Ġste pping +e j +Ġaccus ations +Ġor az +_t ail +Ġindu ced +Ġel astic +Ġbl own +, // +Ġbackground s +âĢĻ une +-s dk +Ġset Interval +Ġincent ives +Ġveget able +_ On +exp anded +p ix +_sh ader +ĠSP DX +@ example +ĠW rapper +.Z ero +Pos itive +Ġsp inner +Ġinvent ed +ĠG ates +оÑĤ оÑĢ +Ġcompar isons +è · +.pr imary +data Provider +add itional +ĉ options +s napshot +.set Horizontal +Ġ" {} +ĠFish er +hal ten +< Type +Ġmax Length +ĠM t +Ġê° Ģ +.jet brains +Ġident ifies +Ġflow ing +ĠDisc ussion +ats by +Ġsch w +ught y +Ġr ivers +.un ique +_PH Y +ed ral +( ll +Ġcs rf +pp ers +ü l +ĠEs pecially +port ed +ĠHarr ison +****** */Ċ +Text Color +ìĬ µ +w ire +Ġstatus Code +ĠFin ish +c ence +ĠMcC ain +ĠW or +( await +Ġ) -> +ĠRegister ed +IN ED +k al +par ison +Ġobj eto +V i +mand a +Ġrenew ed +ĠS of +ess el +.nd array +Ġcr ap +ç® ¡ +.ab spath +( up +Ġclear ance +ĠT W +_C OPY +ĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +Ġforest s +Ġarg uably +ĠA SS +he y +am el +_f ore +ĠSou theast +Ġab used +Ġpract icing +aked irs +ä¸ » +_res ources +Ġp ond +.F ixed +Last Error +ĠPsych ology +Ġ" // +! : +Re usable +Ġmens aje +Ġro spy +Ġb our +Ġvar ieties +Ġem path +(( { +_ org +ĠM es +ĠMag ento +IST ORY +Un less +Ġh j +ĠD uty +J un +, size +Ġpaint ings +Ġdisp ens +d art +Ġbehavior al +Ġr pc +cal culate +fr uit +_m m +ĉp thread +Max Length +Ġc urrencies +_cap acity +ĠO z +Ġfire arm +Ġcoeff icient +Ġbankrupt cy +w art +Ġfat igue +AV A +Ġes pa +_p c +ĠQu otes +_L IGHT +ĠT ickets +Ġrel ates +Ġpublish ers +Ġunlock ed +Ġ// ---------------------------------------------------------------- +ĠInterrupt edException +Ġout look +r n +Ġreb els +W ritten +Ġas ian +ot to +Ġ ĉĉĉĉ +_g pu +T xt +.Image View +Ġsu is +_t ables +.Rec yclerView +Ġwhat soever +è ģ +] ++;Ċ +assert True +_ verify +ĠR ivers +Ġ ][ +J et +id ian +S ibling +Ġgen res +.A ccess +OP S +Ġtr ivial +ภª +al en +в ед +ĠS word +Ġscrut iny +(c b +Ġcomm erce +Ġguarante es +_ad v +ĠL ET +rec io +Ġh ilar +Ġback yard +ãĢ ı +Ġillustr ated +/v endor +. Util +Ġw ow +LO Y +ĠMar shal +"> '.$ +ĠB ak +Ġmod ifiers +d ictionary +ĠSt re +m ultiple +")) , +ĠC ort +'] "). +( admin +ĠCre ator +Int ernet +( ms +log y +DECL ARE +ĠMarc us +<< << +ãģ ł +_m y +(in st +Ġsc iences +ND ER +. enter +Ġit u +Ġbeh ave +P an +omb ies +=' < +')) ;čĊ +ĠM ENU +ĠWork ers +.No Error +Ġbind ings +Ġdis abilities +{ \ +ĠM unicip +Ġco res +ur ple +ĠN okia +us ions +ĠF itness +.handle Change +Ġjav ascript +ìļ Ķ +( dec +Ġpack ing +-de pend +Ġtrans cript +z eros +_ alert +? ",Ċ +lib s +± оÑĤ +Ġ| ĊĊ +tr ained +ĠG ent +ĠR ab +x p +_config uration +å¤ © +_ accept +.rec yclerview +: url +ĠMu hammad +Ġprivile ges +_b ank +uk u +w allet +ĠRO OT +Ġenc uent +? family +ĉ position +Ġc g +Ġprec ip +method s +_f ast +in crement +ĠT iger +_OCC URRED +qu ip +ĠH AS +_d om +Ġw reck +b j +Ġd ern +Ġorg ans +. entries +Ġ_ (' +ram ento +ĠJam ie +Ġp unk +IP P +Ġprogram a +Ġatt ain +Ġpro ves +/s ign +Ġanswer ing +Ġl adder +************************ **** +ĠW almart +ĠCONT ENT +duct or +Ġver bal +ĠP ID +c rypto +_CALL BACK +Ġ= ================================ +Ġpot ent +Ġshort s +.U ri +.un iform +; border +ĠW er +Ġhere in +ll a +ĠI hr +P ixmap +l iteral +! )ĊĊ +g eneric +r ust +_script s +ost o +it us +ĠCoal ition +Ġrem ot +de ploy +ĠEag le +ãĢģ ãĢĮ +Ġimportant e +ĉ object +Ġseason al +ne j +aid u +Bind View +ĠSi erra +-b g +Ġmake Styles +[ offset +G ames +Ġhorm one +AR IO +head s +( select +ĠStart ed +@ param +_de cl +_b log +Ġa ño +\ Api +ĠMil waukee +Pro vid +An imated +Ġcool er +ĠSe ed +. Edit +Ï Ħ +ĠT aking +Ġborder Color +-found er +.Logger Factory +Ġ"" ĊĊ +AL T +ĠL ate +EDI ATE +Ġ);ĊĊ Ċ +af a +Ġcancell ation +At om +ĠB irmingham +emp resa +HE MA +asc al +Ġup side +.V ersion +ĠF older +ĠE ight +ĠV intage +ĠApp Delegate +ĠPre vention +.se parator +ST M +( room +gener ator +Ġc attle +ĉ Z +ĠPart icle +' };Ċ +Ġneighb ours +ĠState less +Ġalt itude +Ġsa int +об ав +Ġconv inc +ĠCont ents +Ġje une +(t s +Serial ization +(c ollection +ĠJ azz +ĠD od +ĠR och +ac io +comm ended +DEF INE +.on load +Ġspecial ty +PL ACE +_MO VE +Ġaccount able +Re uters +Ġf icken +Ġde pr +W ow +V oid +.s pace +à¸ Ĺ +Ġt q +ĠP ets +< $ +(C urrent +ber ries +plan ation +Ġlist Of +ĠTh u +ĠPR INT +Ġm ismo +Ġdo i +ch k +ĠUn icode +( role +Ġvir gin +< Point +_RESP ONSE +-h ouse +ĠVenez uela +EM AIL +Ġp úb +_ex ist +B all +.C L +re ferences +ĠBeautiful Soup +ĉ Expect +TH IS +Ñĥ д +b ane +Ġtemp oral +ER IC +et as +Ġrefresh ing +Ġsec ular +@ synthesize +ac cur +Ġn ella +ĠS OL +.p ipe +Ch annels +èĩ ª +Ġinsert ion +á» ĭ +el ia +Ġadjust able +Can ada +ĠI TEM +Ġcur ves +ĠChe ap +let ing +Ġoptim istic +al lo +Ġpolit ician +_down load += edge +ORT H +Ġmodel o +art o +. rotate +Ġs elenium +æĪ ij +_al ias +Ġrenown ed +.' . +Ġc zy +Ġal les +.Com piler +ĠB ass +Conn ector +.R ole +L INK +Ġc riterion +lem etry +Success fully +/p ng +Ġey eb +asp berry +( gr +Ġd angers +Ġcorrect ed +Ġgl ow +Ġelabor ate +ĠB ears +aw ai +=" '+ +Ġpromot ions +Ġmathematic al +Ġ" ` +_Generic Class +ĠChe f +.S ort +table Name +R IC +Ġvolunt ary +ĠBl ade +-e lect +ĠCom bat +ĠAb ility +Ġab dom +Ġd uck +T mp +åħ ¨ +Ġer ase +.P h +ĠDefault s +p artment +_US B +ê te +; ' +Ġp ads +ĠOb amacare +.T otal +Ġdiv ert +Ġcr icket +Ġrecre ational +( red +ĠC le +R U +Ġmist aken +ĠMont ana +Ġstr ive +_sl ider +ĠPl astic +Ġdecor ated +ĠV P +lic o +ĉf alse +Ġpre fs +( \" +_f alse +i endo +Ġ@ $ +B ucket +act ical +ĠZ hang +.c ols +.B inding +Ġw ax +_ST ORAGE +Ġlaw n +Ġr f +.Sc ene +ĠCal culator +.d esign +Ġres il +л ем +E mploy +ĠPr ices +ĠP WM +ag i +.e valuate +ĉ param +Ġbr ass +bb en +Ġinflamm ation +ull ivan +Ġan not +Ġp H +iam eter +ĠB TC +( box +Story board +Ġcl ay +.assert Raises +| string +.App ly +Ġmatch er +und ed +Ġsatisf ying +Ġìł ķ +Render ing +_app ro +ind rome +AN EL +_f ix +br ush +.M atch +Ġsm iling +on aut +S unday +Ġdelet ion +Ġencour ages +P ull +Ġreven ge +Ġqu arry +tr ade +Ġc ables +(d elta +ites pace +Ġf h +.b unifu +Ġvi el +_IN CLUDED +ĠT ail +ad ar +of s +Ġmet als +g om +_method s +Ġn j +.St d +(w in +$ (' +Ġt urtle +ur on +Ġen rolled +ĠH z +ĠBox Decoration +Ġp ont +rel ationship +B i +³ » +Ġmas cul +Ġsh ades +Ġv r +ĠLog ic +Ġa in +ĠD IST +Ġcoll ar +" profile +Generated Value +ĠP ossible +Ġe ines +ĥ ģ +.time out +ĠE c +Ġjer sey +.D ouble +Ġqual ifying +v or +CRE EN +_A pp +_rec v +Ġali ens +It s +E sc +i ator +ĠE clipse +Ġg h +V ict +ĉ html +to o +. const +Ġant erior +ĠW u +(key s +Ġul tr +_p oly +ĠT ap +ĠB ud +A WS +Ġcrash es +_t ot +Cont in +-h anded +alth ough +ภļ +ific ent +Ġde ve +ut ory +ĠW orth +_M S +Ġfloor ing +Ġsell ers +ĠThank sgiving +Ġp ng +Ġval ores +Ġslee ve +Ġfil le +Ð IJ +Ġappoint ments +Ġv im +User Info +BO OST +Ġpos ed +initial ized +.product s +ĠLeaders hip +man uel +' % +em arks +Per centage +(d ist +. avatar +(h Object +ä» Ĭ +_ iff +ic one +; ) +_n il +Ġab ol +е ÑģÑĤ +Ġven ues +.Con vert +! ')Ċ +.B itmap +sk in +_C OLUMN +Re v +G RESS +g ow +Ġw ished +tract s +.assert False +Ġscreens hot +Ġfo is +Com b +Line Width +ĠGr ab +Ġint ensive +ĉ sh ++ ) +.first Name +_PRO CESS +Ġt ilt +it ored +.L OG +Ġb ak +Ġintention ally +.play ers +(c anvas +)) )čĊ +.Pro vider +_P UBLIC +T alk +ĠL iv +ched ulers +Ġl c +ad ic +feature d +.res ources +Full Name +Ġmean while +B uffers +Ġres olver +ĠS AP +_T E +G NU +ĠForms Module +_ wh +ĠS we +.widget s +Ġcabin ets +Ġsus cept +ĠB ott +activ ex +av ar +ant ics +Ġ" =" +_k wargs +Ġgame Object +ĠAng le +.I ter +mar sh +ĠB irthday +ĠC MS +request s +ĠPear l +_E OL +Ġlin ux +( org +_M ouse +.con structor +Ġz d +Ġk icks +art isan +Ġe ax +K n +pon ge +ĠFin land +Ġmet res +ĠAss essment +part ner +/ pre +! ',Ċ +[ Int +Ġos lo +date picker +/ String +op lay +ĠHe brew +, double +Ġtrab al ++" \ +ĉ EIF +/ text +_F IRST +ĠP ete +Ġe go +Ġextr as +P DO +Ġreg ulate +ĠQ Widget +st s +ĠSh ows +ĠN HS +.c ourse +p thread +ĠF uel +.t imes +Ġ ° +Ġstr ides +($ ('# +( words +Ġrhyth m +Ġsp ont +Ġsens ation +Ġsp ike +C losing +页 éĿ¢ +N umeric +Ġbreat he +Ġfin ale +_F ACT +in ion +Ġch ill +Ġform ally +ANG ED +Ġ' :' +ĠпÑĢ Ð¸ +a q +ĠFab ric +(l at +ĠPr incipal +Ġer ro +oc ale +N om +Ġf ost +_C USTOM +.int ellij +ert ools +Ġcl asse +adi ents +Ġfundra ising +EN E +_OPTION S +_ ob +// }Ċ +Ġprote ctions +.se ed +N V +term inal +;; ; +P redicate +Ġì ¶ +Ġbomb ing +G F +Ġch ew +)) ). +qual ified +] ={ +list en +C ENT +d igest +E ast +Ġd iver +Ġend points +Ġe e +Ġcolle ague +Ġdissert ation +_com mit +_D AT +. rc +Ġbre asts +ĠR ug +ĠP il +Contract s +ĠBry an +Web View +Ġconcent rate +ĠIn ner +Ġ' | +std out +_S ub +> -->Ċ +V ol +ĠS SD +)) ), +. Optional +Ġnurs es +Ġor b +_ pe +);čĊ čĊčĊ +pl aced +ess er +Ġther apeutic +Ġwhites pace +Ġa ston +Success ful +Ġpr aised +ĠW es +Ġe ighth +ir al +Ġvrou w +Ġf action +_b ias +Ġw itch +Ġnp c +(s b +ĠRod rig +_b ig +Dep endency +ĠAb raham +ard i +C AR +n os +Ġabund ance +Ġnut rients +in stein +.V ert +ĠI SS +< U +Ġsum s +_h ist +Ġfar mer +ĠA br +Sh ot +ĠBad Request +Ġh ass +ĠR ails +Ġaffili ated +æĿ ¥ +Ġer f +IN F +ĠView Holder +min i +ĠR oth +Ġfaith ful +ĠPhill ips +AND OM +]. [ +_P AY +ĠAr ctic +f aker +D igit +M ale +std err +se ys +Ġ Å¡ +_rem ote +li que +Ġin def +ĠIndust ries +it ra +_p airs +< iostream +Ġsal aries +ik en +.F rame +PL IC +_S PEC +ĠMed iterr +Ġsystem atic +Ġinter rog +Icon Button +se a +int ro +ĠIss ues +enc rypted +Ġintern ationally +Ġsn printf +Ġpast a +ĠBrad ley +_ Status +AL K +_P AD +.l aunch +< select +Ġhar dest +Ġph y +Ġ(( * +-s lide +ĠNob ody +S u +Ġas ÃŃ +close st +_initial izer +Ġsupport er +-g en +Ġt ales +Ġcor p +_f u +s at +ne ighbor +.M igrations +Ġal gun +Ġsin on +.S pec +? ,Ċ +.G L +m ale +Ġmon itors +yl an +-L icense +.m atches +ĠA BS +ĠM ast +ĠW allet +($ ("# +Dir ty +Ġco pe +Ġinterpol ation +ous ed +ĠJ ets +.F LAG +.C ancel +.Event s +ne ver +ĠM Hz +> D +Ġs ervlet +bast ian +Ġ> & +S ID +_cl k +Ġdiv isions +} ',Ċ +Ġd ildo +Ġpar ade +m ajor +Ġab oard +; ++ +Ġf usion +"}, {" +ĠDialog Result +ĉ arr +- em +_n r +(h andler +.N ET +.Xtra Reports +ĠSh ah +ĠB rief +- , +Ġprec io +ĉĉĉ ĠĠĠĠĠĠ +Ġt ant +ĠGrand e +/ xml +_IC ON +ĠR etro +un que +Ġn ag +to Fixed +X L +Ġdecl aring +ĠCon crete +ĠAm azing +ĉprint k +Ġdeb ates +D ATED +Ġaest hetic +emet ery +Routing Module +ĠNash ville +W AYS +Ġw olf +Ġobserv ers +OT A +ans on +Ġe a +Ġgreen house +ĵį ä½ľ +Ġst air +Ġimmigr ant +_app ly +pe are +ĠBloom berg +_PL AYER +Res p +æŃ £ +Cho oser +ĠI Collection +P eter +Er ro +.detect Changes +Map s +Ġs queeze +ĠHom es +weg ian +Ġformat ting +Ġnegot iate +ul d +ĠN ep +ĠQ B +Ġeconom ies +Ġ*/ , +Ġredu nd +ĠA ber +.IsNullOr WhiteSpace +yc led +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĊ +_S h +Ġske pt +Ġre created +Ġget Type +Ġmarg ins +Ġcolon ial +ch arts +// @ +Ġprocess ors +è¯ ´ +b atis +æĦ ı +ator io +mention ed +P atient +Ġpre y +Check box +_x path +.s kip +ĠMorm on +ĠMemory Stream +CRE MENT +Ġk u +m eld +\ Data +ĠK ernel +il tr +éĢ ģ +( profile +Car bon +RO LE +( pl +] *( +.m emory +Ġmed al +Ġadvis or +it ät +Ġh dr +ier ung +ĠProvid es +( alpha +Ġteen agers +- parser +.L atLng +] ()Ċ +Ġfel ony +ĉĉĉĊ ĉĉĉĊ +BO OK +Ġsl ash +Ġclear fix +ĠPro phet +å® ¹ +right ness +-f i +.k ind +ert on +J im +Ġmanip ulate +Ġworks heet +ol in +st ars +Ġart ifact +_EM PTY +ĉm ain +------------- ' ; +Ġexpress ing +ĠI Q +ĠF act +/************************************************************************ *******Ċ +_m ass +)) : +Ġcon dom +Ġcreate State +omet own +Ġir r +Ġ> ( +> B +iter ation +ãĥ ª +Ġshirt s +ount y +-> $ +_S IGN +ĠD ale +Ġj j +E asy +F re +ĠN y +Ġch lor +match ed +ĠG erm +- UA +ĠN athan +educ ation +-y ard +- che +h ouses +r itional +Ġprox imity +Ġdies em +áºŃ p +Ġd rought +.a udio +ĠLe o +Ġfavor able +in ch +ĠD aw +rib ly +_st udent +id able +O VE +Ġlack s +ounc ing +.b usiness +Ġre open +may be +_G LOBAL +Ġdress es +ĠEd wards +ens ible +ĠHard ware +ĠEx cellent +ĠTime Unit +CTION S +Ġsched ules +Ġseg ue +Op ens +am men +- Identifier +Ġst aring +Ġhapp ily +ĠH ob +' _ +Ġ" ); +ament os +et ched +Ġ/> }Ċ +. Users +Ġinterrupt ed +Contact s +Ġreg istro +in burgh +CH A +_ imp +ph is +s ay +Ġretail er +.N ODE +/ maps +_L AST +ĠCh arge +_g uard +Coll ider +ĠStateless Widget +": [" +(" ../../ +iox ide +ĠS und +Ġ'' ; +un set +add Widget +л Ñİ +el les +alk er +A rc +Ġded uct +G UILayout +ĠV illa +Ġfor bidden +_ where +Ġ\ / +ĠT ib +_A X +] čĊčĊ +ĠB ir +Ġb end +ĠMA KE +ĠM ET +Ġfut ures +Ġweight ed +"" "čĊ +Ġauthor ize +(pro gram +}, {" +Ġcoeff icients +ê s +Per Page +ĠBath room +ĠPublish ing +G PL +Ġsub missions +ĠNUM BER +j Äħ +Ġaddition ally +em pre +ĠSh el +ot yp +S olution +Ġth under +_ ec +ĠĊ ĠĠĠĠĊ +ĠF ellow +Ġk ay +Ġnew State +ONT AL +Im plementation +.L ook +Ġ ents +Ġl ors +ĠB IG +f ab +Ġaver aged +ĠFe edback +ĠW ells +Ġm artial +Ġind ul +ĠComm unist +ĠFore x +ĠAgricult ure +" [ +Ġqu ar +ĠK ont +ĉ view +. Bytes +des ktop +ĠM akes +akes peare +.Null able +Ġspot light +V B +ow y +(t orch +tr idge +_b ounds +Ġapolog ize +.add Item +ant d +* );Ċ +, u +(g en +ç» ĵ +re ator +ĠC ord +ou pper +.m etro +Ġ ew +ĠW ORD +.A fter +Ġdet ained +ĠHam mer +ex isting +Ġo st +Ġmon ument +-c ustom +User ID +ĠN om +Ġre jection +(d im +Ġsingle ton +ĉd ie +ari ance +re ports +] != +eld a +Ġpreval ence +_reg s +." . +Ġfemin ist +Code c +Ġ **Ċ +(label s +_M ARK +FA ILED +Ġadminister ed +W N +ĠĠĠĠĠĠĠĠ ĉĉ +Ġn oun +w ig +Ġg otta +Ġr if +- im +ĠPaul o +ĠCommand Type +] ))ĊĊ +-z ero +Tr aining +Ġl ord +_ art +re ddit +C ert +Ġpes o +R ot +Ġend anger +.d r +user Info +un ts +n v +ĠTrail er +-f irst +(m ake +Ġbenef ici +-bl ack +i ÃŁ +Ġund oubtedly +Ġm ex +ĠAnc ient +( as +Ġdes cent +P ick +Ġrep lica +$ obj +ä hr +Ġar rows +ft y +ĠLib ya +ug a +charg ed +T ur +Ġh omic +iss en +ĠF ake +Ġbe ers +Ġsc attered +( Time +UT IL +Ġbureauc r +/pl ain +Ġstick ing +FA IL +ĠC ovid +Th ird +_p resent +ĠPier re +Ġë ª +Ġ[... ]ĊĊ +Pro b +ĠTra ffic +ica o +do ctor +Ġ), ĊĊ +T abs +al u +ï¼ļ âĢľ +Ġinher ent +_N o +rit is +ĠPro of +.b asename +ä¼ ļ +Ġch im +ĠProt ected +c rit +Ġpr one +Ġк он +ĠHero es +Ġan xious +Ġan os +Ġweek ends +Ġs ext +Ġredu cer += UTF +h alf +ĠS aw +.m m +Ġnue va +.current Target +.l ua +_EXT ENSION +ĉ reg +ĠC trl +_ align +accept able +Ġrush ing +fr ac +Ġbo asts +F ive + ± +ĠTem perature +> ): +Ġchar ter +RE ATED +Ġsubject ed +Ġop c +health y +使 ç͍ +ĠScient ific +Ġfra u +ri ages +à¸ Ķ +.in ventory +ation ale +M ad +min utes +>> ();Ċ +ĠEn v +Ġrecord ings +Ġsusp icion +sql ite +ĉ read +ãģ ¦ +Ġwor ries +.put String +ĠSh anghai +( uid +r er +ĠvÃŃ de +") : +Ġmethod ology +Ġк оÑĤоÑĢ +cc c +av ad +Ġindu ction +ĉ Thread +, string +ạ i +neh men +u ition +Ġ* __ +.em f +Ġì ľ +/th emes +ĠN ine +. One +ĠEm bed +Ġf az +u ations +Ġpriv ately +Ġl ing +[ F +ush i +Ġlaunch es +( KEY +G MT +Ġaim ing +pat ible +ĠB iden +i w +ĠD egree +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġ$ ('< +á rios +to UpperCase +ìł ľ +ĠE UR +Ġovers ight +Ġtable sp +Up dates +.m akedirs +Ġhum idity +/ template +Al ways +( IS +_c ert +D ig +Ġunder way +ort on +ĠHur ricane +Ġsp ends +ĠSeg ment +Ġfl ies +ĠT oggle +ĠLyn ch +Ġs enses +ĠK os +set Enabled +ist ically +Ġtest er +Ġadministr ators +Ġtag ged +Ð ĵ +Ġshort cut +ĠRes olution +Ġsuperv ision +ĠAsh ley +Tr acking +ul atory +and el +ist en +Ġun re +(d iff +ANT S +Ġr ider +Ġs Äħ +.S eries +_ orders +ORIZ ONTAL +Ġret ention +ãĢĤ čĊčĊ +Ġdi agonal +ĠC ancellationToken +_ Internal +Ġru in +.Q t +ocr atic +T el +ĠAn swers +m atic +Ġx p +at em +_j obs +_ any +Ġsen iors +Ġland mark +ĠQ List +Ġman eu +ot ify +/ ";Ċ +/ server +ĠPhil osoph +uten ant +( io +h z +Ġauthentic ated +d v +- Compatible +Origin ally +, function +ãĢĤ čĊ +ĠRepresent ative +as ily +irc uit +.d t +(m ath +.M arshal +[ , +ĠC ities +_ turn +| )Ċ +Ġcant idad +al ter +ĉ ui +ĠNe braska +Ġsk irt +.b g +Shared Preferences +( style +Ġg rief +g ew +Ġsaf eg +ol ang +_l ists +ì Ľ +Ġgran ite +Ġhott est +.j dbc +.C ustomer +Ġâī ¤ +Ġwa ar +_sc ene ++' / +ĠJ TextField +Ġse ating +Ġwe ars +Ġ` / +C ases +ĠY outube +ı m +Ġbal con +, G +Meta Data +- price +SC R +Un ity +Ġtr unk +={` ${ +Ġearthqu ake +Part ial +Ġsub st +Ġelim in +=" '. +//* [@ +Ġsuperv isor +vro let +_ article +Ġp ane +b io +Ġmot ors +N M +F rank +Ġon ion +- word +Item ClickListener +Ġb rit +end encies +Com puter +_r unning +( day +- he +(n amed +ĠS ach +о Ñĩ +c ampaign +.Ab stract +(w rapper +.p ay +Ġu w +Ge o +r ails +/ select +icht e +son s +E VENT +Ġal iment +Pro viders +A wait +_INTER VAL +. off +Ġgl uten +_cl oud +Ġw en +.ex tract +ĉ button +/ MM +Part y +Ġdem ographic +_err no +Ġh iking +(' ')Ċ +", @" +Ġw it +r á +olog ie +ĠSt yles +ĠBrowser Module +.Request Mapping +ic ans +P AGE +cre ation +ĠF erguson +ud ed +num bers +ĠGT K +Ġpresent ations +ĠB obby +_s pan +est yle +Ġilleg ally +abel a +Ġbattle field +cap acity +ter ror +] ");Ċ +Ġwar rior +le ader +ĠDB G +ĠRe venue +Ġvig il +Ġcounter parts +( Error +ACT ER +Ġhe eft +Ġselection s +ze ug +t om +-t wo +. ;Ċ +_st atement +ĠA id +ĠV ul +_r gb +Ġpr izes +Ġedit able +ĉ form +ın ı +.de cor +D emo +lic es +Ġen ctype +rat ulations +ĠR OS +_ch ars +ĠJ ahr +part ial +Ñĥ ÑĤ +ĠRe ceive +ĠL ands +AP TER +Ġch opped +.. " +ĠAn aly +ĠU ID +ĠR adeon +ĠB ee +Ġun m +> M +.find all +Token izer +ĠWH AT +Ġs j +D rawing +E ss +ON D +Ĭ ¶ +(p acket +âĢĶ but +Inv ocation +ĠN uclear +? ;Ċ +Ġgrand es +ĠC rypt +rem ark +Ġ'../../ ../../ +Ġin ability +m agic +c ats +Ġsim ulate +: ${ +in flate +Ġen er +: NO +ip les +Ġmer it +ĠR ated +Ġgl ue +/b log +Ġg ren +Ġthr illed +.C H +unc an +ĠPR IMARY +Ġper sec +Ġfe ared +.M IN +ĠThe ater +é Ĵ +ategor ie +æ® µ +Ġappet ite +s quare +ĠAlex and +.User Id +_g t +_ enter +Ġgradu ates +Fragment Manager +Author ize +-N LS +(M y +Ġtri umph +ust ing +_PARAM S +Char acters +(: ,:, +_B UILD +M Hz +Ġwash ed +Ġun cle +Ste ve +ard own + ${ +_confirm ation +Ġtro phy +Work s +ĠElect ronics +ĠMediterr anean +_m etrics +Ġannounc ing +ĠD AY +_pro to +Ġp ear +base Url +ĉĉĉĉĉĉĉĉ Ċ +Ġcoord ination +: N +.an imate +ĠC otton +_h it +â ľ +Ġjet zt +if ter +(f ields +own load +ific acion +.c uda +ĠLi u +> equals +ĠA ce +ÑĢаР¼ +ĠSuper man +ĠGarc ia +Ġarrest s +ag ar +Ġ{} ) +Ġmac ros +rou pe +ê tre +Ġtw isted +str uments +_ (" +_ vertices +ĠTrans ition +и к +[ max +m ind +Ġaccess Token +Ġun le +m us +c op +ĠF actor +Ġcon ced +Ġre tr +.l inalg +-s lider +ob l +_Static Fields +Ġz ombie +s elling +Ġch ap +Ġsh aking +ĠTrans late +ĠAm sterdam +ĠE TH +_EX TERN +k d +_d isc +Ġpreced ing +Ġpri x +Object Name +_mod ified +ard ware +Ġ?> "> +ĠD W +` ${ +Ġ?> ">ĊĊ +Ġspin ning +_p ending +Match ers +. Keys +ĠP V +en us +ant is +Ġdisc ard +Ġh aul +Ġem pir +Ġpath way +Ġo ak +м ен +-ind uced +Ġimp air +ĠCal gary +.is Hidden +d z +_ include +Ġg m +Ġ' (' +P Y +uggest ions +Ġcommod ity +c ro +/ sub +Ġget Instance +ĠLeg acy +ĠK il +B al +( short +In form ++ x +* r +ĠHope fully +or ate +Ġmach en +Ġtreat y +ĠO ri +.p ublic +-h orizontal +Ġtact ic +Ġb ord +w ares +Ġam mo +ĠL ists +Ġequ ations +/ her +ĠNS W +B ounding +_C ollections +Ġav ail +.Drop Down +è ° +Ġh h +Ġl Ãł +.p b +Ġmemor ial +ĠAT TR +Ġexhaust ed +Ġt sp +ĉ redirect +Ġlik ewise +ST ER +L java +Ġcondem ned +oca ust +(str ict +Ġexem pt +Ġs ms +Ġex agger +S YS +Ġl ounge +: ^ +Ġto dd +de b +ator ial +ĠPort er +Ġtu ition +Ġexem pl +Ġp aren +.line To +Ġkid ney +Ġç a +Ġc ui +ï¼Į 请 +X C +Ġmo ż +Ġnomin ated +l ung +Im Gui +ĠB uzz +Ġstere o +port al +res as +Ġk lass +Ġdraft ed +Ġproject ile +/g pl +(param eters +* )Ċ +Ġassist ed +ĠNS Integer +s itemap +:n th +.View s +.Argument Parser +Ġme er +z ier +ĠD ig +Ċ +Ġpl ag +p ine +Ġblank et +Ġ: - +Ġl cd +------------ --- +(" " +Ġtact ical +ĠRon ald +ex tr +ĠF est +Ġf uer +-n avigation +Ġk b +gh ost +Ġhandle Change +_cl s +() != +Com parator +.v m +ĠCo x +_re view +/ @ +_c ookie +Ġrecogn ised +ld ap +Thread s +ĠSex ual +ĠB earing +(S QL +Ġx r +Ġth igh +URL Connection +ĠSU V +Ġm Context +Ġinc idence +ĠE ste +.s up +_t e +(EX IT +C MD +/ "> +Al most +ĠU ne +Ġand eren +ĠSingle ton +Ġb ore +Th ink +Ġn arc +] initWith +_sh op +(str ategy +! ', +her its +ĠDes k +_m achine +.net ty +ı nda += < +ĠQ R +ĠS idebar +.split Container +Ġon Success +Ġmon key +En joy +(n odes +pect rum +Ġ(* ( +ĉU INT +, height +ĠNetwork s +.t ail +.l inspace +Ġ" ... +List en +Æ ¡ +.Ch annel +- defined +Re peat +ad just +ER M +_ application +.assert NotNull +- stream +Ġr abbit +Ġposition ing +Ġw oke +Ġf ing +Ġmulti player +Ġregister ing +un til +Ã¥ n +( :: +uss ions +Ġpot ato +ĠE quals +.S up +/ap ache +Ġ( = +. ") +.p tr +ĠSpe ech +.cl ip +ĠGab riel +Ġmusic ian +/ issues +.sh op +ĠH ier +_RE T +_b ucket +ãĥ ¡ +av s +Ġro z +fl ower +Write Barrier +ĠMil an +Ġlegisl ature +ĠD oll +Ġprov ing +.concat enate +âķ IJ +Ġg char +cdn js +b les +ĠList ing +л о +.xr Label +ĠS ak +just ice +ĠVal entine +un less +Ġp iger +(r un +Ġtest ified +AN A +ĠRem oves +)) ));Ċ +rec ated +ĠRuntime Method +Ġcon qu +ãĤ ¢ +Ġt issues +ail er +ét é +- Star +Ġfl ames +.set Icon +Ġsup ern +Ġvag ina +- variable +Ġwell ness +C UR +Ġbel le +.get Request +Ġp oco +ben h +ag ens +Ġsp ill +ĠJ ur +Ġdispatch er +н ого +emon ic +(dir name +ĠÐ Ķ +Ġpas se +Ġg anz +ric ing +E U +Ġmuj eres +ess en +.at tribute +j j +ĉĉ ĠĊ +[ ^ +Ġstrtol ower +lex er +ect ar +hot el +.s quare +Ġr all +Ġlower ed +hand led +Mark et +ĠUs es +iv as +.B usiness +ãģĹãģ ¦ +D IV +Ġw asted +Ġav oir +ê m +_ACC OUNT +. et +ĉ SDL +k ap +Ġf ox +up pet +{ },Ċ +", ' +F avorite +P END +ĠA ES +} ), +Ġded uction +Ġpol ÃŃt +Ġcomponent Will +ĠT elerik +_SE LF +Ġm use +C raft +Ġd ens +ठ¿ +( tp +Ġt asty +Ġbal ances +Ġded ication +ĠWall ace +Ġun law +\"> \ +Ġm um +- update +ement e +Ġs oda +Re public +as mine +é ric +( Status +ĠJson Convert +ĠD isk +.Red irect +Ġfilm ing +/m ol +R o +Ġv ille +Ġtrab aj +Ġsyn thesis +reg a +Ġr l +S cheduler +ISH ED +current User +(error s +' h +_b ot +x imo +ĠUS ART +_s uper +_DEC REF +н ой +_RO W +Ġprom otes +ĠT A +Ġhor as +ĠRep resents +Ġname of +ĠEx c +ĠGar age +Ġse ine +, # +Ġher b +/ resources +Ġple aded +.r adioButton +Ġæ ĺ +O ps +ĠN est +c string +ĠDef ence +Ġref ere +_le af +Ġrevel ation +ë § +.execute Update +_W ORLD +Ġexp ans +(" \" +j ab +Ġdoub ts +ĠGe ometry +Ġintrodu ces +Ġsen ators +Ġcan al +.h elper +ĠBi ology +_SE NS +.pre vious +-t ouch +ab it +Ġimpact ed +Ġbr ackets +.d irect +acc um +Ġtest osterone +ĉ action +ĠCh ance +Ġpe aks +CppCodeGen WriteBarrier +Ġun belie +_p ress +.R el +ang led +/ templates +-- >čĊ +l ime +Ġsufficient ly +_ nt +Exp and +.is file +Ġis Empty +Ġq t +Ġmul her +ac ob +Ge orge +å¸ ¸ +Ġass im +as o +Ġcompr ised +O V +(CON FIG +ĉw riter +Ġdes p +Ġten ure +(c r +.p ool +ĠB rend +Ġc ensor +(time out +Ġple a +.W rap +Ġtight ly +ĠW ere +ĠI gnore +abe i +Ġbr idges +Ġcondem n +Ġsimp licity +Ġrout inely +Ġblack s +j b +ĠP it +U tf +Ġ/ Ċ +re load +Ġset Object +/g lobal +Ġf atty +Ġsock s +Could n +Ġerot isk +æĿ ¡ +ĠPress ure +ĠM az +n pos +tol ower +ĠE Q +ute ur +ĠM oment +Ġet a +{{ -- +Ġgraph s +ĠGu ar +r ine +( -- +ĠHttp Status +(st udent +* np +Ġrail way +Ġas ynchronous +_v m +'] ,' +, text +mer chant +(G uid +ĠG ra +ix er +fetch All +.add Listener +fl ip +* $ +> (), +Ġsun light +ass igned +Ġab c +ĠC OLUMN +ĠðŁĻĤ ĊĊ +) ... +Ġen semble +Ġnew line +_S INGLE +ied ad +Ġdark er +orm ap +Ġl ion +pl its +Ġillustr ation +ĠI EEE +Ġv ista +ous ands +****** * +ĠTom my +Ġh ue +S el +Ġa ura +ĠTher apy +Ġanim ator +.con straints +Ġv ague +(" ") +Ġvill ain +Ġbless ing +Ġstring Builder +ĠM isc +ĠD IR +f ax +- node +ĠWalk ing +ĠA U +s ess +Ġgr ill +VERT ISE +ĠF oods +Ġt ournaments +à ĵ +ĠMar sh +Ġw onders +Long itude +.Command Text += input +_enc oder +page Size +Ġget State +> >Ċ +.g rey +p od +Ġread ings +Ġre consider +Start up +Ġexc er +.b alance +_c ycle +_T ime +LOC AL +ĠE FI +ĠRe yn +.set Foreground +by n +Ġdis connected +ACT IVE +Ġembed ding +ick ers +Ġsurround ings +* c +Ġgar ant +Ġb f +Ġw ipe +Ġ ä¸ĭ +_T RA +ado x +ç ķ +Ġsu cks +ĠS ongs +ĠAssoci ates +ĠB ald +ĠB rett +ven ile +Ġv t +Ġin ade +Ġres igned +ĠGl enn +.p attern +.Data Bind +Ñĥ м +Layout Inflater +ch et +ĠTest ament +.m s +Ġp av +ĠReact DOM +ur dy +AD ATA +M u +/ actions +ĠJ s +_ex tract +ĠBr ing +: id +str t +iv ation +Ġoutr ight +az u +loy ment +и Ñı +al do +ĠP ublisher +E ducation +Pa lette +_d rv +Ġ($ ( +ĠAnd a +Ġrem edy +Ġincons istent +te ction +Ġregul ators +Ġshort est +(p air +ĠInstall ation +Ġdefend ants +Ġ( ); +-l arge +M el +Ġthreat en +н Ñı +Ġfet ish +ot ine +_d ic +Ġ< $ +Ġst agger +sp i +$ response +S erv +-b orn +j os +ĉ img +ĉW HERE +_l t +å½ ĵ +.c ost +ĠT ue +.label s +ĠL V +wcs store +ĠJes se +ภ« +Tr ade +Ġpredecess or +ë Ĥ +fin ally +_g eneral +ogg ler +_REG ION +n ement +Ġblog ger +ĠHar bor +ĠD ataset +[ w +Ġattend ees +. ico +max imum +.Un lock +_SY NC +ág ina +Ġdown s +ĠW ii +]) / +Ġkick ing +unic ation +ĠD AC +ĠID S +ĠR ental +Ġcurrent Time +Ġvacc ines +ĠDev il +Ġn ors +_m ouse +urre ction +(n o +Ġ> čĊ +Ġaggress ion +Ġbre eding +.s ymbol +im an +Absolute Path +ĠWH O +_fl ush +- root +arn a +& M +Ġf athers +ĠR ocket +ive au +Ġw ander +Ġcom pos +ĠWar rior +ĠSe at +ĠClin ic +_in voice +(dis patch +Product o +at uring +oss ier +ĠM AY +Ġd agger +Ġsanit ized +ĠR FC +Ġpro ph +Ġur ine +Ġgr ind +ĠExp anded +des cripcion +-f w +ĠK erry += name +Ġch k +Ġnation ally +Ġthe e +In c +Ġ? >> +.R adioButton +.Http ServletResponse +/ Y +ĉf ield +Ġhom me +y per +Ph ysical += v +Ġdr iv +ĠErr ors +Ġc Äĥ +De ath +ĠW INDOW +Ġpo et +ĠSh arp +ĠImm utable +ĉ create +Ġge ht +ĠRe form +ais er +ĠInitial ization +Ġimm unity +.com pose +Ġlat ency +ĠLeban on +ĠPar ad +Ġfu els +ĠEx hib +co h +% ">Ċ +ĠCL I +) initWith +-Z a +_C LEAR +reg n +Ġfin ances +.st andard +_C ATEGORY +.lib rary +Ġtravel ers +_w p +ĠE valuation +start ing +Ġ )),Ċ +ep isode +ĠV ariant +Ġda emon +ĠJul ia +ĠN R +Ġdoub les +< v +/r untime +Ġinterpre ter +ĠIN DEX +ĠHol mes +_D IM +Ġp addle +_ex ample +Ġfore ground +.r outes +Ġs owie +S UCCESS +ĠC DC +ĠB D +_ - +as ured +W riting +Ġcurrent Page +( answer +ĠASC II +à ¨ +Ġsocial ly +yy y +ĠSpecial ist +(c ustomer +ist ani +ke st +ĠM ak +Ġth o +. pt +( comment +ĠCon verter +g am +b ins +. tele +ĠVeter ans +_AL LOC +олÑĮзов аÑĤ +inn amon +; width +oh l +Ġfant as +Ġs ung +ĉ K +( Json +Ġneighbour hood +Ġv ow +Ġs ins +on acci +Ġepoch s +im agen +.Ch ange +.my batis +Se ek +W ER +管 çIJĨ +Ġinter ess +_ Event +eder land +Ġterr itor +Ġci udad +uck ed +Ġsn ack +Ġtransport ed +ĠMan ifest +ĠD AT +_th eta +Ġw ont +.ĊĊ ĊĊĊĊĊĊĊĊ +Ĭ¶ æĢģ +ĠEp ic +De ck +l tra +_Z ERO +Ġ[] ; +/ scripts +Ġ---------------------------------------------------------------- ---------------- +æĥ ħ +Ġwe ed +N BC +Ġrap ed +ĠG ateway +[ M +ĠTime out +ench mark +.View Model +Ġporn os +ĠY a +th ritis +ĠFly nn +Ġme ga +ac in +Ġtrib al +.app le +ĠB lo +â n +ib i +ro v +ĠL ives +^ . +get Request +ĠEst ablish +cont ainers +Ġst arring +Ġcele brities +ĠRel ative +ĠHe ights +Ġtq dm +ĠNorth west +iv ic +ĉ cl +Ġautom otive +ent ric +Ġfort unate +Ġfire place +se ud +nick name +; s +_C AL +h alt +(n s +_de leted +Develop ment +m ovies +Ġident ities +Ġprompt ly +ا ÙĨ +Ġant e +Ġ" ',' +åı £ +imp se +Ġy ap +Type Name +Ġb itch +Ġassoci ates +HE ME +- empty +ĠØ ª +ol vers +Ġpist ol +Sc oped +ag ner +'] ==' +ĠI MP +ex c +Ġo mitted +Ġmind set +Ġ[] ( +Ġor n +_C AM +A vg +Localized String +ĠN atur +Ġcom poser +ĠPlay ing +Ġover d +_ utf +.s k +ĠF ol +$ page +, Object +Ġbe es +al ary +bul let +_lib rary +O ffer +loc ated +Ġ(_ , +âĢľ He +ĠOwn ers +) ).Ċ +Ġb ri +.Ad min +kt ion +лÑİ Ñĩ +Ġerot ici +Cancel led +Ġa gr +re views +_d ma +RI CT +Ġg fx +mp i +pp o +Ġ// @ +Ġupper case +Ġcommit ting +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +User Data +Ġv ai +ĉs ort +Ġcongr at +Ġd ioxide +д а +. area +ĠJosh ua +ĠK och +_b reak +az ure +ist ical +_AL PHA +_ views +Ġelim inating +OM B +en umer +ĠHy dro +(* ( +ERT ICAL +Ġinev itably +Ġst ole +-e ast +ier on +Ġl inger +/d oc +Å º +ĠAl ready +as io +Ġ-- Ċ +Ġabb rev +ĠAt om +h im +ĠINS ERT +s un +âĻ ª +CON NECT +er ator +ĠM anning +Ġ: ( +g as +=> ' +Ġquery set +; }čĊ +ĠPop ulation +uted String +res ident +_F ONT +ĠRes pond +Ġobsc ure +Ġo bservable +ĠContrib utors +k on +ĠMus k +ex ao +ĠT ub +Boot Application +S OR +.H orizontal +.find By +.p ower +Ġposit ively +ven ience +ĠJ ong +Ġwh istle +Ġз наÑĩ +Ġl ending +Ġdestruct ive +Ġon Delete +author ization +(); ?> +_ original +sc ience +at ra +?, ?, +ĠAs c +Ġconvinc ing +$ a +org en +_D ate +ĠPro vide +Ġlon ely +) 'Ċ +ex change +; ?>Ċ +.f ast +S amples +L ondon +'] )čĊ +ĠI onic +Ġp esso +ĠKn ights +ĠR af +_attr s +Ġrepe al +> Main +ĠOrder ed +_N ew +=" "> ";Ċ +ĠS ERVER +ĠHE ADER +_ velocity +ĠIn voke +.timestamp s +Ġs ulf +I QUE +Ġinhabit ants +ph ins +azz o +Ġmon o +Leg end +Ġnon ce +IF E +; ";Ċ +- create +" ",Ċ +per mit +ĠImm igration +Ġpath name +ffect ive +âĻĢ âĻĢ +Ġex ams +- event +ĠT ill +[m id +F IX +; color +( Order +_tra its +Ġorder By +Ġs unt +ĠNich olas +Ø ² +Ġsun ny +in ers +Ġaccess ibility +ĠH B +.com p +ĉ op +Ġminor ities +ethe us +Ġcollabor ative +pr it +H IR +Ġwr aps +ĉd raw +g od +ĠI X +.app s +ĠN M +Ġirre levant +ĠT igers +Ġdi ag +G V +ĠAccess ories +k ont +Ġsimpl ify +ĠF avorite +_t ools +([] );Ċ +Ġtow ers +B es +Ġhun ter +Ġsal on +(b uff +ĉ debug +Ġmal ware +M oving +- options +) +' +ĠLO VE +_S OCKET +_f in +ĠDel aware +Ġsher iff +-in valid +ĠF ULL +Ġп од +el as +" strings +ĠRepresent atives +s urface +res olved +ht docs +)) :čĊ +Ġpress ures +Ġnorm s +Ġpl a +Ġs urname +Ġpost al +ĠDep art +Ġsla ughter +or ida +Ġhe bben +Ġdes ar +comp act +_L ANG +åIJ Ī +op oly +_r ad +ĠST DMETHOD +L azy +ĠĠĠ ĉ +... , +( web +ĠP ont +Ġet was +Ġup ward +_h at +Ġ], ĊĊ +Ġbase Url +Ġworry ing +-add on +(get Class +S PI +Ġcapt uring +) },Ċ +Effect s +Ġcompet ent +Ġf oul +Ġsubscri bing +ĠO BJECT +IX EL +b ucks +( edge +(p ass +ĠPet erson +Ġbo obs +ĠD elay +_s quare +el im +ot ers +_P C +% E +on click +ĠSV G +Ġto pped +Ġf ist +sm art +ĠR alph +( owner +j ours +Ġbron ze +ĠArgument Exception +( original +_S CALE +_c p +Ġrecomm ends +.set Style +S ure +L AND +Ġrepe ating +M att +. Visibility +Ġenter prises +.Set up +(sc ene +ĠRe active +ur ge +b w +.P ut +p ersist +.c ookie +ĠAud i +` s +sup plier +( Form + ¡ +_s o +Į Ģ +ĠLeg ion +t te +N d +L oss +( attrs +.sc atter +Ġg room +Ġgl impse +Ġn ails +Ġcum ulative +Ġf azer +_s ervices +.N um +ib ilit +_res olution +ĠT x +umin ium +op a +.s chedule +sm tp +ภķ +ur ry +ü k +go og +_sign ature +.int o +ĠSte ps +Ġhome owners +ĠNS URL +ĠP AC +ĠĠĠĠĠĠĠĠĠĠĠĠ ĊĊ +> ')Ċ +en h +Ġinc ap +$ MESS +Ġmo ins +ĠF i +Ġoff season +press ions +> .Ċ +ĠGr ass +ĠGo al +_p df +Hand lers +Ġstack s +.get FullYear +=[ ];Ċ +è½ ¦ +, V +(s plit +Ñĥн к +Ġbake ca +Ġ~ /. +pe z +t ails +ĠG len +Ġset Image +ĠCom ic +B LOCK +ĉ This +o ader +Ġcapital ist +_ST EP +( Boolean +ĠCor rect +r ina +Ġconc aten +å® ŀ +() :ĊĊ +Ġun anim +ll i +al ars +- ne +Ġdiv or +ĠKick starter +]. _ +< number +/m enu +GR APH +vis itor +Ġimpro per +_N EXT +Ġb isa +background Color +/ input +Ġmo i +Go al +li qu +Ġmiscon duct +Ġcompr ises +aw ns +ĠP ie +ra is +role um +Ġcur se +y u +_p oll +.current User +ES H +]) [ +Ġstory t +)? ;Ċ +* = +ĠB urg +/ layout +_back end +; ?> * '+ +åĿ Ģ +ac ency +( URL +_h alf += l +Ġlist View +( section +.to Array ++ / +ĠRodrig uez +ist ream +Ġelig ibility +:: - +.new Instance +P B +ĠAs sets +ĠCom posite +ĠL abs +ĠHam as +++ );Ċ +Ġbl k +ĠNe o +L uc +@ login +Ġun aware +.m et +_RE LEASE +( ST +AM IL +ri ke +Ġ( ){Ċ +(s printf +ĠAccount s +ĠV IEW +ĠA j +ãĤ ° +Ġwh isk +Ġid i +Ġro de +Ġih n +ĠElement ary +Q ty +Ġintrig uing +Ġå ¤ +J obs +ĉ offset +ĠAh med +ĠTal iban +Ġè İ·åıĸ +Ġinject ed +.Auth entication +_line ar +.Dec imal +Ġapp les +Ġshare holders +Ġb aked +.d iff +ĠE ddie +ok ers +Ġconfront ed +vo ices +Ġt us +ĠSp in +N ODE +_ Un +CT X +/g oogle +Tem perature +Ġ' '). +Ġmagn ificent +Ġstart Index +semb les +Any one +z k +eh en +ĠD ame +. strict +Ġrepl aces +Ġline back +Ġpush es +Ġche ek +ĠSh i +_BY TES +RE A +ả n +_CON NECTION +G ateway +ĠTr avis +ĠA X +ĠBas ically +ĠUp grade +à ª +th emes +erm o +k or +F emale +_att ach +ĠìĤ¬ ìļ© +Ġpo z +============ ==Ċ +(s ymbol +ĠS ector +__ )ĊĊ +_p adding +ï¼ļ " +Ġf abs +Ġr anged +set Name +Ġp error +â Ĺ +ĠFile Reader +Ġful filled +_C urrent +Ġdom inate +Ġsm ugg +Post Mapping +_for ce +Ġb loc +ĠG iant +(v ideo +ĠC U +System Service +Ġ elf +Ġkont akt +ë ª +ke es +gt k +Ġparam Int +Ġmark up +u ales +Ġaccount ed +Ġgang bang +RY PT +ĠW rong +Ġcred ited +ĠM ESSAGE +Ġfl aws +Ġbb w +Ġmetab olic +ĠO EM +/ event +(C ollectors +mont on +ap pear +Ġopt ed +Ġche at +Ġd av +ĠPro ceed +Ġê ¸ +ank ed +и з +ans k +ĠH ang +ĠC ler +Ġdis gu +Ġc map +.cl js +Ġa ument +le z +ĠJo ined +_re ceived +Ġa erial +ot el +Ġgre et +" s +ĠGen esis +ĠCal if +pan ion +Ġtail ored +m apping +and Expect +.tr ack +at omy +ĠO w +ull ah +.Y es +ĠSimple Name +db h +' en +Ġnons ense +Ġphilosoph ical +(get Context +Ġis so +ĠA CE +start Date +Ġb ÄĻd +ĠAUTH OR +ĠGlo be +Ġinsect s +_A l +ush ing +è® ° +/ Home +ĠLocal Date +need ed +hes ive +Ġill usion +äº Į +Ġtr at +x o +/d etail +_M ATCH +Ġbroad band +Ġw al +ĠIllegal StateException +IRE CTION +Ġnor theast +es ium +ĠClient e +ul ance +nt y +Ġt ecn +Dev ices +Ġgr ains +ĠO g +ĠS EL +ud iant +Ġ++ ;Ċ +Ġexplan ations +oc co +Ġdi ets +Ġco hort +( controller +.Iter ator +-r ich +ro cess +G D +Ġcar bohydr +Ġfri ed +ĠEmploy ment +ìŀ ¥ +ĠLeon ard +_ ${ +qu ares +Ġcompan ions +Ġpar is +Ġstim ulation +ĠZ oo +Ġre levance +ĠCol our +Ġspe ar +ot ional +ĠL ite +ĠK osten +Ġà ³ +_att achment +orph ic +Ġdam it +Ġd lg +Ġthr ive +CH ANGE +ĠApp arently +Ġat ual +Ġroot ed +( images +aw i +ari at +Ġch erry +STAT IC +m nt +ĠUser Id +il let +ĠHis panic +Ġn ak +Ġcent ro +Ġdim s +_initial ize +ı k +ĠCent ers +RE N +Ġevolution ary +ĠTop ics +_d amage +em er +Ġr und +Ġpun ished +Ġcub ic +f air +[] ;ĊĊ +Ġinstant iate +Ġover see +- delete +unte er +start Time +ĠP ipeline +_G AME +ĠC ir +ĉ Null +.Format ting +uc umber +ĠR ide +Ġz oo +Ġcheck er +åIJ Į += C +Ġg rit +"); // +_x y +ĠDe claration +Ġcall able +F oo +ĠList Item +Ġin accur +ml in +ĉ Data +Ġev olving +aw an +Ġca fe +fol k +_ID X +ĠAny thing +ĠPalest ine +ĠGrid View +Ġcol ony +ĠGerm ans +( + +.p id +.js x +ĠSuper ior +Christ ian +ĠL ect +ĉ Game +Ġinstrument al +Anim ations +д ал +ĠMos es +ĉĉčĊ ĉĉčĊ +z s +k te +ä¸ ļ +_D IST +bit map +d B +Ġp ersistence +ÑĢ Ð¾Ñģ +$ l +B ron +Ġ{ | +_ch art +ĠCon sum +Ġh emp +Ġ" ))Ċ +Ġattack ers +Ġknowledge able +Ġc et +Ġvir uses +' I +Ġpitch er +Ġsweep ing += list +apt ops +.de pth +Ġinstruct ed +ĠR us +benh avn +Ġи н +S ports +Ġon set +æĿ ĥ +. RED +_s i +ĠP ST +.on Change +> tag +ĠR oh +_char acter +ĠLaw s +ĠB achelor +_s wap +.re activex +Ġreward ing +Med ium +- [ +ĠRec ently +J oint +part ition +ĠMin utes +Ġind o +Ġabsor bed +ĠG N +_IN D +Ġsab er +Sp awn +output s +ĠJeff rey +Ġmed ieval +h ed +Gu ide +Ġpsy cho +Ġgl am +E lim +äd chen +_pl ain +ĠS au +-f our +Ġanaly zing +QU ERY +Ġtom ato +_button s +V EN +.set Status +. Url ++ ĊĊ +Ġcompl aining +deg ree +conf irmed +Ġsub t +p arsed +Ġtor que +Ġtroub led +ĠT ARGET +Ġtrad emarks +ĠCo ordinate +ĠV iv +Ġ// }ĊĊ +Ġapr ès +.get Position +(Key Code +ĠSil va +Ġmet eor +Ġendorse ment +Over view +ĠP oss +.In ject +Ġeven ly +Ġvisual ization +Ġw char +ĠH DMI +Ġfun ct +ick name +',' ',' +Ġfor wards +Managed Object +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +ĉ server +ĠOut look +ĠChron icle +Ġdub bed +Ġd ok +ĠW ear +.A L +pare n +. Interface +Inter faces +.c od +Ġd ib +.Global ization +ĠAcad emic +Ġass ms +Aut om +Ġl w +ĠN W +Ġ&& čĊ +Ġproble ma +ĠManufact uring +lim its +-m obile +Ġfil me +/ map +Ġdo it +ĠIn k +Ġsu ed +. arr +Ġunder min +ĠPro c +croll View +__ $ +Ġsidew alk +( that +ภ· +[ q +gram mar +Ġt ë +qu ito +Ġspir al +ext ended +Ġf ocal +Ġdig ging +p as +ĠT all +.pro xy +it ures +TR ACT +ĠRe alm +Ġf eder +Ġorient ed +ĠAltern ative +Ġo we +Ġsour ced +ink er +.d et +S ep +ĠQ ui +ĠPal mer +(_ , +s amples +oy er +ull an +que z +Ed ges +Ġsh out +ĠA chie +Ġha ar +_Con struct +Ġprem ature +Ġre vert +'). Ċ +Ġs chn +filter ed +null ptr +S aved +itect ure +CL A +Ġv l +st ell +ĉ Me +ĠL ip +n ational +Ġwh olly +Ġspr ings +.T imer +ĉs rc +els en +åħ ¶ +Ġcommunic ating +ĠQu iz +Ġt eng +Ġge z +ĠOut side +.S ign +(c s +Ġdisput es +ĠWe iss +ann es +> No +ĠB ach +.remove All +re fer +/d ashboard +ĠA jax +Index Changed +ĠWe ak +' "Ċ +Ġs ights +access Token +ĠJ oi +(d omain +ĉc v +Ġcontin uation +Ġpl um +ad ir +.set Message +Ġ ï¼Į +Ġsw allow +ĠL amp +Ġq w +Ġu u +C oin +ub ic +ĠDe als +r ace +Ġdict ator +Ġmem e +turn ed +ĠJul ie +.grid Column +Ġpup py +Ġp am +Ġ) {čĊ +Ġinv iting +Ġf rench +v im +Ġwr apping +Ġ#- }Ċ +([ - +Ear ly +Ġsh iny +.f aces +Ġreb ell +abc def +ä lt +Ġest imation +ph ys +los ures +_RE L +Ġex clusion +ĠSk ype +we ise +-st op +no thing +ĠE gg +is ors +Rich ard +Ġcounsel ing +Ġcomm em +ĠQ MessageBox +ĠSy nd +ĠFro st +ĠCompet ition +ĠAw ake +Ġt ed +ic iones +ĠDev Components +VERTISE MENT +ott i +.run ner +Ġuniqu ely +.fl ag +ĉ rs +_g eneric +Ġ`` `Ċ +ACH INE +Ġme in +( Application +( br +Ġrat ios +: , +ĠXCT est +ustain able +- www +it les +_T EMP +Ġs yst +umeric UpDown +ĉassert True +Ġw f +. peek +ĠBul g +Ġterr ifying +.M ODE +ĠG W +á r +Ġf ic +Ġcommit ments +- tech +ĠL iquid +ope z +z heimer +a ña +-m edia +( animated +_go al +Ġg um +yst one +.S ET +ĠW end +set CellValue +Ġmsg s +c ash +AL LOC +/ aws +Ġmic rowave +.Point er +ĉ Console +_s orted +ĠFil ip +Pro d +Ġ//! < +ing roup +Ġk s +_T RI +Ġteas poon +ĠAT T +Ġrecover ing +ĠG LOBAL +.P ar +Ġ/> ;Ċ +Ġmar ble +ul ators +ĠC ycle +Ġher bs +_m etric +) ! +_C LOCK +_ Button +H arry +è¿ Ľ +Ġstr ains +ĠApp Bar +ĠCh an +/v ideo +Ġb am +.Pro gress +$ f +lem en +Ġir regular +ĠD uncan +ĠM int +-v ideo +ঠ¾ +ó wn +ĠEM PTY +Ġstack ed +ĠH A +_c ut +Ġwhere in +ĠW ays +(count er +è¯ ķ +Form Group +Ġble w +c ourses +Ġproduct os +ry s +ĠRest r +Ġsty ling +> s +Ġp iv +Ġit ertools +get Repository +ĠI k +_dev ices +lay ui +Ġhalf way +Ġfran ç +Ġtun ing +O A +_N ode +ar de +Ġfier ce +lic ted +# čĊ +Ġbreak through +ĠE rik +Ġb ride +Ġ. " +cul us +ins ide +ĠIndian apolis +ĠE E +Ġy og +urre t +.f s +. grad +_c ards +_ac curacy +_ep i +qu eda +/ org +é ªĮ +Ġcom pte +)) [ +Out side +G reater +ĠRender er +. actor +Account s +Id le +_h ours +ern er +Jo ined +Ġmen j +requ ires +ĠO PER +.remove Child +ĉs p +Ġes se +r ift +xF E +ĠSh akespeare +________ ____ +Ġbudget s +Model State +fill able +- component +oc os +ĠBUT TON +/ io +, out +s ms +Th omas +ĠAr med +res ume +Ġrot ating +ĠV ault +Ġse us +. (* +Ġa mino +Ġ[] );ĊĊ +Ġprov oc +no x +.Get Enumerator +==== ===Ċ +æĸ Ļ +_sc roll +Ġfil med +ĠS oci +g ap +g ro +V ote +" But +_R C +An imal +Â Ģ +ib ile +Ġaw aken +ore st +in ja +ĠI van +( Command +Ġ ***** +Î · +Ġkv inder +/h elpers +_c ases +t g +ìĦ ¸ +Register ed +ĉp ass +_d igits +Ġcont our +Ġinf ants +Ġjust ification +ĠFort unately +Con tr +ĠonCreate View +_S AMPLE +Ġallow Null +Ġn ud +Ġfet ched +_e qu +ĠUn able +=\" " +> {Ċ +Ġcommit tees +ist ema ++ ". +ÃŃ an +m ant +Ġsou theast +ï¼Į Ċ +dialog s +PRO JECT +charg er +- port +(u uid +. export +S ix +ĠR P +P rem +Ġconsc ience +Ġmargin Right +_d istribution +y aml +res izing +D ock +ĠLoc ations +G Y +Se ed +B UFFER +oss ip +ull en +Th ings +- self +.p oll +PL AYER +Ġå ® +G ROUP +ĠA way +Ġg ospel +xf d +M ary +ĠPort able +T URE +Ġutil is +Ġse it +Ġstr and +Ġtrans c +Ġ( ^ +ĠAl fred +.m em +.c ircle +Ġ~ / +for cing +Ġr iot +pro x +TH ON +iz ación +ĠN I +ro st +Ġdis pro +_in stances +ï¼Į âĢľ +ograph er +end as +ĠIsa ac +ĠP ine +/d is +Ġcolor With +iter ate +_str ide +Ġpun to +.Event Args +( center +Ġneighb oring +ĠPr ison +ĠMess enger +Ġepid emic +da o +_com plex +Ġgr avel +_D IP +é ment +ĠA ri +_bit map +.qu it +( valid +Ġp end +Ġrespir atory +Ġre bound +Default Value +ãĥ Ń +Ġcomm its +.test s +_f r +it et +.s f +Ġspace craft +c ritical +Ġde pressed +ĠAny Object +Ġun b +Ġdisc ern +(m ysql +L atin +ĠB og +ĠWild life +To File +iox id +@ RestController +Ġ"$ ( +Ġ<< " +Ġdefect s +Ġdat um +h in +Ġreal izar +any ahu +ĠS ig +@ Data +ad aptive +ĠC atherine +.c r +ĠCO OKIE +Ġp ictured +ĠFight er +Query able +ĠAny way +ĠGL FW +_n amespace +_ ft +Ġ] ) +Organ ization +Ġconstit utes +Ġqu and +(ch unk +"/ >čĊ +ĠL akes +main window +Car thy +sp in +(c sv +: red +-com merce +ภ¹ +Ġdiscover ing +Ġe co +_f ac +inc eton +ĠGre ens +j wt +Ø µ +ĠBron cos +ĠGood s +(G TK +Ġreturn Value +Ġsi empre +Ġneut r +w ent +ĠN atal +Ġenthusi astic +á» į +F N +/d atabase +C atalog +Ġbr un +ĠK ash +_P l +isc rim +, width +Ġin mates +Ass ignment +ĠH aven +Ġplay ground +ex am +@ Controller +ul iar +.get Parent +Ġ" ;ĊĊ +: size +iss ors +Ġf is +Ġal c +ens ation +ĠN ixon +Ġmight y +- str +_s pecial +_A DC +ĠTw ig +um bling +- address +Ġher oin +Y TE +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĊ +F riend +Ġa ve +ĠP NG +ĠKurd ish +DataSet Changed +Ġbl ades +br al +St eam +Ġsig u +IRT UAL +ac os +UD P +(d atabase +he c +ĠString s +_scal ar +ĉd esc +ĠT LS +; "Ċ +ĠCor byn +Simple Name +u ell +ĠEnt re +ell ites +- place +Ġfrank ly +ĠE rf +CE L +Ġpa ÃŃs +Ġh edge +Ġlat ent +ĠIR Q +ĠH erald +ĠP rec +ë³ ´ +.T EXT +Sal ary +Ġaut umn +Ġtrav ail +.S um +Ġc ared +M or +Ġint uitive +Ġj ournals +_ IT +ĠT rou +ä¼ ł +Has ColumnName +Com posite +Ġsp ice +_d isk +_CODE S +ĠInt roduced +ion a +Ġnue stra +o ct +ĠĠĠĠĊĠĠĠĠĊ ĠĠĠĠĊ +(param eter +Ġstud ios +Ġproject Id +Ġbd sm +.Sql Client +im izer +ĠC ARD ++ t +a an +.s ol +_Ad just +Ġright eous +ĠLog ging +.f ilters +_T AB +ĉs ys +roph ic +other apy +ĠB rowse +key board +R ON ++ \ +ro pped +Ġext ensively +f k +Ġl ime +year s +Ex c +Ġs ph +Ġche ating +and ro +ÃŃ o +Ġpr ince +o ire +ĠD estination +ĠConvert s +Ġup stream +o led +Ġserv ants +Ġsem antic +Ġcr unch +Ġevent ual +run ner +/ error +Sp in +Ġsecret ly +Ġas semble +.P erson +end error +_ < +Ġp endant +S leep +ĠChem istry +Ġboss es +l k +)) ),Ċ +Block ly +DE VICE +Ġreflect ing +Ġam ple +Mill iseconds +ĠPresident ial +Ġus uarios +ĠN Z +ĠSal ary +ĠA manda +_n p +j ury +Ġkö n +Ġtherap ist +Ġhomosex ual +ĠDr ake +-w indow +ĠLoc ated +.D river +ĠV IDEO +Ġmerch ants +ĠC hest +- lock +/ php +Ġmil ano +_ST YLE +arg er +ide a +G UID +adv anced +me al +Options ItemSelected +=' % +ĠCh am +: data +(st at +Will Appear +Ġinform al +aj i +Ġre productive +ĠC AS +ãģ £ +F UNC +ĠR uth +)+ ( +CON ST +ĠF ans +Ġgroup Id +xffff ffff +Ġsam pler +Ġ}} "> +. the +Ġh ollow +W AY +ĠFac ulty +Attrib utedString +ĠLook s +ĠR ex +j k +ĠM IL +Ġb ard +.L ong +Ġliv est +Ġsk al +ic ism +MA IN +Ġmu cho +B ODY +Ġes e +ĉ use +F oot +.SQL Exception +Ġinherit ance +re ceived +Ġput as +ed is +als a +ĠError Message +Book ing +Ġtr act +ac z +ĠC ant +_reg ex +Ġide ological +Ġj ihad +h os +/s ys +col m +(p ool +Ġest án +ĠP ending +em ás +Ġktó ry +));ĊĊ Ċ +trans actions +Ġw ield +it ere +ert ure +_s s +Ġstretch ing +Ġprison er +.Read All +Ġbes ch +-- ;čĊ +Ġcr isp +_SC AN +Ġa e +Str ict +ĠMin neapolis +ĠBo eing +ar is +re k +_p ipe +Ġpri ests +(E IF +eh icles +ĠInter active +b etween +ĉNull Check +ĠBl air +ĠL t +_in line +eth yl + ¼ +_p ackages +Ġbarrel s +_ he +Ġreg exp +_ pts +_H andler +ing ular +ĠN issan +ĠR anch +Ġper ch +Un supported +Sm ith +ĠLeg ends +M i +Ġg f +st eder +Ġacqu iring +Ġsim ulator +() ," +re ceive +Ġin place +A CTION +ĠWeb Driver +files ystem +< Order +lo pen +ĠHE IGHT +.set Border +į ° +__ [" +Ġcl amp +Seg oe +b ands +to List +amb a +>' +Ċ +Ġcred ible +am at +play ing +.setImage Resource +qu el +Ġpod r +ge om +E k +ĠQ atar +Ġg eld +? ',Ċ +Ġc yl +( ax +ĠW I +ur ally +ĠBr asil +Ġsen za +ale y +on en +Ġb ah +Ġmolec ule +R ad +è¿ ° +AN CH +- background +- agent +Ġprol ifer +: boolean +Ġt ide +erial izer +_ ;čĊ +F ee +** ) +erg y +ĠHon or +.Log ging +ir is +Ġunder mine +ĠD y +Ġt yr +Ġde que +Ġdam er +([] )Ċ +.layout ControlItem +pe ated +C AN +rag ments +L and +) ]);Ċ +ĠS ah +ĠDE CL +With in +ĠN amespace +an other +sem bling +.des cribe +Con sum +ĠF ear +g iven +Or ange +< boolean +Ġstead ily +pa Repository +Ġresult Set +_ ENTER +_re peat +Ġt ones +ĠPRO P +n al +part icle +Ġsign aling +Ġaccess ory +ĉĉĉĉĉĉ ĠĠ +Ġvie le +ĠNo ah +- ag +Ġmur ders +Ġa ired +ĠPL AY +ĠS ullivan +_C ore +Ġul ong +Ġblog ging +> This +Ġdata Index +Ġprint able +ĠE yes +_target s +(P y +. over +Ġbr u +am pton +Ġplaint iff +< Key +b ull +Ġ⣠¨ +Iss ue +.cor nerRadius +C ritical +_p hi +. angle +Ġdynam ically +! ");čĊ +> );Ċ +in vest +.* ĊĊ +Ġt élé +Ġsuper f +Ġcas cade +DT D +Ġviv id +Ġsubsid ies +ĠH ass +Ġcoll aps +Ġcer amic +{} ". +ĠLeak age +-tr ash +coll apsed +-s ocial +ĠCh ad +Ġincl ined +Ġst o +Ġstory board +.p ayment +stack overflow +ĠRaid ers +Ġ# ' +olic ies +ìľ¼ ë¡ľ +em ap +Ġk j +Ġqu ota +ĠGard ens +ë² Ī +ĠAng els +Ġof t +Ġlower case +Ġi Param +Ġche apest +un ta +_p kt +ic ators +Ġle urs +Ġdecre ases +ĉ define +PRE C +amm ers +ĠPre paredStatement +(d irection +Ġcre ws +ark ed +ĠMem phis +ĠS ell +G TK +Ġm aid +: disable +éĽ Ĩ +ĠP f +Ġal beit +open h +?> ">Ċ +.get Source +(s cale +D u +ĠP IL +_ref resh +Ġbet s +(c ar +ĠV on +| --------------------------------------------------------------------------Ċ +ĠGr at +M uch +( Dialog +.stop Propagation +Ġte k +Ġex its +'], $ +Ġphone Number +uc s +ec imal +------------ -- +in p +.po jo +Ġcor pus +Ġpractition ers +.p ic +" testing +Ġstring By +.Not Null +Ġr ang +.D ynamic +_R ender +аÑĤ а +Wait ing +ĠW ik +Ġoverwhel med +% "> +ĠA E +}} >Ċ +u w +_t yp +Ġbuck ets +Ġgre eting +Ġla ughter +Ġant agon +uggest ion +- email +ĉt op +Ġer os +_tr i +Ġiss uing +Ġh á +Ġisol ate +Over flow +, E +Ġnut ritional +ĠAbb ott +Ġn f +.t ouch +.fetch all +_z ip +") }Ċ +Ġam at +ĠC isco +Ġn Ã¥ +PLE X +Ġse i +f oto +.to Json +å¤ ļ +ĠKle in +Ġlib c +Ġmin ers +å ¢ +- print +ĠP ride +T odos +Ġmask ed +Ġset Data +Ġtele fon +Ġunh appy +ĠT ables +ge b +( debug +_all owed +- access +Ġlog istics +Ġg ems +ĠM ature +Ġr sp +ĠAl le +.get Bytes +\ web +ynchron ized +Par agraph +Ġth rottle +.sql ite +cons ulta +ĠSe ah +C e +Ġsub mar +ER E +V ous +Ġre ddit +Ġsql alchemy +-m ile +oc ide +P our +}} ">Ċ +st ead +Ġ@ ( +Ġ[ ]) +ĠAd s +Ġover load +r idden +ĠDes ert +ĠW rap +ĠPortug uese +et z +ĉf irst +Ġmile stone +æĹ ł +Ñĥ Ñī +(s uccess +< Vector +co ol +Ġ[ ]);Ċ +erv als +Ġin vert +" io +cur so +fr agment +Ġfeas ible +.set Position +Ġel m +Ġimag in +@ Spring +Ġb ats +pu és +ga lement +ns ic +gi ene +ell ation +ĠBa iley +Sh ar +ĠT ul +ĠH K +Ġfree zing +gl m +ce ans +-c ut +_c ircle +åij ĺ +n egative +Ġind ian +s alt +Ġt ing +ĉm od +Ġs int +ak in +um l +ĠText Input +Ġpop ped +T MP +Ġpark ed +×Ļ × +ĠF usion +Ġhe ater +ET F +ro zen +h all +ĠM ik +lev ard +- heart +ĉ order +M aking +Ġpled ged +Ġdir s +$ post +ĠH err +stant iate +, "Ċ +.get Color +ĠS AT +Ġtimed elta +ĠM ai +ĉm ethod +Ġid iot +ĠTr av +ident ified +ĠDiv ine +.get Path +D ash +Ġinf iltr +Ġhandle Submit +bro ok +.g eneric +.short cuts +................................ ................................ +Ġdat ings +ĠM V + # +} "ĊĊ +Ġimprison ment +ason ic +rou d +uc ion +æĬ ¥ +Ġdia lect +Ġon Mouse +const expr +.label Control +Ġwe aker +Ġman kind +ĠRE CE +Ġd iz +Ġapp Bar +Ġqu é +f ra +_default s +Ġal iqu +_at om +: indexPath +Ġmiss es +Ġvis ually +ĠH ands +STR U +i ates +_ asset +F inder +mid t +Ġsn acks +(__ (' +. uri +ĠIn strument +ven ir +($ __ +.Dot NetBar +Ġconfig s +Ġguess ed +ि ठ+Ġinitial izer +Ġ? ", +ĠVer izon +man ifest +ge ben +.d etails +G ate +pons ible +ĠEl im +, str +Ġwrit ings +ĠD erek +ĠCo ordinator +Ġpill ow +Ġnotice able +R s +Ġduplic ates +ern els +k J +.z z +oll and +ĠSE CTION +_f name +uff led +'].' ")Ċ +ĠD ollar +Ġem oji +Car ousel +- player +Ġadjust ing +Ġjug a +alleng es +g ene +(body Parser +lop edia +ĠBeh ind +Ġslee ves +Ġdrag ging +ĠChe vrolet +Ġb iz +iv ities +ĠFrequ ency +, char +.W HITE +_pre view +) ';Ċ +_ ax +ION S +.c pu +.input s +UB E +_fe ed +ĠSup plement +! ). +es us +ĠU DP +Ġmicro phone +Ġconf irms +.is NotEmpty +":" ",Ċ +_S CREEN +ĉ expected ++-+- +-+- +ĠH ait +fast call +Ġdep ict +v b +_p icture +ĉd escription +ĠW ife +uc i +Ġv icious +ä» ĸ +ue ba +Ġset User +ãģ ¡ +Ġd iving +Ġoper a +user content +ar ah +) }, +y un +vel t +Ġun covered +Ġh ips +Ġosc ill +Ġassert ing +ĠX i +.re store +ke a +Ġsp elling +Ġder ive +ab we +ĠD ow +.set Type +_v s +Ġco zy +.c ategories +O rg +_m gr +Ġd ungeon +collection View +ĠBl ank +ac ias +ä ä +_clean up +_ACT IVITY +Ġtri angles +.Menu Item +Ġip hone +ĠW on +] ]ĊĊ +ĠCompar ison +.D oc +Ġcan onical +ĠSud an +') { +Up Inside +b uiltin +ENC Y +x be +Ġch uck +Ġcontrad ict +Ġnuest ro +Ġarchitect ural +ĠF ib +Ġcomp ares +* k +C fg +çĦ ¡ +nt en +Match es +ĠDOWN LOAD +_HAND LER +man agement +[ S +EN G +ÂĢ Â +f ang +Ġsl ipped +ĠL anka +esc aping +Ġtack les +ĠPed ro +.P rop +.' ' +.G enerated +.New Guid +at rigesimal +ill on +Ġstat istic +spec ies +hold ing +Dr upal +Ġfundament ally +Ġbond age +Ġres olutions +Inline Data +\ Type +est ion +.w rap +Ġwar riors +ĠLOC AL +Arch ive +Ġembr aced +á» § +.V er +ĠAff ordable +oles ale +ĠAp plied +ĠCon version +m ega +_c am +Ġcer emon +aur us +ĠVol k +.op ens +/ about +ĠSt d +j ournal +()) {čĊ +," \ +( Arrays +ĠD ense +ase ña +än ner +/ stat +user Data +Ġg erman +Ġt z +worth y +Format Exception +ph erd +Ġsm iles +ĠWh enever +( adapter +.bad logic +Ġbrief ing +.Grid Column +- char +dim ension +ĠC opper +Ġnin th +Ġ' {{ +Ġr av +_T able +Ġderiv atives +ĠR aise +ĠF ut +arm or +-p adding +Ġre min +ĉ style +ĠMembers hip +Ġspread s +Ġgall eries +ĠClar ke +Ġcon ception +min ute +Ġab usive +_ad j +Ġterr ific +Ġover t +our cing +Ġentr ada +level s +Ġcrit ique +Ġrespect s +ĠM MA +i ene +Ġenc aps +ĠRay mond +Div ider +iv able +b az +Ġ@ _;Ċ +ĠCl aire +Ġur ging +CE E +Ġtransform er +disc ord +ĠJ ourney +t os +Ġcompet itions +ĠO BJ +ĠB is +Ġrelax ation +id y +_IN STANCE +ĠP ref +d ados +ici encies +ĠMedia Query +ĠC ube +ĠStr ange +g pu +(d ays +_Init Struct +Ġfinger print +em at +ĠGe cko +Ġr ails +ĠL um +str action +ig ung +(m ovie +_d ictionary +_int errupt +ĠQ C +ik ed +append Child +rec ipient +r é +V e +Ġtow el +.last IndexOf +Ġplace bo +ĠW ie +.es p +( Debug +oper ative +Ġdece ased +& id +ĉm utex +el ic +Ġb apt +ĉ čĊčĊ +Ġfar ther +H alf +.dis able +.menu Strip +le ccion +Ġresult Code +Ġc ans +-e lection +f emale +_F IX +aus ible +ĠP OWER +Ġrecon struction +Ġsc ans +.Xtra Bars +âĢĺ s +Rem oved +Ġparagraph s +_m argin +Ġl ymph +Ġb os +ling ton +ĠBapt ist +Ġadvertis ements +ĠMan age +/ yyyy +IO US +ENC ES +ĠF iction +ĉm enu +ĠFile OutputStream +ov an +ĠF eng +Ġsk ipping +get Class +ann i +Ġreb ounds +Ġpublic ity +Ġing res +use ment +Ġthought ful +.Ch art +Ġhat te +pass port +Ġhook ed +ĠL ens +Ġflag ship +Ġst ip +ĠG EN +Ġcl ues +ip v +ĠR ise +ĠG ew +tab lename +Ġfore most +_ validate +_an alysis +oll a +Ġqual ifications +Ġdistrib utions +ĠFl ower +Ġt ense +Ġthank ful +Ġcl utch +Ġun ified +ro ads +Ġsit i +Ġst all +_P RIORITY +c stdlib +_USER NAME +.by tes +? page +ermal ink +ĠVe get +/v nd +- author +.N ONE +ĠCon current +ĠC ry +Ġstart ers +ĠInter action +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +ĠLE VEL +E ll +Ġcom boBox +ĠTh eresa +te k +_H andle +Ġab y +.g dx +, end +(L ocal +O l +kn ife +ar ial +ĠH off +Ġprostituer ade +Do ctor +Inst ances +.Set Value +ĉf rom +Ġlux urious +Ind ent +Alloc ator +_D RAW +(", ", +ĠFr ances +Ġgroup Box +(s chema +Print f +OR IES +- gradient +Ġre put +ar in +_D ONE +in cre +ig nty +Ġex ert +Ġ- . +/ App +-th rough +Ġdecl ining +Ġdess ert +Ġinc umb +Ġdesign ation +.P ORT +, strong +Ġsand box +Ġw ines +ĠP av +$ str +ask ell +Ġh ö +ĠP Y +Get Instance +Text Input +game Object +/ events +created At +Ġlocal Var +ĠWH ITE +per ed +ile ge +eff icient +, color +c ate +ĠC afe +Ġsimilar ities +Ġp umps +ĠHung ary +.User name +Ġsk ate +Ġtouchdown s +Ġacceler ate +ĠH elen +OM EM +ĠK un +_v ol +Ġfind All +ĠMens chen +a head +); " +kom men +Ġpossess ed +.arg max +.trans ition +AR P +OLUM E +(s cript +ĠÐ ĺ +ĠF inding +on ces +I o +B old +Ġrenew al +_D IALOG +Ġdis reg +INT ERN +Ġt oute +Ġelect r +ĠG ross +ĉ true +.F ields +ĠW IDTH +ĠD ent +Ġà ģ +NS Notification +Ġa os +Ġme lee +. Validation +ĠDE C +-depend ent +Ġsu ic +T raits +$ message +ĠD ear +ĉ FILE +l anguages +.P rot +.add r +-g eneration +IC ON +Ġtrans plant +-d escription +Ġch asing +Ġche es +Ġ} */Ċ +Tr ad +qu eries +/widget s +sub package +Ġes pec +Ġcr acked +Ġcompet itor +P urchase +- team +olec ular +or Thunk +& P +Ġrel ent +/ #{ +Ġproduct Id +Ġè ¾ +ĠL av +ĠAl ter +.M ode +AD IO +gr p +æ ·»åĬł +Qu it +Ġdepth s +-c ategory +ĠD ATABASE +S PELL +ĠFal con +ĠQString List +Ġ'' . +ĠIn stitution +d amage +az or +bel ongsTo +ver ages +ĠN ONE +ipp ets +, \Ċ +Ġfoot print +_ archive +n ak +.get Field +ĠRef lection +Ġ' ] +ĠH BO +_dis count +Ġin cest +ĠD odge +ĠW ade +.N O +" encoding +ĠBlock chain +Ġlaws uits +ĠM aint +ch ten +Ġét ait +Ġktó re +_ ctl +(t imer +B attle +iz o +ay ed +I OR +ĠGlas gow +Ġsyn th +_log s +.p ose +_Adjust orThunk +(( & +Ġuns ure +yst ate +íķĺ ëĬĶ +O ULD +. ng +Ġdefault dict +work space +Ġselect ive +Picker Controller +YNAM IC +.method s +Ġpath ways +ĠF ew +K G +CRY PT +follow ing +ĠD LC +ĠS ara +Ġpres et +estruct or +ĠK urt +Ġair plane +Ġo mp +ĠParent s +ĠMart inez +.com plete +Ġbroad ly +Ġsc are +ĠM é +Ġelim ination +Ġpou red +/ sw +Ġcom un +Ġm asc +ĠOrgan ic +ĠString Utils +il ateral +Ġreluct ant +- age +Ġn z +." \ +Ġpast or +ale z +Ġe fect +pro v +/ init +Ġp enn +und s +Ġs size +ĠPro j +bas ename +Ġsh ells +ĠNe ck +ĠEn forcement +vid ed +st own +S phere +$ r +uss en +af il +ĠTele gram +Ġanaly tical +нÑĭ е +us ually +x n +Ġhistor ian +ĠGreg ory +ol ph +ĠUn a +Ġcon tributes +% - +anti ago +ÑĢ ÐµÐ´ +.reg ion +Ġab rupt +ĠUnsupported OperationException +ĠT ASK +_f inish +Ġnot orious +ĠV s +ĠM Q +Ġsun set +Ġun acceptable +ar cer +Ġill umin +ĠOr b +Ġb h +E ste +_dis patch +Ġr ipped +Ġtou jours +ĠPar cel +_ ll +.user Name +.class es +S OURCE +( Number +ел Ñı +Ġhead phones +(s ide +const itution +ann ah +čĊ ĠĠĠĠĠĠĠĠčĊ +Ġcl iff +- ref +Ġmo strar +ĠPow ell ++ y +ĠB G +_f ragment +.P ort +Ġreal izing +param ref +Ġh ometown +@ Table ++" --}}Ċ +F rench +Entity Manager +ĠPl ain +//////////////////////////////////////////////////////////////// //// + ³ +( RE +c apt +Ġorgan isms +Ġj ets +ol ocation +ĠApp RoutingModule +Ġgl orious +æľ į +Ġdisc arded +ĉĉĉĉ ĠĠĠĠĠ +ĠArn old +l ug +Ġpar l +Ġhorm ones +Ġm ah +ĠSon ic +Ġorgan izers +_PL ATFORM +.in v +Ġch ord +vent ional +ĉ of +Ep isode +. Enum +unk t +ĠD h +ĠJ ared +ĠN ak +Ġint ends +End ian +Ġa ustralia +_c v +(res olve +Ġclin ics +lik ed +ASH INGTON +in ha +' * +ĠN P +_b eh +Ġh f +Ġw ür +c ategoria +$ form +Ġsub way +Ġis Active +pop ular +C our +Ġco oldown +Ġa insi +ĠGL uint +ere al +Ġarray Of +Ġh atch +======== == +ress es +_P P +. ^ +_dec ay +ĠB less +met rics +ĠCOPY ING +ĠDump ster +ĠJos é +ĠDesign s +< +Ġ" }Ċ +time zone +Ġe er +max cdn +ĠE SC +ig aret +_conn ected +_re verse +Ġquestion able +ĠUS C +Ġtut ti +Ġdrop out +ĠActiv ities +ĠW inds +')) );Ċ +Ġcon gest +ÄŁ ı +Ġprolong ed +è¿ Ļ +ĠCross AxisAlignment +LE EP +ĠVAL ID +ĠG az +Ġdepend ence +ĠP rix +.Compiler Services +j ump +Ġstr at +c irc +ĠC USTOM +x aa +Ġb mp +Ġb ureau +Ġw aren +N X +( Window +ĠChrist ie +_F E +Ġt n +ĠOm ega +communic ations +Home Page +com pletion +Ġsupply ing +YP ES +á vel +åĪ ¶ +(c lick +\ Contracts +/ questions +Ġe z +AM S +.m esh +Ġ' \Ċ +Rob ot +Json Object +ĠD F +ĠProcess or +_sh ould +.prot obuf +- users +Ġemb ry +F ONT +Ġstart ups +ĠData Source +) # +uro s +_C olor +Ġstand alone +} [ +j d +Ġforg ive +Ġng x +ĠGener ally +Ġconfig urable +/ order +Ġv as +') ";Ċ +ĠR R +ĠT roy +Ġcomprom ised +ĠSw an +int endent +Cent ral +_ keeper +Ġar quivo +ĠRead Only +_cur ve +k v +ent in +è ± +ĠE y +.im read +ĠP am +if fe +at ivity +xb c +Ġgr im +-f illed +names e +'] : +Ġa ur +ĠGib son +.Mouse Event +Ġl ado +avad oc +Ġfam il +ĠM oder +f ps +ãĢĢ ãĢĢ +- example +ĠAl zheimer +ĠU tf +_arg uments +Con clusion +text Content +rem aining +Ġinterrupt s +ĠBack up +ĠM ong +Ġrecept ors +h istor +.cor outines +Ġsh outed +Al arm +Ġcomb ust +Ġg rote +ult ural +( ids +---------------------------------------------------------------- ---------------- +ipl inary +O pts +ĠY ale +local Storage +Ġequ ival +ĠF leet +\ b +* pi +ĠQ Label +æ ¡ +Ġv x +ĠA CL +Ġsu cesso +Ġper c +ĠNot re +Ġan arch +R ing +sp b +Ġstr pos +st ores +ĠMap le +(Main Activity +(" ")) +Ġview Holder +Qu ad +Ġig ual +ors che +.m argin +Ġind ie +Ġfr anc +ĠForm Builder +ĠPart icip +.fl ash +Ġstorm s +U lt +Ġf en +[ new +E ver +=" Ċ +Ġlocal ized +_f ollow +Ġn ave +Ġdomin ance +(t ile +J ournal +ĠV C +Ġpenet ration +ï¼ ķ +Ġcomp artment +Ġb ids +Form atted +****** /ĊĊ +(c ity +âĢĶ it +[ C +Ġuse Callback +a ub +) ?. +ĠV AR +ĠSe bastian +ĠM oss +Ġabund ant +G reg +ÑĤ а +_c i +Ġbib li +CR M +ĠAt tempt +ism e +d ash +ãĢ İ +_m u +.Formatting Enabled +Ind eed +-d irect +Ġsuck ing +Ġp ne +ocab ulary +ĠPack ers +.N avigation +Ġp ied +cri bing +ĠSt uart +.To Double +ĠSecond ary +S aving +ĠD ut +ĠM add +M agic +, H +.document Element +ĠB ST +Ġdiff ers +Ġmore over +_ nd +SE ARCH +п ÑĢав +æ ´ +to Match +Ġdecre asing +-m ember +amp us +( boost +D aily +Data GridView +ĠHttp Context +Ġh ipp +_work ers +-l anguage +é ĵ +Ġconsist ed +ath ing +ĠMer cury +$ content +Ġpract iced +ĠMod ules +_D AY +Ġweakness es +ĠL odge +Ġn ar +ĠM ate +Ġj p +ĠHttp Headers +Ġsm o +ĠT OKEN +] )( +Ġaqu i +sw agen +Ġs rv +ĉ ans +A round +ĠMan uel +Ġfiction al +ĠIM G +Ġ. ' +ĠB erry +Ġwall paper +sex ual +ier o +Ġ çļĦ +ìĨ Į +Backing Field +ĠAd rian +BASE PATH +Ġrepe ats +Ġbl ues +Ġunp redict +_c oll +st acle +ĠT umblr +ĠEl f +Ġass urance +Ġc ensus +ĠIM PORT +END ER +an os +Ġ= ( +ĠEll is +" ĊĊĊĊ +.w in +ĠA bove +al on +_t ick +Ġrepresent ations +Ġæ ķ +w id +ĠAr ms +List a +_f ailure +_c m +.Flat Appearance +Ġthr one +P atch +ĠV oy +eng l +Ġnegot iating +> ` +Ġshoot s +ĠF PS +.Y ear +ĠK iss +enc ión +reet ing +From File +Ġresign ation +Ø · +Ġtw ins +ưỠ£ +Ġge bru +.get Content +.T ree +ĠEmploy ees +ĠF IFA +Ġcert ainty +(C l +Ġtot als +edit able +à¥ Ģ +.Report ing +M as +qu iet +.r ules +ĠV O +con exion +, K +Ġalloc ator +ĠPow der +\ Repository +Be at +_t ipo +Ġ[' ', +_IN TR +Ġ<< < +< hr +") == +ugg age +ĠC raw +Ġé galement +Ġg inger +Ġprim era +Ġprod uto +lt k +.User Name +Ġstr error +m ith +_n b +Ġdis comfort +']; ?> ");čĊ +drop IfExists +ĠB eg +_H AL +Ġcross AxisAlignment +ĠE vidence +Ġpec uliar +Ġinstit ute +ve is +Ġf ft +à ģ +Ġzo ekt +an aly +ĠHom eland +Ġpen etr +udden ly +ĉ element +ĠB ren +ĠTr udeau +ĠCub an +j am +us lim +_e v +Ġst ems +} % +Ŀ å§ĭ +Ġbrand ing +Ġcorrespond ence +.j query +¢ åįķ +ĠRead s +(Http StatusCode +ass in +(s lot +ĠGrad uate +/// < +Ġinform ations +EN ABLE +Ġp uis +Ġfind er +ĠBr is +Ġnett steder +_m id +Ġo gs +ĠSter ling +Ġar rog +str ftime +| ĊĊ +Ġvo x +ĠReg ardless +Ġes o +ĠCom fort +.Boolean Field +Ġu h +AC Y +Ġsque ez +ĠV ic +cont ro +. lo +Ġ ire +ĠCom edy +ë ¶ +Ġorigin ated +Ġsh ipment +| max +_g uid +lev ation +на Ñı +( undefined +ĠD DR +Ġshoot ings +ĠLat ino +END OR +Ġaver aging +Ġgre eted +Ġthe aters +о е +Ġd B +Ġg st +Ġdef inite +. Storage +.h er +Ġa fore +ĠRe ality +ĠGod s +vers ed +Ġhands ome +Ġex cluding +( ad +Qu otes +ĠS cheme +? q +ĠT amil +T icks +Ġp est +' n +Ġporn ography +_mod al +Ġ ---------- +Ġdis posable +F REE +Ġsh ark +C HE +Ġdep icted +Ġdemonstr ations +ĠK illed +ĠR ULE +Ġobs essed +Ġsimpl ified +Post al +Ġconcept ual +Ġp st +L as +_PRO JECT +ucceed ed +ol u +ÄŁ i +Ġpersonal ities +Ġres hape +Ġenc losed +ĉp tr +Ġtutor ials +Ġexpl oded +_DIRECT ORY +åĨħ 容 +Ġcan on +Ġrecogn ise +P AD +ĠAppro x +ĠRest ore +ĠImport ant +Ġheav ier +.Se quential +Ear th +ĠMil k +.set Request +.t em +Ġre construct +Ġskept ical +_Pr ivate +BU F +qu a +: a +Ġse k +Ġd well +oss a +Ġreward ed +и й +(top ic +_part ition +Ġ__ ________________ +Key words +ĠFr anco +L ite +Ġn aken +Ġз а +O BJECT +Ġcraft s +ĠSw ap +.X na +.Con nect +Ġbalcon y +(re al +ĠBarn es +b ir +ĠTw enty +ay an +at ars +ĠProp el +ĠIh nen +Up grade +Ġcur b +- second +Ġn eph +.p res +ìŀ ħ +.se q +Ġp added +" ? +j l +ãĥ ¬ +') a +Co ordinates +Ġen acted +ENT S +Ġl ac +.f inal +ĠPhp Storm +c alled +Ġin quiries +.m iddleware +ĠD owntown +/ ';Ċ +Ġkil omet +ac cel +Ġqu ien +w string +set Data +Ġman era +Ġmod ular +rim p +Ġtar iffs +âĢĻ il +_TH ROW +/c olor +ĠHT MLElement +Ġcar ro +Ġpr ere +Ġplot ting +ĠPos itive +ĠMach ines +OT ES +á» Ľ +ple asant +Ġal te +Ġa inda +th ese +Ġc ors +ip ay +ĠAdvis ory +ĠRub io +j q +Ġl imestone +Ġdet ached +设 ç½® +ten ant +ĠDep th +al ore +ĠÑģÑĤÑĢ Ð¾Ðº +ĠF ORE +ĠL ay +p resentation +) ');Ċ +.sub plots +Ï ĥ +N OW +G ar +hand les +ab ra +put ies +ĠElect rical +M iddle +rop ic +ĠJ D +ĠD yn +ĠB ristol +ĠMc Carthy +Ġstri ker +Ġenumer able +ĠEv an +.default s +qu ences +) || +ĉt oken +â Ĺı +-d ropdown +ST ORE +ĠGraph ic +( pp +Ex pl +Ġup wards +ĠD istributed +ĠW EB +J er +is NaN +çĶŁ æĪIJ +> R +üss en +ef s +Ġun cover +Ġl ud +.cal culate +Ġint ptr +Ġmidfield er +. Headers +Ġm f +ere f +.M etro +ĠSpe aking +: b +Ġcryptoc urrencies +Ġdem ons +ĉ EXPECT +Ġw icked +y outube +: Int +ĠHind i +ĠC AT +ĠØ ¹ +r ar +om ore +/ per +/lic ense +Ġre im +Ġawait ing +Ġle thal +ĠE F +round ed +ĠPl atinum +ĠвÑģ е +.co ords +.De vice +/ item +ĠW enn +compile Components +ĠK inder +.remove Item +Ġand a +bn b +Ġpr a +( transaction +Ġembarrass ing +ĉ BOOL +.content View +Ġevent data +at ore +Ġprovided In +ir ma +Ġz ona +_H W +æ Ļ +Ġst ove +Ġcounter part +_Pro duct +_MAN AGER +Ġinfr ing +ĠE RA +_p arty +Ñ ij +Ġin ici +_ Request +Ġmir acle +Ġcancel Button +S py +at ó +Ġpol ish +ĠNic ole +.display Name +\Request s +Ġuse History +Router Module +Ġst ared +ID ER +Ñĥнк ÑĨи +Ġnot a +$ arr +pec ified +Ġto pp +_DR IVER +/ ng +å ł +_t m +% timeout +< s +Ġ( *) +ĠHttp Request +_TR ACK +(n ote +ĠExp lore +_s erv +Ġç » +B inder ++ ", +. att +ĠEth i +Ġc ódigo +=' \ +.l ines +( Of +å° Ĩ +miss ible +Ġv é +Ġac oustic +Ġcraft ing +n it +.b a +ĠLuc y +Ġi Pod +Ġpup ils +-m ax +_w r +(c p +ĠRE PORT +Ġd ns +ĠRe ferences +Ġundert aken +Ġkø benhavn +Ġch ai +ĠC roat +_ Log +rown ed +_m ed +ĉ date +# __ +Ġcost umes +ĠRe quires +aff le +ç Ĭ¶æĢģ +-S emit +ela ide +еÑĤ од +Ġp estic +Ġd ra +DOC UMENT +Ġ... čĊ +}` }Ċ +ĠA uction +ĠD ock +xxxx xxxx +(get String +ħ į +Ġborder Width +ĠMach inery +Ġpredict able +.S H +Ġam plitude +.for Root +IN avigation +Table Model +at trib +Ġmaneu ver +Ġexc av +B ERS +Ġd apat +Ġinstall ations +.A sync +Ġr ays += âĢĿ +; ččĊ +.c rypto +_db g +ĠEnum erable +Of Size +_epoch s +m w +M ENU +out line +ĠP apers +============ Ċ +Ġuniform s +ĠG ig +- package +ĠJen kins +ĠHome Page +.is Selected +Ġmechan ic +M K +ĠS ounds +//---------------------------------------------------------------------------- -Ċ +Ġresearch ing +Ġinf os +ograph ics +ers et +([' / +ĠTim ber +. agent +.to JSON +_command s +par ing +_ad just +.n ome +(g lm +Status Bar +file path +? âĢĻ +Ġdetect ive +Ġunser er +ĠTib et +EN DED +(se ed +Ġsne ak +Ġam or +=" // +ĠPan thers +all ax +ĠL IVE +ĉD WORD +]= - +Ġtorn ado +/ min +Ġlung s +-c urrent +ĠBook ing +åĪĹ è¡¨ +Ġenjoy ment +ठ° +J A +typ ed +.B tn +f at +ug al +ĠSh ares +Ġdis gr +ĠB AR +ĠFO X +Op code +ĠS z +key down +iction aries +Ġdetail ing +} ))Ċ +Ġp ok +Ġdemonstr ating +Ġnot ation +l ayers +@ if +ĠN PR +.strict Equal +ĠRec ipes +.T ensor +Ġliqu or +Ġdeb ts +.ends With +W heel +.P os +CS V +$ arity +Ġun stable +( loss +ENS OR +Ġele ven +ĠL opez +ĠHop kins +con om +ĠS eth +Ġpo ems +Qu ant +Ġg sl +Ġsy rup +Ġs ibling +Ġc ass +-v ous +ö t +_P ATTERN +_SE CTION +est imated +up grade +.m ongodb +ĠBo at +_C TX +Ġfetch ing +ust in +pi el +M arg +Ref lection +Ġd uct +ĠMunicip al +Ġb x +.Get Current +ml ink +ĠAccount ing +ĠGene va +_P os +Ġpass er +Ġhear ings +com pan +Ġfrag ile +Initial izer +walk er +.M aterial +ĠHun ting +trys ide +Ġk at +Ġcl erk +á Ł +do ing +ĉg roup +Ġsan ction +.l b +ĠL azy +ĠCon straint +P agination +Ġpou vez +ĠInd icates +M ER +Ġcour s +Ġyear ly +Ġgros se +abb rev +ĠD ON +Ġproceed ed +ent lich +Ġproperty Name +ĠTe aching +st adt +Ġc utoff +orn ers +Ġa frica +Ġrend ers +ĠYan kees +ĠTool bar +sp aces +.fill Style +Ġseg undo +_str len +.F irebase +å¤ Ħ +Ġmention ing +\ ( +ĠVal ve +Set ter +Ġsp ans +ĠAl cohol +ĠLet ters +\x e +ĠT K +_B LE +.get Result +< Player +ĠP att +Ġeas ing +Ġtur key +ĠF en +') " +Ġconf ined +Ġin clus +Sup erview +(with Identifier +enc ial +Ġstuff ed +Th eta +Ġeconom ists +} ));ĊĊ +co okies +ĠRo ose +ĠChe ese +Ġfich ier +Ġen forced +AB B +no ÅĽci +_AL LOW +Ġrecru ited +Ġexpend iture +-n ight +Ġassert NotNull +_ex ecute +ĠØ ¯ +IN DEX +_F MT +Ġresc ued +ĠMonth ly +ĠCons ervation +ĠG eb +Ob ama +Ep och +ic ies +ĠOr t +Ġso it +( icon +F riends +m ol +Ġground ed +ĠC ause +ad ena +WE EN +ĠL un +IT IVE +. loop +_un til +Ġcor r +.ed ges +Ġhyp oth +ched uling +trans lator +ĠÐ ľ +R om +ãĢij ĊĊ +ĠX amarin +Ġviol ating +. anchor +--- ĊĊ +Ġtr ader +AD VERTISEMENT +Ġuns ere +ĠD AO +Ġbl ond +ĠP AT +.g lob +Ġè¾ ĵ +Ġsplit ting +Ġun subscribe +Ġatmos pheric +ĠTr im +Ġcit ation +Ġin ference +ĠF t +ĠDar win +find One +ĠG el +( Convert +Ġaccess or +; text +(s orted +Ġjud ged +); \ +: p +Ġme ine +ĠS lim +.Command s +Ġper ceive +coh olic +< Data +.entry Set +Ġassert False +ĠPat rol +ense m +ÅĤ Äħ +¨ ¡ +W IDTH +ĠRes cue +ĠU IF +_THRESH OLD +ĠMich el +ATER IAL +opens ource +ĠD iana +Ġinv ites +_B ODY +Ġreserv oir +Ġro i +c ust +(t c +ï¼ģ ");Ċ +Ġfest ivals +Ġperform ers +Ġclim bed +Ġj ungle +String Length +Ġunlaw ful +ier re +vertis ement +Ġst akes +Ġh ats +Mod ify +ĠLET TER +.H ide +Ġstat utory +_ white +ĠPer l +uten berg +em ple +.W orld +Ġoverlook ed +Ġcon cludes +/* ================================================================ +-w ise +ĉ stream +pop ulation +Ġevent o +Ġillustr ations +ft s +Ġaut of +ĠPro cedure +Ġdes erved +-t imes +Ġg ol +N SError +cre st +ĠPak istani +any ch +get Current +Ġl ar +nt l +ĠRe becca +Ġm ateria +Ġfind By +/ ad +Callback s +ĠAl s +ĠKat ie +ĠObservable Collection +ĠDocument ation +Typ ed +ĠCulture Info +ĠTim othy +Ġlater al +" type +Ġun authorized +Ġteach ings +Ġdebug ger +[ value +Ġal ors +Ġu z +Ġsc atter +Ġdown ward +Ġmig li +status Code +Ġ( )) +ĠM W +Ġм ож +RO SS +.b uf +Ġfair y +ĠInf rastructure +=> " +t lement +$ (" +From String +ĠB ild +Ġconvent ions +_n ative +ĠIns pector +ĠP ist +ub ar +Ġreg s +ĠP ilot +Th us +>' + +Ġc ela +.new s +( Product +L iving +R ussia +Ġfac et +et ical +Ġ[' $ +/ [ +ĠD ire +Ġg ases +ĠIN FORMATION +ĠE at +ĠFor ums +ĠChar acters +_m et +Ġìĭ ľ +Ġk ings +ach ie +ĠL ambda +Ġtim ers +ĠLight ing +ĠCase y +add ir +and ex +. answer +ĠH ip +ĠPr incip +Start Date +Ġ ãĢĮ +t res +Ġ& # +.Max Value +ĠPro blems +Ġlat ex +Of Class +ĠLyn n +// ' +Ġvoy age +Ġshut tle +ĠRoll er +ĠRuntime Error +uy a +D ic +ĉb uilder +Ġbul lying +Ġsimple st +.c alled +ĠL R +Ġmor ality +Ġst urdy +tr acking +.sw agger +_B IND +IT OR +-url encoded +ĠÑ ħ +ĠTr inity +Ġtr aps +Ġ| - +Ġset Text +Ġbarg ain +Ġbr akes +.get Code +Ġmigr ate +Ġrib bon +) return +Ġcharg er +ac om +ADI US +ĠAmb assador +-a fter +Ġann i +ĉs pin +Con cept +ĠHend erson +ĠH OST +.r ank +ĠNor theast +Ġber lin +Ġrequ is +.f eed +Ġsource Mapping +ĠRen contre +. ajax +nest js +Ġtre k +ĠN acional +Ġ& [ +Ġpay able +ort ex +Ġde pt +field Name +Ġcomple tes +ĠR VA +Ġon ions +al ignment +Form ats +Ġ' {$ +Hash Set +ĠB od +.Invariant Culture +Ġsettlement s +Ġhy dr +. updated +vent h +( seconds +="/ " +Ġweb page +( ĊĊ +Ġt ir +Ġto es +ĠBr ick +Ġamb ition +P ot += max +ET IME +Ġdep ot +c alls +ĠNor wegian +` : +Ġbur ger +Ġprofess ors +ĠAl locate +-third s +-ch art +Ġfor d +* N +.k otlin +Ġpaper work +ĠDE VICE +% @", +res pect +(m p +é «ĺ +- if +Ġcush ion +ob ot +Ġpar c +SP ACE +ĠNet anyahu +Ġself ish +fe at +Ġclient es +-to ols +Ġpor ch +Ġj q +. verbose +Ġlib erals +] )ĊĊĊ +p ies +Not Blank +( term +ÈĽ i +_Param s +.normal ize +B ullet +AS IC +(h ex +_client e ++ , +_D I +Ġforth coming +} ")]Ċ +se o +U m +> Name +Ġcomfort ably +irection al +W ITH +/ pr +ĠP oor +ĠVit amin +v ic +G H +Ġprior it +ĠN N +ĠC losed +¤ í +Ġis Open +\ Console +And Feel +.S UCCESS +_OPER ATION +pol ation +ĠT as +ps z +> '. +C URRENT +V endor +host s +ĠE rd +>tag ger +ĠsourceMapping URL +Ġmar athon +_c losed +Ġexem ption +Ġrecogn izes +ides how +' $ +('/ ');Ċ +m its +war z +ĠCh erry +µ ¬ +n or +port e +Ġw l +_back up +.get Boolean +.get Resource +Ġdefinit ive +. EditText +Ġs ÃŃ +.C ONT +ĠPL AYER +.c ards +ĠSh ore +('/ ')Ċ +cl uir +Web Driver +(m onth +-re lease +Ġins pector +å £ +ĠN F +_cl ip +åŃ IJ +Ġinteract ing +.t mp +Ġ'' 'ĊĊ +Ġde e +Ġfro st +"] ))Ċ +ĠPl aces +Th rows +f ork +/ day +i Phone +ĠM IC +Ġfold ing +Ġcro re +ĠCh iefs +pher ical +( price +.Write String +Ġexit ing +] ',Ċ +ight ing +Ing redient +( vertex +Ġscroll View +h f +: new +SE N +se ctor +Ġsp ins +ĠS cheduler +ote chn +sem icolon +Font OfSize +ĠSpecific ally +fl amm +.Object Id +Ġcont a +_per missions +ĉF ROM +IC ODE +/ kg +ĠHot els +-m ed +ĠD in +Ġn avy +get Param +Ġm end +Ġportray ed +ĠMet ropolitan +Paint er +Ġref erral +_g ood +Ġmar vel +osa ic +> (& +. ur +Ġest os +Will iam +Ġtim ber +Ġquel ques +ĠDoc uments +.X aml +Ġbatch es +éģ ĵ +ĠRe leased +T ail +CO OKIE +he id +_st ation +ĠV ia +S ale +ĠRe peat +Ġprom in +ĠZ o +- forward +ĠI on +it ary +Ġj us +- request +Ġproud ly +ĠStream ing +(Mouse Event +ĠS print +_ rotation +Re positories +Ġt art +ĠÑģ в +Ġm appings +è ª +C u +C ycle +Ġb un +ĉl ua +ãĥ ī +Ġ(( ! +Ġcollect ively +ĠCon d +Ġwsz yst +(l ib +openh agen +_s kip +.Column Header +é Ĥ +peri enced +ı è¿° +_p rops +Ġcontr ace +Ġmatch up +ab etic +.m embers +RE CT +(d at +Ġs og +ren om +_M ethod +Custom ers +full name +Z N +re try +Ġk ap +ĠNe u +è Ĭ +add Child +will Return +_p ermalink +Ġener getic +ĠW et +ĠMor r +Ġg cd +count s +, type +d ig +( Login +Ġcr acks +Ġbacter ial +ĠMe at +ĠArm strong +ĠBron ze +Ġapprox imate +_dir s +lig a +ÅĤ ad +Ġkind ness +Ġcont re +ĠE VERY +M ET +Ġannounc ements +g pio +ĠWaitFor Seconds +ĠPhotos hop +Ġdis contin +/ dd +Ġtop ology +an ical +. interface +auc oup +.Hash Set +ARI ANT +(r outes +ĠT eh +Ġh ype +] "). +Ġsl am +Ġbro th +- inter +ĠR id +-m anager +Cancel ar +ĠP agination +Ġsound track +Ġpost erior +Ġscr ub +cre ating +- * +ir teen +.d y +.s ymmetric +Ġ"" . +============ === +Ġch assis +ĠnumberOf Rows +Develop er +_b ins +ĠO UR +ri eb +Pro s +Ġwi ÄĻ +" d +Ġasync io +ze igen +_s pi +.A LL +Ġscre ws +Ch inese +Ġapi Key +Ġun successful +ĠSeah awks +OR G +ç« ł +Ġprofession ally +ĠCou pon +åŃĹ æ®µ +Con vention +Ġpol ym +æī ĭ +Ġsalv ation +Ġengine ered +ĠW rest +ĠG CC +Ġwar mer +Layout Constraint +Ġag grav +Script s +vent ure +Ġrefriger ator +Ġinnov ations +ĠRun ner +N IC +ĠRoll ing +Control Events +Ġlo os +p ac +ĉ panel +ef e +ĠBudd ha +------------ --Ċ +åº ĵ +(for Key +Ġl umin +Ġ( ? +ĠA IDS +, user +im ientos +content Type +ant lr +é ¦ +ĠW elt +Produ ction +m ight +ĠV II +", ( +Ġobserv ing +Ġdeliber ate +( control +Ġwith d +Ġsem ana +ST ACK +uch en +N ice +ĠDeutsch land +ĠSpec ifies +d ma +iz io +ĠF acts +_pop up +ĠDirect ors +{ : +[ R +ĠÑį леменÑĤ +Ġpl at +Ġdirect ing +ä¸ ī +ĠGil bert +â̦ .ĊĊ +.q ml +Ġthere after +Ġdis position +d raft +Ġsurge on +ĠIns ider +Bl end +ĠT rev +tr insic +Top ics +rie ve +_FILE NAME +Ġaut res +J ose +Produ cer +er us +Ġpet it +ĠN EXT +ĠF ilters +Ġreplic ate +"] ). +Ġl enders +] ",Ċ +; charset +Cpp Object +Ġfl oral +ĠT ipo +Ġcirc uits +e asy +(& $ +itt a +ery l +_COMM ON +'}} >Ċ +-back ed +(var iable +( Index +Ġvo ir +_loc ations +++) { +ĠLouis ville +Ġgrat itude +.Mock ito +ĠP owers +ie urs +Ġge ographic +ra le +Ġc ra +ĠSp urs +iph ertext +AC ION +- common +Ġvict ories +ĠFinal s +.sh uffle +-m illion +_PRO C +ass ume +Ġil s +DB C +Boot Test +Ġl avor +.test ing +. ast +"] / +m oid +Ġqual ification +ges ch +ĉ put +Ġair ports +J I +Te acher +_un iform +Ġn ama +ĠB ast +ert ype +c apture +get All +ĠReyn olds +oo led +.com ments +Ġch in +). * +Ġи ли +t gl +ud os +Ġd ÃŃas +ch ai +.pro gram +Ġps z +ĉ icon +ph il +ent ral +_WR AP +ov i +Ġnost alg +In finity +ĉy ield +Ġvit amins +Qu aternion +S ink +_g oods +Ġ ........ +ĠW ings +ur idad +-st ory +"] )ĊĊ +idel ity +Type Def +G tk +Ġí Į +_M ain +Ġche z +ĠR aven +Ġpay roll +Ġfreel ance +LL U +ĠM end +ed ay +Api ModelProperty +.Form BorderStyle +Ġeconom ist +stan bul +Ġfre ight +-A gent +(m eta +Ġsym metry +Ġ' .. +.C alendar +- aut +g f +p ent +yc lopedia +Ġwish ing +ĊĊĊĊĊĊĊĊ ĊĊĊĊ +Ġgentle man +Ġê ³ += # +Ġlect ures +âĢľ In +Ġ! _ +Ġh b +ĠV endor +Recent ly +_n otes +æıIJ 示 +" My +Headers Height +_S O +Ġunw illing +Ġsuper hero +g io +ps y +ĠPe er +j avax +& apos +ĠCr isis +ord inal +Mem cpy +++++++++ ++++++++ +- val +Ġwork book +- ap += k +Ġmetal lic +_ peer +By PrimaryKey +_S D +u ator +_SH ADER +) Math +.Trans form +Ġc ows +Ph i +ĠC lem +(_ (" +ĠL ud +-d elay +ĠSec urities +ĠOrth odox +Sym fony +(re port +Ġent ertain +E PS +iz oph +ex ual +IR D +ä» İ +Ġl ith +Ġsanit ize +Ġfemin ine +IS BN +.auth entication +_p ipeline +/ constants +ĠCON F +Ġluc r +ric ia +.t tf +.set Content +Ġst an +ore an +ĠL loyd +.raw Value +Ġg or +ĠBrow ns +Re gression +Ġlower ing +na issance +Ġbl ows +Ġam azed +Ġun related +Re views +Ġrub y +ĠMod ifier +Ġgi ants +. thread +Ġcontain ment +ĠStart Coroutine +um at +ore lease +ĠR andy +@ endif +D igest +Ġsubur ban +=" );Ċ +Ġann once +. variable +\F oundation +Ġa cre +V an +Ġt uples +d ns +ĠStand ing +_l arge +Ġbox ing +Support ActionBar +ĠFort une +ĠR um +_m ultiple +arch ical +Ġf write +_ quote +Ġfool ish +Ġcompr ising +Ġо п +- selected +v f +ma id +N ama +(d atetime +Ġindirect ly +g art +fix tures +ch os +ĠH alo +Ġrec urring +- news +v il +ĠNurs ing +- produ +ĠH Q +\Http Foundation +enc i +au en +Ġv y +ocr acy +Ġdeleg ation +Ġas phalt +Ġset Selected +k ok +/ rest +met ics +ĠNS Date +Ġtravel led +Ġrec ib +Ġm ime +CL IENT +ĠG U +ĠH ANDLE +/ Q +[ z +Ġbother ed +ĠBB Q +ç as +_ex amples +_F IN +Ġwhite Color +Ġastr onom +-d ir +Ġsovere ign +Ġb reeze +Ġin ning +ĠEd monton +g li +.blog spot +js x +Ġvers a +ĠMoh ammed +.J ob +-t oggler +Ġп олÑĮзоваÑĤ +ard on +Ġnew born +Ġnav al +note q +Ġtum blr +Ġh entai +ĠTyp ically +Ġlo ot +.S prite +Fl ight +Ġw avelength +-s k +ĠEl le +_ exports +Ġ Ñı +ĠI H +izoph ren +Ġí ģ +_pr imary +Ġmo is +ĠB N +Ġsystem ic +Ġdifer entes +IN CT +Ġ'' ĊĊ +$ q +Widget Item +cl ide +$ file +L emma +/ table +ag rid +ĠMongo DB +int e +Ġapp rent +ÂŃ ing +.D b +Ġà Ĥ +ham mer +=' ';Ċ +Ġbro kers +it lement +sembl ies +E le +{ x +Ġlast name +< - +Ġfl atten +_b and +.R oot +.read FileSync +==== == +.r x +? čĊ +Ġmetaph or +T i +con te +Ġdeb it +Ġcont empt +Cpp Type +æĶ ¯ +Form Field +r atio +os opher +Ġimpl ant +P URE +Ġal ta +_man agement +Ġref ine +ĠCheck Box +ĠChar l +- version +cond itional +ven ues +Ġrif les +Ġoff spring +Ġmill ing +Ġshar ply +Ġunder water +( origin +_ Control +Ġ. $ +Pl ugins +Ġdry ing +Ġillustr ates +- u +Ġveget arian +n pc +He art +; ',Ċ +com ma +te enth +as an +/s pec +_m oves +-m argin +Ġing en +³³ Âł +Ġpro jet +Ġo tra +Ġbr as +. utc +Ġsle pt += sub +ab ilit +post er +Ġs dk +ounc ill +Ġw d +Pre paredStatement +ĠDr um +( attribute +ĠEther net +ĉ DB +Cal ifornia +c ube +[ I +.C reated +ĠH M +Ġtr acing +Forms Module +- you +.c urrency +feed ing +Ġt body +L i +acc ion +n as +Ġtr ouver +N ONE +"} ,čĊ +Ġf tp +With Identifier +pol ate +File Info +Ġpurs ued +ĠĠĠĠčĊ ĠĠĠĠčĊ +DE SCRIPTION +} */Ċ +From Nib +Ġdecor ative +_S SL +(ch at +T LS +Ġsurpr ises +al culate +ĠS plash +( Configuration +ĠS EM +im son +/lib rary +< Double +. robot +³³³³ ³³³³ +ĠCP F +ĠUnder standing +Ġcos metic +ĠX t +t ips ++ k +(" ' +ĠP DT +W AR +.get Object +ĠTrad itional +.sl ug +ĠDi pl +=" ", +ĠFil ms +ĠAn im +.h elp +Ġemb assy +ĠBoot s +Ġb unk +-r isk +Ġp ci +Ġ/ \. +ĠI PT +Ġcrash ing +Ġip v +_ ke +ĠRES P +.Log Error +Ġinade quate +I on +ĠF ür +ric ula +Ġshould Be +al ready +']." +G ED +fa q +Ġoption ally +_D is +ĠSuccess ful +ĠC ensus +Ġinc arcer +_C ARD +Ġav iation +ĠG ym +Author ity +.B ean +sh ader +Not Exist +_Text Changed +ĠST OP +( team +" H +w g +Ġgr inder +Ġstri pe +Ġpres ervation +Cl aim +avers al +ware house +target s +Tr ust +Ġal lev +, www +ous se +_ch an +_S ize +system s +Ġobj ection +ĠK ane +Ġcor ros +ĠD SL +Ġu a +ĠM H +ĠStrateg ic +_t cp +Ġê° Ĵ +Ġborrow ed +ĠA ch +ĉ command +Ġg ps +le ston +iche ver +ĠU A +Ġassault ed +Ġspecial izes +ĉ search +Hot el +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +ĠP itch +Ġ Ùģ +READ Y +Ġparent al +Ġg éné +Ġdonn ées +Ġdet ain +T ARGET +Ġprotagon ist +Ġclear Interval +ĠIcon Button +ĠGet All +Type Info +E H +âĢľ They +Ġ{ [ +Ġg ag +Ġ Ú© +ĠD ropdown +.f ree +g one +im ens +Ġinst al +ĉc url +_C AN +ĠB one +ï¼ Ķ +ony ms +-g overnment +.binding Navigator +ĠD ans +ĠMc L +( en +>( _ +ÐĴ Ñĭ +.* ;čĊ += j +-c or +S on +.ToolStrip Item +- around +_X ML +end Date +Ġsl ack +Ġrot ated +Ġno qa +Ġc ottage +Ġencontr ar +_s kill +hou ette +! čĊ +. weather +Ġemphas ized +å® ¶ +ĠÑģ пиÑģ +ĠComp iler +( android +ĠâĢ º +. turn +Ġsup pression +_c alls +Ġ* @ +(str len +.h ex +ĠB ills +ĠR SA +Ï Ĥ +ĠEs cape +ement ia +Ġfront end +Ġp int +_ex c +zz o +[ ],Ċ +Ġ"',' " +. Environment +Ġafore mentioned +Ġend ure +prot otype +ther apy +ss i +D eg +_pl ugins +.user Info +Print er +ĠPRO GRAM +Ġru ins +Ġempir ical +Ġcraw l +ĠBo iler +- comment +.sub plot +_ et +Ġ'. ', +min or +ĠCustom s +Ġy aw +under line +ĠCom o +( (' +(m ean +Ġcha que +ĠBlock s +.r ad +ilib rium +Ġweb driver +Ġmel hor +d ana +ĠAb use +ĠSouth west +ĠP aren +PERT IES +ĉ IL +Ġscre am +v u +Ġin comes +Ġn im +Ġl ace +Ġcompens ate +Re verse +D at +_att ack +Ġn our +ach en +ce k +< Func +w ie +com pressed +-m atch +(" ")]Ċ +im ized +. orientation +.compare To +Ġmass aggi +Ġìľ Ħ +Ġel bow +Ġant ioxid +undred s +/ tools +ĠR OW +an mar +ĠW ow +_t icket +Program ming +Ġthe or +-re view +() )));Ċ +ĠRichard son +ĠP ocket +] [] +am pp +_ health +ĠP OP +ĠNav al +Gu ess +Ġancest or +.Get All +.local Scale +ĠM apper +Ġaccum ulation +Ġsim ulated +ĠDr ivers +Ġd és +cur ring +Ġele phant +Ġadvert ised +Ġmail box +SH IFT +ĠMon ica +Ġan c +Ġward robe +Ing redients +Ġ|| čĊ +ipp y +Ġantibiot ics +av ings +(c x +ĠFerr ari +ĠAn imator +.d type +rem oved +order by +Ġc res +oc ê +Ġp ym +ĠCirc ular +@ index +ĠW arm +S ay +ĠAss istance +Ġcur tain +ĠMont e +IL ER +ĠC VE +ĠD uck +ĠAll ows +_f ire +ĠDer by +Ġre pos +Ġhttp Client +Ġpsych iat +Ġnow adays +Ġcaut ious +ĠComput ing +Ġcompletion Handler +ĠWel sh +ĠB EST +Ġstress ful +_P E +æĹ¥ æľŁ +ĠData Frame +ĉ Integer +_P rint +M oves +Ġtransform ing +.B atch +y ahoo +Position s +ze j +Ġno od +io res +_ * +Ġcl k +ĠF loyd +Ġh ap +font size +Ġn az +.not ification +ĠDep ression +Ġac ne +*** ĊĊ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +.cont ents +yn th +ĠStra ight +')}} "> "+ +Ġtoken izer +Ġsovere ignty +ĠP ence +() ");Ċ +Ġpesso as +.G e +ĠIn cluded +Ġpag ina +Ġex posing +е ÑĪ +_SC RIPT +/$ ', +Th umbnail +× Ķ +webElement X +webElementX paths +press ure +ĠCur ry +_C P +OL UTION +ILE S +prot ect +ool a +Work space +{ };Ċ +ĠU NS +Ġsymp athy +ro ker +Ġrem odel +ĉc ell +Ġat op +.Full Name +Ġfa ut +ĠE asily +_d ynamic +Ġfr amed +Ġmot ive +è· ¯ +s am +Ġmar ca +ĠText EditingController +Ġde structor +cre am +Ġr ude +ĠB old +ĠInd igenous +Ġg ens +Ġrel acion +(s ystem +ĠUIF ont +_char ge +UST ER +E V +.N amespace +Ġmer ger +Ġcal loc +g ang +Bad Request +Ġs per +-d esign +Ġâ ĩ +Ch an +Ġorgan ism +, ) += id +_pl ane +ĠC ases +elf ast +ĠLegisl ature +ĠF aker +Ġinv oking +- utils +(). ' +.f ace +Ġguard ian +my Modal +Ġclip board +ĠAT M +Ġpe as +ĠS ylv +.c alc +ĠContact s +int Value +Ġmodify ing +ĠBar b +. loss +_per centage +Ask ed +(l st +ategor ical +- files +ĠRoman ia +.A c +Ġh ai +ĠF lying +Ġ ż +j p +ĠTr ainer +. arc +_de g +Ġtrace back +Or Fail +F LOW +. old +oy a +g mt +is empty +Ġvacc ination +Ġob solete +recogn ized +Ġru ined +ĠRe in +ĠTr acking +xf b +ا ÛĮ +Ġvæ re +Ġbr yster +ĠIT S +Ġdest iny +Ġsw ear +Ġred es +Ġcl f +Ġfl ipped +ĉ head +Bl uetooth +ĠOver rides +: Boolean +_ = +_l r +sp awn +: index +VAL UES +is key +? ");Ċ +.syn thetic +ĠCheck ing +struct ures +ip ing +Ġvoc als +- Up +ĠManufact urers +ĠMar riage +代 çłģ +Ġgar ner +_C lient +par allel +RI END +Ġvine gar +seg ue +J B +Ġcontact ing +ĠCar roll +Ġout reach +t ensor +_var iant +Ġthe at +lic able +{ | +t iny +_ letter +Ġp encil +HeadersHeight SizeMode +ilt ro +.auto configure +.d rag +.use State +ĠB MI +h int +Com pile +* \ +en ary +Ġl vl +.C ache ++ =" +_t v +ruit ment +Ġf read +Art icles +f ila +Ġpack aged +âĺ Ĩ +AT HER +ĠPl anned +s cheme +Ġdi ary +Ġoff enses +/ F +ĠSt ick +Ġc erc +ĠS lee +ĉĉ ĠĠĠĠĠĠĠĠ +< Image +Ġè® ¾ +- editor +pie ces +ĠD rama +Ġ// //////////////// +ĠT asks +AR C +g ateway +.get cwd +.M etadata +Ġguess ing +åľ° åĿĢ +Ġsm arter +ĠGet Enumerator +Ġe fter +/ operators +ĠGL float +Ġf ør +Ġop aque +ä¿Ŀ åŃĺ +Sp read +SY STEM +Ġinv ersion +ĠBasket ball +Ġsim ulations +Ġden ies +Ġa vez +_list ener +Ġenh ancing +ĠMy th +ĠL akers +_M D +Nd Ex +D ATABASE +Ġt á» +ar th +[ left +Ġcontest s +st ile +(K ERN +_f c +_p m +Ġpres idents +Ġhospital ity +Ġfade In +RO PERTY +_m aps +ĠDefinition s +Ġassess ing +Ġus ar +Ġquant itative +mo z +Be autiful +[ (( +b ons +f requency +Cont ain +Ġpuzz les +ĠCast ro +Ġv illa +Ġkind ly +Font Awesome +ern a +epoch s +_dat as +ĉ ip +.p adding +ĠCont est +Ġed itions +Ġdispro portion +ĠI CO +Ġcome back += value +ri ad +-s ort +Sub mitted +(n etwork +ĠC el +Ġinstall ment +l ashes +.List View +ĠV atican +(Media Type +IV ED +reach able +: Is +ĠC ITY +äº ¬ +ĠHelp ful +Ġba ÅŁ +% čĊ +Ġpsych iatric +Ġrec ycled +FORM AT +ĠG row +b ine +G it +.s s +ĠWe apons +ĠSt y +_ arrow +* self +ire ment +Ġdeg li +App Delegate +_b anner +Ġcoordin ated +ĠWeb cam +Ġcelebr ations +. act +******************************** **************** +( show +Ġweek day +Ġconc erts +ол н +cl in +Ġcr on +ĠN im +.set Vertical +ĠEll en +س ت +ĠS AM +E ff +g z +ste am +Ġant ique +ph ysical +ĠForm Data +.set ter +ĠPO INT +B on +Ġflav our +erv ention +_ENT ITY +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠ +Ġintr insic +Ġæ İ +append To +aram el +) ]) +ĠRecomm end +) m +OutOf Range +Ġkn ight +Ġsat ellites +ĠTit ans +Ġweigh ed +ĠD ana +e ase +Ġs ip +S IM +ĠDevelop ers +mal ink +/ check +_P LL +n ung +Ġdry er += A +.d w +_S QL +Ġsub plot +D ROP +Ġprot otypes +Ġhour ly +display Name +Ġas i +ĠViol ence +Ġastr onaut +Ġdat atype +Ġinformation al +Ġinvestig ative +etermin ed +ren al +; '> +ĉc ol +V G +_ boolean +re cent +Ġ* )ĊĊ +ĠRain bow +om men +Ġl ur +Ġopp ression +(", ");Ċ +ĠFac ility +DEF INED +Ġne on +Ġoff ender +AF P +ĠClean ing +[] ): +Ġund ocumented +.Re positories +ĠG uitar +аÑģÑģ ив +Sk ills +Ġtestim on +rypt ography +ĠAm ber +ĠSt alin +Ġl one +Ġap enas +Ġdies es +ĠAr duino +è½ ¬ +== - +_A ct +Ġc oded +âĸ ł +amb urger +-link s +Ġarm our +.H igh +get Content +st ag +Ġhe ck +ĠìĹ Ĩ +ĠMc Connell +ĠCon cert +ĠAl loc +ä re +.replace All +Ġpart itions +rot t +ĠF le +_T REE +reason able +ĠReport ing +Ġbillion aire +s cores +min s +- eye +M ORE +ab ort +ĠSW T +Ġin verted +ĠTe achers +; n +Ġast ro +н ов +ани ÑĨ +product o +c ountries +ĠO wen +Ġcont amination +Ġv ibe +ĠEll i +.s cript +ĠOl ive +D MA +v ier +: semicolon +-m odule +gress ive +ag u +_ players +Ġresult ados +start ed +scroll Top +==== = +Ġweigh ing +Ġ[[ [ +z ahl +( NS +ĠAssert ion +le ague +.setText Color +ĉ Message +Ġmom s +_A F +. wh +AL S +Ġaut re +] ĊĊĊĊ +.op acity +ĠBudd hist +Ġde af +ĠOrgan isation +(G lobal +ens ch +Ġhead ache +ĠAli en +_in ode +ĠSt ark +Ġæ ī +-l nd +ore f +_fe at +Ġpedest rian +Ġnom inal +Ġbal loon +Ġspr ites +Prototype Of +ĠA post +ĠF EATURE +O H +Ġre cess +ĠDon na +con sumer +$ GLOBALS +ĠG IF +- frame +In icio +Ġpass ages +Date String +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +.by te +B ug +initial izer +p kt +od ium +ĠD ER +. ops +ler i +Ġgift ed +Ġdet ach +ter rain +elt ers +ãģ ı +. loader +ĠN GO +str ncmp +K h +(font Size +ro cket +Ġpreced ent +ĠAur ora +ĠEx periment +is phere +Enc oded +ĠâĢĵ ĊĊ +Ġpy ramid +ĠAnn iversary +of il +ë Ł +( plugin +C oeff +Ġcooper ate +Ġpredomin antly +IS M +Ph rase +_DEF INE +Fl ip +AMIL Y +ĠMark ets +ĠStream Reader +ĠComb ine +Ġmanus cript +z za +, tp +Wh atever +IT ICAL +ighb our +Data Provider +.Text ure +priv acy +.S DK +Ġre charge +Ġc pp +ĠC FG +(h older +(p y +m ot +Ġsav oir +ĠR osa +ĠPC s +Ġí Ļ +.her oku +Ġf ren +ĠR iley +ag ate +Ġs ond +.x lsx +Ġh acked +st ad +G i +Ġsan ity +ĠSql DataAdapter +... ", +ĠP ussy +Ġ **************** +Ġhass le +_P ARENT +ĠU AE +Ġbegin ners +( Client +Ġstatist ically +.h our +ed elta +Ġtr action +uel ve +ar at +Ġsa una +IN VALID +Ġindict ment +AL LE +Ġdiss ent +ĠTyp ography +Ġintention al +s it +ĠAn imals +Ġcoun tryside +Ġu art +} \" +Ġseam less +¾ 示 +Ġaut os +Ġ"' ";Ċ +Fl ush +ANN OT +Ġal gebra +ass oc +ĠW aters +Ġprepar ations +ron ym +[, ] +S ans +Ġarm ies +ipe g +Ġcream y +. art +et re +ĠAn imated +Ġun pleasant +eme an +g reat +i Äħ +ĠEar lier +Ġch ic +Ġpres erving +(ex ec +ĠInvest igation +ĉG PIO +Ġrig orous +ij o += num +Ġtool Strip +) set ++" & +ĠAcc eler +Ġdevelopment al +is posable +Ġflaw ed +re ne +Up dating +Ġwatch dog +Ġden ominator +Ġsubur bs +Ġ... ) +Ġconv ictions +c losure +.I P +Ġtransl ates +.sw t +.Tr ace +Ġmet tre +.is Enabled +ĠEffect ive +.to Int +Ġen chant +Ġst unned +Ġpo i +/ code +ad m +.datab inding +ĠL orem +________________________________ ________________________________ +Ġled ger +Ġcar a +ĠG ir +Ġwa its +Un o +Ġc wd +è¾ ij +ĠT Result +Ġre jo +Ġem itted +ĠWest minster +ä¸Ģ 个 +ne k +_T is +Ġen act +ĉ with +org ia +Ġj ue +Per form +SP ATH +.top ic +ĠD aten +Ạ§ +Ġsit io +_M M +" So +b ial +Ġsc oped +Re quires +ĠT OTAL +ĠCh ancellor +( contents +Ġste alth +dev ices +-p ass +ili h +ĠMal colm +ĠDep ot +Ġconfig ur +a ussian +_con straint +в еÑĤ +G RA +ĠR ates +.dataGridView TextBoxColumn +ĠNob el +it ics +Ġignor ant +ĠReport er +ĠEb ola +ĠSh ock +_re lation +ĠNin ja +) c +Ġt icker +.is Checked +ĠSup pliers +ĠRap id +Level s +âĤ¬ âĦ¢ +ĉ queue +Ġch op +ĠUn ix +re ject +-c alendar +(s ort +è ne +erc icio +Ġh ect +CALL TYPE +rou pon +Ġrent als +auth ors +{ name +ĠF IFO +Ġl assen +ĠN ous +Ġsn apped +Ġfert ility +" log +click ed +Ġplant ing +Ġg b +/ output +PE AT +Ġc ategoria +Ġb ach +Prof essor +in th +"] čĊ +Rec order +ser de +ĠTrans mission +tr ad +Ġtur bo +_VER TEX +\ Event +il ver +Ġbod ily +ĠS ources +Ġkill ings +.xr TableCell +Ġfold ed +/ legal +un er +ĠR ifle +ĠM IDI +_Selected IndexChanged +.Size Type +ĠWeb Socket +Ġsele ccion +S and +ot ros +Ġenv ision +/ etc +ĠMel issa +Sp ot +но е +_ ARM +At tempt +ĠB I +ãģ Ķ +ĠD U +Ġback lash +str ide +/ classes +Ġtext Color +_st aff +ob lin +agent a +.c ollections +ill age +' čĊčĊ +fl atten +_s ales +_M ASTER +T W +_d a +P itch +ph ies +Ġz ombies +ĠV ERY +ĠPharm acy +Ġprogress Bar +Ġhas htag +S idebar +@ stop +(p c +ол ж +MA KE +ĠCor on +Ġkv inner +ĠM aid +b ob +.title Label +Ġsuccess es +ĠDemocr acy +ĠSurg ery +Ġcou gar +Ġcur so +Ġl oro +ist ency +Sen ior +æ k +ĠA AA +ĠBO OK +к о +W STR +Ġ*/ ,Ċ +oy al +.v ector +ĠS PEC +SS F +Ġcomp uls +ĠAppe als +ĠW inston +ĠMock ito +con trib +. available +entity Manager +ari as +_s ale +_r s +Ġdec oding +Ġloc ator +ol ith +Ġk ol +Ġasc ii +ĠR ut +/ interface +ĉĉĉĉĉĉ ĠĠĠ +ĠN umer +.fl ip +-d el +Ġbol ster +on omic +Ġz m +L G +Find By +Ġadapt ive +lo o +Ġv ue +(re verse +_c anvas +. roles +ific ado +ven ient +" As +ĠEn tr +al igned +Ġbere its +/// ĊĊ +.g wt +. employee +_cl i +Ġanticip ate +éĻ IJ +Ġp ik +Ġmush rooms +(t t +Ġo ma +ĠSan chez +_g oogle +. Valid +ĠFile Name +iv ative +k ed +-w ar +Ġm aturity +и д +Ġmin er +Reduc ers +ĠLat Lng +_ST D +D igits +Cal c +-up load +Ġhand ic +ี à¹Ī +egr ated +ĠST M +C lients +ĠTur bo +SY NC +Ġphotograph ers +. Out +.char acter +B UILD +.un lock +Ġar ises +ĠCommand s +(" ");čĊ +_F ORE +; ', ++" ' +. Images +") { +ĠM eyer +Ġneg atively +ĠD LL +Ġex e +Ġdef iciency +Ġwild ly +-s witch +con struction +Ġexception ally +ĠL iz +/j ava +Ġtheir s +ĠCont emporary +l is +.fill Rect +ĠN FC +Ġre he +(num bers +Ġr aster +Ġfig uring +Ġshow c +ĠJ ill +Ġarc ade +ĠConstruct s +md l +(' | +Ġident ifiers +Ġst ellar +( Connection +Ġ" {{ +y or +(m ysqli +Ġdo ve +Of Birth +.dis connect +_h i +Ġzw ischen +ĠGr und +i ros +_A rray +.on click +ans om +An swers +ĉ remove +F a +Ġhur ry +-in f +Ġget Class +ĠReg ulation +ĠFLAG S +m isc +K en +_ heading +G Hz +- entry +Ġbi ography +S ig +-m f +Watch er +âĢľ A +} px +Ġsp icy +_s q +L ost +(tr ack +а ли +Desc ending +< bits +qu ine +ĠAdv oc +_S N +ĠHann ah +PO P +Ġem itter +Ġc yn +ĠC AD +? ). +/ set +ĠS ister +ĠEnd point +Ġmen or +Ġinter p +r k +id le +Ġout fits +. vertex +Ġc lic +ARE N +Ġpost ure +ĠOpport unity +v x +ĠFor bes +.D irection +Ġres ide +Ġremember ing +nest y +Auto resizing +pro viders +ĠA H +Ġhur ting +ĠL ily +eval uate +lij k +p apers +ĠSm ash +ĠL AST +Ġwell s +w asher +_RO LE +ĠD anger +* (( +_re pository +ĠRes olve +ĠRoom s +_R G +ĠQ T +o op +ĠHe ap +Ġslow ing +Ġgrat uite +_c atalog +Ġpol ynomial +L y +pc s +F ox +ĠC yr +Ġdim in +/ month +S alt +Ġh ind +.P ER +For um +c en +_p ol +íĺ ¸ +Ġin ser +( ~ +@ test +ĠGold man +Ġupload ing +F c +Ġkom mer +Ġm itt +_log ged +Ġbu cks +-l ayer +) };Ċ +ĠO M +Ġv eg +col our +Ġоб ÑĬ +Std String +_ que +ĠT ian +Ġspecial ize +и п +Ġк л +tr ial +- edge +Ġm ars +OG LE +Ġempath y +ĠB om +Ġcoll isions +Ġcart e +ĠTe il +ĠM PL +Ġporn ô +Ġa irlines +A ws +N s +ĠSp awn +( use +é» ĺ认 +Ġy acc +st or +Ġconf ess +Ġpe que +r age +? "Ċ +/dat atables +ĠSh ower +__ / +Ġcryst als +Ġbus car +ĠH aus +iz ação +_ entities +ķ Į +ļ Į +x cc +v irt +-che vron +( Result +c ake +COM E +Ġprohib it +ĠCh ess +Ġbe aucoup +ĠÑĩ ÑĤо +R UN +ĠI K +ó ÅĤ +_ Update +Ġsle ek +ĠSpec ify +_c redentials +ÅŁ t +ĠUser Name +ĉ Value +Ġarray List +Ġex changed +ips is +.re lated +ĠSe ite +_B AR +ĠL em +ĠW ATCH +ĠC lients +Ġ. * +ĠEar l +-re port +Ġforeign ers +Ġstrengthen ing +ĉ Description +(g o +.tool bar +Ġcalcul ates +ĉs ource +Ġcz as +Ġre cl +ab o +Ġlocal host +Ġ^ {Ċ +.P op +ĠDes igned +\ Abstract +H old +ĠGuid elines +ipl ine +Ġc aching +.Re ader +_ext ernal +.str ptime +ĠWeek end +-M ar +ĠBe i +Ġ{* } +ĠR ud +Ġexpl or +ĠBou levard +C ash +Ġprep ares +Ġserial ization +ew ater +Ġad c +: ĊĊĊĊĊĊ +Re fer +Ġsc anned +} }ĊĊ +ĠF ul +Ġtour ing +ãĥĥ ãĤ¯ +> (( +sur vey +Ġí ĺ +... ')Ċ +ĠDiv ider +os l +_C ANCEL +_pre pare +st in +ĠHe ath +.Primary Key +ĠâĨ IJ +ĠLocal DateTime +Ġcooper ative +L earning +.en queue +Ġgo og +ĠReg ression +im ates +Ġvoy eur +ĠDr ink +pl ug +Ġl ender +man a +Ġperson nes +yp se +Ġun link +ĠRav ens +Ġhur d +Ġperiod ically +ARG S +ĠG H +char acters +... "ĊĊ +- establish +Ġd n +( condition +ĠGr avity +Ġest as +_f ocus +Creat ure +(s ite +Ġc arr +ĠR L +ĠR I +ĠM oto +AS F +ĠLuck ily +ĉ Route +Ġent ropy +(" ," +Col lect +( contact +ĠFlo rence +Ġpremium s +Ġlif ecycle +Ġb ans +x ef +Web Kit +ĠFlo ating +Ġcos a +Spec ific +ĠLo ans +b read +Ġdes criptors +Ġ{ :. +TH READ +ĠT rent +Ġsc op +Q A +ĠAnt ar +p el +_d ifference +_ch anges +(... ) +ĠR otation +ĠLG PL +ĠJ UST +(T ask +_sub set +ĠTR ANS +åĬ Ľ +ĠSc out +-p opup +Ġsm oked +_C lass +Ġturn over +br akk +ĠRock y +t as +.Regular Expressions +ĠElli ott +ĠSp inner +DU CTION +Ġlib re +Ġmol to +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠ +ĠF TP +m peg +(f eatures +Ġb ald +ĠV id +Ġsh outing +L int +Ġsock ets +Ġpro w +Ġnouvel le +isc ard +ĠS ponsor +Ġconsult a +)) ); +Ind ian +ĠR aspberry +Ġteam mate +ĠJ WT +ĠGh ana +Ġc akes +pr imer +form a +erg arten +_M anager +Ġpre season +G AME +| " +ĠBro ck +Ġoccup y +Ġdecor ations +á nd +Ġc ot +Ġpar an +D isk +rem ain +> ? +Str ong +Ġfr ance +ĠE ra +-c r +.Buffer edReader +ĠParad ise +ĠV AT +ĠAnd ers +Ġlim b +amp oo +Ġimper ative +UT ILITY +ĠRec ognition +Ġragaz ze +Ġpop s +yp ress +Ġemb argo +// {Ċ +Ġsy ll +P TR +åŃĺ åľ¨ +Ġdid nt +Mail er +Ġacad emics +ĠFra uen +ne ider +- rel +Ġrain bow +( In +Ġslic ed +============ =Ċ +(s end +NSMutable Dictionary +v os +(p ackage +Ġord inance +view er +ĠSant os +-s elling +Ġgo v +ett le +Ġfound ers +Ġw aking +sl ashes +-p ound +re cht +ا ت +.on Click +Ġn ord +st änd +_ when +UT ERS +ic c +Ġcaps ule +ĠW id +M arc +ภ¸ +ro red +UG E +LO UD +ĠAud it +ip ients +op ian +ĠS ue +Ġwur den +.H elpers +Ġf actions +[ np +-th an +Ġre co +Ġk as +Ġcmd s +/n etwork +xb f +get Color +Ġbi ased +ĠL ak +D atas +vent s +Ġë ² +_P S +. Validate +Inv oker +Ġne uen +Ġju venile +V ISION +Ġdev ote +Ġlin ha +Ġdiscount ed +\ Config +Ġworth while +Ġskin ny +ĠC ourses +le ys +ĠMort gage +K evin +Ġannounc es +]) * +res ervation +Ġæķ ° +Ġprejud ice +ĠString Comparison +Ġbe ard +-w in +ĠS ão +ĉ ms +j al +ĠE arn +_ ports +ĠN ombre +_C OR +ĠB UILD +.s ound +Y ellow +Ġlineback er +Ġchar itable +j ug +_NON NULL +ĠD ental +"> ${ +ĉm atch +R ussian +Ġvers ch +Ġp inned +Ġadopt ing +Options Menu +P ag +Ġpair ing +Ġt read +erc ises +ĠSp read +) i +ĠB AD +_t f +UI ImageView +pop ulate +b ab +ĠÏ ĥ +[ ++ +Ġopi oid +Ġ## Ċ +d type +ĠStart s +('/ ') +Ġperson als +-mark et +Ġredund ant +ĠEss ential +Ġscrap y +Ġи м +a cl +Ġcre ar +ĠB end +Ġrel ieve +- room +w ife +Ġv Ãł +ĠQ Point +Ġqu asi +Ġmethod Name +\x c +ĠPer u +/ The +. orm +Ġv iz +/p df +Loc ated +Ġconfront ation +ĠChampionship s +Ġhyp ert +Ġd j +ĠUser Info +ĠåĪ Ľå»º +\x b +(s im +Ġ== Ċ +Ġst aging +Ġdr astically +åŃ ¦ +l ords +. less +вед иÑĤе +ĠB ucket +ĠM am +. term +_p i +c zy +.p ub +prec io +ĠV irt +Ġrom an +it at +L ex +_inf os +Ä ° +. other +VE LO +Ġp onder +Ġh anno +( Page +do i +Ġpol ite +Ġprogram mer +D ies +$ d +Ġrep lication +add Column +fr ican +Ġl eng +be er +o it +Ġw asting +yl im +me asure +N eg +Ġpart ie +.con sole +ĠGu inea +TE L +_f act +.ch unk +Ġl ent +Ġall er +Ġठķ +_id le +Ġad missions +JSON Array +Ġv ibration +.h elpers +å¤ ĸ +Ġh en +j ohn +Ġì ĥĿ +Ġjud gement +Ġge en +ter ra +^ { +ĠI z +Ġc â +inst ances +Ġthreat ens +Ġm üssen +Kind OfClass +Ġstoryt elling +_d emo +ri as +Priv acy +h ift +ĠY i +es or +íķ ł +ens itivity +.W riter +ภĤ +D istrict +.get JSONObject +Im pro +(get Resources +ĠS PELL +rodu ce +Ġslow ed +Ġlin ewidth +Ġhonest y +ĠCo ord +ĠF ork +ĠDispatch Queue +ĠCl iff +ĠW iring +_TIM ESTAMP +oll ah +av oid +++ ];Ċ +sem antic +-c ss +Ġv eto +ĠM err +Ġlegisl ators +CEE DED +Ġquestion naire +ĠP ills +Cal culate +(c ore +' e +Ġdis like +ĠPre ferences +_EX TERNAL +è° ĥ +Ġd odge +æľį åĬ¡ +.n ames +.draw Image +_p rom +uck land +Ġ<$ > +ı z +/s ite +é¡ ¹ +rop he +Ġcomp elled +Ġl aptops +Ġun i +C LOSE +Ġcasual ties +ĠUn iform +Term inal +. "," +D AT +(T reeNode +ĠGand hi +(st mt +AX B +* M +Ġumb rella +an imal +Ġgr pc +Ġwhere by +Ġfloat s +ĉ arg +Ġdb g +Ġexceed ing +Event Type +.SaveChanges Async +Ġ{ {{ +Ġow ed +ahren heit +Ġì § +Ġequ ipo +ur ai +Ġid ol +] ")Ċ +_m ajor +Ġentire ty +inger print +ç os +/ account +ĉ right +urs os +ĠE DT +_INS ERT +Ġsh ining +Ġ< : +Edge Insets +Ġcolon ies +. IM +ĉĠ ĉ +RO AD +CC CC +pl acing +Ġget Activity +em acs +' %( +.click ed +ĠTh em +is ia +Bus car +.re name +Ġo ath +Ġafter ward +ĠU FO +AP S +ĠJackson ville +.s ome +Conf irmed +.s can +ig Integer +Decor ator +sh ield +ress ive +.d id +请 è¾ĵåħ¥ +Ġsh utter +D am +Ġparent ing +ey ed +$ item +-de velop +Ġextract s +Ġdecentral ized +ĠEl sa +_sp in +]) + +-in itial +Ġmult itude +Ġsens ory +ĠMODE L +Ġsafeg uard +ì ¹ +Ġhunt ers +ĠT iny +IN O +decor ate +ĠNo Such +H o +( Response +Ġr uler +ĉ short +Ġc aster +Ġclient Id +Ġp db +ëı Ħ +it ic +ĠGame State +Ġnew Item +)ĊĊ ĊĊĊĊ +ou is +n oc +.BL ACK +_V ECTOR +---------- (); +.get P +any e +Ġneur on +if old +ĠK nown +Bit coin +Any way +ay ette +Ġ' [' +Ãł nh +m gr +Ġcor related +Ġn ause +Ġment ality +has Many +ĠF G +amp ie +IT U +F s +.S p +_b etween +Dep endencies +ou g +Place holder += text +ĠMan aging +ocal ypse +åĮ Ĺ +_m ag +f ld +â ij +C AM +ĠHelp ers +Ġd ost +/ out +Ġassass ination +.get Image +ĠKenn y +.' )ĊĊ +){ // +ĠR anger +Ġg ek +Ġsinc ere +< Value +ĠD OT +ĠVict ory +Ġleg ends +Ġpr isons +(ex pression +ĠR abbit +_s entence +Ġbit es +Ġon Failure +ĠâĪ Ī +K im +.g ender +ĠÎ » +Ġ[ . +"] ); +land ing +-d igit +TE MP +ĉ entry +Ġstrt ok +Ġdesc endants +um no +Ġlean ing +Ġspecific s +q n +ĠSp art +Ġpor r +EDIATE K +Ġse per +' aut +ĠSTE P +ĠBorder Layout +Ġret ros +ĠSalv ador +ĠEN GINE +x dc +T weet +v k +Ġì ² +] << +het ics +c oding +Re ach +.re q +gu ide +.s cope +sh irt +rog ate +SET TING +ĠProte in +Ġe ing +. EMPTY +.d f +Ġclear er +Ġc rossover +ĠTo ys +Ġco ated +.M onth +ĠAtt ach +/ run +.t abs +Ġogs Ã¥ +B rown +.D ATE +Ġf os +åŃŠ符 +W ood +-th ree +her ited +Ġ rop +( ac +Ġembod iment +ĠKenn eth +Ġcan non +Ġb idding +čĊ +.get Resources +Ġl ump +_const s +( ext +ĉd ir +â Ŀ +Ġpadding Top +Ġobs ession +Ġb anning +ĠApp Module +Ġpart isan +Ġcatalog ue +Ġmin ors +Ġpitch es +we ep +Ġundert ake +Ġthem ed +aud it +.scroll Top +Ġr er +Ġsympt om +Ġopen ings +.block s +open id +Ġas sh +-s ave +ĠP ig +Ġreg ain +Ġin icial +/f avicon +ĉ exp +Ġsp ices +isk a +claim s +m ak +definition s +Ġcorrespond ent +ĠCann abis +__ ,Ċ +ĠL ucky +ĠGa ussian +ĠN early +C AD +'] ]Ċ +Ġadequ ately +ĠT ITLE +constitution al +-m m +_ override +Ġbl as +.ready State +Ġremin is +Ġrein forced +ĠColl abor +Ġdecor ating +Ġb achelor +ERRU PT +Ġup right +ip ation +ĠNob le +Ġvalue ForKey +Ġset Loading +.I gnore +å ģ +G lobals +ĠM ent +AS SES +Ġlim bs +ĠH UD +inc i +. iv +ĠQ ModelIndex +F use +Ġped al +_F REQ +( verbose +Ġlong itud +ĠChar ter +ê ·¸ +Ġbund les +. ignore +um bo +EM A +.... ... +s x +.C ard +Ġhe ute +Ġste er +j umlah +Ġ{ _ +_Check ed +Ġf ax +ĠG ust +itch ens +Ġ ))ĊĊ +Ġremark ably +/ XML +- remove +_b t +Ġinc ub +.p ackage +.current Thread +ĠHigh lander +.s ide +s plash +Ġ ici += D +Ġp uck +Ġball ots +Ġhug ely +co eff +Ġp Data +.C OLUMN +ĠHe aling +Ġord in +! ), +Ġ' ',čĊ +(m d +ĠS ask +< strong +Ġsurviv or +.s eries +Ġcaffe ine +Ġ` ( +.TRA ILING +_ Input +(" ^ +z d +& );Ċ +ĠP ing +Ġv oucher +.r ating +-sh irts +ĠRetrie ves +.al ibaba +Or acle +_MO V +Old Data +Ġ/* čĊ +Ġg boolean +Ġ=> čĊ +Ġr á +Ġbl unt +ĠImage Icon +if ik +RT C +Ġfib ers +Ġto ile +.s ent +ĠPy Qt +$ app +Ġmed io +Ġgrant ing +Ġtsl int +ĠM ö +(fig size +Ġhur ricane +Ġlif es +Ġà Ħ +rocess ing +_st andard +- option +')) ) +Ġvac ant +å· ¥ +ĠH ollow +handle Change +Ġdiv ider +ĠEngine ers +Ġsv ens +Ġcompl iant +t anggal +ĠC redits +ĠEm irates +Rule Context +Ġreal ization +Ġdistr acted +]+ = +Ġaug ment +ĠD w +ot p +or rent +Edit ar +.st ock +St udy +pe ctions +ĠGame Manager += cut +Ġf lock +ĠRom ans +th em +-h op +Ġscreens hots +Ġ/* !Ċ +Ġconvers ions +Ġnormal ization +(config uration +Ġa eros +_se curity +! 'Ċ +B onus +ĠDR IVER +ĉ Date +t ie +ĠWy oming +St and +it re +Ġsh oppers +Ġdisadv antage +Ġlik ing +ç¬ ij +Ġunderstand able +SE E +Ġh oy +Ġnin ete +Ġcon fer +Ġnow rap +ĠV ern +, čĊčĊ +imest ep +Layout Manager +à · +ĉw ait +PLE TED +J apan +Ġindu ce +Ġå ¯ +оз в +_END POINT +.h orizontal +Ġacceler ated +rim on +IV ES +Trans actions +Le an +ĠSO UR +wh ether +y g +Ġo id +ĠEntity Manager +OUN TRY +Ġfil a +OLUM NS +IN UE +ĠAn chor +TR AN +wo o +block quote +ĠN urse +ĠCar p +Ġrede em +. try +ĠJ P +Ġtimestamp s +Ġ?> ">< +ĠREM OVE +ĠStar bucks +Re ally +Ġflood ed +.C allback +Drop Down +ip ro +Ġt ended +l te +Ġproport ions +- te +ĠR ena +lic ate +for ces +.ex tra +.auth enticate +в од +¡ ° +Ġfor ControlEvents +Ġsen ha +Ġke in +Ġmin ist +ĠPre ference +ĠTele graph +Ñĥ п +str pos +Ġillness es +Ġp igs +Ġget Intent +S ol +Ġ ¡ +(c pu +[ prop +s creens +'); ?> +ĠAct s +Ġstr dup +Ġaver ages +an al +ĠCas ual +Group Box +ĠHand book +/ comments +Ġnumber ed +Ġbroadcast ing +çĽ ij +.native Element +.m u +Ġupdated At +ĠDoes n +.A C +.c oll +Ġrec order +_sh a +B g +b il +Ġbol ts +Ġç ¬ +Ġim posing +ĠInformation en +_flash data +e conomic +Rem ark +uc as +ĠOff icers +ĠT ER +W alk +Ġmerc ado +_g enerate +H Y +Call ing +s nap +script Id +. operation +ĠFl ame +l iness +Ġrent ed +_t oggle +-ch anging +ĠT Y +' util +EE P +Ġgraph ql +ĠUn i +Ġimp ulse +.B asic +Ġenerg ies +M ARY +ĠMar cel +Ġmort al +Ġf res +m ens +m otion +Ġsample d +âĢľ That +id ay +qu ipment +get Int +ĠA bsolute +,' " +un ed +.sh are +Ġ} )( +mm m +ĠR ising +ä» » +Ġun employed +x fa +.f ollow +ĉĉĉĉ ĠĠĠĠĠĠ +sl t +.P hone +Ġkn ives +Ġe ve +on Click +] ))čĊ +ĠW itness +ĉ NS +ĠE OS +ĠSte fan +ĠPri est +âĢĶ which +Get String +. By +Ġup stairs +Ġdetr iment +bro ken +emb ro +Ġnic otine +il ion +Ġaston ishing +_ aff +ĠLess on +Ġaccident al +od or +Ġdec ir +Ġnew Name ++ . +çĽ ¸ +igs list +ĠG ithub +Ġsuccess ive +rac ial +Ġen viron +éªĮ è¯ģ +Ġredirect ed +T OTAL +Ġgrab bing +ĠL ance +Ġfor fe +_C B +å¾ ® +El apsed +_w ay +(Dialog Interface +_me asure +x bb +D og +Dep art +-s rc +res olver +with standing +_sh ell +ĠLast Name +ĠAv iation +Ġbegin ner +("% . +(to ol +Ġн ов +: init +(A PI +ĠMorr ison +vt Color +Ġstap le +/ INFO +Ġsupern atural +Ġste ak +tim eline +zz le +" `ĊĊ +Second ary +ĠNep al +.String Utils +Ġad am +Ġ( ... +Ġsub stitution +Ġboard ing +ĠKey word +ĠAss ault +dbc Template +Ġorder Id +( engine +.assert That +ĠVen us +Ġhomic ide +ĠA val +Ġg utter +ĠSupport ed +/p art +Ġac claimed +H istor +Ġmes es +ü ber +ĠRen ew +Ġgr as +ĠE k +Ġin file +ind y +.m usic +.S croll +ĠA ges +ĠNar uto +ĠG ather +Ġconfirm ing += (" +Ġpitch ed +ole y +Fr ance ++' " +$ total +Ġon de +Ġd itch +_s igma +Ġcontinu ity +re ward +- load +Ġproces o +Lock ed +st aw +Ġsp inal +l azy +! == +j est +Ġd un +ĠRod gers +ĉ grid +Ġlog os +ĠBeng al +.s uper +Provid es +Ġnut rient +.T imestamp +IZ ATION +åĨ Į +Ġf ats +ĠX xx +ct ica +Target s +Ġcont ours +Ġre ordered +: Array +Ġtoler ate +V ir +Ġter ribly +Ġbr icks +(& _ +h b +Port al +ĠB read +. which +ÂŃ t +as InstanceOf +Ġj object +ĉ length +_M T +; ">čĊ +_EX IST +Ġmat ernal +RE L +Ġê²½ ìļ° +he e +Ġlayout s +ĠL ap +ais y +Ġst umbled +ĠU IG +ĠS co +Ġimp aired +RES SED +Ġab uses +V F +AR B +.N AME +r ch +prim ir +_com pleted +Ġp enny +Ch rome +(b egin +ern en +- checkbox +Plain OldData +ĠL PC +r ade +sp ir +Ġcon ceived +T ips +ĠIo T +ĠG an +èģ Ķ +Ġbi ases +Ġconsult ants +ple d +_ ht +associ ated +], ĊĊ +Ġdelight ful +ĠÑĤ ек +Hel vetica +( load +-exp and +_W IDGET +to a +ĠA kt +Ġom n +Ġcl auses +Int el +*/ }Ċ +_reg istration +Ġold Value +Ġrest oring +Ġun real +O VER +ĉĊĉĊ ĉĊ +AT S +_pro be +Ġdiv isor +.update Dynamic +å¹ ³ +Produ ces +st amp +.j boss +ĉt ask +! (: +Ġpsych ic +@ class +M artin +ĠPass ed +clar ations +h el +а Ñĩ +ĉc opy +-b in +z an +ig ram +া ঠ+(s ig +ĠC aval +_ ## +Ġ% = +out lined +ĠAc id +Ġunpredict able +-d ashboard +Hex String ++ c +.P ublic +Ạ© +Ġconvey or +ĠE B +Ġselect s +Ġknock ing +ĠC ec +IBUT ES +owa Äĩ +g atsby +* v +ent ropy +Ġdispatch ed +Ġcam el +ĠSat urn +Ġover weight +( phone +par able +% B +_v ectors +Ġbrew ing +ĠT k +ĠDownload s +ĠS aved +.Pr ice +Ġcur ved +ĠParen thood +è ¶ +.p nl +plet ely +.D ay +Ġadvertis ers +Ġej ec +Ġpr zed +ë ¯ +! ';Ċ +ĠK ush +ĠT AB +Ġquest s +Ġcoinc idence +umm ies +ĠKash mir +ĠEth ics +_g rowth +Ġakt iv +Ġgroup ing +å¢ ŀ +_tr uth +åIJ ¬ +t odos +is et +Tex Coord +ä tt +ĠZ ur +ro ys +_M AGIC +Ġbrew ery +( State +ĠSM ALL +ĠPl ants +it bart +each er +ĠAd elaide +L u +Ġf ick +und les +_load ed +и е +P oll +rit ic +EL Y +Ġ+ ' +ĠProf ession +Ġst amps +ĠS ew +scroll View +Ġcomm unist +/pro blems +}čĊčĊ čĊčĊ +, o +Ġu dp +Ġob ese +appro ve +ancell ation +_G ame +ĠHas htable +adaptive Styles +Ġpossess es +.match er +function al +M rs +ĉs ave +ĠDb Type +Ġk en +get Context +Ġm ans +( rel +ĠBrother hood +) `Ċ +è§ £ +.In formation +OutOfRange Exception +ĠS ek +C as +Ġblog gers +E ither +(" "" +Ġpin ch +Ġco arse +) p +ĠP ulse +Ġlear nt +Ġdent ist +Ġon change +Ġdirect ives +( actions +ny der +ĠSh ir +T rait +_de p +ĠP ET +ĠRE P +.App Settings +cu ador +iden av +Ġenv i +Ġsl ammed +ĠSh oot +Ġdate Format +.j oda +ve ys +Ġ) .ĊĊ +Ġcare g +ĠPar allel +_ translation +.function s +. obs +Runtime Exception +[] = +over view +ĠSch l +Ġno isy +ĠOn PropertyChanged +S ending +Ġunf amiliar +U pon +ĠPrint s +.t yp +Ġflee ing +ĉm ove +( Un +Ġq r +× ľ +_b eta +Ġsk ies +ĉm e +W ND +Ġstick ers +bl as +Ġinsert s +Ġvers es +ĠD ew +Ġtang ible +Ġhe cho +P OL +Ġte ardown +om nia +IB E +.c over +_str ategy +^ - +set Position +u ale +S igned +Ġif ace +as eline +.set Time +ĠMin eral +ĠFight ing +sk ins +Ġdiscrim in +Ġdans k +ĠPr inceton +ac ist +Ġ( ));Ċ +tr acks +imon ial +ad ecimal +EP ROM +ugg le +.Not ification +$ mail +c antidad +ĠJ ung +Ġseek ers +Ġpl ausible +t ier +еР¶ +Ġr apper +ĠMan a +ĠHttp StatusCode +Ġburn t +los es +ĠF oto +ĠJson Object +Inst agram +Ġsys call +Ġreal ities +ĠMAT LAB +:^ {Ċ +TER M +ĠC bd +ĠPar agraph +Ġtrav és +Ġconstruct ing +Ġsw al +Ġp ige +LL LL +-ex isting +G ets +Ġmelt ed +Ġmitig ate +H en +Ġh m +im as +ĠA o +ĠP erez +ĠD AL +Ġëĭ ¤ +Ġdiv is +Storyboard Segue +ĠMod ify +ĠÃľ ber +_O VERRIDE +.p em +unt os +Ġespa ñ +Ġ{ ? +ĠP AY +_ip v +ĠF ury +__ .__ +el ow +-center ed +check s +_ Reg +-J avadoc +ĉ load +ĠLik ewise +ا Ùħ +UN E +.se m +x cb +ĠC ave +_s leep +Ġsil ently +ĠExt reme +.To Upper +ĉC HECK +Ġc ue +ĠQ ByteArray +Ġcorrupt ed +ĠD é +Ġimp ed +Get Name +Ġinaccur ate +Ġso ber +е е +Ġbar code +-- ){Ċ +ink i +Ġé p +Ġd ri +ĠAL T +>>>> >>>> +ont a +[ L +Ġinter es +ver ting +Ġdi agnostics +p dev +è © +ĠIntegr ated +). ' +_g c +$ text +.g ames +ĠT erra +' Re +.trans fer +_F IFO +get Model +Ġbl and +ĠCole man +Ġpr imes +Ġæ Ī +Ġcross es +n k +G ING +Ġ' ^ +ĠB lob +Ġinter course +ĠBl vd +Ġweigh s +_reg ular +ĠPer th +Ġsepar ating +Ġb illed +.tab Control +Ġpup pet +Ġutil ization +Ġâĸ ł +Ġsucc es +Ġl amps +_pro j +E ric +Ġren ovation +ĠFam ilies +ĠB its +part ials +-M en +s olution +Ġd warf +.IN TEGER +ĠLO CK +. ct +Ġexcer pt +ĠP ix +ĠFirst Name +ANT ED +ĠAd mir +-h elp +P rior +ĠAl ign +.IN STANCE +Line Edit +('/ : +Ġin et +od us +.p kl +ĠK Y +up ert +Ġn erves +_grad ient +} ',' +_un ref +Ġs aturated +ĠConn ected +ĠF N +EX IT +Ġtele port +Ġav ait +Page Route +Ġdivor ced +(l ang +f st +ĠT yr +Ġmess enger +if stream +X S +ĠBank ing +Ġinfect ious +ĠM ons +_LO OP +Ġzur ück +Ġobt ener +/re pos +V el +ac ro +Ġuser Repository +style Type +ĠS RC +VML INUX +rec ursive +/ bar +_ch ip +omin ated +ĠN it +âĢĶ to +ĠBudd h +ом еÑĢ +ĠM AG +ĠC HE +_d en +. raises +_de gree +Ġpump kin +_tem plates +_M EDIA +ĠTim eline +Ġb ots +Object Type +Ġbu ys +.post s +C AL +wait ing +ĠDani els +Ġd abei +ĠS igma +il or +ig el +, W +AD S +( panel +ì² ´ +it ating +.p alette +Ġmos quito +Ġt ego +(parse Int +Ġdes pués +p romise +Ġw ij +types cript +ĠT v +_IDENT IFIER +).ĊĊ Ċ +_fl at +its u +US R +ex perience +-f it +ph inx +_th resh +Ġide ally +ĠFre eman +, DB +_r w +çŃ ī +U b +_stat istics +=" ">< +Ġch ore +Ġy ork +inst alled +Add itionally +Ġp stmt +yl ko +:: Ċ +Fore st +Ġhead set +Ġgall on +ÑĢ ÐµÐ¼ +Ġwithdraw n +ĠC andidate +Ġmel ting +Ġfree zer +Ġh l +_HE LP +m ime +( /* +Ġth irst +$ return +member of +еР± +ĠHttp ServletRequest +( ob +_ Result +Ġassert ed +Ġfulfill ing +Ġstret ches +par ated +-f unded +Ġå Ľ +ing les +_c a +. condition +ĠDis plays +Ġor ang +ĠC RE +Ġgl Bind +ĠSelect or +/ type +ĠAlex a +ched ules +ĠPen insula +Ġpar ity +ĉ dest +ĠDo ors +čĊ ĉčĊ +_dim ension +Ġa load +.St oredProcedure +(p aren +ĠBur ke +') ]Ċ +- engine +Ġqu ir +ĠHy brid +ĠDo e +Ġout lines +ĠTrend s +_N V +per iments +ĠH in +? ', +ĉ Text +F UL +Ġsm ells +Ġs lick +Ġmis erable +ĠArray Adapter +Ġparam String +H om +_l iterals +us uarios +Ġprompt ing +_l azy +ĠActiv ation +_ oc +We ak +Ġan ecd +ĠU CLA += re +isse ment +ĠEsc orts +Ex cellent +ĠP ause +Ġre positories +T OR +ari ate +_is o +up dates +hal b +udi ante +ë¡ Ŀ +Ġna ive +ĠP eg +ĠL ounge +ARG IN +(b in +On ClickListener +ĠFA ILED +Ġl ite +Ġd zie +ĠL iteral +iv or +fc ntl +Ġe ats +Ġq ed +Un lock +rid ing +und ai += M +AT TER +Configure Await +ici as +ustom ed +Ġsuccess ion +end Time +ĠJ upiter +Ġjud ging +d ration +_d ocs +.m o +Ġeduc ators +ĠV ine +Con d +[ out +q b +\ Validator +Ġmean ings +Ġpresent ly +Ġdiv iding +otten ham +asc ular +Ġtrail ers +ĠC LOSE +ам и +âĢĻ ai +ĠG ain +w or +Ġpl anner +Ġdistrib uting +v at +month s +x label +H F +V iol +.BASE LINE +еÑĤ ÑģÑı +ĠR otate +Ġtx n +: bold +Ġb loss +Forg ery +( embed +Ġjak o +s printf +the ir +Ġexhib its +- static +he cy +get ActiveSheet +.c lients +ãģ į +_h ide +[ word +C b +add Item +ax e +_r adio +al ion +mod ifier +Ġsat uration +Ġden om +_p ixels +m ess +(f l +at if +Ġse cs +Ġpro stitution +Ġgrand children +Ġparad ise +ĠF eld +_B INARY +it ous +๠Ħ +Ġflash ing +-s ided +Ġcontrad iction +/* ĊĊ +y label +ĠT et +Ġadm ire +res o +Ġlet z +ĠSE ARCH +sl ots +ĠRew ards +ĠH og +ĠNS Data +st ash +F all +ĠA mer +Line arLayout +/ photos +Ġfe ather +Ġ| čĊ +Download s +.Start sWith +Ġ// # +ine Transform +Ġaff id +V tbl +ĠRog ue +scri bed +Ġfa uc +ĠMon roe +Ġdecl ares +mod ern +re on +ay be +P ASS +f ers +_MULT I +ĠMath ematics +Ġsud ah +_ATT ACH +Ġnumber With +ĠSol omon +j in +ograf ia +ö l +_d esign +cul ated +ĠL una +ies z +Ġ=> ' +Ġrevel ations +Al ong +( ed +ĠF ilename +Ġy label +Sec ure +Ġbus ca +agn osis +_RE CE +Ġoverl apping +Ext ent +Ġanticip ation +Check s +ĠALS O +or c +iling ual +it ational +Ġadv ancement +ou ro +ĠP redicate +å¾ Ĺ +er ia +ĠPier ce +or io +Ġmer its +Ġpe anut +.P ackage +ĠCon duct +_SENS OR +Ġbo iling +Ġin tra +ĠI GN +ĠF ur +.Ref resh +ĠRe ach +_dec oder +.Ex p +ĠÑĤ ак +p ill +, Q +ĠGr ill +Ġpop ping +.A g +Ġpro yecto +Ġmile age +Ġec ological +] ]);Ċ +ĠÂ Ń +sub plot +ac ad +ĠTry ing +rec ipes +$ criteria +ĠPers ian +-b ound +M ASK +ĠG esture +Ġk k +ĠP VC +Ġprohib ition +Ġcom ando +ĠLO OK +Sh opping +Ġdist ortion +< Boolean +.Get Length +um pt +\ Product +ell ery +Ġfire wall +form atted +.red is +Ġes a +ĠRh ode +S om +.n on +Ġ' ). +Ġget View +ạ n +pr us +Mat thew +Ġs ia +ĠF ors +G PU +ient ras +_IN ST +Ġol arak +Ġimport ing +T CP +/ ");Ċ +e ither +Ġfresh ly +c ascade +(char acter +ĠJe ep +ot ics +_ UTIL +.Xtra Printing +.first Child +ĠEx cell +Ġd vd +Ġt aller +Ġr as +yp ass +Ġassign s +Ġgri ev +-m ore +J D +ĠBurn s +' >čĊ +.D ependency +.Query String +.O wner +Ġexp iry +Th u +( Vec +Ġhazard ous +Ġr pm +AP ON +Ġadd Target +sv ille +p Net +ĠIm g +ĠTIM ER +.An imation +Ġbe k +Ġass ort +Ġle bih +Ġbody Parser +Ġvibr ating +ID L +Ġbutter knife +int ers +Ġpersu ade +ĠLGBT Q +è ĭ +.s oft +Ġbe ams +_s ur +.D ef +Ġl abs +ĉ plt +Ġsk ins +Ġtransf erring +Ġimag inary +_E nd +; background +Ġl aps +_COM MENT +(S DL +ond s +.Rec ord +ĠIm plements +_t icks +() ))ĊĊ +Ġa rose +] ? +ĠM p +ĠI Command +Ġsculpt ure +Ġcontract ed +< HTML +Ġcal end +at y +/ Sub +Ġkv inn +_ IGNORE +ĠSh ane +ML S +Ġstim ulate +Part ition +Ġm un +ó m +eral a +- account +.B inary +c é +Ġse ize +connection s +ĠĊ ĠĠĠĠĠĠĠĠĊ +ĠDi agnostic +V ISIBLE +ĠRun s +Ġimpress ions +s uite +ob le +~ - +ak ukan +< Person +ĠN os +ĠG ui +.wait For +RE SET +Ġpost pon +Dis cover +arr ison +sh aw +b lood +AJ OR +æĽ´ æĸ° +ĠM use +æĶ ¶ +Ġret aining +ot te +Ġmos que +ĠS ne +Ġstandard ized +Ġmain land +_th ree +unge ons +get Doctrine +Ġwh ale +Ġag g +ĠP orsche +now led +lat ent +ĠRel ation +Ġ// ' +Ġshut ting +ĠRem ix +_c ov +Ġs ailing +Ġv owed +Ġp ots +out u +Ġhair y +cast s +Rel oad +Ġre connect +ter a +.child Nodes +ĠR ack +Ġcurrent Index +Ġall en +Ġ ç͍æĪ· +ĠC ubs +[ X +_SE Q +_RE MOVE +.get Action +(/ ^ +err ar +Ġ ether +cur ve +Ġsl ap +Ġu om +O thers +Ġen gr +Dis position +Ġst aged +E ye +ĠA ux +auth enticate +Ġ$ ? +ĠAndre as +Ġset w +.A rt +Ġforecast s +Ġa unt +-m iddle +Ġmis d +des k +Ġescort e +ĠCas a +rop ical +Ġexem ple +plan et +(U INT +Ġwh ip +ĠPC B +clide an +=" \ +Ġox ide +Ġsucceed s +der ived +ĠEcon om +_co ordinates +ir as +D raft +Ġvisual ize +B rian +_ASS UME +ĠObject Id +Ġtrain ers +_FOR CE +Ġcon soles +- process +lic her +ĠSim mons +T aking +ĠCl aims +Ġdiffé rent +Activity Result +Ġsn s +éĢī æĭ +ĠCr us +Ġll am +r ab +ĠJo an +AA A +ĉf ilter +ish ops +get ting +à µ +Ġquant o +P ast +ov ich +Ġin justice +ĠF LOAT +Ġal right +\ DB +( GameObject +u ish +(b ot +Ġgall ons +ĠR é +ĠS aid +ĠSTDMETHOD CALLTYPE +ais ing +_process or +ell idos +ter dam +ĠBe am +Text Area +Ġret orno +.M ake +Ġ$ ("< +Ġlock down +Ġremed ies +Ġve el +x ee +do ctype +F il +ĠExp and +Ġemp loys +Ġsession Storage +Ph p +P ublish +Ġret al +f abs +ynam ics +Ġtoss ed +ĠnumberOfRows InSection +x path +\ modules +Ġdis astr +ĠM ULT +.M esh +-st age +Ġs df +it ung +ug es +Ġ?> ">' +kin son +Ġк ол +ogn itive +_ li +Ġim minent +Ġaff inity +.sign al +Ġnot ch +ĠSteel ers +max length +K K +ĠEug ene +_P WM +ro i +Ġâ Ĺı +ĠH amburg +.M ust +Ġax e +en ef +Ġamb itions +ĠSpec ies +ĠSt ress +Ġa while +Ġб Ñĥд +Ġwith stand +ĠDec oder +_in ventory +Ġ{ ččĊ +Ġt gt +Ġrail road +W ASHINGTON +Ġnegot iated +N ST +- phone +, U +Ġexerc ising +á» ¥ +_P IXEL +av ors +iter ated +Ġv ampire +ad al +In grese +Ġun g +ject ive +.c ells +Ġn ano +Ġmark down +_R ULE +(event s +Ġl uggage +MESS AGE +ig keit +$ count +Attribute Name +IG INAL +_E nt +ĠB F +ĠCOM MENT +_in i +ĠEurope ans +ĠB elle +åij ½ +) [' +åº Ķ +ĠUse ful +.re ference +() ", +_ grade +ĠK aw +Ġsent encing +Ġsocial ism +mon ster +_L AYER +Ġdee pest +w k +ĠNo ise +### ĊĊ +Ġpr éc +ot le +ÑĤ е +a uf +ib al +Ġcon quer +> Email +Ġamb ulance +O AD +Ġ(" % +ĠF I +.f ixture +Ġter se +ĠĠĠĠ ĉĉĉĉ +Ġsanct uary +ug i +ĠCom parator +Definition s +Ġast hma +Ġl act +Ġhard wood +.c lock +Ġattract ing +ĠM our +(d istance +ic its +Ġbon ne +ĠAC CESS +.Deserialize Object +ĠTyp ed +Ġje u +Ġapp Id +ĠCl ara +ĠH F +ĠRe ich +ipp les +//---------------------------------------------------------------- ---------------- +_del ivery +erial ization +Ġplaint iffs +Sc ient +sh opping +ĠD ummy +ĠW ald +Group Name +Ġins cription +el og +:::: :::: +_ ld +Back Pressed +.R aw +ĠOn Trigger +Ġmuse ums +ĠBe en +ĠAdvent ures +Ġsl ate +Ġlet t +Ġsu nd +ĠG in +ĠMechan ical +.s hip +App Component +Ġdest ined +Ġdw elling +Prof iler +Pre pare +ze ich +Ġsil icon +(h as +Ġ# % +VID EO +Ġcollabor ate +L in +Ġsc opes +( className +(s d +and in +.h am +Service Impl +-des cribed +Ġiron y +st ial +ĠHu awei +(re po +Ġunexpected ly +ĠK ai +.inst all +\x f +Ġexhib ited +_T CP +ĠO x +_CH O +Ġprostitu erte +Ġv ä +Ġsit o +Ġconstitu ents +ĠContin ued +ĠS AVE +r ss +/ message +ub es +Ġmisd emean +Ġtax ation +Ġstory line +h air +ĠFind s +S IG +ver ification +~ = +.h p +Iter able +Ñĭ е +ator i +Ġc tr +R x +_ );ĊĊ +d ag +.p in +Ġp seud +Ġinv o +ÑģÑĤ ÑĢ +_p ix +为 空 +Ġsw orn +âĢĶ or +_reg istry +Ġdis asters +ĠRO I +ĠâĢ ķ +akt u +fore st +be iten +âĢĶ I +ue va +eg t +Ġsp ikes +URE S +ĠRecomm ended +Ġexplo ited +ĠFreder ick +_COMP LETE +ĠDr ugs +!!!! !!!! +ĠR iv +ST OP +RO OM +ĠP ASSWORD +C ookies +.E l +á» Ń +ĠB ert +Ġhash ed +ic ester +Ġdecor ator +Ġquery String +: ;Ċ +Ġ" [" +oto pe +-A meric +ĠMatthew s +UR AL +âĢľ , +Sum mer +f os +_CONT AINER +_A CK +Ġfil tr +_dis p +_ Re +Ġfac ile +а ÑĪ +Ġìķ Ĭ +Ġe ben +Ġspr ink +ĠQ uint +> V +Ġhistor ians +our met +ĠMonitor ing +led ger +c ott +Ġw are +GG LE +c ars +ĠM EDIATEK +Ġvol upt +_ View +HE L +(c opy +(st ats +Ġchrom osome +ĠCurt is +- conf +( asset +Ġhv or +File System +< >();čĊ +oc oder +ĠC annon +) x +ĠSm ooth +ĠS AS +_ ce +ĉ prev +_m ovie +E c +_w all +< Button +ĠF AST +Ġon View +ul an +ĠS UPPORT +Ġgesch ichten +ĠS ons +Im m +$ IFn +Ġfair ness +Ġd pi +ats u +J osh +Equal ity +Ġ} ()Ċ +_ less +ĠR atio +ĠC ats +ĠS tern +Mon ster +Ġmer cury +ü hr +Ġplus ieurs +.des erialize +sc opy +.F alse +) animated +ĠExp erts +Ġ"") {Ċ +.W hen +see also +.un pack +LE M +.select All +Ġperception s +ud ing +ir ling +ĠPrint ing +gram s +ĠFile Stream +erv ille +il og +ic mp +_C ount +Ġlivest ock +- ca +doc uments +Ġpo les +ĉw ant +Ġflu ores +Ġstand point +ĠH uge +Ġradi ans +ĠUIB ar +EDI UM +ĠHistor ic +_h older +ĠMar ines +Ġt ä +.L ight +quir er +ason ry +div ider +ĠFl utter +_f b +restrict ed +ĠEvery body +N ão +Ġkn ot +ĠT witch +Ġhall way +(C ollider +Input Element +? )Ċ +/ off +/ ) +play ed +[ OF +Ġbat ting +_d l +Ġcom edian +Ġé v +ĠD EM +ĠEd en +: white +' ', +Con struction +acer b +Ġtask ed +.man age +Rel ationship +Ġph on +n z +_B GR +Validate AntiForgeryToken +_ air +âĢľ When +Ġgl fw +ĠCon versation +_T OTAL +, Z +Ġg raz +Ġiter able +ĠP ASS +Ġadvert ise +Ġmö glich +/ train +ĠVolk swagen +Ġcreep y +Ġ" )čĊ +QU ENCE +Ġalt ar +Ġed its +comp iled +aw ning +ĠD ungeon +Ġo sg +Navigation Bar +Ġtrend ing +ĠE co +ogg les +cd ot +| - +S ie +ec ret +ĠN egative +ĠL ing +ĠD IM +ĠC WE +ĠCar rier +Ġcar tridge +_us b += os +ĠJack ie +Ġo tras +Ġcommod ities +ĠP resentation +)&& ( +ĠMar tha +ĠCath olics +ĠM ond +об Ñĭ +_ absolute +Ġash amed +pons ors +t al +Ġsad ness +Ġpu ò +F ade +-pre view +ĠRequest s +ĠCal vin +h orn +Reuse Identifier +(pro vider +/app s +ime o +ĉ Class +S amsung +ĠW ORLD +Ġc innamon +dot env +ĠI User +ĠDE V +_C har +.ib atis +et i +/ me +s st +.s ym +ĠRug by +-m aster +aj ar +ĠY EAR +Ġo dp +ĠR oles +Ġbip artisan +ail le +Ġblock er +Ġgre ens +.SE CONDS +Ġbelie vers +ĠL ikes +F LOAT +Ġm ak +Ġg cc +âķIJ âķIJ +(" ~/ +SCRIPT OR +Ġton nes +ĠS ang +Ġtrans pose +enn ai +P red +Ġsoll te +.github usercontent +( print +ĠH ole +çľ ĭ +ad get +Ġprompt s +Ġgen etically +ĠH od +Ġvert ically +_control s +ÑģÑĤ ан +") {čĊ +$ title +Ġ} ),ĊĊ +Ġstate wide +ĠCor respond +ĠAt tr +it ant +Element Type +Ġout ward +Ġfam ilia +( article +Ġbl at +Âł Ċ +Ġgl Get +ĠRe ceiver +Ġ% - +ad am +W inner +Ġtail or +_p wd +ert en +St an +ĉ all +al ive +strt otime +� s +s essions +$ conn +ass ist +Ġchat ting +ĠM ant +Ġ% @ +Ġ"" );ĊĊ +Ġd gv +Ġíķ ¨ +.re peat +_M essage +Ġadvis ers +/ path +Ġk es +) } .ĊĊ +ogen esis +ĠOPTION S +upt ools +Ġmilit ant +Ġex ited +ig ar +ĠCOM M +ĠDis posable +ay cast +Ġrow span +Ġsyn thes +Ġsond ern +ĠĊ +ĠJ acket +R ATION +.getSelected Item +- init +ĠReg isters +_se p +ĠTool kit +.d ict +Ġx label +\ Table +t oc +_com bo +ĠComp act +Ġr ugged +à¥ĩ ठ+-man agement +')}} ">Ċ +ĠSt amp +ı l +ro x +Ġlandsc apes +_NOT E +mon ary +c ab +Ġmo et +x af +rc ode +- cli +_g ate +[ event +SP ORT +g ia +ĠS UPER +/ Login +_sh utdown +int errupt +Ġpret ending +Ġfr inge +ĠRed s +ĠC UDA +ĠUN IX +v it +Ġbr ig +dr v +ĠConn ector +There fore +Ġl ia +D etection +_ actor +Ġtemp file +Ġecc entric +- role +Ġpad x +d ent +West ern +Ġê ·¸ +ĠApplication Record +Ġcampaign ing +_run ner +ĠC ivic +ale igh +Ġdire kt +.s ul +ĠĠ ĉĉĉ +ant en +Ġiss uer +Ġassert ions +( orig +AT IO +Ġlean ed +ä s +.D TO +expl ode +.O bservable +Ġstagger ing +Ġkidn apped +Ġprogram mers +ĠInn ov +.param eter +Ġdom ination +Ġske ptic +Ġæĺ ¯ +Ġavoid s +.Ver ify +ub by +ĠAS N +Ġformat o +ĠBeat les +_b rand +Ġin set +y outu +Ġto c +-f inal +Show ing +ĠD oub +ĠM esa +Ad j +_m edium +Cre ates +(end point +ĉ UP +bb ie +Ġst alk +.datab ind +.S can +ag ents +$ , +ind ividual ++ )/ +ĉv m +(not ification +Ġin ex +ĠClass ification +ren o +Ġo lig +-r ated +Ġform ulation +', { +Ġa cept +_un pack +_C A +.P ow +ĉ im +Ġal uminium +AN O +Ġx n +Ġcó mo +ĠIng redient +Ġseiz ures +åħ ± +ific ador +Ġsigu iente +ĠIn fragistics +Ġduplic ated +ĠDe e +Ġn ø +ĠAC CEPT +(c rate +иÑĤ елÑĮ +- less +Ġinf inity +An alyzer +-D ay +rit t +(c in +ĠG y +Ġmulti plied +uch i +ĠBald win +/ ip +Ġshort cuts +.A DD +Ġvig or +_in struction +( ; +_ eta +è¿ ŀ +utor ials +Ġboost ing +b v +Ġacknowled ges +List ening +FA Q +; b +(( - +Ġarchitect s +Ġz we +Ġpul s +Ġget Count +ver bs +ãĢ ľ +(C ollection +k re +Ġjuris dictions +_b ridge +ĠCr ack +ĠDiff iculty +K O +Res ervation +_re quires +T our +ãģĹãģ Ł +.set Current +Ġk y +ĠAlb any +Ġè § +ll er +agn a +work ers +.bl ank +ĠPr ayer +M IC +Ġresil ience +Te X +ĠL anguages +st udy +ĉc urr +Ġenzym es +Sl ug +ĠíĮ Į +str al +Ġtum ors +Ġseg unda +=' { +in struction +ĠL isp +/ info +Ġ" {$ +,: ), +Ġg v +( ErrorMessage +Ġ' = +}- ${ +.Doc uments +" Well +Ġreminis cent +Ġg az +iro pr +eh r +Ġsup pressed +ers h +.scroll To +Ġcad ena +Ġgame State +ÃŃ m +( conv +ĠTom orrow +ĠC CT +M ongo +ul g +.C amera +.hand lers +m ph +Ġst k +Ġgen etics +AC ING +Tr ivia +ĠB am +(m arker +.St retch +ĠSun ni +ĠBet ty +.t olist +un likely +.Rect angle +ob solete +IL ON +inner Text +emb ourg +a N +ĠV ehicles +un lock +: utf +n ob +ĠSee ing +ĠNE VER +Ġt ls +Ġfil les +Ġbenef ited +ĠCl int +*/ ), +.f old +Ġpos ible +A DED +th ouse +.D AL +ĠO dd +ro kes +ĠSun ny +ĠPartial Eq +_B uffer +ĠLe vi +long rightarrow +eld on +g ages +_w arn +.Create Table +ĠD ip +_ questions +.log ic +Ġ# " +={() => +Ġt ep +Ġju icy +ì Ĥ¬ +en ko +ia lect +Ù ī +Ġon board +Ġæ ı +ĉ rt +_ UTF +ĠQ Action +âĢ ŀ +( Component +(a udio +.h it +g te +Ġprogram med +state Params +Ġpoly ester +f ires +by ss +] =( +_ quality +Of Day +ĠFair y +Ġy elled +op l +(user Name +ĠD ifference +Ġevalu ations +iff any +Ġcycl ists +Ġc idade +Ġtext book +Ġprof iling +__ ), +de a +. activate +Ġindic ations +Ð ķ +Touch UpInside +Ġinval uable +ĠM ASK +Ġcont end +F req +Ġrecru its +(int erval +ĠUser Profile +Ġ'./ ../ +ed u +_C allback +Ġanal ogy +ĠTro phy +app hire +V ideos +ĠCh er +ĠH av +â̦ " +. validator +g fx +ĠU Object +class names +tri angle +ĠEnc oder +.s py +Ġpred ators += status +-s afe +: ",Ċ +ĠIn cluding +Ġ{} ;čĊ +* cos +Ġend ured +.sul ake +Ġnurs ery +Ġfrag rance +Ġre building +Ġn th +ĠFr aser +.set Date +ĠV ince +_RE ST +Ġvent ilation +æµ · +cri bes +.as m +lp Vtbl +ĠA be +uis ine +, array +ĉ className +err als +Ġ' ĊĊ +Check out +Ġsol icit +A ux +_c apture +Ġrib s +rag on +vi ol +top ics +Function Flags +ĠM arty +b ike +ĠT ucker +(k ernel +ĠO ps +Close Operation +/d emo +ild a +ĠlÃŃ nea +APP ING +Ġsu ites +.visit VarInsn +ur us +ĠMin ute +(m anager +Ġbutter fly +Ġap are +Ġw olves +J WT +ĠSal on +ĉd elay +-es lint +is ations +.r pc +)| ( +ĠSnap chat +/m m +M N +cer ies +.text Alignment +ĠFrank furt +Ġad o +(new Value +( access +( Expression +ĠSign In +ĠHait i +_t p +.set Parameter +Min ute +Ġmanual s +ric anes +ĠP TR +ĠOut er +Ġget line +oc ations +_C D +ĠLy on +/g ui +_l ive +id an +.ge om +Ġborder Bottom +im uth +_check point +Ġme u +ĠIr ving +Ġpeu vent +(M AX +ĠAR CH +Ġp ov +.source forge +Ġjam ais +Ġar k +ĠBaghd ad +ĠC LEAR +Menu Bar +Ġtro is +CHED ULE +Ġ# čĊ +(C all +$ order +(M aterial +Ġencontr ado +$ list +ĠMETHOD S +.begin Transaction +_M AG +Style Sheet +Ġmaj ors +Ġindef initely +clean up +Ġhom eland +(d to +D ates +P resentation +ĠD K +={` / +ĉ Key +( Block +_check box +ne eds +Ġon Complete +ric o +Ġgle ich +Ġx m +O OD +B etter +ĠSQL ITE +. Book +x ad +ĠG one +ĉd p +Ġdev otion +Ġst m +Ġobs ess +ĠBack end +Qu eries +I k +// **************************************************************** +Ġdivid ends +.parent Element +} ")ĊĊ +ĠMaterial PageRoute +: num +Ġexp lic +ĠO L +le ast +O ops +iment os +Ġins urers +Ġhero ic +ĉf ields +.img ur +.btn Cancel +ĠDetect ive +(s m +ĠMutable LiveData +.l ab +(( [ +Ġha irst +ĠTrans actions +å¼Ģ å§ĭ +Ġstd Class +uent o +G IS +_c od +Instruction s +C alls +Pointer Type +ĠR w +Ġassort ment +ĠD IG ++ r +_C ERT +Ġinst ability +Ġv ib +on as +Ġro ku +ap ellido +Ġan gl +prene ur +Ġfluid s +ise ase +Ġde ed +qu ist +_CONST ANT +Ġequ ilibrium +_de legate +ĠQuant um +re i +Cap abilities +rect angle +? >< +al ien +ĠJ ug +D NA +T ickets +Occ urs +ĠHaw k +.setHorizontal Group +\ Collection +ff iti +Ġre arr +.setVertical Group +Ġc avity +Ġadult e +Fac ade +- wh +ĠL OL +Ø ° +Ġgrand parents +Sw ift +ĉw x +æīĢ æľī +if en +ff set +B eyond +// }ĊĊ +Ġw ager +Ġb ury +Ġcomm ence +reg istro +sc ient +ĠPer cent +Ġд олж +( identifier +.set Model +Ġs eldom +nt on +Ġappl iance +am us +rys ler +Ġpant ies +engu ins +Ġmim ic +Ġon Changed +Ġal coholic +.reload Data +Ch arge +ĠF ax +Ġj ScrollPane +Emp resa +Ġsh attered +x ba +Font s +? s +Ġpost season +ret ain +_r ates +Ġrequest Code +.t odo +´ s +CH K +ĠKeep ing +enge ance +Ġvs code +IPP ING +Default CloseOperation +_ raise +ĠO culus +ogram s +ra j +pc i +Ġcorros ion +.handle Submit +Access ible +ĠP iano +l ittle +AC L +Äĩ e +.un wrap +ĠCon vers +ĠLe ben +ione er +ĠMer chant +ĠJ orge +Ġembr acing +Ġvent a +á st +Ġvi ene +< QString +Ġexplos ions +Ġdistur bed +." < +m emo +ĠAb original +Ġcomple to +Tex Parameter +Ġuom ini +( agent +Ñĥ ÑĢ +ĠWh olesale +/ am +ĠBook mark +dr agon +Ġglo ve +Ġ" "));Ċ +iv ariate +now rap +In Children +.B r +Ġcon exion +Ġback bone +Ġe clipse +Ġpersec ution +': ĊĊ +/ link +ĠP ero +and as +ĠT ek +. "); +-an alysis +Ġer ad +Mar shal +Ġanch ors +og er +Ġconver gence +st icky +Ġnave g +int ern +_DE SCRIPTOR +ĠConsult ant +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ĠA uch +Ġer re +ÅĽ li +ĠHor izon +col a +Install ation +hot mail +C NN +.C ollectors +ch s +(tr ace +ĠEnc rypt +Ġ---- -- +ĠBase Controller +Ġag ua +Ġre active +id l +Ġclass Names +ĉ Session +ĠDod gers +H ad +_l v +Is Valid +ĠHEL P +ut to +ĠVer ification +Ġget env +_p a +.b mp +: f +ĠLou ise +(' ; +/ socket +Gr anted +.c alendar +( IP +ĠP X +.R oom +Ġprogram m +ens i +Ġtablesp oons +Ġle ve +Ġmo str +.t ipo +/ an +(d i +Ġb iod +Ġdb Context +ĠJS X +ĉ results +. END +ht e +l ify +P recision +èĬ Ĥ +ARS ER +)did ReceiveMemoryWarning +at tempt +IS P +& a +_P OP +ĠT ac +Ġprepared Statement +Ġзап иÑģ +Ġow ing +, start +Ġreview er +Ġr st +Ġprop Types +Ġrock y +_lo cale +ĠStrateg ies +ĠWe ber +.C ascade +_equal To +Ġcos as +ĠDe letes +ĠMax im +Ġsh rimp +re trieve +.In clude +IG IN +ĠO E +] );čĊčĊ +.en umer +Ġco ef +_N ull +R a +ty ard +ĠSh awn +keep ers +Ġq q +_s b +om ens +ĠExec utes +# " +TT Y +ĠValue Type +); */Ċ +ĠAbs olutely +ĠT ottenham +/ art +Ġbless ings +Ġswift ly +b uster +Ġa vid +COM M +, temp +Ġ} ?>Ċ +-g rowing +Ġdeep copy +A ck +egg ies +Ġ__ (" +Ġno ir +terror ism +Ġanth em +ag ency +_PACK AGE +ĠC losure +.reg istry +Ġmamm als +< L +U ICollectionView +ĠLED s +Ġvol ley +( Buffer +_N ATIVE +lib c +impl ode +Scroll Bar +ĠMar ion +.Con tracts +_A t +ĠWe instein +compare To +ĠH ose +en ity +.create Query +_r outer +Ġstim uli +Ġ++ ) +ĠCh amp +ĠBay ern +ass a +.v a +Ġdistrib utors +Ġfile private +Ġdepart ed +cc cc +@ click +ĠL unch +> L +Ġbl uetooth +.De ep +- standing +ác il +Ġro oft +ĠPath s +_iter ations +Invalid ArgumentException +.s pi +ĠUIAlert Action +uy e +sign in +.p riority +ĠEss ays +=' {$ +Ġè¿ ĶåĽŀ +_s igned +.p ersist +Ġred esign +To Lower +ĠNew man += start +ĠIsrael is +asis wa +Spe ech +Ġnum eros +hand lers +ĠW ong +Ġм еÑĤод +We ights +ĠGu jar +te il +ĠNon etheless +_E FFECT +Ġv ect +ĠO sc +Ġco ats +ĠW heat +Ġge ek +ĠPRO PERTY +w orm +_const ants +ĠB oulder +ĠP arm +co le +Ġdefault Center +ĠRou ge +: A +xc f +ĠVen ice +med ian +Ġred emption +F resh +Ġcos m +Ġfig ur +Ġref urb +CO PE +.c d +Ġch ords +ĠS gt +Å į +VP N +ĠS END +ain en +_account s +Ġtent h +Ġdiss olved +< App +ĠCover age +use State +é ro +.. < +Ġì £¼ +Ġdream ing +ĠFore cast +.C ursors +Ġvis as +/ script +_start ed +Ġga str +(P RO +]; // +.T ile +* sin +( Adapter +ĠSand ra +_S IG +ard ash +ĠO val +Ġdescri pcion +(s l +ĠDes criptor +Ġ` $ +/f ree +ĠKey words +Ġt udo +ion ale +(f ound +.x yz +ĠGeneration Type +_DISABLE D +( area +Ġel ites +Ġh ombre +(m essages +ĠR ac +Ġext ingu +ĠEst a +op o +. vel +mouse out +Ġconv olution +ĠHand ling +Ġceil ings +T ek +ĠAre as +.writer ow +< View +ĠCorn ell +_B IN +.in valid +'' 'čĊ +ie ż +_P osition +Ġk idding +PC ODE +Ġwatch er +lo x +Ġâ Ĺ +D ave +_all ow +Ġbis exual +Ġun ordered +ĠSch we +_se gments +Ġt earing +IN LINE +Ġund es +.g oods +.c am +ĠL W +ĉ where +Cal culator +-th reat +- alert +ĠSuz uki +ĠIP A +ĠAtt achment +AC CESS +(d type +O pp +_s ymbols +Ġdans ke +l age +or get +res olution +е Ñĩ +ĠQ Color +ĠBar rett +аÑĨи Ñı += \' +ĠNav Controller +/ ref +(c ountry +_H DR +Ġterse but +pet ition +Ġsu f +cred its +๠Į +x m +ĠDav ies +.re ddit +Ġw oven +ĠO bl +ĠK M +ĠConsider ing +ens ored +.per iod +Ġd dl +$ wp +Ġextrem ist +; \Ċ +Ġk im +al ers +Ġspan ning +Ġco herent +Ġconse gu +.text Label +.g eneral +_d ashboard +л ение +k ick +_P ID +ĠExt ensions +reg exp +ĠCl ause +_m ov +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +ĠR eward +ĠLEG O +A k +=-=- =-=- +ĉ parser +Ġon ze +éĢ Ģ +âĢĿ ãĢĤ +_b all +(r hs +Ġch orus +< count +as urable +Ġwirk lich +ĠEr in +ĠMS NBC +Ġet ter +ĠC ron +_F LOW +Ġ, čĊ +Ġcal idad +ĠFile Writer +ĉ stmt +( Byte +_p at +Ġte lescope +Ġgre ed +ĠT ort +(w rite +\ application +ĉRT LR +ĠConfiguration Manager +Un ix +End Time +In cludes +ĠHar vest +en berg +ĠAustral ians +Ġë ĵ +Ġr n +Ġreput able +Ġbl ending +UL ATION +ĠBrend an +d ad +Ġm ø +ĠW oo +_d c +U ne +Ġr ue +with in +ang ep +Ġp ouch +\" ", +ĠS ic +âĢĿ ), +aly ze +ĠG ef +c overs +Ġd bo +replace All +ĉ Logger +Try ing +[ state +-p iece +éĸ ĵ +beh avior +all ows +l rt +_p ython +ert ura +-c ountry +ĠT G +.UI Manager +b ens +ale x +ĠBre itbart +b ac +Ġpredict s +Ġg ab +Ġcard inal +.Time Unit +ĠVis itor +ĠM ing +Ġliv re +Ġparent Id +port un +Ġdimension al +ĠV est +en ic +à ³ +Ġ Ùĩ +ĠBL UE +Ġitem Count +Ġfe athers +ĉp stmt +ĠPol ar +{ // +und i +Ñĥ ж +z ar +Error Response +ì ĥģ +Rep resentation +* _ ++ ] +pre pend +Ġ' > +Ġlegitim acy +Ġo o +S linky +Ġnation als +. words +; p +tr ap +oman ip +Ġc ues +Ġgradu ating +Ġsem aphore +"] );ĊĊ +ace y +RE ET +Gr ab +ĠFel ix +( Id +_ne ighbors +Ġmeaning less +(d el +Ġj eder +ĠContent Values +.abs olute +/ cl +Ġx b +dat um +Ġtort ured +Ġrub bing +S cores +ĠðŁĺ ī +Ġav ons +Ġam sterdam +E OS +H al +Ġtrust worthy +# = +.EX TRA +Ġman o +is icing +-s upport +ĉc ursor +ĠSp o +aim assage +M ission +[] {" +Ġprint ers +G REEN +Ġt eg +Ġabdom inal +! ĊĊĊĊĊĊ +.Sh ort +аз в +ĠGift s +} ") +(b inding +x ce +âĢ ij +inf os +Form Data +Ġd art +Ġele ms +(in v +Y L +t in +GEN ER +á» ¯ +ĠT aken +uck le +: e +Ġspect ral +.b aidu +/ ');Ċ +Ġgre edy +es ion +,,,, ,,,, +Ġ/> ,Ċ +Internal ServerError +NSNotification Center +ĠA i +Ġsp it +Ġaug mented +Ġstandard UserDefaults +FIN ITY +R ace +: C +ĠRE CORD +ĠHigh light +Ġ' ` +Ġdef icits +Ġne i +Ġresearch ed +T a +Ġc opp +.Get HashCode +): čĊčĊ +On Click +ĠWell ington +Ġrev ival +æ¯ Ķ +éĹ ® +ĠN SS +Ġfor n +Ġint é +ĠKu wait +_fl ip +_ bo +_ \ +Ġocc urrences +ĠScient ists +S RC +og ens +igr ant +RE MOTE +ĠS ID +. opts +u ve +() ])Ċ +Ġlibert arian +ĠGl ide +les en +Ġform e +ow ania +Ġannoy ed +Def s +ĠExec utor +Ġcast s +.set Checked +ĠSh aring +.Serialize Object +Ġselect ors +_ OTHER +ë¯ ¸ +(s uper +( OS +_VER IFY +id unt +< header +Ġ/> ';Ċ +Ġvidé o +ĠNeg ro +ĠL ords +ĠT ours +Ġsoft ly +.re ceive +ĠE RC +Ġdata Set +Bad ge +ĉ Event +Ġper l +Ġ{} \ +(s entence +Or Update +Ġdim inish +P IN +(d raw +.To DateTime +.Equal To +(p in +-p encil +lu ent +ĠCall er +Ġplay ful +- '+ +x ca +sw ick +){ }Ċ +}: ${ +ĠM eth +.get Cell +.b reak +Ġy max +=' Ċ +ĠH iro +( TRUE +as urer +Ġcu er +U ber +. Operation +Ġol an +Ġthr illing +< Response +ĠF emin +Ġtravers al +Ġp oc +Ġset Status +decl ar +std afx +Ġaddict ive +ĠB tn +Ġexplos ives +ĠCook ing +ĠPl aint +Ġaccum ulator +ĠApp ointment +, password +ĠF AR +lu et +Further more +decl spec +_Static s +.D ictionary +"> '. +ĉ valid +" ", +In strument +> J +Ġno str +ĠR ift +_P ort +Ġvec es +[ [' +Ġrall ies +- series +Ġv v +. uc +Ġr tn +State Changed +( ins +ĠCl a +------------ Ċ +c us +ĠRel oad +//---------------------------------------------------------------- -------------------------------- +.se conds +_dest ination +Ġscrew ed +> c +Th ickness +Design er +Ġgr ids +n Äħ +( cookie +T rip +-M obile +Ġv oll +Ġgen ital +Ġconf isc +ĠConfeder ate +Ġweb View +Ġm ise +Ġcl er +(se lection +$ date +Ġshar pen +rag en +And Update +Ġrem ix +Ġh tons +R W +M PI +Ġretrie val +Ġric hest +.Dec ode +:init Components +ĠT Value +S aint +@ include +ĠPER SON +.se p +ĠLD AP +g ba +Ġgro ÃŁe +Ġreli ably +ĠD FS +.getItem Id +Ġprés ent +.get Token +Ġch inese +ĠMe al +Y OU +"> >ĊĊ +b ower +Ġsw apped +/ install +Ġs inks +etr ize +Ġdecl ines +ĉm ysql +ĠC String +ĠMotion Event +.L anguage +R oad +ÑĤ еÑĢ +asc imento +')) -> +. about +( editor +ĠR atings +in come +Å¡ e +.de queueReusableCell +ĠAust rian +Ġs ulla +ĠTrib unal +ĠDid n +ов аÑĢ +Ġins pections +B oss +Ġcock tails +Ġapolog ized +_sub plot +op al ++ =( +Ġreson ance +ib u +Ġë ¦¬ +rom a +res erve +pl s +ĠT ah +ax ies +OP LE +ĠDar ren +ĠZ ombie +_M ap +Ġ] )ĊĊ +ĠQ i +ĠS ail +Ġrestrict ive +Ġeros ion +- par +WH ITE +Ġold u +Ġap erture +Ġbit coins +text o +ĠCom cast +Ġtime less +en kins +Ġfeed er +/ tmp +res den ++' _ +.D estroy +Ġç ok +ĠD OCUMENT +.l ng +.tag Name +Ġk ullan +eg rate +Ġ(* . +ç¼ĸ è¾ij +Ġhand shake +s oc +_ geometry +ĠDam ascus +Min or +ĠK afka +ìĹ ¬ +Fl orida +_com pute +.ex pr +Ġpar alle +ĠD iaz +c ir +[ target +Ġj oking +Ġgl or +(set q +_hand lers +H ang +Ġf err +rim inal +ĉĠĠĠĠ ĉĉ +ent ies +def ines +-t ax +json p +ĠU PS +met ro +__ ;Ċ +ĠUg anda +])) :Ċ +_t d +x ae +l w +. OS +ĠLog ged +ac id +ĠMay o +as pect +Ġvag inal +Ġinitial izing +Ġster oids +f iction +G RE +g end +Ġli abilities +ĠL ets +M ech +( nc +( change +Ġconnect ors +: k +Ġt ast +! ");ĊĊ +th ings +ro phy +luet ooth +ĠSign Up +. ctrl +Ġthere in +ord a +. escape +ig ator +Ġpet rol +Ġspec imen +Ġdeb uted +- Pro +Ġcr ises +.add View +ëı Ļ +-d oor +Ġmon et +Ġmill is +Ġv ier +Internal Enumerator +Ġadmin s +ĠL air +z in +get Query +umb les +L IMIT +ĠV ig +_s ong +< Character +:: . +_h om +_b p +ĠSup ervisor +sub mission +ab ile +Ġno i +Or Create +Ġpe el +Ġon Start +Ġsent iments +veh icles +Ġclass rooms +Ġs zer +Ġb ending +Ġlong evity +Ġa cl +ĠAle ppo +ĠU M +ĠR icht +Ġmultip rocessing +DOM AIN +"," + +_Y EAR +Ġsc rape +Ġsol itary +Ġ"] ";Ċ +/ errors +ìŀ ¬ +ľ ëł¥ +b etter +ĉ number +ĠL F +ĠAc ross +Pub Med +\" " +ĠExcell ence +Ġus ando +ĠU IP +Activity Indicator +_V OID +Ġbre eds +ï½ ¥ +uest as +ĠTre asure +ustral ian +(f ace +ĠT ennis +ĉ Int +ĠHans en +ç µ +: I +Ġâľ Ķ +GR AY +O USE +Ġhe pat +ł í +A IR +ó ż +Ġque ued +vinc ia +ĠChrom ium +Ġcompet ence +ung al +ill i +Ġget By +ĠF inder +Ġincap able +Ġs add +Ġc ites +ĠChurch ill +S dk +More over +As pNet +( Float +$ password +ĠConn or +-s ession +_d m +* )) +Ġde utsch +ĠN X +Ġper ks +_S ORT +_TO OL +_V ISIBLE +.as p +æĪ ĸ +ĠBre ath +D etect +ĠD uel +.c mb +[ it +.Set Bool +Ġnarc iss +Ġab ide +Ġej emplo +ĠâĦ ķ +Ġm ornings +Ġcomput es +.s sl +j t +Ġmuch os +_S S +[ end +Ġbas in +Ġalgun os +ĠCroat ia +lin ewidth +(t ags +(h idden +ÃŃc io +Ġap ar +ĠÐ ¶ +ä¸ İ +. food +ĠR ural +Ġbread th +å½ ± +(s ess ++ ") +ĠP aste +Ġserv idor +ĠBit Set +ĠTr an +la us +v ette +ey es +ĠCL ICK +ĠV III +ĠTurn s +ĠLe Bron +ĠM uj +ĠD eg +ĠAdult s +_s uite +process able +ĠPH Y +g hest +.F ail +ĠSl ack +ce j +\ Carbon +Ġsuper star +Ġhold ings +( forms +Ġ'# ' +M ultip +("[ % +-s olid +/ url +-t ier +[ length +ĠStream Writer +ĠMarket place +get text +_T ICK +ĠFor ge +Ġblack jack +ĠDO ES +ĠM atters +w aves +Ġwhisper ed +Ġl ush +ìĺ ¤ +d igital +Ġwr ink +ĠH ogan +Ġrust ic +.Apply Resources +ĠHard y +os omes +A UT +.ST ATE +Ġnarr atives +ĉ store +b ib +ĉ Scanner +ĠC ody +\ Repositories +Ġre union +and um +âĢĻ h +Ġsn iff +NS Bundle +Ġcompreh end +_US AGE +_ occ +URRE NCY +J NI +Ġspecial izing +Ġvis ions +Ġdol ore +Ġv á +ĠChe vy +ĠSt yled +imp act +all en +Ġk art +ĠTable t +st uff +re esome +аÑĤ оÑĢ +//---------------------------------------------------------------- -----------Ċ +_Ad min +Ġcell phone +Ġaut oplay +Ġcamb io +Ġmar itime +_BO OT +- quarter +Ġlat ina +ĠAJ AX +e quiv +ĠFront ier +ĠX Y +} ]Ċ +ĠR ough +.pro to +Ġcorrect ness +Ġfac il +ĠRe ached +ãģĿ ãģ® +V IS +.p s +Ġstr ncpy +Ġdiff usion +.start Activity +�� � +Ġaccom p +AMES PACE +imon ials +ĠBl ast +aby rin +Ġd ome +Ġextr av +Ġy en +Ġcul inary +P RI +ĠComm unities +n id +_oper ations +.h s +ĠMil ton +Ġno ises +Autoresizing Mask +(c id +}ĊĊ ĊĊĊĊ +] },Ċ +ĠD etection +tab la +Ġlib erties +_D YNAMIC +w get +ĠT ür +ĠP ascal +Trans parent +Delay ed +] () +ĠHer bert +< ActionResult +ch allenge +Ġmush room +.insert Before +ĠR in +Ġhum our +Ġf ø +api Key +alloc ated +Ġconf ession +. ",čĊ +ĉassert That +ĠS ORT +ĠL ORD +Ġexport er +.set Level +p okemon +ash tra +Ġf é +ur ator +(M SG +Ġt up +ĠH ull +Ġyield ed +.Sub ject +\ Route +! ? +ĠÑĥ дал +\ Security +- ar +Ġalleg ation +( Settings +ä nder +Ġell ipse +ĠRetro fit +Ġregul ating +ĠM olly +ĠL ok +_C ustom +ĠProm o +is in +Ġres umed +Ġmet ropolitan +.error Message +: ------------- +Ġpas ado +th ank +_De lete +ĠBright on +, unsigned +ä½ľ èĢħ +Ġaspir ations +-h ow +R ose += (( +_ne eded +_pl ural +< Application +ĠW EEK +ĠUn lock +ĠT EMP +S ou +Ġschizophren ia +Ġt roll +Ġcomplement ary +ĠNET WORK +Ġbl ir +Ġprogress Dialog +" %( +ĠAttribute Set +ĉ ts +.iter items +è¯ Ŀ +Ġesc rit +v ous +_pl aces +H K +Ġseg uir +_f w +ĠR ounded +Ġdis posit +è§ Ĩ +par m +w ow +STRU CTION +. allow +ĠChar Sequence +ĉ extern +Ġprosec uted +Ġmort ar +ĠJ uda +- msg +Ġest ud +.get Description +Ġs ow +amb re +Ġrom a +En h +bon us +Ġsqu at +Ġdist ra +ed Image +Ġpe ppers +-per formance +, ĊĊĊ +, file +ĠM IME +_con cat +AB S +-f ashion +Ġunder cover +One ToMany +Ġre claim +C OPY +Ġb inds +ĠT ape +Ġg ossip +ĠEqu ity +/ Card +. activ +' am +Ġdrain age +< Scalars +ĠonBind ViewHolder +() ?. +Ġs orrow +ĠI b +up y +_U UID +ĠCh arm +ĠElection s +.on Destroy +ĠInterest ingly +ounding Box +_d etection +-h eld +_ unknown +Ġrefr ain +Ġmét odo +Ġe Book +EN OMEM +Ġd ang +Prof essional +Ġd ictionaries +/m ysql +ĠST UD +Ġmas se +s cape +Ġdre i +: name +.log o +Sign Up +Ġt ahun +( theme +ĠFem me +Ġbom ber +ĠJ ade +ĠT ay +Ġsubmar ine +_cl ause +zy ch +Ġsimult aneous +Ġcas os +. boolean +(l hs +Ġcontin ental +-s ale +ĉ env +ĠC ute +ĠFactory Girl +ab us +/ value +Ġj adx +Ġst ern +> >ĊĊ +Ġsurf aced +Ġìł Ģìŀ¥ +pl atz +ĉ email +cept ors +"> ( +Ġep ile +è¯ » +ĠDe bt +åij Ĭ +N OP +" https +: j +Form Item +_L ICENSE +.get Double +ĠAg enda +ĉf inally +(f ilters +( av +ç¾ İ +AP ER +Ġl ava +еÑĢ Ð¶ +)) ))ĊĊ +Ġfault y +_n m +Ġtr ava +(B itmap +Ġspeed ing +> '). +Ġscreen ed +_ roll +ĠMac Book +ĠA UD +Ġdiagn ose +.G enerate +Ġ^ ^ +Ġstr s +[ Test +Ġr ansom +ĠDH CP +eld en +Ġinterpret ations +() ]. +flat Map +Ġline Height +_m ount +ĠW izards +Ġsl uts +eh ler +od al +Ġmilit ia +å ² +earn ed +Ġmis ery +int val +f und +Ġh ides +Ġdi arr +ĠWes ley +Ġx mm +Ġqu em +ĠAr abs +if th +ategor ized +Dis posable +P ure +_NOT IFY +sn ippet +ĠGar rett +.run ning +. weights +Ġ( -- +Ġin variant +äºĭ ä»¶ +ĠAll owed +dir s +Ġpass ions +Ġl ad +ĠFl ush +men us +: block +Ġcompr a +.ch omp +alloc ator +Ġcur ated +ĠKnow ing +ĠPatt erson +Ġtel ah +' ex +Ġdo omed +Ġphil anth +ott y +.st yles +Own ed +Ġallerg ies += params +oc ese +it elist +ĠS ending +b ef +orr ar +ĠN ão +ĠF argo +ĠL ub +ĠComb ined +_g iven +ĉĉĉĉĉ ĠĠĠĠ +Ġreconc iliation +Pattern s +az ard +Ġbiom ass +ĠH ouses +resp uesta +cc o +/top ics +ĠY uk +Ġweaken ed +_c alendar +Ġmulher es +ĠMar l +Ġs ine +ĠT il +ĠSou ls +ĠDe utsche +ĠF OLLOW +Ġpip elines +ĠBever ly +_DIP SETTING +" # +ĠPro to +.b ig +ĠSav ings +ĠT anz +j un +ĠG amma +ĠS add +Ġadvis ors +Ġro ast +Ġun ters +ud ies +_l on +-point er +ĠElement Ref +\ Builder +example Input +.web driver +data Type +ĠQu ite +ĠCelt ics +u il +-def ense +b ish +ĠUI Window +ĠS uddenly +.h ot +.re ason +Ġg ör +AM D +.M ulti +auth enticated +reg ions +; ( +а ÑĢам +ĠKir by +$ route +PREC ATED +ĠDur ham +ow o +ĠPer forms +Ġdisreg ard +n st +ĠP ols +Ġget P +"] : +-col ored +( Keys +ĠAl leg +_mod ify +_ loading +str ained +Ġat roc +_p hr +< Sprite +Ġsatisf actory +m anship +.p ipeline +T ony +Ġth ief +pol ator +( lock +bur st +ĠOptim ization +Ġsurf ing +" Yes +Ġdesc ended +æ Ĵ +_C lear +Ġc ries +ĠFro zen +D IRECT +- Con +ĠLe icester +å¥ ³ +O OM += db +Ġget Message +< Student +_b atches +.M ask +_ eth +\ ) +Ġsom a +C atch +[ ch +Own ers +ind le +: auto +. vert +iv r +.set Location +Ġfl uent +_END IAN +ĠCar lo +cept s +add Action +.o auth +< UnityEngine +re ements +.S kip +? )ĊĊ +.default Props +Ġc abe +ĠSh en +eros is +ĠPro fit +Ġpo is +_C REATED +Ġremove From +(w s +? action +( Field +Ġerr one +.min imum +ĠRetrie ved +Ġd ado +ĠPR IVATE +-s pec +Ġg zip +p data +Ġpos Y +(l ow +Ġqual quer +/ cloud +ê² Į +( common +ĠAr beit +organ isation +Ġtid y +ĠRol and +( ph +.z one +Ġgent lemen +ượ c +å± ± +Ġenc losure +ĠMan afort +ĉ Color +St encil +N ic +Ġthe orem +ĠV G +Ġcol oured +V BoxLayout +uls ive +Drag on +c ff +et est +ens a +of day +.A zure +:UIControlEvent TouchUpInside +_up dates +Ġtrend y +ug as +weak Self +Ġr idge +ib ri +Ġì¶ Ķ +(C G +ĠMon key +.write Int +.tim edelta +ViewController Animated +ĠProvid ence +ãģ Ī +Ġbl ends +/Sub threshold +ĠAp pl +Ġat an +Ġreload Data +umb otron +st üt +O Auth +ĠG iving +ĠìĦ ¤ +ĠFinn ish +check ing +. Embed +sequ elize +Ġinitial izes +ĠOs lo +Ø ¶ +get Extension +_AL T +(bl ank +Ġfatal Error +Ġdem ise +**** *Ċ +ĠX S +(A F +ĠEn s +an tha +ĠP OR +Ġn ich +.N amed +Ġgig antic +ĠObserv atory +.Res olve +ĠPay ments +g uild +Ġcurrent State +============ ===Ċ +ĠS ey +p Data +Ġdead lines +Ġcentral ized +ĠScholar ship +_s upported +.ch rome +() ]);Ċ +Ġc yan +ĠC age +Auth ors +_ čĊ +/ os +k im +de e +.t ex +Ġyours elves +Ġm gr +Ġal k +-inst all +Ġdraft ing +Ġrum or +Ġstat ues +Pool ing +ol ina +AAAA AAAA +/* ---------------------------------------------------------------------------- +Ġextrem ists +Cal cul +ighth ouse +In set +(IN PUT +Ġsynchron ization +iv irus +. axes +ĠG ap +- An +_T emplate +Ġgam er +ĠCr icket +Ġl int +Ġauthor itarian +NS UInteger +Ġred o +Ġadip iscing +_F ETCH +che id +ĠF ang +. indices +t one +д ел +Ġ{{-- < +bra him +Ġsal a +get Code +Ġcommunic ated +start sWith +ert z +Read able +Item Id +oref errer +cred ible +á ria +Ġcombine Reducers +** /ĊĊ +Ġbl iss +Ġad orn +dep ends +ĠRO OM +Ġfr aming +Ġ? ', +aut y +_p ot +_t abs +Ex act +, ", +Ġ'} ';Ċ +Ġarbit r +ahr ain +.getString Extra +Ġ$ \ +Ġoutput Stream +Ġcomm enc +an us +ch y +< Employee +Ġhex atrigesimal +Ġn acional +(serial izers +_put char +_S AFE +ential Action +ItemSelected Listener +.Dis patch +Conf lict +_ about +os aur +Bound ary +Ġclear Color +( Location +ĠMON TH +ĠT aste +- General +ĠW AR +Ġer halten +-s aving +Ġcou pling +-tr igger +m otor +Ġy yyy +ĠPat ent +pt o +Ġmisdemean or +vas ion +ĠAdmir al +à¹ī า +_P WR +Ġdevast ated +fol ios +ITU DE +urre ct +Ġrobot ic +ĠSan ct +ĠHawai ian +.R oute +- condition +Ġr k +/**************************************************************************** Ċ +create Element +ĠK op +ign ant +. rollback +Ġsal ud +_ ', +ĠAN SI +Ex cept +ĠDraw able +.Utc Now +":[ {Ċ +Ġk ole +L ua +ĠBel ieve +Com put +Ġhall uc +ĠSign s +r st +.h u +ĠKN OW +W i +ĠBr ass +ĠR as +@ hotmail +Ġsed iment +Ġap k +Ġì ĥģ +_reg ions +Ġpod ium +< Book +ж е +Ġsix teen +ĠAli as +Ġinfr ared +ĠV ander +ĠLe ading +uc ing +,: ,: +_h or +w at +Ġdé cou +_W idget +S ounds +_n avigation +Ġschn ell +(g enerator +uc ene +Ġrem ake +IP v +Ġré al +_IN CREMENT +Ġhypoth etical +_ ang +Ġof s +Ġ! Ċ +.com pleted +Get Type +Ġkom men +ál ido +add On +Ġz ÅĤ +UL A +_ind icator +'] ĊĊĊ +ap ache +_S elect +ĠGre ene +Wh ats +_an im +Ġrepet itive +m uch +ĠTh reshold +Ġl f +(C ategory +con e +M ix +_MET ADATA +ays ia +Ne ighbors +ĉĊ ĉĉĊ +IP HER +ĠFr ag +ĠC ells +Ġnames paces +( back +ĠRest aurants +sv c +Ġл и +ote ch +-s l +¥ ¿ +ĠW T +ĠRed uction +Ġd otted +ĉf ound +ĠTE AM +B orn +ĠM ush +ĠCompar able +Ġh itch +AT O +Ġmax Height +begin Transaction +ÃŃ v +_b n +Ġher d +Ġrevers al +ĠH ond +del imiter +Ġconf use +Ġh ops +Ġcent roid +Ġcourt room +.decor ators +Ġm pi +ĠImpro ved +IN NER +ĠBang alore +ĠT amb +Ġbo ast +() ))čĊ +Ġil licit +ĠMor occo +greg ator +_res ume +Ġcrack down +Ġport raits +/h igh +( \' +Ġay ud +_fe edback +Ġc ate +/ avatar +Ġhe b +Point Cloud +Ġå ĴĮ +Ġ< ![ +Ġget Resources +} :{ +Oper ating +ĠF og +ĉt ab +ĠResearch ers +Ġfabric ation +.datas ets +ĠCamp o +ĠKa uf +Ġd ll +lig t +] ));ĊĊ +st ellen +ACK ET +l vl +ĠGl ory +.date Time +Ġcomm ute +ĠonCreate ViewHolder +ĠX Element +ĠT okens +< thead +_p ick +ì ¤ +v on +depart ure +(render er +phone Number +(P erson +gen es +ĠL ars +Ġ) {ĊĊ +ĠJson Result +Ġmet odo +VO KE +.get UserId +Acc eler +ĉ required +Ġchampionship s +Build Context +/t ask +/re leases +C ategoria +_over lay +Ġscar ce +_l im +n gr +ah len +ĠArt ificial +sp read +Ġbow ling +.an alysis +SM TP +ĉp assword +Ġbath s +] )){Ċ +current ly +ac iente +_se parator +Ġde ber +ĠDis abled +i ères +Ġâ ķ +_process ing +Ġprotest ing +ĠR OT +gr ab +Ġз ак +Ġpro active +word press +ĠSe ver +ind en +Ġw ikipedia +){ čĊčĊ +_w indows +is lation +Ġun rest +Ġdismiss al +.N UM +_F AST +iss ued +ĠF ACE +_u nder +Ġpl ugged +Ġå ° +ĠbÄĻd zie +ĠI CC +Ġcombust ion +Ġkiss ed +Ġstar red +ĠW atts +Ġspi elen +-p urpose +ĠE val +arg es +, result +techn ology +Ġnational ity +ic us +ĠN ug +ĠÑĤ о +ĉĉĉĉĉĉĉ ĠĠ +col o +Ġg astro +ante ed +OL ID +.b ias +_t ele +.ins pect +Ġve il +. footer +Ġneglig ence +Ġjud gments +Room s +yn n +ĉcount er +occup ation +Ġ çĶŁ +un as +Ġ(^ )( +L ambda +f el +.Param s +Ġд обав +set Layout +Ġdeport ation +Ġlocal Object +ĠPharm aceutical +cept ive +ĠN ome +Equ ipment +F an +Un iversal +ĉ socket +Ġgr in +Ġex poses +Ġhab er +Ġsincer ely +Ġc ams +Ġm ü +en ia +E mer +C rypto +Sl ow +(x hr +! =( +-s ervices +ĠP W +Ġprend re +Ġm ädchen +em ons +озв ÑĢаÑī +.M anager +ì Ļ +Ġg raf +- ra +met rical +/ fl +Ġc emetery +g ens +Ġp ÅĻ +ĠMySql Command +- To +Ġv Ã¥ +Ġa irst +oment um +Ġserv o +m illion +ĠMir anda +" She +Ġadvoc ating +-c aption +ĠAt tribution +Ġwel che +_v endor +ĉ Status +arr is +Ġprint k +"," # +Ġrel ativ +if ferences +izz es +Ġdec imals +ĠPro v +.max imum +Ar n +Ġhelicopt ers +_B OTTOM +ch ure +od ings +' ( +")) );čĊ +( bean +.f d +F und +Ġhang s +app id +/k ernel +.p oi +.Min Value +- validation +L uke +c df +ĠFun eral +ĠS amples +ĉ de +Ġto astr +Ġtax able +Ġcl ustering +Ġ'\ ' +Ġre straint +ec ed +ch ains +ãĢĤ ï¼Ī +_GR APH +Ġfue led +éľ Ģ +H p +å¤ į +T iles +Ġa unque +J C +Ġhost age +ĠE sk +Ġm av +Ġgest ion +Ġb anners +} {$ +.int Value +.' "ĊĊ +_M ATRIX +Ġce ased +ĠG OD +_CAM ERA +.Allow User +tr acked +C ook +b airro +( company +Ġview point +.get Writer +ĠN ets +w ives +Ġ( ))Ċ +example Modal +ĉ child +Ġmyth ology +Ġ// " +_ axes +ib old +.D ark +ĠMax well +Ġg pointer +olic itud +B at +ul ner +bal anced +mail er +Ġcont empor +æīĭ æľº +(" __ +Ġ" )" +re ar +ĠHu ang +] ')Ċ +× © +FT A +ĠCalling Convention +ĠOutput s +P k +.Re ference +lect ual +Ġ) :ĊĊ +Ġbrace let +ug er +ĉ Error +S weet +("/ ");Ċ +h x +Ġun reasonable +Inter preter +Ġlo ft +_product o +Ġsoci etal +.P arser +ĠAd apt +. foo +( where +.F eature +ĠYam aha +g lass +For ge +Ġprohib its +Ġcapac ities +Ġíķ¨ ìĪĺ +Ġper mutation +Ġih m +F ld +el ial +======== ===Ċ +@ Configuration +Ġge ared +ios o +iest a +trans lations +Input Change +Pop ular +ĠPL US +Ġv f +_F ree +b box +Ġcaus al +PI LE +Ġsch ö +Ġiron ic +M ir +. @ +åį Ĺ +Ġè ĩ +R ew +ul ence +fl en +Ġcan Activate +- response +Ġacc ents +ign ored +° F +.Dependency Injection +ĉ point +Ġconting ent +Ġsqu ash +Ġpar ms +ĠC emetery +Ġdelta Time +ĠD OS +Ġvan ished +аÑĢам еÑĤ +ĠD PS +t foot +ĠZ us +_IN STALL +G AN +Ġar b +Ġmunicipal ities +Into Constraints +AutoresizingMask IntoConstraints +, image +_ ignore +Ġdanger ously +quis a +pl uck +Ġhar us +up pe +Http Exception +Br acket +.' 'ĊĊ +ĠT ol +ĠView er +zb ollah +.Code Analysis +ì nh +Ġcorrect amente +.d a +ĠAl ger +× IJ +ba um +ĠPan ther +part icipant +å¿ ħ +-s up +Ġem ulator +Ġf ading +ĠW olver +cre ates +Ġbook ings +.Q uestion +§ è¡Į +Ġstress es +Ġre written +.PI PE +ed es +Ġc bd +": "/ +Ġenh ancements +_s y +B IN +ĠSl ip +Ins pect +ĠW eg +Ġcon gregation +Ġ_ : +_r m +Frame buffer +Ġ'& # +ĠFall out +Is Required +ĠPear son +ĠF ACT +Ġrel ie +ĉ box +ĠShe pherd +ĠWiki Leaks +ĠCollect or +Ġres ized +method Name +Ġevent Type +ĠA then +Des criptors +Ġb ers +- oper +ĠInitial ly +å ¡ +_B TN +ĠĠĠĠĠĠĠĠĠ čĊ +á b +_c ampaign +_w atch +F ord +-date picker +Ġvis c +Ġsat u +_s ms +Ġcont ador +-s vg +ĠDO I +$ args +Ġkn ob +.B OLD +Ġdeb ated +img s +sock opt +tr uth +ĠFe es +Ġh Wnd +_f ood +Ġab ras +Ġnot ions +ĠT od +: create +ĠConf lict +Us uarios +OT OS +Ġm sm +K HTML +([ ( +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġ} ] +w izard +Ġm ientras +Ġdata List +Ġemerg es +Äĥ ng +.Read Int +PG A +ILL ISE +I Enumerator +(t uple +Christ mas +Look AndFeel +og enerated +Ġ# ĊĊ +control led +Ġex quisite +Ġa cest +Read Write +G ain +ãĢį ãĢĮ +Ġcopyright ed +Ġdo om +.Table LayoutPanel +ĠD ort +Ġch ili +Ġwer k +ĠEVENT S +ĠBe acon +Ġship ments +Ġse bagai +up on +ut om +.con verter +.Drop Table +={ }Ċ +f ic +~ ĊĊ +Ġlesb ians +_n a +Fore ign +ĉ then +/ ms +Ġor i +get Property +ĉsn printf +hes ion +ãģ ¤ +"} ," +Ġac rylic +P ers +@ Enable +I sl +(C ard +. Stack +L icensed +_G UID +: title +Ġh ust +Ġprincipal Table +an itize +/ embed +Ġens ured +ĠE GL +ÙĪ Ø± +ĠåĪ Ĩ +/ ,Ċ +Ġfundra iser +Key Name +Ġmarch ed +_VAL UES +ĠSc enario +Ġmet ic +_ass oci +ĠPast or +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉĉĉ +er ate +Ġinv itations +quo ise +Ġbl aming +Ġd aring +UM MY +Ġrich er +em aker +ĠIdent ification +ĠìĿ ¸ +ĠBinding Flags +ch as +Ġresil ient +_p g +Ġre leg +ĠI RA +ST E +Ġtr actor +- loading +ĠPre viously +ĠV acc +/ be +Ġn Ã¥r +Ġurl encode +ĠNor folk +.Re lease +ĠNe utral +ä¸Ń åĽ½ +ĠAr lington +Ġalleg es +ĠW riters +Test er +ĠR ally +Ġc á +ĉ Print +Ġâĩ Ĵ +ĠUser Controller +ĠSeek ing +.V AL +List Node +_ ff +ĠPhill ip +FA CT +Ġc aramel +ĠM ultip +ĠCom pared +ĠSer bia +Ł ³ +Ġrev ive +ĠK anye +Ġver ge +ĠBulg aria +get Body +Ġ| > +ce ph +.DateTime Picker +." ;ĊĊ +ĠT ie +, item +Ġm enn +G as +och a +_v irtual +Ġmaster piece +_se quences +L TE +ĠSub mission +Call er +$ \ +S port +ag us +Constraint Maker +Ġcol oc +Ġw ig +ĠÐ £ +ĉ Array +Look s +ĠGT A +.st eps +atch ewan +_r anges +ext Alignment +ĠBren nan +Ġab straction +uler Angles +.m isc +Ġantib odies +Ġexponent ial +ĠCH ANNEL +exp ense +' y +Ġdetect ives +Ġpur ported +Y STEM +Ġradio active +ĠLat ina +.Enc oding +.T AG +x in +D egree +ur acion +pr ices +ĠRefer entialAction +Ġr arity +Ġp iles +g ende +_project s +_g lobals +.start Time +Ġê µ¬ +SE CTION +_p ublish +F ault +DD L +_p rior +M om +Ġth icker +Ġsequ elize +Ġessential s +str as +in tr +>( () +.man agement +e il +éĹ Ń +A ware +.C ity +ĠAr bit +_D M +_key board +L Object +- webpack +ĠNew port +Ġprincipal Column +leg ant +Ġp allet +Ġfract ure +Ġg mail +.M eta +A bove +.Key Event +j it +_mac ro +_P USH +á» © +/ controller +åĬł è½½ +Ġsuperf icial +exter ity +Ġmens agem +W ind +ist on +.open api +и ÑĢов +ĠSerial izer +uct ive +Ġz ar +Pl aces +.St atic +B a +Ġin advert +ĠIndones ian +_IP V +(h orizontal +Ġget Title +ide press +ĠConsole Color +ip ers +$ out +Ġfest ive +Ġeven ings +.Get Data +uit ka +ĠManual s +uss ed +_M ax +.Ch at +ĠA ircraft += com +FO UND +ap ro +Ġtre asures +_al ive +Ġgad get +ek ing +Button Down +B rowsable +.PER MISSION +P ASSWORD +ĠH ASH +f é +\ TestCase +LO SS +o thers +, J +Ġassh ole +wer k +Ġm ã +. ie +ev il +kont akte +//////////////////////////////////////////////////////////////////////////////// Ċ += sys +ĉ lock +-- ;ĊĊ +_F UN +Fill Color +ó a +pre nd +Ġcompress or +M other +ĠAr cher +.g oto +Ġwür de +Ġbam boo +ï¼ İ +ĠT rees +Ġb umper +Ġsa usage +ĠEl asticsearch +Ġhor izontally +ĠG ul +Im mutable +Ġlos er +Ġabort ed +-d emo +ĠH atch +Ġund e +Ġprocess o +-c all +In come +å ĥ +_ returns +']." ' +(s w +C BS +am ilies +ĠYour self +ĠH olt +.M ON +à§ ĩ +ÑĪ Ðµ +an on +ĠFont Awesome +produ cer +j r +Ġm au +ĉint er +Ġdish onest +Ġmagn a +ĠCollect ive +Ġvra iment +Ġcho ix +st ay +Ġweld ing +r ising +, min +ĠF ate +g lob +RGB A +Ġdet te +V en +Ġembarrass ment +.DE LETE +greg ar +-re nder +(b ucket +"> ĊĊĊ +.wait Key +Bus y +Ġdifferent iation +ĠC ST +.Con stant +Ġline Number +(m atches +Ġweb socket +Ġbar red +Ġpued es +M ono +C ORE +I ID +ĠĠĠĠ čĊčĊ +Ġpúb lico +lean ing +Ġcleans ing +Ġcr is +ĠDev ils +_SET TING +unt ary +. );Ċ +Ċ ĠĠĠĊ +[ curr +ts y +ĠAlex is +rit el +Ġpet roleum +.pre processing +m atter +For Result +- license +Ġtrav ellers +ĠDispatch er +enn ifer +Ġdigest ive +P ED +hib ition +MAS ConstraintMaker +ĠW att +Ben ef +.set View +d to +TE E +ĠPel osi +_EX TRA +Ġmed als +x hr +fore cast +Ġn argin +oun s +-f ill +_CUR SOR +Ġsuperv ised +Ġtur f +ĠEd gar +POS ITION +Ġcategory Id +â ī +_ ER +á»§ a +Sh own +. ll +_POL ICY +(), ' +ĠPre v +ĠString Field +ĉG lobal +ass ed +Through out +o stringstream +.awt extra +Ġslo pes +ĠSe quential +Ġgi orn +Ġz elf +Ġvers atility +lene ck +.c gi +Ġdou bling +ĠBang kok +Ġbu urt +Ġusu ário +st udio +Ġje unes +Ġm uted +Ġ ips +_f raction +&& ( +Ġst unt +'); ?>čĊ +Ġev apor +b able +ĠPR ICE +Ġæ ³ +lu cent +Ġv amp +ĠTechn ician +Ġuniqu eness +M es +ur ban +.param etrize +ĠRe play +S essions +em br +-Americ ans +_PRO XY +Ġp ian +Ġtri e +ĠD estructor +Game State +ĠIM F +ch in +Ġport e +ĠSw al +åŁ İ +Sub string +im ing +/L ibrary +Ġfright ened +w rites +Ġrecurs os +ar Result +_INIT IALIZ +ĠBad ge +_c rc +E ight +ĠDIST INCT +Ġth ro +@ Xml +ĠLegend ary +-t witter +_e asy +Ġ+ ++ +(D ATA +.L ocale +Ġk ä +Ġn urt +Ġcr uis +_ ios +Ġsens ing +_L ine +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +pon g +ole on +Ġwild card +ç͍æĪ· åIJį +Ġbeg ging +R od +ĠÃ İ +_C ELL +Research ers +. selector +_ ing +Ġaspir ing +Ġimm ortal +Ġy min +_ robot +Ġpl ur +B TC +ĠD ID +Ġpier cing +* u +_DEFIN ED +ĠTh i +ita ire +(m edia +- ons +Ġche fs +Ġ"* . +/ AP +Ġraz or +Ġsearch Data +Ġ= & +Ġ ãĢĤ +Ġm ourn +ting ham +Ġo li +ĠVern on +_R S +ŀ æĢ§ +Ġf ácil +ang en +cel ain +Ġa il +le st +ĠQ COMPARE +g ain +ĠÎ µ +ĠK ob +ĠF ault +_config s +ç»ĵ æŀľ +. + +cal ar +(color s +M ul +_ ART +Ġexperiment ing +erm en +ĠAng lo +.Fixed Single +Se a +Ġc txt +.s lider +C ollapse +G rey +Ġf ld +-pro of +.cap acity +get Parent +ĠCom pliance +Ġburg l +- rec +Ġover written +M U +Ġrout ers +ĉ Model +Ġfantas ies +av ian +_p rec +ĠSc andin +Ġ// < +/o ct +Ġceremon ies +Month s +und y +Ġqu ed +ĠN ou +ĠV ibr +.r gb +Ġcit rus +Ġbr aces +-upper case +get Table +Ġdop o +ĠK err +_CH ILD +- cloud +ĉ Matrix +Ġgard ening +S ing +al most +Require ments +ugu ay +( Property +sub scriber +FA ST +re action +(l p +) })Ċ +` ). +.w allet +_ex change +.Max imum +ĠVer b +âĶ ģ +() < +ï¼Ľ Ċ +RO T +C ARD +ub it +{ @ +_k el +ĠTool tip +My SQL +Main Activity +ar f +Ġm align +Ġse inen +ap ist +Ġ< % +Method Impl +M il +ĠM ick +.de pend +< ID +Ġpredict ive +ĠAP PLICATION +le f +dim ensions +Ġconoc er +/ conf +ĠTr acy +F oto +_rem aining += file +Ġpage Index +ĠPar ish +Ġt exas +ĠM AGIC +ĠH ew +d ifference +Ġalt ura +c um +ĉdata Type +Ġcaracter es +avi ours +ĠV OID +è¿ ij +P UBLIC +B io +ĠstringBy Appending +Parse Exception +ĠS uff +ĠN orton +/d etails +.n ull +>> & +ĉ ok +-l ow +. usuario +n ested +X B +OUR S +.Border Color +Ġb row +ĠÐ ķ +cor r +ĠRed skins +.get Tag +.get Transaction +Ġst igma +hard t +ĠPlayer Prefs +als y +uc son +L anguages +ĠOl ivia +Ġt ac +Ġb li +Ġc aval +Ġconsolid ated +Ġper il +Ġde le +Ġform ulated +Ġhigh ways +.sp awn +== $ +ĠN iet +Ġv eggies +yp o +-r ule +ĠV ie +/e pl +Ġenf ants +string Literal +Ġtou ghest +buy er +Ġcov ariance +Ġil i +ĠSoph ie +ĠB AB +Ġ" ), +ĠU k +current Index +_user data +.code c +ĠPun jab +ĠSN P +l ol +adv ance +Ġcom fy +Json Ignore +Ġfashion able +ĠI CON +Ġor a +ĠP ricing +< num +ĠI RC +ER V +ĠMe in +ĠID ictionary +AD OW +is New +ĠDev on +at l +(request Code +ĉ PreparedStatement +IM PORT +Ġmar ital +_SELECT ED +get Response +ar Down +B V +ib Name +ĠP ATCH +ä än +Ġda ar +ĠFile Mode +Ġm arty +.Spring Application +c ene +amp oline +get Size +Rest art +æķ Ī +.project s +ĠEthi opia +Ġstatus es +T ION +(b g +ĠX unit +Temp orary +ĠEng agement +Ġx f +Ġprox ies +Ġgen esis +Pager Adapter +ĠSl ave +Ġsung lasses +ĠCh loe +Ġko ji +ad em +ĉ JSONObject +Î ³ +Ġh ors +* w +ó r +es ch +Ġcritic ised +z ial +ĠSale m +.Vert ical +ĠR ash +> E +ter ing +/s creens +Ġheight ened +аÑĢ ÑĤ +Author ities +_b box +ün st +.font Size +ĠBO OLEAN +div ide +ĠSlo ven +uc er +Ù Ĵ +st ub +Ġnavig ating +: animated +_N OW +_v ect +} {Ċ +@ ( +Ġtele com +Ġcontract ing +ĠAss ange +Ġextract ing +Ġgr ö +c obra +.D IS +Ġcr ab +Ġtw itch +Ġvert s +Ġreject s +ĉ format +Ġreg eneration +.S ys +s olve +ĉd ialog +sh i +m eter +(b est +valid ators +Ġon wards +Ġg uru +Ġmoder ator +ow ied +ex periment +r ub +Ġm qtt +ĠCa ucas +Ġnational ism +Ġm ange +ĉ ImGui +/ Edit +Ġin h +Ġint ellig +ero kee +ĉ export +Ġdiscrim inate +sub tract +ĠM oodle +ens er +ĠGuid es +R AP +-h ot +_gr p +.p icture +X A +Ġinit View +_Com m +Ġoverd ose +Ġ+ ĊĊ +ĠSil ent +show s +Ġinterpol ate +Form ation +Ġb isc +mark ets +( SC +Z e +ĠNetwork ing +Ġad renal +ĠG uns +ete or +Decl ared +orget own +Ġk arena +/ password +_address es +ITER AL +B uzz +ĠCon way +(c ase +P WD +he iro +( act +** čĊ +());ĊĊ Ċ +Ġan v +Ġ. .ĊĊ +(Menu Item +(m ail +_section s +ĉ net +Ġpl ut +Ġw rench +/ object +ĠI st +ĠV IS +/p ub +al ten +Ġguit ars +Ġantibiot ic +ï¼ ĸ + ¹ +Ġ" +" +form ula +Ġbab es +ĠP rompt +Ġen im +/ player +ĉ ref +Ġby Äĩ +Ġconsum es +ĠH ast +ĠT ao +Ġ' ))Ċ +Ġcl am +Ġthigh s +Ġmot if +Api Operation +ĠW L +get C +ĉf lags +oint ments +Ġeconom ical +need le +x ls +pr actice +ut zer +time ofday +- output +Ġfind ById +ĠBudd y +Ðŀ ÑĤ +Se ven +ĠB ark +Ġenv oy +_al gorithm +åĪ © +Ġball istic +ç§ » +r ades +ĉd oc +rodu cing +ĠE ating +Un mount +/data Tables +_b onus +Ġl itt +pp s +) localObject +per f +ĠHel vetica +sh utdown +/ ml +.t okens +ĠHard core +, row +/b g +Sc aler +âĢĶ as +_log its +âĢĻ int +ĉ App +Imp licit +.F printf +ET O +Ġterr a +Ġpossess ing +.r strip +, ), += yes +ĠStr ipe +? = +ne utral +.g ood +Ġk ennen +ĠS ung +f ault +ystate change +Can adian +',' ".$ +ĠM its +æ nd +ĠSTR UCT +ĠURL WithString +ĠCom pass +Ġ-- ĊĊ +ĠNS LayoutConstraint +| min +-ad just +Ġreb uilt +L IGHT +/ se +-m ount +vp n +valid ated +(Q Object +Ġign ition +ĠCharg ers +RYPT O +]initWith Frame +ĠFl uid +Ġcad re +Ġnomin ations +Ne ill +ĠH ou +Ġcurrent s +_g ene +(in p +Par is +z ÄĻ +ag gregate +Ġass oc +weet ed +err at +âĢĵ ĊĊ +Ġ'/ ',Ċ +fix ture +ĠH ighest +amb ient +Ġch mod +Ġcon te +Ġsens ual +Ġgar ment +z ers +ĠPower ed +dom ains +R eward +i omanip +Ġcock pit +out file +Ġbuilt in +Ġins isting +. vars +zip code +Ġ ���� +f ails +Ġconsolid ation +_ oid +Plan et +Ġ= ", +ĉ el +UIL T +ät z +af ari +ĠMc Cl +Tim eline +Est a +Ġfr am +Y E +Ġcere bral +Of Month +ĠP regn +Ġкл аÑģÑģ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +ĠF res +Appro ved +.S pecial +ĠProtest ant +Ġallerg y +_p cm +ĉC opyright +Ġsuper Class +" strconv +ĠMoh amed +Ġ' // +Fore Color +Ar thur +ĠJ ungle +Ġve ins +S ad +Ġback ups +ĠOp inion +û t +Ġinter mitt +ody n +ĠChrist ina +Ġand re +Ġevac uation +pa lette +h orse +ĠRes ident +ĠHass an +.N il +Ġa isle +ĠG rowing +Ġblog info +/s ql +_io ctl +Sc aling +ĠMon ad +_c pp +ĠH utch +ĠApple WebKit +Exp ense +_J OB +Ġpoint less +From Body +ant al +Ġdepict ing +ĠC ELL +Ġref in +ĠC NC +ì¹ ĺ +_dim ensions +ĠS AN +Ġa ft +Ġfoot steps +cc oli +_PH ONE +/m ath +-k ind +ĠMe ans +ich ael +.g una +Ġinaug uration +-dr iving +( delete +Ġtotal Count +_M C +.Ext ension +Com mercial +Ġz Index +< Customer +" g +-sh are +Ġp act +ag ara +ĠS IL +_m odes +ĠM olecular +Ġsystem atically +< G +_s cr +ĠO ro +as ers +Ġb ic +Ġdest roys +PI PE +.Start Position +Ġc á»§a +ire z +.B unifu +_F unction +Ġs ü +_f uture +ĠWe alth +ĠNatur ally +æĢ » +_y es +Ġabrupt ly +String Encoding +ĠCGPoint Make +Ġz h +Ġimp erson +Ġpiv otal +ĠSom alia +Ġsegment ation +_AN AL +ĠLogin Component +Cons ult +Ġtr uncated +] ";Ċ +.get Config +Ġintern ship +B aby +ê° ľ +Ġstrengthen ed +_M I +b asket +Ġnicht s +ĠTV s +ĠSh an +ãĤ µ +rac use +.Re LU +/ interfaces +ĠgetItem Count +Ġret iring +Ġspecial s +Ġentity Manager +bel ief +Ġs older +da ughter +ij kl +Ġutil izes +.f ixed +S U +Ġdr astic +Ġh acks +gr und +ĠM U +ĠSt arter +.Com ponents +_m otor +Gold en +Ġl odge +Ġ )); +ĠCor inth +иÑĩ еÑģÑĤво +ón ico +gre SQL +ĠFl uent +Ġmar c +.Load Scene +.Group s +Ġer h +ĠAut umn +St opped +Ġitalian o +Ġmin ions +ĠAssert ions +Ġm ux +B u +Ġ---------------------------------------------------------------- -------------------------------- +ĉ up +read ystatechange +_M eta +Ġcurrent Date +ĠChap man +Und o +Se an +ap r +Ġpar m +_ icons +ĠSt a +á z +Ġsub division +Ġalter ing +P NG +ponent ial +Ġpost gres +ĠB DS +-ex istent +ĠBrad ford +ĠO MX +_W HITE +_PRO GRAM +q c +Ġtypings Slinky +ĠP ics +_M ETA +IT TER +_sub scription +IRON MENT +ĠHy undai +();ĊĊ ĊĊ +ĠØ ³ +Ġj ac +Ġelimin ates +) });Ċ +Ġcomp rend +ĉ insert +_f aces +"> $ +Ġeb ay +Ġcapt ive +pl iant +ĠCalcul ates +ol ta +est ing +_re vision +Ġm ús ++ m +"," "," +WH AT +Ġcompassion ate +h arga +[ random +Ġmod ulo +(s n +Ġoccup ations +//// Ċ +ĉ board +ĠB alk +wi Äħ +ĠW ifi +.Pro file +:m aj +ĉm at +LOCK S +(j Button +Ġ(' $ +M ur +æĮ ī +b ble +Ġf rog +-h ide +Ġbroad caster +ภŀ +ha led +Ġam using +_predict ions +_in tr +Ġe agle +аÑĤ елÑĮ +Ġget List +ps ilon +Ġcharacter ization +AR DS +Ġre location +Ġr ulers +P AY +ĠDef initely +_A ction +Ġclos ures +Ġfact ual +odyn amic +Ġpreca utions +nie j +ĠPart ies +ĠSub aru +Ġcous ins +ar beit +.m oney +gun ta +( and +get item +.Style Priority +Ġsl id +single ton +Ġg arn +ĠP AS +Ġd azz +a ż +Ġbog us +ĠM og +Ġrival ry +is ol +Ġland marks +ñ as +B ern +ĠSach s +Ġ" )ĊĊ +Ġhost ility +_m ex +m ere +M ot +p ictureBox +Def ense +Ġaffid avit +other wise +.d irectory +_ UnityEngine +-b log +.s kin +ph em +Ap ellido +er chant +[ class +Ġw art +." [ +ale ur +/ back +ĠĠĠĠ ĉĠĠĠ +Ġprecip itation +Ġob struction +Ġp Obj +Ġr upt +UCK ET +ay e +æİ Ĵ +g x +Ġe cl +Ġsecre cy +/ Header +ĠLes b +Ġle i +ĠBullet in +Ġgive away +.H ome +_RO OM +" W +Ġcow ork +_ ra +ĠC ycling +ĠP aw +Ġpup il +/ arch +ĠFile Utils +é¦ ĸ +r sp +Ġfreed oms +ĠL ear +}` ). +Ġbow ls +/b lock +_log ging +Ġmeth ane +Ġhorn s +Ġwonder fully +Ġalter ations +Ġex ile +ls en +_p ause +_L ANGUAGE +ĠUS DA +_m ysql +_AM OUNT +ĠL IFE +Ġyoung sters +Ġri ots +[ E +Ġun forgettable +, },Ċ +Dis posed +ĠAss assin +UN G +ĠNew sp +User Service +: aload ++ ', +Ġsett lers +Ġscre ams +Ġincon venience +.R otate +Ġj ars +ĠP uzzle +Ġm est +ars i +ĠSh arma +| ( +.d s +ĠSac red +_e vt +Ġexpress es +Ġh och +ĠD uch +.c alls +th r +ĠShe ffield +.Alert Dialog +Ġrad ically +Ġtr ous +Ġprev ailing +ĠWW II +âĢĻ n +ens ely +ĠY esterday +ĠSir ius +Ġkill ers +ĠF FT +Ġo val +') :čĊ +Ġìłķ ë³´ +our age +ĠCheck box +Work book +.def er +_f loor +Ġc ouncill +Ġnors ke +mo il +ore a +Ġmarket ed +_S UR +x AA +Ġst ained +e ut +ĠM eng +Ġi eee +. extern +eg ie +Ġr app +ĠPy ongyang +' class +M ob +Ġinitial Value +_w ave +Ġj ab +Ġmascul ine +Ġampl ifier +Ġt ty +Path Component +_ xt +ĠG FP +/ sec +ĉdis patch +mark down +ĠS chn +bo le +· · +mouse move +Ġerr Msg +Ġas ign +_m ono +To Selector +ĠZ u +(R ect +ĠError Code +lat in +ang ible +v tk +CG Size +P okemon +Ġclass mates +Ġattract s +ĠT atto +ult an +ol óg +Ġhalt ed +ठ¨ +ĠK art +Ġ ue +_Init Structure +Test Class +ĠAir bnb +_ ", +Ġchar coal +Ġip c +ĠSt retch +.g lide +lates AutoresizingMaskIntoConstraints +Ġpot ion +ITT LE +Ġcount ert +_h d +pre pared +Ad s +ĠV ampire +rob ots +.Create Index +Status Label +Ġt ucked +af ür +U t +Ġswe ater +_F N +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +ata ka +Ġeyeb rows +ac oes +ud en +.LinearLayout Manager +Ġsw ay +Ġmult in +() )))Ċ +ĠNS UInteger +ĠMy Base +Part ner +uts chen +ĠC ater +.setBackground Color +Ġaccompl ishment +_pro blem +.d td +Ġpage Number +Ġj ackets +Ġcro pped +u els +ĠH ep +Ġc apped +* Math +_callback s +Ġpub b +ĠBrun swick +.res pond +[" _ +Ġbed ding +hyth m +O X +(s peed +Ġpestic ides +Ġ---- --- +.Bl ue +Ġnood les +ĠGo es +Ġs aver +o xy +_com pletion +ĠSw inger +Ġget Date +Ġmind ed +int egration +ĠLot us +(st op +(', ');Ċ +Ġflood s +ĠWork flow +Ġerupt ed +Mac ro +ĠSau ce +Ġevent Name +\ Input +Break ing +ĉ when +_p w +IND ER +ĠWell ness +Ġvox el +ĠM ell +ĠM EDIA +SE NS +ĠFund s +ĠM ild +< Array +- this +ump ed +/f w +ĠDb Context +W I +girl s +H OW +'); ?>Ċ +Ġtempt ing +Ġtest ament +Ġb ible +Ġconsult ed +ĠIndex Error +è¨ ĺ +Ġkey pad +izz o +( ok +Ġwhats app +ĠRemote Exception +Ġteam ed +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +» , +Ġget Time +di ag +iss y +Ġh ed +Ġkn ots +j om +Ġfun nel +-m ails +Ġexport ing +ĠV L +ĠK arn +ĠBuddh ism +ĠAll an +_R ADIUS +Ġw ording +ĠFor get +ĠCor ona +ip hy +Ġlim burg +ugg y +ĠUser Repository +im in +(e le +Ġlabel led +ç¤ ¾ +ĠH erman +.q q +Ġ" ));Ċ +ie ber +.Trans late +ry n +Ġdes env +um d +Sim ply +ĉm ode +R pc +ĠVal encia +Ġstaff ers +Ġsel v +ĠSpi ke +Ġdel ic +Ġer u +_D T +J udge +á» ķ +ĠBas in +.m utable +" url +Ġtar iff +ĠSlee ve +Ġfl are +.drop out +Ġbr ides +)) ,čĊ +_con straints +de struct +Out line +Ġdisappe ars +_lock ed +ĠNS LocalizedString +ck e +ĉ null +ad resse +Ġto pping +ĠJ oker +b ishop +но ÑģÑĤÑĮ +and ering +_ amp += time +_S pace +_P ULL +' = +Ġant iqu +Ġc ach +___ ĊĊ +ON ES +о Ñı +Ġun read +.p olicy +oooo oooo +ëŁ ¬ +Ġu sted +ĠRe ce +Ġal lem +ãĥ¼ ãĤ¹ +ĠThought s +ve illance +istr ate +_l ane +Ġfam ed +.Get Name +Ġsmo other +ĠQual ified +az ers +_ geo +F ax +ĠM inds +ĠR aises +Ġtrans cripts +Con versation +Ġremark ed +ëĤ ĺ +d ling +Ġdeploy ing +Ġshared Application +Ġk p +FontAwesome Icon +_d ummy +reib en +ĠJane iro +Direction s +.get Bean +s ass +Ġcommand ers +v ation +error Code +ĠAl loy +.local ized +Ð ij +Ġdish washer +ĠSou p +N u +_D efault +Ġune ven +Ġ/> ";Ċ +-B ased +Ġseam lessly +- null +ĠX C +Ġst ew +(d elay +AT ORS +ĠWhe eler +" H +e ast +. air +âĢľ But +Object Context +success fully +_l and +Ġfold s +_CO ORD +Ġsub po +.get Address +in str +Material s +Ñĥ ÑģÑĤ +de posit +-l ast +_GR AY += find +Ġmut ant +Ġlesb ienne +let cher +RO UGH +ure ka +.c apture +Ġen n +Ġ([ [ +ĠFl u +Ġtask Id +ĠHus sein +.f older +Ġa usterity +ISTR ATION +_ Impl +注 æĦı +Ġdec ree +- chat +Ġimp lication +Ġguess es +ul kan +An alytics +. plus +COM MAND +е ли +» ĊĊ +_S ITE +Ġequal To +Support FragmentManager +ĠRec ording +å®Į æĪIJ +Ġbag gage +Ġpitch ers +ĠE h +o que +ĉc nt +Ġ=> $ +/ foo +IR A +ĠSat ellite +bor ah +Ġ}} "Ċ +ĠEnd s +ĠSpr ay +, param +.Ch rome +* q +th ought +ibr ated +Ġth ieves +Ġbenefici aries +Enter ed +ottes ville +Ġveter in +By ID +qu ipe +um ption +- unit +Execution Context +@ s +ĠG iov +.Tool Tip +_f riend +( attributes +Ġdump ing +ĠJ C +_D OCUMENT +ĠArm our +( insert +.Horizontal Alignment +ĠQ ed +ãģĦ ãģ¾ãģĻ +/g it +ĠY YYY +ĠCard iff +Ġap a +organ ic +ĠWhere as +Ġæ Ŀ +ĠM ia +Ġdemol ition +Ġsc ars +Ġp ai +Ġre tries +Ġr q +ĠDen is +( Utils +Ġallev iate +ĠP IC +id ue +Ġacknowled ging +Ġ// //////////////////////////////// +ç¡® å®ļ +Ä « +\ Json +.b inary +Ġx type +sign als +ĠAp pearance +& r +} s +C i +ĠI llum +por ate +h og +Ġindex Of +\ Command +_par allel +ĠSher lock +í ĥ +Ġ" ")čĊ +//////////////////////////////////////////////////////////////// //////////////////////////////// +Ġcritic ize +ĠSo ap +ĠMatch er +Ġgr illed +* T +Ġad ore +ull ing +Ġjed och +_ref s +lean up +ĠJ AXB +Ġro ses +ĠL iam +size i +Ġget char +Ġtar de +-to oltip +Ġqual ifier +ĠInter mediate +_W indow +ĠMal ta +Dis connect +ew here +Camp o +Ġirr ational +led o +ĠD N +ARG V +Ġout ro +Ġth irteen +Jose ph +M AR +/g l +J ess +ĠPsych iat +Ġpadding Bottom +- loop +/ fonts +_se en +Te ams +React DOM +(m an +(x path +.get SimpleName +>( * +ĠP vt +Ġel ders +Ġp ies +.user Agent +- region +ĠGree ks +(f ragment +st u +Ġcouncil s +Ġst amina +ĠGod dess +è ¥¿ +Ġphilosoph ers +Ġpers one +ĠL ose +ĠCL R +ĠD ocs +Ġso ak +ĠHOLD ER +Ġb ells +hash Code +R ATE +_WE IGHT +in ous +end ra +oph obic +Ġpro se +Ġfin ely +/o auth +(s pace +ad ge +ĠM ama +Ġstring Buffer +Ġst int +Ġmis ma +Ġvill ains +ĠCrime a +Ġdipl oma +Ġпо Ñģл +ĠBe a +(j oin +Ġíķ ´ +CH AT +per ing +ĠC ros +Ġmon keys +Ġpred s +yl a +,, , +Ġvibr ator +ĠN U +åħ Ī +f ant +z et +Ġb ietet +un ft +sw orth +.F low +Ġpsy ched +ĠContin ental +> t +Ġqu ilt +. UP +Ġexpans ive +Dis pose +(l anguage +C aps +_Z ONE +Ġrec ycle +ĠMan aged +current Color +.b roadcast +sign In +.p rom +ll u +ue blo +Ġpunch es +Ġautom at +Ġassign ing +Ġcreate User +ĠAll ied +Ġconduct or +Ĥ ¨ +Ġs addle +Ġd ni +omed ical +-W est +Positive Button +Ġit alic +? [ +(tr igger +Ġele phants +":" "," +Ġcal iber +raft ed +d igits +Ġmar shal +mill iseconds +mark ers +m om +/ place +Ġhol istic +: t +# , +Ġb oto +Ġnause a +ĠSh ooting +ite ch +Ġtext Status +< Class +ĠDes cribe +Ġbuff et +g il +Ġlog its +std call +mod s +ĠSk ull +ĠB are +h ope +ĠIn tr +F air +ĉ pt +Ġacompan h +Ġf kk +_r pc +Inst alled +_ ans +.get Minutes +â̦ "ĊĊ +- thread +Ġpres chool +AIL S +Ġdiff ic +( convert +ĠN ath +ĠDO J +Ġreg imes +Ġenthusi ast +Ġwarrant ies +Ġfasc inated +_b inding +_N ot +oft en +_R W +/m ail +Ġtitle Label +Ġvill agers +ĠJ iang +Ġsw agger +.Row Index +_img s +rap y +VER AGE +. Up +Ġno op +c io +ĉ ST +Ġdecre ment +Ġmagn esium +_ rotate +S it +Ġnieu we +Ġter med +íķ ©ëĭĪëĭ¤ +Ġur g +_t ouch +Ġsw arm +Ġcl ave +th est +ĠL af +H X +ĠH ulk +Ġplaint ext +ĠSof a +get Session +L ed +Ġecosystem s +he i +ĠK ills +Ġhus bands +Ñħ ÑĢан +(d om +_t iles +Nib Name +Ġdon ating +. acc +Ġlifes pan +.b n +_RG CTX +æ ¥ +ans en +Ġmod elling +Layout Params +ĠonChange Text +rs a +- location +.P e +(b us +(s ong +Ġprodu k +ĠSH OULD +ĠC J +Ġs os +ĠHome Controller +.load ed +(D ocument +.s ocial +t iles +Ġl ame += df +.parse Long +Ġpr ac +Ġdet ox +ĠV E +Ġpunt os +Ġdo ctr +Ġan cor +CA PE +Ġc mb +çĦ ¶ +*) " +:// / +Value Type +Ġmort gages +; q +ĠRock ets +s port +UG C +ct s +ãĤ ģ +ie ur +ĠAppe al +(n b +//////////////////////////////////////////////// //////// +IM ATION +ĠC res +ĠMan ip +C ause +at ypes +man ufacturer +# ---------------------------------------------------------------------------- +Ġsp or +es on +Ġpun ched +Ġbook marks +ĠBul k +Complete Listener +ĠTalk ing +ĠEr nest +Ġrub bish +k ills +ĠDE FIN +Ġneighbour ing +ar lo +ĠP CA +ĉm atrix +lo k +Ġat las +ĠG ur +Ġw yn +-n egative +Ġt ul +Ġre lic +ĠV oltage +ĠPre is +ĠJ NICALL +ĠPM ID +ak et +ĉ attr +Ġet iqu +ĠM J +ĠG mail +cl r +_exec ution +éĶ ® +pos itor +. af +N r +Ge orgia +Top ology +Ġperch é +Ġmus lim +Ġepid emi +Ġsab ot +act us +Ġë ĮĢ +ĠIO Error +. est +p refs +ĠKr ish +.Read Key +NAS A +u ção +_D b +umer ator +W ide +(st atement +.end point +.... ..... +Ġ[ * +stream s +m time +P x +at r +Ġt pl +R oman +Ġscen ic +.n z +ĠSe conds +sub menu +Ġìĭ ¤í +_b undle +Ġde ÄŁ +ĠS isters +pre ferences +Ġport a +Ad visor +max Length +ĠG REAT +__ (Ċ +ole st +ĠLabel s +Ġen fer +ĠĠĠĠĠĠ ĊĊ +ĠThe ft +_F ILL +ĠW ise +) application +un ami +> ())Ċ +ADD RESS +B ST +et zt +ĠQ gs +S ense +Exception Handler +ĠCh u +.get OwnProperty +Ġexerc ised +iot ic +ĠRe leases +Ġp interest +ol ie +is oft +Ġsequ encing +Ġpad re +] ));čĊ +(r adius +.m ed +aint ies +.Object Model +Ġem ple +Ġseg uro +St ars +Ġqual itative +lem n +á» ± +> "). +Ġg x +-c ert +ĠAST M +Ġfull name +Ġte lemetry +ĠCamb odia +_ ul +ĠCl are +C USTOM +Q C +ĠUn s +ĠHTTP S +ĠPark inson +ancy box +',' . +T ue +.get Last +Ġab i +Äħ d +A st +ĠEd iting +.Un ity +j mp +Ġm ats +Ġshared Preferences +Capt ain +.page Size +Ġr tl +Ġan meld +Runtime Object +Ġdemand e +(" ; +se ite +-head ed +ĠK ra +ĠF ONT +` \ +Class NotFoundException +. avg +atic al +A j +Ġpermit ting +Pro j +ERR Q +Ġcre ampie +ĠBuy er +-mod ules +ĠSund ays +| `Ċ +Ġday time +Ġ+ ( +Ġgl itch +ĠOper and +Ġtox ins +iny a +D NS +ĠS as +C ake +ĠNation als +.add To +Ġs inking +Ġcompreh ension +Ġsc or +ag ements +Ġt ard +Ġmarch ing +ĠM TV +Ġs ane +Create Info +Ạ¯ +Ġend Index +ĉ layout +ĠåIJ į +S ITE +ĠT HERE +Ġ[ {' +opath ic +Ġtrans mitter +/ body +Ġp und +ĠC losing +Ġset attr +Ġbound ed +At las +sum ing +(t imes +par er +yn om +fe it +Ġf rem +- leg +ĠBr as +> # +Ġì¶ ľëł¥ +ĠIN STANCE +ĠC ouch +_host s +lik elihood +.M arker +ĠM asks +Ġcere al +util ities +Ġelement al +Ġdist orted +in active +c ry +W L +UPPORT ED +.Th rows +/s chema +ser ie +." ', +ĠBened ict +-p icker +ig gs +ĠPir ate +åij¨ æľŁ +ĠTh ema +ĠSouth ampton +Ġarray With +ĠPaul a +Ġpredict or +- Ass +.user id +Ġper i +Ġexagger ated +ur ate +arse ille +ĠCon cent +ĠP ik +Ġ@ _;ĊĊ +Ġform ations +Ġden omin +"/> .Ċ +ended or +Ġpan cre +Ġam t +Ġon Resume +on Delete +ĠB CH +) (" +m ovement +Ġpot assium + čĊčĊ +ĠMah m +} ";ĊĊ +Ġd q +ĠPublish ers +ĠAm pl +ĠDani elle +Ġt ern +èµ · +no ÅĽÄĩ +e in +ĠAsync Storage +un ger +rou w +Ġsc issors +/ assert +.b ucket +/ archive +_M an +Ġint oler +Ġ() => +ĠÐĴ Ñĭ +Ġsa i +.x y +." čĊ +Ġur inary +es ub +IST ICS +ĠÎ º +Ġcompl iments +Ġtypings Japgolly +ih ar +Exp ansion +ĠS erving +_st udents +ĠX BOOLE +( il +Ġì² ĺ +Ġj ó +(t ol +( JS +ĉC G +ĠD RAW +tw ig +Ġo at +_sm ooth +ĠC SL +Ġos ob +Ġens uing +Ġbank er +ĠBack pack +_p ing +Ġwish list += ax +ĉĠĠĠ Ċ +Dis ney +stead y +"> % +Ġproph ets +ĠZ X +Ġminimal ist +.PL AIN +Se attle +. ordinal +ĠPI PE +Ġret orna +Ġjug ador +ĠB ret +ĠâĶ ľ +Ġpl ush +UL ATOR +Sort ing +.grid y +ect omy +_ activ +r ack +Inter active +ĠAntar ctica +Ġv engeance +en so +_k nown +up plier +.Mod ules +ĠConnection State +éļ IJèĹı +@ FindBy +Ġpl acer +\ model +< ()> +.is Successful +-g ood +b z +ĠDr aco +Ass istant +-ex tra +аб лиÑĨ +Ġhyp ocrisy +Ġt st +ĠA gr +$ txt +Ġlog istic +lic ensed +ĠH of +Ġt at +( iv +Ġinto xic +post Id +_st rike +Ġhum iliation +pc odes +" sync +(rec ipe ++ N +rent e +ĉ Client +ycop g +ĠZur ich +ĠPro files +C ountries +Ġp ict +Ġroll out +requ encies +Ġpatch ed +Ġcar tridges +Ġsh ading +J ar +Ġsalv age +ĠTax es +Ġstand by +apor an +E igen +. angular +ĠN ested +äº « +Ġis Visible +ĠDw ight +_BR ANCH +.D elay +Ġk end +Ġfacilit ated +.flat Map +Ġs anta +ĉS end +/m essages +Ġof Type +ĉs wap +# plt +ĠTur ks +N ES +Ġprogress ively +ĠRes idence +ĠT REE +Ġno en +d io +Ġn elle +Ġsog ar +itt i +week ly +Ġambigu ity +_Set tings +W are +.ne o +_D ST +Ġæĸ ¹ +pre p +lob by +@ email +/m ovie +Ġfun kc +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ÂŃ s +Ġguard ians +- pos +Ġconfig uring +ĠC PS +ĠDe us +Ġvidé os +_ empresa +Ġsl apped +< Model +Ġunders cores +U h +.access Token +SET S +ĠS parse +ĠCal d +: path +ĠS ervers += batch +Ġkn itting +Ġx a +Ġsearch Bar +Ġsn ag +Ġinf used +.b am +le ver +Ġtax onomy +Ã İ +Ġatt aching +Ġh ern +_N OP +Click able +(P arse +ĠDynam o +-b uilder +Ġdere g +Ġsc attering +è¿Ľ è¡Į +an zi +ĠShe pard +"> ',Ċ +_X DECREF +ĠBuzz Feed +_M ARGIN +P LOY +.sm all +Ġm imeType +Ġh olog +ĉc amera +li as +Ġsusp ense +ody nam +b au +Ġgrave yard +_n amed +":" ' +Ġ******************************** **************** +Ġgame Over +ĠLENG TH +ĉs creen +Ġdo InBackground +_depend encies +Ġr tc +/ up +_ ROM +H all +Ġdef iciencies +( te +' # +_e quiv +Ġpre order +ĠA xe +ом Ñĥ +.send File +Ġfil t +ĠLim its +ĠCaval iers +.dis count +âĨ IJ +ĠW it +QRST UV +Ġi j +Ġt egen +Ġ: ", +diff iculty +p unkt +ĠEmail s +ch lor +(f un +.U int +ĠSt all +_ verified +u D +File Type +Ġple asures +Ġjud iciary +Ġsh am +ip ur +_PL US +off ers +( foo +_G T +ĉc ore +ENT ION +ĠLib eration +Command Line +_de partment +.A r +_ne ighbor +ĠSub mitted +ĠĊ +Ġdro its +Ġhomosexual s +Ġab duction +ĉw idget +$ headers +ĠD AR +Ġfl a +th reat +Ġlou is +.Get Property +" Just +(f rames +ry o +prof ession +| i +íķ´ ìĦľ +(s v +Ġun recognized +I onic +F ashion +Screen State +ĠIn coming +Not Nil +Ġsync ing +em ie +Ġtherm o +_pro cs +Ġincons istency +rel igious +.m j +Ġperson n +Ġmoment os +or arily +Ġæ Ĭ +_ne urons +Ill ustr +im oto +il ik +ĠW oj +Tr ading +Ġapp are +Ġentre prises +ach at +Ġ ¬ +Ġne igh +BUTTON DOWN +ĠMah er +ag han +-h ash +" f +Ġclient ele +.add Button +ĉ SP +Q i +Ġgr ated +POS ITE +: > +ĠHow ell +ĠCompar ative +ĠIS C +ÂŃ i +O cean +D avis +ĠFil me +W ins +ĠJ IT +oc cer +ĠC orm +ENCH MARK +rch ive +ica ção +Ġm ata +Ġchild birth +ĠOption ally +En s +Ġx http +Ġel ucid +_Osc InitStruct +)) ):Ċ +Ġint uit +ĠDon ate +Ġcorrel ates +> Delete +Ġequ ipe +Ġb oca +Ġinfl atable +er ah +ĠDateTime Kind +Ġcal ves +\ Lib +Ġem lrt +ĠTr ilogy +ĠP anc +ĠD uis +ĠpelÃŃcul a +WAR DS +_DE TECT +-section al +dh cp +For Row +-de struct +ĠPres enter +/s lick +, on +ĠCit adel +logged in +_sub type +Ġsig ue +Ġc uring +ĠFire wall +Ġfluores cence +ĠItal ians +иÑĤ ÑģÑı +.get Style +In Seconds +j ie +-S mith +Ġx link +Ġsub missive +он ÑĤ +arbon ate +ĠF aul +_go als +ĠCommission ers +chart Instance +_POST FIELDS +Ġmed ial +Ġman os +Ġdel t +sv m +.Ap is +ep hy +Ġasym pt +Ġapp Delegate +Ġimpro bable +ck a +sim d +/ Error +. âĢĵ +ĠP TS +de er +Ġs ina +m agnitude +ID ADE +'] }' +Ġmay ores +ĉ comment +/ console +" @ +v olt +.s ell +ĠM acy +Ġmel od +Ġim ágenes +_ch g +Ġin out +ident e +) '),Ċ +d ni +.b lob +Ġtyp ography +Ġe erie +_O ID +pes an +aj an +Ġch opping +Ġbl uff +ad f +_b ases +.Form atter +Ġ\ % +ĠPage Info +Car rier +ĠCal ibration +com o +-b odied +Ġfinanc ier +ĠIN A +. ERR +Ġhood ie +ĠSan ity +gu arded +.opend aylight +ISM ATCH +High lights +ün k +ani em +anger ed +assign ments +Ġregistr ado +ĠU PPER +ampil kan +ash ire +ĠNik ola +ĠC FL +ĠH DC +Ġp oids +ĠIP s +Ġprevent ative +ips oid +if ix +.c amel +.g a +V olumes +- ste +Y ahoo +_s ibling +H ighest +opt group +Ġkvin na +âĢĿ ãĢĤĊĊ +ĠAppl iances +Ġ" >< +') ")Ċ +ht t +ĠIdent ified +Ġpenc ils +Ġmember Id +Ġappend String +.load Data +Ġmock Mvc +Ġj ub +ĠSl ut +ĠTai pei +st att +Pol it +Ġpart ager +Did Change +Incre ases +) }. +ĠB aba +_CL IP +[ unit +Ġк лÑİÑĩ +Ġalc uni +ĠL ola +Ġcl inging +@ PostMapping +(con cat +Ġss id +ĠFa uc +ok it +ĠRecord ed +á lez +($ ('< +.assertIs Not +Ġk ali +V olt +Ġwarm ly +Ġsca res +get ti +füh rt +_d oes +. EMAIL +im ations +Ġspring fox +ĠDec om +arc y +Ġgl itches +ĠM off +ĠV oll +.b etween +Ġcoord en +ĠPart icularly +GB P +Ġsem ble +East ern +_M SB +]) {čĊ +m organ +ĠE VAL +d ere +HO USE +mo ire +ist ique +_l stm +-com mit +yster ious +Ġtw ink +-th umbnails +en ÃŃ +:' ', +Ġblack out +ĠFlo ors +Ġso fas +Ġou i +lesh oot +ĠRa q +- abs +Ġk ra +M ining +sha ft +.set Columns +Cl azz +PRE TTY +.play list +éĸ ¢ +-Sah aran +M ING +ĉ bl +è® ® +j f +DO CKER +hope fully +( ignore +ĠUsers Controller +ĠMitar beiter +ĠL ES +Ham ilton +-m etadata +ĠK K +ikt ig +Ġwoll te +egr ator +] bool +, current +Ġvalue Type +Ġexcav ation +ol and +Ġv erv +/file path +Auth Provider +Ġpro crast +ĉ ULONG +_MEM BERS +Ġup lift +ĠAut onomous +Ġart works +ĠOut reach +Ġp ore +Home page +Dialog Title +ĠGener ating +PAR SE +Ġsem anas +Ġhuman o +JSGlobal Scope +Ġvol te +Ġb ella +(is instance +Ġpl c +\C atalog +Ġeste emed +éĽ · +(s uffix +Ġswe eps +ĉ ORDER +Ġdo ivent +ĠSw arm +ĠComp iled +get Page +AD R +.R ichTextBox +ĠN aming +ag ged +ĠG ANG +r asing +ode led +Ġg ala +ĠJS Name +dd f +Ġill ust +ĠLans ing +[ port +-de ath +Ġdin heiro +ĠE ighth +Ġb ian +st Ã¥ +Ġvers ión +ĠLinear Gradient +ĠHard ing +. *) +ec zy +$ header +Ġv Ã¥r +Un checked +Ġko je +ĠPal adin +() )), +G iving +() })Ċ +Ġd ips +F riendly +Ġport rays +Ġhel ium +Ġinsurg ency +_ex piry +ĠstringByAppending String +Ġa antal +s lope +m ast +.get Integer +Ġ################ ######## +_PIPE LINE +Ġdens ely +Ġmut ating +m idi +ĠSe it +ay ne +NOW LED +ĠDes mond +ĠF Name +ĠN airobi +\ Context +Ġcalc ular +-d en +Ġc ott +] ):čĊ +ĠRecommend ation +ĠRole x +Ġvalidation Result +.p at +Ġn Ãły +ĠRest Client +ĠG PI +ĠAshe ville +ĠO SP +ĠPER MISSION +ÐĶ Ð°ÑĤа +/ notification +K night +_W ord +ĠB ender +rank ing +Ġpart ida +_res ervation +Ì Ģ +Ġm Name +Ġget ch +Ġb orr +Ġdilig ent +Disc uss +æŃ£ åľ¨ +ape ake +ion ed +-N azi +.c um +ĠK ron +=$ ('# +/s ingle +Ġerot isch +ĠV ib +Ġrat ified +Ġconcert ed +ĠREG ARD +Ġdo br +.Driver Manager +' r +Port able +ĉs uite +Ġrel aciones +ĠD op +emplo i +DO B +Ġcr umbs +Ġx ls +_App lication +(': ', +Ġ---------------------------------------------------------------- --------Ċ +m se +Ġber k +ĠReturn Value +ĠBel ly +Ġcam ar +ĠPe ek +els ing +Ġnot ifies +ĠTr istan +ĠG AR +em me +ĠElev ated +_C SV +(ch alk +Ġtw enties +ĠSearch Result += search +ĠMix ing +ý t +Ġrecru iter +ĠIDE OGRAPH +ĠA go +( Operation +$ values +Ġworld ly +ĠRosen berg +ĠConfigure Services +>* Ċ +Ġsn ork +_op acity +ĠinitWith NibName +i ado +A AC +Ġ] ). +; z +_par agraph +Ġnos es +stand s +if r +_m E +I raq +.P redicate +ena ire +]] ];Ċ +Ġun idad +Ġretire es +_h ello +Ġmode le +ĠUIT ableViewController +f write +_num ero +_vis ited +Ġrece be +( Notification +Fant astic +_sub menu +ĠP EM +ĠCup ertino +approx imately +class ed +.Read String +Ġdomic ile +_P W +Ġball park +ĠK ale +con tra +_f avorite +/ of +Qu ite +ĠOT A +Ġacceler ometer +did n +| ^ +ĠRohing ya +ivic rm +ann abin +обÑĭ ÑĤи +or ado +') + +Ha unted +, ID +( UIAlertAction +ur v +_b el +ĠMex icans +/ terms +ĠPaint er +Input Label +ĠV inci +ĠRos ie +\ uc +< Menu +Ġcool ant +(current User +_d ual +) "},Ċ +& p +Ġconver ged +Ġrestr ain +ĠYugosl avia += target +Ġimp uls +ds a +Search Tree +Ġh box +ĠImp ress +§ Ãĥ +get FullYear +(d a +ĠY YS +.al ignment +.Get Text +.token ize +ĠOlymp us +Ġmur ky +ore station +Ġdiss atisfaction +ĉT Array +_ kses +.Add Singleton +ĠStart Time +Ġfan atic +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +Ġentity Type +. override +Ġ ------------- +ĠDat agram +f out +(with Id +Ġ# __ +Ł èĥ½ +ek yll +.f riends +ame leon +Ġz ach +.simple Button +ret orno +Ġkon k +/s mall +ĠQuick ly +un read +Don ate +Detail View +Ġdu a +Ġpenetr ated +OM UX +Ġn ir +_p data +"], [" +Ġlow es +Ġdop ing +Ġas ymmetric +Ġneed less +our cem +Ġup ro +ĠGu zzle +af b +Ġsext reffen +-c ollar +Ġcol ossal +Mon key +n ish +Ġhandle Message +Incre ased +* dx +ĠChatt anooga +f org +ĠOr den +Ġsh ri +ĠV and +Ġ" @" +Image Sharp +ĠWild cats +pon ible +.sc enes +Ġpaint ers +ĠPf izer +ĠZ ah +To Local +ĠFl am +Ġé taient +)) ^ +ĠSand box +ĠTR ADE +Ġchrom ium +Ġac claim +Ġpac man +´ t +) reader +M ari +.Dispatch er +.A DMIN +ĠRem ed +Sw eden +Ġoverl ays +. er +Ġp ang +Ġclean ly +aven port +Toy ota +patch es +Ġv tx +ĠE is +cl ado +ĠR itch +RO LS +Ġh ade +Ġconspic uous +Ġdo cks +(j q +ĠPrem iership +ĠBe z +ĠâĦ ĸ +ĠÑĥ Ñģл +_tot als +Ġprov a +ĠC ue +Ġsa úde +ĠGame Controller +IM IZE +, port +ãĢĤ ( +.C decl +Instant iationException +Ġcoll age +ĠIO C +Ġb ais +Ġon Finish +-st ars +set Size +Ġmog ul +Ġdis illusion +Ġche vy +(S chedulers +( IR +_loc s +Ġcann ons +Ġcancell ing +/b us +Ġbuf io +ĠY ours +ĠPik achu +Ġter me +r Ã¥ +f ahren +Ġowner Id +Ġoblig atory +Ġcul p +Ġacid ity +-m ult +ĠBam boo +Ġ' "> +_g s +Ġcomp il +n ard +-ex c +Ġrh yme +Ġbut to +s ays +ant asy +ë ¸ +Ġcitt Ãł +Ġche g +Time String +Ġpos itivity +ĠD abei +Ġw ang +Ġes cre +" c +ĉv ideo +ĠRank ed +.str ings +>> >( +Ġин ÑĤеÑĢ +Ġrest a +[: ,: +Ġrend re +Ġdes er +J os +Ġdis ruptions +Ġоп еÑĢ +s ampling +sup press +Ġcontainer View +ĠSeam less +Ġair y +Ġon load +.Window Manager +ĠPL A +br aco +.set PositiveButton +Ġp du +Ġg si +ĠC li +_gr adients +Ñı д +ĠWh isper +c stdint +Ġl äng +Ġform ulations +én om +ourn emouth +[$ _ +Ġordin arily +.set Username +Ġfacult ies +MIT TED +/ values +Ġwe ir +ĠA pt +M Z +ĉc f +uck en +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉĉĉĉĉ +def ense +[i Var +ĠBusiness Exception +Select ors +(co ordinates +ĠRes ets +ĠDr inks +ole ans +(st ypy +_IO C +.x xx +ĠSl ater +ĠBel ize +Ġ/ ************************************************************************ +add in +_ep isodes +Ġis chem +legal ArgumentException +D anny +Ġp ared +.code haus +ĠAss y +ĉ Rect +â ŀ +.list a +Ġв аÑĪ +Ġv ets +HW ND +ison er +Ġx o +Ġor ally +ĠSt mt +.r nn +ĠD PI +ĠStr ikes +.setViewport View +Ġèĩª åĬ¨çĶŁæĪIJ +Y ELLOW +GL enum +part ners +ĠImp licit +Ġtak o +âĢĻ elle +Ġerm ög +total Count +G il +ĉ work +Ġpr atic +in ati +ab ies +ĠSk inner +Ġspir ited +Ġpancre atic +Ġh df +' em +Ġpsych osis +olic it +Ġ" {" +_at ual +Ġé lect +TE AM +Ġd ak +ĠSW AT +.Fragment Manager +Ġprovision ing +l ifetime +_EXTENSION S +ĠC ASCADE +Ġ! [ +(K P +Ġv em +ĠInterr acial +'] },Ċ +sp acer +_k v +W arehouse +R DD +_f sm +.Stretch Image +, Yes +ĠRefuge e +ĠBr inging +Ġv álido +.inter section +Ġsp ooky +_port al +Ġmo th +ĠZ odiac +ĠSOC IAL +M imeType +'] }} +_Bl ue +Ġbot anical +Ġfr ags +Ġfamil ial +- du +Ġse izing +(block s +.r d +.check NotNull +Ġmis er +Ġmax x +ĠK nee +View Item +Inner HTML +D anger +(( __ +Ġprz ypad +create Url +** , +ĠDecor ating +ATEG Y +?> / +.Design er +hex digest +ĠEvery where +all eries +.TEXT URE +.Block s +z ell +Ġpre ço +S uddenly +input Email +(s ync +.b d +gold en +> '); +ĠDick inson +>> (Ċ +ĠQUE UE +Ġget Column +ĠS AND +.p iece +lic er +Fl utter +Ġget Version +Ġresource Id +og l +ÅĤ aw +.Br anch +ĉ web +Ġfr amerate +PP P +Ġfr ay +C NT +Ġinformat ie +'] čĊčĊ +ne as +Header Code +Ġæ ¸ +Ġtr g +raw types +H onda +Ġmark eter +Ġrequest Data +ĠP g +ĉ not +Ġpage Info +Ġakt uellen +ãģķ ãĤĵ +ĠA MS +push ViewController +ĉ AL +Ġv ests +produ ce +-m ême +ĠRah man +F unny +E Z +_ Valid +Ġsquad ron +Ġl ash +Ġ irm +ias co +ĠPar an +Ġpet ites +ĠDec ay +Ġun initialized +priv ileged +Ġm bedtls +å¤ĩ 注 +Ġ^ . +Ġec static +D etroit +Ġpart en +Ġsou venir +.get Login +моÑĤ ÑĢ +en ção +ĠmÃŃn imo +ĠAccess ed +ri ó +M ic +ĠV ocal +.Set String +Ġmens ajes +åĢ į +Ġattr avers +ĠA ph +Ġ' );čĊ +ünd e +Ġench anted +ĠRoot State +ĠCLOSE D +ĉĉĉĉĉĉĉĉ čĊ +Ġcal iente +or ris +Ġphysic ists +h wnd +_v i +Ġráp ido +Ġcapital ized +ed By +Ġmach ining +Ġhub by +ĠSt acy +.B us +dr ink +H ur +Ġprop ia +Unit Test +Ġmiscon ception +__ ));Ċ +/d c +ĠMay weather +_m C +.create From +ĠQ Painter +rops ych +inn itus +ay as +Ġg eg +(d w +Ġus ado +Ġtrick le +Ġann ihil +ĠP asta +Ġ++ Ċ +(Expected Conditions +.post Value +ic ap +ĠDon etsk +_s oup +-p ublish +ĠP b +ment ions +AC CEPT +.P ull +,âĢĻ âĢĻ +Ġret arded +_AT OM +ĠTermin ator +-c ourt +ĠCLLocation Coordinate +Ġrever ence +ĠS SC +ut ely +ĠW ON +ĠG SL +fre i +.get Longitude +Ġopen FileDialog +.B utter +- important +_M ANY +ĠG ong +âĢľ How +Ġg orge += msg +ĠEz ek +create Command +: checked +Ġinf ographic +.W EST +Dir s +Ġguard a +Ġbeet le +< small +- android +Ġcred itor +ĠM éd +Ġfinal ist +Ġab l +ne v +_inter action +ĠMonter ey +j ah +Ġcand ies +ĠQu incy +èª Ń +Ġbatch Size +ak it +Ġo be +(p ara +Ġexperiment ed +Ġcouncill ors +Ġcl ashed +s qu +-st rokes +ĠG K +ĠEx pires +Ġprosec utions +ĠCreat ures +Ġy ö +x lim +_IM P +Entry Point +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +.Default CellStyle +Ġbre ve +ĠBrit ann +Ġsweat y +Ġle th +Ġflash back +per manent +ĠJ DK +_D etails +E uro +p pt +Ġrich TextBox +/ board +Ġtr ance +.c ycle +'); ");Ċ +Ġtox in +_de init +Ġover arching +Ġconfig parser +ĠKaw asaki +.th umb +Ġplay a +ĠJose f ++ _ +Ġzero es +Ġa up +ĠH ari +comm itted +N it +.file Path +ĠDis abilities +man ufact +-al igned +.RE SET +Ġrust y +E y +Ġou sted +cos a +Struct ured +.get D +Ġs ábado +> Loading +_m A +.get Random +bl ings +Ġchees es +tt i +. âĢ¢ +ĠBurg ess +ender it +. ',čĊ +(" "+ +ac b +% p +index ed +_pred icate +nes ia +Ġb ied +ĠC IT +( Pos +_r adi +ä»· æł¼ +B iz +ĠAdoles cent +Ġvi ên +c ycl +_C ancel +Ġcon clusive +Ġappell ate +inform atics +S J +Ġelect ive +role Id +Fetch er +ĉ Command +(" (% +Ġf art +IL A +get Block +A USE +Ġд ан +ĠAr te +Ġnot ifying +Ġge le +.s ame +ĠReg el +ĠBa ÅŁ +.c reation +ĠV N +_comm unity +Ġuns ustainable +SE X +Ġgrid Size +res cia +avers able +(', ')[ +ĠPh elps +á»ķ i +ANCE LED +- IS +.run ners +ĠSt okes +.P rodu +Ġwh ipping +_ac quire +Ġinvestig ación +f ried +.copy With +ĠHard cover +- Se +áŀ¶ áŀ +inv itation +les ai +ĠD orm +ĠÑģпиÑģ ка +Ġconcaten ated +oph il +Ġthink er +/font awesome +ĠLe opard +Ġ"/ ");Ċ +Ġresidual s +ĠMic rowave +Ġconform e +th rop +Ġdis emb +ĠO MG +ĠDisc ipline +ĠAc robat +/re pository +df a +_M ED +buf io +Ġméth ode +_H OLD +ias i +_ legacy +) ččĊ +æ£ Ģ +Get ProcAddress +Ġy ay +ot ence +order id +-t w +Ġdear ly +In coming +/ il +Ġneu rop +uc z +); čččĊ +ĠInnov ative +Ġprof und +ig mat +Selection Mode +re levant +.G O +Ġbru ises +Ġs ach +ode f +Ġre imb +/d esktop +-s pot +und ance +Ent ropy +\ core +Ġsug er +ĠM vc +ĠGN OME +_ind x +ĠYY STYPE +ĠMat lab +ĠC IF +Ġ* )) +Ġproduct List +ĠAl right +ac emark +ÑĤи в +mod ification +int ernational +Ġhom ers +Ġdict s +ĠQ Font +.SQL ite +Ġtransplant ation +ĠMessageBox Button +ĠEl ves +'] ])Ċ +(Q Icon +Ġcin emas +CO ORD +- China +Ġkh ẩu +æĪij çļĦ +Ġskull s +Ġpain staking +f ce +.XR Label +Ġspec ifier +Ġpref erring +/ activity +( Photo +á lt +.l ot +' '. +ann once +.google code +-p df +ĠP oke +_A CL +Ġend owed +dis cover +.om g +Ġwood land +.M agic +Ġvol ont +Not Allowed +Ġch ave +BM W +',' =', +ĠS IX +æĪij 们 +Ġkos her +Ġaspir ation +int l +_ref ptr +'+ Ċ +ment or +.cl ub +Window State +.A RR +Ġz za +Ġmessage Type +.e qu +Th or +Ġin just +Ġg ums +Ġborder Side +//// / +ĠTrans mit +Ġbuf size +Ġh ak +Ġell as +R ANDOM +ĉm c +Ġpe a +ek o +document o +Ġhyster ia +Ġaren as +Ġgun men +Ġm ike +Ġimp unity +atis ation +_Z ero +_COMP ANY +ĠG ors +Ġuse Class +( redis +ĠRUN NING +ĠB air +vel te +Ġ',' . +аÑĤÑĮ ÑģÑı +ö st +encode URIComponent +_re strict +Ġdec als +ĠPed ido +Ġalter cation +Dis plays +ĠApp licants +C US +Text area +ĠAng ola +.f uture +ĠUS HORT +Ġsuppress ing +Ġset zen +AP olynomial +Ġto ch +Ġhall mark +Ġ$ $$ +ĠCHAR SET +.r pm +ĠD ich +---------------- ---- +_p arm +è¿ ĺ +acc iones +h ait +WAR DED +_r outing +ĠN OM +Ġen clave +ĠLot to +ĉf r +complex Content +ĠBall ard +k ube +/w in +.getColumn Model +_RE PLACE +Header Value +Ġest udiantes +Ġap is +Ġb pm +ĠType Name +And Get +rit a +Pl ans +> Note +Ġfet isch +Ġton ed +_g oto +ons ense +Ġm olds +Ġinfiltr ation +ĠGuerr ero +ub bo +ck i +($ (". +_ activities +(ch anges +Ġof App +ĠKe pler +ĠD emp +ĠCont inent +.T icks +ĠUn signed +ĠJah res +Ġfresh men +ĠArch ived +ĠкоÑĤоÑĢ Ñĭй +Ġ' :: +T utorial +C c +Ġtable LayoutPanel +from Json +.level s +_trans ient +Ġendors ing +ĠD IC +la uf +Ġsh red +_E MIT +ific antly +AL A +/ proto +Ġnarrow ing +U tc +Fact ors +Ġsent ient +æŀ IJ +lix ir +ĠC ROSS +met eor +Ġgro in +Ġm db +ĠRot terdam +Ġcom ida +ĠOp Code +ĠDefault Value +Permissions Result +Ġheter ogeneous +Ġm oot +Ġde ceived +-in dependent +ĠObject OutputStream +Ġover power +.d up +Ġl db +Ġdomest ically +Ġbest ellen +Ġlo v +ĠContract ors +Tri angles +Ġfod der +Ġfilm es +ä¼ ģ +Ġrev olver +Startup Script +/ validation +ĠResource Type +i ÅŁ +ĠL az +f ef +Ġlst m +{ * +. attachment +.h its +ew ith +DO G +Al abama +Ġmedium s +.m Context +-c ols +åı ĭ +.not ice +Ġat tn +ĠP acking +ĠL n +_COM PLEX +/ Users +.sav etxt +ĠR ounds +?,?, ?,?, +Ġing l +ĠR OC +_f emale +ĠSt ard +]] ; +Ġwrest lers +Ġtorrent s +Ġsin h + ĊĊ +ë³ µ +s ense +how ever +.Ph ysics +Inf rastructure +ĠSac r +F el +ĠD ISTRIBUT +é ments +ĠValid ates +################################################ ############ +Ġ| / +Ġes l +Ġré seau +ĠB ip +BY TES +_W ATER +Turn ing +EL S +Ġj uxtap +Ġlesb ische +ý ch +( Unknown +Ne o +@ JsonProperty +Ġal umnos +ĠRaq qa +ime i +.get Bounds +.Mouse EventHandler +#### ### +Generic Type +/c ms +Ġturn o +Ġм ин +Ġfolk lore +ĠE vo +Ġconduct ivity +Ġle ben +Ġgear box +-v s +ĠÏ Ĩ +Ġdrink ers +Ġcon exao +ĠTe eth +Ġget Arguments +ĠR AT +ent ious +E duc ++ W +ĠInstitution al +ĠB ord +is Equal +(p wd +Ġign ited +ĠR ousse +Ġimpact ful +ĠM alk +Ġg eral +ĠP ivot +Ġa zt +Ġcsv file +ĠR ope +ĠSOL UTION +ĠArbit rary +Ġlet to +.Mouse Adapter +Ġ} }} +ĠSail or +der a +Put ting +Ġconcentr ates +Ġauth Domain +âĢĿ çļĦ +-f inals +, strlen +Mu on +ĠOrd inary +fire fox +ĠLa TeX +ĠH und +engine ering +/ blue +ed TextBox +(" "); +ĠC DDL +ke pt +ĠGet String +K ir +() =' +ĠO CD +ant ium +$ menu +ĠAppalach ian +Secret ary +ë¥ ĺ +ี ย +Sem antic +Ġ* [ +est one +ung kin +Max Y +-t one +"} ;čĊ +_P art +< Member +tr am +Ġtrans istor +Ġ---------------------------------------------------------------- ----------Ċ +ĠDes de +Ġright ful +ĠCorn el +æ ij +.H OUR +Ġsidel ined +ref errer +m aze +Ġhol ster +Ġcripp led +ĠDate Formatter +oph age +_m D +Ġdes elect +ra ud +ĠPK K +row Data +Ġlock smith +.res ponses +(product Id +_ST MT +Key Type +.Th en +z ee +Ġcr t +ĠGrand ma +@ Resource +Ġbit wise +-c mpr +ãĢĤ www +zeit ig +& display +Cart Item +- No +Ġnum éro +Ġm aur +Ġinst ancia +ĉd t +_n pc +Ġskate board +âĢľ All +ĠCrow d +Ġä n +Ġb raz +ca e +yn et +/p m +/s creen +OPT ARG +ĠV Box +Ġle opard +_g reater +c pt +< dd +Ġmechan ically +osp els +) f +.l wjgl +.get Port +ĠP REF +.Add Transient +pp ard +Ġí ļĮ +Ether net +Ġsal ine +(level s +Ġservice Provider +.A ngle +alt itude +illa ume +Ġs cape +_CAL C +_ quest +ĠDiss ertation +ĠE DM +-C ds +Ġhon orary +st ops +Ġsub dir +ĠV H +ĠChe at +Ġright fully +Q E +.Write Byte +fig ures +enn ie +( DBG +Ġvoks ne +Ġexp ended +UN ICATION +il inx +ĠRec ap +_ verts +Ġtra umat +Ġget Player +Ġverb ess +Ġcultiv ating +Ġiniti ator +Th ông +find First +_per ms +Ġbu c +Ġ""" čĊčĊ +T YPES +object Manager +(Configuration Manager +Ġtim id +Ġsnap chat +Ġcon seg +ĉd istance +_right s +_D es +ĠF lesh +- ver +Ġa fl +fra uen +Ġblas ph +ĠQual ität +ma f +Monitor ing +.D iff +Ġshore line +Ġresponse Body +mem set +< decimal +Smarty HeaderCode +Ġin sets +ĠBinary Tree +amed a +Ġn ihil +ĠN ay +ym ology +ĠW G +Ġt api +ĠInst alled +m aintenance +)} "Ċ +ĠX O +-per iod +s ar +Ġning una +ORM AT +.set PrototypeOf +ĠK b +ĠHen rik +ét ique +ĠLah ore +ĉ Address +Ġmel ts +N y +_adv ance +Ġveloc idad +Ġalum no +Ġsanit izer +Ġph ishing +ĠCom et +Ġch iar +ĉs pec +trim med +(state arr +on nen +Re venue +L ens +Ġcha ired +ĠAss umes +Tr ash +_un set +\ Bridge +Point Size +ĠPol ic +Ġsex uales +ĉd fs +ĠWide String +Ġaccru ed +Y W +_S CHEDULE +Ġk ite +Ġparach ute +[ table +Ġactive ClassName +.Qu ad +Israel i +ĠÅ ĵ +Ġho og +Ġch á»ī +ew ear +Ġtire lessly +set Error +.get Amount +.set Items +ĠM anson +ĠBay esian +_F lag +AC HER +/ original +Ġimm ac +ĠLos ing +' >ĊĊ +L ic +ĠMir age +ĠAssembly FileVersion +Te V +ĠValue EventListener +-s olving +Th o +rou lette +_W P +Ġunint errupted +Ġfield Type +.T yped +Ġam our +Ġmock ery +(v ol +ĠSub committee +ĠR uf +ero x +:UIButtonType Custom +ĠBl ur +Ġwy kon +nc es +ASH BOARD +!! ");Ċ +Ġmurder ers +.d aily +ĠDI AG +j ing +Ġdol phin +Ġl òng +Ġb ö +ĠV ocabulary +.St Object +') "> +Ġz un +Ġscrim mage +tr éal +ĠL ig +[ vi +C ole +Ġfrost ing +.Pl ayers +- translate +Fe els +=\" / +.Butter Knife +Ġ?> ;Ċ +Ġav i +inn ie +.F ailure +Ġsp indle +Configuration Exception +_h op +Ġpos ição +ĠA wait +UIImage PickerController +ĉ day +Ġgen om +C ab +ĠÑĢ ÐµÐ·ÑĥлÑĮÑĤаÑĤ +OR IGINAL +Ġejac ulation +(t cp +SE COND +Ġton ic +ĠList Box +Ġ ĉĉĊ +() >Ċ +Ġqu atre +ượ ng +with Errors +.M aybe +, â̦ +token Id +_UN DEF +Ġfresh ness +ĠAmend ments +.map box +.C V +(b log +_get time +. quest +s parse +Ġres ale +Ġenthusi astically +ĠProstit utas +W a +C argo +.Parcel able +SENS OR +ĠRy u +La ughs +_N ative +/ pg +yst s +Ġphot oc +ç® Ģ +ado pt +.spec ies +conc iliation +Adjust ed +.Firebase Auth +ut tle +ord ination +Ġm unch +ĠSt ake +.p ing +ank er +(QString Literal +Ġsub script +ĠĠ ĉĊ +ĠM CC +_C md +se xy +i ou +ĠM ANY +Ġn anny +TR AIN +Ġflour ishing +ĠW atches +ĠQ Map +ĠF erm +Ġwas m +ĠA bed +_ UD +ĠGlass es ++ v +Att end +.Ch ain +Ġdec ency +ĠSupplement ary +h unter +-t xt +Ġ" }";Ċ +.set WindowTitle +(" +Ġmasc ara +( Profile +åĬŁ èĥ½ +imit é +Ġwild fires +- ROM +.is On +(group Id +Re pair +accum ulate +Ġ< ", +Ġhand written +Ġach eter +ĠM GM +ĠIr ma +->{ _ +ge e +cr iminal +Ġèĭ¥ è¦ģ +Ġmoment arily +") != +_l it +Ġexpires In +." ). +éķ¿ åº¦ +Ġfr ække +vl c +Ġor bs +), $ +Ġvent ured +/ >\ +char m +N uitka +eld ig +aton in +W itness +-l at +Ġset Hidden +Ġrelic s +Ġcons ulate +. IGNORE +" After +Ġset Address +Ġbeste ht +Ġ'' )ĊĊ +.x axis +Ġser ão +Ġmis led +_UN IFORM +ĠV IA +inc r +Ġzen ith +Ġvis cosity +Ġthin ly +.get SharedPreferences +.Error Code +"), " +ĠMillion en +Ġ/> )Ċ +Scroll Indicator +-se eking +ĠPOLIT ICO +as ca +_r l +N avig +(full file +Ġsol itude +Ġju ven +Ġhaul ing +ĠMac ros +ĠG ry +Ġexerc itation +ĠATT ACK +Tick Count +Ġr ites +Ġdo e +Particle System +Ġsl u +Window Text +ĠClass Name +Ġsl ander +ĉ Port +j ong +? a +.D ial +âĢĶ at +$obj PHPExcel +Ġso ar +EN N +appe ared +Ġquot id +em achine +Ġn ip +Ġmicro time +ĠAl ma +; ! +---------------------------------------------------------------- -------------------------------- +ĠPass age +Ġdump sters +ĠEx clude +Ġsuggest ive +ĠCircularProgress Indicator +_cl r +Array Type +ILL A +Elapsed Time +Dr iven +Ġresource Name +ĠG arrison +ser ir +-a head +Ġp innacle +ĠEs presso +S parse +Ġass ays +ĠGirl friend +im id +]=' \ +ONGL ONG +Ġportray ing +L ane +Ġb úsqueda +Ġrein forcements +ĠSpread sheet +ĠArray Collection +, arr +light box +ic ana +< " +build ers +K id +ĠMat SnackBar +EX PR +od cast +ĠFound ations +Ġind s +=' ${ +F izz +-function al +(work space +Ġstem med +_p atches +ĠJar vis +READ ING +Ġdisrespect ful +ĠQ Dom +Ġ$ {Ċ +est atus +Re ached +! .ĊĊ +IL T +ĠN DEBUG +ĠCour age +birth date +ĠT ing +Ġutil izado +án chez +Out door +Ġhand guns +Ref Count +É Ļ +rom o +Ġt ts +.S he +ĠP ane +ãĢij, ãĢIJ +ĠIO CTL +/ black +ins cription +Ġbi opsy +ĠTime Interval +.Test Check +ĠGUI Style +ĠCap ability +ĠBeit rag +don nees +T reatment +.back up +Ġsign ings +ĠB oca +dr m +.M AIN +Ġgo ede +ĠMark up +G REE +ĠBase Service +.C reator +Ġj ails +ĠK ahn +Ip Address +ACH I +Ġinhib ited +Ġ@ $_ +ĠAss ass +Ġenvi ado +Hero es +ÐŁ еÑĢ +ĠM aven +.l s +Ġ ive +| RF +Ġresize Mode +Ġrum pe +_attach ments +T U +Ġtact ile +Attempt ing +Ġro bin +y aw +Ġmerc enaries +ĠHab itat +end date +Ġo xy +ĉR andom +oh on +Is Null +ĠValidation Result +ãĥ ļ +um bed +pp v +Ġar p +ich ick +_r nn +ĠT FT +Tex Image +" On +ĠSam pler +top l +Ġj ane +y ling +ĠUN ICODE +Tab Index +< {Ċ +s uspend +uv ian +, application +ол иÑĩеÑģÑĤво +y at +ez ier +ĠCH UNK +ĠAd ler +/ Add +ĠKey Value +Ġspos ób +Sam pling +ch ers +_AM D +R u +.Must Compile +N ation +Ass oc +Man aging +ĠEng l +_G B +Ġsucc inct +Ġdis liked +ĠI ke +Bullet in +_ARCH IVE +Prop osal +Ġjog ging +.C REATED +Ġch ol +è£ ħ +Į ¨ +-p ush +Ġreserv a +core v +è tre +TH R +Ġincompet ence +Ġchar isma +æĦ Ł +Ġ" == +BT N +ĠLoc ator +iv et +('. ')Ċ +Ġfor IndexPath +ô me +Ġcapac it +w aters +ĠWR ONG +ho a +ĠM IPS +Ġem iss +ĠJacqu eline +(c mp +Ġe ens +Le o +.tim ing +CLUS ION +Ġ(" - +åĵ Ī +.k ode +ĠUnd ert +Ġbew ild +ĠEss en +.h d +Ġren egot +Ġm ower +Ġl sp +Ġpen chant +Ġman oe +Ġag li +Ġrec al +ĠOPER ATION +(^ )( +ĠÎ ½ +ĠSc oped +Ġ@ "Ċ += label +[ loc +Int l +ĠN z +table t +.Column Name +Ġscreen Size +DB us +co oked +- registration +âĢľ One +-n on +ĠwiÄĻ c +Ġcost a +.add Tab +. conditions +ĠH ess +MEM ORY +ĠAval anche +() }}Ċ +Ġtri plet +Ġl abyrinth +ĠNode List +ĠNY T +Ġy eni +d ff +.Html Controls +AV IS +/ Math +Ġmem cmp +Ø§Ø ¡ +оÑģ ÑĮ +c rap +(p ages +Ġl xml +ĠQ DateTime +_t cb +Ġopen id +Ġsyn aptic +ĠMD MA +(s lug +igm atic +en or +Ġcr amped +G OP +Ń IJ +.is File +ĠD ifferential +Ġ=" ";Ċ +ĉĉĉ ĠĠĠĠĉ +ĠC ooke +ĉU FUNCTION +Ġpersever ance +Relative Layout +IMPORT ANT +Ġex on +Ġо н +ib ase +(C ONT +n ovation +ä½ ķ +[ sub +Admin Controller +HTTP Header +cre ar +ĠN IR +ĠDrop DownList +Ġval ide +Ġde hydration +. '] +(W IN +Ġ... \ +Ġphotos hop +ĉ Init +_c ou +Ġtime Zone +dar win +rom atic +Navigation ItemSelectedListener +br ates +] --;Ċ +Ġtraged ies +ĠPed iatrics +SM ART +-A PI +ĠMessage Lookup +ĉ vo +Ġprejud ices +Ġm A +U ps +ĠMISS ING +ĉ ad +C ream +ĠT b +ĠMon a +_ ghost +ĉt ypes +Em b +ĠDocument ary +');ĊĊ ĊĊ +Ġl up +_ Reference +ĠB ATCH +Ġintertw ined +< Cell +ĠCab r +n ation +Ġis Connected +.remove Listener +Ġcon g +_t i +ĠSil icone +Ġê²° ê³¼ +ĠW AN +ĠG ibraltar +/ response +ĉp erson +ch ants +V IP +em ergency +Pixel Format +- Am +Ġsouth western +_pl l +if ers +_ON CE +ĠF ayette +.nc bi +_P anel +.Q ual +Ġpol ys +Ġcreate StackNavigator +� t +Ġlay offs +ĠBl anco +Fe at +ĠV imeo +_ch i +_l ifetime +POINT S +, private +Ġunb earable +print ing +Ġc gi +.B ACK +Ġintern s +ĠNew ly +inf eld +( IB +ĠK ata +ĠDef endants +Th r +é¢ Ħ +_V F +FFFF FFFF +Ġdavid jl +Ġbitter ly +S uggestions +.set Cancelable +FIN AL +ason s +_rw lock +_WRAP PER +Ġhapp iest +(row Index +ós ito +TOT YPE +Autom ation +Log File +Ġcons olation +ãĥ Ģ +Ġt êm +Ġpr er +rg yz +ĠG eg +ĉd to +.default Value +ĠK ami +ĠA SE +optim ized +Ġíı ¬ +Ġorigin ates +err Msg +Ġespa ço +(S YS +ĠMc B +d ance +_det ected +Ġfr ü +ĉĉ ĠĠĠĠĉĉ +< Date +(com b +ĠDec ide +\ Field +ĠProp osed +R ib +Ġdis likes +ĠW ien +ĉ Document +Ġtr af +Ġst oria +ĠT ells +') == +C ri +( VALUE +ĠBurn ett +, void +Ġdan h +Ġc cp +Block chain +:"- "`Ċ +IC lient +IS ODE +Iss uer +) }čĊ +, but +ĠU ph +( Sub +Ġtélé phone +ĠonData Change +Ġmarsh aller +-an alytics +, content +Ġdeb acle +_Value Changed +Ġfa una +Ġ# => +Ġf oyer +'util isation +ĠMü ller +ĠFet ish +Ġdefault Manager +Ġback track +B ah +Exp licit +_A SCII +Ġm Activity +(M sg +Ġê² Į +ĠTER MS +ĠAng ie +HS V +ĠMos que +.N ames +íĬ ¼ +rest e +_p arms +Ġgap ing +Ġcro pping +Data Frame +Ġrespons iveness +_ undo +_tr an +. terminate +Ġitalian e +Ġwalk through +Ġattract iveness +д е +_ST S +_ learn +Ġchocol ates +ier archical +-th inking +Ġ ))) +ish ments +.Log f +ĠTM Z +ĠCan ary +fo il +ĠVacc ine +.v x +ĠSur round +Inter mediate +Ġi ov +v ais +'; ";Ċ +ï½ŀ ĊĊ +éĢģ æĸĻ +â̦ it +Se ats +Cl ar +W ars +ĠHutch inson +ĠHas an +! ')ĊĊ +ĠRich ie +che iden +($ (' +Y ork +Ġl ids +Ġal phanumeric +ĠG lock +.sh apes +Ġspark ing +_ epsilon +uplic ated +.dir ty +]) == +ĠìľĦ ì¹ĺ +Ġsc n +Ġ/ **************************************************************** +_PRE VIEW +_H C +ield ing +f gets +ĠAdd ison +Ġproduct Service +- figure +(ret val +z ano +Ġaut ob +ĉs d +_n umer +ĠSet LastError +ĠF ior +ific ance +Unt itled +Ġin field +Ġ{} ));Ċ +Ġsp ac +Ġro okies +(des cribing +ng en +ி à® +.r df +.M utex +Ġkne eling +ĠQ E +set Max +Read Stream +Ġvent as +s ut +cm peq +.WriteAll Text +ĠEx perienced +$ __ +Ġka um +ĠL IS +Ġdocument os +_HE ALTH +icont ains +Ġart isans +OWN ER +Ġblink ed +get Display +Ġto en +Ġrow Num +Ġav ril +Ġinv is +ĠK ear +toBe InTheDocument +ap ur +Ġr acked +ĠMc Master +_ATTR IB +H az +Ġfact ura +/ ts +ĠÑĢаз меÑĢ +Ġz f +Ġshort fall +.f asta +ĠCONST ANT +.man aged +g ems +Shared Pointer +Ġblur ry +b rightness +( components +Ġ... "ĊĊ +SE LL +ĠIllustr ator +.get Channel +Ġtrou vé +yst ers +Ġvo is +ĠLind en +Ġem ojis +Ġb rawl +ĠMS R +ĠE lo +ĠCroat ian +Popup Menu +L ewis +.J WT +Ġaston ished +B ush +(item Id +Ġdet achment +ĠEnc ore +å° Ķ +Ġre kl +Ġcr am +)$ / +.get Host +_re commend +- HT +_cal ibration +Auth enticate +.firebase app +UN IX +ĉC amera +ĠHE AP +I deal +. office +Ġgoof y +(S ymbol +Ġjou er +_part itions +Ġrapid ement +ĠGN UNET +id User +Ġsuperv ise +( Contact +AW N +ãģ ĺ +Ġna am +Ġa ust +åľ¨ 线 +_soft max +Allow Anonymous +amm able +RO UTE +* D +Ġad en +ĠCrist ina +ĠCrist iano +Ġblood stream +sub class +_person a +CH ILD +-k now +Ġnavigation Options +ĠZuk unft +ĠPix ar +Ty ler +Ġunder world +Ġsincer ity +Ġdispens er +Ġk ter +idd ers +.add Node +- checked +Ġke yst +ĠW TO +.sign als +Ġadvent urer +ĠP ang +\ R += pos +Ġdispens aries +ĠClo set +("{ \" +ide on +Ġnécess aire +() "Ċ +_RECE IVED +Ġrésult ats +Ġmod en +ĠIceland ic +; d +. allowed +(new User +Ġmerc iless +.Wait For +Ġday care +ĠCon veyor +ç ĸ +ð ¬ +ç ĥ +ç Ĺ +ç ł +è Ħ +é ² +å ¦ +çĿ Ģ +å¾ Ī +é ħ +ç ĭ +é ª +æ Ĥ +é ¥ +è ħ +æĥ ³ +å ¨ +é ¹ +ç Ĥ +å Ĵ +ç Į +è´ ¨ +æ ¢ +æ° Ķ +ð « +æķ Ļ +ç Ł +å Ħ +åıij å±ķ +åĪ Ľ +è ij +æ ħ +å ŀ +åģ ļ +æĪ ĺ +æ IJ +å¼ º +æ· ± +åĩ ł +ç ¿ +å © +è ŀ +å§ Ķ +åIJ Ħ +è İ +é ¸ +é º +åı Ĺ +èģ Į +å ĺ +æ ½ +é£ İ +èIJ ¥ +åħ ļ +è ľ +éĤ £ +é¢ Ĩ +ç ij +é ³ +æľ ¯ +ä» Ģ +æĪ ¿ +ç² ¾ +å ª +é Ĩ +å¤ ª +èĤ ¡ +è Ľ +åħ ī +æŀ ģ +åĬ ŀ +è ĵ +ç ĺ +å ´ +å Ĺ +èĬ ± +çł Ķ +å¿ « +å¸ Ī +è¶ Ĭ +è§ Ĥ +æ ¤ +æ ¦ +ç ŀ +èĤ ² +çĪ ± +çĻ ½ +ä¸ ĸ +ä»Ģ ä¹Ī +çľ ¼ +å ³ +è Ĵ +æ ĵ +è¢ « +å¹ ² +çĹ ħ +å£ « +ç Ĵ +è ¸ +æ ¾ +å·¥ ä½ľ +è® © +çĥ Ń +è¾ ĥ +åĦ ¿ +åĬ © +ç§ ¯ +ç ³ +ç ĵ +ç £ +å Ĥ +è ¹ +è ļ +å· ± +çĻ ¾ +åĬ ¿ +èµ Ľ +æ ¨ +æ ¿ +è ĸ +æĿ ij +å¸ ¦ +å¢ ĥ +æĬ ¤ +é Ń +å « +èĩª å·± +æµ İ +ä½ İ +åĮ » +éĺ ² +åĨ ľ +è Ĩ +ç Ĩ +é « +åĨ Ľ +æĪ ı +åį ĩ +æĸ ¯ +ä½ ı +èIJ ½ +åħ » +èĩ ´ +ç Ĭ +ç ĩ +ç ħ +è Ķ +ä¼ģ ä¸ļ +åĽ ¢ +æī į +æł ¡ +åĩ Ĩ +å¥ ĩ +åī ¯ +é ¼ +æ¼ Ķ +é© ¬ +èµ ° +ç¥ ŀ +åħ ĭ +æľ Ľ +æ² ¹ +è¾ ¹ +åį ĥ +å¾ Ģ +åĪ ĩ +æ © +ç ¶ +å Ļ +éĻ ħ +çī Į +社 ä¼ļ +游 æĪı +æĸ ½ +ç ħ§ +æİ § +æ» ¡ +è¯ Ĩ +éĩį è¦ģ +è¶ ³ +çķ Ļ +ç» Ĩ +åį ı +éĢ Ĥ +æ ĩ +æ § +é Ħ +è Ŀ +å¸Ĥ åľº +ç»ı æµİ +ä¹ ł +æĸĩ åĮĸ +éļ ¾ +ä¹ IJ +åĨ ³ +æ¬ ¢ +è§ ī +åĽ Ń +åħ ´ +åħ ħ +ä¸ ¾ +æī ¹ +è ķ +æĬ Ĭ +æĬĢ æľ¯ +ç© ¶ +第 ä¸Ģ +ä¾ ¿ +åĵ į +çİ © +åĿ ļ +èŀ į +åį Ĭ +åĸ ľ +å± Ĥ +ç¦ » +ä» ħ +é Ł +åij ³ +å¿ µ +åŃ £ +ç´ § +ä¹ ħ +é ¤ +é ŀ +è ¤ +åĢ Ļ +åĨ µ +ç Ł³ +åģ ¥ +æĢ İ +å® Ŀ +è¡ Ģ +åŁ Ł +æĹ © +çŁ¥ éģĵ +è´ Ł +åį ļ +å· ´ +äº ² +å± ŀ +ä¸ ¥ +äº ī +å¯ Ł +è º +ç ° +建 设 +产 ä¸ļ +åIJ ĥ +åŃ © +æĹ ħ +æł ¹ +æĿ IJ +ä¼ Ĺ +éļ ı +å® ĺ +åº ķ +å½ © +å¯ Į +æ¸ © +åį « +åī § +çĽ Ĭ +æĬ Ĺ +è´ ¢ +çº ª +æ Ĩ +çĶŁ æ´» +çº ¢ +çĶŁ 产 +è¿ ľ +éĴ ± +åĶ ® +ç¾ ¤ +çı Ń +æ¥ ¼ +éĩ ĩ +èī º +å± ħ +åģ ĩ +è° Ī +æĻ ļ +é ¬ +èĪ ª +å® ³ +è Ĺ +ç į +å µ +çİ ĭ +åº · +è İ· +ç» Ń +äº ļ +é£ Ł +åİ ĭ +æĭ Ľ +èĮ ĥ +è® ¸ +åĽ ´ +é ½ +éĻ į +çº ³ +åĵ ª +æķĻ èĤ² +å·² ç»ı +å¾ · +æŀ Ĺ +å®ī åħ¨ +é¾ Ļ +大 å®¶ +éĿ Ĵ +åº ľ +æ² ³ +åı ¤ +èį ¯ +åĿ ĩ +æĻ º +ä¹ ¡ +çķ ¥ +åĨ · +ç¦ ı +å® ¤ +ç» ´ +æī ¿ +å± Ĭ +è¯ ī +åĪ » +è Ł +æ ª +å°± æĺ¯ +è¿Ļ 个 +ä¸Ń å¿ĥ +ä¸ĸ çķĮ +åŁİ å¸Ĥ +éĿŀ 常 +åĪ Ĵ +åı Į +æĢİ ä¹Ī +åΰ äºĨ +æľ ĥ +åı ² +ä¾ Ĩ +å¾ ĭ +å¥ ĸ +ç» Ī +åª Ĵ +å® ģ +è¯ ¾ +èģĮ ä¸ļ +åħ į +æµ ĭ +æĢ ¥ +æķ ij +çĭ ¬ +èŃ ¦ +é¤ IJ +æĦ ¿ +è´ « +çĸ ij +å ļ +å¥ ¹ +åı Ī +åĽł 为 +ä¸į æĺ¯ +å¤ Ł +æĸ¹ éĿ¢ +éķ ĩ +äº Ĵ +éħ Ĵ +è® ² +çĸ Ĺ +æĺ ¥ +æ¹ ĸ +å¤ ľ +è´£ ä»» +人 æ°ij +åħ ° +çŁ Ń +æķ ħ +åĩ ı +æĻ ® +äº ® +ä¾ Ŀ +åį ° +éĿ Ļ +åĢ ĭ +å¾ ģ +åIJ ¸ +ç¼ º +æĶ » +åĩ Ģ +åħ ¸ +åĽ º +è® ¿ +ç ¹ +ç Ģ +æıIJ ä¾Ľ +ç» ĩ +å¾Ī å¤ļ +çłĶ ç©¶ +è· Ł +主 è¦ģ +æĥħ åĨµ +çŃ ĸ +æŃ » +大 åѦ +æĶ¿ åºľ +å½± åĵį +ä¹ ° +åħ Ń +éĻ © +åħ « +æŁ IJ +è´¨ éĩı +åį ł +å· ® +æĽ´ å¤ļ +æľ ĭ +éĿ © +å® £ +çł ´ +è½ » +åº § +æĺ ¾ +ç¨ ³ +è´ µ +èĥ Į +èī ¯ +çĸ « +æ¯ Ĵ +ä¹ İ +åĢ Ł +è¿ · +çŃ Ķ +æ¿ Ģ +åij ¼ +äºĨ ä¸Ģ +è¶ £ +ä¼ ´ +ä¼ Ļ +è ¼ +ð¬ Ń +åĽ½ å®¶ +æ´» åĬ¨ +çݰ åľ¨ +ç§ij æĬĢ +åį ¡ +ä¸į åIJĮ +个 人 +è®° èĢħ +ä¸į æĸŃ +éĹ » +ä¹ Ŀ +èij Ĺ +ç» ¼ +ä¸ ĥ +æł ij +æľĭ åıĭ +åį ĸ +ä¼ ¤ +æ² Ļ +åĸ Ħ +å¥ Ĺ +è½ ® +ç© ¿ +è¡ ¥ +ä¸Ģ å®ļ +çª ģ +çĿ £ +è¿ ½ +å¨ ģ +åı ¦ +åĽ ° +æŀ ¶ +ç» Ŀ +æķ £ +æİ ¢ +æ´ Ĺ +ä¸ ´ +ä¼ ¼ +è´ ¸ +ä¸ ° +æĺ¯ ä¸Ģ +ç« ŀ +è¿ İ +èģ ļ +è « +æį Ł +æī § +é© ¾ +è¿ Ŀ +è ¥ +è ł +ä»ĸ 们 +æĹ¶ åĢĻ +å® ĥ +人 åijĺ +è¿Ļ æł· +å·¥ ç¨ĭ +åĪĽ æĸ° +åŃ© åŃIJ +å¸ Į +éĥ¨ åĪĨ +éĵ ¶ +代 表 +é¦ Ļ +å¸ ® +æİ¨ è¿Ľ +çĽ ĺ +积 æŀģ +éĥ¨ éŨ +åŁ ¹ +æŃ ¦ +ä¸į ä¼ļ +çŃ ij +éĢ Ļ +çİ© å®¶ +æĭ ¿ +åİ Ĥ +æ¯ Ľ +çģ µ +æŃ Į +ç »¿ +å¦ Ī +çĽ Ľ +é¦ Ĩ +é¡ º +èĦ ¸ +å° ¼ +ä¸ ½ +å¥ ¥ +éģ ĩ +è¯ į +å° ģ +ä¸ Ŀ +好 çļĦ +æĭ ħ +èĦ ± +æģ ¶ +åİ ļ +åĬ ³ +çĽ Ł +æĬ ĺ +åı ¥ +æĢ Ģ +æŁ ĵ +书 è®° +åĨ ł +é² ľ +æ ¦Ĥ +éļ IJ +å¹ ħ +èµ ŀ +å¹ ķ +æ¥ Ń +éģ Ĺ +åĪ ¤ +è ĺ +å ¶ +æĬķ èµĦ +è¡Į ä¸ļ +äº ij +çݯ å¢ĥ +åѦ çĶŁ +åIJĪ ä½ľ +åģ¥ åº· +é£ ŀ +ä¸Ģ æŃ¥ +ä¸Ģ 缴 +åıij çĶŁ +éĺ ¿ +é¢Ĩ 导 +åĸľ 欢 +åºĶ 该 +çĤ º +è® Ń +æĿ Ģ +æ¸ ¯ +交 éĢļ +éĺ ¶ +éĴ ¢ +ä» ¤ +å° ½ +æ¯ į +è¡ £ +ç² ī +é¡ ¶ +ä¹Ł ä¸į +æĬ ĵ +èĭ ¦ +å¹ ¸ +ç¤ ¼ +第 ä¸ī +大 çļĦ +éģ İ +çĥ Ł +éģ ¿ +ä» į +åº Ĩ +æĢ ķ +è° ¢ +çĽ ĸ +å° Ħ +éľ ² +æĸ Ĺ +ç Ĭ¶ +åŃ ¸ +æ¯ ķ +å· ¨ +çŁ ¿ +çļ ĩ +å¸ Ń +çĹ ĩ +æī ¬ +å» ¶ +ä¾ § +æ· ¡ +çļĦ ä¸Ģ +ç¶ ² +æ´ ģ +ç ¸ +è§ Ī +çŃ ¹ +ç§ ĺ +è¯ Ĭ +çı ¾ +èª ī +æ¯ « +ð ¨ +åį ´ +æĪIJ 为 +èĥ½ åĬĽ +é» Ħ +æĹħ 游 +èĪ ¬ +æ¯Ķ è¾ĥ +èµ· æĿ¥ +äºĨ è§£ +èĩª çĦ¶ +ä¸Ģ 次 +åŁº æľ¬ +æĽ ¾ +综 åIJĪ +èı ľ +è§ī å¾Ĺ +第 äºĮ +è· ij +æ³ ¢ +åĢ Ĵ +ç¡ Ģ +åħ µ +èį ī +çĶ ³ +çĶ ° +æĤ £ +è§Ħ å®ļ +èĥ ľ +èµĦ 产 +æ¢ ¦ +æľ Ŀ +è¿Ļ éĩĮ +å¤ « +æĮ ¥ +ä½ Ľ +å® Ī +éĽ ¶ +æĸ ¼ +ç¯ ĩ +å² Ľ +åĵ ¥ +éŃ Ķ +ä¸į åΰ +æī ĺ +åº Ĭ +æ¬ § +èį £ +æ± ĩ +æī © +åģ ı +å¢ Ļ +è® ¯ +å© ļ +æĥ ł +æ´ ĭ +å® ľ +æ¶ ¦ +æħ ¢ +éĢ ı +å® ½ +é¡ ¾ +ç´ ¯ +æ± ¡ +çĪ Ĩ +ç§ Ł +æĥ Ĭ +æ¶ ¨ +é¥ ° +éĺ µ +é¥ ® +æļ ĸ +åº Ł +æĹ Ĺ +éļ Ķ +ç¶ ĵ +åĭ Ļ +å¯ ¦ +éĢ Ķ +æī « +çĥ Ī +éĽ » +åĪ ij +éĹ ľ +éĹ ª +å¥ ĭ +å Ĥ¨ +ç¼ © +ä¾ µ +å ¬ +𬠶 +åĽ½ éĻħ +ç»Ħ ç»ĩ +ä¸ĵ ä¸ļ +åıij çݰ +å¸Į æľĽ +ç»ı èIJ¥ +åı « +æĿ¥ 说 +éļ ľ +ä»» ä½ķ +交 æĺĵ +éĩį çĤ¹ +çļ ® +ç» į +æ´ ¾ +ç§ij åѦ +åºĶ ç͍ +建 çŃij +èĤ ī +æĶ¹ éĿ© +åŁº ç¡Ģ +æ± ī +åĩº æĿ¥ +è¿Ļ ä¹Ī +åĪ ļ +åĿ IJ +ä¸į ä»ħ +ä¼ļ è®® +éĿ ł +åªĴ ä½ĵ +æ° ¸ +åĨ ² +èĭ ı +å¤ ® +çĪ ¶ +åł Ĥ +å®ŀ éĻħ +è¡ Ĺ +ç« ¥ +éĺ ħ +äºĭ æĥħ +åİŁ åĽł +éħ ¸ +以 æĿ¥ +å¨ ± +å® « +åĿ Ĺ +ç» © +éĩ İ +ä¸į å¾Ĺ +ä¼ł å¥ĩ +ç¡ ¬ +åİ ħ +æĹ ¢ +ç» ĥ +èĦ ij +å¼ ± +æİ Į +è´ ´ +æĮ Ĥ +åħ³ éĶ® +å° ļ +é¥ Ń +åº Ħ +çĻ ¼ +åľ ĭ +æİ Ī +个 æľĪ +äº Ī +å¸ ģ +è· Ŀ +æ² ī +ç« Ł +åĨ ¬ +æĬ ½ +éĨ Ĵ +å¼ Ł +è§ ¦ +èģ ĺ +è± Ĩ +æļ ´ +åijĬ è¯ī +è± ª +èµ ¢ +è· ¨ +è³ ĩ +çĪ ¸ +æĬ ± +æµ ª +éº » +ä» ª +è¡ ¡ +å¥ ¶ +çģ ¾ +èµ ¶ +èĤ ¥ +å§ IJ +åĢ º +éľ ĩ +è® ¢ +æ¬ Ĭ +ç · +å» ī +ä¿ Ĺ +å¿ ĺ +å¦ ĩ +ç¼ ĵ +åŃ ķ +æ¼ « +è£ ģ +çĩ ĥ +é» ĺ +çī ¢ +çĪ · +æĬ µ +å® ¾ +æľī ä¸Ģ +è¿ ¹ +è¿ « +è² Į +æľī çļĦ +ð¬ ĺ +è¿ĺ æĺ¯ +æīĢ ä»¥ +ä¹Ł æĺ¯ +è¿Ļ äºĽ +对 äºİ +åIJ § +缮 åīį +èĩªå·± çļĦ +èĥ½ å¤Ł +å¦Ĥ ä½ķ +æľº æŀĦ +åıª æĺ¯ +ç½ij ç«Ļ +åħ¨ éĿ¢ +为 äºĨ +å¼Ģ åıij +æĸ° éĹ» +éĩij èŀį +ç» § +客 æĪ· +ä¸Ģ èµ· +èĮ ¶ +åħ³ 注 +æ°´ å¹³ +åİĨ åı² +å¢ŀ éķ¿ +é ± +åŁº éĩij +åº Ń +åı ¶ +ä¿ ĥ +éĽ ¨ +æ¶Ī è´¹ +èĪ ¹ +çŁ¥ è¯Ĩ +æĪĺ çķ¥ +ç»ı éªĮ +å³ ° +æĽ ² +èĦ ļ +åĨ ° +å¤ ı +å½ Ĵ +ç¬ Ķ +èĻ ij +çĶ ² +åľ Ī +è¯ Ĺ +é½ IJ +容 æĺĵ +çłĶ åıij +éª ¨ +çº ¸ +è· µ +æĹ § +çķ ¶ +åĪ ¸ +è´ · +åı ¬ +ç§ ĭ +æ¶ ² +è¡Į æĶ¿ +çĮ ® +èĤ ¤ +éĢ IJ +è¶Ĭ æĿ¥ +è¶ĬæĿ¥ è¶Ĭ +æĦı è§ģ +èĪ ŀ +åī Ĥ +æ¶ ī +ç¨ĭ 度 +åħ¬ åħ± +æ¢ ° +æľ « +çº ¯ +åĶ ± +æ´ ² +æĬ ¢ +æ¤ į +å¿ Ļ +ä¼ ° +å¼ ¹ +æ³ ī +æľĢ 大 +è¶ ĭ +å· § +ç¦ ģ +æī ¶ +åį ± +çı ł +çĨ Ł +æĭ ľ +主 ä¹ī +æĿ Ĥ +éĻ Ħ +éģ į +æIJ Ń +æĮ ¯ +å¤ļ å¹´ +æķ ¬ +æij Ħ +çº · +å¼ ĥ +æ¹ ¿ +å¨ ĺ +æ¡ £ +é© ¶ +æľ Ĺ +æ® ĸ +æ¦ ľ +åĵ ¡ +ä¸Ģ ä½ĵ +æŁ¥ çľĭ +ç¹ ģ +æµ ĵ +åħ¬ å®ī +æ½ ľ +è´ ¯ +éª Ĺ +æ IJľ +å· ¡ +è ¬ +é Ĭ +å§Ķ ä¼ļ +æĤ ł +åī © +æı Ń +åŃ£ 度 +ð «ĺ +𬠬 +ä ´ +ð ª +ä½Ĩ æĺ¯ +éĥ½ æĺ¯ +å¹³ åı° +åѦ ä¹ł +åĵģ çīĮ +ä¸ Ķ +è¿Ļ ç§į +æĶ¿ çŃĸ +æĭ ¬ +认 为 +ä¸Ģ èά +æłĩ åĩĨ +æĶ¯ æĮģ +模 å¼ı +åħ³ ç³» +çļĦ æĺ¯ +è¿Ļ ä¸Ģ +ä¸į è¦ģ +çĶ ļ +ç²¾ ç¥ŀ +æĭ ¥ +åĪ© ç͍ +ä¿Ŀ æĬ¤ +ä½ľ ç͍ +èĭ ¥ +åĽ½ åĨħ +ä»ĭ ç»į +ä¸Ģ ä¸ĭ +å·¥ ä¸ļ +缮 æłĩ +æľĢ åIJİ +ä»· å̼ +å° į +éĵ ģ +è° ģ +ç»ĵ æŀĦ +éĽ ª +æĻº èĥ½ +ä¼ł 绣 +ä½ĵ èĤ² +çĶŁ æĢģ +æĭ į +æİ ª +åĨľ ä¸ļ +çī¹ èī² +è§Ħ 模 +æĹ¶ 代 +è¿ĩ ç¨ĭ +éĴ Ī +æĿ ¾ +åĶ IJ +åĮ» çĸĹ +çģ ¯ +åζ éĢł +æł¸ å¿ĥ +ä¸į åı¯ +ç³» åĪĹ +åIJ ī +åľ £ +åĢ ij +ä½ ³ +æĿ¥ çľĭ +æ¯Ķ èµĽ +ä¸ĭ æĿ¥ +åĩº äºĨ +å¹² éĥ¨ +å¾® ä¿¡ +å½ĵ åľ° +åį · +åį« çĶŁ +ä¼ Ł +çĸ« æĥħ +è° · +åĩł 个 +éĺ ´ +çĶŁ çī© +å° ¤ +ä¼ Ĭ +èĤ ¯ +éĿ¢ 积 +åĪĽ éĢł +æı ¡ +åľ Ĩ +æĻ ĵ +æĪIJ äºĨ +åĩ ¡ +çĸ ¾ +ç«ŀ äºī +è® ¨ +主 é¢ĺ +é² ģ +è¿ ª +ä¿ Ħ +æĢ ª +ä¸ ¦ +èĻ ļ +æ½ ® +çĥ § +èĢ ³ +æ± ł +éĢĤ åIJĪ +æł¹ æľ¬ +åĬł 缣 +ç͵ è§Ĩ +æ· · +ç¼ ĺ +çª Ĺ +çĬ ¯ +æĥ ¯ +æĦı ä¹ī +åĬŀ æ³ķ +ä¼ ij +æ» ij +åĭ ĩ +æķ ¢ +å¯ » +è¦ Ĩ +éĢ ĥ +ç»ı çIJĨ +åĿ ı +æ³ ½ +ä¹ ĺ +åĪ º +å± ı +é¡ ¿ +äº ¡ +éĤ Ģ +åħ ¼ +åĭ ¤ +æ® ĭ +æĺ ł +æ¯ķ ä¸ļ +æĪ ª +è· Į +å£ ģ +åı¦ ä¸Ģ +羣 å®ŀ +ç£ ¨ +è¯ ļ +å¿ħ è¦ģ +æģ ĭ +æĩ Ĥ +å¾ Ĵ +è° ĵ +æķ ı +æ ύ +èĥ ¸ +æĭ ¼ +å¦ Ļ +è¯ ¸ +èģ Ĭ +æĤ ī +éº ¼ +åĩ Ń +èĪ Ĵ +æ¶ Ĥ +è¿ ģ +æ² ¿ +å¡ ij +æĽ ¿ +æ¾ ³ +å¿ į +èĢ Ĺ +éľ ¸ +åĩł å¹´ +åĪ Ĭ +èĦ ī +èħ IJ +æ¡ Į +çº ł +æ» ļ +æĤ ² +åĨ Ĵ +å¦ ¹ +çķ ħ +çº µ +æij ĩ +å¤ º +è·¯ ä¸Ĭ +å¿ ½ +èĸ ª +æģ IJ +æĦı æĢĿ +å« Į +æı ´ +æ° § +èĢ Ģ +éĺ » +è½ ¨ +å¹ » +æį ķ +åĿ ¦ +åĵĪ åĵĪ +çĭ IJ +æ» ¨ +è² » +è¿ Ł +人 éĥ½ +ç» ĺ +åı ¹ +çµ IJ +æī ° +æ» ĭ +å¥ ij +åĭ Ł +ç¢ º +ð ¦ +éĽĨ åĽ¢ +æĿ İ +å¼Ģ å±ķ +æıIJ åįĩ +åħ¨ åĽ½ +æ±½ 车 +åѦ æł¡ +æł¹ æį® +è¿Ļ æĺ¯ +åĩº çݰ +éĻ Ī +ç½ Ĺ +èİ· å¾Ĺ +åĪ ĺ +éĶĢ åĶ® +æľª æĿ¥ +éľĢ æ±Ĥ +å®ŀ æĸ½ +åĿļ æĮģ +åħ¨ çIJĥ +éĵ¶ è¡Į +æİ§ åζ +é¡ » +åľ° åĮº +æīĵ éĢł +çļĦ è¯Ŀ +帮 åĬ© +ä½ĵ ç³» +è¾¾ åΰ +è§Ħ åĪĴ +åŁ¹ è®Ń +两 个 +æĬ¥ åijĬ +åľ° æĸ¹ +å®Į åħ¨ +æİ ī +ç»ĵ åIJĪ +宣 ä¼ł +æ³ķ å¾ĭ +èīº æľ¯ +ç͵ å½± +èª ª +ä¸Ģ çĤ¹ +è¶ħ è¿ĩ +ç͵ åŃIJ +æĢĿ æĥ³ +æķĻ åѦ +éĺ¶ æ®µ +åķĨ ä¸ļ +çī© æµģ +åĪĽ ä¸ļ +æĸ¹ æ¡Ī +çݰ 代 +æ¡ ¥ +èIJ½ å®ŀ +带 æĿ¥ +产 çĶŁ +ç§ Ģ +æ³ ° +ä¹ ± +åħ· ä½ĵ +åĸ Ŀ +èĵ Ŀ +å® Ĺ +åįĩ 级 +æ·± åħ¥ +ä¿Ŀ éĻ© +ç®Ģ åįķ +çĹ Ľ +稳 å®ļ +è¾ Ĩ +å±ŀ äºİ +å· Ŀ +ä¸į å°ij +åĴ ¨ +举 西 +å½¢ å¼ı +娱 ä¹IJ +æŃ£ 常 +é¸ ¡ +åħħ åĪĨ +å®ŀ è·µ +éĩĮ éĿ¢ +è· ³ +èĻ İ +æĪIJ éķ¿ +æļ Ĺ +çĿ ¡ +ç½ ª +çIJĨ 念 +æĮ ij +èµĦ æľ¬ +å¤ļ å°ij +ä¸ĭ éĿ¢ +å¸ Ŀ +åħ¬ å¼Ģ +æ¸ IJ +éķ · +å± ĭ +欢 è¿İ +å¿ĥ çIJĨ +çĤ İ +æ¹ ¾ +è® ĵ +éĤ Ħ +ç³ ĸ +ä¹ Į +åĬ ± +çī Ļ +èħ ¿ +å² Ĺ +ä¼ į +æĪIJ åijĺ +åŃ Ķ +å°ı ç¼ĸ +èij £ +æ³ ¡ +åħĪ è¿Ľ +åħ § +åĺ ´ +è´ Ŀ +è » +æIJ ŀ +æ³ Ľ +é¸ Ł +ç½ ² +èĽ ĭ +主 ä»» +缮 çļĦ +ä¹ ı +æ´ ¥ +æĪ ´ +严 æł¼ +çħ ¤ +çĮ « +åĶ ¯ +å° Ĭ +çĶ ľ +åŀ ĥ +åľ ¾ +æĭ Ł +çĦ ¦ +é« Ķ +å® ı +æ© Ł +é© » +æĹ ģ +å½ » +éĥ½ ä¸į +æij © +ä» ĵ +ä¹ ³ +å² ¸ +è° ĭ +大 å¤ļ +çģ Ń +èħ ¾ +æŁ ľ +èĪ į +åħļ çļĦ +å° ĺ +åįģ å¹´ +æĭ Ĵ +è£ ¡ +æŁ Ķ +å¹ ¼ +éĶ ģ +ä¸ĵ 项 +æī İ +驾 é©¶ +ç¢ İ +è¢ ĭ +éĶ ĭ +å£ ® +å° ĸ +ç͵ æ±ł +è¿ Ķ +æ¼ ı +å¾ ª +èı Į +èĥ ĥ +è¾ ħ +éĢ Ĵ +èĥ İ +éĻ ª +å¯ ¿ +å¥ Ķ +çĮ Ľ +çº ¹ +çŁ¥ åIJį +å¿ Ĩ +æ¡ ĥ +æ£ ĭ +éĢ Ĩ +çĤ ¼ +ç± į +çī § +æł· çļĦ +è¾ Ľ +åł Ĩ +å®ŀ åľ¨ +ä¼ ı +å® ¿ +èµ ı +è£ Ĥ +åįĬ å¹´ +åĢ ¾ +满 æĦı +æ¢ ¯ +æĦı åij³ +åŃ ¤ +ç¥ Ŀ +æĻ ¶ +èµ Ķ +åģ ¿ +èĦ Ĥ +ç½ ļ +ç¢ į +æ² ĥ +æ ĵį +å´ ĩ +æļ Ĥ +è· ĥ +æIJ ¬ +å© Ĩ +é ī +éī ´ +åħ´ è¶£ +èIJ¥ ä¸ļ +è® Ĭ +èĦ ı +è¾ Ī +å·ŀ å¸Ĥ +è´« åĽ° +ç© · +ä¸Ń å°ı +æ¼ Ĥ +çĻ Į +èľ ľ +ä¼Ļ ä¼´ +çī µ +æĤ Ł +éĻ · +èµĽ åŃ£ +æ¨ £ +åģ ¶ +æĺ Ĩ +è¢ Ń +æį IJ +èī ° +æ Ĥ¬ +çĶ ¢ +èij ¡ +çĽ Ĺ +å© ´ +å° İ +çº ½ +åĢ ¡ +æī ® +è¨ Ń +æĬ ij +ç¡ ķ +è¾ ĸ +éĥ ģ +è¾ © +éĤ » +çݰ åĩº +è¦ ı +å½ ¹ +éĺ Ķ +åī µ +è¯ ± +æĥ ij +æ· Ģ +é¢ Ī +ä¾ ¦ +æģ ° +æ£Ģ å¯Ł +éĨ « +çĦ¶ æĺ¯ +åĭ ĥ +èĮ « +ä ĵ +𠬸 +ä½ľ 为 +çļĦ 人 +éĤ£ ä¹Ī +ç¾İ åĽ½ +è¿ĺ æľī +æıIJ é«ĺ +èĻ ½ +åħ· æľī +åĮħ æĭ¬ +æĪĸ èĢħ +ä¸į è¿ĩ +ä¸Ĭ æµ· +åĮ» éĻ¢ +èµĦ éĩij +çĶļ èĩ³ +åζ 度 +è§£ åĨ³ +èģĶ ç½ij +ç»§ ç»Ń +建 ç«ĭ +è¿Ľ ä¸ĢæŃ¥ +æĿIJ æĸĻ +ä»Ĭ 天 +å¿ħ é¡» +åIJĦ ç§į +çݰ åľº +ä»ĸ çļĦ +å¢ŀ åĬł +é¢Ĩ åŁŁ +åıĤ ä¸İ +æĮģ ç»Ń +ä¹ĭ ä¸Ģ +çī¹ åĪ« +é± ¼ +åħ± åIJĮ +åĬ ª +çİ ī +人 们 +åħĪ çĶŁ +ä¼ĺ åĬ¿ +ä¿Ŀ æĮģ +ä½ľ åĵģ +çī Ľ +æĪIJ æľ¬ +æĶ¶ åħ¥ +åıĬ æĹ¶ +è´Ł è´£ +æİ¥ åıĹ +èį IJ +åıª è¦ģ +羣 çļĦ +导 èĩ´ +æľº åζ +è¡Į åĬ¨ +æĸ° çļĦ +å®Į åĸĦ +为 ä»Ģä¹Ī +ä¸Ń 央 +æĪIJ ç«ĭ +æĦŁ è§ī +åıĺ åĮĸ +åıĹ åΰ +å¹¶ ä¸į +åŃ Ļ +æĸ½ å·¥ +æĺİ æĺ¾ +è¿ĩ åİ» +åıij æĮ¥ +羣 æŃ£ +åŁº åľ° +æĺİ ç¡® +èĥ ¡ +许 å¤ļ +ä¸Ģ å¹´ +æĸ¹ åIJij +æģ © +缸 ä¿¡ +åľ ³ +详 ç»Ĩ +äºĭ ä¸ļ +çĶŁ åij½ +åĴ¨ 询 +æĸĩ æĺİ +çij ŀ +绿 èī² +èİ « +æĦı è¯Ĩ +æĬķ åħ¥ +åĬł å¿« +æ¢ ħ +ç¿ » +å¼Ģ æĶ¾ +æĻ® éĢļ +åįı ä¼ļ +æĪIJ 绩 +ä» Ļ +å¯ Ĵ +è¯ģ åΏ +认 è¯Ĩ +ä¸ ¹ +大 éĩı +è¿ ħ +åģļ åΰ +设 æĸ½ +è´¸ æĺĵ +èĥ½ æºIJ +æĹ¶ æľŁ +ä¸Ģ 天 +æ²» çIJĨ +åĺ ī +å® ĩ +丰 å¯Į +举 è¡Į +æĪIJ æŀľ +èĤ¯ å®ļ +çĭ Ĺ +åĬ¨ åĬĽ +æ£ ® +åĩł ä¹İ +åĽł ç´ł +æ°ij æĹı +æ´ ŀ +ç½ij åıĭ +åIJĪ çIJĨ +广 大 +æ® Ĭ +æ´ Ľ +æĿ ¯ +èĴ Ļ +ç͍ äºİ +èŀį èµĦ +ç¥ ĸ +æľº 械 +举 åĬŀ +èĩª åĬ¨ +åĬŀ åħ¬ +é» ŀ +éĽ Ħ +å̼ å¾Ĺ +çĮ ª +以 为 +æĺ Į +è·Ŀ 离 +åIJ¸ å¼ķ +ç» ķ +éļ Ĩ +计 ç®Ĺ +éĺŁ ä¼į +大 ä¼ļ +å¼ķ èµ· +çī¹ çĤ¹ +èĥ ¶ +å¹´ è½» +æľ¬ 身 +æľº åħ³ +å®ĺ æĸ¹ +éĥ ij +æµ Ļ +è§Ĵ èī² +èij£ äºĭ +为 主 +æĹł 论 +ä¹ł æĥ¯ +æ¥ ļ +æĭ ĵ +绣 计 +åħ Ħ +广 æ³Ľ +åį Ģ +污 æŁĵ +è« ĭ +èĬĤ 缮 +ä¼ ¦ +è¦Ĩ çĽĸ +èĢ IJ +æī¶ è´« +ç»ı åİĨ +éĩįè¦ģ çļĦ +èĤ¡ 举 +æĭĽ èģĺ +åĽĽ 个 +æĩ ī +èĥ ŀ +æij Ĩ +é«ĺ éĢŁ +éº ¦ +åİŁ åĪĻ +èİ ± +æĽ´ 好 +éķ ľ +åĩ Į +åŀĥ åľ¾ +éĢ ² +çģ ° +éĵ º +äºĭ æķħ +çĶ ĺ +空 æ°Ķ +é¾ Ħ +èı ² +çĵ ¶ +æĺ ¨ +æĹ¥ æĬ¥ +æµ ® +åľ° åĽ¾ +åij Ī +大 åĬĽ +ç» ª +å¸ ħ +æľį åĭĻ +ä¸į éĶĻ +乡 æĿij +å± ¥ +å¹³ æĸ¹ +éĹ ² +æī £ +ç´ł è´¨ +èµ ´ +éģ Ń +èIJ ¨ +èĩª 主 +éĩij å±ŀ +èī¯ å¥½ +两 å¹´ +æ³ ¥ +é¢ ľ +ç²¾ 彩 +ä¸Ń åįİ +æĻ ĭ +ä¹ł è¿ij +ä¹łè¿ij å¹³ +æĪĺ 士 +åģļ çļĦ +éª ij +æ» ´ +çĵ ľ +çīĪ æĿĥ +èĤ ł +æľĥ åĵ¡ +çı į +ç¨ ® +ä »¿ +çī© ä¸ļ +åĢĭ 人 +å¦ » +ä¼ ¸ +æ± Ĺ +æĹ º +çIJĨ æĥ³ +æij ¸ +è¿Ŀ æ³ķ +å®Į æķ´ +åİ ¦ +è¸ ı +æĸ ij +æ¡ Ĥ +ä½ĵ åζ +å¸ « +æĿ Ĩ +æ® ¿ +æ¯ ģ +é¦ Ī +è§Ĵ 度 +æ¬ £ +çĥ ¦ +èĤ º +éĩĩ 访 +æij ĺ +æĮ ¡ +æ· ĺ +åħ» èĢģ +çĤ ¸ +è¿ Ī +åİ ī +åĿ Ĭ +è¾ £ +åĩ Ŀ +æ³ ª +çĸ ı +æİ ĺ +åĥı æĺ¯ +éĽ ķ +ç¼ Ŀ +èį · +æį · +åł ¡ +åı¥ è¯Ŀ +çĸ ¼ +æł ı +éģ µ +ç¢ ³ +å·¥ åķĨ +æIJ º +åĪ ¥ +ä¹ Ļ +æĹ ĭ +æĥ ľ +ä¸Ģ 大 +å±Ĥ 次 +èµ ĸ +æĬ ¬ +æ¨ Ĥ +è¯ ŀ +åħ Ĵ +ç¯ ® +èĤ ĥ +å§ ¿ +æĬ ļ +çĵ · +ç͵ åĬ¨ +æĸ° åĨł +æ¶ µ +ç¢ ij +æ· ® +æĹ ¨ +è¸ ª +æ¸ Ķ +æĦ Ī +åı Ķ +åįĹ çľģ +ç¾ © +å§Ķ 书记 +è² ¸ +æ¶ Į +è« ĸ +èIJ Ħ +æı ı +å¿ § +è¾ ¦ +å¦ Ĩ +æī Ń +åij µ +éģ ¥ +è¨ ± +ä» ĩ +åįģ ä¸ī +åī ² +èª į +èĪ ° +é¢ ĩ +é¥ ± +çĭ ł +é«ĺ çļĦ +çµ ± +æħ İ +é¢ ģ +åIJĪ éĢĤ +æµ ´ +èµ ĭ +æĬ ¼ +å¦ ¥ +éĻ¢ éķ¿ +èĢ ķ +è¾ ¨ +æħ ° +åįģ åĽĽ +æľ µ +èĵ Ħ +æŀ ¢ +å» · +æĤ Ħ +æ¶ ¯ +çŁ © +åŃIJ éĩĮ +çĬ ¹ +å±Ģ éķ¿ +é IJ +å¥ ł +ä¼ļ éķ¿ +æĵ ļ +ä¸į åıĬ +åįģ ä¹Ŀ +æ¬ º +èº º +éĺ IJ +çº Į +è¨ » +åĨ Ĭ +èŃ ĺ +é«ĺ çŃī +èħ º +å¤ ķ +ç» ij +åĶ ¤ +èķ ´ +çķ ľ +æħ ĭ +åı Ļ +åı ĥ +å³ ¡ +人 大 +éħ ¿ +éģ © +å¥ ¢ +åı£ æ°Ķ +éĮ Ħ +é ı +åĭ ĺ +è´ ¿ +éļ ª +é ĭ +éļ ¶ +ð ¥ +𬠣 +ð £ +ð« į +𬠳 +ð« ĵ +ð« Ħ +ð« Ł +𨠱 +ä Ĺ +以 åıĬ +æľī éĻIJ +åij ¢ +åIJ Ĺ +çľĭ åΰ +计 åĪĴ +è¿Ľ åħ¥ +缴 æİ¥ +åĪĨ æŀIJ +åıª æľī +设 å¤ĩ +åħ¶ å®ŀ +åĬł 强 +ä¸Ń çļĦ +ä¿Ŀ éļľ +èĢģ å¸Ī +人 æīį +å¾Ĺ åΰ +é£İ éĻ© +ä¸Ģ ç§į +空 éĹ´ +æĪij åĽ½ +ä¹ĭ åīį +ä¸ĵ å®¶ +æĿ ¨ +æĹ¥ æľ¬ +群 ä¼Ĺ +åıĤ åĬł +æķĪ æŀľ +æľī åħ³ +å®¶ åºŃ +åĮº åŁŁ +åĬª åĬĽ +éļı çĿĢ +æĹł æ³ķ +交 æµģ +è¡Į 为 +æ£Ģ æŁ¥ +æľŁ éĹ´ +å¦Ĥ æŃ¤ +èĤ¡ 份 +å½ĵ æĹ¶ +è£ħ å¤ĩ +åĩĨ å¤ĩ +éħĴ åºĹ +è¿IJ åĬ¨ +æıIJ åĩº +å·¦ åı³ +æİª æĸ½ +é£Ł åĵģ +æ¶Īè´¹ èĢħ +åѦ éĻ¢ +æĮĩ 导 +è¿IJ èIJ¥ +éĩį 大 +åĨľ æĿij +éĢł æĪIJ +æĶ¿ æ²» +éĴΠ坹 +æŃ£ å¼ı +åıĸ å¾Ĺ +éĤ£ 个 +éĽĨ ä¸Ń +åıª èĥ½ +å¿« éĢŁ +身 ä½ĵ +åħļ åijĺ +èģĶ åIJĪ +åĬĽ éĩı +éĥ½ æľī +æ ħ§ +å¡ Ķ +åĪ« 人 +表 çݰ +æķħ äºĭ +ä¸Ģ åĪĩ +å° ĩ +èµĦ æĸĻ +åŁ¹ åħ» +éĺħ 读 +æľī 人 +èIJ¥ éĶĢ +çĽij çĿ£ +çݯ ä¿Ŀ +èĢĥ èĻij +æ·± åľ³ +严 éĩį +èĮĥ åĽ´ +å§Ķ åijĺ +çĽij 管 +ä¸ī 个 +è£ħ ä¿® +åħ¬ éĩĮ +åĪĨ åĪ« +çIJĨ è§£ +éŁ © +åĬł å·¥ +认 羣 +ä¸į 好 +åİ» å¹´ +éĻį ä½İ +æľº ä¼ļ +åįı è®® +符 åIJĪ +å¢ŀ 强 +æĬĢ èĥ½ +é¦ĸ åħĪ +ç§ ¦ +ä¸ ģ +å° ¾ +æľī äºĨ +åľ° 产 +æ¸ ł +æĸ¹ 便 +ç§» åĬ¨ +éĢŁ 度 +å°¤ åħ¶ +éĢļ çŁ¥ +åĿ Ľ +éģ¿ åħį +æģ ¢ +è´ ¡ +èģĮ å·¥ +å®ŀ åĬĽ +æĺ¯ä¸Ģ ç§į +åIJ¯ åĬ¨ +çĸ¾ çĹħ +æĿ¥ äºĨ +缸 对 +çݰ å®ŀ +èŀį åIJĪ +åIJĮ æł· +åħ¬ åijĬ +çī¹ æ®Ĭ +ç´ « +ä¸ĭ åİ» +ä¼ł æĴŃ +æľĢ 好 +ä¼ĺ è´¨ +æ² Ĵ +æĮ º +æĹ ¦ +è¯ º +ä¸Ģ åIJį +éģĵ è·¯ +示 èĮĥ +è¿ĩ æĿ¥ +åIJĮ åѦ +é¼ ĵ +æĿ Ń +æľ¬ 次 +åIJĮ æĦı +ä¸ĸ 纪 +ç¾ Ĭ +æ¬ ² +å·¥ èīº +çĵ ¦ +人 士 +æľī æīĢ +ä»İ äºĭ +æľī å¾Īå¤ļ +ä¸į äºĨ +å²Ĺ ä½į +åıĺ å¾Ĺ +åĬ³ åĬ¨ +å¤Ħ äºİ +å¹³ åĿĩ +å½¢ 象 +å¡ ŀ +åħ± 享 +çĿ Ľ +åĪ© 润 +æŃ£ æĺ¯ +å¾Ģ å¾Ģ +缸 æ¯Ķ +æ¨ ª +åĪ · +æµĻ æ±Ł +大 éĥ¨åĪĨ +å¤ļ 个 +æĤ¨ çļĦ +ç͵ åķĨ +å¾® åįļ +å§ĭ ç»Ī +çĬ¯ 罪 +æĺ¯ åľ¨ +ç»Ħ åIJĪ +åİŁ æĿ¥ +æ¸ħ æ¥ļ +åIJĦ åľ° +æĦŁ åıĹ +å½ĵ ä¸Ń +è¶ĭ åĬ¿ +æĻ¯ åĮº +羣 æĺ¯ +ä¾Ľ åºĶ +转 åŀĭ +çĭ Ĥ +èĨ ľ +èĭ Ĺ +å¿ ł +å¾Ī 大 +èĤ¡ æĿĥ +ç¾İ åħĥ +æİĴ åIJį +åĬ¨ çī© +éĶ ħ +å¢ ¨ +主 å¸Ń +å¾Ī 好 +ç»Ŀ 对 +æĿ ľ +转 è½½ +çĴ ĥ +æĿij æ°ij +åIJ ¨ +åĽŃ åĮº +é«ĺ 度 +çī© è´¨ +è¾ ī +æĹ¥ 常 +æı Ĵ +ä¸ī å¹´ +ä½ĵ çݰ +æīį æĺ¯ +代 çIJĨ +ä¸į 管 +æģ Ĵ +åľ° ä½į +ç² ® +èĸ Ħ +æĺİ çϽ +ä¸Ģ èĩ´ +æĽ ¼ +åĵ Ń +åĩ ¤ +åĬ ² +æķ Į +æĪĺ æĸĹ +主 ä½ĵ +åħ¬ å¸ĥ +åıĤ èĢĥ +èĪª 空 +å¯ º +åѦ ä¼ļ +åıį æĺł +ç¾İ 丽 +太 éĺ³ +建 æĪIJ +æħ¢ æħ¢ +åIJĦ 个 +éĤ ¦ +ç»Ħ æĪIJ +ä¸ī 大 +éĶ ¦ +大å¤ļ æķ° +æ¦Ĥ 念 +éŃ Ĥ +åħ¬ çĽĬ +èį Ĵ +身 份 +æ·± åĪ» +åħ © +ç»ı åħ¸ +åIJĦ 项 +èĻ ķ +è¿Ľ æŃ¥ +åįģ äºĮ +æī§ æ³ķ +æĥ³ åΰ +æĦŁ æŁĵ +åķĨ åĬ¡ +å°ı ç»Ħ +èĶ ¬ +çıŃ åŃIJ +åIJĮ å¿Ĺ +éĿ¢ 临 +çĤ Ĵ +å¤ļ ç§į +è§Ĥ çĤ¹ +åĵª éĩĮ +å° Ŀ +å§ Ĩ +èħ ¹ +åŁİ åĮº +太 å¤ļ +çĹħ æ¯Ĵ +åľ¨ äºİ +æīĢ è°ĵ +æĻ ° +æŀ Ŀ +æĭ ĸ +å® ħ +æķ´ æ²» +ä½ı æĪ¿ +åģ · +çĨ Ĭ +èµ ģ +æ° Ľ +æł¼ å±Ģ +åŁºç¡Ģ ä¸Ĭ +èĥ Ĩ +åħ ½ +鼶 åĶ® +åĿ ¡ +女 åŃ© +æĴ ŀ +åħ¨ åĬĽ +åĴ ĸ +èĤ © +çľ ī +èĩ³ äºİ +åħļ ç»Ħ +ä¸Ģ ä»¶ +æĭ Ĩ +äºĭ å®ŀ +åĤ ³ +æ¹ ĺ +ç¶² ç«Ļ +循 çݯ +åIJĮ æ¯Ķ +æĭ Ķ +åĮ» èᝠ+åħ» æ®ĸ +åĽº å®ļ +å®ŀéĻħ ä¸Ĭ +è®° å¾Ĺ +åĪ© äºİ +æĤ ¦ +æĭ ³ +èĤ Ŀ +æķĪ çĽĬ +è© ² +æ°ij 主 +çĹĩ çĬ¶ +é¢ ¨ +å¹¼ åĦ¿ +å§ ij +æĪ Ĵ +ä¸ĭ çļĦ +æ¸ ¡ +å¹´ åºķ +è®° å¿Ĩ +åIJ IJ +大 å¹ħ +å¾ ½ +åħ¬ ä¼Ĺ +ä¿¡ å¿ĥ +çİ Ľ +ä¼ļ ä¸Ĭ +ä¹ Ķ +æijĦ å½± +æ£ĭ çīĮ +éĻ ķ +åºĶ æĢ¥ +æĶ¶ è´¹ +æİ§ èĤ¡ +仪 å¼ı +çŀ ¬ +æīĢ åľ¨ +ç¢ ° +å§ ĵ +é¡ Į +æĶ¯ éĥ¨ +使 åij½ +çĤ ī +å¯ Ħ +ç¿ ¼ +åľ° ä¸ĭ +è¾ ŀ +ä¿ ± +主 æĮģ +è´§ å¸ģ +æģ ¨ +èĤ Į +çĽ Ī +éĶ » +å¿Ĺ æĦ¿ +ç±» ä¼¼ +æĮ ĸ +éĢ » +ç¸ ½ +纪 念 +åķ ¥ +å¼ ¯ +åIJį åŃĹ +åģ¥ èº« +çļĦ å¿ĥ +é© ± +èĥĮ åIJİ +æ³ķ å¸Ī +ç² Ĵ +èĥ½ éĩı +è¾ ° +èī ³ +å½ ¼ +段 æĹ¶éĹ´ +åIJĪ æ³ķ +æĵ ¦ +ç¾ ½ +åİ ¨ +æĪij 说 +äºĭ åĬ¡ +åĩł 天 +åħ ģ +ç¼ ´ +åį ĵ +两 ç§į +çĭ¬ çī¹ +å¸ ¶ +éĴ » +æĥ © +é¢Ĩ åħĪ +è¶³ å¤Ł +å£ ³ +æĦıåij³ çĿĢ +åĪĨ å¸ĥ +ä¹ ĥ +éģ ĭ +ä½ © +è° ± +çģ £ +èį ¡ +è´¯ å½» +å¹ ¾ +ç£ ģ +åħ¸ åŀĭ +åī ĩ +åĨ » +æ¬ ł +ä¸į ä¹ħ +æµ ¦ +éŃ ħ +å¼Ģ äºĨ +使ç͍ èĢħ +è¿Ļ 款 +å° Ī +èĦ± è´« +æĶ» åĿļ +ç®Ĺ æĺ¯ +ç¨ Ģ +æĹł 人 +åł µ +å¥ ı +éĥ½ å¸Ĥ +åı¯ è§ģ +ä¸į åĩº +æ ·» +äº ı +ç¾İ 好 +èĥ ĸ +éŁ µ +æłĩ å¿Ĺ +èĬĤ èĥ½ +æĬ « +å° º +å¯ ¸ +ä¸Ģ 代 +é¢ Ĺ +èĢ ¶ +èĴ ¸ +åĸ ® +æ »¿ +çĮ ľ +æµ Ĩ +åŁ ĥ +åįĥ ä¸ĩ +èµ Į +èģ ² +ä½ľ é£İ +è³ ª +å¯ ¨ +å¹´ 人 +åį° è±¡ +æ¡ ¶ +æĴ ¤ +åįģ äºĶ +æ¯ ħ +æ² ª +åĽ½ æľī +大éĩı çļĦ +å¾ ¡ +å¯ ĵ +è¦ ĸ +æ¼Ĥ 亮 +çľ ł +ç ĤŃ +é» İ +èĻ ¹ +åĪ© äºļ +èŃ ī +æµ ı +åįģ åħ« +ä¸ ¢ +è¾ ½ +æľīä¸Ģ äºĽ +æħ Ī +åģľ è½¦ +å® ł +è§£ æĶ¾ +æľī å¤ļ +éĤ Ĭ +常 è§ģ +æĬ ¹ +çº ¤ +è¦ ª +æ¡ Ĩ +èİ ŀ +æ°§ åĮĸ +è¿Ļ ä»¶ +åĩ ° +æŁ ´ +åıij ç͵ +é¼ ł +转 åĮĸ +å¨ ĥ +æĮ ¤ +ç½ © +å¯Ĩ åĪĩ +æĪij ä¸į +é«ĺ æĸ° +ä¸Ģ ç¯ĩ +è¿Ľ ç¨ĭ +è¡ ° +è¿ĺ ä¸į +ç ħĮ +æĸ° åįİ +èĤ ¿ +æ» © +ä¸Ģ æµģ +è¯ Ī +å®ŀ ä½ĵ +å¤ĸ åĽ½ +èº ² +èµ ł +è¦ º +æ¢ Ŀ +ä¸į è§ģ +è¨ Ĭ +åĮ ¹ +åį µ +çĩ ¥ +æħ ķ +é½ ¿ +å® ´ +é¥ ¼ +èij¡ èIJĦ +å°ı å¿ĥ +æģ ¼ +éĻ Į +æĺ Ĥ +åĥ ¹ +èĬ Ŀ +æ¯ı 个人 +åīį æıIJ +ä½ĵ ä¼ļ +æ¨ Ļ +æIJľ çĭIJ +对 åħ¶ +ä¸ § +èľ Ĥ +æµ ¸ +èª ¿ +åĿ ª +é¢ ĸ +åIJį 为 +ç¬ ¼ +èĪ Į +æľ¬ 书 +èģ ¯ +çº º +ç®Ģ 缴 +éĽ ¢ +ç¾İ çļĦ +éļ ¨ +é«ĺ å³° +è¿Ļ å®¶ +å Ĥ¬ +å° ¸ +ç¡ķ 士 +èŃ · +è° ¨ +æĺ ı +æĶ¿ åįı +è¡ Ķ +ç¿ Ĵ +åľ Ĵ +åĽ½ æ°ij +主 è§Ĵ +è£ ķ +ä¼ ª +åº ŀ +æ°ij èIJ¥ +æĥ § +ç§ĺ 书 +çĹ ķ +çϾ åĪĨ +æº ¶ +æĹł çĸij +çļĦ çľ¼ +æĵ İ +ä¼Ł 大 +å½ ° +åħ¬å®ī å±Ģ +ç³ ķ +å¼ ¥ +åĤ Ļ +ä¹ ¾ +毫 ä¸į +注 æĺİ +åī¯ æĢ» +æĦ ī +æķ ¦ +é¦ ¨ +æĶ Ģ +éĢ Ŀ +åı¯ éĿł +å¤ ¸ +åľ ĺ +éĿ¢ ä¸Ĭ +æĬ ĸ +èĦ Ĩ +é© ° +ä¼ IJ +å¦ ¨ +å®ļ äºĨ +ç³ Ĭ +æŃ ¡ +éĥ¨ éķ¿ +ç§ ī +èĪ Ĩ +åĪij äºĭ +åIJ µ +æ¤ Ĵ +è¡ ĵ +è± « +èı © +åŃ µ +é¥ ² +å°± 好 +åł ª +ä¸ī è§Ĵ +åľº æ¯ĶèµĽ +ä¸į åģľ +æĵ ħ +åħ¨ æĸĩ +æ³ ģ +åѦ ä½į +æ± ° +éł ĺ +åı ł +éļ Ľ +å¸ IJ +çľĭ åĩº +åĮ ł +å±Ģ éĿ¢ +æ³ Į +è° Ĭ +åIJĮ æľŁ +æĬķ æłĩ +å¥ ´ +æĿ¥çľĭ çľĭ +èĦ ¾ +èŀ º +æŃ ī +çĽ ¯ +ç¨İ åĬ¡ +å» Ĭ +æİ © +æħ ¨ +çĽ ¼ +èĬ Ĵ +è® Ģ +æĮ £ +èĮ ħ +æĸ ¥ +æ¤ ħ +åΰ æĿ¥ +èijĹ ä½ľ +çĭ ± +äºĮ æīĭ +ä»İ æĿ¥ +çĸ ² +åºĬ ä¸Ĭ +æĸ° 浪 +æ³ Ħ +å¢ŀ å̼ +ä¸ Ľ +æļ ij +ä»İ ä¸ļ +æ· ĭ +å¤ļ æł· +æľ ´ +份 é¢Ŀ +æŀ £ +西 çľģ +æľ¬ è´¨ +æ·± æ·± +èī ĩ +ç» µ +产 å̼ +æ¼ ł +èħ » +çŃ Ľ +åİ Į +æģ Ń +å«Į çĸij +æĪ ¶ +æ» ŀ +èĨ Ģ +åĬ £ +座 è°Ī +常 æĢģ +çļĦ æĥħ +è¦ ½ +å¯ Ĥ +åĮ Ĩ +èĩ º +é¡ ¯ +çķ ı +éģ £ +åį ľ +çŃī å¥ĸ +è² ¬ +æº ¯ +é İ +çĤ¹ 头 +èĵ ¬ +æ± º +éħ ¬ +éģ Ĭ +è³ ¼ +註 åĨĬ +æľ¬ æĬ¥ +çµ ķ +æ´» æĢ§ +åħ ij +éĮ ¯ +åĨ ¶ +åĸ » +æº ĸ +èĤ ¢ +æº ĥ +æĹ ¬ +åī Ĭ +çIJĨ äºĭ +å± ł +æ² § +èļ Ģ +鼻 åŃIJ +为 æŃ¢ +常 å§Ķ +çµ Ĥ +éĬ · +çĭ Ģ +ä¾ £ +èĥ Ģ +èŃ ° +ç͍ 车 +åĻ ª +æŃ · +åį Ķ +åĪ ¹ +竣 æĺ¯ +é© Ĺ +èIJ Ŀ +çĻ « +çĹ « +æŃ § +å¼ Ĭ +åª ½ +çı Ĭ +è¡ · +éľ ī +åŁº çĿ£ +éļ ± +æ° ¨ +ç» ¸ +å°¼ æĸ¯ +çĥ ĺ +æľŁ åĨħ +è° ħ +éĽ ĩ +éļ Ļ +å ĸī +åī ¥ +çĹ ĺ +æĮ ½ +çĵ £ +æ¹ Ľ +æ¨ ± +æ¾ İ +æ¹ ĥ +åĨ¬ 奥 +æ£ µ +å® ° +åŀ Ĵ +æ§ ĭ +ä¾ Ī +èĮ Ħ +åĺ ¿ +èı ĩ +ç ĻĤ +åĬ ĥ +é į +èĶ ½ +çŀ Ń +æķ ŀ +ä¹ ĸ +éŁ § +è¾ ľ +æĩ Ī +ä½ £ +çŀ » +åŁ Ķ +èĪ ħ +å®ŀ äºĭ +é ¨ +å§ ¥ +çµ ¡ +åĺ » +çķ ¢ +æ²ĥ å°Ķ +è¿ Ħ +èĤ ĩ +æħ ij +ã § +ä ı +ð ł +ð¬ ĩ +ð« Ń +ð« IJ +ã ³ +© ½ +ð« ł +ã Ľ +ð¬ į +é ¿ +ð¬ Ĵ +ã Ļ +𬠤 +ð ¬´ +ð« ĸ +ð ¤ +ã ¬ +ä ² +ð« Ķ +ð« ļ +è¦ģ æ±Ĥ +ä¸Ģ äºĽ +å®ŀ çݰ +èĢĮ ä¸Ķ +åĽł æŃ¤ +çͱ äºİ +åħ³ äºİ +çĦ¶ åIJİ +æİ¨ åĬ¨ +ä¸Ģ æł· +æĮī çħ§ +è¿Ļæł· çļĦ +å½¢ æĪIJ +æľī äºĽ +æĽ´ åĬł +ç»ı è¿ĩ +建 è®® +æ²» çĸĹ +ä½ł 们 +æīį èĥ½ +ä¿ĥ è¿Ľ +åijĺ å·¥ +ä½ĵ éªĮ +èĪ ĩ +åģļ 好 +ä¿Ŀ è¯ģ +æķ´ 个 +æĺ¯ ä¸Ģ个 +éĩĩ ç͍ +çIJĨ 论 +æ¯Ķ å¦Ĥ +ä¸Ĭ çļĦ +æİ¨ èįIJ +çͳ 请 +天 空 +éĥ¨ èIJ½ +åįģ åĪĨ +æĿ¥ èĩª +ä¹ĭ éĹ´ +è°ĥ æķ´ +æ¯ı 天 +è°ĥ æŁ¥ +æĤ£ èĢħ +è¿ĩç¨ĭ ä¸Ń +é¦Ļ 港 +广 åijĬ +éĿ¢ 对 +满 è¶³ +éķ¿ æľŁ +è§Ħ èĮĥ +æķ´ ä½ĵ +æĶ¹ åıĺ +æĻº æħ§ +å¦Ī å¦Ī +å¦Ĥ ä»Ĭ +åIJĪ åIJĮ +éĥ½ ä¼ļ +åĦ¿ ç«¥ +åĩı å°ij +éŁ³ ä¹IJ +ç»ı 常 +ä¸Ĭ å¸Ĥ +ä¼ĺ ç§Ģ +çļĦ éĩįè¦ģ +ä¸Ģ æĿ¡ +æµ· å¤ĸ +åı¦ å¤ĸ +ä¸Ģ å®¶ +åİĭ åĬĽ +大 åŀĭ +çľĭ çĿĢ +åĪ Ģ +幸 ç¦ı +æİ¨ 广 +åIJ Ľ +å¾ IJ +æī¾ åΰ +äºİ æĺ¯ +èĩª 身 +ä¸Ģ ä½į +åľŁ åľ° +åĬł åħ¥ +æİ¢ ç´¢ +æ¢ ģ +主 åĬ¨ +å°± ä¸ļ +女 æĢ§ +çªģ çł´ +ä¸įåIJĮ çļĦ +è¿IJ è¾ĵ +èĩª çͱ +å±ħ æ°ij +æŃ¤ 次 +çļĦ æĹ¶éĹ´ +å®¶ éķ¿ +ä¸Ģ个 人 +æ£Ģ æµĭ +åĨħ éĥ¨ +广 å·ŀ +缴 æĴŃ +ä»İ èĢĮ +è´· 款 +åı¬ å¼Ģ +æĶ¹ éĢł +人 çĶŁ +å±ķ 示 +æ¯ı å¹´ +女 人 +çļĦ æĸ¹å¼ı +æķĪ çİĩ +å±± 举 +æ¸ł éģĵ +ä¼¼ ä¹İ +æ¡Ī ä»¶ +åĪ© çĽĬ +çľĭ çľĭ +å¿ĥ éĩĮ +ç»´ æĬ¤ +å®Ŀ å®Ŀ +ç½ij ä¸Ĭ +论 åĿĽ +å°± åı¯ä»¥ +ä¸į è¶³ +æģ¢ å¤į +å¸ĥ å±Ģ +è´¡ çĮ® +ä¸ĭ éĻį +æİĮ æı¡ +çļ® èĤ¤ +å·¥ åħ· +éĩį åºĨ +åĵģ è´¨ +æİ¨ åĩº +çĶ· 人 +æī¿ æĭħ +çªģ åĩº +èĢĮ è¨Ģ +æ² Ł +åįı è°ĥ +æĺ¯ ä»Ģä¹Ī +æ± ¤ +æĴ ij +çĭ¬ ç«ĭ +çݯ èĬĤ +æī© 大 +æ´ ª +æĿ ° +çĽ IJ +ä» ģ +æ¶ī åıĬ +èĢģ 人 +åį³ ä½¿ +åįĹ äº¬ +éħį åIJĪ +é¬ ¼ +çζ 亲 +ç½Ĺ æĸ¯ +å°ı åĮº +æķĻ æİĪ +åĨ³ çŃĸ +é¢Ħ 计 +æľ¬ 人 +ä¼ ¯ +ç« ¹ +åΰ åºķ +å¸Ĥ æ°ij +åĩº åı£ +éĩĩ è´Ń +æĢ» ç»ĵ +æŃ¦ æ±ī +åĬł 大 +广 举 +æµģ ç¨ĭ +人 åı£ +å¦Ĥæŀľ ä½ł +åĩº åİ» +åĩ ī +åĨľ æ°ij +çݰ 象 +åĬĽ 度 +ç»Ļ äºĪ +åħļ å§Ķ +è¯Ń è¨Ģ +线 ä¸Ĭ +æĢİ æł· +åĦ¿ åŃIJ +ç¡® å®ŀ +ä¹ĭ å¤ĸ +éĥ½ åľ¨ +èī ¾ +çļĦ æĥħåĨµ +éĩĮ çļĦ +åĽ´ ç»ķ +æĽ´å¤ļ çļĦ +ä¾Ŀ æ³ķ +åħ¬ åĽŃ +å®¶ éĩĮ +æ¯į 亲 +ä¸į åĨį +èĭ ¹ +æ³ķ éĻ¢ +飩 åĽ½ +缸 å½ĵ +ä¸į çŁ¥ +è¯Ħ ä¼° +ä¸į ç͍ +顺 åĪ© +éĩį è§Ĩ +è´¢ åĬ¡ +ä»ĸ åĢij +åıij è¡Į +ä¸ĵ éŨ +åħ· å¤ĩ +å¹¶ ä¸įæĺ¯ +è¶³ çIJĥ +é ŀĭ +åıij 表 +æ°¸ è¿ľ +èIJ¥ åħ» +éħį å¥Ĺ +æķ´ åIJĪ +è´ º +åĽŀ çŃĶ +æĶ¶ çĽĬ +ä¹Ł 许 +è» Ĭ +æİ¥ 触 +æĶ» åĩ» +åĽĽ å·Ŀ +æĢ§ èĥ½ +åĽŀ åΰ +èħ ° +ä¹Ł 没æľī +å¼ Ħ +设 ç«ĭ +éĺ² æİ§ +æĬĢ å·§ +éĢļ 常 +è´¢ æĶ¿ +éĥ¨ ç½² +åľº æĻ¯ +æ±Ł èĭı +表 è¾¾ +åĸ · +女 åĦ¿ +èĪ ¶ +çµ ¦ +ä¼ļ åijĺ +æĪĸ 许 +äº © +举 æĸ¹ +天 æ´¥ +è¿ij å¹´ +çľĭ æĿ¥ +æ¯Ķ ä¾ĭ +å² © +éĵ ľ +çİ » +å®ŀ éªĮ +æĢĿ ç»´ +æĭħ å¿ĥ +æ² Ī +身 è¾¹ +æ·± åĮĸ +ç²¾ åĩĨ +ç§ģ æľį +æ¶Ī éĺ² +åİ» äºĨ +ç»Ĩ èĥŀ +çIJĥ éĺŁ +æĺİ æĺŁ +é£Ł çī© +å¾Ī å¿« +让 ä½ł +ä¿¡ ç͍ +å͝ ä¸Ģ +åħ¶ å®ĥ +çŃī æĸ¹éĿ¢ +å¾ĭ å¸Ī +æŃ» 亡 +æ Ł³ +ä¸Ģ æī¹ +ä¸Ĭ 涨 +æľº åľº +å½¢ åĬ¿ +æĦ¿ æĦı +éĽĨ ä½ĵ +æĸ° åŀĭ +æį٠失 +æĽ ¸ +ä¸ĭ åįĪ +æ¯ı 次 +æĪIJ å°± +åħ¬ è·¯ +èĻ « +åĴ ± +西 å®ī +æľĢ ä½³ +ç§ij çłĶ +å¤į æĿĤ +æľº åύ +çα æĥħ +çħ§ çīĩ +å¹´ é¾Ħ +è³ĩ æĸĻ +ç² Ĺ +åĩĨ ç¡® +åĬł ä¸Ĭ +åĩº çīĪ +è° IJ +å®¶ å±ħ +èĥĮ æĻ¯ +ä¸Ģ 线 +äºĭ 项 +åĬ¨ ä½ľ +ç¥ ¥ +æĢ» ä½ĵ +æĪ¿ åŃIJ +ä¹Ł å°±æĺ¯ +大 æ¦Ĥ +é«ĺ æķĪ +åIJ ¹ +æİ ĪæĿĥ +éĻĦ è¿ij +æ¡Ī ä¾ĭ +éĹ ¹ +çΏ çΏ +彩 票 +æĢ Ĵ +举 æĬ¥ +æĻ® éģį +çķĻ ä¸ĭ +è¡£ æľį +æĹłè®º æĺ¯ +åħħ 满 +æ·± 度 +æ¡ ij +æĪª èĩ³ +带æĿ¥ çļĦ +éĻ µ +æĦŁ æĥħ +èµ ļ +åĵª äºĽ +æķ´ æĶ¹ +æĪIJ çĨŁ +å¨ ľ +é¼ » +çŁ Ľ +çĽ ¾ +好 好 +第 åĽĽ +åĨł åĨĽ +è´¢ å¯Į +æľĢ 好çļĦ +车 åŀĭ +éĸ Ģ +åį³ å°Ĩ +åĪĨ 为 +éĿĴ å²Ľ +纷 纷 +ä»Ĭ æĹ¥ +å¹³ è¡¡ +å¹³æĸ¹ ç±³ +éĤ£ ç§į +åĩº çĶŁ +éĿĴ æĺ¥ +人 群 +人 å·¥ +ä¹ĭ ä¸ĭ +æ¹ĸ åĮĹ +åľ¨ æŃ¤ +åįļ 士 +æĹ¶ åĪ» +æ²³ åĮĹ +æĶ¾ å¼ĥ +éĢļ éģĵ +森 æŀĹ +çĸ Ĩ +æķ ¸ +èĬ ³ +æīĵ åĩ» +æĽ ¹ +åĮĸ åѦ +æĥ³ 象 +ä¸ĩ 人 +è´¢ ç»ı +åħĥ ç´ł +ä¼ļ 计 +åħ¨ ä½ĵ +æĦ Ľ +é«ĺ ä¸Ń +æľº éģĩ +声 éŁ³ +æĹħ è¡Į +æµ © +æŁ ± +å°ij å¹´ +åĽ½ å¤ĸ +èijĹ åIJį +çĶŁ åŃĺ +å§ ľ +带 é¢Ĩ +é¢ľ èī² +ä¸Ĭ ä¸ĭ +产ä¸ļ éĵ¾ +æĽ´ 好çļĦ +å² Ń +ä¼ĺ æĥł +便 æĺ¯ +åħ§ 容 +ä¸Ģ åıª +çIJ ´ +梦 æĥ³ +ç§Ł èµģ +å¼Ģ åIJ¯ +è´Ń çī© +åĮħ åIJ« +åĪ© çİĩ +èµ· äºĨ +æľī åĬĽ +éĤ£ éĩĮ +审 æī¹ +对 æīĭ +çݰ éĩij +天 çĦ¶ +çĽ Ĵ +çĪ ½ +å¿ħ çĦ¶ +åĮĸ å·¥ +ä¸ĵ åĪ© +åķ ¡ +å¼Ģ å¿ĥ +人 ä½ĵ +éģĵ 士 +æĢģ 度 +空 è°ĥ +æĭĽ åķĨ +å§ » +第 äºĶ +æ£ Ĵ +ä¸Ģ ç³»åĪĹ +åį± æľº +转 åıĺ +åľº æīĢ +é¸ £ +æĪ¿ éĹ´ +éĢ ¼ +è¯ķ çĤ¹ +对 å¤ĸ +åĩº åı° +åľ¨ è¿Ļ +åİĤ å®¶ +å·¨ 大 +ç®Ģ ä»ĭ +çľĭ äºĨ +åħļ 建 +æĮĩ æĮ¥ +çŁ³ æ²¹ +ä¸į åı¯èĥ½ +èİ ² +ä¸į 太 +åĪĽ æĦı +第 ä¸Ģ个 +è´µ å·ŀ +è¿ĩ äºĨ +æľ¬ æĿ¥ +éģĵ å¾· +çŃĶ æ¡Ī +éĻ ¶ +ä¸Ģ è·¯ +èĤ ĸ +æ¸ħ æ´ģ +æľī æľº +åIJį åįķ +æĿ ± +åij¼ åIJ¸ +ä¸ Ī +ç¦ı 建 +è¯ķ éªĮ +å¼ķ åıij +ä¹Ł 没 +ä¸į ä½ı +çĨŁ æĤī +èIJ ¬ +ä¸į èī¯ +çł ĸ +èĩ´ åĬĽ +çѾ 订 +åIJ Ĭ +ä¾ ¯ +çĺ ¦ +å§ij å¨ĺ +æĸ ¤ +妻 åŃIJ +æĺ¥ èĬĤ +çĪ ¬ +æĽ Ŀ +çĥŃ æĥħ +éķ¿ æ²Ļ +èIJ¥ éĢł +éħ · +éĵ Ŀ +åŁºæľ¬ ä¸Ĭ +åij¨ åĽ´ +ä»Ģ 麼 +认 åı¯ +åĪĨ åŃIJ +ä¸Ģ æĸ¹éĿ¢ +è½ ´ +å¼ · +马 ä¸Ĭ +éĽ ¾ +èĩ £ +å° ¿ +çĶŁ æĦı +å®ī å¾½ +ç¥ŀ ç»ı +åĩº å¸Ń +èᝠåĵģ +çIJĨ çͱ +åįı åIJĮ +æµģ åĬ¨ +åıij åĬ¨ +åĿļ å®ļ +表 æĺİ +åIJİ éĿ¢ +ä¹ī åĬ¡ +å¦ ĸ +æľī åı¯èĥ½ +å¹´è½» 人 +大 éĻĨ +å² ³ +ä¸į èµ· +çŀ¬ éĹ´ +ä¸įå¾Ĺ ä¸į +çѾ 约 +åIJĪ æł¼ +åħļ æĶ¯éĥ¨ +æµİ åįĹ +便 åĪ© +éļı æĹ¶ +å¥ ī +ç§° 为 +产 æĿĥ +åIJ ķ +çĽ Ĩ +课 åłĤ +ç· ļ +æ£ ī +线 ä¸ĭ +èĩª è¡Į +举 æİª +åݦ éŨ +èĩª ä¿¡ +å½± è§Ĩ +ä» Ķ +çĶŁæ´» ä¸Ń +æĿĥ çĽĬ +çϽ èī² +å°± ä¸į +è¿Ľ å±ķ +æ¯ı æĹ¥ +ä¾Ľ ç»Ļ +æĿĥ åĪ© +æĹł æķ° +çIJĨ è´¢ +ä¾Ŀ æĹ§ +ä¸Ĭ åįĪ +è¯Ĩ åĪ« +çĽĪ åĪ© +çł Ĥ +许 åı¯ +åIJĮ äºĭ +åĺ Ľ +éģ ¸ +çĿĢ åĬĽ +éŨ åı£ +ä¸į å¤ļ +åħ¶ 次 +ç¢ § +çī© çIJĨ +åĨħ å¿ĥ +çϾ å§ĵ +æĢ» 绣 +å¹² åĩĢ +积 ç´¯ +åıį é¦Ī +æłij ç«ĭ +社 交 +ç§ © +åįģ ä¸Ģ +éĤ ĵ +驱 åĬ¨ +å±ķ è§Ī +èĪĴ éĢĤ +åŁº åĽł +å·® å¼Ĥ +转 让 +å°ı å§IJ +æł· åŃIJ +ç¿ Ķ +é«ĺ åħ´ +å½±åĵį åĬĽ +æīĭ ç»Ń +缸 åIJĮ +缸 åºĶ +æĻ Ĵ +è§ Ģ +å¸Ĥ å§Ķ +èĬ ¯ +å±ķ çݰ +åľ° çIJĥ +éĤ ª +ä¸Ģå®ļ çļĦ +åħģ 许 +ä¿¡ ä»» +æī ij +éĻ¢ æł¡ +ç®Ģ ç§° +åģļ æ³ķ +ä¹ĭ è·¯ +æĹĹ ä¸ĭ +èħ Ķ +æ¶Ī 失 +ä¸ĸçķĮ ä¸Ĭ +åŁİ 乡 +èĪŀ åı° +å¾Ī 大çļĦ +绣 çѹ +åħ¬ å¹³ +èĤ ¾ +çļĦ 好 +æ± ģ +çľ¼ åīį +éĽ £ +å¹ ½ +åħ± 产 +主 åĬŀ +å¤Ħ ç½ļ +åº Ļ +éģĵ çIJĨ +å¼ µ +æİ¥ çĿĢ +çĮ İ +çģ Į +çͱ æŃ¤ +人 åĬĽ +æµģ è¡Į +ä¾ ł +åı¯ä»¥ 说 +èĴ ĭ +å½¢ æĢģ +æĹ¥ åŃIJ +æ¼ Ĩ +çķĻ åѦ +缸 éĹľ +æľĢ å¤ļ +åĩŃ åĢŁ +åħ¬ 交 +æĮĸ æİĺ +æĿĤ å¿Ĺ +主 人 +éļľ ç¢į +æł¡ éķ¿ +æĸ¹ ä½į +ä¸Ĭ çıŃ +å¤ļ åħĥ +è ĥģ +éŃħ åĬĽ +èĮ Ĥ +åħħ ç͵ +强 大 +çĥ ¤ +å¥ĭ æĸĹ +å®ŀ ç͍ +éĺ ģ +ç»Ļ äºĨ +æľ¬ ç§ij +æł ĭ +æĭ ¨ +æķĻ ç»ĥ +éĥ½ çŁ¥éģĵ +æ¯ķä¸ļ çĶŁ +ç¢ Ĺ +åŀ Ĥ +è® ¼ +å®ģ æ³¢ +åѦ èĢħ +è°¢ è°¢ +åŁİ éķĩ +æĢİä¹Ī åĬŀ +éģ Ķ +æĪIJ 交 +æ½ľ åĬĽ +åį § +æĸ° å¼Ģ +éħį å¤ĩ +主 åĬĽ +åij³ éģĵ +çĥ Ĥ +é£ŀ è¡Į +å« ģ +大 大 +ç»Ļ 大家 +å¤ĸ éĿ¢ +éĨ ī +åıij è¨Ģ +æĹ© é¤IJ +åIJĦ èĩª +å® Ļ +èᣠèªī +æĬ« éľ² +é¡ ŀ +åĨħ çļĦ +èĤ ª +è¾ IJ +æ³ µ +æĬ Ľ +æĺŁ æľŁ +ä¸Ģ 带 +çĶŁ ç´ł +ç»ı éĶĢ +åĩ ¶ +åľ° ä¸Ĭ +åij½ è¿IJ +åĵ ² +ä¸Ĭ åİ» +æĸĩ çī© +è¯ ij +æĮ¯ åħ´ +éķ¿ æĹ¶éĹ´ +ç¥ Ń +åIJĪ èĤ¥ +è¿Ŀ è§Ħ +èģ ª +ä½İ äºİ +éĢĤ å½ĵ +æľī åºı +æľ¬ ç½ij +çķĻ è¨Ģ +æĥ³ æ³ķ +çѾ ç½² +å§ ļ +æĢ§ æł¼ +èĴĻ åı¤ +æŁ ı +åŀ « +åѦ åİĨ +ä»ħ ä»ħ +讲 è¯Ŀ +éĶ IJ +æĢ ĸ +åī ª +èĭ į +åIJ ĵ +强 çĥĪ +åģ¥ åħ¨ +çĸ ¯ +åı¤ 代 +å¥ Ī +ä¸į çĦ¶ +乡 éķĩ +æľĭåıĭ 们 +åĤ ħ +èģ ½ +个 æĢ§ +æ³ķ è§Ħ +å°ı éķĩ +çĶ» éĿ¢ +第 åħŃ +ç¶² è·¯ +åīį æĻ¯ +åIJ¬ 说 +ä¼ł åªĴ +æĿ¡ ä¾ĭ +åĪ« çļĦ +ä¸į æĩĤ +顾 éĹ® +强 度 +éĺ¿ éĩĮ +èµ° åĬ¿ +å¸ ½ +çļĦ ç¡® +åĮº åĪ« +éĮ ¢ +主 管 +ä¸Ģ çľĭ +æĸ ľ +åŃĺåľ¨ çļĦ +ä» ² +åᱠ害 +éĵ Ń +游æĪı ä¸Ń +éħ ± +é¾Ļ 头 +人 å¿ĥ +éĢĢ ä¼ij +æµı è§Ī +åĬ « +éĺ² æ²» +ç® Ń +å± Ī +è¾½ å®ģ +å£ ¤ +è¿İ æĿ¥ +éŀ į +ç͍ æĿ¥ +大 åľ° +ä» ° +éĢļ 讯 +å¼Ģ å·¥ +è£ ¤ +å¦Ĥ åIJĮ +éª ¤ +éĺŁ åijĺ +è½ © +ç¾İ æľ¯ +èĻ Ł +åIJĮ ä¸Ģ +åľ ĸ +书 æ³ķ +æīĵ åį° +åIJ« æľī +éĽĨ æĪIJ +éĹ · +å¸Ĥåľº ä¸Ĭ +æĹģ è¾¹ +åľ° æĿ¿ +产çĶŁ çļĦ +ç² ¤ +éĩį ç»Ħ +è¡Ģ æ¶² +çŃ ĭ +åĬŀ äºĭ +常è§ģ çļĦ +ä¸Ĭ åįĬå¹´ +å±ı å¹ķ +åIJī æŀĹ +å· © +åĸľ çα +ç¿ ł +ä¸ī ç§į +æ¡Ĩ æŀ¶ +举 èİŀ +çĶĺ èĤĥ +èĬ ¬ +åĽ¾ 书 +åĩ¤ åĩ° +æ°Ķ åĢĻ +å° ´ +å° ¬ +两 天 +è¾ħ 导 +åĢŁ 款 +æĹ¥ èµ· +æ´ Ĵ +ä¸Ģ 度 +è¹ Ī +æ½ Ń +æī ĩ +çĻ ľ +æĸ° åħ´ +åĤ ² +诸 å¤ļ +è´ ª +éĻ· åħ¥ +èĪ Ł +èĤº çĤİ +ä¸Ģ æł·çļĦ +åİ ĺ +åľ° çIJĨ +æĬķ æ³¨ +éļ Ĭ +åħī ä¼ı +ä¿Ŀ åģ¥ +åħ Ķ +åħ¬ åĬ¡ +æīĵ çł´ +çĶ· åŃ© +åĬ³ åĬ¡ +ä½ł ä¼ļ +ç͍ åľ° +æº ¢ +åıij è¾¾ +èĤ ļ +è¿ĩ äºİ +èĩ Ĥ +éĢĻ æ¨£ +è½» è½» +ä¸Ń åħ± +åIJĦ åĽ½ +åĶ ĩ +å®ŀ ä¹ł +èĻ ¾ +æ§ ½ +ä¸į ä¸Ĭ +åħį çĸ« +åįł æį® +å·¥ ä¼ļ +åĽ Ĭ +èĪª 天 +åı¯ çα +æĸĹ äºī +çĺ ¤ +å¦Ĥ æľī +éĽ ĸ +对 æĪij +åĩº ç§Ł +好 çľĭ +太 大 +æ°´ åĪ© +åĬ¿ åĬĽ +åħ¨ æ°ij +ç½ ¢ +èµ¢ å¾Ĺ +ç͵ ä¿¡ +车 éĹ´ +æĻĤ åĢĻ +å°ij æķ° +éĵ ¸ +åħ³ èģĶ +ä¸įä»ħ ä»ħ +为 æĤ¨ +åĴ ¸ +æľº åĬ¨ +è£ Ļ +åĵį åºĶ +éģ ł +è² · +ç© ´ +å¢ ħ +éĶ ¡ +çµ Ħ +çģ« è½¦ +è³ĩ è¨Ĭ +åĨ³ èµĽ +污 æ°´ +èª ŀ +å´ Ľ +ç´§ å¯Ĩ +缺 å°ij +å¤ļ 人 +æĢ» 书记 +éĶ Ī +èij Ľ +å¿ĺ è®° +éĻĮ çĶŁ +éķ¿ å¤§ +åħĪè¿Ľ çļĦ +ç¡ ħ +åıij æĺİ +å©´ åĦ¿ +æīİ å®ŀ +èĽĭ çϽ +ä¸Ģ çϾ +缮 åħī +æ ħĮ +åĬł æ²¹ +åIJ ŀ +ä¸Ģ 群 +ä¸Ń ä»ĭ +å¸ ĸ +å¿ Į +èģĮ èĥ½ +广 æĴŃ +çĽij å¯Ł +ç§ĺ å¯Ĩ +çĭ ® +è¿Ļ æĿ¡ +éĢ ¢ +æĢ ¨ +åįģ åħŃ +è© ¦ +说 åΰ +åĩĿ èģļ +æĮĩ 示 +æ° ¢ +å¼ ĺ +éĺ Ģ +æĸ © +éł ħ +ä¸Ģ å¼Ģå§ĭ +æİĴ è¡Į +åľ¨ æĪij +纪 å½ķ +æĬ Ħ +æł ª +说 æ³ķ +ä¸Ń èᝠ+好 å¤ļ +åıª ä¸įè¿ĩ +çķĻ åľ¨ +个 å°ıæĹ¶ +认 çŁ¥ +çķ « +è§ģ è¿ĩ +å°ı å¾® +ä½Ľ å±± +çľ ¾ +讲 è¿° +æ¢ ³ +ç§° åı· +æĹ¥ æĻļ +è¢ ĸ +åķ ¤ +æľª ç»ı +æľĢ æĹ© +æī® æ¼Ķ +è¡Ģ 管 +çº ± +æĥħ èĬĤ +第 ä¸ĥ +æį § +ä» Ĺ +æ¿Ģ çĥĪ +æĹł 线 +ä¸į 容æĺĵ +å¼Ģ å¹ķ +æĸ° çĶŁ +ä¸ĵ 注 +èij ± +åįĹ æµ· +çĩ Ł +èµ· ä¾Ĩ +æ´¾ åĩº +åĦ Ĵ +ä¾ ¨ +è¼ ĥ +åįļ è§Ī +éĢ ¾ +åĮ Ģ +ç»ıæµİ åѦ +æ¸ Ĺ +ä¿Ŀ èŃ· +çī º +çī ² +çİ « +çij ° +æľĢåIJİ ä¸Ģ +æĶ¿ åĬ¡ +æ§ Ľ +èĻķ çIJĨ +éļIJ æĤ£ +æī¿ åĮħ +æ¥ µ +æ¡ © +çĽ ² +导 åIJij +èĩ´ å¯Į +ç¼ Ĩ +æģĭ çα +ä¸į åĬ¨ +ç»Ļ 人 +å· ¢ +表 æĥħ +举 åįĹ +åĨħ å¤ĸ +è¾Ī åŃIJ +åı ī +åįļ ä¼ļ +åĬŁ æķĪ +æ¸ ´ +å± ¬ +æİĴ éϤ +éĢ Ľ +ä¸Ģ ä¼ļ +ä¸į å¼Ģ +å¼Ģ å¥ĸ +é»ij é¾Ļ +é»ijé¾Ļ æ±Ł +å¿« ä¸ī +度 åģĩ +åĿ ¤ +éĤ® ä»¶ +æĩ Ĵ +ä¾Ľ ç͵ +å» £ +好 è¯Ħ +ç§ĺ书 éķ¿ +æĪĺ åľº +好 å¥ĩ +ä¾µ æĿĥ +æĨ ¾ +æľĢ åĪĿ +æī¹ åıij +åİ ķ +è¼ ķ +æŀ ¯ +ä¸ļ åĨħ +è´Ń æĪ¿ +ä¸į åľ¨ +纪 å§Ķ +æīĢ éľĢ +å¸Ĥ éķ¿ +è³ ½ +å¼ķ æĵİ +çģµ éŃĤ +éĬ Ģ +æ» ¤ +çĿ IJ +å¤ļ 项 +åĽŀ 头 +èī ĺ +å¤į å·¥ +éĥ¨ ä»¶ +ç´§ ç´§ +æŁIJ ç§į +使 åħ¶ +æĸ° 人 +æŀ ļ +æ³ķ å®ļ +å·´ å·´ +æ¶µ çĽĸ +ç¨ » +æĭ ¾ +æĻ ķ +è½ ¿ +éĢļ è¡Į +åĵ Ģ +æ³ Ĭ +温 馨 +éĽĨ èģļ +çĨ Ļ +åĩ ij +åįģ ä¸ĥ +æ°Ķ æģ¯ +æıIJä¾Ľ çļĦ +æ³ ³ +奥 è¿IJ +çģ¾ å®³ +åĩĢ åĮĸ +è·¨ è¶Ĭ +åĵª æĢķ +éŁ ¿ +å¢ŀ æ·» +çĦ Ĭ +æ®ĭ çĸ¾ +ç¢ Į +æĤ Ķ +è§ģ è¯ģ +è¾ĸ åĮº +å¿ĥ èĦı +éļ § +åį ¸ +åı¯èĥ½ æĢ§ +æľī è¶£ +åī¯ ä¹¦è®° +åĮĸ å¦Ĩ +ä¿ Ĥ +æ£ ļ +éĨ ĩ +带 头 +éł Ī +追 ç©¶ +æij Ķ +è¿Ļ éĥ¨ +ä¸į 论 +ç¥ ¸ +å ³» +éģ ķ +çĶŁ èĤ² +å¤ ł +å¤ĸ 交 +è¯Ħ 为 +ä»İ å°ı +å°ı å°ı +é ¥¿ +æĴ ¼ +è·¨ å¢ĥ +被 åijĬ +åįĹ å®ģ +身 å¿ĥ +åĨį çĶŁ +æīĢ è¯´ +æĹ¶éĹ´ åĨħ +åĪĹ åħ¥ +éĿĴ æµ· +çα 好 +çª Ħ +èĪ Ī +è¿ĩ 渡 +æ¿ Ł +éĽ Ģ +审 è®® +åĽ½ èµĦ +æŃ¥ ä¼IJ +轨 éģĵ +ä¿¡ 念 +ä¸ī åĪĨ +çĨ ¬ +åѵ åĮĸ +ç¼ ł +éĥ Ĭ +èĪĴ æľį +纪 æ£Ģ +ä¸Ģä¸ĭ åŃIJ +鼻 話 +è² ł +éĴ ¥ +åĮ Ļ +çĹ ´ +è¶ ģ +ç» £ +çĪ µ +è½ ° +éª Ħ +å§ ¨ +æĭ ĺ +çĮ ´ +è® ¶ +è¿Ļ 座 +çį ¨ +æ·ĺ æ±° +çĹħ ä¾ĭ +æ²Ļ åıij +è§Ĩ 为 +头 æĿ¡ +å¿ħè¦ģ çļĦ +åı¯ è°ĵ +è¯Ŀ 说 +ç¯ Ħ +æĹ© çĤ¹ +æŀ¢ 纽 +ç¾ ¡ +çα åĽ½ +çªģ åıij +éĢ Ĭ +æ½ į +èᣠèĢĢ +èŁ ¹ +æ¦Ĥ çİĩ +å¾Ī ä¹ħ +æĥ ķ +è¨ ´ +åľĨ 满 +çļ ± +åĪĨ æ³Į +åħħ è¶³ +çľĭ æ³ķ +è¾ Ł +æĭ ¦ +æĭ © +对 åºĶ +为 æł¸å¿ĥ +èħ Ĭ +å¤ļ ä¹Ī +æµ ij +å®ı è§Ĥ +èĦ ĸ +åIJĪ èµĦ +çĶŁ 涯 +å®ŀ è´¨ +ä¼ĺ çĤ¹ +ç͍ æ°´ +寿 åij½ +æ² « +åIJ ģ +è© ¹ +åĽ½ éĺ² +å´ © +åĿ İ +èĨ ı +ä¸Ģ è½® +éģĹ äº§ +æ¹¾ åĮº +ç» İ +åįķ 纯 +æ¾ Ħ +åīį åĪĹ +身 å½± +é»ĺ é»ĺ +æį ī +çĴ ° +èı Ĭ +æĢ ľ +åħĭ æĢĿ +æĢ» å±Ģ +çĩĥ æĸĻ +ä¸ļ æĢģ +åIJĦ æł· +åĴ ½ +åĩº èī² +åĪĿ å¿ĥ +åı Ľ +çłĶ 讨 +è¡ « +åİĨ ç¨ĭ +ç¦ ½ +è¶³å¤Ł çļĦ +èį Ĩ +çľĭ å¾ħ +è´ © +åĨ³ å¿ĥ +è£ ¹ +å¸Ī èĮĥ +åŀ Ħ +æĿ ł +åĩ ¸ +çĬ¹ 豫 +çĥŃ è¡Ģ +åIJĪ ä¼Ļ +éħ µ +èIJ½ åľ¨ +åįł åľ° +è¡ ¬ +èĵ ī +æĦ ¤ +æ¸ Ĭ +åĪĨ æķ° +ç¬ij çĿĢ +太 å¹³ +çĤ « +æİ¨ ä»ĭ +æĸ¯ åĿ¦ +å½¢ 容 +æĵ Ĭ +æĦŁ åħ´è¶£ +åĨĽ 人 +åĩĮ æĻ¨ +对 çħ§ +åıij çĹħ +å· ¾ +èĪ ī +æª ¢ +ç¬ij äºĨ +ç¡® è¯Ĭ +è´Ł åĢº +壮 大 +æĪ ļ +äºĴ èģĶ +èª ² +èħ ¦ +æĹ ± +åıĹ æ¬¢è¿İ +åį ī +éĻ¢ 士 +æ© ¡ +ä¸Ģ 对 +è¾ ± +æ² Ĥ +åı² ä¸Ĭ +æIJ ı +å´ ĸ +代 è°¢ +ç£ · +é¡ ĺ +æµ ĩ +常 ç͍ +åį ij +åĩº åĽ½ +è¯ ł +稳 æŃ¥ +ç»ı 纪 +å¤ļ å¤ļ +æīĢ å¾Ĺ +为 主é¢ĺ +ä¸Ģ åĪĨ +æł ½ +é¡ § +çº ² +åĥ ħ +å£ ĵ +åĦ ª +ç¿ ° +æİ Ģ +人 为 +åª ³ +æ´ ½ +èĿ ¶ +å¤į åħ´ +ä¼ļ å½±åĵį +åIJĦ çķĮ +éĤ£ ä¸Ģ +é¢ ¤ +çĢ ı +çĢı 覽 +å¯ ŀ +åı¯ æĢķ +åį³ æĹ¶ +çķ ´ +ä¸ĭ åįĬå¹´ +ç¬Ķ è®° +éĻĦ åĬł +çĥŃ æ°´ +å¥ ¸ +ç£ ħ +æĿ ī +æ¸ħ åįİ +éĸ ± +ç° ¡ +å¤Ħ å¤Ħ +åIJĪ éĩij +æ²³ æµģ +ç´ ° +è´Ł éĿ¢ +çļĦ 羣å®ŀ +åύ 械 +èĴ IJ +西 äºļ +å· ħ +ç² ¹ +åİŁ æĸĩ +æŀ ķ +è¡Ģ åİĭ +åļ ´ +å¸ ĺ +åĨ Ģ +æĮ « +ç͵ è·¯ +å°ı ä¼Ļä¼´ +èĿ ´ +æľĢ å¿« +æĭ Į +å® ª +æĸ · +ç¿ ħ +åĴ ³ +åĹ ½ +ç¾ ŀ +躺 åľ¨ +èµĽ 车 +æ² IJ +éĻIJ 度 +为 ä¸Ģä½ĵ +èĴ ľ +å¹ « +æIJ ħ +åĭ ĭ +åī ĸ +纳 ç¨İ +éķ¿ æķĪ +ç½ ķ +åī¯ æľ¬ +ç© į +éĴ © +ç¹ ¼ +åĽ½ åľŁ +è¼ ī +ä¸į å¿ĺ +èѦ 示 +çģ ¿ +å¿ĥ å¾Ĺ +æĦ ļ +忽 çķ¥ +åĽŀ äºĭ +åįł æľī +æ· Ħ +çī ¡ +çĽij äºĭ +ç¿ ¡ +éĴĪ对 æĢ§ +çª ĥ +è£ ½ +èĨ Ŀ +ç³ Ł +港 æ¾³ +太 太 +æ¾ ¡ +ç»Ĩ åĮĸ +åĶ® åIJİ +å®ŀåľ¨ æĺ¯ +ç« £ +çį ² +å̾ åIJij +å¼ķ ç͍ +é¹ ħ +ç¬ij 容 +ä¹IJ è¶£ +æ°ij æĶ¿ +éŨ æĪ· +å± ģ +è¿· 失 +éĶ Į +å°ı 康 +åĭ ī +æ³ ¼ +ä¾ĭ åŃIJ +ä¸ī ä½į +å» ł +èĶ ĵ +广 éĺĶ +èĢ į +èĢģ èĻİ +åĭŁ éĽĨ +èĦļ æŃ¥ +æĭ ¯ +åŃĹ åı· +çĦ ° +é¢ ł +èļ Ĥ +èļ ģ +é£ ¯ +人 æĢ§ +æĴ ° +åİ ¢ +å±Ģ éĻIJ +æľª æĪIJ +åĵª åĦ¿ +大 åıij +ä¸į å®ļ +å¾ģ æ±Ĥ +éĥ µ +åĢº æĿĥ +çα ä½ł +èº ģ +ä»ħ ä¾Ľ +è¿ľ å¤Ħ +éĨ Ľ +åĥ µ +积æŀģ æĢ§ +æİ ¡ +åīį ä¸ī +äºİ ä¸Ģä½ĵ +çŀ Ħ +çĿ ģ +æ² ¸ +åħ± èµ¢ +éĢĢ å½¹ +è´Ŀ å°Ķ +æİ ı +æĪ ² +è¡ į +éĶ Ĥ +ä¸ĩ ä½Ļ +ç§ij åĪĽ +æ¼Ķ åͱ +欧 åħĥ +æ·¡ æ·¡ +éĿĴ å±± +èĹ Ŀ +ç» ½ +令 çīĮ +éĽĨ 群 +ä½ľ çī© +çĢ ij +å¤ ¯ +ç½ij 游 +åħ« 大 +éª ļ +èª ĵ +ä¼ļ å±ķ +åħļ åı² +æ£Ģå¯Ł éĻ¢ +åĸ ĺ +éĺ ± +èĢĮ åĩº +éĢļ 车 +éĴ ĵ +æĥħ 人 +æ¸ Ľ +ä¸Ń ç§ĭ +çĪ Ń +åıª åī© +æĺ Ķ +éĩİ çĶŁ +ç¡ « +èIJĿ åįľ +æĬµ æĬĹ +çĻ« çĹ« +éĻ Ģ +èĶ ļ +å¸ ľ +满 满 +èı ± +éļĨ éĩį +æĺŁ çº§ +æ½ ĩ +åħ¬ åħĥ +è° £ +æ¯Ķ äºļ +æ¡Į åŃIJ +èµ £ +è² ¼ +æĦ¿ æľĽ +é¡ ½ +æ´¾ éģ£ +ç¥ Ľ +åª ļ +éĺ ľ +èij « +èĬ ¦ +æ³ » +å¡ Į +çĭ Ń +å»ī æĶ¿ +å¥ij æľº +æĹĹ èΰ +æĥ « +严 åİī +åıĭ æĥħ +å¦ Ĭ +å¨ ł +åĵª å®¶ +èĨ ¨ +è¶ Ł +æĮ ª +èĻ IJ +é łģ +çŀ © +éº Ł +ç¨ £ +èģĶ éĢļ +åı ® +çİĭ èĢħ +ä¸į ç¡®å®ļ +ç ijľ +è° İ +çī¢ è®° +ç¢ ¼ +æĬ¤ èĤ¤ +é¡ · +çĦ ķ +åģļ 强 +éļ± ç§ģ +éļ±ç§ģ æ¬Ĭ +åıĹ å®³ +ä¸į çͱ +çĥ ¹ +é¥ ª +é© ³ +ä¼ ½ +ä¸Ŀ 绸 +è¥ Ħ +åįģ ä½Ļ +éº Ĺ +æ¬Ĭ åĪ© +èģ ŀ +åı¤ èĢģ +éģ ı +åIJĦ å¼ı +å°± è¡Į +åħ¥ å¢ĥ +ç ĥģ +èľ ĺ +èĽ Ľ +çº ¬ +çŁ « +è» Ł +æ´Ĺ è¡£ +æĦ § +é¢Ħ æ¡Ī +éľ Ĩ +æ·± åİļ +éĺ¿ æĭī +åĨĻ åŃĹ +åį ¦ +éķ Ģ +模 æł· +åĤ į +æIJ į +èĸ ¯ +åł ħ +åħ¬ 积 +è¨ İ +ä¼ł æŁĵ +æ¯ ¯ +çIJĨ å·¥ +åĨ· éĵ¾ +ç«ĭ æĸ¹ +æ¢ Ń +åľ£ è¯ŀ +综 èīº +çİ© ç¬ij +æĥ³ ä¸įåΰ +æijĩ 头 +æ· ¹ +åģĩ æĹ¥ +åĢ ĺ +èĢ ½ +èİ ĵ +åŁ · +èĩª è´¸ +åįĬ 天 +æª Ķ +æ¾İ æ¹ĥ +éķ ij +ä¸ « +éĩĮ ç¨ĭ +å¼Ģ èįĴ +èı ı +å®Ŀ è´µ +èŃ ¬ +åķ Ł +æŁ ł +æª ¬ +é© Ń +æ± Ľ +çĨĬ çĮ« +èķ ī +éļı ä¹ĭ +å± ij +è¾ĥ 强 +èĥ ³ +èĨ Ĭ +éĿĻ éĿĻ +åĴ ª +æĭĽ åij¼ +代 è¨Ģ +ä¿¡ ç®± +è£ħ éħį +æĤ į +åįķ 车 +èIJ İ +å¤ļ 彩 +éĻ ¸ +ä»İ 严 +æ© Ħ +æ¦ Ħ +éĢ ® +éĩĮ æĸ¯ +å§¿ æĢģ +太 æŀģ +éĩ Ŀ +æº ī +è¿ Ń +ç§ ¸ +ç§ Ĩ +å·¥ å§Ķ +æ± ķ +èģ Ĩ +ä½ ¬ +ç¼ ħ +çĶ ¸ +åī¯ å±Ģéķ¿ +éĹ º +èª ¤ +è¤ IJ +ä¸į éĻIJ +èħ ķ +åij ķ +çŁ ¶ +åĨľ å®¶ +管 å§Ķä¼ļ +é¥ º +èĬ ľ +æ¾ Ī +è© ¢ +å¨ģ å°¼æĸ¯ +ä½ķ åĨµ +å°ı ä¼Ļ +奢 ä¾Ī +è¿Ļ ç¯ĩ +è¯ µ +竳 ç¨ĭ +ç´ Ģ +éIJ ĺ +éĤ ¢ +ç³ Ļ +ç¼ Ģ +ä¹ Ĵ +ä¹ ĵ +çī¢ åĽº +åĿ ŀ +å¼ Ī +ä¾ĭ å¤ĸ +å» ³ +è§Ħ 竳 +èĬ Ļ +ç¯ · +èº ¯ +æł Ī +åĿļ å®ŀ +åŁº 建 +çĿĢ çľ¼ +ç· ´ +èij © +ç¼ ļ +æ¦ Ĩ +主 åĭķ +ç¥ Ģ +äºĴ éĢļ +å°¤ 为 +å® Ľ +éª ¼ +æ± ² +ä¾ ĥ +æĤł ä¹ħ +æij § +æĭ ĩ +é« ĵ +éº Ĵ +éĻ Ľ +æŀ ¸ +æĿ ŀ +è´ ¬ +å°ı é¾Ļ +åĵ ® +èĵ¬ åĭĥ +åĮ Ī +çķľ çī§ +å¨ © +个 å¤ļ +æ² ¥ +æĺ § +çĦ ļ +æĬij éĥģ +çĸ ¡ +èĺ ij +éģİ ç¨ĭ +æ© ± +éĿ ĵ +大 çIJĨ +é« ¦ +åĪĨ 辨 +æ¸ ¤ +çĸ ¤ +åĬ¨ èĥ½ +å¼ł å®¶ +ä¸ĩ åįĥ +æ» ¥ +é¥ ¥ +åºŁ å¼ĥ +å¸ ³ +æ¼ ³ +è± IJ +ä» ij +å« ī +å¦ Ĵ +çŀ Ĵ +è¡ ħ +çĭ ¸ +å¾ģ ç¨ĭ +éĤ ¯ +éĥ ¸ +ç¥ Ī +ç¥ · +è¶ ´ +ç»ĵæŀĦ æĢ§ +è§Ĩ åIJ¬ +è¬ Ŀ +çĴ Ģ +çĴ ¨ +åĩº å¤Ħ +è¯ Ģ +å¾ ĺ +å¾ Ĭ +çľ ¨ +åĸ ĩ +åı Ń +åĺ ² +çķ ¸ +å¹² äºĭ +æļ § +æ² Ľ +åĦ Ħ +å» ĵ +åİ¿ éķ¿ +èĥ ļ +çIJ ¢ +çŃ · +éĩ ĭ +ä¾ ® +åIJ © +åĴ IJ +åĮ ¿ +æĬ¬ èµ· +æ³ £ +æ¶ ¤ +éº ½ +æĽ Ļ +åī¯ éĻ¢éķ¿ +åħļ åĴĮ +æķ£ åıij +润 æ»ij +åĵ º +æĥ ¬ +漫 éķ¿ +ä¸į æĩĪ +åŁ ł +åĹ ĵ +èĢģ çĪ· +è® ½ +æĪĺ ç»ĦåIJĪ +æ£ ł +åħ¨ åŁŁ +èł ¢ +è¯ ¡ +åīį çŀ» +æķ Ľ +ä¸Ģ å°ģ +å¹ Ĥ +èİ Ĩ +è¯Ŀ è¯Ń +ç»Ĩ åĪĻ +å± ¿ +åµ Į +éĢ į +åĺ ± +æ¸ ² +çĥ ¯ +çĿ ¹ +é¦ Ĵ +èħ ¥ +æĬĹ åĩ» +çĿ « +èį Ķ +éļ İ +æ³ī æ°´ +è¬ Ĥ +ç Ĥ¬ +åĩı æİĴ +è¸ Ĭ +è ·» +æ· Į +éľ ¾ +å¥ĩ 纳 +å¯ Ŀ +æ¤ İ +æŁ ¬ +æĸ¯ åŁº +åħ¬ ç«ĭ +è¨ ĵ +é£ Ļ +é© ¿ +åĤ µ +èĽ Ļ +ç¯ĩ 竳 +åĪĨ æĶ¯ +ä¸Ĭ å¹´ +çŃ Ŀ +ç¼ ¤ +èĢģ æĹ§ +åĻ ¬ +æľ ¦ +èĥ § +æ¶Ī è²» +æĵ Ķ +æ¦ ´ +æ¿ Ĵ +ç³ ¯ +æ³ ¸ +æį Ĩ +ç» ļ +èµ İ +çIJ IJ +èµ Ĥ +æħ ® +æ² Į +çĦ Ļ +æĴŃ æĬ¥ +æ· ĩ +åĪĩ åħ¥ +çij ķ +çĸ µ +éģ ´ +ç¨ ļ +ç© © +èŀ ĥ +æ£ ķ +æĨ § +æĨ ¬ +ä¼ º +æ¯ Ĺ +æį į +æĬ ī +ç´ Ĭ +å¼ Ľ +æĭ Ń +æĹı èĩªæ²» +åĿ · +ç« ¶ +è© ³ +è¿Ħ ä»Ĭ +è° ´ +çŀŃ è§£ +æŁ ¿ +é¢ Ĭ +ç° § +çĥŁ èĬ± +ä¾ ¥ +çĿ ¦ +éħ Ŀ +æ° ĵ +çIJ ī +å§ Ĭ +æ² ® +æħ · +èľ ķ +çij ļ +éĩĩ çŁ¿ +åł ° +åºķ èķ´ +èĨ ³ +è¾ ķ +éŁ Ń +åĴ Ļ +ç² ½ +åī Ķ +æ² ¦ +èĤ ´ +éķ ¶ +æĺ ¼ +è¾ Ĺ +å© ª +åĮ ® +æĸ ĵ +æ± ¶ +éĥ ´ +éł » +çª Ĵ +è¢ ± +åĽ ± +èĢ ĺ +è ļĮ +çĭ Ļ +çĹ ¹ +ç¥ ī +æı ® +æ· Ĩ +ç£ ĭ +éĺ ª +æ « +ã ¸ +Ļ ¶ +ã ij +𣠲 +ä ¢ +ã Ń +𬠨 +ð¬ Ģ +𬠮 +𬠯 +ð¬ ľ +𪠨 +ð« Ĺ +ð¬ Ĭ +𬠱 +ð¬ Ł +ä İ +ð ¡ +ä ĥ +ã ł +ð © +ð© ¾ +𬠺 +ð¬ Ļ +ãĢ Ķ +ãĢ ķ +çļĦ æĹ¶åĢĻ +æľīéĻIJ åħ¬åı¸ +ä¹ĭ åIJİ +ä¸ļ åĬ¡ +åķ Ĭ +èϽ çĦ¶ +æĭ¥ æľī +äºĴ èģĶç½ij +éĤ£ äºĽ +ä½ł çļĦ +åĨ³ å®ļ +éϤ äºĨ +åĽ¢ éĺŁ +åı¯ æĺ¯ +以 åIJİ +社 åĮº +çļĦ éĹ®é¢ĺ +å¹¶ ä¸Ķ +æķĻ å¸Ī +å°± ä¼ļ +天空 éĥ¨èIJ½ +æľĢ ç»Ī +å½ĵ çĦ¶ +ä¹Ł æľī +ç¡® ä¿Ŀ +æĥ³ è¦ģ +è´Ń ä¹° +人 çļĦ +åIJ ´ +çļĦ åıijå±ķ +ä¸į çŁ¥éģĵ +软 ä»¶ +æĪij们 çļĦ +çζ æ¯į +åī ij +èĢĮ æĺ¯ +å®ī æİĴ +åIJİ æĿ¥ +çļĦ åľ°æĸ¹ +èµ µ +èĢĥ è¯ķ +çªģ çĦ¶ +ä¸Ģå®ļ è¦ģ +åζ ä½ľ +è¯Ħ ä»· +åħį è´¹ +è´¹ ç͍ +绣 ä¸Ģ +çĦ¶ èĢĮ +è¿Ļ 次 +éĿĴ å¹´ +人 ç±» +äº ¦ +让 人 +è´Łè´£ 人 +éĩĩ åıĸ +çļĦ äºĭæĥħ +ä¹Ł ä¼ļ +车 è¾Ĩ +æĽ´ æĺ¯ +强 åĮĸ +æĪij åĢij +以 åīį +ä¼ĺ åĮĸ +å§Ķåijĺ ä¼ļ +åĽ° éļ¾ +å¹´ 度 +ä½į äºİ +æĮĩ åĩº +åĨį æ¬¡ +åĬŀ çIJĨ +æ¯ı 个 +对 æĸ¹ +è¿Ľè¡Į äºĨ +æľĢ é«ĺ +课 ç¨ĭ +身 ä¸Ĭ +æĽ¾ ç»ı +åĮ» çĶŁ +å®ī è£ħ +æľ ± +è¿IJ è¡Į +åıĮ æĸ¹ +æľĢ 大çļĦ +æŀĦ 建 +è¿ŀ ç»Ń +çļĦ å°ı +她 çļĦ +çŃī çŃī +æĶ¹ åĸĦ +åIJĦ ç±» +éģĩ åΰ +æľī çĿĢ +人 çī© +æĢ» æĺ¯ +è¿ħ éĢŁ +åζ å®ļ +å®ĥ 们 +å®ĺ ç½ij +è¿ĺ è¦ģ +ç»Ī äºİ +æĪ¿ åľ°äº§ +è¯ģ æĺİ +èĤ¡ 票 +åºĶ å½ĵ +èĭ± åĽ½ +è¿IJ ç͍ +æľĢ æĸ° +享 åıĹ +让 æĪij +æĻļ ä¸Ĭ +å¾ ŀ +å°ı 说 +å°¤åħ¶ æĺ¯ +è®Ń ç»ĥ +åħ¨ å¸Ĥ +æĮij æĪĺ +æľī çĤ¹ +带 çĿĢ +çļĦ ä¸ľè¥¿ +é£İ æł¼ +é»Ħ éĩij +å¼ķ 导 +æŃ¤ å¤ĸ +æľĢ è¿ij +追 æ±Ĥ +强 è°ĥ +ä¹Ł åı¯ä»¥ +æĦŁ åΰ +èĩª æĪij +çī¹åĪ« æĺ¯ +æĪIJ éĥ½ +éĢIJ æ¸IJ +å¿« ä¹IJ +ä¹ĭ ä¸Ń +æĬķèµĦ èĢħ +ä»ĸ们 çļĦ +æ° ı +å·¥ä½ľ 人åijĺ +äºĨ ä¸Ģ个 +åķ ¦ +ä¸Ģ åĢĭ +åŁº å±Ĥ +æ²Ł éĢļ +第ä¸Ģ 次 +å¹¶ 没æľī +çļĦ å·¥ä½ľ +åľ¨ è¿ĻéĩĮ +æŀ ª +æĶ¯ æĴij +æĹ¶ å°ļ +æĿ¥ åΰ +æĶ¶ è´Ń +éĿ© åij½ +æĺ¯ ä¸įæĺ¯ +讨 论 +ä¸ļ 绩 +å°± èĥ½ +ç«ĭ åį³ +è¡Ĺ éģĵ +åľ¨ ä¸Ģèµ· +æľĪ 份 +é«ĺ 端 +å¾Ī éļ¾ +ä¿Ħ ç½Ĺæĸ¯ +æīĭ 段 +åģļ åĩº +ä¼Ĺ å¤ļ +å®ŀ è¡Į +æīĵ å¼Ģ +游 客 +ä¾Ŀ çĦ¶ +å°± åĥı +离 å¼Ģ +说 éģĵ +æĸ° èĥ½æºIJ +æº ª +äº ķ +令 人 +ä¸Ģ åľº +æĪij æĥ³ +两 人 +èĩ³ å°ij +çļĦ çĶŁæ´» +æĺ¯ 个 +èĭ± è¯Ń +æ²Ĵ æľī +æĢĿ èĢĥ +éĻIJ åζ +åı° æ¹¾ +ä¸Ģ æĹ¦ +çļĦ ä¸Ģ个 +é«ĺ 级 +åĬŀåħ¬ 室 +å¾· åĽ½ +æĪij å°± +å®ļ ä½į +éĢĤ åºĶ +æĮĩ æłĩ +åħ¨ çľģ +ä¸Ĭ è¿° +å®ĥ çļĦ +åĽŀ å®¶ +欧 æ´² +éĵģ è·¯ +é¼ĵ åĬ± +çļĦ å½±åĵį +é«ĺ æł¡ +天 ä¸ĭ +é«ĺ è´¨éĩı +æĿŃ å·ŀ +èµĦ 讯 +æĶ¾ åľ¨ +æľī ä¸Ģ个 +å°± è¦ģ +ä¸Ĭ éĿ¢ +è§£ éĩĬ +éĢIJ æŃ¥ +å°½ 管 +æľī ä»Ģä¹Ī +çļĦ äºĭ +çĻ» è®° +人æ°ij å¸ģ +è§Ĥ ä¼Ĺ +è§Ĥ å¯Ł +ç͵ èĦij +çļĦ åIJĮæĹ¶ +ä½ľ ä¸ļ +宣 å¸ĥ +çļĦ ä½ľç͍ +åĽŀ æĿ¥ +éļ¾ ä»¥ +æīĢæľī çļĦ +å°ı åѦ +æıIJ åīį +æ¤į çī© +åĩ ¯ +ä¸Ĭ äºĨ +å°± åľ¨ +åħĪ åIJİ +æīĭ æľ¯ +éĥ Ń +éĿ¢ åīį +æ¯ķ 竣 +äºĮ æĺ¯ +红 èī² +éĺ³ åħī +èĭ¹ æŀľ +å¾Īå¤ļ 人 +ç»Ļ æĪij +åĵ ¦ +çľ¼ çĿĽ +éł Ń +ä¸Ģ æĺ¯ +åıijå±ķ çļĦ +åıį åºĶ +æĪ¿ å±ĭ +æľŁ å¾ħ +ç§į æ¤į +æĸĩ åѦ +åį³ åı¯ +é¦ĸ 次 +èĭ± éĽĦ +å¤ļ 次 +åĮħ è£ħ +æ²³ åįĹ +ä¹ĭéĹ´ çļĦ +ä»į çĦ¶ +åIJ¬ åΰ +èij£äºĭ éķ¿ +è§Ħ åĪĻ +ä¸Ģ 份 +大 ä¼Ĺ +使 å¾Ĺ +è¿Ľ åı£ +ä¸Ģ çīĩ +æĢ§ çļĦ +çļĦ 大 +æĪij æĺ¯ +äºĴ åĬ¨ +æ° £ +çļ Ĩ +åħ¬åı¸ çļĦ +ä¸Ģ è¾¹ +åıĬ åħ¶ +èī¯ å¥½çļĦ +æĭĵ å±ķ +å½ĵ å¹´ +广 åľº +åģļ äºĨ +åŁº äºİ +æıIJ éĨĴ +åħĦ å¼Ł +èĢģ æĿ¿ +è¿ij æĹ¥ +çĬ¶ åĨµ +注 éĩį +åĪļ åĪļ +è°ĥ çłĶ +å¿ĥ ä¸Ń +æĬĬ æı¡ +éļı åIJİ +ä¸į å¤Ł +åĪĽ ä½ľ +ç«Ļ åľ¨ +缸 äºĴ +çĸ«æĥħ éĺ²æİ§ +å¹´ 代 +带 åĬ¨ +伤 害 +竣 çĦ¶ +å¼ķ è¿Ľ +ç´¯ 计 +让 æĪij们 +åĽŀ æĶ¶ +æĬ¥ åIJį +åĬ© åĬĽ +èģĶ çĽŁ +çŃĸ çķ¥ +åij¨ è¾¹ +åĭ Ĵ +è¿ĺ åľ¨ +æµģ éĩı +寻 æī¾ +ç͵ åĬĽ +èι èζ +è¿ĺ èĥ½ +æĭħ ä»» +çļĦæĥħåĨµ ä¸ĭ +çļĦ åİŁåĽł +缺 ä¹ı +çIJĥ åijĺ +å²ģ çļĦ +çĶ· åŃIJ +å·¥ èµĦ +è¿ijå¹´ æĿ¥ +åij Ģ +æıIJä¾Ľ äºĨ +她 们 +å®¶ åħ· +çĩ ķ +è½» æĿ¾ +æł¡ åĽŃ +èĢĥ æł¸ +åį± éĻ© +åħļ ç»Ħç»ĩ +æĢ» ç»ıçIJĨ +çļĦ æĸ° +çİ» çĴĥ +è¿Ļ ä½į +对 æŃ¤ +å®¶ 人 +çļĦ è¦ģæ±Ĥ +温 度 +æĮĩ æķ° +缴 åΰ +æŃ¤ æĹ¶ +æ¹ĸ åįĹ +éĥ½ è¦ģ +ä½ľ åĩº +åIJĦ ä½į +èĢĥ çĶŁ +ä¾Ŀ æį® +说 è¯Ŀ +æĪij ä¹Ł +å·¥ åİĤ +åıĺ æĪIJ +ä»ĸ 人 +æĪij è§īå¾Ĺ +åIJĦ 级 +ä¼łå¥ĩ ç§ģæľį +ä¸Ĭ åįĩ +好 åĥı +åĬł éĢŁ +äºĮ åįģ +è¢ ģ +è£ħ 饰 +éĥ½ èĥ½ +ä¸Ģ å¼ł +åĬ¨ æĢģ +å¹´ çļĦ +è¿Ļ å°±æĺ¯ +ä¹Ł è¦ģ +èµĦ æł¼ +æĪĺ äºī +æĦŁ è°¢ +åŁ¹ èĤ² +天 æ°Ķ +女 士 +åı¯èĥ½ ä¼ļ +çļĦ 产åĵģ +ä¹Ł å°± +主è¦ģ æĺ¯ +åĪº æ¿Ģ +ç»Ļ ä½ł +大 æķ°æį® +åĮ» åѦ +åĪ ¤æĸŃ +ä»ĸ 说 +表 æ¼Ķ +äºļ æ´² +ä¸ĵ é¢ĺ +ç«ŀäºī åĬĽ +éĤ£ æł· +å±ķ å¼Ģ +å¹³ æĹ¶ +æİ¥ ä¸ĭæĿ¥ +æī¿ 诺 +æ³ķ åĽ½ +åħ³ å¿ĥ +ä¼ļ æľī +éĤĢ è¯· +é¢Ħ éĺ² +对 æİ¥ +好 äºĨ +åĴ± 们 +çļĦ æĦŁè§ī +æĢĿ è·¯ +éĥ½ 没æľī +çļĦ æĸ¹æ³ķ +女 åŃIJ +åı¸ æ³ķ +è¿ĺ ä¼ļ +è¶ĬæĿ¥è¶Ĭ å¤ļ +åĽł çĤº +æµ· åįĹ +人 æķ° +å°Ĩ ä¼ļ +ä¸ļ 主 +é¤IJ 饮 +å±ħ ä½ı +åıij åĩº +è¿ij æľŁ +å¼ķ é¢Ĩ +æľºåύ 人 +åĩºæĿ¥ çļĦ +çľĭ è§ģ +ä¿ Ĭ +让 ä»ĸ +ä¸į æĥ³ +å·¥ä½ľ çļĦ +è¡¥ åħħ +æµ ħ +çī¹ å¾ģ +ä¸Ĭå¸Ĥ åħ¬åı¸ +ç¾İ é£Ł +广 西 +æ¯ı ä¸Ģ个 +èIJ½ åľ° +åĵģ ç§į +åĴĮ è°IJ +å½» åºķ +é«ĺ èĢĥ +æĺ¨ 天 +åīį å¾Ģ +çĽij æµĭ +çϾ 度 +åľ¨ ä¸ŃåĽ½ +çļĦ éľĢæ±Ĥ +亿 ç¾İåħĥ +åѦ æľ¯ +æĶ¶ åΰ +æĿ¿ åĿĹ +ä¸Ģ 段 +æŀĦ æĪIJ +ä¼ģä¸ļ çļĦ +表 éĿ¢ +æķ´ çIJĨ +ç»ĵ å©ļ +人 å®¶ +åģľ æŃ¢ +åѦ ç§ij +æĺ¾ å¾Ĺ +ä¼ij æģ¯ +é¢Ħ æľŁ +æĪĸ æĺ¯ +çļĦ 主è¦ģ +åºĶ 对 +èµ° äºĨ +ä¸Ń éĹ´ +èµ° è¿Ľ +åijĪ çݰ +æIJŃ éħį +é¹ ı +æĺ¯ åĽłä¸º +æĥħ 绪 +å®ļ æľŁ +社ä¼ļ 主ä¹ī +çŃī 级 +磼 çĽ¾ +é£ŀ æľº +èĩ³ ä»Ĭ +æĶ¶ éĽĨ +çļĦ æķħäºĭ +åĪĩ å®ŀ +å®ŀçݰ äºĨ +å½¢ æĪIJäºĨ +åįĹ æĸ¹ +ä¸Ń åѦ +æµ· æ´ĭ +åIJ¦ åĪĻ +æĭį æijĦ +大åѦ çĶŁ +åĩºçݰ äºĨ +æĦı å¤ĸ +ä¹Ł èĥ½ +çļĦ èĥ½åĬĽ +åĿIJ åľ¨ +åĪĻ æĺ¯ +èĢĥ å¯Ł +å°Ĭ éĩį +éĺ² æŃ¢ +ç´§ å¼ł +读 书 +åĩº è¡Į +å°± æľī +å±¥ è¡Į +çݰ代 åĮĸ +åĽ½ åĬ¡ +åĽ½åĬ¡ éĻ¢ +ç»´ ä¿® +åİŁ åĪĽ +æĺ¯ æĮĩ +ä¼ij éĹ² +çĤ ® +æĸ° æĹ¶ä»£ +éĢĻ åĢĭ +ä¸į æķ¢ +å®Į ç¾İ +ç»Ĩ èĬĤ +éŃ ı +èͬ èıľ +é¢Ĩ导 çıŃåŃIJ +è¶ħ 级 +è¡Į æĥħ +人工 æĻºèĥ½ +åį° åº¦ +åŁºç¡Ģ 设æĸ½ +åıĪ æĺ¯ +èᝠçī© +åIJ¸ æĶ¶ +åį´ æĺ¯ +éĥ İ +å¥ĸ åĬ± +çļĦ æľĭåıĭ +ä¿Ŀ çķĻ +è§Ħ å¾ĭ +æĸ° çĸĨ +è¿ĺ åı¯ä»¥ +æİ¥ è¿ij +æŃ¤ åīį +æī¹ åĩĨ +æĢİä¹Ī æł· +çļĦ ä½įç½® +ä¸Ģ åĿĹ +æĭĴ ç»Ŀ +顾 客 +ä¹Ł åľ¨ +ä¸Ģ çĶŁ +éĥ¨ éĺŁ +å¹´ åīį +æĸ¹éĿ¢ çļĦ +å°Ŀ è¯ķ +羣æŃ£ çļĦ +ç¦ģ æŃ¢ +è¿ĺ 没æľī +æ°ij çĶŁ +èµ° åIJij +èĦ¸ ä¸Ĭ +å½ĵ 天 +éĽĨåĽ¢ åħ¬åı¸ +çļĦä¸Ģ ç§į +西 æĸ¹ +åĽŀ åºĶ +ä¸Ģ 声 +常 常 +æıIJ åΰ +èħ¾ 讯 +æľį è£ħ +为 ä½ķ +äºij åįĹ +å°± ç®Ĺ +ä¼ł æī¿ +åıį èĢĮ +ä¸ĩ åIJ¨ +è´¢ 产 +å¦Ĥ ä¸ĭ +æĹ¥ åīį +åİŁ æľ¬ +æľĢ éĩįè¦ģçļĦ +认 è¯ģ +ä¸Ģ éģĵ +ä¿¡æģ¯ åĮĸ +å¾Ĺ åΰäºĨ +é̲ è¡Į +æĪij è¦ģ +éĢļ ä¿¡ +室 åĨħ +èµļ éĴ± +æĶ¶ èĹı +è§£åĨ³ æĸ¹æ¡Ī +æĪ¿ 产 +çĭ ¼ +æ´» åĬĽ +ç»ıæµİ åıijå±ķ +çŃī å¾ħ +ä¹Ł å¾Ī +åĿ ij +å¾Ī 好çļĦ +éļ¾ åº¦ +ä¸į å¦Ĥ +人æ°ij æĶ¿åºľ +åĩº åıij +åīį æľŁ +æ¼Ķ åijĺ +女 çĶŁ +èģļ çĦ¦ +审 计 +é¢Ħ æµĭ +ä¾Ŀ æīĺ +äºĶ å¹´ +è¡¥ è´´ +æ¸ħ æĻ° +éª Ĥ +çľĭ èµ·æĿ¥ +çļĦ åŃ©åŃIJ +é¢ij éģĵ +ä½ı å®ħ +éĿ¢ åIJij +æľĢ ä½İ +æĹ¢ çĦ¶ +ä¸Ģ å¥Ĺ +æķ° åѦ +群 ä½ĵ +åĮĹ京 å¸Ĥ +å±ħ çĦ¶ +æ°Ľ åĽ´ +éĢĶ å¾Ħ +çļĦ åŁºç¡Ģä¸Ĭ +èģĮ è´£ +åı¯èĥ½ æĺ¯ +åĨĽ äºĭ +æĪIJ æķĪ +åŃ©åŃIJ 们 +计ç®Ĺ æľº +èµ ¤ +产ä¸ļ åıijå±ķ +å·¨ 大çļĦ +å·¥ 人 +çĶŁ éķ¿ +éĥ½ åı¯ä»¥ +çļĦ æľºä¼ļ +èµĦ è´¨ +çĹĽ èĭ¦ +ç²ī ä¸Ŀ +å¢ ĵ +å¹³ å®ī +管 éģĵ +è·Ł çĿĢ +饮 é£Ł +åķĨ å®¶ +å¤ļ å®¶ +åı¸ æľº +åºĶ该 æĺ¯ +éĢı éľ² +认 å®ļ +è¡Įä¸ļ çļĦ +çļĦ ä¼ģä¸ļ +æ¯ı ä¸Ģ +èĮĥåĽ´ åĨħ +è¾ĥ 大 +è´ ¤ +大 èµĽ +å¤ļ äºĨ +é¸ ¿ +临 åºĬ +åľ¨ è¿Ļ个 +çļĦ åĨħ容 +éĶĢ éĩı +å¾Ī å°ij +åŃ Ł +ç»´ æĮģ +åĴĸ åķ¡ +æľ¬ åľ° +èī² å½© +å¹¶ éĿŀ +èĢĮ å·² +温 æļĸ +èIJ § +æĬĵ ä½ı +èĢĮ ä¸įæĺ¯ +åĸ Ĭ +çļĦ åħ³ç³» +çī© åĵģ +éĤ£ æĺ¯ +åĨľ 产åĵģ +è¿Ļ æĹ¶ +å©ļ å§» +æ°´ æŀľ +æĶ¶ èİ· +ä»ĺ åĩº +客æĪ· 端 +æ¼Ķ åĩº +åħ¨ æĸ° +è¿Ļ ä¹Łæĺ¯ +æĺ¯ çͱ +è§Ĥ 念 +æľī 个 +éĢł åŀĭ +èĥľ åĪ© +ä¸ī æĺ¯ +è¶ħ å¸Ĥ +åħļ建 å·¥ä½ľ +æĶ¾ å¿ĥ +线 è·¯ +æĭĽ çĶŁ +åIJĥ é¥Ń +è½ ī +å°½ éĩı +è§ģ åΰ +åIJĮæ¯Ķ å¢ŀéķ¿ +åįİ ä¸º +æĪij å¸Ĥ +æıIJ åĩºäºĨ +æ°ij èѦ +åįļ çī© +åįļçī© é¦Ĩ +è¯ļ ä¿¡ +åīį éĿ¢ +å±± 西 +è¾ħ åĬ© +转 ç§» +æĽ´ 为 +丰å¯Į çļĦ +åį ¢ +å¿« éĢĴ +æĺ¾ èijĹ +çī© èµĦ +åΰ è¾¾ +æľī åĪ©äºİ +åij Ĩ +åŃ©åŃIJ çļĦ +ä¸į ä½Ĩ +çłĶç©¶ éĻ¢ +çͳ æĬ¥ +æļ ¨ +æ°ij éĹ´ +åį » +çļĦ å£°éŁ³ +å¸Ĥåľº çļĦ +ä¸Ģ åı¥ +çľģ 级 +æĿ¥ çļĦ +åĵª 个 +æīį ä¼ļ +åĪĨ éħį +èĶ ¡ +ä»ĸ åľ¨ +åħ± æľī +å¡ ĺ +èĴ Ĥ +éľ į +åıĤ è§Ĥ +ä¸Ī 夫 +ä¾Ŀ éĿł +æľī æĹ¶ +äºĨ å¾Īå¤ļ +ä¸ĸçķĮ æĿ¯ +å®¶ æĹı +ä¸į éľĢè¦ģ +大 å¸Ī +èŀį åħ¥ +éĿŀ æ³ķ +çĹħ 人 +åIJİ æľŁ +大家 éĥ½ +ç½ij åĿĢ +åİŁ æĸĻ +便 å®ľ +æ¶ Ľ +仿 ä½Ľ +å·® è·Ŀ +åı¦ä¸Ģ æĸ¹éĿ¢ +产åĵģ çļĦ +èµ « +æĥħåĨµ ä¸ĭ +éĴ¢ éĵģ +æľ¬ ç«Ļ +纳 åħ¥ +å·² æľī +æľī 没æľī +ä¼° 计 +é£ ĺ +æľŁ è´§ +åĢĭ人 è³ĩæĸĻ +ä¸ĵä¸ļ çļĦ +çĪĨ åıij +èĩ´åĬĽ äºİ +çİ°åľ¨ çļĦ +æľī åĵªäºĽ +çł´ åĿı +æķ°åŃĹ åĮĸ +åľ° éĿ¢ +é»ij èī² +å¹¼åĦ¿ åĽŃ +çļĦ ç²¾ç¥ŀ +äº Ń +导 æ¼Ķ +çݰ æľī +æŃ¦ åύ +èĭı å·ŀ +çİ Ħ +æ±Ł 西 +å»¶ 伸 +论 æĸĩ +è¾ĥ 为 +çİ© æ³ķ +é¼ İ +åIJĮ æŃ¥ +éĩĬ æĶ¾ +æĽĿ åħī +åĿļ åĨ³ +å§Ķ æīĺ +å°Ĩ åľ¨ +äºĪ 以 +ä½ľ æĸĩ +èĢĮ åľ¨ +ä¼ĺ åħĪ +åĽŀ åİ» +ä¿® å¤į +åĽ½åĨħ å¤ĸ +çŃĸ åĪĴ +åıij æĶ¾ +å¿ĥ æĥħ +çļĦ åİĨåı² +éĿ¢ è¯ķ +举 åĮĹ +ä¿¡ åı· +ç²® é£Ł +è¯ģ 书 +æŁIJ äºĽ +è¿IJ ä½ľ +åĨ² åĩ» +çĥŃ çĤ¹ +æĹ¶ æĹ¶ +æĹ¶æĹ¶ 彩 +åľ° çĤ¹ +ä¸Ģä½ĵ åĮĸ +éļ¾ é¢ĺ +æĽ ° +ç«ĭ åĪ» +æĺ¯ éĿŀ常 +åħ± åĴĮ +åħ±åĴĮ åĽ½ +æ¿Ģ åĬ± +æľīæķĪ çļĦ +å¤Ħ ç½® +该 åħ¬åı¸ +æ£Ģ éªĮ +èѦ æĸ¹ +è´ ¾ +äºĨä¸Ģ ä¸ĭ +ä»Ĭ åIJİ +çħ ® +ç͍ åĵģ +读 èĢħ +æĪij åľ¨ +åĽŀ å¤į +ä¸Ģ 座 +è¿ĺ 没 +å®ļ åζ +没 æĥ³åΰ +å¤ ¹ +ä¼ł éĢĴ +ä¸Ģ 款 +强 大çļĦ +çļĦ è¡Į为 +å¤ı 天 +åıijåĬ¨ æľº +é¢ĨåŁŁ çļĦ +å®ŀéªĮ 室 +ä¸Ģ æĬĬ +æĺ¯ 为äºĨ +éĻķ 西 +æĭħ ä¿Ŀ +è¾¾ æĪIJ +è¦ģ æĺ¯ +æĺİ å¤© +ç»Ļ ä»ĸ +建ç«ĭ äºĨ +ä¸į è¡Į +ä¸Ń æĸĩ +åľ° 说 +åIJİ çļĦ +çĽij æİ§ +éĢ ¸ +æĢ» éĥ¨ +æľ¬ æĸĩ +é¹ ¿ +æĻ¯ è§Ĥ +çļĦ 缮æłĩ +èĽ ĩ +åĨ ¯ +ä¸Ń åĮ» +æķĪ åºĶ +产 éĩı +åŃ Ŀ +è´¦ æĪ· +è¿Ŀ åıį +èij£äºĭ ä¼ļ +京 举 +责任 ç¼ĸè¾ij +åķı é¡Į +çα å¿ĥ +èѦ å¯Ł +é¤IJ åİħ +å¸Ĥ æĶ¿åºľ +天 天 +æĸ° é²ľ +éĥij å·ŀ +è¶ħ è¶Ĭ +å½ Ń +çŁ¥è¯Ĩ 产æĿĥ +åĽŀ å¿Ĩ +è·¯ 线 +å»ī æ´ģ +éĿĴ å°ijå¹´ +åıĸå¾Ĺ äºĨ +çľĭ åΰäºĨ +é¦ ¬ +ç²¾ åĵģ +åľ° éĵģ +æĮģ æľī +ä¸ĭ äºĨ +æľī æĹ¶åĢĻ +ä¸Ģ 人 +æĴ Ĵ +ä»Ķ ç»Ĩ +èĢģ åħ¬ +äºĭå®ŀ ä¸Ĭ +èģĶ èµĽ +ä¾ĽåºĶ éĵ¾ +é¢Ħ ç®Ĺ +åζéĢł ä¸ļ +å®īåħ¨ çĶŁäº§ +俱 ä¹IJ +俱ä¹IJ éĥ¨ +çļĦ æł¸å¿ĥ +æīĵ ç®Ĺ +å½± çīĩ +æIJŃ å»º +ä¹Ł ä¸įä¼ļ +æĭħ å½ĵ +å±Ĥ éĿ¢ +åѦ åijĺ +临 æĹ¶ +缸 ç»ĵåIJĪ +对 æ¯Ķ +ä»ĸ æĺ¯ +æĸ° åĮº +è¿Ľ åİ» +çϾ å¹´ +ä¿ © +å°½ å¿« +ç͵åŃIJ åķĨåĬ¡ +æĽ´ æľī +æ¸ħ çIJĨ +åı¦ ä¸Ģ个 +åĤ » +ä»Ģä¹Ī æł·çļĦ +æĺ¯ æľĢ +åij¨ å¹´ +å¾Ī 容æĺĵ +åĽ¢ ç»ĵ +ç´ Ħ +æĹ© å·² +çļĦ åıĺåĮĸ +éľ ŀ +æĹ¥ ä¸ĬåįĪ +失 åİ» +ä¸Ń åľĭ +çļĦä¸Ģ äºĽ +å°ı åŃ© +ä¸ĭ è·Į +éĶ» çĤ¼ +é ij +éij « +å¿ĹæĦ¿ èĢħ +èĤ¡ å¸Ĥ +èµĽ äºĭ +许åı¯ è¯ģ +åı¯ æĮģç»Ń +åijĬè¯ī è®°èĢħ +éĢ» è¾ij +å¼ķ åħ¥ +çļĦ è¿ĩç¨ĭä¸Ń +è§Ĩ è§ī +èĩªæ²» åĮº +è¯ģ æį® +è£ħ ç½® +第ä¸ī æĸ¹ +å¹´ æĿ¥ +å¹¿ä¸ľ çľģ +带æĿ¥ äºĨ +éķ¿ æ±Ł +访 éĹ® +å·® ä¸įå¤ļ +æĺ¯ æĪij +éģŃ éģĩ +æĬĵ 好 +é«ĺ è¾¾ +å¹¶ åľ¨ +èĩª è§ī +ä¾ĽåºĶ åķĨ +æĥħ æĦŁ +ä½ı äºĨ +çļĦ èģĮä¸ļ +çļĩ å¸Ŀ +西 éĥ¨ +åĴĮ å¹³ +çļĦ åĬĽéĩı +æ± ª +åħħåĪĨ åıijæĮ¥ +æĬķ è¯ī +èµ· åΰ +äºĴ 缸 +æ¾³ éŨ +æİ¥ åΰ +æ°´ æ³¥ +模 åŀĭ +ä¸Ģ åįĬ +ç§© åºı +æĪij们 åľ¨ +æī¿ 认 +ä¸Ģ éĥ¨åĪĨ +åįł æ¯Ķ +å¦ĩ 女 +ç² ĺ +äºĨè§£ åΰ +ä¸Ģå®ļ ä¼ļ +åIJĦ 大 +èµ° åĩº +为 大家 +é«ĺ éĵģ +åı¯ä»¥ åľ¨ +ä½Ĩ åľ¨ +çĶŁæĢģ çݯå¢ĥ +èı ¯ +çļĦ ä»·æł¼ +麻 çĥ¦ +æ¿Ģ åıij +éĤ£ å°± +çļĦ æł·åŃIJ +为 æŃ¤ +天 åľ° +çļĦ 缮çļĦ +åĢº åΏ +å·² ç¶ĵ +åĽĽ 大 +åIJĮæĹ¶ ä¹Ł +å½¼ æŃ¤ +æĭ¿ åΰ +åIJ« éĩı +åįģ 大 +éļ¾ éģĵ +å¼ Ĺ +ä¸Ģ 段æĹ¶éĹ´ +çħ§ 顾 +æķ°æį® æĺ¾ç¤º +æĪIJ为 äºĨ +èµ° åΰ +æľ¬ åħ¬åı¸ +ç»Ī 端 +ä¹Ł ä¸įæĺ¯ +头 åıij +大 约 +é£İ æĻ¯ +æ¶Ī èĢĹ +审 æŁ¥ +äºī åıĸ +æ³ķ æ²» +äºĭ çī© +ç¼ĵ è§£ +æĥ ¨ +缸åºĶ çļĦ +çļĦ æķĪæŀľ +åıį å¤į +åıijçĶŁ äºĨ +éĢĻ äºĽ +ç»ĥ ä¹ł +åݨ æĪ¿ +å¼Ģ æĭĵ +欣 èµı +夫 妻 +ä¸į ä¸Ģæł· +产 èĥ½ +èĬ¯ çīĩ +è¦ģ ç´ł +åıį 对 +çİĩ åħĪ +è´§ çī© +æĹ¥ ç͵ +ä½ľ å®¶ +æĶ¹ è¿Ľ +æĪIJ åĪĨ +åĽł èĢĮ +åĩı èĤ¥ +æ½ ĺ +å±±ä¸ľ çľģ +åĬ Ŀ +åŁ ĭ +æŃ¦ è£ħ +æ±ĩ æĬ¥ +ä¸Ģ个 æľĪ +çĥŃ éŨ +大 éģĵ +æ´» åĭķ +éĥ½ å¾Ī +ç͵ 梯 +ç´§ æĢ¥ +åĢº åĬ¡ +客 æľį +ä¸Ģ éĥ¨ +ä½ł æĺ¯ +çݰ çĬ¶ +æŃ£ç¡® çļĦ +ä¹ĭ å¤Ħ +ç¼ĸ åζ +ä½ł åı¯ä»¥ +çŃī åľ° +èİ ī +对 è¯Ŀ +æ·ĺ å®Ŀ +è°ĥ èĬĤ +æİĴ æĶ¾ +åºĵ åŃĺ +ç´ ļ +çļĦ ä¼ĺåĬ¿ +æĿĥ å¨ģ +以ä¸ĭ ç®Ģç§° +ä¸Ģ 项 +èģļ éĽĨ +ä¼łç»Ł çļĦ +æ·· åIJĪ +è¿Ļä¸Ģ çĤ¹ +ä¸Ģ çľ¼ +æĹł éĻIJ +èİ·å¾Ĺ äºĨ +éĢī æīĭ +åζ åĵģ +åįı ä½ľ +çĭ¬çī¹ çļĦ +ä¸Ģ 级 +è¿Ļ个 éĹ®é¢ĺ +æĸ Į +æĺ¯ æĪij们 +æķĮ 人 +æ¸ħ æ´Ĺ +ä¸Ģ缴 åľ¨ +å°ı ç±³ +çļĦ è¿ĩç¨ĭ +åľ¨ åĮĹ京 +ä¸Ģ æĶ¯ +æĹ© ä¸Ĭ +æĸĩ èīº +ç¦ı åĪ© +é£Ł ç͍ +æĦŁ åĬ¨ +åħ¨ ç¨ĭ +æĶ¯ åĩº +æĸ° 建 +å¸ ķ +æĺ¾ çĦ¶ +羣 çļĦæĺ¯ +æĸ°éĹ» ç½ij +èĥ½ åIJ¦ +åįı åĬ© +亲 èĩª +å¾Ī æľī +çϼ å±ķ +æĦı 大 +æĦı大 åĪ© +ç͵ ç½ij +æĹ¥ çĽĬ +çĨ ± +èĤĮ èĤ¤ +çĶ· æĢ§ +ç»Ħ 建 +çŃī éĹ®é¢ĺ +æ¶Ī éϤ +æĬ¤ çIJĨ +å¡ij æĸĻ +ä¹Į åħĭ +ä¹Įåħĭ åħ° +åķĨ æłĩ +çIJ ³ +æĸ° æīĭ +çļĦ çī¹çĤ¹ +åĴ ¬ +å½ĵ ä¸ĭ +设计 å¸Ī +èµĶ åģ¿ +第 åįģ +æĻºèĥ½ åĮĸ +å¼Ģåıij åĮº +åı¯ä»¥ éĢļè¿ĩ +åħ±äº§ åħļ +åİī 害 +çģµ æ´» +æĹ¶ åħī +éĥ¨ ä½į +人 æĸĩ +è¿Ľ æĿ¥ +ä¹ĭ æīĢ以 +ä¸ī åįģ +çļĦ åѦçĶŁ +éĺ² æĬ¤ +åĽ½ 产 +æ·±åľ³ å¸Ĥ +éĤ£ å°±æĺ¯ +åΰ ä½į +çī¹ æľĹ +çľĹ æĻ® +å®ŀ æĹ¶ +åı° çģ£ +èĢĮ ä¸į +æĮĩ å®ļ +åĿ Ŀ +èħIJ è´¥ +çī¹ å®ļ +å¢ŀ éĢŁ +æłĩ çѾ +æĪ¿ ä»· +æĦ ģ +贯彻 èIJ½å®ŀ +æĢ§ è´¨ +çłĶç©¶ çĶŁ +ç¾İ 容 +æī¹ è¯Ħ +ç©¶ 竣 +人åĬĽ èµĦæºIJ +éĸĭ å§ĭ +åĽŀ å½Ĵ +èIJ¥ åķĨ +èIJ¥åķĨ çݯå¢ĥ +ä¸ŃåĽ½ 人 +çļĦ åŁºæľ¬ +è¯Ŀ é¢ĺ +æłĩåĩĨ åĮĸ +西 èĹı +åĭ ¾ +çļĦ 设计 +ç®Ģåįķ çļĦ +å¤į åζ +æ¸IJ æ¸IJ +以 å¤ĸ +èģĶ åĬ¨ +两 次 +æĢ§ åĴĮ +æĽ´ 大 +çļĦ åIJįåŃĹ +éŁ ¦ +ä½ł è¦ģ +å¢ĥ å¤ĸ +æĹ© æľŁ +åĪĿ æŃ¥ +è´¦ åı· +害 æĢķ +æĺ¨ æĹ¥ +åĪļ æīį +ç¥ŀ ç§ĺ +ç²¾ å¿ĥ +æµģ éĢļ +åħ¨ æĸ¹ä½į +以 å¾Ģ +ä¹Ł å°Ĩ +æĺ¯ ä¸ŃåĽ½ +åĽ½å®¶ 级 +å°Ĩ åĨĽ +æij Ĭ +æľĢ 为 +第ä¸Ģ æĹ¶éĹ´ +æ¶Ī æ¯Ĵ +å°Ĩ äºİ +å¨ģ èĥģ +èĭ± æĸĩ +æīĭ ä¸Ń +çIJĥ è¿· +è§Ĥ çľĭ +离 å©ļ +æľ¬ åľŁ +åĪĨ æķ£ +æĻ ´ +è¦ģ 注æĦı +浪 è´¹ +管 æİ§ +åĩº åĶ® +æĢ» è£ģ +ä¸Ģ éĺµ +å¨ ĩ +äºĶ 个 +å½ĵ åĪĿ +çºł 纷 +ä¸ĵ ç͍ +å¤ĩ æ¡Ī +åĪĿ æľŁ +å®ĥ æĺ¯ +åĮº åĿĹ +åĮºåĿĹ éĵ¾ +大 è¿ŀ +è¿Ļ ç±» +åıĺ æĪIJäºĨ +éĤĦ æĺ¯ +åįļ 客 +çı¾ åľ¨ +ä¸Ģ æĸ¹ +å®ĮæĪIJ äºĨ +è¿Ļ个 æĹ¶åĢĻ +åħ¨ å¹´ +ä¸Ĭ 线 +ç½ IJ +ç«ŀ èµĽ +åĩºçīĪ ç¤¾ +åĵ¥ åĵ¥ +å¯ « +å¾Ĺ 以 +èĬ± åĽŃ +äºĨ èµ·æĿ¥ +èĦ±è´« æĶ»åĿļ +çļĦ åİŁåĪĻ +讲 è§£ +æ¶Ī åĮĸ +æį٠害 +æļĤ æĹ¶ +å¾Ĺ çŁ¥ +éĢĤ ç͍ +éŨ åºĹ +è§£ 读 +æĻ® åıĬ +人æ°ij æ³ķéĻ¢ +åī¯ ä¸»ä»» +å¿ĥ çģµ +è¯Ĭ æĸŃ +ç¾İ 女 +æŁ ¯ +å¹´ 以æĿ¥ +æ´» è·ĥ +åĢŁ åĬ© +åħ± 建 +è¯ī 讼 +æĶ¾ æĿ¾ +çªĹ åı£ +ä¼ģ æ¥Ń +åĬł æĭ¿ +åĬłæĭ¿ 大 +ä¹° äºĨ +主 æµģ +æĩĤ å¾Ĺ +å°Ĩ åħ¶ +éĢı æĺİ +å·¥ä½ľ ä¸Ń +èĤ¡ ä»· +æ¡£ æ¡Ī +没æľī ä»»ä½ķ +åijĬ çŁ¥ +å¹´ åĪĿ +æĹ¥ ä¸ĭåįĪ +åİĤ åķĨ +èĬĤ å¥ı +主 导 +è£ Ŀ +åħ³éĶ® è¯į +èģĬ 天 +åĨĻ ä½ľ +æĶ¹éĿ© å¼ĢæĶ¾ +æľī æľĽ +éĢļ æĬ¥ +èIJ Į +æĢ» é¢Ŀ +çŁŃ æľŁ +ä¸Ģ çķª +çĶŁæ´» çļĦ +åĮĸ çļĦ +æĺ¥ 天 +è¿Ļ åľº +æĸ°å¼Ģ ä¼łå¥ĩ +æĺ¯ è¦ģ +å°ļ æľª +åıĺ æĽ´ +ä¸Ģ åij¨ +客 è§Ĥ +æĹ¥ èĩ³ +é¹ ° +çİ ² +å°Ĩ æĿ¥ +客 人 +åıĺ éĿ© +说 äºĨ +åİŁ çIJĨ +èģĮ åĬ¡ +åıĪ æľī +ä¸Ģ åı¥è¯Ŀ +æĦŁ åıĹåΰ +ç¬Ķ èĢħ +ç§» æ°ij +西 åįĹ +ä¹ĥ èĩ³ +æŃ£ è§Ħ +åĪĿ ä¸Ń +çĬ ¬ +å½ĵ äºĭ +å½ĵäºĭ 人 +æĪij们 è¦ģ +åħ¥ åı£ +éĤ£ æĹ¶ +æľīéĻIJ 责任 +å°ij 女 +è¿Ļä¹Ī å¤ļ +åĪĨ åħ¬åı¸ +å®ĩ å®Ļ +çļĦ éĢīæĭ© +å§IJ å§IJ +åıij èµ· +è» į +æĽ´å¥½ åľ° +éĻĨ ç»Ń +æľ¬ æľįåĭĻ +å« © +èµ¶ ç´§ +èĦĤ èĤª +第äºĮ 天 +æĪij ä¼ļ +两 ä½į +æķ ² +åħ¬å®ī æľºåħ³ +ç§ijæĬĢ åĪĽæĸ° +å°º 寸 +è¾IJ å°Ħ +å®Ĺ æķĻ +转 æį¢ +åĩº çİ°åľ¨ +ä¸Ģ é¢Ĺ +æľŁ éĻIJ +åIJĮåѦ 们 +åĮĹ æĸ¹ +ä½ł å°± +ä¸Ģ带 ä¸Ģè·¯ +èĢģ å©Ĩ +游æĪı çݩ家 +çļĦ ç»ĵæŀľ +è¡¥ åģ¿ +å¤ĸ è´¸ +对 å¾ħ +ç»´ çĶŁç´ł +ç»ıéĶĢ åķĨ +è¿ĺ å°Ĩ +åŃIJ 女 +æĽ´ é«ĺ +ä¸į 大 +éī´ å®ļ +让 ä»ĸ们 +æīĢè°ĵ çļĦ +æŃ» äºĨ +帮 æī¶ +åĵ² åѦ +以ä¸Ĭ çļĦ +çļĦ åħ³éĶ® +æĹ© å°± +æĬ¥ ä»· +éģµ å®Ī +æī© å¼ł +æĺ¯ å¾Ī +å¼Ģ éĢļ +æĸ° åĬł +æĸ°åĬł åĿ¡ +ç¿» è¯ij +询 éĹ® +é¸ Ń +ä½ĵ åĨħ +两 个人 +çĪ ¹ +éľ ľ +乡æĿij æĮ¯åħ´ +çĿ¡ è§ī +å®ĺ åijĺ +åĪĽ å§ĭ +åĪĽå§ĭ 人 +ä¼Ĺ 人 +åį³ ä¾¿ +çĸ« èĭĹ +ä¼ģä¸ļ å®¶ +æ¸ £ +ç²¾ åĬĽ +å¤ĸ éĥ¨ +èģª æĺİ +è¿Ļ ä¹Ł +å½ķ åıĸ +åĨ² çªģ +åħ¨ 身 +åŃ£ èĬĤ +忽 çĦ¶ +çļĦ æĢģ度 +åĤ¨ å¤ĩ +ä¿Ŀ åħ» +çļĦ æĥ³æ³ķ +ä¸Ĭæµ· å¸Ĥ +æIJº æīĭ +çļĦ ä¿¡æģ¯ +åķĨ åľº +çļĦ æĢĿæĥ³ +æĿĥ åĬĽ +毫 æĹł +æĢĢ åŃķ +硬 ä»¶ +åĨħ èĴĻåı¤ +æİ¢ 讨 +åħ» çĶŁ +çļĦ 表çݰ +空 ä¸Ń +æģIJ æĢĸ +å¾Ī é«ĺ +ç»ıæµİ 社ä¼ļ +ä¸Ĭ æĿ¥ +å»¶ ç»Ń +éĩį å¤į +éĺ² èĮĥ +çļĦ å½¢å¼ı +æľĪ åºķ +èĢģ 年人 +绿 åĮĸ +å±± åĮº +æĭ¿ åĩº +æĹħ 客 +æĽ´ æį¢ +åħ¬ 主 +èĬĤ 约 +åħ¨ åİ¿ +åĽŀ æĬ¥ +çIJĨ æĢ§ +çĸ¯ çĭĤ +æ¶ī å«Į +åī§ æĥħ +åĨ¬ åŃ£ +åIJİ ç»Ń +è¿Ļæĺ¯ ä¸Ģ个 +æ¼Ķ 讲 +ä¸Ģ å±Ĥ +æľīåħ³ éĥ¨éŨ +æĹł å¥Ī +ç§į ç±» +缸åħ³ çļĦ +æĪĸèĢħ æĺ¯ +æī¶ æĮģ +å¤ļ æķ° +çļĦ ä½ľåĵģ +ä¸ĭ ä¸ĢæŃ¥ +å¸Ī åĤħ +é«ĺéĢŁ åħ¬è·¯ +好 åıĭ +ä¼ĺç§Ģ çļĦ +è¿Ľ äºĨ +æģIJ æĢķ +äºĨ åIJ§ +大 è§Ħ模 +çļĦ ä¸ĸçķĮ +æĢĢ çĸij +å· · +åħ´ å¥ĭ +æĪ ° +æĿij éĩĮ +æľĭåıĭ åľĪ +åĨ¬ 天 +ä¸Ńåįİ äººæ°ij +åįı åķĨ +è¯Ħ éĢī +æĹ Ń +å¢ŀåĬł äºĨ +åıĹ ä¼¤ +ä¸Ģ èĤ¡ +便 æį· +ä¸ ij +é¹ ¤ +å¤ĸ è§Ĥ +å·¥ç¨ĭ å¸Ī +åĴĮ åħ¶ä»ĸ +è¿Ļ å°± +ä¸Ńå°ı ä¼ģä¸ļ +西 åĮĹ +åĽ½æľī ä¼ģä¸ļ +èĭ¥ æĺ¯ +åı¯ æĥľ +çĶŁ æĹ¥ +åĩ ½ +ä¹° åįĸ +ç¥Ŀ ç¦ı +人æ°ij 群ä¼Ĺ +åħī æĺİ +åħ¬ å¯ĵ +æĺ¯ è°ģ +æĪij çŁ¥éģĵ +è¯Ń æĸĩ +æķı æĦŁ +ä¸įéĶĻ çļĦ +æĿ¥ 讲 +æ³¢ åĬ¨ +çļĦ 第ä¸Ģ +åľ° éľĩ +åľ¨ åħ¨åĽ½ +骨 å¹² +å®ī ç½® +å®¶ ç͵ +ä¸İ æŃ¤ +ä¸İæŃ¤ åIJĮæĹ¶ +åıĹ çģ¾ +çĥŃ çº¿ +çļĦ æĬĢæľ¯ +æµĭ éĩı +ä¾Ŀ èµĸ +ä¸ŃåĽ½ çļĦ +çī¹ æĢ§ +è¾ĥ é«ĺ +è¸ © +ä¼ļ åľ¨ +建 éĢł +导 èĪª +æĥ³ èµ· +åħ¨ ä¸ĸçķĮ +建 æĿIJ +ç¯ Ģ +çļĦ åŁºç¡Ģ +èĩªåĬ¨ åĮĸ +åīį åIJİ +çĿ¡ çľł +æİ¨ è¡Į +æį® äºĨè§£ +ä»Ģä¹Ī æĹ¶åĢĻ +ä¸į åĸľæ¬¢ +çħ¤ çĤŃ +éĤ£ä¹Ī å¤ļ +å¸Ĥåľº åĮĸ +ä¸į管 æĺ¯ +ç«ĭ åľº +éĥ½ 没 +课 é¢ĺ +æĪij们 å°Ĩ +è¿ĩ çļĦ +åĨį åĬłä¸Ĭ +çĪ ¾ +身 æĿIJ +çĶ· 女 +è¿ľ è¿ľ +çĶ· çĶŁ +èĩªèº« çļĦ +è´Ł æĭħ +çϾ ä¸ĩ +西 çıŃ +西çıŃ çīĻ +åĩĢ åĪ©æ¶¦ +æ¾³ 大 +澳大 åĪ©äºļ +ä¸į åİ» +æī¿ åıĹ +楼 çĽĺ +å¢ĥ åĨħ +æ·· åĩĿ +æ··åĩĿ åľŁ +æĢĿæĥ³ æĶ¿æ²» +å¸Ĥ åĮº +æĭĽ æłĩ +åĽ¢ ä½ĵ +è¿Ľ 度 +åĨĽ éĺŁ +åıį å¼¹ +äºĨä¸Ģ äºĽ +æİ¥ å¾ħ +çļĦ åŃ¦ä¹ł +éħį éĢģ +é£Łåĵģ å®īåħ¨ +æĽ¿ 代 +æĺ¯ 以 +éĢļ ç͍ +çłĶç©¶ æīĢ +ç¦ ħ +æī Ķ +éļĶ ç¦» +ä¸ĩ å¹³æĸ¹ç±³ +çļĦ è§Ħå®ļ +ç»Ļ æĪij们 +æ¿Ģ åħī +ä¼ļ åĩºçݰ +çŁŃ ä¿¡ +ç©¿ çĿĢ +æ²Ī éĺ³ +æķĻ æĿIJ +éĺ² çĸ« +ä¼ĺ èī¯ +约 å®ļ +æĪij çľģ +åħ¬ æ°ij +éģ¸ æĵ +é쏿ĵ ĩ +å·² æĪIJ为 +ä¸į å¿ħ +ç¥ĸ åĽ½ +å¹¶ æľª +åľŁ 壤 +å¾® ç¬ij +äºĭä¸ļ åįķä½į +çļĦ 游æĪı +åħ¬ 示 +åIJĪçIJĨ çļĦ +çª Ŀ +æ°Ķ 象 +å®¶ ä¸Ń +亮 缸 +åį« æĺŁ +è®° è½½ +è§Ĩ éĩİ +åľ°åĮº çļĦ +ä½Ĩ ä»ĸ +èĤĮ èĤī +äºı æįŁ +åĬŀ åѦ +ä¸Ģ è¡Į +è¯ŀ çĶŁ +åıijå¸ĥ çļĦ +çļĦ æľįåĬ¡ +çļĦ çłĶç©¶ +åij¨ æľ« +产ä¸ļ åĽŃ +é«ĺ 温 +æĪIJåĬŁ çļĦ +æŃ¥ 骤 +åŃĺ åĤ¨ +åŃIJ åħ¬åı¸ +让 她 +ä¸Ń æľī +åĺī 宾 +å¦ ® +æĺİ å¹´ +äºĨ åIJĹ +äºī è®® +æĪ Ī +ä¸Ģ æľ¬ +ç¾İ丽 çļĦ +ä½ł 说 +大 人 +æĶ» çķ¥ +ä¸į æľĥ +å¾ħ éģĩ +ä¸Ģ è¾Ĩ +çīĪæĿĥ æīĢæľī +æ°ij ä¼Ĺ +åĬ٠夫 +å±ķ ä¼ļ +大 èĦij +æ¯ı æľĪ +å°ı 麦 +æµĻæ±Ł çľģ +çļĦ æīĢæľī +ä¸ĭ æ»ij +èĵĿ èī² +è¦ģ æĥ³ +åѦçĶŁ çļĦ +å½ĵ ä½ł +ä½ľ æĪĺ +å®¶ 乡 +å¤ļ åIJį +é«ĺ äºİ +åĿļ 强 +è¿ŀ éĶģ +åIJİ æŀľ +人 äºĭ +ç´ ħ +æ¿Ģ åĬ¨ +è¿Ľ æĶ» +ç© Ĩ +ä¸ ĺ +让 èĩªå·± +以 æŃ¤ +夫 人 +å¼Ģ 设 +æ°Ķ è´¨ +鸡 èĽĭ +çĦ¡ æ³ķ +åIJĥ äºĨ +åĪĨåĪ« 为 +èģĶåIJĪ åĽ½ +å½ĵ 代 +å¦Ĥæŀľ æĺ¯ +è¿ľ ç¨ĭ +åĸ Ĥ +è®° ä½ı +æ¸ħ åįķ +åIJĪä½ľ ä¼Ļä¼´ +åİ» åģļ +æķħ éļľ +模 æĭŁ +å¸Ī çĶŁ +åīį æĿ¥ +ç͵è§Ĩ åī§ +çĥŃ çα +éľ² åĩº +é«ĺ å±Ĥ +ç͵ åύ +纪 å¾ĭ +å¼Ģåıij åķĨ +éķ¿ å®ī +è½½ ä½ĵ +çļĦ å°±æĺ¯ +被 人 +åıĹ çIJĨ +篮 çIJĥ +èİ İ +交 ç»Ļ +æľªæĿ¥ çļĦ +两 大 +åIJķ å¸ĥ +çŃī 人 +çļĦ æĹ¥åŃIJ +åIJĪä½ľ 社 +æĮij éĢī +åŃĺ æ¬¾ +ç³»ç»Ł çļĦ +æĬĬ å®ĥ +没æľī ä»Ģä¹Ī +ä»İ æŃ¤ +ä¸Ń åįĪ +çĸ¼ çĹĽ +å·© åĽº +浪 漫 +缸åħ³ éĥ¨éŨ +éķ¿ åŁİ +纤 ç»´ +ä¸Ĭ éŨ +çĪĨ çĤ¸ +èµ· çĤ¹ +çļĦ éĢļçŁ¥ +èĢĮ æĿ¥ +çļĦ èĢģ +æīĭ éĩĮ +è¯Ń éŁ³ +è¾Ľ èĭ¦ +æ±Łèĭı çľģ +ç͍ äºĨ +身份 è¯ģ +æľī åĬ© +æľīåĬ© äºİ +çī© èģĶç½ij +åĩº éŨ +å¼Ł åŃIJ +æĥ ¹ +è¿Ļä»¶ äºĭ +æĪij们 åı¯ä»¥ +çļĦ çĶŁåij½ +æľīä¸Ģ ç§į +åºĹ éĵº +åıĮ æīĭ +çļĦ æ¶Īæģ¯ +èĢIJ å¿ĥ +å°´ å°¬ +éĤ£ 天 +é¦ĸ æī¹ +æĺ¯ä¸Ģ å®¶ +人 æ°Ķ +åıį æŃ£ +æĪij åĴĮ +å®ł çī© +ä¸į 对 +寻 æ±Ĥ +缸 ä¼¼ +åľ¨ ç¾İåĽ½ +åı« åģļ +åĹ İ +ç«ĭ è¶³ +ç͍ éĢĶ +åħ Ĩ +大 æ°Ķ +åIJij ä¸Ĭ +ä»ĸ å°± +é¡¹çĽ® 建设 +èĭ¥ å¹² +æĺ¯ æľī +æ¿Ģ æĥħ +çļĦ æĦıä¹ī +æĺ Ń +严éĩį çļĦ +å¯Ĩ éĽĨ +èĪŀ è¹Ī +èᣠèİ· +èİ· æĤī +æ±Ł åįĹ +åģĩ å¦Ĥ +æĪ· å¤ĸ +线 ç´¢ +ç§ģ 人 +转åŀĭ åįĩ级 +çļĦ ä»·å̼ +åįķ çĭ¬ +èĢģ çϾå§ĵ +å°į æĸ¼ +åĽ½éĻħ åĮĸ +ä¼° å̼ +æľįåĬ¡ ä¸ļ +èĩ Ń +æİī äºĨ +è§£åĨ³ äºĨ +ä¹Ł ä¸įèĥ½ +åħ ¹ +æĸ¯ çī¹ +æķħ æĦı +è¿ĩ 度 +èĬĤ æĹ¥ +çϽ çĻľ +çϽçĻľ é£İ +ç»§ æī¿ +äºĨ ä¸įå°ij +äºĮ 人 +è§ģ éĿ¢ +æĥ³ æĥ³ +å¤į åIJĪ +康 å¤į +åİ¿ åŁİ +åľ¨ åĽ½åĨħ +åľº åľ° +é϶ çĵ· +è¿Ļ 项 +çľ¼ ä¸Ń +çł ¸ +æĦŁè§ī åΰ +æŀľ çĦ¶ +æĶ¾ åħ¥ +约 æĿŁ +æİĴ æŁ¥ +车 主 +çļĦ æĦıæĢĿ +æĸ° åŁİ +æĥ³ çĿĢ +éģ Ĥ +èĮ¶ åı¶ +ä¹° æĪ¿ +åĨľ æĪ· +é«ĺ æīĭ +çİī ç±³ +æĸ°åĨł èĤºçĤİ +çħ§ æĺİ +æĮĩ åįĹ +è¸ ¢ +æķij æı´ +æĻ¯ çĤ¹ +ç¨İ æĶ¶ +çļĦ æīĭ +æŃ£ 好 +è¦ģ æĬĬ +éļı æĦı +åħ¶å®ŀ æĺ¯ +ç»Ļ èĩªå·± +è°Ī åΤ +æ¯ı天 éĥ½ +æĢģ åĬ¿ +é¢Ħ 约 +åİĨåı² ä¸Ĭ +å®Ŀ è´Ŀ +åīį è¿Ľ +ä¹Łå°±æĺ¯ 说 +çļĦ æĦıè§ģ +åı£ 罩 +åİĺ ç±³ +èĬ± è´¹ +ä½ĵèĤ² æĬķæ³¨ +åħ¬ä¼Ĺ åı· +èijĹåIJį çļĦ +å¼Ģ æĪ· +æĭį åįĸ +å²ģ æľĪ +åĨħ æ¶µ +å®Įæķ´ çļĦ +é«ĺ åİĭ +åħ¬åĬ¡ åijĺ +使ç͍ çļĦ +çĶŁäº§ 线 +妹 妹 +èµ° 访 +æĺ¯ åı¯ä»¥ +åľ¨ å®¶ +æļ´ åĬĽ +æ³° åĽ½ +è´¨ çĸij +ä¸į éģİ +天çĦ¶ æ°Ķ +缺 çĤ¹ +å°ı åŀĭ +ä¸įä»ħ æĺ¯ +é»ij æļĹ +æ¢ ¨ +æĸĩ æĹħ +è¦ģ æľī +ä¸Ń å±± +çļĦ æķ°æį® +å¾Ĺ å¾Ī +以 便 +对 ä»ĸ +åĬł 以 +çϼ çı¾ +设 å®ļ +èĤļ åŃIJ +éĿ ĸ +å¥ī çĮ® +ä¸į åıĺ +åı£ ç¢ij +åľ¨ åĵªéĩĮ +ä½ IJ +è¿Ļ 两个 +çļĦ æĸ¹åIJij +æŀ « +äºĮ 次 +çīĩ åĮº +éł IJ +ç£ Ĭ +æĭ¿ çĿĢ +å·²ç»ı æĪIJ为 +ä¹ĭ ä¸Ĭ +å®Ĺ æĹ¨ +奶 奶 +é«ĺæĸ° åĮº +社 æľĥ +è·Ł 踪 +æľįåĬ¡ ä¸Ńå¿ĥ +æī ¯ +æīĭ æĮĩ +礼 çī© +宿 èĪį +ç͍ å¿ĥ +æıIJé«ĺ äºĨ +亮 çĤ¹ +ä¸į æĦ¿æĦı +æĴŃ æĶ¾ +å¤ļå°ij éĴ± +没 ä»Ģä¹Ī +æķ° åįģ +æĢ» çĽij +çļĦ åŁİå¸Ĥ +æī¾ åΰäºĨ +åĨħ åľ° +åΰ çİ°åľ¨ +æĪĺæĸĹ åĬĽ +åİŁ å§ĭ +åĥ § +åĢĴ æĺ¯ +æľĢ åħ· +è´«åĽ° æĪ· +éĢģ åΰ +级 åĪ« +åĩº èµĦ +æĪª æŃ¢ +ç§į åŃIJ +èĥ½ ä¸įèĥ½ +幸 è¿IJ +èĸ ĩ +项 éĵ¾ +æĮĤ çīĮ +ä¸Ģ 樣 +ä¹ĺ 客 +èIJ½ åIJİ +ä½Ĩ æĪij +æĹ© åľ¨ +åĬ¨ 漫 +å¹³ çŃī +对 ä½ł +ä¸į æĢķ +å¤ĸ çķĮ +å¤ļå¹´ æĿ¥ +é¦ĸ 个 +æ²³ åįĹçľģ +æĪĸ åħ¶ä»ĸ +éķľ å¤´ +åįĹ æĺĮ +ä¸Ģ éĿ¢ +éĢłæĪIJ çļĦ +å´ Ķ +çŃ Ĵ +æķĻèĤ² éĥ¨ +åľ° åŁŁ +æĺĨ æĺİ +å·´ é»İ +æīĭ 游 +ä¸Ģ æĹ¶ +çł į +é¡¶ 级 +åħ± 计 +åİŁ æ²¹ +è¾ī çħĮ +说 æĺ¯ +æĸ°åįİ ç¤¾ +ç»ıåİĨ äºĨ +ä¸į æŃ¢ +è¦ģ ä¹Ī +èĢħ çļĦ +æĢ» æĬķèµĦ +è¡Į é©¶ +ä¸Ĭ å¸Ŀ +å¹´ 纪 +çIJ ¼ +ä¼ł 说 +ç²¾ èĭ± +æĸ¹ éĴĪ +æ±Ł æ¹ĸ +æĪIJ çĤº +æĢ» éĩı +æĬķ æĶ¾ +åĬ¨ çĶ» +èĹ ¤ +ç͵ æºIJ +éĴ Ļ +åIJĮ è¡Į +æĻ®éĢļ çļĦ +åĽ¾ä¹¦ é¦Ĩ +è¯Ī éªĹ +æħĪ åĸĦ +è¿Ļ 份 +主æĮģ 人 +å°± è¿Ļæł· +èĢĮ æĪIJ +èĩªè¡Į 车 +ä¸ŃåĽ½ çī¹èī² +èĤ¿ çĺ¤ +åIJ ¾ +å¼Ł å¼Ł +åıĹ çĽĬ +éĢīæĭ© äºĨ +æĺİæĺ¾ çļĦ +æĬ¥ èĢĥ +ç¬ij éģĵ +éĽĸ çĦ¶ +温 å·ŀ +éĿŀ æ´² +ç§į ç§į +åıĤåĬł äºĨ +è´§ è¿IJ +éļı 便 +å°± 没æľī +ç¸ £ +央 è§Ĩ +ç©¿ è¶Ĭ +çļĦ çݰ象 +åĩł 次 +çļĦ é£İéĻ© +æŃĮ æĽ² +æľ¬ å±Ĭ +å¹´ åĨħ +ä¸į è¶ħè¿ĩ +è¿ĩ å¤ļ +å¿ħé¡» è¦ģ +ç»ĵ 论 +åĢŁ éī´ +ç¥ŀ å¥ĩ +æľŁ æľĽ +ä¸ĵ 享 +éĿŀ常 éĩįè¦ģ +æĦıè¯Ĩ åΰ +åIJĪ å¹¶ +æĬĬ èĩªå·± +å¥Ĺ è£ħ +éŃĶ æ³ķ +å¤ı åŃ£ +ä¸į åĥı +å¢ĥ çķĮ +æĥĬ åĸľ +æľīä¸Ģ 天 +çĦ¦ çĤ¹ +æĪij 认为 +åħ° å·ŀ +ç͵ æ°Ķ +èģĶç³» æĪij们 +ç§ij æĻ® +她 说 +çļĦ æĸĩ竳 +å¥ĩ æĢª +åıĭ 好 +饮 æĸĻ +çļĦ æĶ¯æĮģ +çŃĶ åºĶ +éĩį éĩı +çij ¶ +åĩı è½» +ç§ijåѦ å®¶ +å·´ 西 +éĩijèŀį æľºæŀĦ +åħļ å§Ķ书记 +貸 款 +ç²¾ èĩ´ +ä»İ æľª +åį° åĪ· +åĽŀ 顾 +é¦ĸ éĥ½ +åıij èĤ² +éĹ® éģĵ +è¾¾ åΰäºĨ +å¿į ä¸įä½ı +æīį æľī +æįIJ èµł +ä½Ľ æķĻ +ä¸į æ¸ħ +éĺŁ éķ¿ +缸 åıį +æĬ¥ èѦ +大 åħ¨ +欧 缣 +帮 å¿Ļ +çļĦ æĻĤåĢĻ +缮 å½ķ +è¶³ 以 +èī° éļ¾ +ä»ĸ ä¹Ł +å·¥ ä½ľèĢħ +头 èĦij +缺 éĻ· +æĪIJç«ĭ äºĨ +å°± å¼Ģå§ĭ +认 åIJĮ +é»Ħ èī² +çĹħ æĥħ +覺 å¾Ĺ +è¿Ļ 两 +ä¿¡ ä»° +åľĭ å®¶ +ä¸įä»ħä»ħ æĺ¯ +çĭ¬ å®¶ +èά çļĦ +æĿIJ è´¨ +æµ· ä¸Ĭ +çĤº äºĨ +æľºåĬ¨ 车 +缸å½ĵ äºİ +å¤ļåħĥ åĮĸ +æĽ´ 大çļĦ +èĽ ® +åģĩ æľŁ +å¼ı çļĦ +交éĢļ è¿IJè¾ĵ +çľģ å§Ķ +ä¸į ç®Ĺ +æĶ¾ ä¸ĭ +éĹ ¯ +人 åľ¨ +港 åı£ +æĹ¨ åľ¨ +åij½ 令 +æŁIJ 个 +å¹³ 稳 +åıª 好 +人 人 +äº ŀ +äºĮ ç»´ +äºĮç»´ çłģ +æŀģ 为 +åĪ« å¢ħ +åħ¶ ä½Ļ +大 äºĭ +主管 éĥ¨éŨ +æĹł éĶ¡ +éĹ µ +éģŃ åΰ +说 è¿ĩ +为 ä½ł +è§£ çŃĶ +éªĮ æĶ¶ +çļĦ ç»ıéªĮ +åĮ¹ éħį +çģ« ç®Ń +豪 åįİ +æŁIJ æŁIJ +çļĦ æĹ¶ä»£ +书 éĿ¢ +æģĴ 大 +å»¶ éķ¿ +ä¸Ģ åIJĮ +æľª èĥ½ +交 æį¢ +çĶ¢ åĵģ +çŃī åΰ +åĪĨ 离 +æīĵ ç͵è¯Ŀ +å¹² çĩ¥ +è¾ĥ å¤ļ +å¤ļå¹´ çļĦ +èĥĮæĻ¯ ä¸ĭ +为 ä¾ĭ +æijĺ è¦ģ +å´Ľ èµ· +æŃ¤ åĪ» +æľī æľºä¼ļ +æĿ¡ 款 +é¢Ĩ导 å°ıç»Ħ +çļĦ 身ä½ĵ +åįķ ä¸Ģ +央 è¡Į +ä¸įæĸŃ æıIJé«ĺ +ä»·å̼ è§Ĥ +èĬ ½ +èIJ į +æ³ķå¾ĭ æ³ķè§Ħ +ä¸į éĶĪ +ä¸įéĶĪ éĴ¢ +åĩº äºİ +èĻļ æĭŁ +æį® æĤī +çĥ¦ æģ¼ +åħ¨ æĸ°çļĦ +æī« æıı +çĻ» éĻĨ +èīºæľ¯ å®¶ +çļĦ é£Łçī© +çļĦ åŃĺåľ¨ +客 åİħ +æĪij们 å°± +æŁ¥çľĭ æĽ´å¤ļ +è¯Ħ 审 +å¸Ĥ åł´ +è¬ Ľ +å·¨ 头 +ä¸ŃåĽ½ ç»ıæµİ +äºĨ èĩªå·±çļĦ +åĨ³ è®® +çĽijçĿ£ 管çIJĨ +æĬķ 票 +åĨį 度 +è¡Į çĤº +注 åħ¥ +ä½ľä¸º ä¸Ģ个 +æ¯ı个人 éĥ½ +åįķ åħĥ +è¦ģ çŁ¥éģĵ +被 称为 +ä¹ĭ éĻħ +è§£ éϤ +ä¸ ¸ +æº « +ä¸ī æĺŁ +é²ľ æĺİ +ä¹Ł éĥ½ +æĹ¶ æľº +åĩº æīĭ +æĥħ å½¢ +åķĨ è´¸ +éĢī 举 +对 èĩªå·± +çĶŁ åĬ¨ +åħĭ æľį +个 ä½ĵ +èĭ ij +ç¨ ± +大 åݦ +æĺ¯ 对 +åĪ© æģ¯ +è¿IJåĬ¨ åijĺ +åĮĸ è§£ +åīį æ²¿ +æĦŁ æģ© +æĢ» ä¹ĭ +é«ĺæĸ° æĬĢæľ¯ +åĿĩ 为 +åħ¨ åĮº +æ°Ķ æ°Ľ +åı¯ä»¥è¯´ æĺ¯ +ä½ı 宿 +åħļåijĺ å¹²éĥ¨ +åĹ ¯ +è·µ è¡Į +çļĦ ä¸ĵä¸ļ +èĢĥ éªĮ +èķ ¾ +åħ¬ åŃIJ +çļĦ çĬ¶æĢģ +æ½® æµģ +ä¿¡ æīĺ +è´ ¼ +åIJĦ æĸ¹ +æķij åĬ© +éĿŀ常 çļĦ +æ¡¥ æ¢ģ +åħ¬ æĸ¤ +ä¼¼ çļĦ +çľĭ 好 +å±Ģ éĥ¨ +å®ī éĿĻ +éħį ä»¶ +常 è§Ħ +å¼Ģ 车 +第äºĮ 次 +ä¸Ĭ 级 +åıĤ èµĽ +å®¶ å±ŀ +强 åĬ¿ +åľ¨ ä»ĸ +åIJij åīį +ä¹ĭ åľ° +éĥ ¡ +è¡Į ç¨ĭ +èѦ åijĬ +è§Ħå®ļ çļĦ +åķĨ åŁİ +äºĶ 大 +æķĻ å®¤ +åįģ è¶³ +æīĢ以 åľ¨ +å°Ĩ ç»§ç»Ń +çŃī æĸ¹å¼ı +å®¶ ä¼ģä¸ļ +交 ä»ĺ +çĤ¹ è¯Ħ +ç»ĵ ç®Ĺ +ä¹Ł åı¯ +å¤ĸ æ±ĩ +è¿Ļç§į æĥħåĨµ +æİĪ äºĪ +å¸ĥ ç½® +æĪIJç«ĭ äºİ +é¢Ħ èѦ +管çIJĨ 人åijĺ +å©ļ 礼 +ç»ĵæĿŁ åIJİ +åħ¥ éĢī +æĹł æ¯Ķ +åĴĮ åıijå±ķ +çϽ éħĴ +çİ© åħ· +ä¸ĩ ç¾İåħĥ +çļĦ æĪIJ绩 +æĭį çħ§ +èĢĥèĻij åΰ +ä¼ģä¸ļ åıijå±ķ +äºĨ 个 +çĶŁ æ°Ķ +çļĦ 女人 +äºĶ åįģ +çĪ· çĪ· +纽 约 +éĥ½ 被 +ä¸Ĭ 课 +çĽ ¡ +ä¼łç»Ł æĸĩåĮĸ +æ½ľ åľ¨ +åıij å°Ħ +ä¸Ģ 身 +éĺ² å®Ī +åĪ ® +é¢ĺ 缮 +åľ¨ åĨħçļĦ +ç¾İ 好çļĦ +è¿ĻéĩĮ çļĦ +ä¸Ģ ä¸Ŀ +人 åĿĩ +åĢ¡ 导 +身 åIJİ +æī© å±ķ +大 éŨ +å°± 被 +该 é¡¹çĽ® +æŀ¶ æŀĦ +ä¸Ģ åı£ +ä¿¡æģ¯ æĬĢæľ¯ +å¼Ģ ä¸ļ +æĶ¶ åıĸ +ç½ij 页 +æĶ¯ æı´ +å°ģ éĹŃ +å¡ij éĢł +大 èĥĨ +å¿«éĢŁ åıijå±ķ +çľĭ ä¼¼ +æ¸ Ŀ +è¿Ļæł· ä¸Ģ个 +模 åĿĹ +注æĦı åΰ +çł´ è§£ +èĩª ä»İ +åijµ åijµ +ä¹ĭ å¾Į +ä¹ĭ æĹħ +è·Ł æĪij +æ³ķ 人 +æİĴè¡Į æ¦ľ +åĿļ å®Ī +好 å¤Ħ +çŁ³ 头 +å¹¶ å°Ĩ +èĪ ± +æŃ ĩ +两 岸 +å¤ļ ä¹ħ +象 å¾ģ +个æĢ§ åĮĸ +çļĦ è§Ĵ度 +å¸ Ĩ +ç¦ı å·ŀ +æŁ¥ å¤Ħ +两 åĽ½ +åIJ¸å¼ķ äºĨ +é¦ĸ å¸Ń +大 åĵ¥ +é¤ Ĭ +涨 å¹ħ +éĢī ç͍ +許 å¤ļ +èIJ½ æĪ· +åĵĪ å°Ķ +åĵĪå°Ķ 滨 +åģļ ä»Ģä¹Ī +以 åħį +é¾ į +æĹł éľĢ +åΰåºķ æĺ¯ +æĢ ¡ +åijĬè¯ī ä½ł +éĺ² æ°´ +è¿Ļ æĹ¶åĢĻ +欢 ä¹IJ +转 åIJij +è¿Ļ个 åľ°åĽ¾ +åħ¥ é©» +èįī åİŁ +æĹ¶ä»£ çļĦ +åıĺ åĬ¨ +åĬłå¼º 对 +åģ¶ å°Ķ +å®Ī æĬ¤ +æ°Ķ 温 +人 éĹ´ +æľĿ é²ľ +ç»ı è´¹ +åĽŃ æŀĹ +å·¥ åľ° +è§Ħ æł¼ +åĩł åįģ +è¯ķ åĽ¾ +å¦ ĥ +éĤ£ æĹ¶åĢĻ +å¼ĺ æī¬ +ä¸ļ çķĮ +çļĦ éĢŁåº¦ +ä¼ļ ä¸įä¼ļ +èIJ¥ æĶ¶ +å°ıå¾® ä¼ģä¸ļ +çľĭ è¿ĩ +æĬĬ ä»ĸ +éģµ å¾ª +è¿Ļ è¾¹ +没æľī 人 +å£ ¶ +æ¹ĸ åįĹçľģ +æŀģ åħ¶ +çļĦ人 çĶŁ +ä»ĸ è¿ĺ +转åĮĸ 为 +èµ° è¿ĩ +æĬ± çĿĢ +çīĽ å¥¶ +ä¸ĩ 亩 +å¿ĥ æĢģ +æĹ¥å¸¸ çĶŁæ´» +ä½ĵ æ£Ģ +æĻ ĥ +çŃī é¢ĨåŁŁ +æĩī 該 +åı¯ä»¥ çľĭåΰ +æī¾ ä¸įåΰ +èĢģ å¹´ +æĬĬ æĪij +积 åĪĨ +梳 çIJĨ +ç» ³ +çļĦ æĶ¿æ²» +å¸Ŀ åĽ½ +éĻª ä¼´ +æ´Ľ éĺ³ +åħ¬ æŃ£ +å¼Ģ åı£ +çī¹èī² çļĦ +åĽ° å¢ĥ +ä¸Ĭ æľī +ç«ĭ ä½ĵ +æīĵ å·¥ +åķ¤ éħĴ +åľ¨ éĤ£éĩĮ +éĤ£ è¾¹ +个 åĪ« +ä¸Ģå®ļ æĺ¯ +çļĦéĩįè¦ģ æĢ§ +主 å¼ł +åĴĮ æľįåĬ¡ +ä¸Ĭ ç½ij +è¡¥ åĬ© +åıª éľĢ +å¼ ¦ +éģ ® +åĬĽ äºī +度 è¿ĩ +èij ¬ +é¡¿ æĹ¶ +éĦ ī +纺 ç»ĩ +åľ° åĿĹ +ä¿¡ç͍ åį¡ +ç½ļ 款 +åijĬè¯ī æĪij +éĽ Ļ +书 çĶ» +è¨Ń è¨Ī +æĢ» ä¼ļ +åΤ åĨ³ +ä¿¡ èªī +个 èĤ¡ +å¹³ 常 +æĢİ éº¼ +ä½ĵ çİ°åľ¨ +é»Ħ æ²³ +åĽĽå·Ŀ çľģ +羣 缸 +åIJĦ项 å·¥ä½ľ +åĬ¨ åijĺ +å³° ä¼ļ +ä¸Ģ æľŁ +æľī ä¸Ģå®ļçļĦ +é«ĺ度 éĩįè§Ĩ +ç¹ģ èᣠ+åıijçݰ äºĨ +ç½ij 红 +æīĭ æ³ķ +å®¶ åĽŃ +仪 åύ +è¾ĥ ä½İ +çļĦ å®īåħ¨ +æ¡ IJ +ä»ĺ 款 +æĬij åζ +åįĵ è¶Ĭ +æŃ£ éĿ¢ +åĵ ij +强 åζ +ä»Ĭ天 çļĦ +æĪĺ èĥľ +楼 å¸Ĥ +æĭ¿ ä¸ĭ +é¢ľ å̼ +举 éĥ¨ +çłĶ åζ +çļĦ æĪĺçķ¥ +åľ¨ ä¸Ģ个 +ä¸ī 人 +å®Į äºĨ +æĸ° æĬĢæľ¯ +ç»ıæµİ æķĪçĽĬ +å¯Į æľī +æ¾³ æ´² +åĬ© çIJĨ +é¢Ĩ åıĸ +è° Ń +çĩĥ çĥ§ +ç´ł åħ» +éĤĦ æľī +è¿Ľ èĢĮ +ä»Ģä¹Ī æĺ¯ +çłĶç©¶ ä¸Ńå¿ĥ +éĢĤ ç͍äºİ +æİ¥ æĶ¶ +失 æľĽ +äºĮ 级 +éĹ´ çļĦ +åİŁ æłĩé¢ĺ +èªį çĤº +æį ¡ +对 çĿĢ +对 éĿ¢ +ä¸Ń åİŁ +éĵ ĥ +çĶŁäº§ çļĦ +åıijå¸ĥ ä¼ļ +士 åħµ +è¿Ļ åı¥è¯Ŀ +ç¼´ 纳 +ä¸Ģ个 个 +åѸ çĶŁ +çĸij éĹ® +交 èѦ +示èĮĥ åĮº +天 使 +åľ¨ ä¸Ĭæµ· +åIJĮ æĻĤ +è½» æĺĵ +å͝ä¸Ģ çļĦ +çĥŃ éĹ¹ +ä¹IJ è§Ĥ +çļĦ 身份 +åĸĦ äºİ +大 åİħ +èĤ¯å®ļ æĺ¯ +éĺ² çģ« +å¤ĸ åĩº +æį® 说 +é¡¹çĽ® çļĦ +ä¸Ģ åı° +èĻļ åģĩ +ä¸Ģ ç¬Ķ +ç«ĭ æ³ķ +严 èĤĥ +æī¿ åĬŀ +åįģ åĩł +çļĦ 空éĹ´ +æľ¬ ç½ijç«Ļ +åģļ å¾Ĺ +ä¿Ŀ 温 +æľĪ åĪĿ +åľ¨ ç½ijä¸Ĭ +åIJĦ æĸ¹éĿ¢ +ä¸ī 天 +交æĺĵ æīĢ +è§£ æŀIJ +åħļ ä¸Ń央 +è¿Ľ åĩºåı£ +åĴĮ 社ä¼ļ +次 æķ° +ä¹ĭ å®¶ +ç»´ 度 +æ´¾åĩº æīĢ +产çĶŁ äºĨ +带 æľī +å¾Ī 强 +æľīäºĽ 人 +å¹´ åIJİ +äºĨ 许å¤ļ +å¯Ĩ 度 +åѦ æľŁ +çıł æµ· +æľĢå¤ļ çļĦ +è¾¹ ç¼ĺ +容 éĩı +第äºĮ 个 +ä¸Ģ缴 æĺ¯ +ä¸į ç¦ģ +æŃ ² +ä»ĭç»į äºĨ +ä¼ĺ éĽħ +æ¯Ķ è¼ĥ +èģĮ ä½į +温 æŁĶ +æľī éĴ± +æľĢ é«ĺçļĦ +åįļè§Ī ä¼ļ +ä¸į æĪIJ +éĶĻ äºĨ +è¯ģ çĽij +è¯ģçĽij ä¼ļ +æĪIJ 人 +åĿĩ åĮĢ +æľī åĪ© +è¶Ĭ åįĹ +æīĵ äºĨ +好 åIJĥ +ç³» çµ± +è·Ł éļı +çļĦ åľ°ä½į +æŃ£ å¦Ĥ +ç¨į å¾® +åį° åıij +åĪĽ ç«ĭ +é£İ åħī +å°Ĩ æĪIJ为 +ä¸į é«ĺ +é¢ij ç¹ģ +设 æľī +ä¼ ŀ +æĭĨ éϤ +å½± åĥı +æ¸Ĺ éĢı +å¹´ å¼Ģå§ĭ +ç½ij æĺĵ +è¦ģ åģļ +ç͵åĬ¨ 车 +羣 å¿ĥ +æµ· åĨĽ +ä¼ł æĿ¥ +å·® åĪ« +è°¨ æħİ +çĥŁ åı° +åįĥ å¹´ +è¯ģ å®ŀ +çIJ ª +çļĦ åħ·ä½ĵ +åΰ å¤Ħ +ä¸į å®ľ +èľ Ģ +èĥ½åĬĽ åĴĮ +çīº çī² +çļĦ éĴ± +大 éĺŁ +é¦ĸ è¦ģ +ä¸į æĦ¿ +çİ« çij° +人æ°ij ç½ij +è¿ĺæĺ¯ è¦ģ +åĽĽ å¹´ +æį٠伤 +çļĦ åģļæ³ķ +éĿ Ī +è¡Ķ æİ¥ +åIJĪ æĪIJ +没 人 +éŨ æ§Ľ +ä¿¡ è´· +çļĦ 缸åħ³ +举 é£İ +社 ä¿Ŀ +ä¸ĭ 游 +åĿĹ éĴ± +è¿ĩ åIJİ +çļĦ åºĶç͍ +é¥ ¶ +é¢ģ åıij +ä¸Ģ å¤Ħ +åįİ å¤ı +为 ä¼ģä¸ļ +åıª ä¼ļ +ä¾µ 害 +çļĦ åĬŁèĥ½ +åѸ ç¿Ĵ +ä¸Ńåįİ æ°ijæĹı +åıijå¸ĥ äºĨ +è¿İ æİ¥ +æĪij èĩªå·± +è¿ĺ éľĢè¦ģ +太éĺ³ èĥ½ +åİ» ä¸ĸ +æĺ¯ ä½ł +åIJĪ åĬĽ +ç»ĺ çĶ» +åı° åĮĹ +çĿ£ ä¿ĥ +åĮĹ éĥ¨ +æľī å¤ļå°ij +å¾Ī éĩįè¦ģ +åĪĴ åĪĨ +åı· 线 +æĶ¾ 大 +ä¼ļ 被 +èİ· å¥ĸ +ä¹ĭ åĨħ +失 åİ»äºĨ +çݩ家 们 +éĩĩ éĽĨ +å£ ¹ +å®¶ ä¼Ļ +çϽ 天 +åĽłä¸º ä»ĸ +社ä¼ļ æ²»çIJĨ +å¼Ģ åĪĽ +ç͵ ç¼Ĩ +æĸ° ä¸Ģ代 +å¹¶ è´Ń +å°± å·²ç»ı +çļĦ 社ä¼ļ +éϤ éĿŀ +åı¯ä»¥ ç͍ +å© ī +æ¯Ķè¾ĥ 好 +å®ŀ ä¸ļ +åĪĽ åĬŀ +æıIJ èµ· +é» ĥ +ä½ı åľ¨ +å¸Ĥ æĶ¿ +éĿ¢ä¸´ çļĦ +èĥ½ åľ¨ +çŁŃ çŁŃ +羣 人 +æĺİ æĺİ +èµĦ åĬ© +çļĦ ä¸įåIJĮ +å°ı æľĭåıĭ +é¢ĺ æĿIJ +ç¾İ åij³ +æĺŁ åº§ +ä¸į ä¸Ģæł·çļĦ +çľĭ ä¸Ĭåİ» +ä¸Ģ æł¹ +广 å·ŀå¸Ĥ +åıijçĶŁ çļĦ +é«ĺ ç§ijæĬĢ +ä¸Ģ è¾ĪåŃIJ +交 åıī +ä½ĵç³» 建设 +åĽłä¸º æĪij +çıį æĥľ +ä¸Ĭ åѦ +æĪĺ æľ¯ +æŃ¤ ç±» +交 å¾Ģ +æĮī æij© +人们 çļĦ +åħ¶ 實 +åİŁ æĿIJæĸĻ +渴 æľĽ +缸 å¤Ħ +å¾® å¾® +æ® · +ä¹ĺ åĿIJ +å¼Ģå±ķ äºĨ +é«ĺ åĵģè´¨ +æĹłäºº æľº +ä¸įæĺ¯ å¾Ī +çļĦ æĬķèµĦ +èĬĤ çľģ +èĩ ī +ç²¾ éĢī +çļĦ æłĩåĩĨ +åįĹ éĥ¨ +认è¯Ĩ åΰ +å¹³ éĿĻ +èĹ ¥ +æī« é»ij +æī«é»ij éϤ +æī«é»ijéϤ æģ¶ +éĢĻ ç¨® +建çŃij éĿ¢ç§¯ +ç¡® ç«ĭ +管çIJĨ åĬŀæ³ķ +æĦı å¿Ĺ +ä¸ ¨ +让 åŃ©åŃIJ +æķij çģ¾ +å½ĵ ä»Ĭ +çģ« çģ¾ +åIJĦ éĥ¨éŨ +ä¾µ çĬ¯ +æ¯ı åij¨ +æı ½ +ä¸Ģ次 æĢ§ +åħ¶ä»ĸ 人 +éĶĻ è¿ĩ +ä¸İ åħ¶ +åĭĩ æ°Ķ +çĩĥ æ°Ķ +é¦ĸ å±Ĭ +æľį 饰 +ç² ¥ +å®Į æ¯ķ +å°± æĬĬ +åĬŀäºĭ å¤Ħ +ä¸Ģä¼ļ åĦ¿ +离 ä¸įå¼Ģ +å¦Ĥæŀľ æĤ¨ +ä»ĵ åºĵ +导 å¸Ī +åIJĪéĢĤ çļĦ +毫 ç±³ +å®īåħ¨ æĢ§ +ä¾Ŀ çħ§ +产ä¸ļ åĮĸ +ä½ł çľĭ +羣çļĦ å¾Ī +åѤ çĭ¬ +éĺ² å¾¡ +å¾Ī ç®Ģåįķ +é£İ æ°´ +ä½Ĩ ä¹Ł +æİ¨ åĩºäºĨ +æ°ijèIJ¥ ä¼ģä¸ļ +çłģ 头 +å¤įæĿĤ çļĦ +ç»ĦæĪIJ éĥ¨åĪĨ +åħħ满 äºĨ +è¿ij åĩłå¹´ +çľģ æĶ¿åºľ +æľī å¿ħè¦ģ +éĻ ³ +ä¹ĭ ç±» +ä¹ĭç±» çļĦ +æĢ§ ä»· +æĢ§ä»· æ¯Ķ +åķĨ åºĹ +å¸Ĥ å̼ +人æīį åŁ¹åħ» +æ·± åıĹ +管çIJĨ å±Ģ +æģIJ æĥ§ +ä»ħ æľī +æĬµ è¾¾ +æµ· åħ³ +èµĭ äºĪ +äºĭ åĦ¿ +ä»· éĴ± +æīĭ ä¸Ĭ +èĩª å¾ĭ +åħ³ çα +享 æľī +éģĹ æĨ¾ +å¾Īå¿« å°± +æĽ´ å¿« +æłĩ è¯Ĩ +åºĨ ç¥Ŀ +ä¹Ł 好 +ä¸į æĺĵ +æĪij å¾Ī +æĶ¹éĿ© åıijå±ķ +å¤ĸ åľ° +æĬµ æĬ¼ +è¯Ĺ 人 +åİķ æīĢ +æĸ° åªĴä½ĵ +èĸ Ľ +è°Ī è¯Ŀ +ä¸Ģå®ļ ç¨ĭ度 +èµ° åľ¨ +æľĢ 强 +åĬŁ çİĩ +åħ± è¯Ĩ +大 æ¡¥ +ä¸ĭ æĸ¹ +å¤ĸ èµĦ +ç¢ ± +å·¡ è§Ĩ +æ¹ĸåĮĹ çľģ +个 çϾåĪĨ +个çϾåĪĨ çĤ¹ +çļĦ 责任 +çļĦ åĵģçīĮ +åĬ© æİ¨ +åĪĽéĢł äºĨ +ä»» èģĮ +å¿« æį· +æĿij åºĦ +åİ» çľĭ +æīį èĥ½å¤Ł +å± ¤ +æĪij å®¶ +æĺ¯ä¸Ģ 款 +ç¾ ħ +åĨ° éĽª +æŀģ 大 +çģ¯ åħī +éĨ ĭ +ä¸İ åħ¶ä»ĸ +æıIJåĩº çļĦ +éĿł è¿ij +è°ĥ åĬ¨ +å°½ åı¯èĥ½ +åıij åĬĽ +ç»Ļ 她 +éĢĤ éĩı +è·¨ åĽ½ +åħĪ è¡Į +æĸ° æĿIJæĸĻ +ä½ľ äºĨ +满 äºĨ +ä¸į 满 +çļĦçľ¼ çĿĽ +çľĭ å¾Ĺ +è¿Ļ ä¸Ģ次 +é½IJ åħ¨ +çļĦä¸Ģ éĥ¨åĪĨ +ä¸ Ļ +æ¸ħ æĸ° +說 æĺİ +身边 çļĦ +æīĢæľī 人 +å½° æĺ¾ +è± ¹ +åį ¿ +è¿IJ 转 +æĮĩ å¼ķ +å¸Ĥ åħ¬å®īå±Ģ +åıĤ å±ķ +ä¹ĭ æĹ¶ +éĩijèŀį æľįåĬ¡ +èµĦæľ¬ å¸Ĥåľº +èĥ½ 让 +å¿ĺ äºĨ +天 åłĤ +æ¯Ķå¦Ĥ 说 +éĬĢ è¡Į +èĽĭ ç³ķ +çĶ © +æł¸ å®ŀ +æĻ® 京 +ä¼ĺ ç¾İ +åı£ èħĶ +漫 çĶ» +çľ¼ éĩĮ +äºĨ ä¸ĭæĿ¥ +æĪij们 ä¹Ł +ä¾ į +为 ä¸Ńå¿ĥ +å¥ĩ 迹 +éĿĴ çĿIJ +æĪªèĩ³ 缮åīį +åĩº ä¾Ĩ +æĢ» åħ¬åı¸ +å¼¥ è¡¥ +ç®Ĺ æ³ķ +å·¥ä½ľ 室 +æīĢ以 æĪij +æ°´ åĪĨ +æīĢ å±ŀ +ä¸į 说 +ä½Ĩæĺ¯ åľ¨ +è¦ģ åİ» +åĪĽä¸ļ èĢħ +ä¸į æ¸ħæ¥ļ +åĽĽ åij¨ +æĺ¯ ä»İ +çļĦ æł¹æľ¬ +çģ ¶ +æ¯Ľ æ³½ +æ¯Ľæ³½ 举 +æµ· åı£ +åĽĽ åįģ +ä¹Ł 被 +èģ · +ä¸Ģ æīĭ +绩 æķĪ +çļĦ çĶ·äºº +书 ç±į +ä¸Ģ èĦ¸ +大 äºİ +鼶 éĥ¨ä»¶ +åħ³ æĢĢ +å¹³ ç±³ +æļ´ éľ² +å¾Ĺ å¤ļ +ä¸ī 级 +æľ¬ åij¨ +两 èĢħ +对 ä¸ŃåĽ½ +åıª è§ģ +欧 ç¾İ +å¦Ĥæŀľ æľī +å·²ç»ı æĺ¯ +çľĭ å®Į +çģ« éĶħ +èµ IJ +ä¸Ģ éģį +æĦŁ åĨĴ +ç»ĵ å±Ģ +ä»ĵ åĤ¨ +å®ŀ åľ° +å̻ ç»ıçIJĨ +ä¹Łä¸į çŁ¥éģĵ +碰 åΰ +åIJĪ è®¡ +客æĪ· çļĦ +ç½Ĺ 马 +æĦī å¿« +é£ Ľ +çĥŃ çĥĪ +伦 æķ¦ +åĮ» ä¿Ŀ +éĺ¿éĩĮ å·´å·´ +åĨį 说 +为 åŁºç¡Ģ +çĶŁäº§ ç»ıèIJ¥ +è¿ĻäºĽ 人 +åĪĹ è½¦ +æ²³åĮĹ çľģ +è¿Ļ 段 +æ´»åĬ¨ ä¸Ń +å© · +çĶŁ çIJĨ +ä¸ŃåĽ½ 人æ°ij +éĦ Ĥ +åIJ¬ åıĸ +å¤į ä¹ł +æľī çĽĬ +æĶ¶ æĭ¾ +å¾Ī åı¯èĥ½ +ç½ij绾 游æĪı +们 çļĦ +èµĭ èĥ½ +éļ¾ å¾Ĺ +åĪĨ æīĭ +羣 è¯ļ +åħ¬åı¸ åľ¨ +åĿĩ è¡¡ +åı£ åij³ +çīµ å¤´ +ä¸Ģèά çļĦ +轿 车 +çŃī äºİ +æ²ī é»ĺ +æĪij éĥ½ +å°ı ç¨ĭåºı +ä¸Ģ åī¯ +æī¿ è½½ +åľ° è´¨ +çķĮ éĿ¢ +ç͵ æľº +çĦ¦ èĻij +éĶĢåĶ® é¢Ŀ +æĸ° 车 +ä¸Ĭ 游 +主 æ¼Ķ +éļIJ ç§ģ +åıijå±ķ æĪĺçķ¥ +çļĦ åĬªåĬĽ +å¼Ģ åħ³ +è§£åĨ³ éĹ®é¢ĺ +çĿ£ 导 +对 æĬĹ +å¾Īå¤ļ 人éĥ½ +æĹł æķĪ +产åĵģ è´¨éĩı +å®ī å¿ĥ +åįİ äºº +ä¸į 符åIJĪ +èĩª å®¶ +éĺµ å®¹ +çļĦ åIJĦç§į +çļĦ çIJĨ念 +çļĦ æĸĩåĮĸ +为 èĩªå·± +å±± æ°´ +游 æ³³ +éľĩ èį¡ +çĶŁæ´» æĸ¹å¼ı +è¿ľ 离 +çŁ³ åĮĸ +æŃ¤ äºĭ +æĺ¯ 羣çļĦ +çļĦ æ¯Ķä¾ĭ +ç͍ ç͵ +奥è¿IJ ä¼ļ +ä¿Ŀ å®ī +èĽĭçϽ è´¨ +çļĦ å¿ĥçIJĨ +å· « +åı· çłģ +æ°Ķ ä½ĵ +åıij æĶ¹ +åıijæĶ¹ å§Ķ +åĮ» å¸Ī +æ¶Ĥ æĸĻ +æĺ Ĭ +å¸Ĥ 级 +ä¸ĸçķĮ çļĦ +åĪĨåĪ« æĺ¯ +çł´ 产 +ä¸Ģ æĿ¯ +æĭī å¼Ģ +å¹³ åĩ¡ +çļĦ åıijçĶŁ +åĬ¨ æīĭ +ä¸Ģ缴 以æĿ¥ +æīĭ å·¥ +éĩĮéĿ¢ çļĦ +æĹł åħ³ +ä»ĭ åħ¥ +èµ° ä¸Ĭ +å°±æĺ¯ è¦ģ +å¹´ éĹ´ +åĩº çı¾ +å½± éŁ¿ +å¹ħ 度 +éĽ ģ +éģĵ åħ· +缮çļĦ åľ° +åIJİ èĢħ +ä¸Ĭ æ¼Ķ +äºĨ åĩł +æ®ĭçĸ¾ 人 +å¿Ļ ç¢Į +æĺ¯åIJ¦ æľī +å¹¶ 对 +ä¼ļ 导èĩ´ +æ°´ åºĵ +ç»Ĩ èĩ´ +åIJİ æĤĶ +å¿ĥ æĢĿ +åģļ äºĭ +åİĤ æĪ¿ +çĿ ¿ +è¿IJèIJ¥ åķĨ +头 éĥ¨ +çļĦ è§Ĵèī² +æĺ¯ ä»ĸ +æĹ¢ æľī +å°ıæĹ¶ åĢĻ +强 åĬ² +主 æĴŃ +åħ¨åĽ½ åIJĦåľ° +æį ı +æįŁ åĿı +åķĨ ä¼ļ +ä¿Ŀ ç½Ĺ +çľģ å¸Ĥ +éļ§ éģĵ +æľī ä¸įå°ij +è¦ģ åľ¨ +建设 é¡¹çĽ® +ç³ĸ å°¿ +ç³ĸå°¿ çĹħ +æĿ¡ä»¶ ä¸ĭ +ä¼ĺè´¨ çļĦ +é¦ĸ åıij +å½ĵæĹ¶ çļĦ +丰 çͰ +大 çĽĺ +缸 ç»§ +å®ģ å¤ı +åħ¥ ä½ı +æĪij è¿ĺ +åħĭ æĸ¯ +å®ļ ä»· +å¹³æĸ¹ åħ¬éĩĮ +çļĦ çŁ¥è¯Ĩ +æĪij们 ä¼ļ +åħĥ å®Ŀ +ä½ĵ éĩį +è³ £ +对 æĪij们 +çŁ³ å®¶ +çŁ³å®¶ åºĦ +ç²¾ åįİ +å½¢ çĬ¶ +åıĹ åΰäºĨ +ä¿® 订 +ç¾İ åľĭ +é«ĺ æ¸ħ +çľ¼ éķľ +è§īå¾Ĺ èĩªå·± +带 ç»Ļ +åĶ® ä»· +éŨ 票 +åŃķ å¦ĩ +ç͵è§Ĩ åı° +åıij ä½ľ +çļĦ åij³éģĵ +éķ¿ è¿ľ +åħ¬åħ± æľįåĬ¡ +æŃ£å¸¸ çļĦ +æľī è¿ĩ +é£İ æĥħ +æ¯Ķ éĩį +åIJ » +管çIJĨ å·¥ä½ľ +综åIJĪ æĢ§ +å·² 被 +说 èµ· +æİĴ æ°´ +ä¸įæĸŃ åľ° +æĥħ æĢĢ +è¾ĵ éĢģ +è¿ĩ æķı +çļĦ åı¯èĥ½æĢ§ +æľį ç͍ +æľī 许å¤ļ +å§Ķ åī¯ä¹¦è®° +åĮĸå¦Ĩ åĵģ +æļĤ åģľ +æĬķèµĦ 人 +çıŃ çº§ +说 çĿĢ +åįĹ åĮĹ +åĪĨ è¡Į +çıł å®Ŀ +å¯ ¶ +å¢ŀ å¤ļ +被 åĬ¨ +ç®Ĭ çļĦ +éĹľ ä¿Ĥ +çļĦ èĦ¸ +æĥ Ł +ä¸į ä¸Ģå®ļ +ç¶ Ń +çģ« çĪĨ +ç§Ł éĩij +çŀ § +éĩį 建 +è· ª +ä¸Ģ 種 +çļĦ åIJĪä½ľ +å®ī æħ° +ä»į æĺ¯ +ä¸ĵä¸ļ åĮĸ +è°ĥ è§£ +ä¸į 妨 +éĢĻ æĺ¯ +å¿ħ éłĪ +ä¼Ĭ æľĹ +å¾Ĺ äºĨ +æľįåĬ¡ å¹³åı° +å§ ¬ +åħĪ éĶĭ +çİĭ åŃIJ +çļĦä¸Ģ åĪĩ +æĢ» çIJĨ +åĵ ¼ +çª ij +çļĦå¿ĥ æĥħ +çļĦ éĩį大 +çij Ł +ä¸Ģ ç¬ij +åıijå±ķ ä¸Ń +åģ¥åº· åıijå±ķ +åĵģçīĮ çļĦ +ç¦ ® +ä½Ļ 人 +ä»Ĭå¹´ 以æĿ¥ +æķ° çłģ +çѾ è¯ģ +åİ» æī¾ +åŁºéĩij ä¼ļ +æĬ± æĢ¨ +æŃ£ å½ĵ +çıŃåŃIJ æĪIJåijĺ +ä¸į åIJĪæł¼ +åζ å®ļäºĨ +ç¼ĵ æħ¢ +åζ 约 +æłı 缮 +å¸Ĥåľº ç»ıæµİ +ç»ĦæĪIJ çļĦ +严 å³» +æĹ¥ 讯 +ä¸ĢçĤ¹ çĤ¹ +æĺ¯ æĢİä¹Ī +çļĦ çħ§çīĩ +éĺ» æŃ¢ +模 ç³Ĭ +ç¼ ¸ +éģķ åıį +æIJ¬ è¿ģ +éĩij éĴ± +å½ ¬ +ä¸į å®ī +æĪĺçķ¥ åIJĪä½ľ +å¡« åĨĻ +讲 ç©¶ +åħħåĪĨ åĪ©ç͍ +èĥ½ å¤ł +èij¡èIJĦ éħĴ +éĩĩç͍ äºĨ +åľ¨ ä»Ĭå¹´ +ä¸Ńå°ı åѦ +åľ¨ æĦı +çļĦ åİĭåĬĽ +ä¸į 幸 +åζ èᝠ+åı¯ä»¥ 让 +被 è¯Ħ为 +ç»Ĩ èıĮ +æĪı åī§ +åįĬ 导 +åįĬ导 ä½ĵ +è§Ĩ è§Ĵ +åĸľ æŃ¡ +å¾ģ æĶ¶ +è°ĭ åĪĴ +æŀģ 大çļĦ +çĤ¹ èµŀ +è®°èĢħ ä»İ +两 åIJį +èĩª åĬ© +èµ· æŃ¥ +æĬ¤ 士 +å®Ŀ 马 +太 åŃIJ +å°ıå°ı çļĦ +温 æ³ī +åĩºç§Ł 车 +ç§Ł æĪ¿ +两 å®¶ +éľĩ æĴ¼ +ç§ī æī¿ +ä¸Ģä»¶ äºĭ +çĥΠ士 +å®ĺ åħµ +转 身 +ä¹IJ åĽŃ +çĻĮ çĹĩ +模 èĮĥ +æĦ £ +è¿ĩåİ» çļĦ +代 ä»· +çļĦ æ¦Ĥ念 +åĩł çϾ +è´µ éĺ³ +æĭħ å¿§ +éĢĤ å®ľ +çݯå¢ĥ ä¿ĿæĬ¤ +çĥ « +ä½ł æĥ³ +æŃ¤ åIJİ +ä½ł ä¹Ł +çį İ +éϤ æŃ¤ +éϤæŃ¤ ä¹ĭå¤ĸ +è°ĥ 度 +ç§ij 缮 +æīĢ说 çļĦ +åĬ ĩ +忽 è§Ĩ +ä¸ī 次 +ä¸Ģ æĹ¥ +åŀĤ 缴 +ç«ŀ æĬĢ +éĿ¢ åĮħ +大 æĪĺ +æIJº 带 +å¦Ĥæŀľ 没æľī +åħ» æĪIJ +åĩº è¡Ģ +çα好 èĢħ +æīĵ éĢļ +èµ· è¯ī +åijĪ çݰåĩº +æŃĮ æīĭ +åľ¨ å¤ĸ +é¢Ĩ导 å¹²éĥ¨ +åĨ ¥ +èĪĨ 论 +æıIJ åıĸ +éĺ¿ å°Ķ +æľĽ çĿĢ +ä¸ī äºļ +è² ¡ +åĪ ·æĸ° +æĻļ æĬ¥ +è¿ĺæľī ä¸Ģ个 +åĨ° ç®± +ç½ij çĤ¹ +åĩº åħ· +强çĥĪ çļĦ +æĪij çĽ¸ä¿¡ +å¸ĮæľĽ èĥ½ +çīĻ é½¿ +äºĭ å®ľ +ä¸ļåĨħ 人士 +代 æĽ¿ +åıĺ å½¢ +éĽ ² +è°ĥ æİ§ +åĪĽæĸ° åĪĽä¸ļ +æĭĨ è¿ģ +æł¸ æŁ¥ +éĢ Ĺ +åħ¥ åѦ +æĦı åIJij +æı Ľ +ä¸ĭ 次 +ä¼ł è¾ĵ +ä»ĸ们 åľ¨ +èĢĮä¸Ķ è¿ĺ +æĹ¥ åľ¨ +æķĻ è®Ń +æ´» çĿĢ +çļĦ æľīæķĪ +å¤įå·¥ å¤į +å¤įå·¥å¤į 产 +æĺ¯ä¸Ģ ä»¶ +çŃī çĿĢ +å¾ © +åĭĩ æķ¢ +éģŃ åıĹ +å¥Ķ é©° +讲 座 +说 å®Į +ç»Ļ åĩº +è° ¦ +è¯Ĭ çĸĹ +çĽ² 缮 +客 è¿IJ +å°± è¿ŀ +å¼Ģ åħĥ +å¼Ģåħĥ æ£ĭçīĮ +ä¸įæĸŃ æıIJåįĩ +ç͍æĪ· çļĦ +æĴ ķ +ä¾Ľ æ°´ +ç¶ĵ æ¿Ł +ä¸Ń åĮ»èᝠ+èģĶ æĥ³ +åħ¬äº¤ 车 +èĪª çıŃ +æĬĢ è¡ĵ +å¼ķèµ· çļĦ +å° ¹ +èµĦ æ·± +åĽ½èµĦ å§Ķ +èĺ Ń +é¼» åŃIJ +éĹ ½ +æİĴ éĺŁ +è§Ĥ åħī +éģĹ åĿĢ +举 京 +é¥Ń åºĹ +ä¸įæĸŃ çļĦ +å°±æĺ¯ ä¸Ģ个 +éķ¿ ä¹ħ +çļĦ è§ĤçĤ¹ +å¨ ¶ +æĪij çİ°åľ¨ +çķ ° +å¾Ĺ åĩº +å¿ħ å®ļ +ä¸į åıĹ +åıª éľĢè¦ģ +åĽ° æī° +ç§ijåѦ æĬĢæľ¯ +çīĽ èĤī +è¾ĥ é«ĺçļĦ +è·ij æŃ¥ +æ² ¾ +èı© èIJ¨ +æľĢ å¾Į +ä¿Ŀ å¯Ĩ +æ²» å®ī +éĤ ± +常 è¯Ĩ +èĦ¸ èī² +åĮĹ å¤§ +æ±ĩ èģļ +æijĨ èĦ± +é¾Ļ头 ä¼ģä¸ļ +女 åıĭ +çŃī å·¥ä½ľ +ä¸Ń ç¾İ +èģĮ åľº +èĦij è¢ĭ +åĨĻ çļĦ +饲 æĸĻ +åĬ³ åĬ¨åĬĽ +å± ¯ +æĮģ èĤ¡ +åĽ¾ åĥı +è¿ĩåİ» äºĨ +è² ¨ +è¾ ² +éĹ® æĪij +è·Ł ä½ł +çĶŁ æŃ» +审 ç¾İ +é¢Ĺ ç²Ĵ +ä¸Ń æĸ¹ +åĬł çĥŃ +æĹħè¡Į 社 +çϼ çĶŁ +ä¸į åłª +åĤ · +æ¥ ł +åĬŀ æ¡Ī +æŁ Ħ +æĹ¢ æĺ¯ +å¤Ħ åĪĨ +羣å®ŀ çļĦ +æĬ¥ 纸 +å¸Ī çζ +å®īå¾½ çľģ +åī¯ ä¸»å¸Ń +ä¹ĭ éģĵ +导 å¼¹ +åŃ¦æł¡ çļĦ +åŁİå¸Ĥ çļĦ +è°Ī åΰ +æ¢ Ĺ +å¹³ éĿ¢ +说 ä»Ģä¹Ī +é¢ij çİĩ +éķ¿ ä¸īè§Ĵ +çļĦ åĪ©çĽĬ +é» ¨ +è±Ĩ èħIJ +å®ŀéĻħ æĥħåĨµ +æŀĹ ä¸ļ +纪æ£Ģ çĽijå¯Ł +ä½ı éĻ¢ +çļĦ æķ´ä½ĵ +åīį è¡Į +æĮ ¨ +çħ¤ çŁ¿ +å̻ è£ģ +å°ı åIJĥ +æŀģ 端 +å©Ĩ å©Ĩ +çݰ è´§ +è¯Ĺ æŃĮ +éĴ¥ åĮĻ +缩 çŁŃ +ä½Ĩ è¿Ļ +æĸ° åĵģ +è¿Ļ 对 +çŁ¥åIJį 度 +å¿ĹæĦ¿ æľįåĬ¡ +大 å±Ģ +è¡¡ éĩı +ä½ĵçݰ äºĨ +æ¡ĥ èĬ± +åIJ¸å¼ķ åĬĽ +åł ¤ +æĵħ éķ¿ +åĴ Ĵ +缸 æľº +ä¸Ģ ç«Ļ +ä¸Ģç«Ļ å¼ı +æľĢ ç¾İ +æ°¸ ä¹ħ +çļĦ éĥ¨åĪĨ +åĪĨ å·¥ +å·¥ç¨ĭ 建设 +æIJŃ è½½ +æ°´ ä¸Ń +èĮ ¨ +çļĦ æĵįä½ľ +绣 æ²» +çķħ éĢļ +åħļçļĦ åįģ +è¼ ¸ +æ¸ ¬ +ç¾İ è§Ĥ +ä¸į åĪ© +åıį æĢĿ +éªĦ åĤ² +æłĩ çļĦ +æĿĢ äºº +éĺ¿ å§¨ +é£Ł æĿIJ +åIJĥ çļĦ +åIJİ åĨį +çŁ £ +两 ä¾§ +æ¸ħ æ°´ +è¿Ľ çIJĥ +å¼Ģå§ĭ äºĨ +åIJ¬ äºĨ +çĦĬ æİ¥ +çŁ ® +å¨ Ł +为 人 +éĢģ ç»Ļ +åĨĴ éĻ© +æķ · +ç»Ī æŃ¢ +æīį çŁ¥éģĵ +è¿IJ æ°Ķ +éĢļ é£İ +æĥĬ è®¶ +ç§ijåѦ éĻ¢ +æıIJ éĹ® +太 åİŁ +缸åIJĮ çļĦ +ä» ķ +èģ ĸ +æĥħ æ³ģ +é¢Ĩ导 人 +åĩºæĿ¥ äºĨ +沿 线 +éĻ ½ +æĦŁ è¦º +ä»į åľ¨ +æ© Ļ +约 为 +åĸĿ éħĴ +ç͍ èᝠ+ä¸ĭ ä¸Ģ +æ³ķ å®ĺ +顺 åºı +åģļ ä¸Ģ个 +åĭ ¢ +æŃ ª +ç͵ ç«ŀ +ä¼´ éļıçĿĢ +ä¹ĭ åĬĽ +ä¹ĭ 人 +äºij 计ç®Ĺ +åĪ«äºº çļĦ +ç§ijåѦ åıijå±ķ +第 åħ« +å¹² æī° +女 ç¥ŀ +è¿Ļæł· åģļ +å¤Ħ åľ¨ +æ°´ è´¨ +éķ¿ æĺ¥ +å¸Ĥåľº éľĢæ±Ĥ +ç»´ æĿĥ +è̳ æľµ +æĸĩåĮĸ çļĦ +奶 ç²ī +ä¼ł è¾¾ +æīĭæľº çīĪ +æĽ¾ åľ¨ +äºĮ æľŁ +åİŁåĽł æĺ¯ +æºIJ 头 +åıĪ èĥ½ +è£ ¸ +æĬĢæľ¯ åĪĽæĸ° +æĸĩåĮĸ æĹħ游 +åıij 票 +å¹´ 级 +ä½ł ä¸į +ä¹ĭ å¿ĥ +æķ° çϾ +åIJij å¾Ģ +èĢģ å®¶ +åľĭ éļĽ +çļĦ é«ĺ度 +æľĿ éĺ³ +æ¸ħ éϤ +èĩª æľī +书 ä¸Ń +游æĪı è£ħå¤ĩ +ä¸ĩ å¤ļ +驾驶 åijĺ +ä½ł çŁ¥éģĵ +åĽ½ åºĨ +é£Ł åłĤ +æİ¥ åı£ +æĢ» æķ° +åħ¶ä»ĸ çļĦ +çĶŁåij½ çļĦ +ä½ł åľ¨ +çļĦ 缮åħī +è¿Ļ æĸ¹éĿ¢ +éĥ½ 说 +çĸĹ æ³ķ +åĭĩ 士 +åľ¨ åħ¨çIJĥ +ä¿ĿéĻ© åħ¬åı¸ +çĿ£ æŁ¥ +åĸĦ èī¯ +表 å½° +è¹ ² +è·¯ 段 +æľĥåĵ¡ è¦ı +æľĥåĵ¡è¦ı ç¯Ħ +æĪ· åŀĭ +ä¿ĥ 使 +ä¿® 建 +é«ĺ æ°´å¹³ +åģļ åĩºäºĨ +主 åľº +è¡Į èµ° +空 çϽ +æľī人 说 +è¿Ļ个 ä¸ĸçķĮ +åIJį ä¹ī +å®Į ç¾İçļĦ +羡 æħķ +åıĬ åħ¶ä»ĸ +åı¯ ç͍ +æĭ IJ +è¾ĥ 大çļĦ +æĬĢæľ¯ åĴĮ +å°¼ äºļ +çϾ è´§ +æı ī +éĢī è´Ń +éĺŁ åıĭ +ä¼ł æĦŁ +ä¼łæĦŁ åύ +åıªè¦ģ ä½ł +为ä»Ģä¹Ī è¦ģ +ä¸ĵ注 äºİ +ä½Ļ é¢Ŀ +åħ¸åŀĭ çļĦ +缮åīį å·² +欲 æľĽ +èģĶ ç»ľ +æµģ ä¼ł +çļĦ å®¶åºŃ +åı· åı¬ +çıį è´µ +ä¼Ł 大çļĦ +éī´ äºİ +è·Ł ä»ĸ +产 çī© +ä¸į å·² +è¿Ŀæ³ķ è¡Į为 +头 ä¸Ĭ +åĪĨ è§£ +åı¯ä»¥ çľĭåĩº +æł¡ åĮº +åŃĹ ä½ĵ +ä¿® çĤ¼ +çĶļèĩ³ æĺ¯ +微信 åħ¬ä¼Ĺ +åıĸ 代 +èIJ¥ä¸ļ æĶ¶åħ¥ +æ½į åĿĬ +ä½ł èĥ½ +社ä¼ļ ä¿Ŀéļľ +æ¯ĶèµĽ ä¸Ń +污水 å¤ĦçIJĨ +夫 å¦ĩ +ä¸Ģ å¹ħ +沿 æµ· +åı£ æĦŁ +ä½Ĩ åį´ +å½ĵ æĹ¥ +çļĦ æľĢ大 +æ¯ı ä¸Ģä½į +没 äºĭ +çī¹ åĪ¥ +å¼Ģ åѦ +è·¯ éĿ¢ +å¿ĥçIJĨ åѦ +æĶ¾ ç½® +éĩįåºĨ å¸Ĥ +ä½ł èĩªå·± +æ¶Īè´¹èĢħ çļĦ +ä¸Ģ æ³¢ +èѦ æĥķ +å᧠室 +注 å°Ħ +é£İ 鼨 +沿 çĿĢ +åijĬ 訴 +表 çݰåĩº +åĽĽ æĺ¯ +åı¤ åħ¸ +æĽ´ éĩįè¦ģçļĦ +好 äºĭ +çľ¼ 泪 +æ¨ ĵ +审 åΤ +碰 æĴŀ +车 ç«Ļ +è¿Ľåħ¥ äºĨ +éĽĨ åIJĪ +æł¼ å¤ĸ +宾 é¦Ĩ +æĶ¯ä»ĺ å®Ŀ +她 æĺ¯ +æĺ¯ å¦Ĥä½ķ +人 次 +çļĦ æĪIJåĬŁ +æĹł åĬĽ +æµ· æĭĶ +æĺ¥ åŃ£ +éĥ½ ä¸įä¼ļ +çŃī å¤ļç§į +ä¸Ģ个 å°ı +åģľè½¦ åľº +让 æĽ´å¤ļ +è¿Ļ çĤ¹ +æĪIJ åĵģ +éĴ ī +éģĩ è§ģ +çıŃ ä¸»ä»» +æĦı æĦ¿ +çļĦ åIJĮåѦ +游 è§Ī +åİĭ 缩 +åľ¨ ä¼łå¥ĩ +å¼¹ æĢ§ +æĹ¥ åĨħ +ç¦ı建 çľģ +è§Ĵ èIJ½ +åĪĨ å¼Ģ +ä¼ļ 让 +å¤ĸ åĽ´ +çĨŁæĤī çļĦ +çĨ Ķ +ä¸ĩ è¾Ĩ +å¤ľ éĹ´ +车 身 +ä¸Ń æľŁ +å®ĮåĸĦ çļĦ +åĵģ ç±» +åıĭ è°Ĭ +éĢīæĭ Ķ +éªij 士 +å½ ¦ +çļĦ çľĭæ³ķ +åĽ½ çİĭ +è¾£ æ¤Ĵ +åıijå¸ĥ æĹ¶éĹ´ +åı¤ åŁİ +éļı æľº +ç« ĸ +å¼Ģ è¾Ł +ä¼Ĺ çĶŁ +没 åĬŀæ³ķ +åįĥ éĩĮ +æĿ¥æºIJ äºİ +çļĦ æĿĥåĪ© +æ¯Ķ åĪĨ +满æĦı çļĦ +ä¿® è¡Į +åĿ ł +大 æµ· +èİ ¹ +åĩº 身 +è« ĩ +åħ³ èĬĤ +åIJį 人 +éľĢè¦ģ 注æĦı +æĹ© æĻ¨ +å¤ĸ åįĸ +åıĪ è¦ģ +æ¶ī æ¡Ī +çĶ³è¯· 人 +éĻĦè¿ij çļĦ +åĬłå¿« æİ¨è¿Ľ +æĸ° å¹´ +大 è¡Ĺ +ä¸Ģ é»ŀ +èĭı å®ģ +æĤĦ æĤĦ +èĦ¾ æ°Ķ +å¸Į èħĬ +éļı åį³ +æķ¢ äºİ +å®ŀè·µ ä¸Ń +æĺ¯ 没æľī +æľīè¶£ çļĦ +æĿ¥èĩª äºİ +è£ģ åΤ +女 åŃ©åŃIJ +èĩ³ åħ³ +èĩ³åħ³ éĩįè¦ģ +æĻº åĬĽ +èµ° åĩºåİ» +çŁŃ æĿ¿ +大 åĽ½ +çļĦ 认è¯Ĩ +å¹´ å¤ľ +åĨį åΰ +åIJĮ æł·çļĦ +å¯Ĩ å°ģ +å¤ĸ交 éĥ¨ +çĶŁ æķĪ +æĤ¨ åı¯ä»¥ +ä½ł åĢij +è¿ĩ å¹´ +å¼ ĵ +è¡Į æĿİ +æ¯Ķ èµ· +身 é«ĺ +è¿Ļ个 人 +ä¸Ń å¤ĸ +éģĵ æŃī +çĽ¯ çĿĢ +亲 åŃIJ +éĹ ¸ +çϽ äºij +èĦĸ åŃIJ +ä¸ĢåĪĩ éĥ½ +æ· ij +è° ľ +åģ¶ çĦ¶ +éĿł è°± +é«ĺ 管 +ä¸ĭ åıij +æĶ¾ åΰ +ç±» åĪ« +ä¸ĭ åĪĹ +æ·· ä¹± +åIJĪæ³ķ æĿĥçĽĬ +çݯ çIJĥ +æľīæķĪ åľ° +åķĨ æĪ· +æ¹ĸ 人 +æµ· 岸 +æĬķ 产 +两 个æľĪ +éĥ½ éĿŀ常 +å¢ŀ强 äºĨ +æĿ¥ åΰäºĨ +åī© ä½Ļ +æĤ¨çļĦ åŃ©åŃIJ +æµģ æ°´ +æŃ£ ä¹ī +天 çĮ« +åģļ è¿ĩ +ä½ķ æĹ¶ +æĪij åİ» +çľģ 份 +å¥ĸ éĩij +该 å¦Ĥä½ķ +ä¸ĭ çıŃ +åģ¶ åĥı +æijĨ æĶ¾ +æĸ° 模å¼ı +æĬķ è³ĩ +è·¯ åı£ +åĨľæ°ij å·¥ +大 åѸ +ä»¶ äºĭ +æł¹æľ¬ ä¸į +æµĵ 度 +æµĵ åİļ +è½® èĥİ +æĪ¿ ä¼ģ +éĿŀ常 好 +ä»İ ä¸Ń +人 æł¼ +ç¿ ģ +æĹ¶éĹ´ åĴĮ +è¿Ļ ä¸įæĺ¯ +åΏ åķĨ +æĥĬ 人 +åύ å®ĺ +åĩĨ åĪĻ +æĥħ æĻ¯ +æĽ´ é«ĺçļĦ +åѦ å®¶ +泡 沫 +åľ°æĸ¹ æĶ¿åºľ +å°± çŁ¥éģĵ +åij¼ åIJģ +ç»ı è´¸ +èĬ± éĴ± +æľī ä¸Ģ次 +æĦŁ æħ¨ +ä¸Ģ åįĥ +å¤ľ æĻļ +詹 å§Ĩ +詹å§Ĩ æĸ¯ +è¦ģ éĹ» +ç» Ĵ +æºIJ äºİ +çļĦ è´¨éĩı +注æĦı äºĭ项 +æħ¢ æĢ§ +稳å®ļ çļĦ +建设 åĴĮ +æĻ¯ 象 +éĩı åĮĸ +çļĦ 話 +è¯Ħ 级 +æº ľ +红 åĮħ +éĢļ éģİ +社ä¼ļ 责任 +æĸ° 产åĵģ +åĨ· éĿĻ +çľĭ ä¸įåΰ +èģĶ éĤ¦ +éŃ Ħ +çļĦ åīįæıIJ +çļĦåīįæıIJ ä¸ĭ +è¾ĥ 好 +çļĦ æĦŁæĥħ +客æĪ· æıIJä¾Ľ +çĭ¬ èĩª +å¢ŀ æĶ¶ +æĸĩ çĮ® +æĭ¼ åij½ +管çIJĨ åĴĮ +æµģåĬ¨ æĢ§ +åħ¨ å®¶ +ä¸Ĭ æĸ¹ +æİ¨åĩº çļĦ +ä¸ī åĽ½ +ä¸Ģ个 æĺ¯ +æĸ° ä¸Ģè½® +æĸĩåĮĸ éģĹ产 +æ® º +大 æ¹¾åĮº +éĥ½ éľĢè¦ģ +çļĦ å®ŀéĻħ +ç· Ĭ +大 å¥ĸ +åħī èĬĴ +便 äºİ +çļĦ 表æĥħ +æ¼Ķ ç»İ +红 åĨĽ +å½ĵ æĪij +æ²» æĦĪ +é¢Ŀ 度 +éĿ ľ +ä»»ä½ķ 人 +è¡Ĺ 头 +çī¹ æĸ¯ +çĸ¯ æĭī +åĮ»çĸĹ æľºæŀĦ +ç»Ļ åŃ©åŃIJ +è§Ħ 磩 +è£ ľ +çļĦ 身影 +ä¸ĵ æłı +æĿ¥ 临 +ç«¥ å¹´ +å¤į èĭı +è¨ Ĥ +åŀĭ åı· +åĽ¾ æ¡Ī +ç®Ģ åİĨ +æĭ ± +èį· åħ° +ä»» æĦı +æī¿ æİ¥ +è¿Ļ æīį +客 车 +æľĿ çĿĢ +éłħ 缮 +åı° é£İ +çļĦ æĪ¿åŃIJ +éª ı +æĿ± 西 +éģĹ ä¼ł +è¶Ĭ å¤ļ +äºĨ ä»ĸçļĦ +ä¸Ĭ åij¨ +管çIJĨ åĪ¶åº¦ +失 ä¸ļ +çĶ· åıĭ +æİ¥ ç§į +å¨ģ åIJį +çĴ° å¢ĥ +åıijçĶŁ åľ¨ +个 åĽ½å®¶ +åĪĽæĸ° åıijå±ķ +æĶ¹åıĺ äºĨ +åģ¥åº· çļĦ +å̼å¾Ĺ ä¸Ģ +å̼å¾Ĺä¸Ģ æıIJ +åĽ¢ ä¼Ļ +åģĩ 设 +åı° ä¸Ĭ +è§ĦèĮĥ åĮĸ +éĻª åIJĮ +座 æ¤ħ +åı¯ æĢľ +åħĭæĢĿ 主ä¹ī +æ³ķå¾ĭ 责任 +ä¸Ģ é¡¿ +æĬ¬ 头 +为 éĩįçĤ¹ +è¿ľ æ´ĭ +éĢı è¿ĩ +åħ¨çIJĥ åĮĸ +è¶£ åij³ +票 æĪ¿ +æ¯ı 人 +åIJĦç§į åIJĦæł· +äºĨ åĩºæĿ¥ +ç»Ŀ对 æĺ¯ +ä¸ĭ å±ŀ +ä¸Ģ åıĮ +è¿Ļ åĿĹ +æĬĹ çĸ« +è¦ģ çĤ¹ +å½¢æĪIJ çļĦ +æĪij çľĭ +ä¸ĩ éĩĮ +èĢĥ çłĶ +为 åħ¶ +æ°ij 宿 +å¤ļ ä½į +大 èĩ´ +ä»ĺ è´¹ +åħ¥ æīĭ +å±ħ å®¶ +æīĢåľ¨ åľ° +人 身 +è¿ĩ å¾Ĺ +è¯ķ è¯ķ +访 è°Ī +åĬł éĩį +å°± ä¸įä¼ļ +çĶŁäº§ ä¼ģä¸ļ +åĽŀ åĽ½ +åºķ 线 +èµ¶ åΰ +æĶ¯ éĺŁ +æĪij们 éĥ½ +éĤ® æĶ¿ +缴 èĩ³ +éĴ¢ çIJ´ +åħ ľ +çłĶ讨 ä¼ļ +æľĪ 亮 +åĿļæĮģ 以 +åħ¬å®ī éĥ¨ +éĴ¢ 管 +å°ı çϽ +ç½® ä¸ļ +èģ ĭ +书 åĨĻ +æĿ ı +éħį æĸ¹ +èĢĮ åıĪ +çijŀ 士 +çķĮ çļĦ +èĢģ 大 +æĪIJçĨŁ çļĦ +å¹² ä»Ģä¹Ī +ä¸ĵ项 æĸĹäºī +çŃī å¤ļ个 +èĦ± 离 +ä¸ī 个æľĪ +çłĶç©¶ åijĺ +æĹĭ 转 +æŀģ èĩ´ +åħį è´£ +åħįè´£ 声æĺİ +å¾Īå¤ļ çݩ家 +车 ä¸Ĭ +交 äºĴ +å·² æĺ¯ +ä¸Ģ å°ı +çļĦ éĩįçĤ¹ +èĬ± äºĨ +ä¸į æĺİ +æľīåħ³ è§Ħå®ļ +çĬ¹ å¦Ĥ +çľ ¸ +å¯ ¡ +çļĦ è¡£æľį +åĮħ 裹 +身 åŃIJ +å¸ĪèĮĥ 大åѦ +äºĭ åħĪ +线 æĿ¡ +æ³ķ åζ +åħ» æĬ¤ +稳å®ļ æĢ§ +éĤ µ +åŀĦ æĸŃ +é¡ į +èĢĥ åı¤ +æĿł æĿĨ +èĭı èģĶ +æ°´ ç͵ +åħ·ä½ĵ çļĦ +æ¿Ģ æ´» +æĪij æł¡ +åĪļ å¼Ģå§ĭ +åĩ¸ æĺ¾ +ç¦ ¾ +åħ¼ èģĮ +éĢı éģİ +åľ¨ 游æĪıä¸Ń +社ä¼ļ åıijå±ķ +好 çİ© +å¹» æĥ³ +ä¸į 代表 +注æĦı åĬĽ +æ£ į +ç͍ æīĭ +ç¾İ 人 +许å¤ļ 人 +å¾Ī æĺ¯ +çļĦ çłĶåıij +æīĵ åĩº +åIJĪä¼Ļ 人 +ä¸Ģ å¤ľ +ç¼ĵ ç¼ĵ +ä¿® æŃ£ +æĦŁ çŁ¥ +ç»Ī 身 +æ¿Ģ ç´ł +çݯå¢ĥ ä¸ĭ +次 ä¼ļè®® +ç»ıæµİ å¢ŀéķ¿ +æī Ľ +åıij éħµ +åĪĨæŀIJ å¸Ī +åľ¨ æľªæĿ¥ +主è¦ģ æľī +ä¸Ģ åŃ£åº¦ +çļĦ 说æ³ķ +ä»İæĿ¥ 没æľī +è´§ 车 +缩 å°ı +太 è¿ĩ +æķĪ åĬĽ +ä¸į ä¸ĭ +æĬķ 稿 +èᝠä¸ļ +ç»Ħ éķ¿ +ç«Ļ çĤ¹ +å¾Ī åĸľæ¬¢ +éIJ µ +åĬ¿ 头 +æ¼ı æ´ŀ +æĦ¤ æĢĴ +åħħ å®ŀ +åĪĽä¸ļ æĿ¿ +çĪ ª +æľª å¿ħ +åºķ éĥ¨ +å¾Ĺ åĪĨ +人æ°ij åĮ»éĻ¢ +äºĮæīĭ æĪ¿ +å·²ç»ı 被 +大 楼 +æĸ° æĪ¿ +辦 æ³ķ +ç͍ åĬĽ +æĭĵ 宽 +åĨħ åľ¨ +æĴŃ åĩº +饰 æ¼Ķ +ä¹Ł 让 +ä½ľ çĤº +çī©ä¸ļ 管çIJĨ +åį´ ä¸į +为 ä¸ŃåĽ½ +å±Ģ åĬ¿ +ä¸į èĤ¯ +æľĢ æĸ°çļĦ +åı¯ä»¥ éĢīæĭ© +æĺ¾ çݰ +å°± ç®Ĺæĺ¯ +åľ¨ æł¡ +é¾ Ł +两 æĿ¡ +çļĦ å®ŀåĬĽ +è¶Ĭ 好 +她 åľ¨ +å¿ł è¯ļ +ä¹Ł éľĢè¦ģ +游æĪı æĵįä½ľ +è¶ħ åĩº +å¦Ĥæŀľ ä¸į +æīĢåľ¨ çļĦ +ä½ł è¿ĺ +以 åĨħ +æľī ä¸Ģå®ļ +åı¯ è¾¾ +è·ij åΰ +åī Ľ +建ç«ĭ åģ¥åħ¨ +æķ´ 车 +åīį æĸ¹ +éĹ´ æİ¥ +çѹ å¤ĩ +çĸ² åĬ³ +离 å¼ĢäºĨ +æ± Ŀ +éĿ¢ éĥ¨ +ä¹ĭåīį çļĦ +åıĺ 为 +å¦Ĥæŀľ 说 +对 ä»ĺ +åĿĩ åı¯ +被åijĬ 人 +ç²¾ ç¾İ +èģļ ä¼ļ +çĿĢ æĢ¥ +è°· æŃĮ +ä¸Ģ åı· +红 åĪ© +ä¼łå¥ĩ 游æĪı +å» ĸ +è´ ŀ +ä¹° åΰ +éŃ ļ +ä½ĵ è´¨ +å°ij äºĨ +æ³ī å·ŀ +åIJ Ł +ç»Ŀ ä¸į +é»ij æģ¶ +é»ijæģ¶ åĬ¿åĬĽ +ä¸Ĭ æĺł +çļĦè¯Ŀ é¢ĺ +ä¸ĩ人 次 +ä¸ĸ éĹ´ +ç͍ å·¥ +è´¯ ç©¿ +å®Ŀ çŁ³ +ä½ł 好 +åĪĩ åī² +强 åĽ½ +åĽŀ èIJ½ +æ°´ æĻ¶ +模 仿 +æ´ª æ°´ +éĢĻ éº¼ +åįģä¸ī äºĶ +ä½ ij +éĻ Ħä»¶ +çļĦ å¢ŀéķ¿ +éĻĦ å±ŀ +çݰ å·² +帮 ä½ł +éĩij çīĮ +é«ĺ åİŁ +åľ¨ å®¶éĩĮ +éĺ² èħIJ +ç¡®å®ŀ æĺ¯ +宣 讲 +天 æīį +ç»ıèIJ¥ 管çIJĨ +éĶħ çĤī +åIJĪ ä¸Ģ +è§Ĥ èµı +éķ¿ è¾¾ +主ä¹ī æĢĿæĥ³ +éĤ£ 麼 +é£İ äºij +为主 çļĦ +æļij åģĩ +æĮģ ä¹ħ +å¼Ĥ åľ° +å¼Ģ éŨ +模 æĿ¿ +æī¹ 次 +ä¸į 便 +天 çĶŁ +åĩł 个æľĪ +ä¸ĵ ç§ij +åı¦ æľī +åħ¬å¸ĥ çļĦ +æĩ · +åľº åIJĪ +çļĦå¿ĥ æĢģ +è¿ĺ 好 +å®ŀ æĪĺ +èĢģå¸Ī çļĦ +åħ© åĢĭ +åı¯ åľ¨ +éĤ£ ä½į +å¥ł å®ļäºĨ +ä¿ĥ éĶĢ +æı´ åĬ© +ä¸ĩ çī© +æĥħ æĬ¥ +é¦ĸåħĪ è¦ģ +æĸĩåĮĸ åĴĮ +éĥ½ å·²ç»ı +ä¸Ĭ ä¸ĸ纪 +åĨľ åľº +大 æī¹ +æĺİçϽ äºĨ +çļĦ æĪIJéķ¿ +çļĦ æ¯ĶèµĽ +失 误 +åģļ æĪIJ +ä»Ĭ天 å°ıç¼ĸ +é¢Ĩ è¢ĸ +æıIJåįĩ äºĨ +å¾IJ å·ŀ +ä»į æľī +è¿ĩ 滤 +å¹½ é»ĺ +çĥŃ éĩı +ä¸Ģ é¦ĸ +æ¼Ĥ亮 çļĦ +åĩł ç§į +åĢ¡ è®® +å°±åı¯ä»¥ äºĨ +æİĴ åĪĹ +éĩį éĩį +ä¼ģä¸ļ åĴĮ +ä¸ĵ å±ŀ +çħ İ +亲 æĪļ +çϾåĪĨ ä¹ĭ +稿 ä»¶ +è¿ĺ å¾Ĺ +人 åĵ¡ +äºī 夺 +æĽ´ 容æĺĵ +大 èĩªçĦ¶ +鼻 èħ¦ +太 空 +åľ° å¤Ħ +å¤ ¢ +ä»ĸ 对 +å¿ħ å°Ĩ +ä¸į å½ĵ +严 è°¨ +åĩº åľº +å·²ç»ı æľī +é¢Ĩ åĨĽ +é«ĺ æ¡£ +ä¸Ģ æīĢ +æł Ĺ +让 åѦçĶŁ +æĽ¹ æĵį +æŁIJ ä¸Ģ +伸 åĩº +èĬ± åįī +æ¸ħ éĨĴ +èģĶç³» æĸ¹å¼ı +åĪĨ å±Ģ +èħ ³ +æ©¡ èĥ¶ +éķ¿ å¾Ĺ +绿 åľ° +è¢ į +çļĦ èīºæľ¯ +女 æľĭåıĭ +ä¸Ń è¶ħ +离 åŃIJ +å¤ļæł· åĮĸ +éĺ³ åı° +ä½İ 碳 +ä¸Ģ ç±» +çŃīæĸ¹éĿ¢ çļĦ +å¾Ĺ 好 +模 åħ· +ä¸ĩ 亿 +çķĻ æĦı +临 æ²Ĥ +å°ij éĩı +çľĭ åIJij +ç»ıèIJ¥ èĢħ +çķĻä¸ĭ äºĨ +åĿı äºĨ +åijĬ åĪ« +羣 çIJĨ +ç¼´ è´¹ +æĬĬ ä½ł +çļĦ ä»»åĬ¡ +æĪij 对 +ä¹° åħ¥ +çĻ» ä¸Ĭ +æľī 两个 +ä¸Ģ 头 +æĵį æİ§ +åħ¨ è¦ĨçĽĸ +çĿĢ æīĭ +å¢Ļ éĿ¢ +å¤ļ æĸ¹ +åı¯çα çļĦ +ä¹Ł åı¯èĥ½ +æľĢ æľī +è¿ĻäºĽ éĥ½æĺ¯ +æĥ ¡ +å® ® +å¾Ī å°ı +éĹ®é¢ĺ æĺ¯ +åĿĩ æľī +å¾ģ éĽĨ +说 åĩº +æľī æĦı +é¢ Ĥ +æī¬ å·ŀ +åķĨä¸ļ 模å¼ı +çĶŁ èĤĸ +æįIJ 款 +å² Ĥ +ç¾İ æĻ¯ +è¿ĺ 羣 +æĭ¥ æĬ± +身ä½ĵ åģ¥åº· +æ·± å¤Ħ +çľ¼ ç¥ŀ +çļĦ 形象 +ä¼ĺ è¶Ĭ +å½ĵ æĪIJ +åĮº åĪĨ +åİ» éϤ +注 å®ļ +å§IJ 妹 +åĮº åĨħ +é© ļ +æļĹ ç¤º +æĺİ äº® +æħ° éĹ® +å¸Ĥåľº 份é¢Ŀ +çĮª èĤī +çļĦ èµĦéĩij +åİĨ ç»ı +å§ĭç»Ī åĿļæĮģ +çĶŁ æľº +ä¸į 顾 +éĩij åĪļ +大 声 +éĻķ 西çľģ +é² į +åĨľä¸ļ åĨľæĿij +æľī 害 +éŨ è¯Ĭ +æ¯ı ä¸Ģ次 +çļĦ åĽłç´ł +é¢Ŀ å¤ĸ +åİ¿ 级 +çļĩ åIJİ +åĽ½ ä¼ģ +é¦ĸ éĢī +ç¼ĸ åĨĻ +æĭ¿ èµ· +åģ· åģ· +ä¸İ ä¸ŃåĽ½ +åįĸ å®¶ +ç»Ļ ä»ĸ们 +ç¥ŀ è¯Ŀ +åѸ æł¡ +æĪij ä¸Ģ缴 +çŁ¥éģĵ äºĨ +åį Ĵ +åĴĮ åľ°åĮº +ä»Ģä¹Ī éĥ½ +çĶ» å®¶ +æľ¬ çĿĢ +ä½Ļ åIJį +审 çIJĨ +ä¸Ģ åIJij +åıijå±ķ è¶ĭåĬ¿ +åĮº éĹ´ +注åĨĮ èµĦæľ¬ +çIJ ¦ +ä¸į åı¯ä»¥ +çļĦ åĦ¿åŃIJ +å̼ çıŃ +ä¸¥æł¼ çļĦ +å®ŀä½ĵ ç»ıæµİ +æľī æĿĥ +æĪij åıĪ +éĵ¶ æ²³ +ç«ĭ 马 +æĿĢ äºĨ +åĮħ 容 +管 å®¶ +身 é«Ķ +éĵ ħ +å°ı åŃIJ +管çIJĨ ç³»ç»Ł +æľīçļĦ 人 +é£İ ç͵ +æĻºèĥ½ åζéĢł +ç²¾ ç¡® +æĭĽåķĨ å¼ķ +æĭĽåķĨå¼ķ èµĦ +äºĮæīĭ 车 +åİ¿ å§Ķ +èīº äºº +å¥ ķ +è¿İ æĿ¥äºĨ +ç»ĵæĿŁ äºĨ +çļĦ ä¼łç»Ł +æĭ¼ æIJı +奥 迪 +çĸij æĥij +ä¹ĭ æĹ¥èµ· +æłĩå¿Ĺ çĿĢ +åľ° åįĢ +è¯ł éĩĬ +åΰ æľŁ +åħ¨ éĥ½ +çŁŃ æļĤ +æĺ¯ æĪijåĽ½ +æĪij å·²ç»ı +æ»´ æ»´ +天 èµĭ +对 她 +åį«çĶŁ éĹ´ +çĶŁäº§ åŁºåľ° +æĹ¥ è®° +çļĦ æķĻåѦ +åĵ ĩ +æ°ij äºĭ +è¿ĺ åİŁ +æīĭ ä¸ŃçļĦ +çļĦ èī¯å¥½ +æ· « +ä¸Ńåħ± ä¸Ń央 +åĪ ĥ +åĵ Ħ +åľ¨ ä»ĸçļĦ +å°Ī æ¥Ń +åľº éĿ¢ +éĤ» å±ħ +çĹ Ĵ +å¦ Ħ +å¤ĸ ç§ij +ä¸į éĢĤ +举åĬŀ çļĦ +é Ĥ¹ +åħļçļĦ 建设 +çϼ 表 +è·¨ çķĮ +æ²ī æ·Ģ +大 çīĩ +è¶Ĭ é«ĺ +å°Ĩ æĺ¯ +è§ī éĨĴ +åĤ¨ åŃĺ +å¢ŀ 大 +ä¸į 让 +æķ´ å½¢ +å¹³åı° ä¸Ĭ +åĩł ä½į +è¯ī æ±Ĥ +好 ä¸į好 +åľ į +æĸĩ æľ¬ +é̲ åħ¥ +ç´ į +æł¹ æĵļ +èįī æ¡Ī +åħŃ ä¸ª +åĭ ¿ +åζ æĪIJ +饮 æ°´ +æ°¸ æģĴ +èĩª æĿĢ +åı¸ 马 +éļ¾ çĤ¹ +为 æĪij们 +å¼ § +åī© ä¸ĭçļĦ +åĩĨå¤ĩ 好 +çļĦ æľĢä½³ +èģĶåIJĪ ä¼ļ +æĤ£èĢħ çļĦ +æĪijä¸į çŁ¥éģĵ +ä¸ĭ ä¸Ģ个 +åıijå±ķ æĸ¹åIJij +ç¬ ¨ +æīĢ以 æĪij们 +åĨĻ äºĨ +éĢł æĪIJäºĨ +æ²Ļ æ¼ł +çŃĽ éĢī +çģ¾ åĮº +ä¸Ĭ çľĭ +éħ ¶ +æ»ļ åĬ¨ +éļ¾ åħį +åIJī åĪ© +ä¸Ģ ä¸Ģ +ç²¾ å¯Ĩ +伸 æīĭ +礼 仪 +åħ¨ æĺ¯ +è¶Ĭ 大 +ä¸Ń æłĩ +åıĸ åĨ³ +åıĸåĨ³ äºİ +éĢĶ ä¸Ń +讨 åİĮ +æīĭ åĨĮ +第 ä¹Ŀ +åŃĶ åŃIJ +çĦ¶ å¾Į +ä¸Ģ åħ± +æµ· æĬ¥ +款 å¼ı +æķ´ 天 +è¾¹ çķĮ +è·¯ è¾¹ +æĻĭ 级 +åIJIJ æ§½ +çļĦ åħ³æ³¨ +æĪij 没æľī +å°±æĺ¯ åľ¨ +缮 çļĦæĺ¯ +åį³ä½¿ æĺ¯ +é¡¶ å°ĸ +å·²ç»ı åľ¨ +å®īåħ¨ éļIJæĤ£ +æłĩ æĿĨ +åįĹ éĢļ +ä¼ļ 对 +座 ä½į +èµ¢å¾Ĺ äºĨ +åİŁæĿ¥ çļĦ +身 为 +书 åºĹ +è¢Ń åĩ» +ä»Ĭ æĻļ +以 èī² +以èī² åĪĹ +æĬĸ éŁ³ +åį´ æ²¡æľī +丧 失 +çļĦ å±ĢéĿ¢ +åįģåĽĽ äºĶ +çŃī 缸åħ³ +æ±ĩ æĢ» +å¤ĸ 表 +为 æ°ij +éľĩ æĥĬ +å¥Ĺ è·¯ +çĬ¯ç½ª å«Įçĸij +å°Ĩ 以 +çİĩ é¢Ĩ +éħĴ åIJ§ +è¡Įä¸ļ åıijå±ķ +å¹´ èĩ³ +åύ æĿIJ +åĴĮ æĬĢæľ¯ +æľĢ å°ı +è¿Ļä¸Ģ åĪĩ +èģĮ ç§° +å½ĵ ä½ľ +æİĢ èµ· +åĴ ĭ +ä¸Ń éĥ¨ +æīĭ èĩĤ +ç½¢ äºĨ +媳 å¦ĩ +æ´½ è°Ī +æĹ¶ä»£ ä¸ŃåĽ½ +人çĶŁ çļĦ +æŀģ éĻIJ +ç¦ Ħ +åĮº æĶ¿åºľ +æľ¬ éĴ± +礼 åĵģ +çļĦ éĤ£ä¸ª +侦 æŁ¥ +太å¤ļ çļĦ +å®ŀæĸ½ æĸ¹æ¡Ī +é«ĺ æłĩåĩĨ +æĮĩæĮ¥ éĥ¨ +å̾ æĸľ +çī¹èī² ç¤¾ä¼ļ +çµIJ æŀľ +éĴ» çŁ³ +ç§» æ¤į +çī¹ ç§į +èĩª æĦ¿ +æĭľ çĻ» +åįķ 身 +åį´ åıĪ +åĪ¥ 人 +åIJĪ è§Ħ +æľº ç͵ +çī¹ æĦı +å½ĵåīį ä½įç½® +ä¹° å®¶ +åIJĪ çº¦ +èĤ© èĨĢ +为 åĩĨ +å®¶ è£ħ +çļĦ çĥŃæĥħ +éĿŀ éģĹ +çļĦ éŃħåĬĽ +åİŁ åijĬ +社ä¼ļ åIJĦçķĮ +ä¹° çļĦ +å¤ļ åIJĥ +éĽķ å¡ij +èµ· ä¹ī +åĬł åī§ +éĤ£ä¸Ģ åĪ» +å°Ĩ è¿Ľä¸ĢæŃ¥ +æ¡Ĥ æŀĹ +æĽ´ 强 +对 ä¼ģä¸ļ +æĹł æĦı +ä¹łè¿ijå¹³ æĸ° +æµģ 失 +å¾® 软 +缸 对äºİ +座è°Ī ä¼ļ +主 èIJ¥ä¸ļ +主èIJ¥ä¸ļ åĬ¡ +ç§ģ åĭŁ +å±ķ示 äºĨ +常æĢģ åĮĸ +è² ´ +符 åı· +å¹´è½» çļĦ +å°± éľĢè¦ģ +ä¹Ł æĽ¾ +çļĦæĥħ 绪 +è¾¾ æłĩ +èĩ ¨ +ä½į å±ħ +ä»ħ 为 +é¦ĸ å®¶ +éĺ´ éĺ³ +ä¸įåĨį æĺ¯ +åĽłä¸º å®ĥ +ä¼ģä¸ļ åľ¨ +çĺ ¾ +åIJ¬ è§ģ +åİŁ æľī +åζ è£ģ +å¯Ĥ å¯ŀ +éĢļè¿ĩ 对 +æ»ij éĽª +è¿Ļ å¼ł +çļĦ çIJĨè§£ +æĸ° ä¸ŃåĽ½ +è¿Ļ åĦ¿ +ä½İ ä»· +æĥ³ è¿ĩ +çļĦ ä¿¡å¿ĥ +建çŃij çī© +çļĦ é¢ľèī² +ä¸į åºĶ该 +æĹłçĸij æĺ¯ +å¼ķèµ· äºĨ +åħ¨ åijĺ +æĿ° åĩº +è¿Ļæĺ¯ æĪij +èª ° +èĺ ĩ +éĺµ åľ° +åħħ å̼ +çŁ¿ ä¸ļ +çĿĢ ä»ĸ +ä¿¡ 访 +ä¸ĩ è¾¾ +æij© æĵ¦ +å¼Ģ 端 +èı² å¾ĭ +èı²å¾ĭ 宾 +车 åŃIJ +æľ¬èº« çļĦ +çģ«è½¦ ç«Ļ +常 å·ŀ +为 代表 +为代表 çļĦ +广 ç͵ +亲 人 +åı³ æīĭ +éĽĨ è£ħ +éĽĨè£ħ ç®± +çļĦ åį°è±¡ +æ©Ł æľĥ +åĮĨ åĮĨ +åħī ç͵ +大 æĸ¹ +è¿ĺ æľª +åĪ© 好 +ç»Ŀ 大å¤ļæķ° +åľ¨ è¿Ļç§į +ä¸Ģ ç»Ħ +æĸ° èĤ¡ +转 åıij +æ³ķ åºŃ +æĹł æīĢ +éģĵ è·¯ä¸Ĭ +çŁ¿ å±± +èij ī +æĶ¶ åĽŀ +ç§° ä¹ĭ +ç§°ä¹ĭ 为 +æıŃ éľ² +åı£ 岸 +åIJ ¼ +å¿ĥ æĥ³ +çļĦ 梦æĥ³ +éĽ ¯ +ä¹ĭ åĪĿ +å¥ĸ 项 +订 éĺħ +èĵĿ 天 +åĿ¦ åħĭ +ç«ĭ æ¡Ī +èģĶ æīĭ +ä½Ĩæĺ¯ æĪij +帮 æĪij +ä»ħ 代表 +说 æĪij +çļĦ è¶ĭåĬ¿ +æ¯Ķè¾ĥ 大 +èµ° å»Ĭ +éĩįçĤ¹ é¡¹çĽ® +èµĮ åľº +åIJį çīĩ +æĦŁ åı¹ +åľ¨ åľ°ä¸Ĭ +åıij çĥŃ +èĮĥ çķ´ +çļĦ éģĵè·¯ +éĩij èī² +ä»ĸ åıĪ +ä¼ļ 产çĶŁ +æ°ij åĽ½ +å®ĺæĸ¹ ç½ijç«Ļ +æĶ¶çĽĬ çİĩ +çļĦ åΰæĿ¥ +çļĦ åĬŀæ³ķ +æĶ¹ åζ +ä¸ĩ ç§ij +ä¸į äºĪ +è¿ĻäºĽ éĹ®é¢ĺ +çα ä¸Ĭ +çIJĥ åľº +è´£ 令 +æİĪ è¯¾ +åľ¨ é¦Ļ港 +ç»Ĩ èħ» +å¤ļ ä¸ĩ +åIJĮ å¹´ +大 使 +æĸ ĭ +ä¹Ł 为 +æĥł å·ŀ +åIJī 祥 +çͰ åĽŃ +åĽ½å®¶ éĺŁ +éĩį çĶŁ +åľ¨ åħ¶ +é¦Ļ åij³ +è´Ł èį· +亲 åĪĩ +èĩª 豪 +没 éĶĻ +åĽłä¸º åľ¨ +æĺŁ æĺŁ +éĤ ij +è¿ĺæľī å¾Īå¤ļ +æij© æīĺ +æij©æīĺ 车 +æŃ¥ è¡Į +管çIJĨ ä½ĵç³» +èĦļ ä¸ĭ +éģİ åİ» +æ±ī è¯Ń +对 ä¸įèµ· +çļĦ ç»ıåİĨ +åıĬ 缸åħ³ +ä¸įå°ij 人 +éĩį ç£ħ +åĬ³åĬ¨ èĢħ +大åĬĽ åıijå±ķ +æĢİä¹Ī åģļ +çĭĹ çĭĹ +举åįĹ äºļ +åĭĩ äºİ +åħ¬ éĸĭ +çĵ· çłĸ +åıĤ çħ§ +广æĴŃ ç͵è§Ĩ +举 åĬ¨ +æ±Ł 西çľģ +æķĪ èĥ½ +å͝ æľī +éĿ¢ è²Į +èĩªåĬ¨ 驾驶 +æ¦ľ åįķ +å½ĵ æĪij们 +仲 è£ģ +æľ¨ æĿIJ +ç±³ åħ° +çϽ éĵ¶ +çļĦ 人éĥ½ +å°± åĥıæĺ¯ +æŃ¥ åħ¥ +åįł ç͍ +åĩ» è´¥ +让 大家 +ä¼ļ è®©ä½ł +åİ¿ æĶ¿åºľ +è¦ģ ç͍ +çŃī å½¢å¼ı +åįĩ é«ĺ +责任 æĦŁ +å¤ĩ ç͍ +ä»ĸ 认为 +æ¸ħåįİ å¤§åѦ +ä»ĸ èĩªå·± +éĸ± è®Ģ +太平 æ´ĭ +éĶģ å®ļ +çŃ Ĩ +è¿Ļ çīĩ +æī§ æĶ¿ +è¿ĶåĽŀ æIJľçĭIJ +å°± æŃ¤ +éģĩ åΰäºĨ +å¼Ģå¹ķ å¼ı +管çIJĨ éĥ¨éŨ +å§¿ åĬ¿ +设 æĥ³ +åĽĽ åŃ£ +æĬĢæľ¯ 人åijĺ +å·® çĤ¹ +è¾ŀ èģĮ +èĢģ 師 +çļĦ æĦŁåıĹ +ä¹Ł éĿŀ常 +å¹´ ä¸ĬåįĬå¹´ +æĢª çī© +èĮĥ æĸĩ +æĪĺ å½¹ +åIJ« ä¹ī +åħ¨ è¿ĩç¨ĭ +èĢĮ éĿŀ +éĢļ讯 åijĺ +è¿Ļæł· æīįèĥ½ +æľº ç»Ħ +è£ ı +çķ¶ çĦ¶ +èµĮ åįļ +åIJĦ æľī +å·¥ä½ľ æľºåζ +äºĭ åIJİ +åī§ éĻ¢ +å±Ĭ æĹ¶ +åĺ´ éĩĮ +主 线 +ä¸Ģ åľĪ +主è¦ģ åİŁåĽł +å°¸ ä½ĵ +åĮ»çĸĹ åĻ¨æ¢° +ä½ł æĢİä¹Ī +ä½Ĩ çͱäºİ +æĹ¶ 空 +çĶ· æľĭåıĭ +çĶľ èľľ +é«ĺ åľ° +æĻ ĸ +èĴIJ éĽĨ +åĩĿèģļ åĬĽ +å¤ĩ åıĹ +æĸĩ åĪĽ +马 æĿ¥ +马æĿ¥ 西äºļ +æŁ´ æ²¹ +使 人 +æķĻ ä¼ļ +ç§ĭ 天 +æĺİ çıł +åħŃ åįģ +çݯå¢ĥ ä¸Ń +æ¸ħ æĻ¨ +积æŀģ åıĤä¸İ +å·ħ å³° +为 æľŁ +çѾ åŃĹ +æĦŁ æ¿Ģ +ç§ĭ åŃ£ +æĿij åŃIJ +æ¢ħ 西 +æļ´ 鼨 +çĶŁæ´» åľ¨ +çªĹ æĪ· +æģ¶ åĬ£ +纯 ç²¹ +åľ¨ æİ¥åıĹ +没 èĥ½ +è¡Į 人 +åĭ º +æĭ¨ æīĵ +ä½ľ åĩºäºĨ +çļĦ 主é¢ĺ +æľª ä¾Ĩ +ä¸Ń æľĢ +æ¾ ľ +é«ĺ è¡Ģåİĭ +åħ´ èµ· +æŃ£ èĥ½éĩı +åŁ¹è®Ń çıŃ +æİ¥ åħ¥ +çĦ¶åIJİ åĨį +åѦçĶŁ 们 +é¢ĨåħĪ çļĦ +çģ« çĥŃ +ä¸ĵ èģĮ +æĪĸèĢħ 说 +建 è¨Ń +é» ı +对 åħ¬åı¸ +çī¹ æľīçļĦ +åħī èᣠ+å½ĵ åľº +éĿ¢ åŃIJ +èµĦ产 管çIJĨ +æĹ¶æľŁ çļĦ +çŀ İ +åįİ ä¸ľ +åıĪ ä¸Ģ次 +èĥİ åĦ¿ +å®ļ çĤ¹ +头 çĹĽ +æ¶² ä½ĵ +æĺ¯ä¸Ģ ä½į +帽 åŃIJ +å¹´ èµ· +ä¸į ä½İäºİ +è¾ĥ å°ij +éĿ¢ä¸´ çĿĢ +å±Ĥ å±Ĥ +èĿ´ èĿ¶ +èī° èĭ¦ +éĺ¿ æł¹ +éĺ¿æł¹ å»· +æ¦Ĥ æĭ¬ +请 éĹ® +èµ· åºĬ +å±Ģ å±Ģéķ¿ +稳 åģ¥ +å¦Ĥæŀľ æĪij们 +éħĴ ç²¾ +æĪ· åı£ +æĦŁ æĤŁ +æĪij们 éľĢè¦ģ +æĬĢ èīº +èĩª åªĴä½ĵ +è¿Ľ åĮĸ +æ¿ĢçĥĪ çļĦ +ä½ĵ 温 +èļ ķ +èĩ´ è¾ŀ +宪 æ³ķ +ä¸Ģ çŃīå¥ĸ +çĵ¶ é¢Ī +æĥł æ°ij +èµ° è·¯ +çݰ ä»» +åķĨ éĩı +ä¸ĭ 车 +åĪ ł +責 ä»» +èŀįåIJĪ åıijå±ķ +ç´ł æĿIJ +æ²¹ ä»· +åģļ 人 +çŀ ª +æĶ¹éĿ© åĪĽæĸ° +çļĦ åĮºåĪ« +è·¨å¢ĥ ç͵åķĨ +æ¶īåıĬ åΰ +æīĺ 管 +æĪij è¿ĺæĺ¯ +åĿIJ æłĩ +ç½ij 讯 +å½ĵåľ° çļĦ +追 溯 +åľŁ è̳ +åľŁè̳ åħ¶ +åºķ ä¸ĭ +åĩł åįģå¹´ +ç©¿ è¿ĩ +çĶŁæĢģ æĸĩæĺİ +æİ¨ èĸ +æİ¨èĸ ¦ +éł Ĩ +åĴ³ åĹ½ +åĪĨ æĪIJ +çĹķ 迹 +æĪ· ç±į +éĥ½ ä¸įèĥ½ +æĻļ ä¼ļ +åĢ © +ä½ĵ åĬĽ +è¿Ļ个 èģĮä¸ļ +æĹł å½¢ +åıª æĥ³ +è¿Ľ åıĸ +æĿĢ æŃ» +èĦ Ĭ +äºij åįĹçľģ +æľª çŁ¥ +ç¾İ èģĶ +ç¾İèģĶ åĤ¨ +å¤ĸ å½¢ +诱 æĥij +çĽ £ +è¡Į 使 +åłĨ 积 +çĨŁ ç»ĥ +éĺIJ è¿° +æľĢ大 éĻIJ度 +å·¡ æŁ¥ +夺 åĨł +ä¼ģä¸ļ æĸĩåĮĸ +çĭ® åŃIJ +ä¿Ŀ å®Ī +ä¸ºæł¸å¿ĥ çļĦ +æī© æķ£ +åζéĢł åķĨ +æŁĶ 软 +为ä¸Ģä½ĵ çļĦ +游 çİ© +çĶŁ çĹħ +幫 åĬ© +åͱ æŃĮ +æīį åı¯ä»¥ +宽 æĿ¾ +è¦ģ æ¯Ķ +æĺ¯ æĢİæł· +çģ° èī² +çİĭ åĽ½ +æIJħ æĭĮ +计 éĩı +åij¨åĽ´ çļĦ +æĻºèĥ½ æīĭæľº +常 åĬ¡ +常åĬ¡ åī¯ +é© ´ +å°Ĩ è¿ij +寻 常 +ä¸ŃåĽ½ å¸Ĥåľº +容 åύ +å±± ä¸Ĭ +èĥĮåIJİ çļĦ +亲 å¯Ĩ +æīĢ以 说 +éİ ® +çļĦ çIJĨçͱ +大 åŁİå¸Ĥ +常 å¹´ +æĹħ游 ä¸ļ +å°±æĺ¯ è¿Ļæł· +åĨį æĿ¥ +é«ĺ ä½į +åĨħ 饰 +æŀĦ éĢł +ä¸Ģ èµ·æĿ¥ +çͳ è«ĭ +å·²ç»ı å¼Ģå§ĭ +çļĦ åĬ¨ä½ľ +被 è¿« +éģį å¸ĥ +åīĸ æŀIJ +å°ı äºĭ +å¿ĥ ä¸ŃçļĦ +ä½ĵåζ æĶ¹éĿ© +çļĩ å®¶ +æķĻ åłĤ +åIJĥ å®Į +åĽ½æ°ij åħļ +æĺİç¡® äºĨ +åıijå±ķ è§ĦåĪĴ +第ä¸Ģ æŃ¥ +å¾Ĺ èµ· +åľ¨ åĵª +çļĦ è·¯ä¸Ĭ +é» Ķ +çķ¶ æĻĤ +大åĬĽ æĶ¯æĮģ +åıĮ éĩį +çŁ¥éģĵ èĩªå·± +åIJĪä½ľ åįıè®® +æ°Ķ åĬ¿ +éķ¿æķĪ æľºåζ +ç½ķ è§ģ +åĽŀ æĿ¥äºĨ +ä»ĸ ä¼ļ +ä¸Ń æĸ° +ä¸Ńæĸ° ç½ij +çļĦ åķĨåĵģ +èµł éĢģ +決 å®ļ +å¸Ĥåľº çĽij管 +çķĻ åѦçĶŁ +ç͵ åİĭ +äºļ 马 +äºļ马 éĢĬ +è¿ĺæĺ¯ æ¯Ķè¾ĥ +ä¿ĥè¿Ľ äºĨ +æµģ åħ¥ +æijĦ åĥı +æijĦåĥı 头 +æıIJ åıĬ +åıij æİĺ +æī¾ åĩº +æ¢Ŀ ä»¶ +ç¹¼ çºĮ +æĪij åĸľæ¬¢ +å¥ İ +æ¦ľ æł· +å¼Ģ èĬ± +æ²ī éĩį +åŁº åĩĨ +ä»ħä»ħ æĺ¯ +轨éģĵ 交éĢļ +åĶIJ å±± +çŃī ä¸Ģç³»åĪĹ +ä¸įè¿ĩ æĺ¯ +åŃĺåľ¨ çĿĢ +èĬ± çĶŁ +å¤ · +ç»Ī ç©¶ +ä¹Łæĺ¯ ä¸Ģ个 +åįģ åŃĹ +èĸª éħ¬ +伤 å¿ĥ +æĺ¥ ç§ĭ +åĨ· åį´ +ç²¾ çģµ +çļĦ åľ°åĽ¾ +æ¯Ķ çī¹ +æ¯Ķçī¹ å¸ģ +æĢ§ åĪ« +ä½Ļ ä¸ĩåħĥ +ä¸įå¿ĺ åĪĿå¿ĥ +å¿ĥ çĸ¼ +æĽ² 线 +é«ĺ ä½İ +è¦ı å®ļ +æĻ¯ èī² +è¦ģ 说 +åħ¬åı¸ å°Ĩ +æ¶² åİĭ +è¿Ŀ 约 +åİļ 度 +åºŀ 大çļĦ +è¿ĺæĺ¯ å¾Ī +é¦ĸåħĪ æĺ¯ +çµ ² +åĬ¡ å®ŀ +並 ä¸Ķ +å¢ŀ è¿Ľ +ç»Ħç»ĩ å¼Ģå±ķ +èµ·æĿ¥ äºĨ +è¾ĥ å°ı +导 游 +两 åľ° +ç¿ ĺ +çģ¿ çĥĤ +é£İ éĩĩ +æĶ¯ 线 +æĶ¯çº¿ ä»»åĬ¡ +娱ä¹IJ åľĪ +天津 å¸Ĥ +åĮħ åĽ´ +æľ¬ èµĽåŃ£ +éĩįè¦ģ 讲è¯Ŀ +åıĮ åIJij +åįİ ä¸½ +éĶ ¤ +åĦ¿ 女 +åįĸ åĩº +ä¾Ĩ 說 +ä»ĭç»į ä¸Ģä¸ĭ +åIJ¦ 认 +åĭ Ŀ +æĻ®éĢļ 人 +çļĦ åĬ¨åĬĽ +涨 åģľ +åŁºéĩij 管çIJĨ +ä¸Ģ个 éĩįè¦ģ +è¿IJ æ²³ +çħ ŀ +è´¢æĶ¿ éĥ¨ +è¡Įä¸ļ åįıä¼ļ +éĥ½ å°Ĩ +è¨Ģ 论 +ä¸ĭ ä¾Ĩ +墨 西 +墨西 åĵ¥ +åĽłä¸º ä»ĸ们 +æĢİä¹Ī åĽŀäºĭ +åĬłå¤§ 对 +èĬ Ń +çīĮ åŃIJ +ä¼ļ 使 +妹 åŃIJ +ç«Ļ éķ¿ +å¿ħ å¤ĩ +æłij æľ¨ +æģ¶ æĦı +æ²³ éģĵ +å¯Į è£ķ +ç¹ģ åįİ +代表 åĽ¢ +æµij 身 +é¦ĸ ä½į +èĪªç©º åħ¬åı¸ +鼻 å½± +ä¸ĵ è¾ij +æ°´ æºIJ +ä¸Ń æ¯Ĵ +並 ä¸į +èĢĮ åİ» +é ĥĿ +äºİ æŃ¤ +æĸĩåĮĸ 建设 +èĤ¯å®ļ ä¼ļ +å¸ĮæľĽ 大家 +æıı åĨĻ +ä½İ è°ĥ +æĸ°åħ´ 产ä¸ļ +æ·Ħ åįļ +æĶ¾ å¼Ģ +çļĦ æĢ§æł¼ +çĸ¾çĹħ çļĦ +æķ´ é¡¿ +线ä¸Ĭ 线ä¸ĭ +éĢī 项 +çļĦ 认åı¯ +æķ´ é½IJ +çĶļ ä¹Ī +çľģ åĨħ +åı¤ 人 +æ°ij ä¿Ĺ +çī¡ ä¸¹ +éŨ çªĹ +éĤ£ æł·çļĦ +çĽijäºĭ ä¼ļ +ç¿¡ ç¿ł +ç¦ ¹ +åįĥä¸ĩ ä¸įè¦ģ +æĶ¶ 缩 +çļĦ æĸĩåŃĹ +åĴĮ å°ļ +æĮĩ 令 +åħ±äº§ åħļåijĺ +çļĦ çĪ¶äº² +å®Į å·¥ +åĬ¡ å·¥ +马 æĭī +马æĭī æĿ¾ +æµĭ è¯Ħ +å² ļ +ä¸į åģļ +ä¸ĥ å¹´ +åĿĩ ä»· +主 è§Ĥ +å¾Ī ä¸įéĶĻ +èĤ¡ä¸ľ 大ä¼ļ +äºĶ ä¸Ģ +é£İ åIJ¹ +å¼Ģ éĩĩ +è¿Ļä¹Ī 大 +èĥ½ çľĭåΰ +èĢĥ è¯Ħ +åį³ ä¾¿æĺ¯ +çݰ代 åĨľä¸ļ +æ¯Ķè¾ĥ é«ĺ +è¦ģ çľĭ +没 äºĨ +è§£ 決 +çݯ æ¯Ķ +åĨ² åĬ¨ +æ·± å¤ľ +åĩł åįĥ +ä¿ ı +ç½ij æ°ij +å°± 没 +ä»ĸ 表示 +éĩı åŃIJ +æĹ©é¤IJ åĬłçĽŁ +åįĬ å²Ľ +æIJŀ ç¬ij +ä¸Ĭ æĬ¥ +å¯ © +é¢Ħ 订 +èľĤ èľľ +æŁ¥ æī¾ +ä¼Ĺ æīĢ +ä¼ĹæīĢ åij¨ +ä¼ĹæīĢåij¨ çŁ¥ +æĹ© æĹ¥ +åıij æī¬ +åĴĮ 个人 +åĬłåħ¥ äºĨ +åĸ® ä½į +åĪĨ æĺİ +第ä¸Ģ æī¹ +ç¾İ åĨĽ +æĿĢ æīĭ +éŨ å¤ĸ +åķĨ åľĪ +ä¸Ģ åĪ» +çļĦçľ¼ ç¥ŀ +éľ Ħ +äºĽ ä»Ģä¹Ī +åĬł æ·± +æ¯ı ä½į +å¸Ĥ éĿ¢ä¸Ĭ +åıĶ åıĶ +çļĦ éĤ£ç§į +粤 港澳 +è´´ å¿ĥ +æĸĩåĮĸ 产ä¸ļ +红 æĹĹ +åĺī åħ´ +æĶ¶ çĽĺ +å®ĮæĪIJ åIJİ +ä¼ģä¸ļ 管çIJĨ +纵 横 +ä¸į ä¿¡ +æĪIJ éĥ½å¸Ĥ +æ´Ĺ 澡 +举è¡Į çļĦ +çĶ¢ çĶŁ +ç©¿ ä¸Ĭ +åĪļ 好 +åħī 线 +æīĵ æŀ¶ +è¿Ļ æľ¬ä¹¦ +åĶ®åIJİ æľįåĬ¡ +åĩł åĪĨ +ä¸Ĭ 次 +ä¸į åĪĨ +产 åIJİ +éģ¿ å¼Ģ +ç»Ī æŀģ +代表 大ä¼ļ +æ¼Ķ æĬĢ +åĽŀ è´Ń +åѦ è´¹ +éĺ» ç¢į +ä¸Ģ大 æī¹ +ç«£ å·¥ +åĨ³ å®ļäºĨ +ä½Ĩ å¦Ĥæŀľ +ç͵ æµģ +ä¸Ŀ 毫 +èĥ½å¤Ł åľ¨ +éĶĢåĶ® æĶ¶åħ¥ +åľ¨ åŃ¦æł¡ +æ°´ åĩĨ +è§Ĩ 线 +èĩª åľ¨ +åķĨä¸ļ éĵ¶è¡Į +为äºĨ 让 +çį² å¾Ĺ +çݩ家 æľĭåıĭ +éĿ¢ èĨľ +åĪĨ åī² +åī§ æľ¬ +ç« Ń +说 å¾Ĺ +æĥ³ çŁ¥éģĵ +çļĦ人 çī© +èĮħ åı° +åIJĮ ä¸Ģ个 +æķ°æį® ä¸Ńå¿ĥ +çĶ Ħ +åĸľ æĤ¦ +ä¸ĭæĿ¥ çļĦ +å®ļ åIJij +æŀģ åħ· +çļĦ åľŁåľ° +éĤ£ åĢĭ +æijĦ åħ¥ +äºĨ æĪijçļĦ +马 è·¯ +åħ¨ 社ä¼ļ +è®® æ¡Ī +å±ĭ åŃIJ +åIJį åı« +åĮ ª +åľ¨ å¤ĸéĿ¢ +åįİ åįĹ +åıij è´§ +å¯Ĵ åĨ· +é«ĺçŃī æķĻèĤ² +详ç»Ĩ çļĦ +个 é¡¹çĽ® +çĶŁäº§ åĬĽ +æĹ¶ 常 +å°± æľĥ +ä¸ĩ èĤ¡ +éĻĮçĶŁ 人 +æıı ç»ĺ +å½ĵ çĦ¶æĺ¯ +æĭī åĬ¨ +éĵ¾ æĿ¡ +æī£ éϤ +ä¸Ģ缴 éĥ½ +å°ı åŃ©åŃIJ +伤 åı£ +第äºĮ å±Ĭ +è´Ń ç½® +çļĩ 马 +æĹł èģĬ +表 åĨ³ +诸 å¦Ĥ +åĵį èµ· +é£İ æļ´ +ä¸Ģæµģ çļĦ +ç ·¨ +è§£æĶ¾ åĨĽ +室 å¤ĸ +å°± è¿Ļä¹Ī +å³ ¶ +æīĢæľī 人éĥ½ +æIJľç´¢ å¼ķæĵİ +çļĦ æĪIJæľ¬ +åħļ æĶ¿ +åıijè¡Į 人 +çļĦ äºĭå®ŀ +对 该 +åıĹ æįŁ +ä¿Ħ ä¹Į +é²ľ èĬ± +åĨľ èᝠ+æŀģ éĢŁ +æĢ¥ æĢ§ +两 ä¼ļ +ä¸Ģèά æĿ¥è¯´ +æµ· é²ľ +åĨ Ī +ç͍ 人 +çĶ¨äºº åįķä½į +åĢ ª +åĦª æĥł +æł¹ æºIJ +åĽ¢ è´Ń +ç¾İ æ´² +ä¸ĭ è¡Į +å¹´ æľ« +èľ ¡ +è¯ģ ä»¶ +åľ¨ æĪijåĽ½ +ä¸į åºĶ +æĮī æĹ¶ +åłª ç§° +åľº ä¸Ĭ +å¹²éĥ¨ èģĮå·¥ +æľī å¾Ī大çļĦ +æķ°åŃĹ ç»ıæµİ +æ¼Ķ ç»ĥ +æį® ç»Łè®¡ +å¾Ģ æĿ¥ +广åijĬ æľįåĬ¡ +çļĦ è·Ŀ离 +æŃ ¸ +è¨Ģ è¯Ń +被 èªī +被èªī 为 +åĭī 强 +å°Ĭ æķ¬ +ä¸ĩ 亿åħĥ +ä¸ŃåĽ½ åĽ½éĻħ +å¹² é¢Ħ +å¹´ 产 +èĢķ åľ° +èĮ İ +åį³ æĺ¯ +æĺ¨ æĻļ +æĪIJ为 ä¸Ģ个 +çºł æŃ£ +åij½ åIJį +é¢ģ å¸ĥ +çĮľ æµĭ +ä¿ĿèŃ· æĶ¿çŃĸ +æĭ ¢ +æ´» æ³¼ +çŃī éĥ¨éŨ +åѦ åΰ +å¢ŀå̼ ç¨İ +èĪª 线 +åĨ ¤ +åįģ åĩłå¹´ +æİ§èĤ¡ èĤ¡ä¸ľ +ä¸Ģ éŨ +个 å·¥ä½ľ +ä¸ªå·¥ä½ľ æĹ¥ +æĸ° 西 +æĸ°è¥¿ åħ° +论 è¯ģ +ä» Ĩ +åı¦å¤ĸ ä¸Ģ个 +æĶ¹ ç¼ĸ +严 ç¦ģ +åĸľ 好 +个人 ä¿¡æģ¯ +满æĦı 度 +åĵ ¨ +å¸Ī èµĦ +æĶ¹ 为 +ç«ŀäºī 对æīĭ +åĩº çĤī +åķĨ 人 +大 æ£ļ +æĮĩ导 ä¸ĭ +å¦ĩ ç§ij +è¼ ª +æī ģ +åIJĮæĹ¶ è¿ĺ +å¹¶ éĢļè¿ĩ +æĪĺ éĺŁ +èĶĵ å»¶ +ä¿ ŀ +éĢĤå½ĵ çļĦ +åīį è¾Ī +åĵģ åij³ +湿 åľ° +æĪIJ åŀĭ +ä¸į åıªæĺ¯ +æĥ© ç½ļ +åĩºåı° äºĨ +çİ© 游æĪı +æīį åıijçݰ +åºĶ èģĺ +å¤ĸ æĿ¥ +åįł é¢Ĩ +å±ķ æľĽ +å« Ĥ +港 èĤ¡ +æ¡Į ä¸Ĭ +æĶ¯ æŁ± +çļĦæĥħ å½¢ +广éĺĶ çļĦ +æĶ¯ è¡Į +å´© æºĥ +æľĪ ä¸Ń +æľĪä¸Ń æĹ¬ +ç»į åħ´ +临 è¿ij +æĬ¤ æłı +æļ ® +åįķ èģĮä¸ļ +è¾¹ å¢ĥ +æĹ¥ çħ§ +ä¸Ģ åłĨ +缴 å¾Ħ +åħ±åIJĮ ä½ĵ +æĸ°åįİ ç½ij +æīĵ 好 +ç͵åĬ¨ 汽车 +ä¸į æĺİçϽ +éĢĻ è£¡ +缼 大 +çİĭ æľĿ +åĨį ä¸Ģ次 +åĬŀåħ¬ åİħ +è´¨ æĬ¼ +åIJĪ åĩ» +人们 对 +鼶 é£Ł +éĥ½ä¸į çŁ¥éģĵ +çļĦ è¯Ńè¨Ģ +åĭŁéĽĨ èµĦéĩij +åĬ¨ èĦī +å½ ¤ +è¿Ļ åĩłå¹´ +çŁŃ è§Ĩé¢ij +太 é«ĺ +常 å§Ķä¼ļ +åĬł çıŃ +éĩį å¿ĥ +åªĴä½ĵ æĬ¥éģĵ +没 æ³ķ +éĹ» åIJį +çĥŃ åº¦ +å¹¿æ³Ľ çļĦ +åħŃ å¤§ +çī© ä½ĵ +ä¸į 该 +é¢ĺ 主 +精彩 çļĦ +为 è¿Ľä¸ĢæŃ¥ +èĻ ŀ +åĽº çĦ¶ +è´µå·ŀ çľģ +çºł ç»ĵ +代çIJĨ 人 +æ³ķå®ļ 代表 +åı¦ä¸Ģ ç§į +ä¸į åIJ« +æĭ¯ æķij +ä¼ļ ç»Ļ +è¯Ĺ è¯į +åIJĮ ç±» +å¾Ĺ ä¸įåΰ +æĬĵ ç´§ +以 åħ¶ +åħ¥ åħļ +è¿ĺ åı¯ +æľŁ åĪĬ +å¾Īå¤ļ æĹ¶åĢĻ +æĹ¥ åIJİ +åħ¬ 约 +ä¸Ģ 举 +æ¯Ķè¾ĥ å¤ļ +éĩij æ²Ļ +æį ŀ +æİĴ åĩº +æŃ¦ æľ¯ +ä¸į æĸ· +ä¸Ń èĢĥ +ä¿¡ èµĸ +ä»İä¸ļ 人åijĺ +çģ« çĦ° +éĨĴ æĿ¥ +ä½İ 温 +é̾ æľŁ +åĬ± å¿Ĺ +éħ ¥ +åı¯è°ĵ æĺ¯ +è¿Ļ æĦıåij³çĿĢ +é¢ł è¦Ĩ +åĮĹ京 大åѦ +ä¸ĵ 线 +åıĬ 以ä¸Ĭ +è¨ ª +èĢĮ åIJİ +çŁ¥ ä¹İ +ä¸Ģ对 ä¸Ģ +å¨ĥ å¨ĥ +çģ¾ éļ¾ +åħ¨ å±Ģ +æīĢå¾Ĺ ç¨İ +å®ŀ æĥł +èļĤ èļģ +ä¹Ł çŁ¥éģĵ +温 åĴĮ +èIJ½ ä¸ĭ +åŀĭ ä¼ģä¸ļ +åĨį ä¹Ł +ä¾Ľ çĥŃ +é«ĺ æ½® +çĢı覽 åύ +çļĦ 巨大 +åħΠ天 +å¹´ ä¸ŃåĽ½ +类似 çļĦ +çIJĨäºĭ ä¼ļ +空 éĸĵ +çģµ æĦŁ +åĬĽ æ°Ķ +带 ä¸Ĭ +ä¸į好 æĦıæĢĿ +æľī ä½ķ +å·² åľ¨ +åıĸ åĩº +è¿Ŀæ³ķ çĬ¯ç½ª +åŃ¦ä¹ł 贯彻 +åľ° 带 +楼 梯 +çŃī æĥħåĨµ +ä»İ åīį +çļĦ ä¹łæĥ¯ +ç³Ł ç³ķ +å°± èĥ½å¤Ł +è© ķ +ä¸Ģ å¾ĭ +æĮ« æĬĺ +åİŁæĸĩ åľ°åĿĢ +å½ĵ å±Ģ +ä¸į éĢļ +æķ° åįĥ +éĺŁä¼į 建设 +æĹ¶ èĬĤ +åģļ èµ· +çļĦ è®°å¿Ĩ +ç½ij绾 å®īåħ¨ +åĩ¡ æĺ¯ +æ° ¯ +éĽķ åĪ» +åŁĥ åıĬ +æĪij åı¯ä»¥ +çĽij çIJĨ +æĽ´ åħ· +åŁİ 管 +èĭ ¯ +åı¥ åŃIJ +èĭ¥ æľī +ä»İæĿ¥ ä¸į +缸åħ³ è´Łè´£ +å®īåħ¨ æĦŁ +æĽ´ è¦ģ +çļĦæĥħ æĦŁ +çī¢ çī¢ +è¾ĥ 好çļĦ +æ° ® +ç¬ij è¯Ŀ +车 å±ķ +ä¹ĭ ç¾İ +ç®Ģ 约 +ç±»åŀĭ çļĦ +èĢģ åĮĸ +çľĭ ä½ł +è¿ĩ åĪĨ +éŨ åīį +ä¸Ģ éĹ´ +æĥ³ åİ» +åª Ľ +åľŁ è±Ĩ +åıĪ ç§° +ä¸Ń ä¿¡ +åŃĺ éĩı +马 äºij +èĩ´ 使 +åħĪ åīį +èĢģ åŃIJ +æīĵ æī® +æ¯ķä¸ļ äºİ +æ¯ķä¸ļ åIJİ +ç¾İ好 çĶŁæ´» +å·¥ä¸ļ ä¼ģä¸ļ +就好 äºĨ +èħIJ èļĢ +çıį çıł +åΰ è¿ĻéĩĮ +æīĢéľĢ çļĦ +è¿Ļæĺ¯ åĽłä¸º +çIJĨæĥ³ çļĦ +å·®å¼Ĥ åĮĸ +é ® +é® ® +äºļ 太 +æĹł ç©· +æıIJ çݰ +ä¸ĵä¸ļ æĬĢæľ¯ +çĶ¢ æ¥Ń +åѦ åŃIJ +ç§ij å¹» +åįłåľ° éĿ¢ç§¯ +ä¸į åĩĨ +æľªæĪIJ 年人 +æĶ¶ å½ķ +è¿ĺ 款 +éĴ¢ çŃĭ +æ¼ ¢ +å¾Ĺ æĦı +综åIJĪ ä½ĵ +æŀģ é«ĺ +åįķ è¯į +é«ĺæķĪ çļĦ +骨 头 +æī§ çĿĢ +缼 ä¸ĸ +模 çī¹ +æĽ´ èĥ½ +ç»Ŀ æľĽ +对åºĶ çļĦ +æ¨ Ĭ +æĸ° ä¸ī +æĸ°ä¸ī æĿ¿ +æģ° æģ° +åIJį å®¶ +æł¸å¿ĥ æĬĢæľ¯ +个 å°ı +æĢİä¹Ī ä¼ļ +说 ä¸įå®ļ +西 çĵľ +åĵ İ +ç¢ Ł +å¿ħ ä¸įåı¯ +å¿ħä¸įåı¯ å°ij +ä¹ĭ éĸĵ +åĪĨ 管 +交éĢļ äºĭæķħ +å¼Ģ åĬŀ +å¾ģæ±Ĥ æĦıè§ģ +äº ¨ +鼻åŃIJ éĥµ +鼻åŃIJéĥµ ä»¶ +ä¿¡æģ¯ æľįåĬ¡ +ä½ł è§īå¾Ĺ +缴 è§Ĥ +å·² å®ĮæĪIJ +åĪĨ ä¼ļ +åĽŀ åįĩ +éļ » +好 人 +äºĨè§£ ä¸Ģä¸ĭ +åį« æµ´ +æľĢ çα +åºŀ 大 +客 æĪ¿ +çijŀ åħ¸ +éĥ½ ä¸įæĺ¯ +é¤ ¨ +èĹ ī +çļĦ åIJĦ项 +为 缮æłĩ +çļĦ è®¤çŁ¥ +å½±åĵįåĬĽ çļĦ +夸 å¼ł +佩 æĪ´ +æ±ĩ çİĩ +çļĦ çαæĥħ +æĺ¥ é£İ +æĺ¯ æĪijçļĦ +æ¨ ¹ +åįĬ å°ıæĹ¶ +å±± åİ¿ +å±± 西çľģ +èĢĮ è¿Ļ +æĽ´å¤ļ ä¿¡æģ¯ +è¿ĺ æľīä¸ĢäºĽ +ç²¾ ç»ĨåĮĸ +ç¾İ åѦ +çͱ æĸ¼ +ä»ħä¾Ľ åıĤèĢĥ +å¾Ī é«ĺçļĦ +åıł åĬł +è¿Ļä¹Ī 说 +å±ķ åĩº +åĽĽ å¤Ħ +ä¸ĩ å®¶ +æĭĽ åĭŁ +çļĦ 强大 +æĤ£ æľī +å°ı äºİ +ä¹Łè®¸ æĺ¯ +对 èĩªå·±çļĦ +èģĮä¸ļ æķĻèĤ² +æĿ¥ è¿Ľè¡Į +æ¡£ 次 +æīĵ èµ¢ +éĥ½æľī çĿĢ +åº ¸ +è¯Ń æ°Ķ +çͲ éĨĽ +空 åĨĽ +车 åĨħ +åĽłä¸º ä½ł +å®ŀ æķĪ +æĥħ ä¾£ +åıijè¾¾ åĽ½å®¶ +éķľ åŃIJ +æ¯į å©´ +ä½Ĩæĺ¯ ä»ĸ +积æŀģ æİ¨è¿Ľ +大å¹ħ 度 +çļĦ 女åĦ¿ +é¤IJ æ¡Į +åIJ¬ å¾Ĺ +çļĦ 积æŀģæĢ§ +好 åIJ§ +æĹ¥ æ¶Īæģ¯ +æľī ä»»ä½ķ +æ¯Ĵ åĵģ +æĹ©çĤ¹ åĬłçĽŁ +第ä¸Ģ 天 +å°½ åĬĽ +æł ĸ +主 æīĵ +æĺ¯ä¸Ģ åIJį +çĪĨ æĸĻ +äºĭä¸ļ åıijå±ķ +å¾® åķĨ +äºİä¸Ģä½ĵ çļĦ +çĶŁ çĮª +èĩªçĦ¶ èµĦæºIJ +çŀĦ åĩĨ +è§Ħ模 åĮĸ +å¹¶ ä¸İ +èĤ¥ èĥĸ +å®¶ ç͍ +大 çĪ· +é¢Ħ åijĬ +æĿ¥ åģļ +éĺ³ åİ¿ +æŀĦ çŃij +é¢ģ å¥ĸ +åİĨåı² æĸĩåĮĸ +æľįåĭĻ æĪĸ +æĢ» åĨ³èµĽ +åıij åŀĭ +æĪij 羣çļĦ +æĽ ¦ +åıĤ ä¼ļ +èĦĨ å¼± +åĩĨ åħ¥ +èħ¹ éĥ¨ +åı¸ 令 +æĤ² åī§ +天 ä¸Ĭ +åı£ ä¸Ń +ä¸ĩ 个 +åѦ ä¸ļ +æıIJ åĢ¡ +两 è¾¹ +大 èĤ¡ä¸ľ +åı¤ éķĩ +è¡Ģ ç³ĸ +çļĦ ç¨ĭ度 +æ£ī èĬ± +åIJİ åı° +å°± åĮ» +æķ´ æķ´ +èĴ ² +çĽĪåĪ© èĥ½åĬĽ +ç± ½ +èĦ « +çľĭ éĩį +å®¶ éķ· +èģĺ ç͍ +èµĽ éģĵ +åīį èĢħ +建 èѰ +å¾ĭå¸Ī äºĭåĬ¡ +èīºæľ¯ åĵģ +æľī èĩªå·±çļĦ +åIJ¦ å®ļ +社 åĽ¢ +åij¨ äºĶ +带 åΰ +å·¥ä½ľ ä¼ļè®® +èĤ¡ æľ¬ +å¤ĸ åĮħ +å®¶ åħ¬åı¸ +çĽij çĭ± +èĪ Ĭ +åIJį æł¡ +西 æ¹ĸ +è¶ħè¿ĩ äºĨ +åįĹ å±± +ç»Ħ ä»¶ +å̼å¾Ĺ 注æĦı +æĮ£ æīİ +äºĭ 迹 +ç¶ĵ çĩŁ +ç§ij 室 +好 åIJĹ +æ¤ħ åŃIJ +åľĪ åŃIJ +ä½Ĩ 她 +æµģ çķħ +åIJĦèĩª çļĦ +èģĮ åijĺ +è¡į çĶŁ +åħ¨ åľº +æĴ¤ éĶĢ +åį´ è¢« +å®ģ éĿĻ +åīį æīĢ +åīįæīĢ æľª +åīįæīĢæľª æľī +主 ä¸ļ +åĮĹ ç¾İ +è¯Ħ å®ļ +åĵģ å°Ŀ +大家 éĥ½åľ¨ +主 å¸ħ +ç»Ĩ å¿ĥ +ä¿¡æģ¯ æĬ«éľ² +çļĦ ç«ŀäºī +éĢĻæ¨£ çļĦ +ç§ijåĪĽ æĿ¿ +éĩĩ æijĺ +票 æį® +éĢIJ å¹´ +èĭ± è¶ħ +è¡Įä¸ļ åĨħ +人 寿 +åIJİ åĭ¤ +å¦Ĥ æĦı +ç¬Ķ è¯ķ +æ·¡æ·¡ çļĦ +ä¸į èĪĴæľį +ä½ĵ 积 +ä¹Łä¸į è¦ģ +éĿ¢ æĸĻ +æł· æľ¬ +ç¥ ģ +æĮī è§Ħå®ļ +大æ¦Ĥ æĺ¯ +æĥħåĨµ è¿Ľè¡Į +åIJĦ åįķä½į +çļĦ ç¬ij容 +åĩºèī² çļĦ +代表 æĢ§ +çļĦ ç¾İ好 +éĴ ¦ +å¾® çĶŁçī© +è¶Ĭ æĺ¯ +æĸ¹ åı¯ +å¹² èĦĨ +éģĬ æĪ² +çļĦ åħ´è¶£ +éĹ® è´£ +åĽłä¸º æĪij们 +èĢĥ éĩı +çĶŁ çĶŁ +éĺ» åĬĽ +ä¸į åħģ许 +æıIJ è®® +åĩı æĮģ +åıªæĺ¯ ä¸Ģ个 +æĪij æĬĬ +åıijçݰ èĩªå·± +å¢ŀ å¹ħ +å¦ į +èĹĿ è¡ĵ +ä¸Ģå®¶ 人 +åĪĨ 级 +çļĦ æķ°éĩı +è½® èŀįèµĦ +çŃī åĽłç´ł +大 夫 +èģĺ 请 +é£İ æľº +绽 æĶ¾ +ä»»ä½ķ ä¸Ģ个 +éł Ĥ +éĺ¶ çº§ +æĬĬ 她 +è¿Ľ åĨĽ +èĥ½ åģļåΰ +åŁ¹è®Ń æľºæŀĦ +çī© æĸĻ +ç«¥ è¯Ŀ +æĮĩ导 æĦıè§ģ +éĺ ® +æ·±åħ¥ æİ¨è¿Ľ +主 æľº +æ¸Ķ ä¸ļ +ä¸į æľį +æµĵ éĥģ +è¡Ĺ ä¸Ĭ +ä¾Ŀ 次 +æĹ¶ 段 +æ¢ µ +çļĦ åĸľçα +å¾Ī éķ¿ +åĪĿ 级 +æŀľ æĸŃ +æĬ¢ æķij +é¼ĵ èĪŀ +ä¾Ľ éľĢ +æ·±åħ¥ å¼Ģå±ķ +产ä¸ļ éĽĨ群 +åĻª éŁ³ +åIJ¬ çĿĢ +æ·±åĪ» çļĦ +å¿į åıĹ +ç͵ ç£ģ +强 èĢħ +æ»ĭ åij³ +æĽ¼ èģĶ +åı¯ä»¥ 缴æİ¥ +大 ç±³ +æŃ· åı² +æĶ¿åĬ¡ æľįåĬ¡ +åħ¬ å¼ı +社 群 +éģĵ士 èģĮä¸ļ +ä¹ĭ æĥħ +æµ· æ°´ +æ¼Ķ å¥ı +åºĹ éĩĮ +迹 象 +åıijå±ķ çIJĨ念 +é«ĺ 空 +åij¨ åĪĬ +åĽŀ åΰäºĨ +ä¸į éĢĤåIJĪ +åłµ å¡ŀ +åĬ Ī +æ°´ ä¸Ĭ +çĢij å¸ĥ +纳ç¨İ 人 +çĩĥ æ²¹ +å·¥ç¨ĭ é¡¹çĽ® +峡 è°· +æľī éĴĪ对æĢ§ +åľĨ å½¢ +æľ¬ å¸Ĥ +è¿Ļ è¯Ŀ +管çIJĨ èĢħ +ç¡®è¯Ĭ çĹħä¾ĭ +æĬĬ æīĭ +彩 èī² +ä¸Ĭ åīį +夯 å®ŀ +ç¾Ĭ èĤī +å¾Ģ å¹´ +æĵħ èĩª +è¿· 人 +èĪª æ¯į +ç²¾ ç»Ĩ +åľ¨ æĪijçļĦ +åĪĽ æĬķ +麦 åħĭ +æľĪ ç»ı +åĮĹ æµ· +ä¹ĭ æĺŁ +åı¶ åŃIJ +å¸Ĥåľº ç«ŀäºī +è¿Ļ äºĭ +åıĥ èĪĩ +产 åľ° +åĶ ī +åķĨåĵģ æĪ¿ +èĪª è¿IJ +ä¼ĺ å¼Ĥ +ä»ĸ们 æĺ¯ +鼨 æ°´ +è¯į æ±ĩ +åĨľ çͰ +欧 éĺ³ +çŁŃ 线 +管 ç½ij +æł¹ åŁº +åıªæľī ä¸Ģ个 +éŀĭ åŃIJ +å¸Ĥ å§Ķ书记 +åĪ» æĦı +è¡Į 车 +åıĪ è¢« +åı¯éĿł æĢ§ +è´ ± +ä»» åij½ +åºĶ åľ¨ +å°± å¾Ĺ +æľįåĬ¡ ä½ĵç³» +æĶ¿ æĿĥ +åıijè¨Ģ 人 +è¿ĩ å¾Ģ +两 åıª +èϽ 说 +éĢģ ä¸Ĭ +ä»Ģä¹Ī äºĭ +æķ£ æĸĩ +æİĮ æİ§ +èĸĦ å¼± +ä¸ĭéĿ¢ å°± +主è¦ģ åĨħ容 +å¾Ī éĩįè¦ģçļĦ +å°± 说 +çϽèī² çļĦ +éĤ£ä¸ª æĹ¶åĢĻ +ç»ı纪 人 +çļĦ æ¯į亲 +ç¬Ķè®° æľ¬ +åºķ å±Ĥ +è¿ij 代 +è§£ 说 +è²ł 責 +æľĢ大 åĮĸ +åķĨ éĵº +æł¡ åıĭ +æ² ģ +ä¸į åĩºæĿ¥ +éĻ· éĺ± +ç¨ ħ +åħ¬å¸ĥ äºĨ +åĩĢ å̼ +çĽ¸å¯¹ è¾ĥ +ç¬ Ľ +æł¸ ç®Ĺ +åįİ ä¾¨ +æĢ¥ æķij +æĮº 好 +åħĴ ç«¥ +äºĮ èĥİ +åĩº èĩª +åĿ Ł +æīĭ ä¸ĭ +å± ¡ +åĪĽéĢł æĢ§ +ä¸¥æł¼ æĮīçħ§ +åĨį åİ» +举 缣 +人 æµģ +äºĨä¸Ģ 声 +å°ıæĹ¶ åīį +è´µ æĹı +éľ ĸ +ä¹Łæĺ¯ éĿŀ常 +éĢ ± +çľĭäºĨ çľĭ +ç¹ģ æ®ĸ +èĩ³ æŃ¤ +é¢Ħ å¤ĩ +å¾Ī æĺİæĺ¾ +æ¼Ķ èīº +åĿIJ çĿĢ +ä¿Ħ åĨĽ +åľ¨ è¿ĩåİ» +ä¹ĭ äºĭ +æĬĵ èİ· +åĿIJ ä¸ĭ +çͱ ä¸ŃåĽ½ +ä¹Ł å¼Ģå§ĭ +çŃĶ å¤į +åŀĥåľ¾ åĪĨç±» +éĴĵ é±¼ +åIJĦ 種 +缸 éģĩ +ä¸įåģľ çļĦ +æī¹ éĩı +éĩįè¦ģ ä½ľç͍ +å§Ķ å±Ī +åħŃ å¹´ +ä¸ĥ åįģ +ä¹ĭ æĪĺ +é£İéĻ© 管çIJĨ +éŁ³ æ¨Ĥ +è¡ĮæĶ¿ å¤Ħç½ļ +æľ¬ äºĭ +æĴ° åĨĻ +èģļ åIJĪ +éĢĤ æĹ¶ +æIJ¬ å®¶ +ç¢İ çīĩ +缼 å®´ +ç®Ģ æ´ģ +åı¬ éĽĨ +ç®Ģ åĮĸ +åĮĹ京 æĹ¶éĹ´ +第ä¸ī å±Ĭ +æĿ¥ åĽŀ +常ç͍ çļĦ +京 æ´¥ +京津 åĨĢ +梦 å¹» +è¯ķ è¡Į +æľº åºĬ +åΰ æľĢåIJİ +åĬ© æīĭ +åĪĨ 彩 +åĩº åĵģ +åι 车 +åIJ¯ åıij +ä¾§ éĿ¢ +æ¯ı å½ĵ +缸åħ³ è§Ħå®ļ +ä¸ĸ 人 +è´Ń 车 +å¿ĥ 缮 +å¿ĥ缮 ä¸Ń +äºĶ éĩij +è¿ĺ è®°å¾Ĺ +ä¾Ŀ çĦ¶æĺ¯ +æıIJ æ¡Ī +ç͵åķĨ å¹³åı° +åģļ åΰäºĨ +æĿľ ç»Ŀ +å®ī åįĵ +ä¸ĸçķĮ åIJĦåľ° +åīį éĢĶ +æ´Ĺ åĩĢ +å¥ĭ åĬĽ +åŁİå¸Ĥ 建设 +å¤ļ åĬŁèĥ½ +ä¼ļ éĢłæĪIJ +åıijå¸ĥ ä¼ļä¸Ĭ +ç©¶ 竣æĺ¯ +åĪĨ 红 +çŁ¥ èŃĺ +éĿ¢ æĿ¿ +æĹł 声 +æĢ¥ éľĢ +失 çľł +çΏ å¦Ī +äº Ĥ +åħ¨ æĻ¯ +ç»ıåħ¸ çļĦ +åī§ ä¸Ń +é¢Ĩ导 ä¸ĭ +åħļ åĨħ +åħ¥ ä¾µ +æĭī æĸ¯ +ä¸Ģ å¹ķ +åĬł ä¹ĭ +èĤ Ĩ +èĭ± æł¼ +èĭ±æł¼ åħ° +å·§ åħĭ +å·§åħĭ åĬĽ +ä¸Ģ å¿ĥ +èģ Ĥ +å¾Ģå¾Ģ æĺ¯ +管çIJĨ å±Ĥ +çĻ» åħ¥ +建ç«ĭ èµ· +建 åĽ½ +åŃIJ 宫 +åºĶ ä»ĺ +æİ¢ ç©¶ +第ä¸Ģ ä½į +ä½Ļ å®¶ +çŃī æ´»åĬ¨ +æīĢ èĩ´ +è¾ĥ å¿« +æĺ¯ éĿŀ +æıIJ åIJį +äºĮ èĢħ +åıªåī© ä¸ĭ +åħ¶ä¸Ń åĮħæĭ¬ +ç¼ĸ ç¨ĭ +çł´ ç¢İ +ä¸Ń 举 +å·¥ä½ľ æĬ¥åijĬ +çѾ åIJį +éħĴ ä¸ļ +çŁ¥ æĻĵ +çĥŃ å¿ĥ +éĿŀ åĩ¡ +èIJ¥ä¸ļ æī§ +èIJ¥ä¸ļæī§ çħ§ +人大 代表 +ä¸Ģ个 æĸ°çļĦ +å¨ģ æµ· +éĤ£ 人 +涨 ä»· +æ¶Ī çģŃ +éļ¾ å¿ĺ +ç¶ĵ é©Ĺ +åı£ è¢ĭ +ç³» æķ° +æĸĩ ä¸Ń +好 转 +æĸ° 鼶åĶ® +讲述 äºĨ +å¼Ģ çĽĺ +çķĻ ç»Ļ +æħ¢æħ¢ çļĦ +æĤ² 伤 +æľ¬ æľŁ +äºĨ å¤ļå°ij +è¿Ļ 让 +åIJĮ çŃī +æ¸ħ æĺİ +个 åŁİå¸Ĥ +æºĸ åĤĻ +åĩłä¹İ æĺ¯ +强 åĬĽ +ä¿ ¯ +æ°´ 稻 +åĽºå®ļ çļĦ +æł¸ åĩĨ +说 æľį +顯 示 +è¿Ļ å¥Ĺ +æĻºæħ§ åŁİå¸Ĥ +å±ĭ é¡¶ +ä¸į æĿ¥ +çĶŁ é²ľ +çŁ¥ æĥħ +æĬķ 身 +åijĬè¯ī æĪij们 +ä¸ī åĽĽ +ä¸ĩ ä¸Ģ +è¾Ĩ 车 +为 ä¹ĭ +åΰ æĹ¶åĢĻ +è¿Ļ æīįæĺ¯ +åIJį çīĮ +åºŁ æ°´ +åݻ年 åIJĮæľŁ +å¹´ éĻIJ +éģĭ åĭķ +åıĮ çľ¼ +è¦ģ ç´§ +对 çŃĸ +åľº é¦Ĩ +çϾ ç§ij +è¶Ĭ éĩİ +å¯Į åIJ« +大å¤ļæķ° 人 +æľĢ å°ij +åı¬ åͤ +åħ¸ èĮĥ +åĨľ æľº +æŃ£ æĸĩ +åºĶç͍ äºİ +æ·± èĢķ +ä¿ Ń +ä»Ģä¹Ī ä¸ľè¥¿ +å¥Ĺ é¤IJ +å½ĵ éĢī +å·¦ æīĭ +è°ĥ çIJĨ +æĻļ é¤IJ +éļ¾ åħ³ +åĩŃ è¯ģ +çα 人 +æĮĩ è´£ +è´£ ç¼ĸ +çļĦä¸Ģ 款 +éĵ ² +åįģ 个 +èĢ » +æľįåĬ¡ åķĨ +åľ° çĭ± +è¿ŀ å¿Ļ +åĽ° æĥij +çļ ĵ +ä¸į åIJĥ +çİ°åľ¨ å·²ç»ı +çĽĺ çĤ¹ +ä¸įåģľ åľ° +管çIJĨ 模å¼ı +è¿Ļ 段æĹ¶éĹ´ +æ¤ ° +礼 åĮħ +æµģ 转 +æī« çłģ +éĽĨä¸Ń åľ¨ +æ±Ĥ åĬ© +åįĬ 个 +å¿«éĢŁ å¢ŀéķ¿ +å¾Ģ ä¸ĭ +è¯Ħ åĪĨ +å°± æĥ³ +åķĨåĬ¡ éĥ¨ +æľī éĹ®é¢ĺ +èİ· åĪ© +æ¯Ľ çĹħ +æĦŁ åºĶ +èī¯ æĢ§ +åĪĨ æŃ§ +åĨ ī +æĪij们 çİ°åľ¨ +è¦ģ åĬłå¼º +å·§ å¦Ļ +èŀº æĹĭ +åĪĩ æį¢ +çĭ Ħ +顺 çķħ +å°¤åħ¶ æĺ¯åľ¨ +èĬĿ 麻 +éļ¾ è¿ĩ +æĹĹ å¸ľ +å¤į åį° +å¤įåį° ä»¶ +å¿ħ éľĢ +对å¤ĸ å¼ĢæĶ¾ +éļ¾ åıĹ +åİŁæĿ¥ æĺ¯ +ç®Ĺ äºĨ +é«ĺ å±± +离 èģĮ +çµĦ ç¹ +çµĦç¹ Ķ +å±ģ èĤ¡ +çϾ å®¶ +éģĩ ä¸Ĭ +æĺĶ æĹ¥ +ä¸į 容 +çĽij管 éĥ¨éŨ +主 æĦı +æµģ åŁŁ +è·Į å¹ħ +èĩ³ ä¸Ĭ +åĪ« 说 +æĺ¯ æ¯Ķè¾ĥ +å®ıè§Ĥ ç»ıæµİ +å¸Ĥåľº 主ä½ĵ +污æŁĵ çī© +æķij æ²» +丰 æĶ¶ +åŃĺ æĶ¾ +åĩ Ħ +éĩij å±± +æį¢ äºĨ +ä¸ĵ 人 +éĹľ æĸ¼ +æĹ¢ è¦ģ +åĽ½ è¶³ +éļ ĭ +åıį åĩ» +èµ· 身 +åħĪ æĺ¯ +å¸ĮæľĽ èĥ½å¤Ł +åζ 订 +åºĹ éĿ¢ +åĸ Ģ +æķĻ ä½ł +éĻį æ¸© +åĬĽ æ±Ĥ +ä¸ī çϾ +çī© ä»· +丢 失 +å¢Ļ ä¸Ĭ +éĥ¨ 份 +æł· æĿ¿ +ä¹ĭ æĦı +ç½ij å°ıç¼ĸ +ä¸ĸ ä¸Ĭ +è°ĥ è¯ķ +污æŁĵ éĺ²æ²» +å½± éĻ¢ +å®Įåħ¨ åı¯ä»¥ +éĢļ åħ³ +ä¹īåĬ¡ æķĻèĤ² +没æľī åĬŀæ³ķ +èĢ ¿ +å¦ ³ +æĹł æĥħ +å¾Ĺ çĽĬ +å¾ĹçĽĬ äºİ +æľŁ çĽ¼ +娱ä¹IJ åľº +çͲ æĸ¹ +ä¸Ģ æ±½ +çĹ ° +çĸij ä¼¼ +æĸ°æµª å¾®åįļ +强 è¡Į +å½ĵ ä»ĸ +èĥ º +ç͍æĪ· æıIJä¾Ľ +åĮº å§Ķ +æĦ¿ æĻ¯ +æĬĺ æī£ +失 踪 +è¿« åĪĩ +åŃĹ æ¯į +åĴ ¯ +èªį èŃĺ +ä»Ģä¹Ī æĦıæĢĿ +çĽĴ åŃIJ +å½ķ éŁ³ +建设 å·¥ç¨ĭ +ä¸ļ ä½Ļ +å®ŀè·µ æ´»åĬ¨ +羣 空 +çĤ ĸ +åľ¨ è·¯ä¸Ĭ +主è¦ģ åĮħæĭ¬ +该 æĢİä¹Ī +æĢ» æľī +æĢ§ æĦŁ +æ°ij èĪª +å¼Ģ åºĹ +欺 éªĹ +çªģ åĩ» +缺 失 +æī§ ä¸ļ +åľ° éģĵ +å¹¶ æĹł +æ°ij åĬŀ +ç»Ħç»ĩ çĶŁæ´» +æĪij å¦Ī +è¨ĺ èĢħ +管 åζ +æī¾ 个 +èĹ » +çĤİ çĹĩ +äºĴ åĬ© +æµıè§Ī åύ +çݩ家 æĿ¥è¯´ +éĻįä½İ äºĨ +è£ Ķ +æĮ£ éĴ± +åķĨ æľº +æĶ¹ è£ħ +æµģ 浪 +æĶ¿ æ³ķ +èĢģ 头 +çĶŁäº§ åĴĮ +ç© Ĺ +亲 çα +亲çα çļĦ +å±¥ èģĮ +åŁİ éĩĮ +ç»Ĩ åĪĨ +åĬ³åĬ¨ åIJĪåIJĮ +åľ¨ æĹ¥æľ¬ +å¨ģ å°Ķ +åį« è§Ĩ +éĢ£ çµIJ +çĿĢ éĩį +æĬĺ 磨 +åĽ¾ 为 +çľ · +å·¥ åºı +æĵ ģ +æĵģ æľī +ç½ijç«Ļ åľ°åĽ¾ +çļĦä¸Ģ 大 +ç»Ħç»ĩ å®ŀæĸ½ +æĬĽ å¼ĥ +åĴĮ æĶ¯æĮģ +æ³ķ åĪĻ +浪 æ½® +çݰ æľīçļĦ +åĩł çİĩ +为 客æĪ· +åįģ ä¸ĩ +è ¹Ħ +çªģåĩº éĹ®é¢ĺ +åıĥ åĬł +éĥ½ä¼ļ æľī +çĽ ¤ +è°ģ éĥ½ +æīĭ åĬ¨ +缴 è¾¾ +çĤ¹ å¤ļ +éĺ¶ å±Ĥ +ä¸į ä½³ +éĤ£ 段 +滨 æµ· +æĺ¯ åĽ½åĨħ +æĪij å¸ĮæľĽ +åIJĽ åŃIJ +è§Ĥ éŁ³ +åģļ é¥Ń +æ±½ è»Ĭ +åħ³ ç¨İ +çľ¼åīį çļĦ +æ°´ éĿ¢ +è̳ æľº +追 踪 +æİ¨ éĢģ +éĴ± åĮħ +æģ¶ å¿ĥ +æµ· åŁŁ +å· į +å¼Ģ æĿ¥ +表 æĢģ +仪 表 +å¹³ åİŁ +åįģ å¤ļå¹´ +ä¹Ł æĹłæ³ķ +åħ¼ 顾 +è¡£ æŁľ +æł½ åŁ¹ +æĪ¿ æºIJ +设ç«ĭ äºĨ +ä¸ĩ åIJį +æķ° é¢Ŀ +è¦ģ åĿļæĮģ +åIJīæŀĹ çľģ +请 èģĶç³» +ç»ıåİĨ è¿ĩ +çļĦ æľ¬è´¨ +åħ¥ éŨ +æľ¬ æ¡Ī +çİĩ è¾¾åΰ +åı° éĺ¶ +éĴ ŀ +æĪij èĥ½ +èݲ èĬ± +éĴ ł +ä¸Ģ äºĭ +åİŁ æľīçļĦ +æ¯ı åĢĭ +æ¯Ķäºļ 迪 +æ£ĭçīĮ 游æĪı +ä¸įä¼ļ æľī +å½Ĵ æĿ¥ +äºĶ çϾ +è¿ĩ é«ĺ +鼷 è¾¾ +ä¸Ģèµ· åİ» +æķĻ å¯¼ +å°± è¯Ĭ +å°± å¾Ī +ä¸įåIJĮ äºİ +ä¿ º +å¸ĸ åŃIJ +æĶ¿åįı å§Ķåijĺ +çĸ«æĥħ å½±åĵį +åĪĨ è£Ĥ +为ä»Ģä¹Ī ä¼ļ +äºĶ æĺŁ +å°ij åĦ¿ +æĬ¢ éĻ© +梦 è§ģ +è®°èĢħ éĩĩ访 +å±± è·¯ +æĪij 个人 +æ²Ļ 滩 +è¹ Ń +æĶ¹ è®Ĭ +æĸ°åŀĭ åĨł +æĸ°åŀĭåĨł çĬ¶ +åĮ» æĬ¤ +åĮ»æĬ¤ 人åijĺ +æµ· å°Ķ +åħ³äºİ æĪij们 +éϤ å¤ĸ +åº ļ +宣 åijĬ +ä¸ī åįĥ +æ¦ ¨ +ç§ijæĬĢ å¤§åѦ +ä¸ĥ åħ« +顺 åºĶ +çΏçΏ å¦Īå¦Ī +éĢī åıĸ +åī§ çĥĪ +乡æĿij æĹħ游 +积æŀģ æİ¢ç´¢ +表çݰ 为 +å¾Ī æ¸ħæ¥ļ +大 åĨĽ +æĿ¥ ç͵ +å¥Ĺ æĪ¿ +çݰ è¡Į +享 åıĹåΰ +çľĭ çĤ¹ +åĽºå®ļ èµĦ产 +以 人为 +以人为 æľ¬ +ä¸į å®Į +éĻį 鼨 +åģļçļĦ äºĭæĥħ +å¹¶ äºİ +顽 强 +èĢ ¸ +åĺ´ å·´ +缸åħ³ ä¿¡æģ¯ +æĪij 没 +æĪĺçķ¥ æĢ§ +æĢĿ 念 +åĪĺ å¤ĩ +åĬ© æĶ» +é£İ è²Į +éĿ¢å¯¹ éĿ¢ +积æŀģ å¼Ģå±ķ +çĸĹ æķĪ +çľĭ 书 +缺 åı£ +åĽ½æ°ij ç»ıæµİ +使ç͍ æĿĥ +éģ¥ è¿ľ +å¡« è¡¥ +第ä¸ī 人 +åįĬ å¤ľ +æŃ¦æ±ī å¸Ĥ +æĪij åıijçݰ +ä¼ĺæĥł æĶ¿çŃĸ +é£İ åı£ +å°± ä¸įèĥ½ +为 主è¦ģ +æµģ åĩº +å´ĩ æĭľ +å¹¶ ä¸įèĥ½ +é«ĺ ä¸ī +ä¸ĸçķĮä¸Ĭ æľĢ +æĥ³ å¿ħ +åħ¶ æīĢ +åĢĻ éĢī +åĢĻéĢī 人 +ä¸į çα +åī¯ ä½ľç͍ +人æ°ij æĹ¥æĬ¥ +æĪij ä¸įæĺ¯ +å®ŀ çī© +ç͵ åİĤ +ä¹Ł ç®Ĺæĺ¯ +æľī éĹľ +æľī èĥ½åĬĽ +æĮĤ åľ¨ +çľ¼ ä¸ĭ +约 ç¿° +å°ı åѦçĶŁ +èµ· åΰäºĨ +å·¥ 夫 +åIJĮ å¿ĥ +åĿ¦ è¨Ģ +çł Į +åıijæĮ¥ äºĨ +èģĮä¸ļ éģĵå¾· +è¿ĻäºĽ å¹´ +念 头 +èĢģ é¼ł +åħ¨ èµĦ +åħ¨èµĦ åŃIJ +ä¸Ģ åij³ +å¤ļ ä¸ĩåħĥ +æł¼ æľĥ +éķ¿ éĢĶ +带 èµ° +èĭ± 寸 +æĸĩ ä½ĵ +对 ä»ĸ们 +åĵŃ äºĨ +å¡« æĬ¥ +çīĪæĿĥ 声æĺİ +ç͵ 线 +è´Ńçī© ä¸Ńå¿ĥ +饱 满 +ä½İ 头 +强 è¿« +ä¿Ŀ æ´ģ +欧 åĨł +缸 è¿ŀ +认 è´Ń +çģ« æĺŁ +é«ĺ å°Ķ +é«ĺå°Ķ 夫 +èij« èĬ¦ +æłĩ 注 +çļĦ çIJĨæĥ³ +æł¸ éħ¸ +æł¸éħ¸ æ£Ģæµĭ +åĬ ī +ä¸Ģèά æĺ¯ +æĢĿ ç´¢ +轨 迹 +çĥŃ å¸¦ +éĻ £ +åĩĨç¡® æĢ§ +æĪ´ çĿĢ +åľ¨ çĶŁæ´»ä¸Ń +æīĢ èĥ½ +æľ¯ åIJİ +带 ä½ł +ç¥ ł +æ®ĭ éħ· +ä¹Ł åıªæĺ¯ +çͳ è´Ń +举åĬŀ äºĨ +æľī æĦıä¹ī +æĹº 缼 +åľ¨ ç¶² +åľ¨ç¶² è·¯ä¸Ĭ +å¾Ī大 ç¨ĭ度 +管 è¾ĸ +çĸ«æĥħ æľŁéĹ´ +触 æij¸ +éĺ¶æ®µ æĢ§ +ä¼ļ è§īå¾Ĺ +çļĦ çĶ»éĿ¢ +æİ¥åıĹ äºĨ +表达 äºĨ +éĤĵ å°ı +éĤĵå°ı å¹³ +åħļ é£İ +åħļé£İ å»īæĶ¿ +åķĨ åѦéĻ¢ +åħij æį¢ +é£Łåĵģ èį¯åĵģ +éĿŀ常 好çļĦ +çľ ¯ +纳 ç±³ +åĬ¨ æijĩ +åĽŀ éģ¿ +çľĭ èijĹ +款 项 +åħ« å¹´ +åģļ 个 +æĸĩ æ¡£ +éĩijèŀį ç§ijæĬĢ +åħ¶ä¸Ń æľī +äºĨä¸Ģ ç³»åĪĹ +æĹĹèΰ åºĹ +ç§° èµŀ +éĽ¢ éĸĭ +åζ åĨ· +å®¶ éŨåı£ +åįģ å¤ļ +ä¼´ ä¾£ +çľĭ çĹħ +æĭī çĿĢ +æī Ĵ +çĸ² æĥ« +å°ijæķ° æ°ijæĹı +åĽ¾ å½¢ +è½ § +å¢ŀ éĩı +饲 åħ» +çģ« å±± +æ¯ı 个æľĪ +ä½ľä¸º ä¸ĢåIJį +è½´ æī¿ +æĸĩ 书 +ç¼ ķ +åħ·ä½ĵ æĥħåĨµ +çĹĽ çĤ¹ +缴 éĶĢ +å¡ Ĭ +ä¹Ł æľĥ +çĥŃ æ½® +å¹³ æ°ij +æ¼Ķåͱ ä¼ļ +æķĻ çłĶ +éĢĥ éģ¿ +ä¸Ģ è´¯ +å°± è¶Ĭ +å®ŀ å®ŀåľ¨ +å®ŀå®ŀåľ¨ åľ¨ +ä¹łè¿ijå¹³ æĢ» +æº º +å¿ĥ åºķ +éķ¿ å¾ģ +媽 媽 +第ä¸ī 次 +åĩº æ¼Ķ +çĭĢ æ³ģ +å°Ķ æĸ¯ +代çIJĨ åķĨ +çĨ ı +çļĦ 对象 +ç͵ éĩı +è¡Į åĪĹ +åĽ½ 人 +è·ij äºĨ +åįĶ åĬ© +èIJ¥ è¿IJ +å¸Ī åħĦ +æ¦ ® +æĥ³ åĥı +æĢ§ 强 +ç§ijåѦ çłĶç©¶ +å»¶ å®ī +ä¸¥æł¼ èIJ½å®ŀ +é¢Ĩ ä¼ļ +缸 å·® +è·¯ 人 +çĶ « +æľī ä»·å̼ +æľīä»·å̼ çļĦ +ç¾İ åĽ¢ +æ°ij主 çĶŁæ´» +æĪij æīį +ç¾İåĽ½ 人 +æ°Ķ åij³ +åıį å°Ħ +çļĦ åĨ³å¿ĥ +大 è±Ĩ +交 代 +è¿Ľ åĩº +åıį æĬĹ +æĮĩ çļĦæĺ¯ +ä»· ä½į +è¿Ľ é©» +ä¸Ĭ çϾ +ä½į åĪĹ +ä¸ŃåĽ½ ä¼ģä¸ļ +çļĦ好 å¤Ħ +主 ç¼ĸ +æ±½ æ²¹ +ä½Ĩ æĪij们 +æĢİä¹Ī çľĭ +é»Ħ å±± +å¤ļ åªĴä½ĵ +åIJİ åį« +èİ·å¾Ĺ æĽ´å¤ļ +åĬ¡ å¿ħ +为 å¥ijæľº +é¦ĸ 饰 +ä¸ĩ åįļ +è¶ĬæĿ¥è¶Ĭ 大 +ä¸ĵ项 è¡ĮåĬ¨ +å¥ĭ è¿Ľ +ä»į çĦ¶æĺ¯ +è´¨ æĦŁ +å¦Ĥæŀľ ä¸įæĺ¯ +ç«Ļ èµ·æĿ¥ +ä¹¾ éļĨ +åı¯æĢķ çļĦ +å¯Į è´µ +æ¸ħ ç®Ĺ +åIJij ä¸ĭ +åĢ ļ +çļĦ çŃĶæ¡Ī +èι ä¸Ĭ +çļĦ羣å®ŀ æĢ§ +çŃī åĬŁèĥ½ +åĸľ åī§ +å¨ģ åĬĽ +æĸ° é¢ĸ +æł¸ ç͵ +æĬ¥ éĶĢ +æķħ 乡 +ä¼´ éļı +éŀ Ń +å¦Ĭ å¨ł +åĪĨ åĮĸ +æľī å¾Ī大 +æĢİä¹Ī 说 +æĻĤ 代 +产 åĩº +ä»ĭç»į 说 +å¤ĦçIJĨ åύ +èĨ¨ èĥĢ +åī¯ å¸Ĥéķ¿ +çļĦ 妻åŃIJ +æł· åĵģ +åIJĮæ¯Ķ ä¸ĭéĻį +åħĥ å·¦åı³ +ç͍ èĩªå·±çļĦ +é«ĺ éĽĦ +æĺ¥ æĻļ +ä¹Ł æľīå¾Īå¤ļ +çľ¼ çIJĥ +æķ£ æŃ¥ +ä»ĸ们 éĥ½ +第ä¸Ģ å®¶ +åĬŀ 好 +å®ī éĺ² +ä¸Ģ ä¸ĩ +åľ¨ éĩĮéĿ¢ +éŁ³ é¢ij +åı£ åı· +ä¸Ģ è¶Ł +ç¦ı çī¹ +é³ ŀ +æĥĬ èī³ +æĸ° å¨ĺ +绿èī² åıijå±ķ +ä¸Ń å¼ı +ä¹Ł åıªæľī +çݰ 身 +åı¯ ä¾Ľ +æ¯ı ä¸Ģ个人 +第ä¸ī èĢħ +åľ° å½¢ +éĴ¢ ç»ĵæŀĦ +çĽijçĿ£ æ£ĢæŁ¥ +åı« æĪij +èĩ´ æķ¬ +æ´Ĺ æīĭ +ä¸ĭ è°ĥ +康 çĨĻ +æĪIJ交 éĩı +ä¹Ł æĪIJ为 +åħī æ»ij +å®Įæķ´ æĢ§ +çģ ¼ +ç¶² éłģ +éķ¿ å¯¿ +éģ© ç͍ +çļĦä¸Ģ 项 +çŀ© 缮 +æĬĬ èĩªå·±çļĦ +éĵ¶è¡Į åį¡ +å°± å¿ħé¡» +ç¾İ çϽ +éŀį å±± +æľ¬ é¢Ĩ +ä¸Ģ ç¢Ĺ +æīĵ æ³ķ +æĤ¨ 好 +对 åŃ©åŃIJ +æĬ¥éģĵ ç§° +ä¼ł åĩº +大 èĩ£ +ç¬ ĭ +çĽ ı +é¾ ļ +缴 线 +æĻº åºĵ +ç§Ł 车 +é£İ åij³ +çľĭ ä¸Ģä¸ĭ +æİ¨ éĶĢ +éĥ¨ éĥ¨éķ¿ +è´¨éĩı åĴĮ +åĪĬ çĻ» +å·¥ä¸ļ åĮĸ +çİĩ 为 +鼶 ä»¶ +硬 åĮĸ +ä¸Ĭ åįĥ +ç»ıéªĮ å̼ +å¹³ è¡Į +声 éģĵ +æľįåĬ¡ è´¨éĩı +çĶŁ çĶ¢ +æľĢ 容æĺĵ +ä¸Ģ æŀļ +å¹´ æĬ¥ +åħ¬ ç½ij +åħ¬ç½ij å®ī +åħ¬ç½ijå®ī å¤ĩ +çļĦ èĥ½éĩı +å®ŀéĻħ è¡ĮåĬ¨ +è¦ģ ä¸įè¦ģ +æĹ¥æľ¬ 人 +è̶ 稣 +ç¼ĸ åī§ +æ¶ © +åį° å°¼ +ä¸Ĭä¸ĭ 游 +åĩł åı¥ +ä¸Ń éĵģ +ç°¡ åĸ® +èĩª 带 +çĶŁ äºİ +ä¸Ģ åı£æ°Ķ +åĭ¤ å¥ĭ +éĻį ä»· +å±ķçݰ äºĨ +å¸ĥ æĭī +ä¼ļ éĢīæĭ© +çļĦ ç»ıåħ¸ +好 æľĭåıĭ +车 éģĵ +æķ´ åĢĭ +åľ ĵ +éķ¿æľŁ 以æĿ¥ +æĬķ å½± +çļĩ åĨł +è¿ĩ 大 +åijĬè¯ī ä»ĸ +ä¼ģä¸ļ æıIJä¾Ľ +æĬ½ 象 +éĢĤ 度 +çļĦ 女åŃ© +èµ· ä¼ı +çļĦ åĬŁæķĪ +ä¸ĵ项 æķ´æ²» +åı¯ éĢļè¿ĩ +ä¸įåIJĮ ç¨ĭ度 +å¼Ĥ è®® +åĩĢ èµĦ产 +åij Ĺ +ä»Ģä¹Ī åij¢ +å·¡ éĢ» +è¸ı ä¸Ĭ +ä½Ĩ å®ĥ +ç²¾ 度 +管 å±Ģ +第ä¸Ģ åIJį +åĨħ åŃĺ +æijĨ åľ¨ +åī© ä¸ĭ +主ä½ĵ 责任 +çĤ¹ åįĬ +以 èĩ³äºİ +åħ»èĢģ ä¿ĿéĻ© +æĦŁåıĹ åΰäºĨ +çŁ¥åIJį çļĦ +å¯Į 豪 +妥 åĸĦ +åŃĻ åŃIJ +éĵ Ĥ +说 èĩªå·± +让 æĤ¨ +æķ° æİ§ +çļĦçľ¼ åħī +注 éĶĢ +çļĦ çģµéŃĤ +è¿ĺ ä¸įéĶĻ +éĹ® ä»ĸ +èĩªä¸» çłĶåıij +èĵ ĭ +ç´« èī² +åĽ½å®¶ å®īåħ¨ +è¾½å®ģ çľģ +ä¹Ł æ¯Ķè¾ĥ +ç¾İ èĤ¡ +ä¸įç¡®å®ļ æĢ§ +å¿ĥ 头 +æĪ ³ +级 åĪ«çļĦ +论 è¿° +çļĦ åĽŀçŃĶ +ä¿Ŀè¯ģ éĩij +çŃī è¡Įä¸ļ +幸ç¦ı æĦŁ +æŃ§ è§Ĩ +æľº 票 +æ´¾ 人 +èĩ´ åij½ +åĺ´ è§Ĵ +æĸ°éĹ» ä¸Ńå¿ĥ +æĶ¾å¼ĥ äºĨ +å®ľ å±ħ +åĨĻ ä¸ĭ +éĹ® çŃĶ +è¿ĻéĩĮ æĺ¯ +å¤ļ åľ° +åĮºåŁŁ åĨħ +åīµ æĸ° +çľĭ ä»ĸ +æī§æ³ķ 人åijĺ +åĬ¨ æľº +éŁ³ åĵį +çļĦ åij½è¿IJ +é¡¶ éĥ¨ +åĵ Ł +éĥ½ æľĥ +æīĵéĢł æĪIJ +æĦı åĽ¾ +çļ ĸ +åĢĴ åħ¥ +å·´ èIJ¨ +åĬ© åѦ +å¤į åı¤ +åIJ¯ ç͍ +åĽ½éĻħ å¸Ĥåľº +åĤ¨ èĥ½ +é»ijé¾Ļæ±Ł çľģ +ä¹ĺ 车 +è¿IJåĬ¨ ä¼ļ +ä¿Ŀ åĪ© +çŁ³ æĿIJ +çµ ® +çĤĴ ä½ľ +çļĦ ä¿¡ä»» +å°± æĪIJäºĨ +åı¯ è§Ĥ +çļĩ ä¸Ĭ +è¿Ļ åĩłå¤© +ä¸Ģ éĶ® +åĨ· åĨ» +ä¿Ŀ åį« +æł¸ æ¡ĥ +åIJĪä½ľ åħ³ç³» +éĢģ åĩº +æĹĹ ä¸ĭçļĦ +åľ¨ ä¹İ +为 广大 +åįĪ é¤IJ +ä¸ĵ 访 +æĪĸ å°Ĩ +éĿĴå²Ľ å¸Ĥ +å¥Ķ è·ij +æĹ¥ æĬ¥éģĵ +å¥ij åIJĪ +æĸ° æĺ¥ +ä¸į å°ıå¿ĥ +两 ä¸ī +æĦıæĢĿ æĺ¯ +åĨ· èĹı +çļĦ çĹĩçĬ¶ +æĢ§ åij½ +è¶ħ æłĩ +å¯Ĩ 碼 +ç§ijæĬĢ èĤ¡ä»½ +äºĨä¸Ģ æī¹ +çĿ£ å¯Ł +åªĴ ä»ĭ +å°Ħ æīĭ +ä¿® åħ» +çīĩ åĪ» +éĢĤåIJĪ èĩªå·± +åıªè¦ģ æĺ¯ +åIJĥ è¿ĩ +éĩij éĵ¶ +缴 å±ŀ +åѦ éĹ® +åİĭ åζ +çªĹ å¤ĸ +æĶ¶ åΰäºĨ +åħ¨åĽ½ 人大 +ä½Ĩæĺ¯ 对äºİ +åľ¨ æķ´ä¸ª +çļĦ èĥĮåIJİ +åĩıå°ij äºĨ +åıį èħIJ +åıįèħIJ åĢ¡ +åıįèħIJåĢ¡ å»ī +æĹ · +åĪĨ æľŁ +åľ¨ æ·±åľ³ +æīĵ çĿĢ +æī« ä¸Ģ +æī«ä¸Ģ æī« +æĶ¿åºľ éĥ¨éŨ +æİ¥ è¿ŀ +å±ŀäºİ èĩªå·± +åŃIJ å¼¹ +åIJĮæł· æĺ¯ +æĢ» åħ± +车 ä¼ģ +æ¢ ĵ +åħ¬ é¡· +åıij 声 +éĴ Ľ +èµ°åĬ¿ åĽ¾ +主 èIJ¥ +åĸ Ķ +æķ°æį® åĪĨæŀIJ +ä¸į è¿ľ +æľī åIJį +æľīåIJį çļĦ +åģ¿ è¿ĺ +å¾Ī ä½İ +è®ĵ 人 +èĿ ī +é«ĺ è´µ +å°ij 许 +æ° Ł +å¹ ¢ +亲 æĥħ +è¿Ļä»¶ äºĭæĥħ +ç͍ é¤IJ +缸åħ³ æĸ°éĹ» +å°± åºĶ该 +ç»Ī çĤ¹ +æĺ¯ å¤ļå°ij +çĻ» åľº +è¯ķ 管 +è¯ķ管 å©´åĦ¿ +åģļ 大 +åģļ大 åģļ强 +çļĦ ä¾ĭåŃIJ +åħ« 个 +æĺİ æĹ¥ +çĤ ³ +èµ° åİ» +éģ º +å¢ © +ä½ĵä¼ļ åΰ +åĴ ı +ä¸ĭ è¾¾ +å¤į åıij +追 éĢIJ +æīĵ åĵį +çļĦ éļ±ç§ģæ¬Ĭ +åħ·æľī ä¸Ģå®ļ +è¿Ļä¹Ī å¤ļå¹´ +æłij æŀĹ +æľĢ éķ¿ +åIJĮ èĥŀ +åħī æ³½ +åŁŁ åIJį +æĮĩ åIJij +åıĹ害 èĢħ +æłij èĦĤ +æľīå¤ļ 大 +大 éĿ¢ç§¯ +æĹł ç¼Ŀ +æĶ¹ æŃ£ +æĽ´å¤ļ çļĦæĺ¯ +æľŁ æľ« +æŃ ¼ +ä¹ī ä¹Į +éĤ£ ä½ł +çļĦ 第ä¸Ģ个 +èĮ µ +å° § +èį « +ä¸įä»ħ åı¯ä»¥ +æ¶Į çݰ +æĢ» éĿ¢ç§¯ +æĸ°éĹ» åıijå¸ĥ +æ°ij ç͍ +å°± 读 +æīĵ è´¥ +å¤ĸ è¯Ń +æĪij们 ä¸Ģèµ· +é¢Ħ å®ļ +çĥ¹ 饪 +æľĢ 主è¦ģ +æľĢ主è¦ģ çļĦ +çīĮ çħ§ +åĽł åħ¶ +ä½İ ä¸ĭ +ä¼ļ åIJĮ +è§ģ è§£ +éĹ´ éļĶ +æķĻ ç¨ĭ +å° ī +å¸Ĥ ä¸Ńå¿ĥ +åħ³éĶ® æĺ¯ +æµ· åįĹçľģ +çī¹åĪ« æĺ¯åľ¨ +ä¸ŃåĽ½ 大éĻĨ +åħħè¶³ çļĦ +æĹ¢ èĥ½ +åĤ³ çµ± +çijľ ä¼½ +åħ¥ åĽ´ +æħ¢æħ¢ åľ° +æĬ¥ éħ¬ +æī¹ å¤į +å·¥ä¸ļ åĽŃåĮº +ä¸İ åıijå±ķ +èĥ¸ éĥ¨ +åľ¨ ç½ij绾 +åľ¨ç½ij绾 ä¸Ĭ +交 è°Ī +æĽ´ æĶ¹ +åįłæľī çİĩ +ä¸Ŀ绸 ä¹ĭè·¯ +è¡ Ľ +çłĶ åΤ +åĪ ª +åĪª éϤ +è¿Ļ åıª +çļĦ æ°Ķæģ¯ +åĬł å·ŀ +éĴ § +çIJĨäºĭ éķ¿ +ä¸ĸ å®¶ +æµģè¡Į çļĦ +å¾Ī æľīåı¯èĥ½ +们 éĥ½ +ç»ıèIJ¥ 模å¼ı +è¡Įä¸ļ ä¸Ń +éĢļçŁ¥ 书 +åij½ é¢ĺ +æľ¬ ç¶²ç«Ļ +æ²Ļ çī¹ +åıij åħī +é«ĺ ä»· +å·² çĦ¶ +åıĮ åįģä¸Ģ +ä¸Ĭ è¯ī +ç¿ħ èĨĢ +è¿Ļä¸Ģ å¹´ +大ä¼ļ ä¸Ĭ +éĩ ī +å®Įåħ¨ æĺ¯ +å¾Ĺ 太 +ä¸Ģèά 人 +è¿ĺ ç®Ĺ +æĬĺ åıł +æĬķ æľº +çĤ¹ çĩĥ +çݰéĩij æµģ +åħĶ åŃIJ +ç½ij æł¼ +æİ¥ è¿ĩ +ä¾Ľ è´§ +éĺ´ å½± +åİŁ åħĪ +æį £ +å·¦ ä¾§ +åħĭ æĭī +æīĵ åį¡ +ç§ij æ¯Ķ +æ±ĩ éĽĨ +åľ°çIJĨ ä½įç½® +è¯Ħ å§Ķ +ç»ĵåIJĪ èµ·æĿ¥ +è¿Ľåħ¥ åΰ +åı¯ è¡Į +åı¯è¡Į æĢ§ +让 å®ĥ +åĪ¶åº¦ æĶ¹éĿ© +çĶĺèĤĥ çľģ +åĵ Ĺ +åģı åģı +è¡£ çī© +ç¥Ŀ è´º +æºIJ èĩª +å¹¶ä¸į 代表 +åĽ½ 度 +好 åĿı +æĿ ĸ +æĿŃ å·ŀå¸Ĥ +湿 度 +é² ¸ +åįļ 彩 +æ³° å±± +æĿij èIJ½ +æĸ° èģŀ +èĤ ĭ +åı¤èĢģ çļĦ +çļĦ ç§ĺå¯Ĩ +ä¸Ģ个 éĹ®é¢ĺ +éģı åζ +åįĥ 亿 +è¿ĩ 硬 +å°Ħ åĩ» +èĩªçĦ¶ æĺ¯ +产 åĮº +çĤ¹ çĤ¹å¤´ +åı¯ä»¥ 帮åĬ© +说 å®ŀ +说å®ŀ è¯Ŀ +æĪij åıªæĺ¯ +ä¹ĭ ä½Ļ +åIJĮæĹ¶ ä¹Łæĺ¯ +ä¸ŃåĽ½ éĺŁ +建æĪIJ åIJİ +ä¹IJ è§Ĩ +åij¨ å²ģ +èᝠåºĹ +éĩij åįİ +严éĩį å½±åĵį +è´¨ åľ° +æĹħ éģĬ +åħµ åύ +æķĻèĤ² æķĻåѦ +离 åİ» +åIJĦå¼ı åIJĦæł· +ä»ĭ ç´ +ä»ĭç´ ¹ +å¼Ģ 头 +å°Ĩ èĩªå·±çļĦ +åIJ¬ åĬĽ +ä¿¡æģ¯ ç³»ç»Ł +ä»İ æł¹æľ¬ +ä»İæł¹æľ¬ ä¸Ĭ +æİĮ 声 +欢 åĸľ +å±ķ åĮº +åķ ¸ +太å¤ļ äºĨ +éĹ² ç½® +èĥ¡ èIJĿåįľ +å§Ķ å®£ä¼ł +å§Ķå®£ä¼ł éĥ¨ +åįĹ éĺ³ +å·ŀ åĮº +ä¸İ æĹ¶ +ä¸İæĹ¶ 俱 +ä¸İæĹ¶ä¿± è¿Ľ +å«Įçĸij 人 +èī¯ å¿ĥ +头 é¡¶ +è´¢ æĬ¥ +ä½Ľ æ³ķ +å¾ µ +åİŁ ä»¶ +åĭ ŀ +çĶ· 篮 +å¤ĸåĽ½ 人 +è¿Ŀ 纪 +æī¾ äºĨ +æįķ æįī +缸 è¯Ĩ +æIJľ éĽĨ +çļĦ ä¼Łå¤§ +ä¸ī ç»´ +å°±è¡Į äºĨ +çĭIJ æľĪ +çĭIJæľĪ å±± +å¸ĮæľĽ éĢļè¿ĩ +èĢĮ 对äºİ +éĿ¢ å°į +åĨĽ åĽ¢ +è¡Ĺ åĮº +æĤ¬ æĮĤ +便 ç§ĺ +æľīä¸Ģ çĤ¹ +ä¼ļè®® ä¸Ĭ +ä¸ĭ æīĭ +廣 åijĬ +äºĶ è¡Į +çŃī åĢĻ +ç´§ç´§ åĽ´ç»ķ +æĭ¿ äºĨ +æ¡Į éĿ¢ +ç¥ŀ æĥħ +éĽĦ åİļ +çŀ ³ +楼 ä¸ĭ +å½ ª +äºĭ åıij +åĨį è§ģ +é¤ ĺ +é¢Ħ åĶ® +åİ» çľĭçľĭ +æĪij们 åºĶ该 +ä¸ī å®¶ +æµ Ĭ +ä¹IJ éĺŁ +çľĭ ä¸įè§ģ +èĦij åŃIJ +æĮģ æľīçļĦ +çϽ èıľ +éĹª çĥģ +åĸĿ æ°´ +æİ§åζ ç³»ç»Ł +ä¸ĵ åĮº +æľĿ å»· +æĪij å¿ĥéĩĮ +å±ķ åİħ +èľĺ èĽĽ +åĨ» ç»ĵ +ç² ª +åº IJ +åIJij 社ä¼ļ +åĨ³çŃĸ éĥ¨ç½² +çŁŃ æľŁåĨħ +æĸ° ä¸ļæĢģ +æľ Ķ +æĹ¶ æĬ¥ +使 ä¹ĭ +åĽł åŃIJ +åıĤä¸İ èĢħ +çļĦ 年轻人 +æīĭ 表 +å°ģ éĶģ +为ä»Ģä¹Ī ä¸į +åIJ¸ çĥŁ +æ¯Ĵ ç´ł +åĪij æ³ķ +磫 æŃ£ +身 æĹģ +åİŁ è°ħ +çĽij æĬ¤ +æŃ¤ å¤Ħ +éļ¨ æĻĤ +æŀľ å®ŀ +åĮ»çĸĹ æľįåĬ¡ +ä¸į åIJĪçIJĨ +æIJŀ 好 +çļĦ èĦļæŃ¥ +å¤ĸ å¥Ĺ +ç¶ĵ éģİ +æĶ¾ ç¼ĵ +åģľ çķĻ +æĺŁ çIJĥ +çļĦä¸Ģ éĿ¢ +åĩł ä½ķ +è½® åĽŀ +æ¯Ľ å·¾ +ä¿® çIJĨ +ä¸įçŁ¥ ä¸į +ä¸įçŁ¥ä¸į è§ī +æķ´ 个人 +æ¯ģ çģŃ +åı° å·ŀ +使ç͍ 寿åij½ +é»ij çϽ +æij¸ ç´¢ +é¼ł æłĩ +éĿ© æĸ° +éº µ +ä¸ĵéŨ 为 +å¾Īå¤ļ æľĭåıĭ +å·¥ä½ľ ç»Ħ +åIJĪ å½± +çĤº ä»Ģ麼 +æŀģ 度 +çļĦ è¿ĽæŃ¥ +å½ĵ ä¹ĭ +å½ĵä¹ĭ æĹł +å½ĵä¹ĭæĹł æĦ§ +è´´ è¿ij +å°º 度 +åľ¨ çİ°åľº +éĻį 临 +åħ»èĢģ éĩij +ç£ ķ +åı¯ä»¥ 使 +管çIJĨ æ°´å¹³ +æľ¬æĬ¥ è®°èĢħ +æ³ķ 令 +åį¡ è½¦ +举 æµ· +å¤ļ éĩį +åħ¶ éĹ´ +ç´ Ļ +éĩį大 é¡¹çĽ® +æ±Ĺ æ°´ +ç»Ħ å§Ķä¼ļ +ä¿¡æģ¯ åħ¬å¼Ģ +ä¸į论 æĺ¯ +ä¸Ģ åIJ¬ +èĴ¸ æ±½ +æıŃ ç§ĺ +è¶ħ éģİ +触 åıij +å© ¦ +åħ³èģĶ äº¤æĺĵ +å°± ç»Ļ大家 +好 ä¹ħ +åĢŁ è´· +游æĪı è§Ĵèī² +å¼ĢåIJ¯ äºĨ +æİ ł +åħļçļĦ åįģä¹Ŀ +ä¸ĭ 鼨 +çŁŃ æĹ¶éĹ´åĨħ +å¯ ħ +导 åħ¥ +å·¥ä½ľ ç»ıéªĮ +ä¹Ł åıªèĥ½ +鼷 éľĨ +è·Ł è¿Ľ +åį¡ éĢļ +é¢ĩ æľī +æľº ä½ĵ +æĪĺ士 èģĮä¸ļ +女 主 +ä½ĵåζ æľºåζ +è¶³ åįı +èĪĴéĢĤ çļĦ +åĢŁ åı£ +æī¹ åΤ +æķ° å̼ +è« ¾ +éĺ¿æĭī 伯 +åĺ İ +æħ ¶ +è¾¾ 人 +å¼Ģ æ°´ +大 鼨 +温 室 +ä½İ è¿· +ä»į æĹ§ +éªĹ åŃIJ +亲 å±ŀ +çIJĨ æĻº +æľ¬ åŁºéĩij +å¨ ħ +åĨĻåŃĹ æ¥¼ +å¢Ļ å£ģ +å® µ +èϽ çĦ¶æĺ¯ +顺 çĿĢ +åħ« åᦠ+åķĨ ç͍ +ä¸į 失 +è¿· èĮ« +顺 便 +æļij æľŁ +欺 è´Ł +é¢ij é¢ij +该 æł¡ +æĸĻ çIJĨ +æ·± æĥħ +åīį éĶĭ +ä¿Ŀ èŃī +èģĮä¸ļ çĶŁæ¶¯ +åħ¬ å¼Ģåıij +åħ¬å¼Ģåıij è¡Į +åħ¥ æĪ· +éł ĵ +å̾ åIJ¬ +éŃ ģ +æĦī æĤ¦ +åĽŀ åIJĪ +åħ¨åĬĽ 以 +åħ¨åĬĽä»¥ èµ´ +åĥ¹ å̼ +èĥ½åĬĽ 强 +ç»ı å¼Ģ +ç»ıå¼Ģ åĮº +è¿ľ æĸ¹ +çļĦ éģĵçIJĨ +缴 åįĩ +缴åįĩ æľº +为主é¢ĺ çļĦ +ç»Ļ æĤ¨ +è¿ĺ æĥ³ +æ¯Ķ æĪij +åĨľ çī§ +æµ· åºķ +çŃ¾è®¢ äºĨ +对äºİ æĪij们 +æĹ¶ 许 +éĶ® çĽĺ +å®ŀéĻħ æİ§åζ +çļĦ æ¨¡æł· +åıįæĺł äºĨ +代 åĬŀ +åĮ» ç͍ +éĽĨ ç»ĵ +åıijå±ķ åīįæĻ¯ +æĮĩ çĿĢ +åįİ åĮĹ +è¿Ļ åĩłä¸ª +åIJį æ°Ķ +åĤį æĻļ +èĩª åıij +æ³¢ åħ° +大åĬĽ æİ¨è¿Ľ +èĩª ç§° +èįĨ å·ŀ +æIJį 害 +äºĨä¸Ģ åı¥ +æľĢåĪĿ çļĦ +éĩijèŀį å᱿ľº +æĢĢ å¿µ +è¡Į åĭķ +女 æİĴ +ä¸į è§£ +ä¼ł éĶĢ +转载 请 +饰 åĵģ +åıª 为 +ä¸İ ä¼Ĺ +ä¸İä¼Ĺ ä¸įåIJĮ +èĥ½ èĢĹ +èı© æıIJ +è¿ij 两年 +è¿Ķ 乡 +马ä¸Ĭ å°± +äºĮ çŃīå¥ĸ +æ°´ 管 +æ³ķ åѦ +çģŃ çģ« +大 å§IJ +åij¨ 转 +æľī æľŁ +æľīæľŁ å¾Ĵ +æľīæľŁå¾Ĵ åĪij +å°į æĸ¹ +ç¥ŀ èī² +æ²¹ èĦĤ +ä¸ī çĤ¹ +ä¸į åĪ©äºİ +äºĭä¸ļ éĥ¨ +å°± è·Ł +å¼Ģ æĶ¯ +å°ı 女åŃ© +åħ±åIJĮ åĬªåĬĽ +çĶļèĩ³ è¿ĺ +è¿Ļ åIJį +è¿Ļ ç¬Ķ +çݯ åį« +æľī ç§į +è§Ĩ åĬĽ +çĨŁ çŁ¥ +åħ¬ç§¯ éĩij +æ¶Īéĺ² å®īåħ¨ +é¢ĩ 为 +大 èħ¿ +éĿ ¶ +çī¹ æķĪ +æľįåĬ¡ åĮº +å¼Ģ åĩº +深度 èŀįåIJĪ +æĹł å¿§ +æŁ¥ éĺħ +ç»Ī ç»ĵ +ä¿Ŀ ç¨İ +è¨İ è«ĸ +å½ĵ åģļ +è·³ èĪŀ +å¯ § +女 çİĭ +è®°èĢħ åľ¨ +åħ¨ 产ä¸ļéĵ¾ +è´¯ éĢļ +åħ´ ä¸ļ +éĻį åΰ +å°ģ éĿ¢ +åħ¨éĿ¢ æİ¨è¿Ľ +奶 èĮ¶ +éĢī åĿĢ +äºĨä¸Ģ åľº +åIJĮ ä¼´ +è®® 论 +æIJ ĵ +诸 èijĽ +诸èijĽ 亮 +å¹² åĺĽ +æµģ æĦŁ +ä¸ĵä¸ļ çŁ¥è¯Ĩ +ç͵ ç«Ļ +åĩı å¼± +åĩº åħ¥ +åIJĦ çľģ +éĿŀ常 é«ĺ +åľ° 毯 +åıij æĸĩ +çĦ ī +çĥ§ çĥ¤ +å£ģ 纸 +æģ¶ åĮĸ +èĬ ¸ +èĥĸ åŃIJ +çĩ Ĵ +çľģ éĴ± +çϾ 强 +çIJĨå·¥ 大åѦ +éĴ¢ æĿIJ +åĽ½æľī èµĦ产 +æĪĺ æľº +æ³Ħ éľ² +åIJİéĿ¢ çļĦ +æ°´ èµĦæºIJ +æ¢ħ èĬ± +åĨĻ çĿĢ +ä¹ĭ 声 +æĹł åı¯ +æĺİ æľĿ +ç«ĭæĸ¹ ç±³ +ç· £ +æĶ¾ è¿ĩ +ç¦ı çͰ +å¾Ĺ ä½ı +åıĹ ä¼Ĺ +ä¸Ń 级 +çĹħ åıĺ +ä¸Ģ çŀ¬éĹ´ +æĿĥ éĩį +人æĢ§ åĮĸ +åĮ»çĸĹ åį«çĶŁ +ä¸įåΰ ä½į +æĻºèĥ½ å®¶å±ħ +饮 ç͍ +æ¼Ķ åıĺ +é«ĺ ç´łè´¨ +ä¹Ļ æĸ¹ +åģľ çķĻåľ¨ +èİ· æī¹ +ç©¿ æ¢Ń +客 åľº +æĮ½ åĽŀ +京 åŁİ +çĶŁåij½ åĬĽ +實 éļĽ +çĩ Ī +åĨį çݰ +çݰå®ŀ ä¸Ń +æľī ä¿¡å¿ĥ +çĸı éĢļ +åĺ´ åĶĩ +鼷 éĶĭ +èıľ åįķ +éħ ¯ +è¶ħ é«ĺ +å¾Ī é«ĺåħ´ +çĶŁ æ®ĸ +éĢł ä»· +误 åĮº +æĨ ĭ +好 æ¶Īæģ¯ +å´ Ń +以 èĩ´ +å¼Ģ çİ©ç¬ij +çĽij è§Ĩ +å·¡ å¯Ł +å¾· å·ŀ +æĹ© æĹ© +éĹª ç͵ +æĪª åĽ¾ +åı¯ä»¥ æł¹æį® +æīĭ èīº +æİ¥ 轨 +ç§į æĹı +æĢĢ éĩĮ +åİ» åĮ»éĻ¢ +ä¸Ģ äºĮ +å¼Ģ éĺĶ +åĩı éĢŁ +ä½Ĩ ä»İ +éĢĻ ä¸Ģ +åĩı åħį +主é¢ĺ æķĻèĤ² +å¼Ģå·¥ 建设 +è¹ ¦ +æľĪ 饼 +ä¸ĭ æ²ī +å°Ĭ 严 +éĻ ĩ +å®ŀ æľ¨ +å»ł åķĨ +声 ç§° +èĢĥ åľº +å¸ĥ é²ģ +èĩª æĿ¥ +èĩªæĿ¥ æ°´ +éĴ ¾ +å¹´ 以ä¸Ĭ +大 åıĶ +ä»ĸ å·²ç»ı +åħ¨ æĿij +èģĶç³» ç͵è¯Ŀ +为 导åIJij +åΤ å¤Ħ +对 éĺµ +缮 æ¨Ļ +åIJį é¢Ŀ +客 æ°Ķ +横 åIJij +çŃī åĨħ容 +åĩł çĤ¹ +è°Ī 论 +ä¸į ä¹ı +å±ķ çݰåĩº +è¾ĥ éķ¿ +éĢĨ 转 +å°ı æĻĤ +æĺ¯ å¤ļä¹Ī +æľ¬ æľĪ +è¿ij è§Ĩ +æĪIJç«ĭ 以æĿ¥ +代表 çĿĢ +æĬ¥ å¤į +æĪı æĽ² +è¨Ń åĤĻ +åħ¥ èĤ¡ +å¾ģ æľį +é«ĺ åĩº +èĪŀåı° ä¸Ĭ +å¿ĥ åĬ¨ +两 çĤ¹ +缸 çķ¶ +èĻ Ľ +主 页 +åĩł å®¶ +æĹł ä¸į +åįı å®ļ +æĸ IJ +å¯ĵ æĦı +åħ¨ 线 +æįķ é±¼ +åı¯ä»¥ ä»İ +æľī è¿Ļæł·çļĦ +æģ¶ éŃĶ +åĮħ åŃIJ +æģ ¤ +å¼Ģå¥ĸ ç»ĵæŀľ +ä¸į æŃ» +èĹ į +弯 æĽ² +æµ· 峡 +éĶĢ æ¯ģ +çļĦ çĭ¬çī¹ +示 æĦı +ä¸įèĥ½ åĨį +èĥ½ æĬĬ +éĺ² çº¿ +ä¸įå°ij äºİ +æ± Ģ +çļĦ éĤ£ä¸Ģ +羣 æĥħ +åŀ ® +被 æīĵ +åĽ½ å®ī +ç¾İ å¦Ļ +è¿Ļ åĩł +åĩº éģĵ +æľįåĬ¡ äºİ +æĪIJæŀľ 转åĮĸ +æīį åįİ +天 é¹ħ +åĩł 个人 +åĢĺ èĭ¥ +è̽ 误 +æĬĹ æĪĺ +è¡Į éĬ· +æĿ¥ è¢Ń +åĢŁ éĮ¢ +èįī èİĵ +ä¸¥æł¼ æī§è¡Į +举è¡Į äºĨ +å¤ĸ ç±į +å·² è¾¾ +æĿij åħļæĶ¯éĥ¨ +è¡ Ŀ +éĻį èĩ³ +æµ· éĩı +é¤IJ é¦Ĩ +æĢ¥ å¿Ļ +æ·± è¿ľ +å¾Ģ è¿Ķ +ç¨İåĬ¡ å±Ģ +å¹¿æ³Ľ åºĶç͍ +è®® åijĺ +æĹł æķĮ +çľ¼ åħī +çĥŃè¡Ģ ä¼łå¥ĩ +æŃ IJ +äºĨ äºĽ +è¿Ŀ èĥĮ +è¿Ļ æĺ¯ä¸Ģç§į +ä¸į 稳å®ļ +大家 åĪĨ享 +表 çı¾ +åīį åįģ +è·¯ è¿ĩ +æĴ © +åIJĮ æĥħ +ä¹ł ä¿Ĺ +åıij è´¢ +åºĶ æľīçļĦ +æĿİ æŁIJ +èĤ Ľ +马 åħĭ +éĢļ åijĬ +å·¨ 人 +ä¸Ģ åĽ¢ +éĢĻ æ¬¡ +ä¸į äºĨè§£ +æĸ½ è¡Į +èij¡èIJĦ çīĻ +åıĺå¾Ĺ æĽ´åĬł +æı £ +åĪĽæĸ° èĥ½åĬĽ +çķħ éĶĢ +表 æī¬ +æ¯Ķ åĪ© +æ¯ĶåĪ© æĹ¶ +åĮ»çĸĹ ä¿ĿéĻ© +æĵį 纵 +伤 亡 +æµİ å®ģ +åıĺ äºĨ +æľ¬æ¬¡ æ´»åĬ¨ +åľŁ 豪 +æĥ³ åĬŀæ³ķ +æĺ ķ +å½ĵ æĻļ +åĩº å±Ģ +çĥŃ è®® +è°Ī è°Ī +æĻĭ åįĩ +åĬ¿ å¿ħ +çĻ» å±± +éĤ£ åĦ¿ +åIJĥ åΰ +ä¹ĭ åŁİ +å¿« æĿ¥ +æ¹Ľ æ±Ł +第ä¸ī 个 +åħ¨éĿ¢ æıIJåįĩ +å¥ĸ åѦ +å¥ĸåѦ éĩij +æĬķåħ¥ 使ç͍ +é½IJ é²ģ +åı¯ä»¥ æĬĬ +åĴĮ ä»ĸçļĦ +è´ŃæĪ¿ èĢħ +æŃ£å¼ı åIJ¯åĬ¨ +åįİ æ¶¦ +ä¸įæĸŃ å®ĮåĸĦ +éĴ¢ æĿ¿ +ç´¯ 积 +满 èĦ¸ +åĽĽ æĸ¹ +è´¢ çī© +ä»ĸ们 ä¼ļ +å¤ı æĹ¥ +éĤ£ 个人 +éĿł çĿĢ +çĤ¹ äºĨ +çĤ¹äºĨ çĤ¹å¤´ +æ© ĭ +åıΠ好 +åıĪ好 åıĪ +åıĪ好åıĪ å¿« +éĺµ éĺµ +å°ģ 建 +æľ¬ çͰ +çī©ä¸ļ æľįåĬ¡ +èĩªè´¸ åĮº +åIJ ı +便åĪ© åºĹ +åĽ½å®¶ æłĩåĩĨ +éĿ¢ ç²ī +èī° è¾Ľ +æĶ» åħ³ +æīĵ åĮħ +车 éĺŁ +人 éĢī +åı¯ ä¸įæĺ¯ +äºĮ åįģå¹´ +åIJį å¸Ī +浦 举 +åħ¬ è¯ģ +è¿IJ éĢģ +æĺ¯ æľĢ好çļĦ +æŁĶ åĴĮ +çİĭ æŁIJ +çĹħ æĪ¿ +åĨ¶ éĩij +ä¸Ģä»¶ äºĭæĥħ +åį ¤ +åı¯ æİ§ +çī Ł +æĭ Ĥ +å·² äºİ +人 éĢł +çĶŁçī© åĮ»èᝠ+ä½ĵ çݰåĩº +èĤ² åĦ¿ +èĢģ å®ŀ +åľĸ çīĩ +è« ¸ +ç´¯ äºĨ +æĦŁåħ´è¶£ çļĦ +åĽ¾çīĩ æĿ¥æºIJ +ä¹Ł æĺ¯ä¸Ģç§į +æ¾İæ¹ĥ æĸ°éĹ» +æĹ¶ 表示 +åħī è¾ī +æĬ¥ åºŁ +å²ģ æĹ¶ +éħ ® +æ£Ģ ä¿® +åıĺ éĢŁ +åıĺéĢŁ ç®± +åľ¨ èģĮ +éı ¡ +æį Ĥ +çĿ£ åĬŀ +æ°¸ ä¸į +åģļ ä¸ĢäºĽ +åİĨ æĹ¶ +å·¥ç¨ĭ æľºæ¢° +æģ° å½ĵ +å°± åľ¨äºİ +ç§° åij¼ +éĢļ常 æĺ¯ +æł· å¼ı +åij¨ ä¸Ģ +èĭ± éķij +åĿĩ 线 +ä¼ł éĹ» +ç͍æĪ· ä½ĵéªĮ +èµŀ åIJĮ +骨 æĬĺ +为主 ä½ĵ +æ±Ł å±± +æ¸ħ æľĿ +æĶĢ åįĩ +ä¸į çĽ¸ä¿¡ +éĿ ´ +æŃ¦ åĬŁ +åĭ¤ åĬ³ +æĿ¥ æī¾ +å°Ĩ æĮģç»Ń +丫 头 +æ¨Ļ æºĸ +è£ ´ +深深 çļĦ +åŃķ èĤ² +è§ĦåĪĴ 建设 +æ¸ħ çν +ç²¾åĩĨ æī¶è´« +æīĵçł´ äºĨ +è¿Ļä¸Ģ 天 +å·¥ä½ľ æĢ»ç»ĵ +æĹħ ç¨ĭ +举 èIJ¥ +æĶ¾ å°Ħ +æľī åĩłä¸ª +éĿŀ çī©è´¨ +åIJĥ å¾Ĺ +åĹ ¨ +ä¼ļ åıijçĶŁ +篮 æĿ¿ +å¼Ģ å°ģ +麻 å°Ĩ +èıı æ³½ +ä¸į åIJĪ +ç³»åĪĹ äº§åĵģ +èѬ å¦Ĥ +ç¾İ èªī +èĩªå·± åĸľæ¬¢ +交æĺĵ ä¸Ńå¿ĥ +åIJĪ åͱ +使 æĪij +åĥı ç´ł +带 éĺŁ +ä½Ĩ 对äºİ +æĬĬ è¿Ļ个 +èĤĿ èĦı +åįķ纯 çļĦ +æĶ»åĿļ æĪĺ +缼 ä¼ļ +åijµ æĬ¤ +æª Ģ +èµ¶ ä¸Ĭ +æ¥ Ĭ +ä¹ħ äºĨ +ç¡ Ŀ +çŃĶ é¢ĺ +ä¿ĿæĮģ çĿĢ +è§ģ è¯Ĩ +çĤ¹ åĦ¿ +åįĬ 个æľĪ +æ» ĩ +浸 泡 +ä¼ł éĢģ +åľ¨ å¸Ĥåľºä¸Ĭ +ä¹ĭ 乡 +çī¹ éķ¿ +éĽ ŀ +èª ł +身 å¤Ħ +æŁł 檬 +身 ç©¿ +çľģ åħ¬å®ī +çľģåħ¬å®ī åİħ +åıĻ åĪ©äºļ +åĩł åĪĨéĴŁ +人 åĢij +åľ° 段 +èĩª åѦ +ä¹Ł è¶ĬæĿ¥è¶Ĭ +èģĮ æĿĥ +æĸ § +èĩ » +å½Ĵ 纳 +驾 é©Ń +éĥ¨åĪĨ åľ°åĮº +没æľī æĥ³åΰ +æĴ ĩ +ä¹Į é²ģ +ä¹Įé²ģ æľ¨ +ä¹Įé²ģæľ¨ é½IJ +èĤ² 人 +çļĦ æŃ¥ä¼IJ +å»¶ æľŁ +æ²¹ æ°Ķ +åģļ å®Į +åľ£ åľ° +丰 åİļ +宽 带 +åı¯éĿł çļĦ +åºŃ éĻ¢ +åŃ ľ +å°ı康 社ä¼ļ +å®īåħ¨ 管çIJĨ +å¹´ 第 +æİĴ 污 +èĥĮ åĮħ +å®¶ ä½ı +åħ¶å®ŀ å°±æĺ¯ +ä¼ļ è§ģ +帮åĬ© ä¼ģä¸ļ +ç½ij è´Ń +æĺ¯ ä¸įä¼ļ +飯 åºĹ +æŃ» åİ» +åħįçĸ« åĬĽ +æľ ķ +åĸĿ äºĨ +è½» å¾® +个æľĪ åĨħ +ç»Ħ åĽ¢ +åĴĮ å®ĮåĸĦ +é¸ ½ +æıIJ éĢŁ +西å®ī å¸Ĥ +ä¸Ńå¿ĥ 主任 +æĹ¶éĹ´ 为 +æľŁ æĿĥ +è¶ ķ +ä¸įä»ħ è¦ģ +æľį ä»İ +é¡ĺ æĦı +ä¸į å°ı +ä¸įå°ı çļĦ +ç° ĩ +çª ¦ +åĪĩ æĪIJ +åĵĪ åĪ© +天 羣 +ä¸Ģ次 次 +éĩij å¸ģ +æĢİä¹Ī èĥ½ +ç½ij è´· +ä¼ļ计 å¸Ī +çŁŃ 缺 +对 æłĩ +åıĺå¾Ĺ æĽ´ +åīį åĩłå¤© +éĺ² æ±Ľ +彩 èϹ +åĵģ ä½į +表 æł¼ +严 å¯Ĩ +æ¯Ľ åĪ©çİĩ +çļĦ åį±å®³ +å½ķ åζ +æ°´ åĬ¡ +èĥ½å¤Ł 让 +å¹³ æĿ¿ +ä¹³ æĪ¿ +è¸ı å®ŀ +é¦ĸ åĪĽ +é¦Ļ èķī +æĬ¥ 表 +ä¸Ģ æĬ¹ +åĩºçĶŁ äºİ +è²» ç͍ +åĩº 让 +åIJĪæ³ķ æĢ§ +å°¼ åħĭ +åĨ° åĨ· +é¦Ļ æ°Ķ +åı· ç§° +èµ· çłģ +åŁİ åİ¿ +çİ© èĢį +ä¸Ĭ éĻIJ +ä¼ļè®® ç²¾ç¥ŀ +æĹģè¾¹ çļĦ +便 ä¼ļ +æıŃ æĻĵ +çİ© æĦı +éĽª å±± +åIJij çĿĢ +ä½ĵèĤ² åľ¨çº¿ +说æĺİ ä¹¦ +åĮĸ èĤ¥ +åħļç»Ħ 书记 +åĬ¨ 人 +ä¹ĭ æīĢ +æľĪ èĩ³ +æľĢå¿« çļĦ +èĬĤ åģĩæĹ¥ +ä¸ĵ åľº +èĢĥ ä¸Ĭ +çª Ł +é²ľ è¡Ģ +è¾ĥ强 çļĦ +æĤĦ çĦ¶ +å¤ļ个 åĽ½å®¶ +çªĹ å¸ĺ +æŀģ å¤§åľ° +ä¸įç͍ æĭħå¿ĥ +è¿Ļä¹Ī åģļ +åĥ¹ æł¼ +ç¾İ丽 乡æĿij +å°ıæĹ¶ åĨħ +ç´§ è¿« +大 çģ« +èĥ³ èĨĬ +æĵįä½ľ ç³»ç»Ł +æ®ĭ çķĻ +åĨĻ åĩº +ç¦ģ å¿Į +åĬłçĽŁ åºĹ +è¿ij çϾ +便 åı¯ +æķ´æĶ¹ æİªæĸ½ +éĩĩ访 æĹ¶ +åĶIJ 代 +æ·±åĮĸ æĶ¹éĿ© +çŁ ¢ +éĥ½ åĸľæ¬¢ +è¶ĬæĿ¥è¶Ĭ é«ĺ +èĬ± æľµ +头 çĸ¼ +å®ī 康 +å¢ŀéķ¿ çİĩ +çľ¼ çľĭ +å°±æĺ¯ 为äºĨ +èĢĮ 导èĩ´ +åĬłå¿« 建设 +èĬ± æł· +åĨħå¿ĥ çļĦ +æĺĨ å±± +è³ĩ æºIJ +åĽŀåΰ å®¶ +èıĬ èĬ± +æ°´ éĩı +å¾ģ ä¿¡ +è¡ĮæĶ¿ åĮº +ä¹ĥ æĺ¯ +æĬķèµĦ é¡¹çĽ® +å«ģ ç»Ļ +ç¥ŀ åľ£ +ç¨ ł +æľ¬æĿ¥ å°± +éĢIJ ä¸Ģ +èģĮä¸ļ æĬĢæľ¯ +ä¸įèī¯ ä¿¡æģ¯ +æīĺ è¿IJ +åIJ¯ 示 +ä¹ĭ åħ§å®¹ +éŁ ¶ +奢 åįİ +æıŃ ç¤º +æĪIJ为 ä¸ŃåĽ½ +æ¶Īè´¹ åĵģ +åħ¬ ç͍ +æIJŀ å®ļ +请 ä½ł +æŁ ļ +åĨħ è¡£ +ä½Ĩ ä»ĸ们 +ä¿Ŀ 湿 +该 åİ¿ +饱 åĴĮ +æİ¨ åIJij +èµĦæĸĻ æĺ¾ç¤º +ä¸į å½±åĵį +人 人éĥ½ +åıijå±ķ 壮大 +åħ»èĢģ æľįåĬ¡ +çĶŁæ´» æ°´å¹³ +åIJĦ åİ¿ +ä½ł éľĢè¦ģ +说 çļĦæĺ¯ +å¤ĸ åªĴ +æŃ¤ 人 +次 è¦ģ +追 èµ¶ +åºĶ该 å¦Ĥä½ķ +æĹ¥ åĩĮæĻ¨ +çķ¥ æľī +éĥ½ æĥ³ +游 ä¹IJ +è¿Ļ款 游æĪı +å¹³ æ·¡ +æĺ¯ä¸Ģ åĢĭ +å¤ĩ èĢĥ +åζ æŃ¢ +ä¸Ģå®ļ èĥ½ +å¾Ĵ å¼Ł +以 çĤº +åįĥ åħĥ +äºĶ åħŃ +迪 士 +迪士 å°¼ +éĺ³ æĢ§ +åĨ¬å¥¥ ä¼ļ +å°±æĺ¯ åĽłä¸º +æĮĤ éĴ© +æ¦Ĥ åĨµ +åıªè¦ģ æľī +æ²¹ çĶ» +åľ° æłĩ +ä¸Ĭ è°ĥ +产ä¸ļ åĽŃåĮº +åħ« åįģ +æ£ ± +æ¶² æĻ¶ +æĿij å§Ķä¼ļ +çŃ¾çº¦ 仪å¼ı +è¿Ļ åħ¶ä¸Ń +åĨĻ éģĵ +示èĮĥ åŁºåľ° +éĩİçĶŁ åĬ¨çī© +鼻åŃIJ ä¿¡ç®± +åĽ½éĻħ è´¸æĺĵ +人 æĿĥ +ä¿Ŀ 管 +èĭ¥ æĤ¨ +åİĭ æĬij +é» Ľ +åľ° çľĭçĿĢ +éĻ ° +ä¸Ģå¹´ å¤ļ +ä»İ 容 +ä¸Ń æĸŃ +å¯Ł è§ī +ç§» 交 +éĶ ¯ +æĪĸ许 æĺ¯ +ç¶ ł +两 项 +æľĢ åĸľæ¬¢ +æľĢåĸľæ¬¢ çļĦ +å¤ľ éĩĮ +åIJĮ ä»ģ +åĪĽæĸ° 驱åĬ¨ +è°ģ èĥ½ +é£ ¾ +åħī åѦ +åİ Ħ +èĦ± é¢ĸ +èĦ±é¢ĸ èĢĮåĩº +è¿ ¦ +æĺ¯ ä¸įåı¯èĥ½ +çª ¥ +èĥ½ 满足 +宽 度 +伦 çIJĨ +åı¯ä»¥ èİ·å¾Ĺ +转 ä¼ļ +å±± æĿij +éĵº 设 +åĩº åĩ» +æĸĩåĮĸ èīºæľ¯ +ä¼ļè®® 室 +æŃĮ 声 +æ» Ķ +èIJİ ç¼© +æľįåĬ¡ åijĺ +åıij表 äºĨ +æĸ¼ æĺ¯ +æĺİç¡® è§Ħå®ļ +ç»´ å¥ĩ +æ°´ 产 +æĬķ ä¿Ŀ +éĺ´ éģĵ +èµ¶ å¿« +夺 å¾Ĺ +ä¸ĭ åįķ +çµģ åħ¬åı¸ +çݯ ç»ķ +å½ Ī +ä½ľé£İ 建设 +æĹħ游 æĻ¯åĮº +æľī æĽ´å¤ļçļĦ +丰å¯Į å¤ļ彩 +çIJĨè´¢ 产åĵģ +åĩº å·® +ä»İ严 æ²» +ä»İ严治 åħļ +缸 å¹² +æ»ĭ 润 +主åĬŀ æĸ¹ +åī§ åľº +æ»ļ çIJĥ +æ©Ħ æ¦Ħ +èĩªä¸» åĪĽæĸ° +éĢļ å¾Ģ +æł¼ å°Ķ +çļĦ ä¼ĺçĤ¹ +èĥĮ ä¸Ĭ +çª ľ +çĪĨ åĩº +å¹³ æķ´ +ä¸Ģ èĦļ +åħ¨ä½ĵ åijĺå·¥ +éĻIJ å®ļ +åŁİéķĩ åĮĸ +æ· ³ +éĢ® æįķ +è¡ĮåĬ¨ 计åĪĴ +æīĵ å¾Ĺ +åİļ éĩį +纪å½ķ çīĩ +åĿļ ä¿¡ +央 ä¼ģ +åĨį ä¹Łä¸į +天 涯 +åıĤèĢĥ èµĦæĸĻ +æľī æ¯Ĵ +åIJ¸ 纳 +è¶Ĭ åıij +éĩįè¦ģ æĦıä¹ī +åĽ½éĺ² éĥ¨ +è¿Ļ个 è¡Įä¸ļ +æĻ® æŁ¥ +å¼Ĥ æĢ§ +å»¶ è¿Ł +å°ı å¹ħ +èī² æĥħ +综åIJĪ æ²»çIJĨ +æŃ£æĺ¯ åĽłä¸º +产ä¸ļ ç»ĵæŀĦ +çłĶç©¶ æĬ¥åijĬ +åģľ ä¸ĭ +éķ¿ èĢģ +éĩĿ å°į +åįĹ京 å¸Ĥ +çģĮ æºī +转 è¿IJ +欺 è¯Ī +éĢł åģĩ +åĪĨå¸ĥ å¼ı +æĦŁ è§¦ +æĪij å½ĵæĹ¶ +åıij è§ī +åĽ¾ 纸 +æĶ¹ èī¯ +çĭł çĭł +åĨ² åĪº +æĸ° 京 +æĸ°äº¬ æĬ¥ +ç¥ŀ åύ +秸 ç§Ĩ +çĪ º +å°Ĩ è¿İæĿ¥ +å·¥ ä¿¡ +工信 éĥ¨ +éĻIJ éĩı +æŃ¢ æįŁ +åѦä¼ļ äºĨ +åįİ çĽĽ +åįİ缼 é¡¿ +å¾Į ä¾Ĩ +ä¸ĭéĿ¢ æĺ¯ +ä¸ĭéĿ¢æĺ¯ å°ı +æIJ¬ è¿IJ +ç¾İæľ¯ é¦Ĩ +æ¸ħ åĩī +å¤ļå¹´ åīį +è© ŀ +åįĥ ç±³ +表 è¿° +æ±Ł éŨ +åĬłæ²¹ ç«Ļ +æľ¬ èĥ½ +导 读 +åĽ´ è§Ĥ +å¹¶ åIJij +åŁºæľ¬ æĥħåĨµ +æīĵ å¼ĢäºĨ +è¿Ļ ä¸ī个 +æ±ķ 头 +强 æľīåĬĽ +强æľīåĬĽ çļĦ +è¿Ľ åľº +ä¹Ŀ æ±Ł +çIJĥ æĺŁ +好çľĭ çļĦ +大 æĪ· +æ¹ ¯ +å¥ĩ å¦Ļ +ä¹IJ åύ +æĪijçļĦ å¿ĥ +çľī 头 +åĨľä¸ļ çĶŁäº§ +ç¼ĸ çłģ +åŁº ç¤ +åŁºç¤ İ +天 æĸĩ +åĢĭ人 è³ĩè¨Ĭ +åİ» è¿ĩ +èģĨ åIJ¬ +æĶ¾ åģĩ +ä¸į åħ·å¤ĩ +æ·Ģ ç²ī +大 佬 +åħ¨ 天 +åħ¨éĿ¢ 建æĪIJ +éļIJ å½¢ +ç¼ħ ç͏ +åIJ ³ +è¡ĮæĶ¿ æī§æ³ķ +åŁİ åł¡ +èİ« æĸ¯ +èİ«æĸ¯ ç§ij +æīĢæľī æĿĥ +éĽĨ åľĺ +å±Ģ åī¯å±Ģéķ¿ +åĩłä¹İ 没æľī +æ´ģ åĩĢ +ç͵影 èĬĤ +åŃ© ç«¥ +æīĢ åģļçļĦ +æ¸ħ 代 +æĸ° çīĪ +éĵĿ åIJĪéĩij +为 æĬĵ +为æĬĵ æīĭ +åΤ å®ļ +çī¹ äº§ +æīĭ æ©Ł +ä¸įåı¯ æĪĸ +ä¸įåı¯æĪĸ 缺 +å¸Ĥåľº è§Ħ模 +åĿ ¯ +åĮ» åѦéĻ¢ +å¿« è¦ģ +èĮ ľ +æĬĺ èħ¾ +äºĨ è¿ĩæĿ¥ +æĬ¥åijĬ æľŁåĨħ +çī© ç§į +ç»Łè®¡ å±Ģ +æī© 建 +æ¶ ħ +责任 人 +éĺ İ +è¯Ħ è®® +å¾Ģ äºĭ +æīĢ ç¤º +æķ´ æ´ģ +éĹº èľľ +æĹħ éĢĶ +å®ŀ è®Ń +ä¹ĭ ç§° +å·´ 士 +éĢŁåº¦ å¿« +ä¸įä»ħ å¦ĤæŃ¤ +å®Ŀè´µ çļĦ +åºŁ çī© +æ²³ æ°´ +æİ¥ 纳 +ç²¾ æ¹Ľ +åħ¶æ¬¡ æĺ¯ +顺 å¾· +åħ¬åħ± åį«çĶŁ +è¤IJ èī² +ä¸į æĥľ +æĬĢæľ¯ æľįåĬ¡ +æİ · +æ±Ĥ èģĮ +ä¸ī 峡 +æĬķåħ¥ åΰ +太 åIJİ +åIJ¯åĬ¨ 仪å¼ı +缴æİ¥ å½±åĵį +æĸ° 款 +个 乡éķĩ +çϾ 亿 +åº « +ä¹Ł æŃ£æĺ¯ +åı¶ çīĩ +æľĢæĹ© çļĦ +æĪĺ 绩 +å·¥ æľŁ +æĻļ æľŁ +è¿Ļæł· 说 +è¯į è¯Ń +ä¾ Ħ +æķ£ çĥŃ +éĽĨæĪIJ çĶµè·¯ +åIJį è¯į +æĻº åķĨ +æĭ¥ åłµ +çĭĤ 欢 +è¿Ļ èά +æµ´ 室 +åijķ åIJIJ +æľªæĿ¥ åıijå±ķ +ä¸īä½į ä¸Ģä½ĵ +åªĴ é«Ķ +ä¸įå¾Ĺ 转载 +åĽłä¸º 她 +æĺ¾ç¤º å±ı +ä¾Ľ æļĸ +éĨ« éĻ¢ +æľī æĦıæĢĿ +æľīæĦıæĢĿ çļĦ +娱ä¹IJ åŁİ +åįµ å·¢ +åĪĽéĢł åĬĽ +竳 èĬĤ +人大 常å§Ķ +èĢĮ çİ°åľ¨ +å¤ĸ å©Ĩ +å¢ŀ æĮģ +äºĶ åįĥ +èĢģå¸Ī 们 +æ´Ľ æĿī +æ´ĽæĿī 磶 +æİĮæı¡ äºĨ +ä¸ŃåĽ½ æĸĩåĮĸ +æĸ° æĶ¿ +主è¦ģ ç͍äºİ +åıij çĥ§ +类似 äºİ +åĮĹ æŀģ +æĪij们 认为 +å¼¥ 漫 +åħ¨çIJĥ ç»ıæµİ +é¢ IJ +ä¸Ģèµ· è£ħä¿® +æĶ Ĵ +æĭī èIJ¨ +帶 ä¾Ĩ +åĨ· æ°´ +ä¸ī åĨľ +æĿ¿ æĿIJ +è¿ŀ è¿ŀ +éĵ ® +ç»ıèIJ¥ çIJĨ念 +å±± é¡¶ +å¾Ī æĥ³ +çĺ « +å§ĭç»Ī ä¿ĿæĮģ +åľ¨ 广å·ŀ +ä¸įåIJĮ æĦı +åıĺ åİĭ +åıĺåİĭ åύ +产 éĶĢ +表 éĿ¢ä¸Ĭ +æīĢ以 ä»ĸ +ç»ıéªĮ 丰å¯Į +éĥ¨ å§Ķ +åħµ åĽ¢ +æīĢ è¿° +æķ¦ çħĮ +ç»ıèIJ¥ èĮĥåĽ´ +åı£ è¯Ń +失 ä¿¡ +æ¯ı个人 çļĦ +æīĭ æĮģ +æģIJ æħĮ +åł¡ åŀĴ +é¦ ħ +éĵ¸ éĢł +æĭ¿ åĩºæĿ¥ +æİ¢ æµĭ +大家 ä¸Ģèµ· +å¥ § +å®ŀè´¨ æĢ§ +å°ı åĦ¿ +èĩº åįĹ +èĩºåįĹ å¸Ĥ +å¼Ģåıij èĢħ +åı¯ æł¹æį® +ç®± åŃIJ +饺 åŃIJ +å¿Ļ çĿĢ +æĿ¥ ä¸įåıĬ +缸 ä¼ł +åĽ½ ç½ij +èħ¹ æ³» +è¿ĻéĩĮ æľī +é£İ æĻ¯åĮº +åıĤ ä¿Ŀ +æŃ» èĢħ +æĪ´ ä¸Ĭ +æ©Ł æ§ĭ +è¯ķéªĮ åĮº +ä¼ł æİĪ +æµ· è¾¹ +泪 æ°´ +缸åħ³ åĨħ容 +éĥij å·ŀå¸Ĥ +åħij çݰ +两 åij¨ +èĬľ æ¹ĸ +ç͵åŃIJ ä¿¡æģ¯ +红 å¤ĸ +æĹħ游 å±Ģ +å¾Ģå¾Ģ ä¼ļ +è¿ħ çĮĽ +ä¼ł 羣 +æ¸ħ æ¾Ī +å°± è¿ij +微信 群 +ç³»åĪĹ æ´»åĬ¨ +ç»ı常 ä¼ļ +è§Ĥ æµĭ +å¿ĥå¾Ĺ ä½ĵä¼ļ +éĻĪ åĪĹ +åĮĹ æĸĹ +è« ® +è«® è©¢ +è¿ĺæĺ¯ ä¼ļ +æµĭ ç®Ĺ +æĺŁ ç©º +宽 容 +çī©ä¸ļ åħ¬åı¸ +æĪĴ æĮĩ +å¸ħ æ°Ķ +ä¸ĢæŃ¥ æŃ¥ +åħ± 鸣 +åĨ³ ä¸į +æİ¥ 管 +å¦ĩ èģĶ +æ¯Ķ åĸ» +é²ģ è¿ħ +æĮģ çºĮ +缸 亲 +å¨ģå°¼æĸ¯ 人 +ç«ĭ 项 +åĪ Ŀå§ĭ +èĩª åζ +è¿Ī è¿Ľ +ä¸Ĭ æ±½ +å®ı ä¼Ł +æł¹æľ¬ 没æľī +æĸ°åĨł çĹħæ¯Ĵ +åĵª ç§į +康 åħ» +è¡° èĢģ +å½ķ åĥı +é«Ķ é©Ĺ +ç»ij å®ļ +é¢Ŀ 头 +äºĶ æľĪ +èĬ± å¼Ģ +ä¸Ģ线 åŁİå¸Ĥ +åΰ åľº +æĬķ éĻį +çĹĺ çĹĺ +åıĹ ä¸įäºĨ +æīİ æł¹ +æĽ´ ä½ķåĨµ +æĬ½ æŁ¥ +åĩº è·¯ +审议 éĢļè¿ĩ +ä¸į åĥħ +èī² è°ĥ +çϾ ä½Ļ +èĤł éģĵ +æ·±åİļ çļĦ +马 åĬĽ +æĹ© æĻļ +æŃĮ èĪŀ +éĺ² æĻĴ +æľĢåIJİ ä¸Ģ个 +樱 èĬ± +å°ıä¼Ļ åŃIJ +åľ¨ å½ĵåľ° +å°ıä¼Ļä¼´ 们 +èµ· æºIJ +åħ¨ åªĴä½ĵ +ç° ½ +éħ± æ²¹ +æĹłè®º å¦Ĥä½ķ +裤 åŃIJ +åģľ äº§ +ä¸įçͱ å¾Ĺ +çīµ å¼ķ +ä¼ł åĬ¨ +ä¹Ŀ é¾Ļ +åĬł åĽº +ä¹Łä¸į æķ¢ +æĬĢæľ¯ æĶ¯æĮģ +ä¸Ĭ å²Ĺ +ç»ıéªĮ åĴĮ +æł¼ æŀĹ +åIJ¸ éĻĦ +æľªæĪIJ å¹´ +奢ä¾Ī åĵģ +追 æį§ +好 ä¸į容æĺĵ +èķ´ åIJ« +ä¿Ŀ å®ļ +æĬ¥ ä¸ļ +æµ· åĨħå¤ĸ +ä½ł çİ°åľ¨ +æ²¹ èĢĹ +è´¨éĩı 管çIJĨ +æ½ľ æ°´ +丽 æ±Ł +转 åħ¥ +è¿Ļä¹Ī ä¹ħ +æĺİ ä»£ +责任 åζ +éĩį å·¥ +大 å·´ +触 åıĬ +èµ· åĪĿ +大 å¦Ī +æĸ¯ å¡Ķ +åĨĽ å·¥ +书 éĻ¢ +å³ ¨ +æİ¨ çIJĨ +è¿Ļç¯ĩ æĸĩ竳 +è¿ģ ç§» +åľ¨ åIJĮä¸Ģ +ç»Ĩ ç»Ĩ +åīĬ å¼± +书 æĪ¿ +ç¶ĵ 常 +è¯ķ é¢ĺ +æĤ£ ä¸Ĭ +çĻ«çĹ« çĹħ +åĨ² æ´Ĺ +å¤ĸ æı´ +åħĭ åζ +åįģ æľĪ +åģļ ä¸įåΰ +ç¾İ åĮĸ +å¦Ĥ æľŁ +è¿ĺ éľĢ +天 åºľ +å°± æĦıåij³çĿĢ +çļĦç¡® æĺ¯ +éªĹ å±Ģ +å°ıç»Ħ èµĽ +è© © +ä¹Ŀ å¹´ +æĻĵ å¾Ĺ +çłĶç©¶ 人åijĺ +大 éħĴåºĹ +ç§ij åѸ +åħŃ åIJĪ +çķĮ å®ļ +车 è½½ +å¼Ģ çĿĢ +毫 æĹłçĸij +毫æĹłçĸij éĹ® +è¿IJ ç»´ +ç¦ģ åĮº +èĦ± èIJ½ +讲 å¸Ī +产ä¸ļ åŁºåľ° +é«ĺ æĢ§èĥ½ +åħī 彩 +çݰ éĺ¶æ®µ +åĩ ¿ +è¾ĥ å·® +饮 çĶ¨æ°´ +éĸĭ çϼ +ç½ij åIJ§ +çĮ´ åŃIJ +æŃ¦ æŀĹ +å®ī åİ¿ +ä¸įåı¯ æĢĿ +ä¸įåı¯æĢĿ è®® +éĬ· åĶ® +è´« ç©· +为 åķ¥ +éº ĵ +å¹¾ åĢĭ +è§Ħ模 以ä¸Ĭ +æı ļ +被 åĽ° +缺 å¸Ń +å¿« é¤IJ +æĬ¢ åįł +æĻ Ł +å¤į æ´» +æľ¬æĬ¥ 讯 +åĪĽ ä¸ĭ +æµ· 滩 +éĩı 产 +å¦Ĥä½ķ åİ» +车 ä½į +å¯ ĩ +äºĮ åįģåĽĽ +ç»ıæµİ æįŁå¤± +éħįå¥Ĺ 设æĸ½ +åŁºæľ¬ éĿ¢ +äºī 论 +就好 åĥı +çłĶç©¶ æĪIJæŀľ +éĻĪ è¿° +æīĵ åĬ¨ +ä¸ĭ å·´ +ç§Ĵ éĴŁ +对 人ä½ĵ +æĬĢæľ¯ çłĶåıij +åİŁ åŃIJ +æĺ¯ä¸Ģ 项 +äºĨä¸Ģ 份 +æĮĩ çͲ +ç͍ éĩı +è¿ĺä¸į å¤Ł +æĶ¿åºľ éĩĩè´Ń +çŁ¥è¯Ĩ çĤ¹ +ä¸ŃåĽ½ 梦 +å¾Ī å¼Ģå¿ĥ +礼 è²Į +éĿŀ常 å¤ļ +éĿŀ常å¤ļ çļĦ +åĽ ļ +æĹħ é¦Ĩ +å°½ æĥħ +æŃĮ åͱ +æ²Ļ é¾Ļ +车 åİ¢ +客 æµģ +åģı å·® +积累 äºĨ +æ¡ Ķ +çĶ» çĶ» +ä¹Ł åºĶ该 +åºĶç͍ ç¨ĭåºı +èĥĥ èĤł +以 å¾Į +豪 å®ħ +æ·± åĬłå·¥ +缴 è¨Ģ +åĮĸ çŁ³ +åĽ½ éģĵ +ä¸ĥ 个 +ä»İèĢĮ 使 +èĤł èĥĥ +æĹ¥ è¶ĭ +çζ åŃIJ +ç· © +æĭĽ çīĮ +产 å¦ĩ +çķª èĮĦ +æĪij éĻ¢ +建çŃij å·¥ç¨ĭ +å±ķè§Ī ä¼ļ +å®¶éķ¿ ä»¬ +åĨľ ä½ľçī© +æĹ¥ å¤ľ +æĶ» æĵĬ +è§Ħ éģ¿ +èĪŁ å±± +便 æ°ij +åħ« åŃĹ +ä¸į æĽ¾ +æĶ¯ éħį +çĨ¬ å¤ľ +人 é¡ŀ +ç´Ģ éĮĦ +ç»ıèIJ¥ æ´»åĬ¨ +大 涨 +å¸Ĥå§Ķ 常å§Ķ +åĪĨ éIJĺ +ä¸Ģ个 èģĮä¸ļ +çĹħ åĽł +è¿Ļ 对äºİ +ä¸įå¾Ĺä¸į 说 +åıijç͵ æľº +æľīæīĢ å¸®åĬ© +缮æłĩ ä»»åĬ¡ +åĽł åľ° +åĽłåľ° åζ +åĽłåľ°åζ å®ľ +å°Ĩ è¾¾åΰ +ç²Ĺ ç³Ļ +稳 åĽº +å« £ +çİ°åľ¨ å¾Īå¤ļ +ä¸ĸçķĮ 级 +å¼ł æŁIJ +çĤ¹ ç¼Ģ +èij µ +社ä¼ļ ç»Ħç»ĩ +å¾Ģ åIJİ +åĬł æģ¯ +åĻª 声 +æľī åħ´è¶£ +为æĤ¨ æıIJä¾Ľ +æ²¹ æ¼Ĩ +ç¬¬åĽĽ å±Ĭ +çļĩ 宫 +ä¹Ĵ ä¹ĵ +ä¹Ĵä¹ĵ çIJĥ +éļ¨ èijĹ +éģ© åIJĪ +åįĹ éĿŀ +æĵ ´ +西 æ´ĭ +åĬł å¯Ĩ +æĪIJåĬ٠䏾åĬŀ +åı£ æ°´ +æĪIJ 年人 +æīĢ æıIJä¾ĽçļĦ +éļĶ å£ģ +åľ¨ 京 +å½ĵåľ° æĹ¶éĹ´ +çŃī åIJĦç§į +é£İ æ°Ķ +å±ĭ éĩĮ +ä¸Ģ åŃĹ +çļĦæĹ¶éĹ´ éĩĮ +åĺ¿ åĺ¿ +å¿« 讯 +ä¸Ń åľº +ä¸Ģ çĵ¶ +æ» ķ +é¢Ĩ è·ij +好 èݱ +好èݱ åĿŀ +没 åħ³ç³» +åĩº å¢ĥ +ä¸įæĺ¯ ä¸Ģ个 +éĥ½æĺ¯ éĿŀ常 +éľĩ åĬ¨ +èİ· èĥľ +åįļ å¼Ī +æĬļ åħ» +对 ç«ĭ +æľįåĬ¡ æľºæŀĦ +è°£ è¨Ģ +社ä¼ļ ç§ijåѦ +åIJ¬è¯´ è¿ĩ +æī ³ +æīĵ 磨 +åı£ æľį +好 åĥıæĺ¯ +以åıĬ åħ¶ä»ĸ +çī¹ è´¨ +亲 è¿ij +ä¸Ģ ç»ı +æ¶ Ŀ +éŃĶ æľ¯ +éģĵè·¯ 交éĢļ +è§Ħ模 æľĢ大 +å®ŀæĸ½ æĦıè§ģ +ä¹ ŀ +ä¸Ģ ä¸ĸ +åŁ· è¡Į +è±Ĩ çĵ£ +åĪĹ ä¸º +æķħ 宫 +çĶŁ åij½åij¨æľŁ +ä¸īç§į èģĮä¸ļ +详ç»Ĩ ä»ĭç»į +å®Į å¤ĩ +岩 çŁ³ +éļı æīĭ +é£ ² +æķĪæŀľ åĽ¾ +ç§ĭ åĨ¬ +åĬŁ å¾· +è§Ħ竳 åĪ¶åº¦ +æĹ¥ æ¸IJ +æīĢ éľĢè¦ģ +æīĢéľĢè¦ģ çļĦ +å²Ľ ä¸Ĭ +åĩº åľŁ +åĽ¾ æĸĩ +ç§ijæĬĢ è¿ĽæŃ¥ +éĢļ èĥĢ +èĢģ 太太 +èĭĹ æľ¨ +éĵ¶ å·Ŀ +å¸IJ 篷 +éĿŀ è¦ģ +éħį ç͵ +å¤Ħ å¢ĥ +èĤ¡æĿĥ æĬķèµĦ +ä¸Ģ缴 åΰ +åĿĩ çͱ +æĬĹ æĹ¥ +æį® ä»ĭç»į +ä½ł åĸľæ¬¢ +åĪĽæĸ° åŀĭ +åıĺ è¿ģ +è§Ĩ å¯Ł +å®Įåħ¨ 没æľī +åħĥ æĹ¦ +åı¯ ä¿¡ +åı¦ è¡Į +æĿij 级 +åħ¥ åľº +æIJŃ æ¡£ +ä¹Ł åĽłæŃ¤ +æį¢ æĪIJ +ä¸į è´Ł +äºĨ 大éĩıçļĦ +éģĶ åΰ +å¸Ĥ åİ¿ +å¹´ è¼ķ +å¿« æīĭ +å¸Į å°Ķ +èĩª èIJ¥ +éĽª èĬ± +æIJ ģ +çľ¼ ç§ij +æŃ£ 確 +çļĦ å§¿æĢģ +åĿļå®ŀ çļĦ +æĮĩ 纹 +æªĶ æ¡Ī +ç½® äºİ +佩 æľį +豪 éŨ +åĵ Ĵ +æģ° 好 +檢 æŁ¥ +åĪĿ è¡· +大 åĶIJ +约 ä¼ļ +èĴ¸ åıij +çѹ åĪĴ +å¹´ ç»Ī +è¡Į æ¥Ń +åħ± éĿĴ +åħ±éĿĴ åĽ¢ +ä¼ļ å¼ķèµ· +ä¸Ń ç§ij +ä¸Ńç§ij éĻ¢ +æĮ¯ åĬ¨ +åį´ åıijçݰ +ä¸įåĬ¨ 产 +èĮ ¹ +æĪ¿éĹ´ éĩĮ +è´§å¸ģ æĶ¿çŃĸ +æ²» çĻĤ +æħİ éĩį +å¡ŀ å°Ķ +åĽ½ ç±į +åĽł æŀľ +çŃī çī¹çĤ¹ +å±± è°· +ä¸ĭ è¼ī +è®ĵ æĪij +饮 éħĴ +è¿Ļ个 游æĪı +ç»Ŀ 大éĥ¨åĪĨ +åĴ¨è¯¢ æľįåĬ¡ +å¹² æ´» +è®® ä¼ļ +æ¦Ĥ è¿° +åĪĨ åĮº +æŃ» åIJİ +ç«Ļ çĿĢ +主è¦ģ é¢Ĩ导 +åIJĮ åŁİ +大 æłij +对 åѦçĶŁ +社ä¼ļ ä¿ĿéĻ© +å¢ŀ èµĦ +主人 åħ¬ +å®£ä¼ł æķĻèĤ² +æĸĩåĮĸ 交æµģ +客 æĪ¶ +çŁ¥åIJį åĵģçīĮ +æ»ŀ åIJİ +äºĴ è¡¥ +æĦŁ äºº +åī ¿ +åIJİ ä»£ +äºī 龸 +æķĻèĤ² åŁ¹è®Ń +éĿĻ èĦī +ä¹ı åĬĽ +说 åĩºæĿ¥ +çİĭèĢħ èį£èĢĢ +åĢ « +åįĩ èµ· +éķ ģ +åĩº 游 +éĢļè¡Į è¯ģ +å·¥ä½ľ å²Ĺä½į +åĮł å¿ĥ +æĭ¿ æĿ¥ +æ´Ĺè¡£ æľº +æĪijä¸į æĥ³ +é¢Ħ è§ģ +æ¼Ķ 示 +ä¸Ģ缴 没æľī +è·Ł 她 +对çħ§ æ£ĢæŁ¥ +ç° ¿ +ä¸ĵ å¿ĥ +è®® äºĭ +åīį 端 +åį¡ å°Ķ +è¨Ń å®ļ +设置 äºĨ +å©ļ 纱 +åľ¨ åĽ½å¤ĸ +åı³ ä¾§ +è³¼ çī© +å¥ĩ èij© +å¢ŀåĬł å̼ +好 è¿IJ +åĽ½éĻħ æľºåľº +ä¸ĭ ç§° +缮åīį 为æŃ¢ +ç¥ŀ ä»Ļ +å®ĥ åı¯ä»¥ +æ¾Ħ æ¸ħ +èĥ½ 使 +游 åĩ» +游åĩ» éĺŁ +åĩ ¹ +ä¸įè¦ģ åĨį +åĨ³ èĥľ +åĨ³ æĪĺ +æĭ ½ +缼 åħ¸ +å¾Ī好 åľ° +æľĢ ç¾İçļĦ +åĥ ļ +å·´ åŁº +å·´åŁº æĸ¯åĿ¦ +æľĢ éĢĤåIJĪ +é«ĺ èģĮ +ä¿Ŀ å§Ĩ +æİĪ æ¬Ĭ +说åΰ è¿ĻéĩĮ +æİ¨ å¼Ģ +çİĩ è¾¾ +ä¸īåĪĨ ä¹ĭä¸Ģ +管çIJĨ ä¸Ńå¿ĥ +交 æ±ĩ +森æŀĹ åħ¬åĽŃ +å¾Ģ ä¸Ĭ +éªij è¡Į +æį® æŃ¤ +纽 带 +ç» ŀ +ä¸ī æĸ¹ +æĦıä¹ī ä¸ĬçļĦ +æİ¨ è¿Ł +å¤ļæł· æĢ§ +æĥ³ èµ·äºĨ +æİĴåIJį 第 +å·¨ é¢Ŀ +æĿŁ ç¼ļ +å®ī å®ļ +äºĭ 實 +çļĦ æĦ¿æľĽ +è£ħå¤ĩ åζéĢł +人 å±ħ +人å±ħ çݯå¢ĥ +å¿ĺè®° äºĨ +该 游æĪı +楼 ä¸Ĭ +å¼Ģ ä¼ļ +æģ ³ +åıĭæĥħ éĵ¾æİ¥ +ç¡ Ĵ +ç»ĻäºĪ äºĨ +åģı 好 +åĵ ī +交éĢļ å®īåħ¨ +éĽ Į +æ²» çĹħ +è§īå¾Ĺ å¾Ī +衬 è¡« +å¿ĥ æĦ¿ +æ´ŀ å¯Ł +æ°ij æ£Ģå¯ŁéĻ¢ +æıIJ çĤ¼ +è¦ģ è¿Ľä¸ĢæŃ¥ +驾 车 +æĻ® æĥł +æķ ĸ +ç¦ı éŁ³ +éĢģ è¾¾ +è§ĦåĪĴ 设计 +æīĭ å¥Ĺ +å®ī ä¿Ŀ +è¿ĺä¸į å¦Ĥ +åīį è¿° +æłĩ è®° +ç´§ æİ¥çĿĢ +æ§ IJ +深深 åľ° +满满 çļĦ +æĺ¥ è¿IJ +æĹ¥ 产 +çα æĬ¤ +åħ¨ æĹ¥ +åħ¨æĹ¥ åζ +转 åĬ¨ +ç¥Ń ç¥Ģ +ä¹° ä¸ľè¥¿ +对 æľªæĿ¥ +æ¶Ī失 äºĨ +åļ´ éĩį +ä¸ī æĿ¡ +éħ¸ 奶 +éĽĨåĽ¢ èĤ¡ä»½ +西 è·¯ +åıª å¾Ĺ +éĢģ åİ» +çĭł æĬĵ +åĪ©ç͍ çİĩ +ä¸ĭ åij¨ +å¥ĭ æĪĺ +æĺ¥èĬĤ æľŁéĹ´ +è´Ł 责任 +æĺĤ è´µ +å°¾ å·´ +ç¯ĩ æĸĩ竳 +åħ ® +è®Ĭ æĪIJ +å¹ ¹ +çĻ» éĮĦ +ä½ Ī +å·¥ åĮł +åĵªæĢķ æĺ¯ +åıį åĵį +ç§ ĥ +åĩº 轨 +æĹ¥ åĨĽ +åIJį èªī +æķı éĶIJ +æľįåĬ¡ æ°´å¹³ +çħ§ å°Ħ +ä¼Ĭ æĭī +ä¼Ĭæĭī åħĭ +åĨħ éĺģ +èĬĴ æŀľ +ä¸ĩ åĪĨ +éĢĢ æ¬¾ +缴æĴŃ éĹ´ +æĭ¿ åΰäºĨ +å°İ èĩ´ +空æ°Ķ ä¸Ń +客æĪ· æľįåĬ¡ +è¿IJ åĬ¿ +ç»ĵ çŁ³ +ä¸į å¿ħè¦ģçļĦ +èĥ¶ åĽĬ +çIJĨ ä¼ļ +æĬ½ åĩº +空æ°Ķ è´¨éĩı +æ¯ķ 竣æĺ¯ +åĨ· æ¼ł +ä¸Ģ å¦Ĥ +ä¸Ģå¦Ĥ æĹ¢ +ä¸Ģå¦ĤæĹ¢ å¾Ģ +æĤ£ çĹħ +åĬł æĮģ +èµŀ åĬ© +é« ® +åij½ ä¸Ń +æĦıä¹ī ä¸Ĭ +ä¸į èĪį +åģļ æ¢¦ +æīĵ æī« +æĺŁ åħī +æĸŃ è£Ĥ +åħ¨ å¥Ĺ +è£ģ å®ļ +马 åħĭæĢĿ +骨 骼 +ä¸Ģ è·¯ä¸Ĭ +å®ļ æĹ¶ +å·¥ç¨ĭ æĬĢæľ¯ +å½¼ å¾Ĺ +æ±² åıĸ +ä¸Ģ è§Ī +åIJµ æŀ¶ +ä¿Ĺ ç§° +æłª æ´² +åºŁ æĹ§ +è¡Į æĺŁ +åıijçĶŁ åıĺåĮĸ +é¦ĸ ä»ĺ +åįģåĪĨ éĩįè¦ģ +æĬĬ è¿ĻäºĽ +ç¥ŀ å·ŀ +æıIJä¾Ľ åķĨ +æ¥ · +å± İ +çĬ¶ åħĥ +åŁİ å¢Ļ +çľĭ ä¸Ģçľĭ +çĶŁäº§ èĥ½åĬĽ +åŁºæľ¬ä¸Ĭ éĥ½ +æīĵ æī° +åĪĿ 次 +åĩº 示 +åħ¶ä¸Ń ä¸Ģ个 +çĶŁæĢģ ç³»ç»Ł +æīĭ æİĮ +æµİåįĹ å¸Ĥ +åľĭ åħ§ +æŃ£ å̼ +å¹¾ ä¹İ +æİ¨èįIJ éĺħ读 +è¿Ń 代 +è°ĥ ä¾ĥ +饮 åĵģ +å¢Ļ ä½ĵ +åıĺ çݰ +äºĨ 好 +äºĨ好 åĩł +ä¸į çķĻ +çĪ ² +å°½ æĹ© +æŃ£åľ¨ è¿Ľè¡Į +åĩº éĻ¢ +æĿĢ å®³ +æıIJ 款 +åıijå±ķ 空éĹ´ +åīį 身 +ä¸įæĸŃ å¢ŀ强 +æ·± å±Ĥ次 +容 纳 +éĤ£ 份 +å·¥ä½ľ æķĪçİĩ +æľ¬ åĽ½ +失 èIJ½ +æŃ£ åĽłä¸º +èĬĤ æ°´ +ä¸ĭ ä¸Ģ代 +çłĶåıij ä¸Ńå¿ĥ +ä¸į çIJĨ +å®Į 好 +ä¿ĿæĬ¤ åĮº +ç»ĵæŀĦ è°ĥæķ´ +å¥ł å®ļ +宣 ç§° +éĺ» æĮ¡ +æĴ¤ 离 +ä¸į æĸ¹ä¾¿ +åĴ ķ +ç¬ijäºĨ ç¬ij +çݯå¢ĥ 污æŁĵ +ä½ı æĪ· +ç»Ŀ ç¼ĺ +éϤ å°ĺ +é«ĺ å°ļ +æĢİä¹Ī åı¯èĥ½ +éĿ¢ èī² +åķĨ æ¥Ń +çĸ ¹ +èµĦæºIJ ä¼ĺåĬ¿ +è¾ĸåĮº åĨħ +èĢĢ çľ¼ +æij§ æ¯ģ +ä¸ĸçķĮ ç»ıæµİ +å¼ķ æĿ¥ +ä¸Ģ åĪĻ +æĭĩ æĮĩ +æĬµ 御 +éĽ į +åĩĨå¤ĩ å·¥ä½ľ +çıł ä¸īè§Ĵ +ç¨Ģ åľŁ +èİ·å¾Ĺ æĦŁ +æĪIJåĬŁ çİĩ +ç½ij 约 +ç½ij约 车 +èĦ IJ +æķ¬ ä¸ļ +éĩij ä»· +ç²¾ é«ĵ +ä¹° 车 +åħ³ åı£ +åĨį å¤ļ +æŀģ åĵģ +åIJĦ å®¶ +举æĬ¥ ç͵è¯Ŀ +èļ Ĭ +æĸ¹ å½¢ +ç§ijæĬĢ æĪIJæŀľ +æľĢ好 æĺ¯ +éĹ® åĢĻ +红 éħĴ +åĽĽ ç§į +ç¿Ĵ æħ +ç¿Ĵæħ £ +åŀ ¦ +éĤ£ åıª +é¢Ĩ æĤŁ +çľ¼ éĥ¨ +æ³° å®ī +ä»» æľŁ +磨 æįŁ +æĽ¿ æį¢ +åħ¸ 礼 +符åIJĪ æĿ¡ä»¶ +è¿ĺæľī ä»Ģä¹Ī +åħ±äº« åįķ车 +åı¯ åĪĨ为 +åŃ£ åIJİ +åŃ£åIJİ èµĽ +举èİŀ å¸Ĥ +å¿ĥ æĦı +æīŃ æĽ² +ä½ľä¸º ä¸Ģç§į +è¿Ļ éĥ¨åĪĨ +åıĤä¸İ åΰ +ç½ij çIJĥ +實 çı¾ +ç»Ħ è£ħ +åIJij å¤ĸ +å·¥ä½ľ æĸ¹æ¡Ī +åįģ æĿ¡ +課 ç¨ĭ +颤 æĬĸ +åĵ © +éĤ® å¯Ħ +äº ¢ +åħį è²» +ç§ ¤ +åºĶæĢ¥ 管çIJĨ +åĽĽ äºĶ +éºĴ éºŁ +å¾Ĵ æŃ¥ +è¨ĺ å¾Ĺ +çĴ IJ +æĺ¯åIJ¦ ä¼ļ +æĦıè§ģ åıįé¦Ī +éļ¾ æĢª +çª į +交 æİ¥ +两 åįĥ +æĩī ç͍ +æľŁ éĸĵ +æIJ¬ åΰ +è®® é¢ĺ +碧 æ¡Ĥ +碧æ¡Ĥ åĽŃ +åģļ çĶŁæĦı +éĻĽ ä¸ĭ +è· ĭ +èĢģ人 å®¶ +带 åĽŀ +æŀ¸ æĿŀ +è¡Į éķ¿ +åĨħ容 ç®Ģä»ĭ +æ¢ ¢ +æĮĩ æİ§ +éĩį çĹĩ +ç½ijåıĭ 们 +çı¾ 代 +ç±» 产åĵģ +å¥Ķ æ³¢ +æ¸ º +ç²ī ç¢İ +è¿Ļ åıªæĺ¯ +æ£Ģå¯Ł æľºåħ³ +é½ Ĭ +æĪ¿ ç§Ł +å¾· æĭī +å²ģ 以ä¸Ĭ +纯 åĩĢ +åĪĨå¸ĥ åľ¨ +èĥ½ å¾Ĺåΰ +ä¸į å°½ +ç«ŀ ä»· +çļĦ 带é¢Ĩ +çļĦ带é¢Ĩ ä¸ĭ +ä¸ŃèᝠæĿIJ +æĿij éķĩ +ä¸įåı¯ éģ¿åħį +éľ² 天 +å°ı å§ijå¨ĺ +çī© ä»¶ +èijĹä½ľ æĿĥ +æĭĺ çķĻ +éĥ½ è§īå¾Ĺ +æĽ² æĬĺ +æ·»åĬł åīĤ +åı¬ åĽŀ +æīİå®ŀ æİ¨è¿Ľ +æĬĦ è¢Ń +åĮĸ 身 +缴 èIJ¥ +ä¹Ł å¸ĮæľĽ +èį£èªī ç§°åı· +åįĸ ç»Ļ +æľī ä¸įåIJĮçļĦ +å¥ĩ çī¹ +éĥ½ 认为 +å¦ ŀ +æĪIJéķ¿ ä¸º +辩 æĬ¤ +主 æķĻç»ĥ +æ³ķå¸Ī èģĮä¸ļ +æ¤į åħ¥ +ç´¢ å°¼ +åIJ¬ è¿ĩ +ä¹łæĥ¯ äºĨ +夺 åıĸ +éŁ ĵ +æľ¬è´¨ ä¸Ĭ +æİ¥ åĬĽ +äºij 端 +è¦ģ åģļ好 +è·¯ çģ¯ +åįıåIJĮ åıijå±ķ +æľī å¾ħ +æ°´ åŁŁ +æIJľçĭIJ é¦ĸ页 +è´¨éĩı å®īåħ¨ +åįģäºĮ äºĶ +åĵ® åĸĺ +èĵ¬åĭĥ åıijå±ķ +åIJį 声 +身 亡 +çİĭ åºľ +åİŁåĪĻ ä¸Ĭ +çĥĺ å¹² +éģĹ æ¼ı +éĿ¢ 缮 +åĽ½ ä¼ļ +ä¸Ģ缴 éĥ½æĺ¯ +æľīä¸Ģ ä½į +éħį æľī +éĻª çĿĢ +ä¼ģ åĽ¾ +æĮī ä¸ĭ +èĵĿ åĽ¾ +æ© ĺ +大å¤ļ æĺ¯ +辩 论 +æĹĭ å¾ĭ +æĬ¥ éĢģ +æĿ¡ è§Ħå®ļ +åĬ¨ éĿĻ +åĮΠ奴 +æĭľ 访 +ä¸Ģ åĪĢ +ä»ĸ çŁ¥éģĵ +主 æĿĥ +ä»ĸ æĽ¾ +æĴŃ ç§į +å£ģ åŀĴ +çī¢è®° 使åij½ +åľ¨è¿Ļ æĸ¹éĿ¢ +æīĭ èħķ +æĶ¯ æŀ¶ +ä¾Ĩ èĩª +éĩį å¡ij +å¤ļ å±Ĥ次 +ä»ĭ è´¨ +éĿ¢ åŃĶ +æ½® 湿 +åİ¿ åŁŁ +游æĪı å½ĵä¸Ń +å£ ŀ +åĪĹ åĩº +èµĽ åĮº +å¤ļ åįĬ +éĩįçĤ¹ å·¥ä½ľ +æĪij们 å¿ħé¡» +æŁı æŀĹ +é²ģ èĥ½ +æĸ½ å±ķ +åIJĦ åĮº +åħį ç¨İ +èµĽ åIJİ +æľĢ éĩįè¦ģ +ä¸Ģ个 好çļĦ +è¿Ŀæ³ķ è¿Ŀè§Ħ +äºĨè§£ æĽ´å¤ļ +æķ¬ 请 +ç¬ijçĿĢ è¯´ +ä¸įæĸŃ åıijå±ķ +æijĦå½± å¸Ī +以 éĺ² +çĤ¸ å¼¹ +声 åĵį +ç¤ ģ +æĩ ¿ +èĪĨ æĥħ +èĩªçͱ è´¸æĺĵ +æķı æį· +ä¸ī大 éĺ¶æ®µ +èĭ Ķ +æĹº åŃ£ +ä¸į 满æĦı +微信 åı· +ä¿® 为 +çł´ è£Ĥ +éĢĥ 离 +æ¯ı èĤ¡ +è¾¾ ä¸įåΰ +æ¯ıå¹´ éĥ½ +çģ¯ ç¬¼ +æŃ¤ åŁºç¡Ģä¸Ĭ +åĥı 个 +åĪĨ 娩 +æĻ ¾ +ä¸į èĩ³äºİ +红 线 +误 è§£ +举 è·¯ +æ·® å®ī +产 åѦ +产åѦ çłĶ +èī¾ æ»ĭ +è»ĭ çĹħ +åīįæıIJ æĺ¯ +æ¯ı ä¸Ģ天 +ä¸ĥ 大 +æłij åı¶ +èµ° å¾Ĺ +è¿Ļ 两ç§į +æİı åĩº +æİ IJ +é¢Ĩ导 èĢħ +ä¸Ģ æľµ +个å¤ļ æľĪ +ä¸Ń åħ³ +ä¸Ńåħ³ æĿij +课åłĤ æķĻåѦ +大 åĴĸ +éģĭ ç͍ +è¯ļ æĦı +ç»Ħ åĽ¾ +è¯ķ çĿĢ +ä¹Ķ æ²» +è¿ĺ ä¸įæĺ¯ +æľī æĽ´å¥½çļĦ +åIJİ å¤ĩ +æĸ°çĶŁ åĦ¿ +æ°Ķ è¡Ģ +æ²¥ éĿĴ +å±ı éļľ +æ¥Ń åĭĻ +æĪij 以为 +éķ¿ çĽ¸ +èĢģ çΏ +éķĩ æ±Ł +æľºæ¢° 设å¤ĩ +ä½Ĩæĺ¯ å¦Ĥæŀľ +åĿļå®ļ ä¸į +åĿļå®ļä¸į ç§» +åĨ² éĶĭ +ç®Ģ缴 æĺ¯ +åĤ¨ èĵĦ +纯 ç͵åĬ¨ +漫 æŃ¥ +举 èµ· +æģ¶ æĢ§ +è¨ĺ éĮĦ +èģĮèĥ½ éĥ¨éŨ +åħ¨ éķ¿ +鼻 è¦ĸ +ä¹³ èħº +ä½ķ å¤Ħ +æ¶Ī æŀģ +æŃ£ å¤Ħäºİ +å®ī å®ģ +æĪIJ éķ· +åıĻ è¿° +æºĥ çĸ¡ +ä½Ĩ çİ°åľ¨ +女 æĺŁ +å©´ å¹¼åĦ¿ +æĬķ èŀįèµĦ +éĹ® éĹ® +æıŃ å¼Ģ +è¯ ı +åIJį å½ķ +èĺij èıĩ +åIJĬ é¡¶ +æ¹ĸ åĮº +åįĸ åľº +建 ç¯ +å»ºç¯ ī +èİ ½ +åIJ¬ åIJ¬ +ç«ŀäºī ä¼ĺåĬ¿ +åĩº ä»» +æľī 两ç§į +橱 æŁľ +è¤ ª +è¯ķ åį· +ç»ıæµİ æĬĢæľ¯ +æ·± å±Ĥ +éĩįè¦ģ åĨħ容 +é£İ æİ§ +çĬ¶æĢģ ä¸ĭ +éĥ¨ éĸĢ +广 æ±½ +è§Ĥ æij© +éģĹ çķĻ +转 è´¦ +æĮģ ä»ĵ +æĢ» 计 +åľĺ éļĬ +æĪ¿ 举 +éĺĢ éŨ +åħ¬ åħ³ +åħ³ åĪĩ +èĤ ĺ +æķ¸ æĵļ +ä¸ī åįģå¹´ +è§ģè¯ģ äºĨ +å± Ĩ +çģ° å°ĺ +æ¦ľ é¦ĸ +è¦ĨçĽĸ çİĩ +ä»Ļ 女 +çĶŁäº§ æĢ» +çĶŁäº§æĢ» å̼ +æĪ¿ è´· +æ±Ł åĮº +åħħç͵ æ¡© +çϾ åIJĪ +確 èªį +转 ç§»åΰ +éĥ½ æĹłæ³ķ +纪念 é¦Ĩ +çŃ¾ç½² äºĨ +å¹¶ä¸į å¤ļ +æĮ ł +ä¸į太 好 +ä¸ĸ 代 +误 导 +é«ĺå³° 论åĿĽ +åħ¼ 容 +龸 æ°Ķ +æĿ¥ 访 +æīĢ å¸¦æĿ¥çļĦ +æĺ¯ä¸Ģ éĥ¨ +æĻļ é¥Ń +åİĨ 代 +åIJ¦ åīĩ +ä¹ħ ä¹ħ +æľīæķĪ æľŁ +诱 åıij +æĢ» èµĦ产 +æľ¬èº« å°±æĺ¯ +çĶŁäº§ åİĤå®¶ +æĹ¶ 髦 +èĢIJ ç͍ +ä»İå°ı å°± +æĿ¡ 约 +èĭ± åĭĩ +ä¿Ĺ è¯Ŀ说 +寺 åºĻ +å¿ĥçIJĨ åģ¥åº· +ä»Ģä¹Ī äºĭæĥħ +æ±ī åŃĹ +çķĻ ä½ı +åįĹ è·¯ +ä¸ī 项 +丢 äºĨ +æĥ³ åΰäºĨ +çѹ éĽĨ +éĻĦåĬł å̼ +西 è£ħ +ä¹ĭ ä½ľ +åģļçļĦ äºĭ +çķ¶ æĤ¨ +çķ¶æĤ¨ åľ¨ +é¦ĸ 款 +ä¸įåľ¨ ä¹İ +å·¥ç¨ĭ æĸ½å·¥ +éļIJ éļIJ +åıĺ 身 +沿 éĢĶ +æĤł æĤł +ä¿Ŀ æļĸ +çĶŁæ´» åŀĥåľ¾ +渤 æµ· +æŃ¦ ä¾ł +女 主è§Ĵ +举 ä¾ĭ +æ ·¨ +çϽ é¢Ĩ +è£Ļ åŃIJ +è¿Ķ è¿ĺ +è¿Ī åĩº +é¾Ļ éŨ +ç»ıæµİ ä½ĵ +æĶ¶ å®ĺ +çķĮ éĻIJ +è·³ åĩº +åįĩ å̼ +绵 éĺ³ +çĸ¤ çĹķ +çľĭ æ¸ħ +æĭĴ çµķ +è¥Ħ éĺ³ +课 å¤ĸ +åŃIJ åŃĻ +æŃĮ è¯į +æĪIJ åIJį +溶 æ¶² +åĦĴ å®¶ +åķĨä¸ļ åĮĸ +辨 åĪ« +å¤ļ è¾¾ +ç½ij åºĹ +ä¹Ŀ 大 +ä¹Ŀ大 ç²¾ç¥ŀ +æŃ¤ 举 +è¿ŀ è½½ +ä¸Ģ åĢĭ人 +èī² æ³½ +æ¶µçĽĸ äºĨ +è¦ı åĬĥ +åĽ½ æĥħ +åį«çĶŁ åģ¥åº· +积æŀģ åĵįåºĶ +æĭ Ļ +åζ åĬ¨ +æĥ³è±¡ åĬĽ +çļĦ ä¹IJè¶£ +å¼łå®¶ çķĮ +å´ İ +éĩį åŀĭ +å¤ĸ å¢Ļ +æĶ¾ åѦ +è®¤çľŁ åŃ¦ä¹ł +è´¬ å̼ +æ³ķ æ¡Ī +æĬ¤èĤ¤ åĵģ +éĻ·åħ¥ äºĨ +请 æĤ¨ +åŀ ¢ +æķĻèĤ² èµĦæºIJ +交æĺĵ å¹³åı° +æĹ¶ è£ħ +ä¼łæŁĵ çĹħ +æ¹ĸ æ³Ĭ +èµĦ 管 +åݨ å¸Ī +éĹľ éį +éĹľéį µ +åĵĪåĵĪ åĵĪ +çĽĹ çªĥ +çĶľ ç¾İ +åºĦ åĽŃ +缮åīį å·²ç»ı +è¾¹ ä¸Ĭ +çģ« èĬ± +æĬ¥ è®°èĢħ +æģĭ æĥħ +ç´§ åĩij +æ°´ æµģ +è¿Ļæĺ¯ æĪij们 +æ³¥ åľŁ +æĽ¾ ä»» +æĸ¹ è¨Ģ +åij¨ åħŃ +åı· 楼 +ä¼ij åģĩ +误 ä¼ļ +åĽ½ åĢº +åīį å¤ķ +两 å¼ł +éĹ « +éŃĶ é¬¼ +æĬĬ æĮģ +èĬĤèĥ½ çݯä¿Ŀ +æ¸ħæ´ģ èĥ½æºIJ +èĤ¥ æĸĻ +é«ĺ é¢ij +å°± æľīäºĨ +交 ä¼ļ +没 éĴ± +éĽħ æĢĿ +è¦ģ åıĬæĹ¶ +åŁ¹åħ» åѦçĶŁ +欣 åĸľ +çĥŃæ°´ åύ +é¾Ļ æ¹ĸ +äºĮ 楼 +æĸ°æµª è´¢ç»ı +æĸ° åĬ¨èĥ½ +èµ£ å·ŀ +æĭ³ 头 +æµģ åIJij +ä¹Łæĺ¯ å¾Ī +åıij åĶ® +ä¸Ń åIJ«æľī +åIJĵ å¾Ĺ +å·¨ æĺŁ +æĹł æīĢè°ĵ +æ¯Ľ åŃĶ +åħ¬åħ± 交éĢļ +çĤİ çĥŃ +èµ· èįī +åĬłçĽŁ åķĨ +说 ä¸įåĩº +大åѦ æ¯ķä¸ļ +å·¥ä¸ļ åĽŃ +éłĺ åŁŁ +åºĨ åħ¸ +æµģ 产 +èģ² éŁ³ +ä¼¼ä¹İ æĺ¯ +è´§ æºIJ +æ·± åĪĩ +æ²»çĸĹ æĸ¹æ³ķ +èµĦæºIJ éħįç½® +ç¶² åıĭ +çĶ £ +äº ¥ +躲 åľ¨ +社 ç§ij +è»Ł é«Ķ +女 è£ħ +æŃ¡ è¿İ +综åIJĪ å®ŀåĬĽ +æł¼ å°ĩ +åħļåı² åŃ¦ä¹ł +æľĢ åŁºæľ¬ +æľĢåŁºæľ¬ çļĦ +çľĭ æľĽ +åıĹ è´¿ +ä¸įä»ħ èĥ½ +ä½ķ å¿ħ +ä¸Ģ个 å°ıæĹ¶ +ç¾ Į +æĭĽ æĶ¶ +çĤĴ èĤ¡ +æĿij å¹²éĥ¨ +缸 çα +æ½ľ èĥ½ +ä¹ į +æĹ¶ è¾° +欣 æħ° +éĵ¶ è¡Įä¸ļ +çĭŃ çªĦ +éĩįçĤ¹ é¢ĨåŁŁ +çݰå®ŀ çĶŁæ´» +éĮ¯ 誤 +æĸ° è§Ħ +滥 ç͍ +æĹ¶ ä¸į +æĹ¶ä¸į æĹ¶ +帳 èĻŁ +ç¨Ģ 缺 +åIJij 举 +ä¿Ŀåģ¥ åĵģ +çıŃ éķ¿ +äºĴ åĭķ +笼 罩 +æ½ Ľ +æļĸ å¿ĥ +è½° çĤ¸ +åºĨ 幸 +è²Į ä¼¼ +æĵ º +èĢIJ 磨 +ä¸ĵä¸ļ 人士 +ä¸Ģèά éĥ½æĺ¯ +æ¼³ å·ŀ +åħ¨ èĩªåĬ¨ +å½ķ ç͍ +大 è·Į +æľīæķĪ æĢ§ +èĩª åĭķ +ä¸ī个 æĸ¹éĿ¢ +港 åĮº +ä¿¡ 貸 +éĢļ è¯Ŀ +é«ĺ 涨 +æ³Ħ æ¼ı +éħį ä¸Ĭ +åħļ å·¥å§Ķ +被 认为 +被认为 æĺ¯ +ä¸įä¼ļ åĨį +è°ĥ åīĤ +åıĤ èĤ¡ +èĦ± åıij +å¿ł å®ŀ +åĨħ åĪĨæ³Į +ç¹ģ å¿Ļ +åıĮ åĪĽ +é©» æĿij +åĪĴ ç®Ĺ +éģİ ä¾Ĩ +åľ£ ç»ı +èıľ 鸣 +æĭ¼ å¤ļå¤ļ +ä¸ŃåĽ½ 汽车 +çĥŁ èįī +缴 æµģ +äºĨä¸Ģ åı£æ°Ķ +ä½İ æĪIJæľ¬ +æī¾ åĽŀ +èĩª åįij +總 æĺ¯ +æĸĩåĮĸ åĪĽæĦı +天 æ²³ +樱 æ¡ĥ +éªij åħµ +éĩĮéĿ¢ æľī +çİ ® +èĥ½ æī¾åΰ +éĢĥ è·ij +åĪĩ å°Ķ +åĪĩå°Ķ 西 +以ä¸ĭ æĺ¯ +å²³ éĺ³ +çļĦ æ¦Ĥçİĩ +æĬµ åζ +å¸Ī äºĭåĬ¡ +å¸ĪäºĭåĬ¡ æīĢ +åĩĨ æĹ¶ +屬 æĸ¼ +订 è´Ń +åįłæį® äºĨ +ä¸Ń éĢĶ +å° ĭ +é»ij 马 +åİ¿ åħ¬å®īå±Ģ +ä¸ĥ æľĪ +èī² ç´ł +å¿ĥèĦı çĹħ +æĹ¶ éĻIJ +æ¯į åħ¬åı¸ +å¹ķ åIJİ +ä¸Ĭ æ¦ľ +å̾åIJij äºİ +纸 ä¸Ĭ +æ¡ ĵ +éĽĨä½ĵ ç»ıæµİ +æĥħ å¢ĥ +è¦ģ åģļåΰ +ç©į 極 +åıª æĢķ +æ¹ĺ 西 +çļ± çº¹ +åħ¨ åľĭ +çĦ¡ è«ĸ +好 æĦŁ +åįķ ä»· +è¿Ľç¨ĭ ä¸Ń +æĺĨ ä»ij +åĪĽ 客 +åħħ æĸ¥ +åħĪ æĬĬ +该 æĢİä¹ĪåĬŀ +åĵģ å¾· +åħ¨éĿ¢ åıijå±ķ +è¨Ī åĬĥ +æĢ» å·¥ä¼ļ +ä½Ľå±± å¸Ĥ +æĬĹ è¡¡ +å¼Ģ åľº +éĴ± å¸ģ +åıĭ 们 +å«ī å¦Ĵ +ç´¢ èµĶ +è®Ĭ åĮĸ +æĮ¤ åİĭ +æĮij è¡ħ +çŃī ä¸Ģæī¹ +æĿ¨ 欢 +ä¸ĵå®¶ åѦèĢħ +èĥ½ è¾¾åΰ +èµ° è¿ij +è´«åĽ° åľ°åĮº +éĻIJ æľŁ +ä¸į 平衡 +åĽ½åĨħ å¸Ĥåľº +èµĽ åľº +éħį èµĦ +è¦ģ èĢĥèĻij +ä¸ĩ åı° +æľĪ æľ« +éĶ ¥ +åŃ « +æİ¥è§¦ åΰ +åĩº 产 +æķĻ åѸ +ä½ľ å¼Ĭ +çļĦ æľĢåIJİä¸Ģ +ä¿ĥ æĪIJ +åIJ¸ åıĸ +æ½ľ èīĩ +被 éªĹ +è¾ĵ äºĨ +çĭIJ çĭ¸ +åįĩ éĻį +è¿ĻäºĽ ä¸ľè¥¿ +æĬķèµĦ åŁºéĩij +çĶŁçī© åѦ +ç½ij绾 èIJ¥éĶĢ +åIJij è®°èĢħ +èįī åľ° +æĢ ¯ +æľįåĬ¡ èĥ½åĬĽ +éĥģ éĹ· +åįķ åĵģ +å¾Ĺ 罪 +æĺĵ äºİ +个å¤ļ å°ıæĹ¶ +éĩį ä»» +ä¸Ĭ å®ĺ +æľ¬ éĩij +çı¾ åł´ +溢 ä»· +æĺŁ è¾° +æ´»åĬ¨ çİ°åľº +丹 麦 +å¸Ŀ çİĭ +æŁ¥ æĺİ +åŃĺåľ¨ äºİ +é¦Ļ æ°´ +æĬ½ æ£Ģ +å®ŀéĻħä¸Ĭ æĺ¯ +æĸ° å¾ģç¨ĭ +è´¢åĬ¡ 管çIJĨ +æİ Ľ +åĨľ åİĨ +éĥ½ èĥ½å¤Ł +éĤ¯ éĥ¸ +羣 實 +ç» Ĭ +åĨµ ä¸Ķ +ç½® 身 +ç¥Ī 祷 +çĿģ å¼Ģ +æĮĩ çĤ¹ +å¼Ģ æľº +西 å®ģ +åĮĹ çº¦ +积 æ°´ +åĩº åĬ¨ +åıijå±ķ 模å¼ı +转 æĬĺ +èĢĥ çĤ¹ +æľī ç½ijåıĭ +è´«åĽ° æĿij +æĪij们 çŁ¥éģĵ +åĪĨ éĶĢ +å±± èĦī +æ¯Ķ æĭŁ +ä¼° ç®Ĺ +æĶ¹ 建 +壮 è§Ĥ +ç§ī æĮģ +æı ª +ç¦ Ģ +åĮĸåѦ åĵģ +ä¸ŃåĽ½ åζéĢł +ä¸Ģ æŀ¶ +æīį è¡Į +æĭĽ å¾ħ +åıĺ æį¢ +åīį 线 +幸 好 +è¿Ļæł· çļĦè¯Ŀ +å¿ĥ è¡Ģ管 +æĢ§ çĸ¾çĹħ +åħ¨ èĥ½ +åĪij 侦 +ä¿¡æģ¯ åıijå¸ĥ +æĺ¾ çĦ¶æĺ¯ +éĿĴ éĵľ +åIJĥ ä»Ģä¹Ī +ç͵ ä»· +æ³ķå¾ĭ è§Ħå®ļ +çħ ² +çĵ· åύ +èĤī ç±» +æıĴ åħ¥ +åĹ ľ +è¿Ł è¿Ł +ä¸ĢçĤ¹ éĥ½ä¸į +è¿ĺ åĮħæĭ¬ +èĪį ä¸įå¾Ĺ +æłĩå¿Ĺ æĢ§ +æľĪ 以æĿ¥ +ç³ĸ æŀľ +éĥ½ åºĶ该 +çݯå¢ĥ åį«çĶŁ +èĪª è¡Į +éĥij éĩį +ç½ij æĬķ +åįģ ä½³ +ç§ģ ä¸ĭ +æļ´ è·Į +åĬłå¿« åıijå±ķ +产åĵģ çłĶåıij +åĪĽéĢł åĩº +æĢ» è§īå¾Ĺ +åºķ çĽĺ +èķ Ĭ +åĩºå¸Ń ä¼ļè®® +主 æĿ¿ +æĹ¥æĻļ éĹ´ +å®ĺæĸ¹ å¾®åįļ +å¼ķç͍ æĹ¥æľŁ +åī¯ æķĻæİĪ +ç͵åŃIJ 产åĵģ +è¡° éĢĢ +çķĻ åŃĺ +çģ« åĬĽ +çĴ § +çļ Ĥ +åħ¼ åħ· +éĩį è¿Ķ +é¢Ĩ çķ¥ +åĪĩ éϤ +åĨįçĶŁ èĥ½æºIJ +å®ŀåľ¨ 太 +çIJĨ论 ä¸Ĭ +ä¸ī å±Ĥ +ä¸ĸçķĮ åIJĦåĽ½ +å®ľ æĺĮ +è̳ è¾¹ +宽 æķŀ +æ±ī æĹı +çϽ çϽ +è¿ĻéĩĮ éĿ¢ +çĶŁæ´» ä¹łæĥ¯ +èµŀ èµı +çĶ· 士 +ä¸Ń ä¿Ħ +车 祸 +åīĤ éĩı +éϤ åİ» +å·¦ è¾¹ +çŃij çī¢ +çīĽ å¸Ĥ +å®¶ åĬ¡ +åķ ĥ +ç½® æį¢ +ç´« å¤ĸ +ç´«å¤ĸ 线 +å¾Ģ åīį +åĬĽ åѦ +ç´§ è·Ł +缮çļĦ åľ¨äºİ +ç» ® +ç¥ Ĥ +宣 è¨Ģ +äºĮ æ°§åĮĸ +äºĮæ°§åĮĸ 碳 +æĹł ç¼ĺ +ç²¾ éĢļ +è¨ º +å¼ķåıij äºĨ +æľĢ åħĪ +æ´¾ é©» +ä¸į å¿į +æĪij çΏ +å¹´ ä¸ĭåįĬå¹´ +æ·ĭ å·´ +没 éĹ®é¢ĺ +åºĹ åĨħ +è·Ł æĪij说 +çĶŁäº§ çĶŁæ´» +è§Ĥ æľĽ +æ¸ į +被 æī§è¡Į +被æī§è¡Į 人 +èĪ ľ +æİ º +ä¸Ģ ç§Ĵ +èįī åĿª +åij¼ åĴĮ +åij¼åĴĮ 浩 +åij¼åĴĮ浩 çī¹ +人æ°ij éĵ¶è¡Į +çĦķ åıij +è¯ģåΏ 交æĺĵ +çķ Ķ +æľº èĥ½ +å¦ ¾ +æĻļ å¹´ +å·¥åķĨ èģĶ +åİŁ åŀĭ +è§Ĵ度 çľĭ +æĬ¥ 社 +è¯į æĿ¡ +躲 éģ¿ +éĩį åIJ¯ +å¤ķ éĺ³ +èĤ¡æĿĥ 转让 +åľ¨ ä¸Ģ +åľ¨ä¸Ģ æĹģ +社ä¼ļ åĮĸ +åıijå±ķ åİĨç¨ĭ +æĭĸ æ¬ł +使 èĢħ +ä¸İ åIJ¦ +æĸ° å±ĢéĿ¢ +ä»Ĭ天 æĪij们 +é½IJ èģļ +对 æĪij说 +éĢĴ 交 +æľª æĽ¾ +èİ Ĭ +éĸ ī +亲 æīĭ +è§Ĵ éĢIJ +æľī é»ŀ +ç¨İ çİĩ +ä½İ 声 +é»ĺ å¥ij +æĻ® æ³ķ +大 ä¸ĵ +第äºĮ 大 +ä½ı åĿĢ +æĶ¾ è¿Ľ +äºĮ æĪĺ +亲 身 +åĽº åĮĸ +ä¸ĭ 乡 +åħ³éĶ® æĬĢæľ¯ +åĽŀ æĥ³ +æĬ¥ åĪĬ +æ¶Ĥ æĬ¹ +èĹı çĿĢ +ç¥Ŀ æĦ¿ +åįĩ 温 +çĶļèĩ³ è¿ŀ +åħ¬åħĥ åīį +ç¾İ æĸ¹ +è¯ļ å®ŀ +æĹł åģ¿ +åīµ æ¥Ń +å°ıå¿ĥ 翼 +å°ıå¿ĥ翼 翼 +两 æīĭ +温馨 æıIJ示 +仿 羣 +æĥ ¶ +èĥ¡ åŃIJ +å·¥ä½ľ ç«Ļ +硬 çĽĺ +ç« ¿ +åĤ³ éĢģ +åħ¨ æł¡ +é²ľ æ´» +çĴĢ çĴ¨ +ç»ĵ å°¾ +æį¢ æĿ¥ +æĪ Ģ +ä½İ ä½į +ä¸ĩåħĥ 以ä¸Ĭ +åĬł åĪĨ +æİ¨ä»ĭ ä¼ļ +çIJĨ èµĶ +å¾· å°Ķ +æĬĹ è®® +æ´ ¼ +åĸ § +åŁİ éĻħ +å¾Ī æ£Ĵ +人 æŃ»äº¡ +ä¼ļå±ķ ä¸Ńå¿ĥ +äºĴèģĶ äºĴéĢļ +èĸĦ èĨľ +éĩį é»ŀ +ç¦ģ æ¯Ĵ +åĨ· ç¬ij +大家 åı¯ä»¥ +é¦ĸ 缸 +è¿ij è·Ŀ离 +æµ® çݰ +ç§ĺ è¯Ģ +èµ· é£ŀ +æIJ ¶ +羣 åģĩ +æģ ķ +å°ı åºĹ +æ°ij çľ¾ +åıijå¸ĥ åħ¬åijĬ +ä¾§ éĩį +å¾ĺ å¾Ĭ +æĢ Ķ +æª IJ +æķ° 缮 +åī¯ ç§ĺ书éķ¿ +两 åı¥ +éļIJ çŀĴ +åıĮ åıĮ +æīĭ æĦŁ +èij¡ 京 +éģĹ å¿ĺ +é¬ ¥ +è¿Ļ个 åľ°æĸ¹ +说 çļĦè¯Ŀ +å·¡ åĽŀ +è¿Ŀ 竳 +æī¾ å·¥ä½ľ +æĶ¯ çIJĥéĺŁ +裡 éĿ¢ +æĺ¾ç¤º åĩº +èĩ³ å°Ĭ +两 级 +åīį æ®µæĹ¶éĹ´ +çĺ¦ èº« +èĤ¢ ä½ĵ +æ¯į 親 +æīĭç»Ń è´¹ +汽车 è¡Įä¸ļ +æİ© çĽĸ +æİ§èĤ¡ éĽĨåĽ¢ +åı£ å¾Ħ +æĶ¿çŃĸ æİªæĸ½ +æµ· 绵 +åħ¨ éķĩ +äºĭ åħ³ +å¸Ń æī§è¡Į +å¸Ńæī§è¡Į å®ĺ +éĤ£ 次 +åı¯èĥ½ åĩºçݰ +ä¸Ńå¿ĥ åŁİå¸Ĥ +ç¿» 身 +ä¹Ł ç®Ĺ +ä¾µ çķ¥ +åĸĩ åıŃ +æ¯ı次 éĥ½ +è§ ħ +éĻ¢ éĻ¢éķ¿ +å§ĭ äºİ +èѦ åĬ¡ +èᝠæĿIJ +å±ł æĿĢ +æľ¬èº« å°± +éļıæĹ¶ éļı +éļıæĹ¶éļı åľ° +åĶ® åįĸ +æĹłäºº 驾驶 +é¢ ħ +åĵģ 質 +åĺ² ç¬ij +è·ij åİ» +åħĭ éĩĮæĸ¯ +çķ¸ å½¢ +ä¿® 饰 +磩 éĺµ +éŁ³ä¹IJ ä¼ļ +æŁ³ å·ŀ +é½ ¡ +ä¼ļ è°Ī +æŃ£ çīĪ +ä¹Ł åIJĮæł· +æļ§ æĺ§ +è¡ĮæĶ¿ éĥ¨éŨ +ä¹ĸ ä¹ĸ +èĤ¤ èī² +æĹ¶ ä»» +羣 åĪĩ +æľĪ ä¸ĭ +æľĪä¸ĭ æĹ¬ +举æĸ¹ è´¢å¯Į +è£ħä¿® åħ¬åı¸ +éĢĢ è¿ĺ +åĭĺ å¯Ł +åĵ¥ 伦 +åĵ¥ä¼¦ æ¯Ķäºļ +çĭ¬ ä¸Ģ +çĭ¬ä¸Ģ æĹł +çĭ¬ä¸ĢæĹł äºĮ +è°ĥ åij³ +åİĭ è¿« +åħ¨çIJĥ æľĢ大 +åī¯ æł¡éķ¿ +æĽ´ ä½İ +åĪĨéĴŁ åIJİ +åĽŀ ä¾Ĩ +åζ åīĤ +åijĬè¯ī 大家 +çĤ¹ éĴŁ +åįģä¸ī å±Ĭ +åij¨ åĽĽ +è¿Ļæł· ä¸Ģ +è¿Ļæł·ä¸Ģ æĿ¥ +èĭ Ł +æľĽ åİ» +æĪIJ è¯Ń +å½ĵ åį³ +ç¬ij 声 +ä¹ĭ åĬ¿ +åĪijäºĭ æ¡Īä»¶ +æĮĤ çĿĢ +ä½ķ ç§į +å°ı 游æĪı +åĽ½å®¶ æĪĺçķ¥ +åĨ· åĨ· +å®ľ 宾 +æIJº ç¨ĭ +è¶ĭ äºİ +åıį çľģ +常 说 +ä¸ĩ æĪ· +åĥµ å°¸ +åįĥä¸ĩ åĪ« +åıijçݰ éĹ®é¢ĺ +åı¯ çŁ¥ +éŨæĪ· ç½ijç«Ļ +åģ¥åº· 产ä¸ļ +åı³ è¾¹ +æµ· è¿IJ +è¿ij ä¹İ +åĮ» æ²» +æĢ» ç®Ĺ +ä¸Ģ åĪĨéĴŁ +æĭ § +ä¹Ł æľīä¸ĢäºĽ +ä¾Ľç͵ åħ¬åı¸ +å»ī ä»· +帮 ä»ĸ +æŃ¤æ¬¡ æ´»åĬ¨ +åıªèĥ½ 说 +èĬ ĭ +çīĩ 段 +åŃĺåľ¨ éĹ®é¢ĺ +ä½łä¼ļ åıijçݰ +è½® å»ĵ +ç½ij éĢļ +滨 æ±Ł +æİĪ ä¿¡ +é»İ æĺİ +ä¸į å±ŀäºİ +约 åįł +éķ¿æ²Ļ å¸Ĥ +èĥļ èĥİ +åħĥ ä»¶ +éĻĨ åĨĽ +è³¼ è²· +æĮĩ æľĽ +å®ŀä¹ł çĶŁ +çī¹çĤ¹ æĺ¯ +çıł æ±Ł +çľĭ ä¸įåĩº +ä¸įè§ģ äºĨ +ç¼ ī +éĺµ èIJ¥ +åĶIJ æľĿ +没 å¿ħè¦ģ +åĽ½åľŁ èµĦæºIJ +ç»ıæµİåѦ å®¶ +åIJĪèĤ¥ å¸Ĥ +çIJ¢ 磨 +ç¡® åĪĩ +åŁİå¸Ĥ åıijå±ķ +çŃ· åŃIJ +人æ°ij æľįåĬ¡ +满 åĪĨ +è¿· ä¿¡ +ä½ľèĢħ æľ¬äºº +æĸĩ竳 æĿ¥æºIJ +ç«Ļ ç«ĭ +æŀĦ æĪIJäºĨ +è¾Ľ åĭ¤ +è¶ħ 强 +éĶ ļ +åīįä¸ī åŃ£åº¦ +å°± è§īå¾Ĺ +å´ĩ é«ĺ +è¶Ĭ ä¾Ĩ +è¶Ĭä¾Ĩ è¶Ĭ +å¸Ĥåľº èIJ¥éĶĢ +综åIJĪ ç´łè´¨ +åŃ ļ +ä¾® è¾± +äºĮ åŃĹ +å·¥ä½ľ ä»»åĬ¡ +åı²ä¸Ĭ æľĢ +æľĢ ä¼ĺ +åIJ© åĴIJ +表 çϽ +èİ« åIJį +èİ«åIJį åħ¶ +èİ«åIJįåħ¶ å¦Ļ +å¹ £ +åIJĮå¿Ĺ 们 +建设 çĶ¨åľ° +åĦ Ģ +éħį åģ¶ +å¼ © +åͱ çīĩ +æīĭ èĦļ +åħ¼ ä»» +åģľ æĶ¾ +æŃ£ å®Ĺ +æĸ° åĨľæĿij +åĤ¬ çĶŁ +æīĢ åŃ¦æł¡ +念 ä½Ľ +åͤ éĨĴ +åħ± åĪĽ +æĭī ä¸ģ +èĥĮ çĿĢ +çĶŁæĢģ ä¿ĿæĬ¤ +åı£ 头 +æĸ¹åIJij çĽĺ +調 æķ´ +æĭĽèģĺ ä¿¡æģ¯ +åħ¶ä»ĸ åĽ½å®¶ +ç®Ģ æĺĵ +åĮ¿ åIJį +è¯Ħ æµĭ +æĺ¯ä¸Ģ 座 +çīµ æīĭ +è¶³ 迹 +çIJĨè§£ åĴĮ +æľĢ åıĹ +å¿ĥ è·³ +çζ 親 +éĿŀ常 åĸľæ¬¢ +èĭ¦ éļ¾ +æĬĢ å¸Ī +æ°ij æĦı +æĪĺ åĽ½ +æĽ¿ è¡¥ +æ´¥ è´´ +ä¸ŃåĽ½ ä¼łç»Ł +åIJĦ è¡Į +åIJĦè¡Į åIJĦ +åIJĦè¡ĮåIJĦ ä¸ļ +第äºĶ å±Ĭ +èį· èĬ± +æĦı èŃĺ +票 ä»· +åĪĨ æµģ +æĿİ çϽ +æ±Ł åĮĹ +æİĴ æĸ¥ +ä½ĵ éĩı +åĮħåIJ« äºĨ +åĪĺ æŁIJ +çݰ å¦Ĥä»Ĭ +å·¥èīº åĵģ +è¿Ļç§į æĸ¹æ³ķ +åĬŀåħ¬ 楼 +ç͵ å·¥ +çħ Ļ +åį¡ çīĩ +å¹´ å¹´åºķ +ä¸ĵ项 èµĦéĩij +åĮ» ç§ij +åĮ»ç§ij 大åѦ +åĽŀ头 çľĭ +ä¸į å±ij +èĩª 驾 +没 æĶ¶ +æīĵ çĮİ +èĦ¸ éĥ¨ +åıĥ èĢĥ +å°Ĩ 士 +è´«åĽ° 人åı£ +çIJĨæĥ³ 信念 +é£İ å°ļ +人æīį éĺŁä¼į +çij ¾ +æĿ¥ è¿ĻéĩĮ +æ´Ĺ 涤 +å¹´ èĸª +èĭį çϽ +ä¸ĩ äºĭ +课 æľ¬ +åºĵ éĩĮ +çī¹ æ´¾ +ç´¾ åijĺ +èµŀ ç¾İ +ç©¿ æĪ´ +製 ä½ľ +èµŀ æĪIJ +ä¸Ģ ä¾§ +å½ĵåľ° 人 +æĭ İ +纸 è´¨ +ä½Ļ 个 +éĶĤ çĶµæ±ł +æľº åŀĭ +éĻ¢ éϢ士 +åģļ å·¥ +å¼ł è´´ +ç¥Ľ æĸij +æ®ĸ æ°ij +å¥ij 约 +æ¹ĺ æ½Ń +æIJ ĸ +åŃĺ è´§ +交éĢļ 大åѦ +è¶ģ çĿĢ +æĸĩçī© ä¿ĿæĬ¤ +å¤ĩ æĪĺ +éĩĩ 纳 +åįĬ æľĪ +æľĢ åħ³éĶ® +æľĢåħ³éĶ® çļĦ +æİ¥ éĢģ +æĶ¶ åī² +åıį åĢĴ +çĥ Ľ +æ ½Ķ +ä¼Łå¤§ å¤įåħ´ +çļĦè¯Ŀ è¯Ń +容 å¿į +å®ļ éĩı +æķ Ĺ +åĵģçīĮ 形象 +æīŃ è½¬ +åĽ½å®¶ éĩįçĤ¹ +èĨĿ çĽĸ +ä¸Ģ 楼 +大 éϏ +éĤª æģ¶ +åĽŀ åij³ +çĮ ¿ +çĿ¡ åīį +æĹł è¾ľ +çĹħæ¯Ĵ æĦŁæŁĵ +æľºæ¢° åĮĸ +çĤ¹ 亮 +溶 è§£ +åĩłä¹İ æīĢæľī +è·ij éģĵ +ç͵è§Ĩ æľº +åı ¨ +æijĩ äºĨ +æijĩäºĨ æijĩ头 +èĩª è´Ł +综åIJĪ åĪ©ç͍ +èĩª å¦Ĥ +åİŁ ä¾Ĩ +ä¹Łä¸į æĥ³ +èĬĤ 课 +è¿ĩ åī© +çͲ çĬ¶ +çͲçĬ¶ èħº +æĸ° ä¸ĸ纪 +èĩªä¸» åĵģçīĮ +é«ĺ å±Ĥ次 +ä¸Ģ è§Ĵ +è¡Į äºĭ +ç¥ĸ åħĪ +å©ļ åIJİ +éĹ´ éļĻ +ç¼Ŀ éļĻ +è¿Ļ æĶ¯ +ä¸įæĸŃ åĪĽæĸ° +å¾® åŀĭ +æĽĻ åħī +享 ç͍ +ä¸ŃåĽ½ ç§»åĬ¨ +éĹŃ çݯ +æī§ æĦı +åıijå±ķ æł¼å±Ģ +æł¸å¿ĥ åĮº +éªļ æī° +åħļåĴĮ åĽ½å®¶ +ä¸ŃåĽ½ æĶ¿åºľ +帶 èijĹ +ä¸ĩåįĥ çĵ¦ +åħ© 人 +äºİæĺ¯ æĪij +åĽº ä½ĵ +çªģ å¦Ĥ +çªģå¦Ĥ åħ¶ +çªģå¦Ĥåħ¶ æĿ¥ +éĩĮç¨ĭ ç¢ij +çα ç¾İ +æŁ¥ éªĮ +åıĮ èµ¢ +éĹª åħī +楼 å®ĩ +æĻ ı +æľī è¶³å¤ŁçļĦ +æŁĶ æĢ§ +ä¿¡æģ¯ å®īåħ¨ +管 线 +å¹¶ ä¸įä¼ļ +åύ ä»¶ +ä½ł åºĶ该 +çĿĢ å®ŀ +æĺİ æ¸ħ +æĬĹ çĶŁç´ł +æīĵ æŃ» +å®Įåħ¨ ä¸įåIJĮ +èĬ± æ¤Ĵ +æĶ¾ 宽 +ä½İ 端 +åĽĽ èĤ¢ +åĮĹ京 èµĽè½¦ +éĽĨ å¸Ĥ +æľª å©ļ +大å¹ħ æıIJåįĩ +建çŃij 设计 +çĭ¬ æľīçļĦ +æİ¢ éĻ© +æ²³æµģ åŁŁ +æħķ 容 +被 çĽĹ +åĵº ä¹³ +èı ģ +æĥ¬ æĦı +è¶ĬæĿ¥è¶Ĭ 好 +广大 群ä¼Ĺ +å¾· èĤ² +å¸Ĥåľº ä»·æł¼ +奥 å·´ +奥巴 马 +èĬĤ缮 ä¸Ń +两 款 +ä¸ĩä½Ļ åħĥ +ç»´ å°Ķ +çĶŁçī© ç§ijæĬĢ +åIJ¬ èµ·æĿ¥ +çł ļ +æĭŁ å®ļ +æ²¹ çͰ +声 èªī +建çŃij ä¸ļ +éĻIJ è´Ń +çīĩ åŃIJ +çķľ ç¦½ +ç½ij é¦ĸ页 +ä¼Ĺ çѹ +æĴŀ åĩ» +åīį ä¸įä¹ħ +åīį ä¸ĸ +åĽĽä¸ª æĦıè¯Ĩ +æµĭ ç»ĺ +éĺ² ç©º +漫éķ¿ çļĦ +æ²IJ æµ´ +æ¯Ķè¾ĥ ç®Ģåįķ +æµĭ å®ļ +åĽŀ è°ĥ +让 人们 +èĴĭ ä»ĭ +èĴĭä»ĭ çŁ³ +ç»ĵ æĻ¶ +å¢ŀæ·» äºĨ +æĿ¡ è¯Ħ论 +åī¯ ä¼ļéķ¿ +ä½ı æīĢ +ç»Ļ åĩºäºĨ +è°ĥ éħį +æ² ĸ +æľī ç͍ +æľīç͍ çļĦ +ä¸ĢæĿ¡ é¾Ļ +éĩİ å¤ĸ +ç¼ĺ åĪĨ +æ°¸è¿ľ ä¸įä¼ļ +æŀľ æłij +大åıij å¿«ä¸ī +麻 éĨī +äºij éĽĨ +åİ» åĵªéĩĮ +åħ¥ å¸Ĥ +ä»» æĢ§ +建 æ¡£ +建档 ç«ĭ +建档ç«ĭ åį¡ +ä¸Ģ 棵 +社 åįĢ +缸 ä¼´ +åļ · +å¡« åħħ +ä¸Ģ æĹı +ç¾ ģ +åıĸ è¯ģ +èΰ éĺŁ +åİĤ åĮº +è¡· å¿ĥ +åıijå±ķ éĺ¶æ®µ +é«ĺ 强度 +åĹĵ åŃIJ +é¢Ĩ è¡Ķ +楼 主 +大 èĴľ +æŀķ 头 +ç²® æ²¹ +é»Ħ çĵľ +æĵ Ĵ +å°ı çĭĹ +æĶ¹éĿ© å§Ķ +åįģ åĪĨéĴŁ +é²ľ èī³ +åħ³ ç¾½ +çĭĢ æħĭ +å®ŀç͍ æĢ§ +å°ij è§ģ +é£ŀ æī¬ +çͰ éĩİ +æIJ Ĥ +è¿Ļ个 è¯į +åºĶæĢ¥ é¢Ħæ¡Ī +è§Ĵ度 æĿ¥çľĭ +æķ¬ çķı +æ³ķ å®Ŀ +åĸĦ æĦı +æīĵ æĸŃ +对 åĨ³ +çµķ å°į +åĢŁ æŃ¤ +å¼Ģ æºIJ +å°ı 說 +ç¥ º +å²ģ 以ä¸ĭ +éĢĢå½¹ åĨĽäºº +ä¸įä¹ħ åīį +åĩº åİĤ +讽 åĪº +æĿ¥çľĭçľĭ åIJ§ +éŃĶ åħ½ +çķĻ ä¸ĭæĿ¥ +å±ħ 室 +åłħ æĮģ +çľĭ äºĨä¸Ģ +çľĭäºĨä¸Ģ çľ¼ +éĽĨåĽ¢ æĹĹä¸ĭ +æĪĺ æĪĺç»ĦåIJĪ +è®¤çľŁ èIJ½å®ŀ +汽车 产ä¸ļ +çī©çIJĨ åѦ +æķ µ +éĴ Ŀ +åĽ¢ éķ¿ +ä¸įæĸŃ æī©å¤§ +èĤ© è´Ł +åıijå±ķ 缮æłĩ +è³ĩ éĩij +åīį ç½® +ä¸ŃåĽ½ åı¤ä»£ +æŃ» åĪij +åħħåĪĨ ä½ĵçݰ +åħ³ éŨ +ç¾İ æĦŁ +æīĵ åħ¥ +æĬijéĥģ çĹĩ +å°ij çĪ· +æłij æŀĿ +æ¶Īæģ¯ ç§° +æ´Ľ åħĭ +åį ¯ +è¿Ī åIJij +æİ¨ åĭķ +ä»İä¸ļ èĢħ +åİ» ä¹° +欢 å¿« +æĭ¥ æĮ¤ +马 æ¡¶ +æĬĬ æİ§ +æĶ¿ åħļ +å¼ł æī¬ +客 æłĪ +红 æĺŁ +éĢģ æĿ¥ +åħ¨åŁŁ æĹħ游 +èĩª ç§ģ +åįģäºĮ æĿ¡ +åı¹ æģ¯ +ä¸Ģ èīĺ +ä¿Ŀ è´¹ +æĸ½å·¥ çİ°åľº +æľī 幸 +ç»Ń èĪª +åı¯èĥ½ æľĥ +èĥĮ åıĽ +ä½£ éĩij +ä¸ī çŃīå¥ĸ +å¾Ī 满æĦı +游æĪı åľ¬ +群 éĩĮ +æŀĦ ä»¶ +åºı å¹ķ +太 æ¹ĸ +æľ¨ è´¨ +æĻĭ æ±Ł +çµĤ æĸ¼ +è·³ è·ĥ +åĢºæĿĥ 人 +çŃī 诸å¤ļ +æĶ¾ åĩº +åħ³éĶ® æĹ¶åĪ» +æĦŁæŁĵ èĢħ +é£ŀè¡Į åijĺ +èĥĨ åĽº +èĥĨåĽº éĨĩ +æĬ± æŃī +åij¨ äºĮ +æĸ° æĹ¶æľŁ +åĨ·éĵ¾ çµģ +è¿Ļç§į æĸ¹å¼ı +该 æĿij +åĽŀ é¦Ī +åŁºçĿ£ æķĻ +人 åıĤ +æŀ¯ çĩ¥ +æī¹åıij å¸Ĥåľº +åħħåĪĨ èĤ¯å®ļ +å¸Ĥ æĶ¿åįı +äºĭ æ¥Ń +龸 çİĭ +çĥŃ æIJľ +åįģä¹Ŀ 大 +ä¼´ æľī +ç¾İåĽ½ æĢ»ç»Ł +åŁİå¸Ĥ 管çIJĨ +ä¸ĭ 令 +èĥ¸ åı£ +åıª çŁ¥éģĵ +åij¨ ä¸ī +ç͍ æĪ¶ +éŃ ¯ +å¿ĥ è¡Ģ +带头 人 +åĮ» åĬ¡ +åĮ»åĬ¡ 人åijĺ +æİ§åζ åύ +ä½ľåĵģ åĨħ容 +æĪĺ åıĭ +åİĨ å¹´ +ä¸į åħĭ +ä¸įåħĭ ä¸įåıĬ +æĹ¥ æŃ£å¼ı +è±IJ å¯Į +ç¨İ è´¹ +æĹ¶ æķĪ +å±ķ ä½į +è¡¡ éĺ³ +æĪ¿ 貸 +çĪĨ 款 +ä¹IJ æĦı +çĶ· 主 +å¯ ¬ +æľĥ èѰ +ä¹ĭ å¤ľ +åIJĮ 樣 +ä¸įè¦ģ 太 +ä¼Ĭ æĸ¯ +ä¼Ĭæĸ¯ åħ° +åŁºæľ¬ åİŁåĪĻ +åİ» æİī +ä½İ ä¿Ŀ +个 交æĺĵ +个交æĺĵ æĹ¥ +èģĬ èģĬ +åĽĽ ä½į +åħļç»Ħ æĪIJåijĺ +主è¦ģ ä»İäºĭ +å½± éŁ³ +åĨĴ åĩº +åij¼åIJ¸ éģĵ +è¾¾ å°Ķ +æľ¨ åľ°æĿ¿ +诡 å¼Ĥ +çģ¯ åħ· +çģ« çĥ§ +è§£ èĦ± +æĦĪ åıij +æ¹ĸ å·ŀ +é£İ ä¿Ĺ +æĸ° å½¢åĬ¿ +æĸ°å½¢åĬ¿ ä¸ĭ +è² Ŀ +èĦ ĵ +åĬ¨åĬĽ çĶµæ±ł +é£ŀ èι +飧 æĢ§ +åĪ© çī© +åĪ©çī© æµ¦ +ä¸į 认è¯Ĩ +ç¼ĸ ç»ĩ +ä½ľ åĿĬ +èģĮä¸ļ æĬĢèĥ½ +çľĭ è¦ĭ +åĽ´ æ£ĭ +æĺı è¿· +å½Ĵ å±ŀäºİ +æĤ¬ å´ĸ +éĨ« çĻĤ +å®ĭ 代 +åºĦ æĿij +èĹ ķ +çĮĽ çĦ¶ +çĩĥæĸĻ çĶµæ±ł +å®ŀä½ĵ åºĹ +ä¸įè¶³ 以 +æĥħ ç· +æĥħç· Ĵ +å»Ĭ åĿĬ +ç͵ åı° +åºĶ åĬĽ +ä¸Ńå°ı åѦçĶŁ +èĥ¡ åIJĮ +éī´ åĪ« +åĨħ ç½® +ä¹± 象 +æ¬Ĭ çĽĬ +å¼ĢæĶ¾ å¼ı +åįļ æĸĩ +讲 课 +çŃī åİŁåĽł +ç©· 人 +交 æĽ¿ +æĬ¤ çħ§ +åıijå±ķ æľºéģĩ +客 åķĨ +åıį ä¹ĭ +ç±³ é¥Ń +å¹¶ åıij +å¹¶åıij çĹĩ +æ±ī åŃIJ +æŀľ åĽŃ +对æĪij æĿ¥è¯´ +åģı åIJij +æī¹ 示 +读 åIJİ +读åIJİ æĦŁ +æĺİ æĻº +åĽ´ çĿĢ +åıį 转 +æĿ¨ å¹Ĥ +ä¸ĵ åįĸ +ä¸ĵåįĸ åºĹ +åıĹ éĻIJ +åºŁ è¯Ŀ +æŀģ å°ij +åįĪ åIJİ +è¿Ľ ä¿® +åīĬ åĩı +æľ¬ç§ij çĶŁ +ä¼ĺ éĢī +åħī çħ§ +åıĻ äºĭ +åıĸ æļĸ +åĮĹ è·¯ +æ¦ ķ +èİĨ çͰ +楼 å±Ĥ +天 èĬ± +天èĬ± æĿ¿ +çĤ ľ +å·²ç»ı æľīäºĨ +è¶ ¾ +çͳ åįļ +ç͵ éĺ» +åĬŁ è¯¾ +æŃ¥ æŃ¥ +éĤ£ä¹Ī 容æĺĵ +æŃ¤ æĸĩ +ä½ ° +计 è¾ĥ +çīĩ éĿ¢ +ç͵影 éĻ¢ +ä¸į åħ¬å¹³ +ä¸ī æľŁ +æĹħ游 èµĦæºIJ +å¤ļç§į å½¢å¼ı +è£Ĥ ç¼Ŀ +åIJİ æİĴ +硬 度 +åĽŀ æļĸ +éģĵ æķĻ +è´« è¡Ģ +æ¸ħ é¦Ļ +伤 çĹħ +æĦı 義 +çļĦ ç¼ĺ +çļĦç¼ĺ æķħ +åºĦ 严 +åıªæĺ¯ 为äºĨ +æīĵ æĬĺ +以 ä¾Ĩ +滿 è¶³ +çİĽ 丽 +風 éļª +æĸĩ ç§ij +éħįå¤ĩ äºĨ +è¿Ľ é£Ł +æ¶ ¡ +è·¯ ç¨ĭ +åı« 声 +ä¸Ńå¿ĥ åŁİåĮº +æľīæīĢ ä¸įåIJĮ +å¼µ è²¼ +é¢Ħ æĬ¥ +æľīå¤ļ ä¹Ī +è¿Ľè¡Į åħ¨éĿ¢ +æĽ¾ ç¶ĵ +ä¸ī 代 +å®ı 大 +æ¸ħ æī« +éĢī åĩº +åĵª ä¸Ģ个 +主 義 +ä¾Ŀ æĵļ +çļ® éĿ© +èµ¶ æĿ¥ +çŃĽ æŁ¥ +æ¨ Ł +ä¿Ŀ èįIJ +åIJĥ æĥĬ +æľĭåıĭ们 对 +ä»ĸ æĺ¯ä¸Ģ个 +åºŁ æ°Ķ +æ» ħ +è´¢ ç¨İ +æĿij æĿijæ°ij +èµĦ产 è´ŁåĢº +å®ī å¨ľ +缮åīį åĽ½åĨħ +æĦŁè§ī èĩªå·± +çµIJ åIJĪ +éͦ æłĩ +éͦæłĩ èµĽ +æĽ´ æ·± +åŁº æķ° +éħ¿ éħĴ +çī¹èī² äº§ä¸ļ +åİĭ å®ŀ +ä¾Ŀæ³ķ 追究 +æ·¡ å®ļ +ç®Ģ缴 å°±æĺ¯ +å£ĵ åĬĽ +æ°ij å¿ĥ +ä¸į åIJĪéĢĤ +çͱæŃ¤ åı¯è§ģ +èµŀ èªī +æ¾ ¤ +åĩłå¹´ åīį +åIJī ä»ĸ +çł´ æįŁ +轻轻 åľ° +å²Ľ 屿 +æĦı å¢ĥ +ä»Ģä¹Ī åı« +åģĩ è£ħ +éĢģ è´§ +å¹ķ å¢Ļ +妥 åįı +åĽ½ æĹĹ +äºĨ å¾Īä¹ħ +åĪĨ辨 çİĩ +ç´ Ķ +éĺ³ åĮº +åĩŃ çĿĢ +åģľè½¦ ä½į +京 éĥ½ +éĶ £ +æĵ ¾ +è¿Ľ éŨ +åĪĺ æµ· +åĽĽ 级 +女 è¶³ +è¡ĮæĶ¿ 审æī¹ +éģ¥ æİ§ +ä¸į éĮ¯ +å¾Ĺ å¾Ī好 +为 缮çļĦ +ä»į æľª +ç²¾ è£ħ +éĢį éģ¥ +å°½ 头 +çºł ç¼ł +éłĺ å°İ +æĭħ è´Ł +æĪĸèĢħ åħ¶ä»ĸ +åıªä¸įè¿ĩ æĺ¯ +åı® åĺ± +åģĩ åĨĴ +æļĸ æ°Ķ +çĽIJ åŁİ +被 è§Ĩ为 +诺 è´Ŀå°Ķ +ç»ĻäºĨ æĪij +è¿ij åįĥ +éĩį åĽŀ +éĨĴ äºĨ +ç͵ è§£ +忽çķ¥ äºĨ +èĥĮ éĥ¨ +æĸĩæĺİ åŁİå¸Ĥ +æº ħ +è² ĵ +æĬµ æĮ¡ +åĸľæ¬¢ åIJĥ +éĿĻéĿĻ åľ° +å¾Ī æ·± +åŁºç¡Ģ çŁ¥è¯Ĩ +è¿ĩ éĶĻ +çIJĨ ç§ij +交æµģ åIJĪä½ľ +èĪ Ķ +調 æŁ¥ +æħĪ æĤ² +éĴ ° +èĩ´ ç͵ +å®£ä¼ł æ´»åĬ¨ +åıĺ éĩı +çļĦ人 æĿ¥è¯´ +æĹ¶ éļĶ +ä¸į管 ä½ł +缸 è¿ij +è´µ éĩijå±ŀ +ä¹Łä¸į åı¯èĥ½ +ç²ī æľ« +åįĹ çĵľ +çϽ 马 +åħī æºIJ +éĩij å¥ĸ +çĭ¬ è§Ĵ +çĭ¬è§Ĵ åħ½ +妨 ç¢į +ç»Ļ åĬĽ +ä½Ĩ ä»į +å¼łå®¶ åı£ +èIJ¬ åħĥ +渲 æŁĵ +éķ¿å¤§ äºĨ +è®°èĢħ äºĨè§£ +æĢĢ çĿĢ +è¦ģ åѦä¼ļ +游æĪı 代 +游æĪı代 ç»ĥ +äºĮ çϾ +æĦıè¯Ĩ å½¢æĢģ +çİ º +计åĪĴ çĶŁèĤ² +æī¾ åĩĨ +åħ° èĬ± +è¿Ļ座 åŁİå¸Ĥ +污 æ³¥ +å®ĺæĸ¹ 微信 +å½Ĵ å±ŀ +æ°§ æ°Ķ +éģİç¨ĭ ä¸Ń +åį°è±¡ æ·±åĪ» +稳 妥 +çµIJ æĿŁ +åŃķ æľŁ +çī¹ æĿĥ +åĿļ åĽº +顺 åĬ¿ +æŀľ èͬ +éĨ« 師 +åİ ® +ä¹Łæĺ¯ å¦ĤæŃ¤ +é¦Ĵ 头 +缸 åĬ© +å¹² 线 +ä¸Ģ æľ¬ä¹¦ +ç» ¥ +æĮ¯ å¥ĭ +èĤ¾ èĦı +åĭķ çī© +é£ŀ è·ĥ +èıľ åĵģ +å¤ļ ä½Ļ +å¤ļä½Ļ çļĦ +éĢĿ ä¸ĸ +æģĭ 人 +å¼Ģåıij åĪ©ç͍ +顺 丰 +éĩİ å¿ĥ +æł¡ å¤ĸ +æģIJ é¾Ļ +éĿ¢ åħ· +éķ¿ è¾Ī +éļı å¤Ħ +éļıå¤Ħ åı¯è§ģ +ç´§ 缺 +éĩį ä¸Ń +éĩįä¸Ń ä¹ĭ +éĩįä¸Ńä¹ĭ éĩį +奥 æĸ¯ +奥æĸ¯ åį¡ +ä¸Ģ个 å¤ļ +ä¸Ģ个å¤ļ æľĪ +ä¸įåı¯ 缺å°ij +æĸ° æł¼å±Ģ +æıIJ æĮ¯ +è¡Į è´¿ +æ¼Ĥ æµģ +èģĬ åŁİ +åħ´ 建 +è´¨ æ£Ģ +ç§ģæľį 游æĪı +æĽ´ éĩįè¦ģ +è´ ® +çħ ľ +转åıĺ 为 +è¿Ļ 两年 +ä¿Ŀ é²ľ +æī§ æķĻ +çĥ ¨ +å¼Ģåıij 建设 +è¿IJèIJ¥ 管çIJĨ +误 å·® +京 åī§ +å¸IJ åı· +å·¥ä½ľ ä½ľé£İ +ä¸ĸ ä¿Ĺ +çϽ 宫 +天 åĽ½ +å¤©åĽ½ ç»§ç»Ń +å·´ æĸ¯ +èIJ¥ åĪ© +åĵģ æł¼ +æĿijæ°ij 们 +æĪ¿ 车 +çŃī çĹĩçĬ¶ +å¦Ĥ å®ŀ +å® ¸ +å±Ĥ 级 +éĶĻ è¿ĩäºĨ +ç»ĵ å®ŀ +ç¬ij èĦ¸ +羣å®ŀ æĢ§ +éĥ½å¸Ĥ æĬ¥ +é¥Ń èıľ +åºĶ 注æĦı +æĬ½ çĥŁ +伪 éĢł +åīį ä¸Ģ天 +éŃĶ é¾Ļ +éŃĶé¾Ļ 令çīĮ +约 è°Ī +绣çѹ æİ¨è¿Ľ +让 ç͍æĪ· +åħ¨éĿ¢ èIJ½å®ŀ +å¼Ħ å¾Ĺ +è°Ī æģĭçα +鸣 æĪIJéķ¿ +鸣æĪIJéķ¿ è®° +æ´ĭ æ´ĭ +çĸı æķ£ +éĿ¢ç§¯ 约 +æµĵ 缩 +æĸ¯ é¡¿ +çĶŁæĢģ åľĪ +æī§ 导 +ç§» éĢģ +齿 è½® +æł¹æľ¬ å°±ä¸į +缩 åĩı +èµ° ä¸ĭåİ» +çĿ« æ¯Ľ +ä¹Łä¸į éĶĻ +åıįæĺł åĩº +èĭ¦ æģ¼ +缸åħ³ æĶ¿çŃĸ +é«ĺ 楼 +ç²ī èī² +æĬķèµĦ é¢Ŀ +ä¸į ç»ı +ä¸įç»ı æĦı +å®ģ æĦ¿ +èĪĮ 头 +æ»ĭ çĶŁ +å®ģ åİ¿ +åīįåĪĹ èħº +åĩ ³ +é£Ł 欲 +åıĸ èĥľ +éĻ¢ åŃIJ +ç´łè´¨ æķĻèĤ² +滨 å·ŀ +æĬ¢ æĬĵ +å¼Ĥ åij³ +åĴ ļ +åĬ į +宽 éĺĶ +æļ´ 涨 +æĥł åıĬ +è§Ħ ç¨ĭ +ä¾Ľ åħ» +éĢģ å¾Ģ +å±± åºĦ +举 äºļ +å±ķ é¦Ĩ +è§£ éĶģ +æĹł è§Ĩ +éĻį èIJ½ +è¿ŀ äºij +è¿ŀäºij 港 +åıĤ è°ĭ +çİ ĸ +ç¬ ĥ +èĢĹ è´¹ +æī¿ å¾· +社ä¼ļ æķĪçĽĬ +åįĹæµ· ç½ij +åĪĽ 伤 +èIJ ± +åħħ æ²Ľ +ç½ijç«Ļ 建设 +大 åºĨ +åĨį éĢł +åŃĹ æł· +åħ¨æ°ij åģ¥èº« +èĮ« èĮ« +æµ® åĬ¨ +åīį åı° +å¢ŀ 设 +éĢĽ è¡Ĺ +åĢĴ éĹŃ +æ³ķå¾ĭ 顾éĹ® +çĸ ® +çĹħ çĹĩ +空 åīį +请 æķĻ +èĥľ ä»» +æĿĢ èıĮ +æĪĺæĸĹ æľº +ç»ĺ åζ +å¤Ħ æĸ¹ +çªģ åĽ´ +çĮ« åĴª +æĬ¥åijĬ æĺ¾ç¤º +ç¿ Ł +çķ¶ åľ° +æľĢ éļ¾ +纪 å§Ķ书记 +ä½İ åİĭ +èĻļ 空 +è¿Ļéĥ¨ ç͵影 +产ä¸ļ åįĩ级 +è°· çα +è°·çα åĩĮ +æĬ¼ éĩij +女 æĸ¹ +éĴ» çłĶ +æļĹ æļĹ +è¿· ä½ł +æīĢ è¬Ĥ +å¨ģ å»ī +å¼Ģ æľĹ +å² Ķ +çģ« çĤ¬ +åIJĪçIJĨ æĢ§ +åħ¬ åĬŀ +ä¼ļ ä¼ļéķ¿ +éĺ´ è°ĭ +å¼Ģ å±Ģ +æĻ®éĢļ è¯Ŀ +åį¡ æĭī +å°ij åIJĥ +éĹª èĢĢ +æŀľ æ±ģ +æī§è¡Į åĬĽ +è° Ľ +æĬ¢ åĬ« +é«ĺéĢŁ åıijå±ķ +éŁ ¬ +åįĹ æ²Ļ +é«ĺçŃī åŃ¦æł¡ +æį¢ 个 +åı¯èĥ½ åŃĺåľ¨ +æĬ Ĵ +è°± åĨĻ +被 æĬĵ +æĿ¯ åŃIJ +èĬĤèĥ½ åĩıæİĴ +æ°ĶåĢĻ åıĺåĮĸ +åĪĨ åĪ¥ +ä¸Ń æŀ¢ +欢 åij¼ +åħī 纤 +è¿Ļ 群 +çľ¼ çķĮ +åħ±åIJĮ åıijå±ķ +çݰ ä»Ĭ +éĹ» è¨Ģ +çī¹èī² å°ıéķĩ +æķij 人 +éĻį æ°´ +ä¸ĸçķĮ ä¸Ģæµģ +å°± é¤IJ +çŀ ¥ +å¤į ä»ĩ +ç¾½ æ¯Ľ +ç¾½æ¯Ľ çIJĥ +è´© åįĸ +æºIJ æ³ī +æĢ»ä½ĵ è§ĦåĪĴ +åĬ¨ æĦŁ +ä¸Ģ 审 +åĢŁ éĴ± +è§ģ æķĪ +èĬ± èįī +åIJĮ ä¸ļ +æŁ¥ è©¢ +åĽ½éĻħ åIJĪä½ľ +ä¾Ľ åĽ¾ +åģ ´ +æł ĵ +缸 éĢļ +è°Ī åıĬ +è¿ĩç¨ĭ å½ĵä¸Ń +é¦Ļ èıĩ +åįģåĽĽ æĿ¡ +ä¸Ģå¼Ģå§ĭ å°± +ä¸ĵ åijĺ +æĺİ é¡¯ +æīĵéĢł åĩº +ä¸ĭéĿ¢ æĪij们 +æľº æ²¹ +åı° è¯į +åŃIJ å¼Ł +æľĢ 常è§ģçļĦ +æĪij è®°å¾Ĺ +ç» ° +æĤ¬ æµ® +è¿ĺ 羣æĺ¯ +æĮĤ åı· +åıĭ åĸĦ +éĩį 伤 +çħ§ 亮 +æŃ¦ èѦ +åĩºçݰ éĹ®é¢ĺ +è¸Ĭ è·ĥ +åľ°çIJĥ ä¸Ĭ +å¸Ĥ 人大 +åıĹ害 人 +å² IJ +åIJĮ åѸ +éĩijèŀį å¸Ĥåľº +æľīçļĦ çݩ家 +å¸Ĥ æķĻèĤ² +å¸ĤæķĻèĤ² å±Ģ +åIJĦ å¼Ĥ +ç·ļ ä¸Ĭ +æģ º +æľī 大éĩıçļĦ +åķĨ æĬ¥ +åįķ åįķ +åħ¨ é¢Ŀ +ä¾ĿæĹ§ æĺ¯ +好 åĩłä¸ª +åĸ µ +éĩį æķ´ +çĶŁæ´» è´¨éĩı +æİ¢ 访 +åį° èĬ± +缼 è¡Į +å¾® è§Ĥ +èĪį å¾Ĺ +åºŁå¼ĥ çī© +积 èĵĦ +å®ļ å±ħ +æĤ ¼ +èĮ ¸ +çļĦ 帮åĬ© +çļĦ帮åĬ© ä¸ĭ +亿 åIJ¨ +åŃĶ éĽĢ +è¿ĻæĿ¡ è·¯ +é¥ µ +æĦĪ åĬł +éķ į +ä½ľ æ¡Ī +èįĶ æŀĿ +太 å°ij +è·» 身 +åħ¬çĽĬ æ´»åĬ¨ +çϽ æĸij +æĬĢæľ¯ æ°´å¹³ +å¸ § +æĹł çŁ¥ +åºĶ该 æĢİä¹Ī +éĢĢ å¸Ĥ +æ¸ Ń +åħ» çĮª +é© ¼ +群 å²Ľ +大 åį« +ä¹ĺ çĶ¨è½¦ +èı² å°Ķ +è´´ åIJ§ +åģľ ä¸ĭæĿ¥ +æľīæľº ç»ĵåIJĪ +åĪ» èĭ¦ +çļĦ åľ° +çļĦåľ° æŃ¥ +è¯Ĭ æīĢ +å¼Ģ æĪĺ +èĢģ çīĮ +çѹ çłģ +åħ«å¤§ 以æĿ¥ +楼 æĪ¿ +åŃĻ æĤŁ +åŃĻæĤŁ ç©º +åħĴ åŃIJ +第ä¸Ģ æĿ¡ +社交 åªĴä½ĵ +æĥ³ èµ·æĿ¥ +大 æ´ĭ +æĭ¼ éŁ³ +è¿Ľ åįļä¼ļ +è¿ĩ åħ³ +æ² ¼ +ç©¿ æIJŃ +éĤ£ ä¸Ģ天 +çł´ éŨ +æĬķæłĩ 人 +èµ¢ å®¶ +èĻļ å¼± +æ¿ ĥ +å®ī æ£Ģ +客 å®¶ +çĭ¬ç«ĭ èij£äºĭ +æīĭ åĬ¿ +åīµ éĢł +åľĨ满 å®ĮæĪIJ +为主 线 +好å¥ĩ å¿ĥ +é¢Ĩ åľŁ +çª ĸ +åħ¸åŀĭ æ¡Īä¾ĭ +çªģåıij äºĭä»¶ +åºķ æ°Ķ +头 æĻķ +å®Ľ å¦Ĥ +è§ ¸ +æ¸ħ æ·¡ +åļ ¼ +åģľ ç͵ +ç²ī å°ĺ +éĻįä½İ æĪIJæľ¬ +æĶ¾ æīĭ +è®°èĢħ 表示 +æĭĸ å»¶ +éª ĩ +æ®ĭ å¿į +çľģ æķĻèĤ² +çľģæķĻèĤ² åİħ +é«ĺ é¢Ŀ +éĦ Ļ +æ¥ ŀ +åĨħ ç§ij +èIJ¥ä¸ļ é¢Ŀ +åŁº çŁ³ +æµģ æ·Į +主 æĹ¨ +éĺIJ éĩĬ +建 åįİ +æĥĬ åı¹ +çī¢åĽº æłijç«ĭ +æĺ¯åIJ¦ åŃĺåľ¨ +建 åĨĽ +éĽ¾ éľ¾ +åħ¬ 认 +åħ¬è®¤ çļĦ +æ°¨ åŁº +æ°¨åŁº éħ¸ +åīį åĩłå¹´ +åι éĤ£ +æ±Ł 举 +å·¥ æ¥Ń +ä¸ĢçĤ¹ ä¹Łä¸į +ä¿® 士 +äºĨä¸Ģ éģį +åĪ ģ +æ»ļ æ»ļ +åĪĨ æł¡ +羣 çα +è¡Ģ èĦī +æĢ¥ åī§ +ä¸Ģ群 人 +ç¾ ¯ +æĪIJ é¾Ļ +ç²¾ç¥ŀ çĹħ +缸åħ³ 人åijĺ +éĿĵ 丽 +ä¸ī åŃ£åº¦ +åĪĴ å®ļ +ä¸ĸçķĮ 第ä¸Ģ +éĢļ ä¿Ĺ +åķĨä¸ļ åľ°äº§ +åĬŁèĥ½ æĢ§ +èµĦæľ¬ 主ä¹ī +详 è§ģ +æĬĵ æįķ +æĸĩ æĺĮ +å®Ŀ å®ī +è£ħéħį å¼ı +æºIJ æºIJ +æºIJæºIJ ä¸įæĸŃ +çĶŁ æĢķ +纵 åIJij +å£ ½ +çľ¼ è¢ĭ +èĤī ä½ĵ +åı¤ ä»Ĭ +èŀį åªĴä½ĵ +åģ ī +æł¼ æľĥåĵ¡ +çĥ · +åĬŁ ç͍ +æīŃ çŁ© +绿èī² éĢļéģĵ +åī§ ç»Ħ +å¼± åĬ¿ +è´¨éĩı éĹ®é¢ĺ +éĻIJ é¢Ŀ +éª Ĩ +éģµ ä¹ī +å¯Ŀ 室 +æĥ³ 念 +åł± åijĬ +ä»ħ 次 +ä»ħ次 äºİ +èŀį åĪĽ +æĭĽèģĺ ä¼ļ +åºĬ åŀ« +转åŀĭ åıijå±ķ +ä¸ŃåĽ½ çĶµä¿¡ +åIJ¬ è¯Ŀ +è«ĭ æ±Ĥ +大éĥ¨åĪĨ 人 +æ´» å¾Ĺ +åĵŃ æ³£ +è¶ Ļ +åıijçĹħ çİĩ +ä¸į 符 +åĨĽ å®ĺ +é¢Ī æ¤İ +æĸ°åĨł çĸ«æĥħ +æŁ¬ åŁĶ +æŁ¬åŁĶ 寨 +ä»»ä½ķ å½¢å¼ı +人 éĻħ +人éĻħ åħ³ç³» +æĢ» æī¿åĮħ +å¹³åĿĩ æ¯ı +æģŃ åĸľ +åĦ ĺ +åħµ 马 +è¿Ł åΰ +å·¥ 伤 +çīĪæĿĥ å½Ĵ +çīĪæĿĥå½Ĵ åİŁ +æĭ¥ æĬ¤ +ç³Ĭ æ¶Ĥ +å¹² æ¶ī +å°ij ä¸įäºĨ +æĥ³ æī¾ +è´¹ çİĩ +该 éĻ¢ +èŀį åĮĸ +è¿İ åIJĪ +è§ĨåIJ¬ èĬĤ缮 +æł¼ ç¶²ç«Ļ +çľī æ¯Ľ +欢è¿İ 大家 +å®¶åºŃ æķĻèĤ² +ä¾µ èļĢ +ç»Ļ ä½łä»¬ +è¡Ģæ¶² 循çݯ +å¯Ħ æīĺ +å°ĸ åı« +以ä¸ĭ åĩłä¸ª +è¿ĺ 以为 +åħ¶ä»ĸ çݩ家 +ç¬ij ç¬ij +æīĵ åIJ¬ +èĩªçĦ¶ ç§ijåѦ +åŁº ç«Ļ +ä¹Ŀ å·ŀ +ä¿Ŀ 驾 +ä¿Ŀ驾 æĬ¤ +ä¿Ŀ驾æĬ¤ èĪª +æĶ¾ çľ¼ +çŁ¥åIJį ä¼ģä¸ļ +ç¸ ® +ç¨ ½ +æļ ĩ +使ç͍ 網路 +é¢Ħ çķĻ +大 象 +åıijæĺİ ä¸ĵåĪ© +æĸĩ 娱 +éĢł ç¦ı +湿 润 +éĿ¢ æĿ¡ +æ¶Īè´¹ åįĩ级 +è®Ĭ å¾Ĺ +åĩł åIJį +ä» Ħ +认 æ¸ħ +è¿ľ æĻ¯ +æıĴ 座 +诸 侯 +åıĺ æĢģ +ç¦ı 彩 +è´§ æŀ¶ +失 æİ§ +ç§»åĬ¨ 端 +ä¸Ĭ åı¸ +éĢł 纸 +å¸ĥ æľĹ +çĴ ĩ +åı° åįĹ +åĮĹ京 åĨ¬å¥¥ +èĵĿ çīĻ +éķ¿ çŁŃ +æĬĺ å°Ħ +ç»ij æŀ¶ +å¯Ĵ åģĩ +转 åŁºåĽł +æĢ¥ äºİ +æŃ£ åĵģ +åħħ 滿 +大 纲 +æĬĹ ä½ĵ +è¨ĵ ç·´ +æĶ¶ ç´§ +æ¯Ķ è³½ +åħµ åĬĽ +æľ¬ æĽ¸ +äºĮ 代 +æĢ¥ è¯Ĭ +æĸĩ æ¡Ī +ç»ı åķĨ +æĻ¨ æĬ¥ +æ£ ĺ +æĢ»ä¹¦è®° åľ¨ +åıĹ éĤĢ +äºĶ åĽĽ +å²Ń åįĹ +çα åIJĥ +åŁĥ å°Ķ +å¿ĥ å¢ĥ +è¦ĨçĽĸ éĿ¢ +å®ŀåľ¨æĺ¯ 太 +æł¹ åºķ +纷纷 表示 +åĹ ħ +éļıçĿĢ æĹ¶éĹ´ +åİĨåı² æĤłä¹ħ +éħ ī +æĢ» éĺŁ +主é¢ĺ æ´»åĬ¨ +éĹ® åį· +é©¿ ç«Ļ +æı¡ ä½ı +åı¯èĥ½ 导èĩ´ +æ°ij éĸĵ +éĸĭ åķŁ +ä½Ĩ ä¸įéĻIJ +ä½Ĩä¸įéĻIJ äºİ +åįģ éĩĮ +å¨ ¥ +æįŁ èĢĹ +çĸı 导 +çݯ æ°§ +ç¥ŀ éĢļ +çα å°Ķ +çαå°Ķ åħ° +æľ´ å®ŀ +å¿« æĬ¥ +æĶ¶ åıĹ +æĪĸ 許 +èĥĮ éĿ¢ +æĸĩåĮĸ ä¼łåªĴ +ä¸ī åĢĭ +æĶ» åĬ¿ +å®ī 举 +å®ī举 å°¼ +åĿĩ å·² +顾 èĻij +éĦ Ń +è¿Ļå®¶ åħ¬åı¸ +åħ¬åijĬ ç§° +æıIJä¾Ľ ä¼ĺè´¨ +稳æŃ¥ æİ¨è¿Ľ +å¤į è¯ķ +å°Ĩ é¢Ĩ +è°Ī èµ· +å¨ Ħ +è¿ŀ 线 +æ©Ł éĹľ +åºĶç͍ åľºæĻ¯ +çĶ» åĥı +è´¢ è¿IJ +ä¿Ŀ éļª +çĹħ çIJĨ +æ¯Ľ 主å¸Ń +ä¸Ŀ 毫ä¸į +çα å¥ĩ +çαå¥ĩ èīº +ä¸ĵå®¶ ç»Ħ +åij¼ åͤ +éĭ ¼ +çģ ¸ +é¢ĨåħĪ åľ°ä½į +æıIJ æĭĶ +龸 éģĵ +å±± åĿ¡ +èĿ İ +沸 èħ¾ +该 项 +ä»Ĭ çĶŁ +ä¸Ģç¯ĩ æĸĩ竳 +æĸ¹å¼ı è¿Ľè¡Į +é»ij 客 +æĶ¹ åĬ¨ +主 é¡Į +æķ£ å¸ĥ +ä»Ģä¹Ī åľ°æĸ¹ +åĮĸ åIJĪ +åĮĸåIJĪ çī© +éĿĻ ç͵ +æĢ» æĶ¶åħ¥ +å§Ķ ç»Ħç»ĩ +å§Ķç»Ħç»ĩ éĥ¨ +éĿĻ æĢģ +èĢģ åŃĹåı· +室 åıĭ +éĥ½ä¸į æķ¢ +æŀ¶ åŃIJ +çģµ æķı +审 è§Ĩ +æĤ£ åĦ¿ +å±± 寨 +èĸª èµĦ +é©° æı´ +éĥ¨åĪĨ åĨħ容 +好 ä¼¼ +æĪIJåijĺ åĽ½ +åľ¨æĪij çľĭæĿ¥ +åħ³æ³¨ 度 +éĻĪ æŁIJ +è¿Ļç§į äºĭæĥħ +éĢī å®ļ +ç²¾ åŃIJ +å£ģ çĶ» +æ±Ł æ·® +é«ĺ æĺĤ +æł¼ åĬĽ +è¼ © +åѦ åłĤ +æĤ¨ åIJĮæĦı +ä¸ĢåĪĩ éĥ½æĺ¯ +æ½ ¤ +éĸ ĥ +å¸ĮæľĽ èĩªå·± +ä¿ ĺ +æ±Ł åİ¿ +æ³ ¾ +ç§ij æķĻ +æīĵ è¿Ľ +ä¸į æħİ +å¯Ĵ åĨ¬ +æ¸Ķ æ°ij +鼷 æĸ¯ +主 å®° +æĹħ游 度åģĩ +ç͵åŃIJ éĤ®ä»¶ +æ±Ĥ å©ļ +éļİ æ®µ +åģ¥èº« æĪ¿ +注æĺİ åĩºå¤Ħ +äºĭæķħ åıijçĶŁ +级 以ä¸Ĭ +åŃĺ æ´» +æĸ½ èĤ¥ +èľľ èľĤ +åµ © +æĮĸæİĺ æľº +æĬĹ æĭĴ +ä¼ł 导 +æĺ¯ä»Ģä¹Ī åij¢ +ä¸Ĭå¹´ åIJĮæľŁ +建 åħļ +çĶŁ æħĭ +ä¿Ŀ ä½ı +款 车åŀĭ +人 èĦī +éļIJ èͽ +失 æķĪ +éģ¿ åŃķ +ç®Ģ 便 +谢谢 ä½ł +å®Ī ä½ı +æĶ¾ æĺł +è¨Ī çķ« +çݰ代 çµģ +é¤IJ 廳 +æķħ å±ħ +大 大å°ı +大大å°ı å°ı +çī¹åĪ« 声æĺİ +éģį åıĬ +å¿ĥçIJĨ åĴ¨è¯¢ +è³ ´ +çĮ® è¡Ģ +å·²ç»ı è¾¾åΰ +æīĵ æĭĽåij¼ +åıĮ è¾¹ +ä¸Ģæĸ¹éĿ¢ æĺ¯ +å´ĩ å°ļ +éĺ¿ å¯Į +éĺ¿å¯Į æ±Ĺ +æĮģ æľī人 +è± ģ +é£İ çŃĿ +åĬ¨ èį¡ +äºĨä¸Ģ ä¼ļ +äºĨä¸Ģä¼ļ åĦ¿ +ä¸ĩ 象 +çľĭ ç͵è§Ĩ +åįģä¸ī æĿ¡ +çĮĽ çĥĪ +è¦ģ ä¸įçĦ¶ +太æŀģ æĭ³ +å¼ķ çĪĨ +ç»ıè¿ĩ å¤ļå¹´ +游æĪı éĩĮçļĦ +é¾Ļ æ³ī +æłĩ éħį +è®ĵ ä»ĸåĢij +éĢł æŀĹ +åĮºåŁŁ æĢ§ +亿 ä¸ĩ +æĪĺçķ¥ å¸ĥå±Ģ +éķĩ æĶ¿åºľ +åĶ® 票 +çĶŁäº§ å·¥èīº +éķĩ åħļå§Ķ +ä¸Ńå°ı åŀĭ +æľ¨ è̳ +æ²³ è¾¹ +èĦ¾ èĥĥ +欢è¿İ æĤ¨ +åıĺ å¼Ĥ +缤 纷 +åŀĥåľ¾ æ¡¶ +辩 è¯ģ +车 åºĵ +æ¯Ķ çİĩ +åħ´ æĹº +详ç»Ĩ äºĨè§£ +å®ī å±ħ +çħ§ æĸĻ +æĸ¹ æīį +èµ ¦ +åĨ ķ +å¥Ķ èµ´ +å®Ŀ 鸡 +åľº åĿĩ +缮åīį æŃ£åľ¨ +åIJŀ åϬ +è¿° èģĮ +æĩ µ +å¥ĩ çijŀ +ä»į å°Ĩ +èĪī 辦 +å·¥åķĨ å±Ģ +å¡ij èĥ¶ +åĬŀ å®ŀäºĭ +æĸ¹ æĸ¹éĿ¢ +æĸ¹æĸ¹éĿ¢ éĿ¢ +æĸĩåĮĸ èĬĤ +åħ¥ èģĮ +é¸ ¥ +ç©¿ éĢı +以 ä¹łè¿ijå¹³ +åį± éļª +æľ¦ èĥ§ +åİĨåı² æĢ§ +æķŀ å¼Ģ +ä¼Ļä¼´ åħ³ç³» +çŁ¿ åĮº +åĽ½éĻħ åľ¨çº¿ +ä¼łå¥ĩ éĩĮéĿ¢ +è¿ij äºĽ +è¿ijäºĽ å¹´ +åĬ£ åĬ¿ +æĶ»åĩ» åĬĽ +æĻº éĢł +ç¦ § +çİĭ åħĪçĶŁ +éĨ« çĶŁ +åĽĽ 项 +å®ŀ æĻ¯ +åĪĿ åĪĽ +å¿ĥ 裡 +æĻ¶ ä½ĵ +交 éĻħ +让 æ¶Īè´¹èĢħ +课 æĸĩ +æİĴ æ°Ķ +å¹¶ä¸į æĦıåij³ +缸 声 +第ä¸Ģ å±Ĭ +åİŁ èijĹ +éĽ ľ +没æľī 太大 +è¡¥ æ°´ +çµģ ä¼ģä¸ļ +第äºĮ æī¹ +åħ¶å®ĥ éĹ®é¢ĺ +æİĮ éŨ +责任 å¿ĥ +é¤IJ åħ· +ç¾Ĭ æ¯Ľ +没æľī å¿ħè¦ģ +ä¹IJ åĽ¢ +è¿Ľ åŁİ +ä¸ĢçĤ¹ åĦ¿ +身 å½¢ +çļ®èĤ¤ çĹħ +æĺ ± +å¢ŀ èĩ³ +èģ² æĺİ +æıIJ è´¨ +ä½ĵèĤ² åľº +çѹ 建 +é¬ Ĩ +车 çīĮ +éļĶ éŁ³ +è´Łè´£ åIJĮå¿Ĺ +丰 ç¡ķ +ä½Ľ éĻĢ +äºī åIJµ +åº ¶ +æ·¡ æ°´ +å°ı çĶ·åŃ© +ç§ģ èĩª +åĮĸ è¿Ľç¨ĭ +æĪĺ士 æĿ¥è¯´ +æ²¹ èħ» +èĦ±è´« èĩ´å¯Į +æĹ¥å¸¸ å·¥ä½ľ +交 èŀį +åĨľ è´¸ +åĨľè´¸ å¸Ĥåľº +åĵĪ çĻ» +ç͵ è´¹ +èµ ĺ +åıĮ èħ¿ +æĵĶ å¿ĥ +æĿ¥ 形容 +使åij½ æĦŁ +éĤ£ä¹Ī ç®Ģåįķ +èĬĻ èĵī +åĢŁæ¬¾ 人 +ç§Ģ 丽 +è®ĵ ä»ĸ +严åİī æīĵåĩ» +è³ ŀ +æļ « +çħ¤ æ°Ķ +çά ä¸Ĭ +æ½ĩ æ´Ĵ +太 ä¹ħ +åij½ åIJį为 +è·¯ çͱ +è·¯çͱ åύ +é© ¯ +æıIJ æĹ© +æĬĹåĩ» çĸ«æĥħ +åĩ Ľ +交 åıĭ +éĶĢåĶ® æ¸łéģĵ +毫ä¸į çĬ¹è±« +èIJ¥ åľ° +çłĶç©¶ 表æĺİ +é±¼ ç±» +æį¢ å±Ĭ +æİ¡ åıĸ +çī Ĩ +缼 å¼Ģ +æ²§ æ¡ij +åºŃ 审 +ç»ı æŁ¥ +åĬł å¼· +缸æ¯Ķ äºİ +ä¸ĵ çıŃ +ä½ĵ åŀĭ +被 害 +被害 人 +æĶ¶ 款 +åħ·æľī èī¯å¥½ +é«ĺå³° æľŁ +åģı ä½İ +åĦ Ł +åĨľä¸ļ ç§ijæĬĢ +ç®Ĭ æĥħåĨµ +å¦Ĥæŀľ çݩ家 +éķ¿ çº¦ +第åħŃ å±Ĭ +åħ¬å¼Ģ æĭĽèģĺ +åĪĩ æĸŃ +è¿« 使 +çĸĹ ç¨ĭ +第äºĮ ç§į +ä¸į åħį +å¹² èѦ +çŁ³ 榴 +åĹ £ +两 ç±» +çε 士 +åŁİ乡 å±ħæ°ij +æŃ¤ 项 +缴 è¾ĸ +缴è¾ĸ å¸Ĥ +åij¼ åºĶ +éĴ ¯ +ç¦ı å¾· +æľº 身 +æĵį åľº +æ¿Ĵ 临 +人群 ä¸Ń +èĤ¡ æ°ij +åŃ ½ +æ³ķ åħ° +é¨ İ +糯 ç±³ +æĢ» çļĦ +æĢ»çļĦ æĿ¥è¯´ +åħ¸ éĽħ +æĸ° éĻĪ +æĸ°éĻĪ ä»£è°¢ +缮 çĿ¹ +é¢Ħ è¨Ģ +è·Į çł´ +æĸ° ç¯ĩ竳 +æ¯Ĵ æĢ§ +åĸĿ èĮ¶ +æŁ¥ èİ· +亮 丽 +çĶŁäº§ åķĨ +æĶ¹ æĪIJ +为äºĨ æĽ´å¥½ +æ·± 交 +深交 æīĢ +æİ ĥ +ä¹Ļ èĤĿ +泸 å·ŀ +åħĪè¿Ľ æĬĢæľ¯ +è¾ĵ ç»Ļ +æķ£ æĪ· +æĢĿç»´ æĸ¹å¼ı +åºĹ 主 +è°ĭ æ±Ĥ +游æĪı æĬĢå·§ +ä¸Ģå¹´ 级 +çľ¼ è§Ĵ +ä¸Ńä»ĭ æľºæŀĦ +å·§ åIJĪ +éĺ² çĽĹ +导 è´Ń +æĪ Ĭ +æĽ´ éĢĤåIJĪ +åŁºæľ¬ ä¿¡æģ¯ +马 ä¸ģ +åħ»æ®ĸ åľº +åıį è¿ĩæĿ¥ +æİ¨ å´ĩ +å¯ĨåĪĩ åħ³æ³¨ +åŁºéĩij ç»ıçIJĨ +æĮī éĶ® +åĨħéĥ¨ æİ§åζ +æĪIJåijĺ åįķä½į +æľ¯ è¯Ń +åζ æľį +åĪļ éľĢ +æ£Ģ ç´¢ +大大 æıIJé«ĺ +åģ¥åº· 管çIJĨ +èĩª æŃ¤ +客æĪ· éľĢæ±Ĥ +丰 èĥ¸ +èµ· éĩį +èµ·éĩį æľº +æ¬ł 缺 +æ¡Ī åŃIJ +æĥħ人 èĬĤ +åħļ æł¡ +è¢ ľ +该 åī§ +迷失 ä¼łå¥ĩ +ç»ļ 丽 +åķ ª +æĹł ç§ģ +é̲ ä¸ĢæŃ¥ +第ä¸Ģ 竳 +åύ åħ· +åĨľ èµĦ +確 實 +åºı åĪĹ +娱ä¹IJ å¹³åı° +èŀįèµĦ ç§Łèµģ +èµĦæºIJ åħ±äº« +èģ½ åΰ +æIJŀ å¾Ĺ +ç»§ç»Ń ä¿ĿæĮģ +åIJ¯ èĴĻ +çľ º +ä¸Ŀ è·¯ +设æĸ½ 建设 +æİ¥ åľ° +æİ¥åľ° æ°Ķ +第ä¸ī åŃ£åº¦ +åŁº è°ĥ +åıij éŁ³ +社ä¼ļ èµĦæľ¬ +éĽĩ 主 +è¿ŀ èĥľ +没 åķ¥ +å» ¢ +èµ¶ èµ´ +æ¼Ķ åĮĸ +åı¤ æĢª +çİĭ çĪ· +é¢Ħ åħĪ +å¼Ģ åħ· +åĽŀ é¦ĸ +åľ°ä¸ĭ æ°´ +å°ıç¼ĸ ä¸Ģèµ· +èµİ åĽŀ +åľ° è²Į +åĪĿ ä¸ī +åı¯ ç͍äºİ +éģĹ è¿¹ +è¿Ļ æī¹ +èĸª æ°´ +å¿ħçĦ¶ ä¼ļ +æ² ½ +éį ĭ +第ä¸Ģ éĥ¨ +åĪĬ çī© +å®ŀ ä¾ĭ +æ¸ħ åĩĢ +ä¸Ĭ èµĽåŃ£ +åĽ¾ 表 +éĤ® è½® +åĵª 裡 +缸 è§ģ +æī° ä¹± +æ¯ı æ¯ı +è¿Ļ è¾ĪåŃIJ +ç¡« éħ¸ +äºī 缸 +溯 æºIJ +åĩº ä¼Ĺ +çİī çŁ³ +åħ± çĶŁ +æĹ¶éĹ´ 段 +éĩįè¦ģ æĮĩ示 +æ¶Īè´¹ éľĢæ±Ĥ +éķ¿ éķ¿ +éķ¿éķ¿ çļĦ +å®ī æĬļ +å¢ŀ é«ĺ +æľ¬ è½® +亲 çľ¼ +é£İ æ³¢ +èĢģ å¦Ī +æĶ¶è´¹ æłĩåĩĨ +åĨħ éĻĨ +æĮ¥ åıij +åįĩ åѦ +èĥ¸ åīį +åģı è¿ľ +纯 æ´ģ +æĸ½å·¥ åįķä½į +身 ä»· +è´¢ åĬĽ +çº ¶ +è£ħ çͲ +æĺ¾ç¤º åύ +毫 åįĩ +æ·± çŁ¥ +è̶ ç© +èĢ¶ç© Į +è¾ĥ éĩı +åľ¨ è¿ĩ渡 +åľ¨è¿ĩ渡 æľŁ +èĮ Ĺ +ä¸Ģ个 æĺŁæľŁ +èĬ · +è´¿ èµĤ +æ¿ ķ +æĩĤ äºĭ +ç§ § +åħħ å½ĵ +åĽ½ ç«ĭ +èĬ± çĵ£ +éĤĦ è¦ģ +åħ¬ åľĴ +触 åĬ¨ +æ³° å·ŀ +ä»Ģä¹Ī æł· +æ»ĭ åħ» +è¯Ħ åΤ +æĮ¥ æīĭ +èĦ Ī +å§¥ å§¥ +è¿IJ è´¹ +æ¯ħ åĬĽ +å¿ĥ æĻº +ä¸į æİĴéϤ +第ä¸ī 代 +éĢĢ è´§ +æĺŁ éĻħ +æ°¸ åĪ© +æĬ¤ åį« +çıŃ è½¦ +è¨Ģ è¡Į +ç¹ ª +主åĬ¨ æĢ§ +å·¥ç¨ĭ è´¨éĩı +éĥĬ åĮº +ä¸Ģ æłĭ +ä½Ĩ å®ŀéĻħä¸Ĭ +ä¸ī大 èģĮä¸ļ +åij¼ åı« +女 åħĴ +è¯ģåΏ æĬķèµĦ +èĢĥ æħ® +çĤ« èĢĢ +æ²» 好 +åĺ ¶ +èĥ ¤ +åħīä¼ı åıijç͵ +åĩł æŃ¥ +æīĢ æīĢ +æīĢæīĢ éķ¿ +çħ§ æł· +åĵ¥ 们 +è¯ Ľ +è¿Ļä¸Ģ åĪ» +çŁ¿ çī©è´¨ +ä¸įå¾Ĺ å·² +åIJĮ 缣 +ç»Ĩ å¾® +è·¯ èĻİ +çϾ èĬ± +æ·· æ²Į +ä¸Ĭæµ· è¯ģåΏ +éĢĢ ç¨İ +èµŀ åı¹ +æī®æ¼Ķ 游æĪı +åIJį åĪĹ +åIJįåĪĹ åīį +åIJįåĪĹåīį èĮħ +ç±³ å°Ķ +ä»Ģä¹Ī åİŁåĽł +å®īåħ¨ ä¿Ŀéļľ +ä¸Ģåıª æīĭ +ä¹³ ä¸ļ +ä¸į çĶĺ +æĥħ åķĨ +æĮ¡ ä½ı +åİŁåĽł ä¹ĭä¸Ģ +è¿Ļ 两天 +çĥĺ çĦĻ +è± ¬ +ä½ł 以为 +没 è§ģè¿ĩ +åĵªå®¶ 好 +åīį ä»» +è¿Ľ è´§ +éĢĢ åĽŀ +串 èģĶ +èĩ³ æĸ¼ +åĨ° æ·ĩ +åĨ°æ·ĩ æ·ĭ +æŁ¥çľĭ 详æĥħ +çı¾ 實 +æİ¨ æµĭ +æİ¥ æīĭ +éļ¶ å±ŀäºİ +åŁİå¸Ĥ 群 +æĿİ åħĪçĶŁ +çŁ¿ æ³īæ°´ +çī¹ ä»· +æĽ´å¤ļ 精彩 +ç¨ĭ å¼ı +读 æĩĤ +å±ı èͽ +奥 æŀĹ +奥æŀĹ åĮ¹ +奥æŀĹåĮ¹ åħĭ +红 èĸ¯ +å¥ ® +å®Ŀ çİī +ç¶² 絡 +è² § +欧 å¼ı +çϽ ç³ĸ +èĩªçĦ¶ çģ¾å®³ +åijĬè¯ī 她 +å» ļ +çĤ¹åĩ» æŁ¥çľĭ +é£İ 湿 +èµĦ产 éĩįç»Ħ +ä¹Łä¸į ä¾ĭå¤ĸ +åįĬ 个å°ıæĹ¶ +åIJ¸å¼ķ æĽ´å¤ļ +æĹ¶éĹ´ èĬĤçĤ¹ +æĶ¶ 纳 +åIJ¸ æ¯Ĵ +èĢģ 乡 +çIJ ħ +æľĢ çµĤ +åıį æĦŁ +ç͍ 微信 +çĶ¨å¾®ä¿¡ æī« +éĢŁ çİĩ +大 çĨĬçĮ« +åı¯ æĥ³ +åı¯æĥ³ èĢĮ +åı¯æĥ³èĢĮ çŁ¥ +åĴ § +èµ° åħ¥ +碳 éħ¸ +èĮĥ åĨ° +èĮĥåĨ° åĨ° +被 åΤ +积æŀģ æİ¨åĬ¨ +è¶³ è¶³ +ç²Ĵ åŃIJ +大 å®Ĺ +大å®Ĺ åķĨåĵģ +ç½ij绾 ç§ijæĬĢ +æĽ¼ åŁİ +å·² ä¹ħ +å·²ä¹ħ çļĦ +秦 çļĩ +秦çļĩ å²Ľ +ä»» æķĻ +å͝ ç¾İ +æ·¡ åĮĸ +æ¡Ĥ èĬ± +çŁ¥è¯Ĩ åĪĨåŃIJ +æĩĴ å¾Ĺ +主 åħ¬ +设计 çIJĨ念 +è³ º +æīĢ æıIJä¾Ľ +æīĢæıIJä¾Ľ ä¹ĭ +æĶ» åħĭ +åĤ ¾ +è¯Ń æ³ķ +åįĥ åı¤ +éĸĭ æĶ¾ +第ä¸Ģ èĬĤ +éĤĦ æ²Ĵ +éĢĥ çĶŁ +æ³ Ĺ +åİ¿ å§Ķ书记 +ä½ľèĢħ æīĢæľī +çħ ½ +ç» ħ +æł ħ +æľ´ ç´ł +çijķ çĸµ +åĮħ åĮħ +æ°ij主 åħļ +ä¸į è¿ľå¤Ħ +å¥ĩ å¼Ĥ +åĺ» åĺ» +æī ¼ +ç¿» å¼Ģ +æĢİ èĥ½ +éģ´ éĢī +è§£ éĩĭ +å¹¼ ç¨ļ +è¦ģ 好好 +è¶´ åľ¨ +ç´¢ åıĸ +ç»Ī çĶŁ +åħ¨ æµģç¨ĭ +éģ© çķ¶ +åįıè°ĥ åıijå±ķ +æĬ¥ ä»ĩ +ç§ijæĬĢ åĽŃ +ä»Ģä¹Ī éĥ½ä¸į +æľĢåIJİ ä¸Ģ次 +ç»Ļ人 ä¸Ģç§į +æł¸ å®ļ +被 åĪĹåħ¥ +æĦı æĥ³ä¸įåΰ +èĢĥ æŁ¥ +åľ¨æŃ¤ ä¹ĭåīį +æīĵ çIJĥ +è¶ĬæĿ¥è¶Ĭ å°ij +å®ļ å¾ĭ +è¡ĮæĶ¿ æľºåħ³ +ä½ıæĪ¿ åħ¬ç§¯ +å°ıå§IJ å§IJ +ä¸ī èı± +ä¿® è¡¥ +èŀĥ èŁ¹ +西 çͲ +æĢ ł +çŃī å¤ļ项 +产ä¸ļ éĽĨèģļ +ä»·æł¼ ä¸Ĭ涨 +åħ¬åħ± åľºæīĢ +è¢ĭ åŃIJ +æĨ§ æĨ¬ +çļĦæĸ¹å¼ı æĿ¥ +åΰ è´¦ +çģ ½ +å·´ èı² +å·´èı² çī¹ +æ¼Ķ ä¹ł +èŃ¦ç¤º æķĻèĤ² +çķı æĥ§ +å¼ķ æµģ +æĶ¶ æĶ¯ +å±Ĥ åĩº +å±Ĥåĩº ä¸į +å±Ĥåĩºä¸į ç©· +æijĩ æ»ļ +辦 çIJĨ +纵 è§Ĥ +æķij æµİ +å®¶ éĥ½çŁ¥éģĵ +åĮ ¯ +å°ı 鸣 +ä»» åĭĻ +计 åħ¥ +ç«ŀ éĢī +å¼ĢèįĴ æĹ¶æľŁ +åij¨ æģ© +åij¨æģ© æĿ¥ +交 ç»ĩ +çķ¢ æ¥Ń +æł¹æį® èĩªå·± +æĸ°äºº çݩ家 +åѵåĮĸ åύ +éĩĩ æļĸ +å¹³åĿĩ æ°´å¹³ +åħ¬å¼Ģ 课 +失 åĪ© +伺 æľį +çĬ ģ +忽 æĤł +主è¦ģ éĽĨä¸Ń +æ¤į æłij +æ¯Ĺ éĤ» +èĩº çģ£ +åĩºåĽ½ çķĻåѦ +æĬĹ éľĩ +æĥ© æĪĴ +å¹´åºķ åīį +åĴ¸ éĺ³ +æ°ij å±ħ +大çIJĨ çŁ³ +éĿ ³ +éķ ĸ +æ¸ħ è¿ľ +è£ħ è½½ +èĩ Ģ +å½± ä¸ļ +å¼Ł åħĦ +æĤ² è§Ĥ +çĿĢçľ¼ äºİ +æįį åį« +åī¥ å¤º +ç¯ Ĩ +å¾Ī éķ¿æĹ¶éĹ´ +è¥ Ł +第ä¸Ģ çϾ +ä¸ĢåĪĨ éĴ± +æĸ°éĹ» è®°èĢħ +éķ· æľŁ +æ³ķ æĪĺç»ĦåIJĪ +è°ģ çŁ¥éģĵ +èħ° éĥ¨ +æ±ī åł¡ +åħ¥ çĿ¡ +åįĸ æİī +æ¶Īè²» èĢħ +æĥ¯ ä¾ĭ +æĥ³ äºĨ +æĥ³äºĨ æĥ³ +èĢģæĹ§ å°ıåĮº +ä¼ł è¨Ģ +åĪĨæķ° 线 +æµģ 泪 +ç»Ħç»ĩ é¢Ĩ导 +äºļ åĨĽ +å¢ŀå̼ æľįåĬ¡ +å¾ ¹ +ä¼ ¶ +äºĽ 许 +å¸ĥ èݱ +强 æĤį +宫 å»· +绿 èĮ¶ +åĮ ¡ +å¾Ī æŃ£å¸¸ +æĺ¥ å¤ı +æ¯ Ļ +è¯Ħ æ¯Ķ +åĩ¡ äºĭ +æĬī æĭ© +åĢĴ éľī +éĩį 度 +åįıä¼ļ ä¼ļéķ¿ +å¿§ èĻij +ä¸ĭ ä¸Ģç¯ĩ +沪 æ·± +æĪ İ +æīĵ ä»Ĺ +åįĪ é¥Ń +å¹´é¾Ħ 段 +ä¸ŃåĽ½ è¶³çIJĥ +设计 æĸ¹æ¡Ī +åºĶç͍ æŁ¥çľĭ +é¢Ħ æĸĻ +åĹ ¡ +ç¥ĸ çζ +çļĦä¸Ģ åijĺ +æ´Ĺ å¹²åĩĢ +åİĨåı² æĸ° +åİĨåı²æĸ° é«ĺ +çĭ¬ åħ· +æħĭ 度 +æīĵ 交 +æīĵ交 éģĵ +é»Ħ çŁ³ +çĽ¼ æľĽ +çī§ åľº +转 弯 +åįĩ åįİ +åĨį ä¹Łæ²¡æľī +èĭ± æīį +æĽ´ åIJį为 +åĢŁ ç͍ +çºł éĶĻ +ç»Ŀ对 ä¸įä¼ļ +çİĭ çīĮ +çĽĨ åľ° +失 è°ĥ +好 象 +é³ ¥ +ä¿Ŀ ä¿® +åĽĽä¸ª èĩªä¿¡ +头 çļ® +åİŁ åīĩ +æĬ¥ æ¡Ī +奴 éļ¶ +å³ Ļ +è°ĥ æĸĻ +ä¹Ł 許 +èIJ½ åΰ +èIJ½åΰ å®ŀ +èIJ½åΰå®ŀ å¤Ħ +çĦļ çĥ§ +çĶŁæ´» çݯå¢ĥ +åºĶ åıĬæĹ¶ +è¶Ĭ è¿ĩ +æĦŁ è¬Ŀ +æĻ¯ å¾· +æĻ¯å¾· éķĩ +çĬ Ģ +身 éĤĬ +ç¨İåĬ¡ æĢ»å±Ģ +åĩĢ åľŁ +ä¾µ åįł +åĬ¨ å·¥ +å¹´ ä¹ĭ +å¹´ä¹ĭ ä¹ħ +第äºĮ èĬĤ +åĬ¨çī© åĽŃ +第ä¸Ģ 书记 +éħ ļ +çĶŁäº§ 设å¤ĩ +æŁIJç§į ç¨ĭ度 +åľ Ń +åĩŃåĢŁ çĿĢ +éĺħ è§Ī +çϽ æ²Ļ +æ²¹ çĥŁ +çªģçł´ åı£ +åıĹ å½±åĵį +åı¯ä»¥ æĽ´å¥½ +å³° å̼ +æĿĤ è´¨ +宿 è¿ģ +çĽĺ æ´» +æ¿Ģ èµ· +åĦ¿ ç§ij +åĿIJ èIJ½åľ¨ +æĮª å¨ģ +æµ· å²Ľ +绣 绣 +éĻ ¨ +ä¼ĺ äºİ +å°Ī å®¶ +ä¸Ģ éĤĬ +èIJ Ĭ +äºĨä¸Ģ åı£ +æ²ĥå°Ķ æ²ĥ +æŃ£å¸¸ 使ç͍ +æĻ®éģį åŃĺåľ¨ +丰 满 +çĶ» åį· +åºĶ æĶ¶ +åºĶæĶ¶ è´¦ +åºĶæĶ¶è´¦ 款 +å®Įæķ´ çĥŃ +å®Įæķ´çĥŃ æ¦ľ +注 è§Ĩ +çĨ Ħ +èº ¬ +éĶĢåĶ® 人åijĺ +è¶ĭ åIJij +çĦ¦ æĢ¥ +åįģå¹´ åīį +ä¼łç»Ł 产ä¸ļ +質 éĩı +åĩ¤åĩ° ç½ij +èµĦæºIJ æķ´åIJĪ +æ¶Į åħ¥ +æĸĩåĮĸ ä¼łæĴŃ +çķĮ 第ä¸Ģ +æ°´ æ³µ +宫 殿 +æİ¢ 寻 +ä¿® åīª +æĦı è¦ĭ +ç´Ĭ ä¹± +æĽ ī +çϽ è¡£ +èĻİ åį« +ç´§ æī£ +å¤Ħå¤Ħ éķ¿ +åĪĽå»º å·¥ä½ľ +红 æŀ£ +饼 å¹² +äºĨ åįĬ天 +ä¼ļå½±åĵį åΰ +çĽ¸ä¿¡ 大家 +èħ¾ é£ŀ +å°± å¦ĤåIJĮ +ä¸ĭéĿ¢ å°ıç¼ĸ +æ°ijèIJ¥ ç»ıæµİ +æĻ ¦ +è£ħ æī® +é»ij å¤ľ +常 å¾· +å·¥ä¸ļ 大åѦ +æĺİ çŁ¥ +éĺŁåijĺ 们 +åIJ¬ 课 +æ¯ı éļĶ +羣æĺ¯ 太 +åIJĪä½ľ åħ±èµ¢ +çIJĨ åıij +æīį å¹² +çľĭ èµ·ä¾Ĩ +殿 ä¸ĭ +å®ī éĺ³ +æīĢ äº§çĶŁçļĦ +éĽĩ ä½£ +æĬ¬èµ· 头 +æį® æĬ¥éģĵ +éļĨéĩį 举è¡Į +交 éĶĻ +è¶ħ é¢Ŀ +åĮĸ çĸĹ +é¡ Ĩ +纵 æ·± +çĪ±åĽ½ 主ä¹ī +éĻ¢ åī¯éĻ¢éķ¿ +è® ³ +羣æŃ£ åģļåΰ +åѤ åįķ +èĩªçĦ¶ èĢĮ +èĩªçĦ¶èĢĮ çĦ¶ +ä¿® 身 +èĬ ¹ +æģ¯ æģ¯ +æģ¯æģ¯ 缸åħ³ +驾 æł¡ +æİ© 饰 +æ³½ è¿ŀ +æ³½è¿ŀ æĸ¯åŁº +举 æŃ¢ +管çIJĨ ä½ĵåζ +åħ¶ä¸Ń ä¹ĭä¸Ģ +æĿ¾ å¼Ľ +æĭ¦ æĪª +åį« åģ¥ +åį«åģ¥ å§Ķ +ä»İ åݻ年 +åĤ ¢ +è´Ń 票 +åĽ¾ æłĩ +æ²³ 西 +æ°ijæĶ¿ å±Ģ +ç§ģ èIJ¥ +å¤ĸåĽ½ è¯Ń +å¹² è´§ +æĵ¦ æĭŃ +åľ° ä¸Ń +åľ°ä¸Ń æµ· +æµĵ æµĵ +æµĵæµĵ çļĦ +å§ĭ 建 +å§ĭ建 äºİ +ç¶ĵ æŃ· +è·¯ æ¼Ķ +æļ´ é£İ +åŁº è¾ħ +æī¶è´« å·¥ä½ľ +ä¸Ģ缴 å¤Ħäºİ +æĥħ è¶£ +äºĮ åŃ£åº¦ +åİĮ æģ¶ +顺åĪ© å®ĮæĪIJ +æŁ¥ å°ģ +é¡¶ 端 +ä¸į åŃķ +ä¸Ģ大 åłĨ +被 æ·ĺæ±° +æĺ¯ ç͍æĿ¥ +æľĢ åIJĪéĢĤ +亮 çľ¼ +å¹¶ä¸įæĺ¯ å¾Ī +ç§ijçłĶ éĻ¢ +ç§ijçłĶéĻ¢ æīĢ +ç² Ł +é¢Ī éĥ¨ +é»ĺé»ĺ åľ° +é«ĺä¸Ń çĶŁ +æĹıèĩªæ²» åİ¿ +æķĻåѦ è´¨éĩı +æĪĺ çģ« +åĿİ åĿ· +æIJŃ ä¹ĺ +è¯Ĺ æĦı +åĪij èѦ +åĩº æ±Ĺ +åįģåħŃ æĿ¡ +请 åıĬæĹ¶ +åĨľä¸ļ 大åѦ +èIJ½ åı¶ +æĢ» èĢĮè¨Ģ +æĢ»èĢĮè¨Ģ ä¹ĭ +æĿľ åħ° +æĿľåħ° çī¹ +éĻª ä½ł +åħ¬ æĬ¥ +çķĻè¨Ģ æĿ¿ +éĺħ åİĨ +ç«¶ çĪŃ +ç»Ļ åĪ«äºº +æĹ¥æĬ¥ 社 +åĿIJ èIJ½ +åĿIJèIJ½ äºİ +éĩij åŃĹ +éĩijåŃĹ å¡Ķ +åĽ ¤ +è¯Ŀ åī§ +æĮģç»Ń æİ¨è¿Ľ +æ¼ı æ°´ +詳 ç´° +æĢĢ æĬ± +åıĺ å¹» +饥 饿 +éļIJ 身 +个 èµĽåŃ£ +åĵ¡ å·¥ +æģ¢å¤į æŃ£å¸¸ +äºĨ 好å¤ļ +æĺŁ å·´ +æĺŁå·´ åħĭ +åħī çݯ +å¸ħ åĵ¥ +çϽ éĽª +ç¨į ç¨į +计 æıIJ +æĦĽ æĥħ +éİ ĸ +ä¿¡ éĺ³ +è§Ģ å¯Ł +å¦Ĥæŀľä½ł æĥ³ +缸æ¯Ķ ä¹ĭä¸ĭ +è§£ å¼Ģ +æīĵåį° æľº +身 躯 +ç²¾ç¥ŀ æĸĩæĺİ +èĤ¡ æĮĩ +å¾® åĪĽ +红 èĮ¶ +èĩ´ çĻĮ +æģ© æĸ½ +èħ¿ éĥ¨ +大åŀĭ å¤ļ人 +å®ī åĢį +è¾ħ导 åijĺ +èĪª éģĵ +å¸ĥ å°Ķ +åįĹå®ģ å¸Ĥ +ä¸ĬçıŃ æĹı +ä¾§ ç»ĵæŀĦæĢ§ +追 éļı +å½ĵåľ° æĶ¿åºľ +èµ° åĩºæĿ¥ +éĩijèŀį ä¸ļ +丼 书 +é¡¹çĽ® ç»ıçIJĨ +è¿ĩ æĪ· +骨 æŀ¶ +è¡ Ļ +ä»Ģ 麽 +èħ ĭ +è¦ģ 害 +åľ¨ åºĬä¸Ĭ +代è¨Ģ 人 +並 å°ĩ +åIJĦ个 æĸ¹éĿ¢ +è°´ è´£ +åħ± æĮ¯ +åį³å°Ĩ åΰæĿ¥ +èĤº çĻĮ +ä¾Ľ éĶĢ +丼 æŀĹ +èµ ĥ +åįģä½Ļ å¹´ +åĭĺ æİ¢ +飵 åij³ +èĭ¦ ç¬ij +æľĢ大 ç¨ĭ度 +éĩįçĤ¹ åħ³æ³¨ +ä¹ĭ 举 +满 æĢĢ +åıĹåΰ å½±åĵį +æĭĽ æĬķæłĩ +è¡¥ é½IJ +西 红 +西红 æŁ¿ +é¬ § +è£ħ åᏠ+éĤ» éĩĮ +èĤĩ äºĭ +æİĴ æ¯Ĵ +åѤ åĦ¿ +鼶 è·Ŀ离 +å®ŀ å¹² +çľĭ æŁ¥çľĭ +æĶ¶è´¹ ç«Ļ +ç» · +åħ¬çĽĬ æĢ§ +éĢĴ ç»Ļ +æĶ» æīĵ +æĺŁçº§ éħĴåºĹ +æĺİ åªļ +ç፠ç«ĭ +è¯Ŀè¯Ń æĿĥ +ä¸ĢæŃ¥ ä¸ĢæŃ¥ +书æ³ķ å®¶ +æľªç»ı æİĪæĿĥ +çŁ³ èĨı +åĩŃ ä»Ģä¹Ī +çļĦ æĹ¥ +çļĦæĹ¥ åŃIJéĩĮ +诱 人 +çϾåĪĨ çϾ +èĪĪ è¶£ +å¼ł åħĪçĶŁ +èĢģçĪ· åŃIJ +æ³¢ çī¹ +åŁºéĩij 份é¢Ŀ +æ²Ļåıij ä¸Ĭ +å¥ĭæĸŠ缮æłĩ +æ°¢ èĥ½ +æ²ĥå°Ķ çİĽ +義 åĭĻ +éŁ³ ç®± +æ²ī 浸 +æ²ī浸 åľ¨ +èĭ± åľĭ +çģ¯ çģ« +è¿Ľ 项 +两 端 +ä¹Ķ 丹 +èĦ¸ é¢Ĭ +åıijå±ķ æ½ľåĬĽ +åĭķ ä½ľ +åĵĪ ä½Ľ +å®´ ä¼ļ +æ§ į +ç«ĭ å¿Ĺ +ç¡ķ士 åѦä½į +åĭĭ 竳 +è¿Ļ åľºæ¯ĶèµĽ +æĮģ å¹³ +éķĢ éĶĮ +èĭ± çī¹ +èĭ±çī¹ å°Ķ +æķĻ èģĮå·¥ +åĬŁ åĬĽ +该 æ¡Ī +ä¸Ģ æ¢Ŀ +åĺī å¹´ +åĺīå¹´ åįİ +è¿« ä¸įåıĬ +è¿«ä¸įåıĬ å¾ħ +è¿Ļ个 æĹ¶ä»£ +精彩 æĴŃæĬ¥ +人 èĦ¸ +人èĦ¸ è¯ĨåĪ« +æ£Ģå¯Ł å®ĺ +å°ı èħ¿ +éĨĴ 缮 +åħļ æĢ» +åħļæĢ» æĶ¯ +æĪ Ł +èĮ« çĦ¶ +è±Ĩ æµĨ +主 æ²» +éĿĴæµ· çľģ +åĪijäºĭ 责任 +çł ° +ä¹ĭ æ¬ĬåĪ© +äºĶ å®ĺ +è¿· æĥij +åħ¥ åºĵ +å®¶ 纺 +å¼¹ ç°§ +åįģäºĶ æĿ¡ +ç»Ļ å®Ŀå®Ŀ +èĪªç©º èĪªå¤© +å¾Ģ å¤ĸ +å¼ķ åĬĽ +çľ¼ çļ® +æ¶ī è¶³ +æĿ¥ 宾 +åľ¨çº¿ è§Ĵèī² +çĥŃ éĶĢ +æµģ éĢĿ +泡 泡 +éĻį å¹ħ +è´ŁéĿ¢ å½±åĵį +红 楼 +红楼 梦 +éļĶ çĿĢ +ä¾¥ 幸 +许 ä¹ħ +åĴĮ çĿ¦ +èŃ ½ +使ç͍èĢħ æĪĸ +ä¹° åįķ +è¿ ´ +é£İ æīĩ +æķĻ å¸« +æ¡ĮåŃIJ ä¸Ĭ +å¾Ī æ¼Ĥ亮 +åł± å°İ +第ä¸Ģ åŃ£åº¦ +ç©© å®ļ +æĤ² åĵĢ +çĿĢåĬĽ æīĵéĢł +æĮ Ł +è·¯ æ¡¥ +åij IJ +åľ£è¯ŀ èĬĤ +çļĩ åŃIJ +ä»ĩ æģ¨ +éħĿ éħ¿ +ä¸į éĹ´ +ä¸įéĹ´ æĸŃ +æĮĩ å°ĸ +ä¸ŃåĽ½ ç½ij游 +åŀ £ +æĦıè§ģ 建议 +æ¯ħ çĦ¶ +亮 度 +èģĶ è°Ĭ +å½ķ åħ¥ +åĦ ² +å¨ĺ å®¶ +ç§ij å°Ķ +ä¹Łæ²¡ ä»Ģä¹Ī +æł¹æį® ä¸įåIJĮ +åı¶ ä¿® +å̼ å®Ī +æľ« 端 +åĪ ¨ +åĤµ åĭĻ +èģ¯ åIJĪ +å¥ĩ å¹» +èĻļ æŀĦ +é»Ħ æĺı +å¹³ åĿ¦ +æµģ æ°ĵ +æĸ° åŁºå»º +æĮ½ æķij +åįİ å°Ķ +åįİå°Ķ è¡Ĺ +æľĢ åıĹæ¬¢è¿İ +ç»Ń 约 +å¼Ĭ 端 +éŃĶ æ³ķå¸Ī +éŃĶæ³ķå¸Ī åĴĮ +åħ·ä½ĵ åĨħ容 +çIJī çĴĥ +æī© 容 +èĮ¶ åĽŃ +主ä¹ī èĢħ +ç«ĭ éĿ¢ +æİ¥åıĹ éĩĩ访 +åĩº åħ¥å¢ĥ +ç§ij åįı +éĴ ³ +çµIJ æ§ĭ +ç»ĵæŀľ æĺ¾ç¤º +åı° è´¦ +å°± æĿ¥çľĭçľĭ +èĩª æķij +åıį æĩī +åİ» åĵªåĦ¿ +è¿Ļ é¦ĸ +è¿Ļé¦ĸ æŃĮ +åIJ¬ ä¼Ĺ +å¤ĸ 壳 +ä½ĵèĤ² é¦Ĩ +實 æĸ½ +èŀº ä¸Ŀ +æĭī åįĩ +çĮĽ åľ° +åħ¨åĽ½ 人æ°ij +æĤī å°¼ +æĹı 群 +åĽ¢ åijĺ +两个 å°ıæĹ¶ +åľ¨ çݩ家 +åľ¨çݩ家 ä¸Ń +çĶľ çĶľ +æĬķ è¡Į +åįĶ æľĥ +éĻ ¡ +åĬłå·¥ åİĤ +æ¦Ĩ æŀĹ +æŃ» è§Ĵ +åĨħ å¹ķ +æīĢæľī æĥħèĬĤ +åĪ· åį¡ +æ°´ èĤ¿ +èĥĥ åı£ +å«Į å¼ĥ +æ²® 丧 +ä¸īå¹´ 级 +æ¶Ĥ å±Ĥ +å¿ĥ 仪 +å¿ĥ仪 çļĦ +å¤ Ń +é¦ĸ è½® +æĹłè®ºæĺ¯ åħ¶ +éĢı æ°Ķ +äºĮ åįģäºĶ +ç® « +åĬŁ åĬ³ +çѾ ä¸ĭ +æ²ī è¿· +æķij åij½ +éĹª éĹª +åIJĥ äºı +å±ķ åĵģ +åį³æĹ¶ åıijçĶŁ +ç¶ ľ +ç¶ľ åIJĪ +æłĩ æĺİ +çľĭ ç͵影 +åħ¬ 竳 +éĺ¿ æ£® +éĺ¿æ£® 纳 +身 åĪĽéĢł +身åĪĽéĢł çļĦ +æ¸Ľ å°ij +å̼å¾Ĺ åħ³æ³¨ +鼶åĶ® åķĨ +æįĨ ç»ij +è¸ı åħ¥ +èĽ Ł +æŁ´ 纳 +èĢģ åħµ +绿èī² çݯä¿Ŀ +é¹ Ń +麻 æľ¨ +æıŃ çīĮ +è¿Ļ款 车 +ç¾İ å¾· +ç¾İå¾· åħ¬åı¸ +æ¶ § +è°ģ çŁ¥ +æ´ĭ èij± +æ¯į æł¡ +ä¸Ģ éĹª +çĶ· 主è§Ĵ +æĹłçº¿ ç͵ +å±ł å®° +æĺ¯ éŁ©åĽ½ +æĺ¯éŁ©åĽ½ 娱 +容 è²Į +åĿĩ 使åħ¶ +太 å¿« +å¹´ çͱ +å¹´çͱ 缼 +èĭ¦ èĭ¦ +åĬĽ è¿ĺæĺ¯ +åĬĽè¿ĺæĺ¯ èĩª +æĨ © +èģ¯ çµ¡ +åĶ ¾ +åħ·æľī æĪĺ士 +追 éĹ® +åłĨ æĶ¾ +åıį 驳 +å®ŀäºĭ æ±Ĥ +å®ŀäºĭæ±Ĥ æĺ¯ +åѸ éĻ¢ +åįģ åĩłä¸ª +æķij æĬ¤ +æķijæĬ¤ 车 +ç½ij绾 ä¼łæĴŃ +åįģåħ« å±Ĭ +éĥ¨ åī¯ +éĥ¨åī¯ éĥ¨éķ¿ +çĹ´ è¿· +管çIJĨ æĿ¡ä¾ĭ +èŀį 为ä¸Ģä½ĵ +æĢ» 产å̼ +è³ ĵ +ä¸ĥ æĺŁ +çıŃ ç»Ħ +绣 é¢Ĩ +请 大家 +éĩij éϵ +èĪħ èĪħ +æµ· æ¹¾ +æĸ½ çŃĸ +享 èªī +éº ¥ +端 åįĪ +绿 åŁİ +確 ä¿Ŀ +å·´ æĭī +åĨĴ çĿĢ +æħ· æħ¨ +个人 è§ĤçĤ¹ +ä¹Ļ çĥ¯ +ç¡ħ è°· +éĸĭ å±ķ +å°ļ 书 +åĿļ 飧 +åº µ +èĢģ é¾Ħ +èĢģé¾Ħ åĮĸ +羨 çľ¼ +绿 æ°´ +绿水 éĿĴå±± +书 é¦Ļ +主åĬĽ åĨĽ +æīįæĺ¯ 羣æŃ£ +æĬ¢ åħĪ +æĪIJå°± æĦŁ +éĩį æŀĦ +éĴ¢ åİĤ +æĪIJ 份 +èĬ± 纹 +ä¹ĭ äºī +å¹² ç»Ĩèĥŀ +æĹ¢ åı¯ä»¥ +ç¹ģ çIJIJ +æĦļ èł¢ +éĿŀ常 æĺİæĺ¾ +ä½ĵ 彩 +æĬĢ æ³ķ +æĿĨ èıĮ +å¹¿æ³Ľ åħ³æ³¨ +åĮĹ å®ĭ +å§Ĭ 妹 +åįı åĬŀ +æ·® åįĹ +çĥ ı +æ´Ĺ èĦ¸ +åıĹ è®¿ +åıĹ访 èĢħ +éĩįè¦ģ åĽłç´ł +å½±è§Ĩ åī§ +综èīº èĬĤ缮 +èľķ åıĺ +äºĮ 线 +äºĮ线 åŁİå¸Ĥ +ä¼Ĭ å§ĭ +çıĬ çijļ +èĩª æŁ¥ +åħ¥ åĽŃ +åĩ¶ æīĭ +åħ¬ è¯ī +éģĩ éļ¾ +éĩĩçŁ¿ çŃī +èĩª çIJĨ +åĸ· æ¶Ĥ +æī© åħħ +éĢı è§Ĩ +é«ĺéĢŁ å¢ŀéķ¿ +åĽ¾ çĶ» +ç¾ ¹ +èĤĩ åºĨ +è¾ľ è´Ł +èµĶ ä»ĺ +è· ¡ +åģ¥åº· æĪIJéķ¿ +以ä¸Ĭ åѦåİĨ +åıĸå¾Ĺ 以åıĬ +æ²ī 积 +åįģä¹Ŀ å±Ĭ +缸éĹľ æľįåĭĻ +æī§ åĭ¤ +åī¯ åİ¿éķ¿ +å¯ ° +åģľ æ»ŀ +æ·¹ 没 +çŁ³ çģ° +çį ¸ +åĢ ¦ +ç¾İ åªĴ +æķĻ æ¡Ī +åĬł çĽĸ +åħ¬å¼Ģ èµĽ +å¥ł åŁº +æĺĨ èĻ« +çŀ ħ +磷 éħ¸ +äºī åĪĽ +çİĭ æĻĵ +ç¼ĵ åĨ² +åİļ åİļ +åİļåİļ çļĦ +æŀ£ åºĦ +ç²¾ çĽĬ +ç²¾çĽĬ æ±Ĥ +ç²¾çĽĬæ±Ĥ ç²¾ +åĪĨæĶ¯ æľºæŀĦ +å®ŀæĸ½ ç»ĨåĪĻ +æĸ° èµĽåŃ£ +總 çµ± +éĢł è¡Ģ +é¢ĩ åħ· +é»Ħ åŁĶ +è¡Ģ èĦĤ +交éĢļ å·¥åħ· +å³ ¥ +æĹıèĩªæ²» å·ŀ +寺 éĻ¢ +確 å®ļ +æ¦Ĥ念 èĤ¡ +æĦŁ å®ĺ +æŁľ åı° +åĶ Ķ +çŀŃè§£ 並 +æĢ» ä»· +åIJ¸ åħ¥ +æĢ ¼ +æĻļ éĹ´ +å±Ĭ æ¯ķä¸ļçĶŁ +çĶŁ å§ľ +éĺħ读 åħ¨æĸĩ +å¾Ĺåΰ æľīæķĪ +æIJľ æķij +åİĨ æĿ¥ +èŃī æĺİ +åĥ » +èĨ³ é£Ł +åĦĦ åħĥ +æīĵ åİĭ +宾 客 +åķ ¼ +ä¸ĢçϾ å¤ļ +æ·±åħ¥ 人å¿ĥ +æ¢ħ å·ŀ +çłĶ åѦ +åħ³ ä¹İ +è¼ Ľ +亲 åıĭ +éħį æĸĻ +æĪij çĪ±ä½ł +è´¸æĺĵ æĪĺ +æľī èī² +æľīèī² éĩijå±ŀ +æįIJ åĬ© +为 é¦ĸ +为é¦ĸ çļĦ +å¯Į åĬĽ +çĶ· ç¥ŀ +é³ ³ +æµĩ æ°´ +åIJ ± +æĺİç¡® æıIJåĩº +åı¹ äºĨ +åı¹äºĨ åı£æ°Ķ +礼 æĭľ +è¿Ļ个 åIJįåŃĹ +ä¿¡ å¾Ĵ +å¿Ĺ 强 +éĻIJ æĹ¶ +æĶ¶ è²» +åĨľå®¶ ä¹IJ +å°ıé¾Ļ èϾ +èIJ½ å¹ķ +æ§ Ł +åѦ 龸 +æĪĸ å¤ļ +æĪĸå¤ļ æĪĸ +æĪĸå¤ļæĪĸ å°ij +座è°Ī ä¼ļä¸Ĭ +æ¶ ¼ +éŃĶ çİĭ +å² ± +é¡¶ å±Ĥ +é¡¶å±Ĥ 设计 +èĦij åŃIJéĩĮ +éĻ¢ åŃIJéĩĮ +轩 è¾ķ +身å¿ĥ åģ¥åº· +èħ ij +éĹľ 注 +åıĤåĬł ä¼ļè®® +ä¸Ńåįİ æĸĩåĮĸ +追 寻 +å®ī çĦ¶ +é£Ļ åįĩ +éŁŃ èıľ +é¸ ¦ +åĤ¨ éĩı +çĶ· æĸ¹ +å¤ĩ 份 +æijĶ åĢĴ +润æ»ij æ²¹ +é̼ è¿ij +çͳ è¯ī +鸣 ç±» +çŁ³æ²¹ åĮĸå·¥ +åĿļ æŀľ +è¿Ļå®¶ ä¼Ļ +æĭĴ ä¸į +羣 çļ® +è·Ŀ éĽ¢ +è¿ĺ æĮº +éĽķ åĥı +åĪĿ æģĭ +æıIJä¾Ľ æĽ´å¤ļ +æŁ¥çľĭ åħ¨æĸĩ +æķ°åŃĹ è´§å¸ģ +åĸī åĴĻ +åı¦ä¸Ģ ä½į +åĤ¬ åĮĸ +åĤ¬åĮĸ åīĤ +ä»İæĿ¥ 没 +å¯ĨåĪĩ 缸åħ³ +éĥ¨ 主任 +产åĵģ ç»ıçIJĨ +並 åIJĮæĦı +èIJ½ åħ¥ +å±ıå¹ķ ä¸Ĭ +åħ¬åı¸ 竳ç¨ĭ +æį¢ åı¥è¯Ŀ +æį¢åı¥è¯Ŀ 说 +ä½į æĸ¼ +ä½ Ķ +åĩ» æĿĢ +缸 è¾ĥ +缸è¾ĥ äºİ +ç²½ åŃIJ +åįĹ æŀģ +宫 é¢Ī +è£ģ åijĺ +æĺİ ç»Ĩ +ä»·å̼ éĵ¾ +åĽĽä¸ª æĸ¹éĿ¢ +æĥħåĨµ æĿ¥çľĭ +æĮij åīĶ +æ® ĺ +æŀģ åĬĽ +çĸij éļ¾ +æĬµæĬĹ åĬĽ +æĢ¥ éĢŁ +æĪ Į +ä½İ ä¼° +éĹª è¿ĩ +æģ ¬ +èµŀ æī¬ +ä»ĸ å¦Ī +æĪIJ为 ä¸ĢåIJį +æ´Ĺ 礼 +é¢Ħ计 å°Ĩ +åħĪè¿Ľ åįķä½į +è¼ Ķ +éĢĥ èĦ± +çݰ åŃĺ +èĢģèĻİ æľº +åįģä¸ĥ æĿ¡ +åı¦ä¸Ģ åįĬ +温 æĥħ +åī¥ ç¦» +ä¸ĸ è´¸ +å®ĺ åı¸ +å¾Ī å·® +éĹ´ è·Ŀ +请 注æĦı +åı² è¯Ĺ +åĪ© åύ +è¿IJ ç®Ĺ +沦 为 +該 使ç͍èĢħ +èĮ ¬ +éͦ 绣 +åı² æĸĻ +çģµ æ´»æĢ§ +èģĶ ç¤¾ +æĹł åĬ© +æĬĹ æ°§åĮĸ +èıľ èĤ´ +éĢł èι +æİī èIJ½ +å¤į æŁ¥ +åĭĥ åĭĥ +åij¼ 声 +給 äºĪ +åIJĮäºĭ 们 +ç½ ° +è¯ķ æİ¢ +åħ³éĶ® åŃĹ +æįIJ çĮ® +ç»Łè®¡ æķ°æį® +åĪĽ ä½ľèĢħ +ä¸ĭ åįĬ +ä¸ĭåįĬ åľº +æī¿æĭħ 责任 +端 æŃ£ +ç©¿ è¡£ +ä¼ł çIJĥ +åĬ© éķ¿ +åĩ ± +éķ¶ åµĮ +é£ŀ ç¿Ķ +è¾ĵ åįµ +è¾ĵåįµ ç®¡ +ä¸ĩ åħ¬éĩĮ +æİ¨å¹¿ åºĶç͍ +å¿« æ¨Ĥ +ç§ ½ +èī° å·¨ +åIJ¬ å®Į +åĿļ 硬 +奥 åľ° +å¥¥åľ° åĪ© +é¢ ĵ +èĻIJ å¾ħ +ä¾Ľ æ±Ĥ +éľī ç´ł +伪 è£ħ +乡 åľŁ +åĩ¡ æľ¬ç½ij +åĩ¡æľ¬ç½ij 注 +ä¼Ĭ åĪ© +è¡¡ æ°´ +æĽ´ åĥıæĺ¯ +åĪĨéĴŁ å·¦åı³ +è¦ı 模 +äºĶ åĪĨéĴŁ +åºĹ åĬłçĽŁ +åĽ° éĽ£ +åħ³ åģľ +æĢĿ 绪 +åĴ½ åĸī +缸 符 +çĥ¦ èºģ +æĻĤ æľŁ +åijĪ çı¾ +è§£ æķ£ +诱 导 +éļĶ çĥŃ +çĮ ¶ +åįĹ å®ĭ +æ·±åħ¥ äºĨè§£ +çŃĶ çĸij +æĺ¼ å¤ľ +åįĥ ä¼ı +åĬ³åĬ¡ æ´¾éģ£ +红 è±Ĩ +åĿı äºĭ +çĤ¹ æ»´ +å°±ä¸ļ å²Ĺä½į +约 åIJĪ +åħį éϤ +éĢĨ åĬ¿ +éĩį éĩijå±ŀ +å®ĺ 宣 +ä½İ å»ī +æģ¨ ä¸įå¾Ĺ +å¾Ĺ 天 +å¾Ĺ天 çĭ¬ +å¾Ĺ天çĭ¬ åİļ +ä¸Ģå°ģ ä¿¡ +æĬ½ å¥ĸ +è¾Ĺ 转 +çķĻ å®Ī +çķĻå®Ī åĦ¿ç«¥ +çŃĶ åį· +å·¨ åŀĭ +æľĢ好 ä¸įè¦ģ +æµĻæ±Ł 大åѦ +æĨ ¨ +æı¡ æīĭ +éĴĪ ç»ĩ +æİĴ 骨 +çĤ ½ +å°ģ è£ħ +åįĢ åŁŁ +空æ°Ķ åĩĢåĮĸ +åħī å½± +åĢĴ å¡Į +å§ļ æĺİ +æ¤į 被 +åѦ åīį +åѦåīį æķĻèĤ² +èĬĿ åĬł +èĬĿåĬł åĵ¥ +缩 æ°´ +ä½ Ł +åľ¨çº¿ åĴ¨è¯¢ +èµı æŀIJ +éĿĴ èĽĻ +æĬ± ä½ı +èĮĤ åIJį +åħ¨åĬĽ æīĵéĢł +åįļ士 åѦä½į +æ²§ å·ŀ +åĻ ¢ +æĿĤ çī© +åĪ» çĶ» +æį ħ +å¾® éĩı +å¾®éĩı åħĥç´ł +ä¸Ģ åĽŀäºĭ +鸡 èĤī +åĪ©æ¶¦ çİĩ +æīį ç®Ĺ +å¾® å¦Ļ +棵 æłij +è´ª 婪 +åĩı å̼ +梦 å¢ĥ +åı¯ è§Ĩ +åı¯è§Ĩ åĮĸ +广大 å¸Ĥæ°ij +ä¸ĵä¸ļ ä»İäºĭ +ç»ı 纬 +ç´§ çĽ¯ +çŁ¥ å·± +è¤ ļ +æĸĩåĮĸ åºķèķ´ +åݦéŨ å¸Ĥ +临 港 +对åħ¶ 羣å®ŀ +岸 è¾¹ +è¦ĸ çĤº +æĬĹ çĻĮ +åĶIJ å®ĩ +ä¸įå¾Ĺ è¶ħè¿ĩ +å¨ģ æħij +æ¡Ĩæŀ¶ åįıè®® +èµ° ç§ģ +åĽ¢ å§Ķ +夸 大 +æ¬ Ħ +ç¥ŀç»ı ç³»ç»Ł +æijĦå½± ä½ľåĵģ +èĬ ¥ +å®ī åºĨ +æµ· 滨 +æŀĦ æĢĿ +çīµ æĮĤ +åı © +éĺIJ æĺİ +éģ ģ +ç²¾ æ²¹ +ç©´ ä½į +æĬ¤ 身 +æĬ¤èº« 符 +æĮĩ å°İ +åŃĺåľ¨ ä¸Ģå®ļ +å¯Ĥ éĿĻ +æµ·å¤ĸ å¸Ĥåľº +éĿ ¡ +综åIJĪ å¾ģ +ä¿ IJ +è¨Ī ç®Ĺ +æĺİ æľĹ +äºļ è¿IJ +äºļè¿IJ ä¼ļ +åīįçŀ» æĢ§ +åĮ® ä¹ı +产ä¸ļ æī¶è´« +èĦij æµ· +èĦijæµ· ä¸Ń +åħļçļĦ é¢Ĩ导 +åĪĺ éĤ¦ +æµģ æĺŁ +æĵ Ĥ +æĶĢ çĻ» +åĴ Ķ +ä¸Ģä¸ĭåŃIJ å°± +è¯Ĭ æ²» +使 åĬ² +åīµ ä½ľ +éĵŃ è®° +éĴ± è´¢ +æĹ¥æĬ¥ è®°èĢħ +çĥŁ çģ« +èĥľ è´Ł +åįļ 主 +ä¸ŃåĽ½ èģĶéĢļ +ç½ijç«Ļ é¦ĸ页 +å°± å¤Ł +å°±å¤Ł äºĨ +æīij åħĭ +å±ħ å§Ķä¼ļ +è° ¬ +å®īåħ¨ äºĭæķħ +åķĨ çĶ¨è½¦ +循çݯ ç»ıæµİ +æ· ¤ +èĢĥ è¯ģ +å®Ŀ èĹı +å®Į ç»ĵ +çłĶåıij æĬķåħ¥ +å² ij +æģŃ æķ¬ +离 éĢĢä¼ij +æ°´ 墨 +å© ¶ +è¯Ĺ åı¥ +å®ģæ³¢ å¸Ĥ +å¼± çĤ¹ +åģľ çīĮ +奶 æ²¹ +å¥ĩ纳 æ²³ +æĨ Ĥ +社ä¼ļ å®ŀè·µ +è´Ŀ 壳 +çłĤ æµĨ +èι åıª +宣 æī¬ +综åIJĪ æķ´æ²» +åĤ ij +æ°ijæĹı æĸĩåĮĸ +éĩį çݰ +积 æ·Ģ +åħ¬ çĦ¶ +çħ ī +缸 èģļ +æ± ¾ +纹 çIJĨ +çĩĥ çħ¤ +æŃ¤ ç§į +ç¾İ å¦Ĩ +åįĥ çĵ¦ +çIJ Ľ +驾驶 è¯ģ +éĺ¶ æ¢¯ +ä¸Ŀ ä¸Ŀ +å¾Īå¤ļ äºĭæĥħ +åħī éĺ´ +èijĹä½ľ æ¬Ĭ +åħ§ éĥ¨ +çĽ¸å¯¹ æĿ¥è¯´ +éĸ Ĵ +éľĩ æħij +說 話 +æĨ ij +ç«¥ è£ħ +ä½ıæĪ¿ åĴĮ +ä½ıæĪ¿åĴĮ åŁİ +å·²ç»ı è¶ħè¿ĩ +侦 å¯Ł +çŁ¿ çī© +ä¾Ľ 大家 +çī¹ éĤĢ +ç¨ĭåºı åijĺ +çķľçī§ ä¸ļ +æ° ª +çij ª +åĢĴ åľ¨ +åĢĴåľ¨ åľ° +æ¯ Ģ +梯 éĺŁ +æİ¥ èijĹ +æĬĹ èıĮ +è¤ ĩ +ç¬ Ļ +æ¯Ķ ä¸Ĭå¹´ +鸡 汤 +åŃ¦ä¹ł æĪIJ绩 +æĸij æĸĵ +åħΠ坼 +åĪĹ ä¸¾ +è°ĥæŁ¥ æĺ¾ç¤º +æ© « +ä¹Ŀ åįģ +è°¢ 飵 +è·¨è¶Ĭ å¼ı +女æĢ§ æľĭåıĭ +èIJ¥åħ» ä»·å̼ +å®ŀè·µ ç»ıéªĮ +èĭı å·ŀå¸Ĥ +çĵ¶ åŃIJ +æĸ° çļĦä¸Ģ +æĸ°çļĦä¸Ģ å¹´ +æĺİ æĻ° +å®ł çα +åŃŠ第 +æľĹ 诵 +纳 æĸ¯ +éĢĨ è¡Į +è«ĭ æĤ¨ +è«ĭæĤ¨ æıIJä¾Ľ +èĥ¸ æĢĢ +第ä¸ĥ å±Ĭ +强 壮 +代 åŃķ +æ±¶ å·Ŀ +å®¶ åĸ» +å®¶åĸ» æĪ· +å®¶åĸ»æĪ· æĻĵ +èħ ® +åIJ¯ 迪 +æĹł éļľç¢į +èĻķçIJĨ åıĬ +æĿ¥ åİĨ +å®ŀ åĬ¡ +ä¹Ł éļıä¹ĭ +æĬĢèĥ½ åŁ¹è®Ń +åѤ ç«ĭ +åī ģ +éĥ´ å·ŀ +æĶ¶ æķĽ +éł» éģĵ +èᣠ幏 +èİ« è¿ĩäºİ +æŃ¤ æĻĤ +纪å§Ķ çĽij +纪å§ĶçĽij å§Ķ +缸 éĤ» +åı¦ä¸Ģ è¾¹ +çªĴ æģ¯ +æľīå¾Īå¤ļ ç§į +æ¯ı éĢ¢ +éĹ® ä¸ĸ +ç´¯ ç´¯ +éĿĴæĺ¥ æľŁ +è·¯ åĨµ +åħĭ èݱ +è¿Ħä»Ĭ 为æŃ¢ +æĥĬ å¥ĩ +è·¨ 度 +éħ¿ éĢł +åĩ ĭ +è¿ij ä¸īå¹´ +åĨħ 马 +åĨħ马 å°Ķ +æı į +è¿Ľå±ķ æĥħåĨµ +èĮ § +æľīåºı æİ¨è¿Ľ +æĢ» åĨłåĨĽ +æĪIJ绩 åįķ +éĽ»è©± åıĬ +ç´§å¯Ĩ ç»ĵåIJĪ +åºĬ ä½į +é¹ Ĭ +æķ£åıij çĿĢ +åĭŁ èµĦ +æ°¨ éħ¸ +彩 ç¥ŀ +è®Ģ åıĸ +éĩį æ¸© +ä¸Ń åŃĺåľ¨çļĦ +ç¾İ éºĹ +ä¸įæĸŃ å¢ŀåĬł +è½® æµģ +æİ¥ åIJ¬ +å¹´ 产å̼ +åįĥ åħĭ +æĪĺåľº ä¸Ĭ +çħ§ é¡§ +å¹²éĥ¨ éĺŁä¼į +åį° ç«ł +ä¸Ģèĩ´ æĢ§ +è¿ŀ å¤ľ +åħħ è£ķ +é»ij åIJįåįķ +åĩĢ æ°´ +ä¸Ģ大 æĹ© +åĮħ 袱 +çĬ¯ è§Ħ +çIJĨ è«ĸ +æŀģ æĺĵ +éª ¸ +å¨ĺ å¨ĺ +åĽ¢ åľĨ +亿åħĥ 以ä¸Ĭ +åĪ©ç͍ æĤ¨çļĦ +带æĿ¥ æĽ´å¤ļ +ä¸Ń央 空è°ĥ +æľĪ èĸª +çĮľ æĥ³ +åĪº 客 +ä½ľ æģ¯ +åįķ è°ĥ +äºĴ åĪ© +å¦Ĥæľī ä¾µæĿĥ +å°ı å·§ +åįģ åł° +åĵĪåĵĪ åĵĪåĵĪ +è¾¹ éĻħ +æłĩ è¯Ń +åĪĩåħ¥ çĤ¹ +éĢĨ è¢Ń +è¯ķ åīĤ +绿 è±Ĩ +è® ļ +åŁºçĿ£ å¾Ĵ +å£ ¬ +åħ¨ æĺİæĺŁ +éĢī ç§Ģ +èĪĮ å°ĸ +ä¸įåIJĮ ç±»åŀĭ +çĥŁ åĽ± +çģµ æ°Ķ +åĮº 管å§Ķä¼ļ +åĨľ åī¯ +åĨľåī¯ äº§åĵģ +èĶļ æĿ¥ +沪 æĮĩ +åħ»æ®ĸ æĪ· +æĸĹ å¿Ĺ +é¦ĸ é¢Ĩ +è¡Ģ èħ¥ +åĬł ç´§ +ä¸Ģèĩ´ 好è¯Ħ +第ä¸ī èĬĤ +æī¬ å°ĺ +交éĢļ æŀ¢çº½ +鼶 ç¢İ +é»ij æ´ŀ +çľĭ ä¸įæĩĤ +å±ŀ å®ŀ +主 åŁİåĮº +å¨ Ľ +å¨Ľ æ¨Ĥ +ç¬ij æĦı +èϹ æ¡¥ +åIJĦ个 çݯèĬĤ +çķ¥ å¾® +èĢķ èĢĺ +æľ¬ åľºæ¯ĶèµĽ +æĪIJ è´¥ +éĢī èĤ¡ +èªŀ è¨Ģ +çŃĶ è¾© +èĩª ä¹ł +æ£ º +ä¸ĩ 欧åħĥ +åģľ å·¥ +对åħ¶ è¿Ľè¡Į +积æŀģ éħįåIJĪ +ä¹¾ åĿ¤ +å¦ĸ æĢª +èļĮ åŁł +èµĦ产 è¯Ħä¼° +è°ĥ çļ® +éϤ å¤ķ +åĽ´ å¢Ļ +æľį å½¹ +æ·± æ¸Ĭ +é¢Ħ åζ +ç ĥ½ +å®ī 稳 +建 æŀĦ +çĭĻ åĩ» +主åĭķ 註åĨĬ +éĥ½æľī èĩªå·± +æİĴåIJį 第ä¸Ģ +麻 è¾£ +çĢ ļ +çĥŁèĬ± çĪĨ +çĥŁèĬ±çĪĨ 竹 +èĩªçĦ¶ ä¿ĿæĬ¤ +ä»Ļ å¢ĥ +为äºĨ éģ¿åħį +åĨ· åºĵ +è§£æĶ¾ æĢĿæĥ³ +åĪĿ äºĮ +ä½ĵ è´´ +é¦ĸ å¯Į +迪 æĭľ +æļĤ ç¼ĵ +æĶ¯æĮģ åĬĽåº¦ +侦 æİ¢ +马 åĪº +åĮĹ æ±½ +ç¹ ŀ +è°İ è¨Ģ +éĢ£ çºĮ +å· ³ +ä»»ä½ķ æĹ¶åĢĻ +车 èģĶç½ij +åįķ 项 +å¸Ń åį· +建çŃij æĿIJæĸĻ +ä¸Ńç§ĭ èĬĤ +ç¡ķ士 çłĶç©¶ +ç§ģ ç«ĭ +åħļåĴĮ æĶ¿åºľ +æľ¬æ¬¡ 交æĺĵ +èººåľ¨ åºĬä¸Ĭ +ç½ijåıĭ è¯Ħ论 +å¦ Ŀ +害 ç¾ŀ +åħ¬ç«ĭ åĮ»éĻ¢ +ä¸ ŀ +çĶŁçī© è´¨ +åºĶ éĤĢ +æĬ½ åıĸ +åĩł å¼ł +æijĺ ç¼ĸ +ç»ĺ æľ¬ +详 è§£ +强 硬 +æľĢ åħĪè¿ĽçļĦ +æĭĽ èĤ¡ +æĭĽèĤ¡ 书 +åįĥ æĸ¹ +åįĥæĸ¹ çϾ +åįĥæĸ¹çϾ 计 +éħį éŁ³ +驾 çħ§ +å¾ģ æĪĺ +èªĵ è¨Ģ +æĭľ å¸Ī +æĭľå¸Ī åѦ +æĭľå¸ĪåѦ èīº +æĬ± åĽ¢ +ç±³ ç²ī +éĿŀ常 éĢĤåIJĪ +èĪª æµ· +å±¥ 约 +åįģåħ« æĿ¡ +éĶ» éĢł +éĩįè¦ģ 举æİª +åıijæĮ¥ ä½ľç͍ +æ· ļ +人 社 +人社 å±Ģ +è¯ķçĤ¹ å·¥ä½ľ +éĺľ éĺ³ +æ¡ĥ åľĴ +æ°ij ä¼ģ +æ´ģ çϽ +è´µ 宾 +åħ¬ 社 +è§ī æĤŁ +è®°å¿Ĩ åĬĽ +æľĥåĵ¡ 註åĨĬ +æŃ¤ æ¡Ī +麻 çĹ¹ +çı Ģ +æĸ© èİ· +çĶ· åŃ©åŃIJ +å±ĢéĻIJ äºİ +åĭĺ æŁ¥ +åIJĥ 饱 +èĬ¬ åħ° +æ£ķ èī² +ç¦ı ç¥ī +çͳ èĬ± +æµ· çĽĹ +èĶ ij +æĸĩ åѸ +æ´»æĢ§ çĤŃ +缴 éĢļ车 +è°¢ éĤĢ +躺 çĿĢ +åľ ĥ +æ¯ıæĹ¥ ç»ıæµİ +åħ¬åħ± æĸĩåĮĸ +讲 æķħäºĭ +å¯Ł çľĭ +æĤł éĹ² +åľ° åĿª +æ¶Į çݰåĩº +é«ĺçŃī éĻ¢æł¡ +èĮĦ åŃIJ +éĺ² åį« +ä¾ĭ è¡Į +æĺ¾ éľ² +æĸ° 常æĢģ +ç»Ŀ ä½³ +å¯Į æ°ij +以 人æ°ij +以人æ°ij 为 +éĤ¢ åı° +å±ķ æ¼Ķ +çϼ å¸ĥ +è´Ł è½½ +åģı 离 +æ°¸ éģł +éĩįè¦ģ åİŁåĽł +åįıä¼ļ ä¼ļåijĺ +éļ¾ æ°ij +çĶŁäº§ 车éĹ´ +çģµ åĬ¨ +两年 åīį +æĸ¹ åľĨ +æ´» ä¸ĭåİ» +ä¸ĸçķĮ è§Ĥ +éªĹ åıĸ +ç¾İ è²Į +èĥ½ çľĭåĩº +çϼ æı® +è§Ĥ å½± +åī ĥ +åIJĪèµĦ åħ¬åı¸ +å© § +å¹² æĹ± +åħŃ ä¸ªæľĪ +尤为 éĩįè¦ģ +èĤ ½ +秦 åĽ½ +æīĺ ç¦ı +建çŃij å¸Ī +åįĩ级 æĶ¹éĢł +å°ı é¢Ŀ +å°ıé¢Ŀ 贷款 +两个 ç»´æĬ¤ +æĭį æĭį +åı¯ çĸij +æį¢ åıĸ +æŃ¦ 士 +èµĸ 以 +èµĸ以 çĶŁåŃĺ +æĮ ļ +殿 åłĤ +èĩªçĦ¶ çķĮ +ç£ģ åľº +å¦Ĥä½ķ çľĭå¾ħ +ä»ĬæĹ¥ 头æĿ¡ +西 åŁŁ +èİ· è¯Ħ +風 æł¼ +ä¿Ħ åĽ½ +æīĵ æĭ¼ +å®£ä¼ł çīĩ +å¾Ī æĸ¹ä¾¿ +ä¾Ľç»Ļ ä¾§ +纪念 ç¢ij +毫 åħĭ +èĬ³ é¦Ļ +å·¥åķĨ éĵ¶è¡Į +请 çĤ¹åĩ» +ç¼ ª +æĹłæķ° 次 +èᝠå¸Ī +èħ ¸ +游 èīĩ +åĮ ¾ +å·¡ èĪª +æ²»çIJĨ ä½ĵç³» +èIJ¥éĢł èī¯å¥½ +æ·· æ·Ĩ +éĢļ çķħ +åĬ³ ç´¯ +ä»ĵ ä½į +å¢ŀ éķ· +éļIJ 约 +æĿĤå¿Ĺ 社 +åħ» èĤ² +åı¯èĥ½ åıijçĶŁ +èĢĥ 試 +西 ä¾§ +åĬł åĢį +主æĮģ åı¬å¼Ģ +çķ¢ ç«Ł +éĹ® 询 +æµ· æ£ł +èĹ © +注æĺİ æĿ¥æºIJ +æ£Ģ çĸ« +请 åģĩ +æĬļ æij¸ +èĵĦ çĶµæ±ł +è·Ł ä¸įä¸Ĭ +çݰ代 社ä¼ļ +çѹ èµĦ +ä½ĵèĤ² 彩票 +å»¶ 误 +è¾Ľ è¾£ +éĿ¢ 容 +åį° è®° +çģŃ äº¡ +ç´ł é£Ł +åħ´ èĩ´ +éľĢè¦ģ ç͍ +éľĢè¦ģç͍ åΰ +å®Ŀ å¦Ī +ç£ĭ åķĨ +éļ¶ å±ŀ +è´¡çĮ® åĬĽéĩı +åħ¬åħ± èµĦæºIJ +大 éĺª +åĨĽ è®Ń +æĤ¬ 念 +社ä¼ļ 稳å®ļ +å¹²äºĭ åĪĽä¸ļ +æľī æĿ¡ä»¶ +æľīæĿ¡ä»¶ çļĦ +ä¸Ģå¹´ ä¸Ģ度 +åİ ¥ +强 奸 +豪 车 +æİĮ æŁľ +æ°´åĪ© å·¥ç¨ĭ +å³ ª +积æŀģ ä½ľç͍ +æµ· æ·Ģ +æµ·æ·Ģ åĮº +çĥŃ æĴŃ +åĿļæĮģ ä¸įæĩĪ +åıĮ èĦļ +绣 æĪĺ +ä»»ä½ķ 人éĥ½ +åľ°ä¸ĭ 室 +åĨ¶ çĤ¼ +è°ħ è§£ +æ¸Ķ èι +太éĺ³ åŁİ +被 æįķ +计ç®Ĺ åύ +西 åĮ» +èĪĴ å¿ĥ +æ¡ ¦ +éģ ² +åĬ ij +è¨ Ĺ +èİ º +åĸ ¬ +çĵ ¯ +åĺ ĺ +åł ķ +æķ Ŀ +åij ¦ +èĭ ŀ +æŃ ¹ +æĵ ¬ +æ£ Ħ +èĪ µ +å¥ ª +çļ ĭ +æĶ ¸ +åľ © +ç¤ Ļ +ç¢ ĺ +éı Ī +æĦ ķ +ç¹ ³ +èĺ ¸ +è² Ĥ +æ¼ ² +æij ¹ +æĶ Ŀ +åŃ ¢ +èķ Ń +é¨ ° +æ½ ¼ +éħ ° +æĴ ¥ +è¹ ¬ +é¨ Ļ +è¸ ¹ +éģ IJ +çĺ Ģ +èĽ ¤ +æĤ ĸ +çĴ ŀ +ç£ IJ +æİ ° +è¾ Ĭ +å¾ ij +æİ ĸ +éģ ŀ +éĤ ¸ +éĽ ı +æĨ İ +æľ ½ +çį » +ç® Ķ +è¤ ¶ +æļ ¢ +æĺ µ +çı Ĥ +æĤ ¸ +åģ µ +åĻ ľ +å£ ¯ +æĴ ® +æģ į +å© ķ +ç¯ ± +éĺ Ļ +çī ł +è£ ĺ +è³ ¢ +éĩ ľ +éĵ ł +èİ ĺ +æ® Ĩ +çĻ ¸ +è´ ı +ç² ± +å« ¡ +åĨ ¢ +è¤ Ĵ +æĩ Ĭ +éľ ĵ +å¡ µ +æĭ £ +å» Ł +é£ ½ +é¢ Į +åļ İ +æ· º +èĨ ł +åİ Ń +åļ ĩ +åij ĥ +çĴ ĭ +çŃ ± +æĭ · +èį § +éĶ ° +åŃ ° +èĵ ĵ +èĨ ½ +æŀ ī +åĸ ½ +çĽ Ķ +çŃ IJ +ç¾ ļ +è ħĮ +è¾ « +æ³ ĵ +çĶ ¬ +èŁ ² +åĸ ª +å¦ ĵ +è¬ Ģ +çĤ Ĭ +æĽ ľ +æ± IJ +è´ Ī +èį Ģ +æĬ ł +ç¢ ¾ +æ« ĥ +éŀ ł +èij Ĩ +ç¥ ¯ +å½ Ŀ +é¦ į +åĮ £ +æľ Ń +åĿ Ĥ +ä¿ ij +èĵ ® +çij Ľ +æī ī +èĩ Ł +è² « +çİ ¥ +æ· ¼ +åİ ² +é³ Į +å³ Ń +åij Ľ +é § +é§ IJ +éģ · +ä¿ ª +æĢ Ĥ +è¾ į +å± į +åĭ ģ +å¥ ļ +éļ ħ +éĴ ´ +è¼ Ŀ +å® ¦ +èIJ ĥ +çĺ ĭ +æĨ ¶ +æĤ ħ +è¾ Ļ +åij ľ +çł º +éĢ ŀ +æµ ļ +éĸ £ +èĸ © +éĻ ĭ +çĤ Ļ +èª ķ +ä¸ Ł +é¹ ½ +ç± Į +è´ ° +éĭ ª +çľ © +æĴ IJ +èĨ º +éŀ ĺ +ç¾ ² +çª ® +ç´ IJ +æ® ´ +çº ¾ +èº į +ç´ ĭ +çĦ ĸ +çĶ º +çī ½ +çĤ ¯ +ç¼ Ķ +æ¯ ĵ +å¬ ° +æ¢ § +äº Ł +è¢ ħ +çį Ħ +è¿ ¥ +æ¼ ¾ +çĿ ij +ç¸ ¾ +é¦ ĭ +é¤ ħ +æ ¹Ħ +æĺ ĩ +æŀ Ń +èĸ ° +æŁ ij +æ¦ » +åĻ Ĺ +åĻ ´ +æ£ £ +åĶ § +çĨ ¹ +è¼ ¯ +å¢ Ł +é² ² +æĪ Ľ +èī ¦ +èĬ ® +åĺ Ł +å¸ ¥ +å¿ » +çĮ Ŀ +å¯ µ +è³ ¦ +èĽ ¾ +æ» ¾ +çĤ ķ +éĵ ¬ +èĴ ¿ +éĴ ¨ +çĥ Ļ +ç² ķ +æĥ ¦ +æº § +é¢ į +éħ £ +å³ ¦ +ç± ģ +çĥ ĥ +åĨ Ĺ +åı ģ +çĽ § +ç½ µ +éĴ Ĺ +å¬ ī +è° ı +ç³ § +è¾ Ń +æ· ¬ +èŁ Ĵ +è¯ © +è¦ ĥ +çĻ ĸ +é½ Ĵ +çĪ IJ +ç® į +ç¼ İ +ç£ º +è¯ « +è¤ ² +æĵ ł +èIJ ¦ +çĿ ¬ +è° į +éĦ ° +æł ¾ +é¡ ı +ç¸ ± +æ¡ ¨ +éĨ ¬ +è¥ ² +è® ª +å© º +èį Ł +åĮ Ŀ +çĨ ł +èĽ Ĭ +æ¸ ļ +å´ ½ +é² ¤ +åķ ° +åĮ ķ +ä¸ IJ +è® ¥ +åı ½ +åı ¼ +çļ ¿ +è¿ Ĥ +åIJ Ĩ +å± ¹ +èĩ ¼ +è® ¹ +é© ® +çº « +æ± ŀ +æĬ ¡ +èĭ ĩ +åIJ ł +åIJ Ń +åIJ ® +å² ĸ +ä½ ĥ +çĭ Ī +åº ĩ +åIJ Ŀ +éĹ ° +æ± ¹ +å¿ ± +æĭ Ħ +æĭ Ĺ +èĮ ī +èĭ Ľ +èĮ ģ +çŁ ¾ +èĻ ı +åij » +åĴ Ħ +å¿ ¿ +èĤ ® +çĭ ŀ +çĸ Ł +çĸ Ļ +çĸ ļ +æ³ ŀ +å¸ ļ +å± ī +è¿ ¢ +é© ¹ +ç İ· +çıĬ ó +çıĬó ł +çıĬół Ħ +çıĬółĦ ģ +æĮ İ +æĭ ´ +åŀ Ľ +èį ¤ +æ® ĥ +çĽ ¹ +åĵ Ĩ +è´ » +æ¯ ¡ +çĭ ° +çĭ ¡ +æŁ Ĵ +æģ ĥ +è¯ ¬ +è¢ Ħ +è¯ ² +èļ ¤ +èĢ Ļ +åŁ Ĥ +æį İ +æį Į +æ¢ Ĩ +é ħĮ +çł ¾ +æ® ī +åĶ ł +æĻ Į +èļ £ +èļ ª +èļ ĵ +é¸ ¯ +åĶ ģ +åĶ Ĩ +åĢ Ķ +èĪ Ģ +è± º +èĥ ° +é¸ µ +é¸ ³ +é¦ ģ +ç¾ Ķ +æ¶ £ +æ¶ ķ +æĤ ¯ +è¯ ½ +è° Ĩ +ç¥ Ł +ç» ¢ +æį º +æį ¶ +æį » +æİ Ĥ +èı ł +èIJ ¤ +éħ Ĺ +çľ ¶ +åķ Ħ +èļ ¯ +èĽ Ģ +åĶ ¬ +å¸ · +éĵ IJ +éĵ Ľ +åģ İ +å¾ Ļ +èĦ ¯ +è± ļ +çĮ ĸ +çĹ Ĭ +æ¶ ® +æĥ Ń +æĤ ´ +æĥ ĭ +è° ļ +æı © +æIJ Ģ +æIJ Ķ +æ¦ Ķ +æ¤ Ń +éĽ ³ +åĸ ³ +è· Ľ +èľ ĵ +èľ Ĵ +é¹ ĥ +éĶ Ħ +çĶ ¥ +çŃ ı +çĮ © +çĮ ¬ +çĮ ¾ +çĹ ¢ +çĹ ª +æĥ ° +çª ĺ +è° ¤ +éļ ĺ +å© ¿ +é¹ ī +çij Ļ +æĸ Ł +æ¤ ¿ +éħ ª +éĽ ¹ +åĹ ¦ +è· · +è· º +è· ¤ +èľ Ī +èľ Ĺ +å¹ Į +é¦ ı +èª Ĭ +æ¼ ĵ +è¤ Ĥ +èĶ Ĺ +èĶ ¼ +åħ ¢ +è£ ³ +èľ » +èĿ ĩ +åĺ Ģ +éĶ ¹ +ç® ķ +ç® © +çĺ © +çĺ Ł +æ¼ ± +å¯ ¥ +éª ¡ +æĴ µ +æĴ ¬ +è± Į +åĺ ¹ +èĿ ł +èĿ Į +èĿ Ĺ +èĿ Ļ +éķ IJ +ç¨ ¼ +ç¯ ĵ +èĨ Ľ +é² « +çĺ ª +é² ¨ +æĨ Ķ +ç¿ © +è¤ ¥ +ç¼ Ń +åĻ © +çĵ ¢ +éľ İ +è¸ ± +è¹ Ĥ +èŁ Ĩ +é¹ ¦ +ç¯ ¡ +çĺ ¸ +çª ¿ +ç¼ ° +èĹ IJ +è¹ ĭ +èŁ ĭ +èŁ Ģ +èµ ¡ +èĩ Ĭ +é³ Ħ +ç³ ł +æĩ ¦ +åļ £ +éķ ° +é³ į +ç° ¸ +çĻ £ +é³ ĸ +é¬ ĵ +èł ķ +éľ ¹ +èº ı +é» ¯ +çĵ ¤ +çŁ Ĺ +ä¹ Ĥ +ä¹ ľ +åħ Ģ +å¼ ĭ +åŃ ij +åŃ ĵ +å¹ º +äº ĵ +å »¿ +ä¸ ı +åį ħ +ä» ĥ +ä» ī +ä» Ĥ +åĪ Ī +çĪ » +åį ŀ +éĹ © +è® £ +å¤ ¬ +çĪ ¿ +æ¯ ĭ +éĤ Ĺ +éĤ Ľ +èī ½ +èī ¿ +åı µ +ä¸ ķ +åĮ ľ +åĬ ¢ +åį Ł +åı ± +åı » +ä» ¨ +ä» Ł +ä» ¡ +ä» « +ä» ŀ +åį ® +æ° IJ +çĬ ° +åĪ į +éĤ Ŀ +éĤ Ļ +è® ¦ +è® § +è® « +å° » +éĺ ¡ +å° ķ +å¼ ģ +èĢ Ĵ +çİ İ +çİ ij +åľ ¬ +æī ¦ +åľ ª +åľ ¹ +æī ª +åľ ® +åľ ¯ +èĬ Ĭ +èĬ į +èĬ Ħ +èĬ ¨ +èĬ ij +èĬ İ +èĬ Ĺ +äº ĺ +åİ į +å¤ ¼ +æĪ į +å° ¥ +ä¹ © +æĹ ¯ +æĽ ³ +å² Į +å± º +åĩ ¼ +åĽ ¡ +éĴ ĩ +ç¼ ¶ +æ° ĺ +æ° ĸ +çī Ŀ +ä¼ İ +ä¼ Ľ +ä¼ ¢ +ä½ ¤ +ä» µ +ä¼ ¥ +ä¼ § +ä¼ ī +ä¼ « +åĽ Ł +æ± Ĩ +åĪ ĸ +å¤ Ļ +æĹ ® +åĪ İ +çĬ · +çĬ ¸ +èĪ Ľ +åĩ « +é Ĥ¬ +é¥ § +æ± Ķ +æ± ľ +æ± Ĭ +å¿ ĸ +å¿ ı +è® ´ +è® µ +è® · +èģ ¿ +èī ® +åİ ¾ +å¦ ģ +çº ¡ +çº £ +çº ¥ +çº ¨ +çİ ķ +çİ Ļ +æĬ Ł +æĬ Ķ +åľ » +åĿ į +æĬ ĥ +ã§ IJ +èĬ « +èĬ ¾ +èĭ Ī +èĭ £ +èĭ ĭ +èĬ ¼ +èĭ Į +èĭ ģ +èĬ © +èĬ ª +èĬ ¡ +èĬ Ł +èĭ Ħ +èĭ İ +èĭ ¡ +æĿ Į +æĿ ĵ +æĿ Ī +å¿ ij +åŃ Ľ +éĤ ´ +éĤ ³ +å¥ ģ +è± ķ +å¿ Ĵ +æ¬ ¤ +è½ « +è¿ ĵ +éĤ ¶ +å¿ IJ +åį £ +éĤ º +æĹ ° +åij ĭ +åij Ĵ +åij ĵ +åij Ķ +åij ĸ +æĹ ¸ +åIJ ¡ +èĻ ¬ +åIJ ½ +åIJ £ +åIJ ² +å¸ ı +å² Ī +å² ĺ +åħ ķ +åĽ µ +åĽ « +éĴ Ĭ +éĴ ĭ +é ĴĮ +è¿ ķ +æ° Ļ +æ° ļ +çī ¤ +ä½ ŀ +ä½ ļ +ä½ Ŀ +ä½ Ĺ +å½ · +ä½ ĺ +ä½ ¥ +è± ¸ +åĿ Į +èĤ Ł +å¥ Ĥ +åĬ ¬ +çĭ ģ +é¸ ł +é¥ ¨ +é¥ © +é¥ « +é¥ ¬ +åº ij +åº ĭ +çĸ Ķ +çĸ ĸ +èĤ ĵ +éĹ ± +éĹ ³ +çĤ Ģ +æ² £ +æ² ħ +æ² Ķ +æ² ¤ +æ² ı +æ² ļ +æ± © +æ± ¨ +æ² ¨ +æ± ´ +æ² Ĩ +æ² © +æ³ IJ +æĢ ĥ +æĢ Ħ +å¿ ¡ +å¿ ¤ +å¿ ¾ +æĢ ħ +å¿ ª +æĢ Ĩ +å¿ Ń +å¿ ¸ +è¯ Ĥ +è¯ ĥ +è¯ ħ +è¯ ĭ +è¯ Į +è¯ Ĵ +éĻ Ĥ +éĻ ī +å¦ © +å¦ ª +å¦ £ +å¦ Ĺ +å¦ « +å§ Ĵ +å¦ ¤ +åĬ Ń +åĪ Ń +éĤ ° +çº Ń +çº ° +çº ´ +çİ ¡ +çİ Ń +çİ ł +çİ ¢ +çİ ¦ +çĽ Ĥ +å¿ Ŀ +åĮ ¦ +åĿ © +æĬ ¨ +æĭ ¤ +åĿ « +æĭ Ī +åŀ Ĩ +æĬ » +åĬ ¼ +æĭ ĥ +æĭ Ĭ +åĿ ¼ +åĿ » +ã§ Ł +åĿ ¨ +åĿ Ń +æĬ ¿ +åĿ ³ +èĭ · +èĭ ¤ +èĮ ı +èĭ « +èĭ ľ +èĭ ´ +èĭ Ĵ +èĭ ĺ +èĮ Į +èĭ » +èĭ ĵ +èĮ ļ +èĮ Ĩ +èĮ ij +èĮ ĵ +èĮ Ķ +èĮ ķ +è ĮĢ +èĭ ķ +æŀ ¥ +æŀ ĩ +æĿ ª +æĿ ³ +æŀ § +æĿ µ +æŀ ¨ +æŀ ŀ +æŀ ĭ +æĿ » +æĿ · +æĿ ¼ +çŁ ¸ +ç łĢ +åĪ ³ +å¥ Ħ +æ® ģ +éĥ ı +è½ Ń +éĥ ħ +é¸ ¢ +çĽ ± +æĺ Ļ +æĿ ² +æĺ ĥ +åĴ Ĥ +åij ¸ +æĺ Ģ +æĹ » +æĺ ī +çĤ ħ +çķ Ģ +èĻ ® +åĴ Ģ +åij · +é» ¾ +åij ± +åij ¤ +åĴ Ĩ +åĴ Ľ +åij ¶ +åij £ +åĴ Ŀ +å² ¢ +å² ¿ +å² ¬ +å² « +å¸ Ļ +å² £ +å³ ģ +åĪ ¿ +å² · +åī Ģ +å¸ Ķ +å³ Ħ +æ² ĵ +åĽ ¹ +ç½ Ķ +éĴ į +éĴ İ +éĴ ı +éĴ Ĵ +éĴ ķ +éĤ ¾ +è¿ ® +çī ¦ +ç« º +è¿ ¤ +ä½ ¶ +ä¾ ij +ä¾ ī +èĩ ¾ +ä¾ Ĺ +ä¾ ı +ä¾ © +ä½ » +ä½ ¾ +ä¾ ª +ä½ ¼ +ä½ ¯ +ä¾ ¬ +å¸ Ľ +ä¾ Ķ +å¾ Ĥ +åĪ ½ +éĥ Ħ +ç± ´ +çĵ ® +æĪ Ĺ +èĤ ¼ +äı Ŀ +èĤ ± +èĤ « +è¿ © +éĥ ĩ +çĭ İ +çĭ į +çĭ Ĵ +åĴ İ +é¥ ¯ +é¥ ´ +åĨ ½ +åĨ ¼ +åº ĸ +çĸ ł +çĸ Ŀ +åħ ĸ +åĬ ¾ +ð¬ ī +ð¬ī ¼ +çĤ ĺ +çĤ Ŀ +çĤ Ķ +æ³ Ķ +æ² Ń +æ³ · +æ³ ± +æ³ ħ +æ³ ł +æ³ º +æ³ ĸ +æ³ « +æ³ ® +æ² ± +æ³ ¯ +æĢ Ļ +æĢ µ +æĢ ¦ +æĢ Ľ +æĢ ı +æĢ į +ã ¤ +㤠ĺ +æĢ © +æĢ « +æĢ ¿ +å® ķ +ç© ¹ +å® ĵ +è¯ ĵ +è¯ Ķ +è¯ ĸ +è¯ ĺ +æĪ ¾ +è¯ Ļ +æĪ ½ +éĥ ĵ +è¡ © +ç¥ Ĩ +ç¥ İ +ç¥ ĩ +è¯ ľ +è¯ Ł +è¯ £ +è¯ ¤ +è¯ § +è¯ ¨ +æĪ ķ +éĻ Ķ +å¦ ² +å¦ ¯ +å§ Ĺ +å¸ ij +åŃ ¥ +é© ½ +èĻ ± +è¿ ¨ +ç» Ģ +ç» ģ +ç» Ĥ +é© · +é© ¸ +ç» ī +ç» Į +éª Ģ +çĶ ¾ +çı ı +çı IJ +çı ij +çİ ³ +é¡ ¸ +çı ī +çı Ī +æĭ ® +åŀ Ń +æĮ Ŀ +æĮ ŀ +åŀ ¤ +èµ ³ +è´ ² +åŀ ± +åŀ Į +åŀ § +åŀ ĵ +æĮ ¦ +åŀ ł +èį ļ +èį ij +è´ ³ +èį ľ +èİ Ĵ +èĮ ¼ +èĮ ´ +èĮ ± +èİ Ľ +èį ŀ +èĮ ¯ +èį ı +èį ĩ +èį ĥ +èį ł +èĮ Ń +åŀ © +èį ¥ +èį ¦ +èį ¨ +èį © +åī ĭ +èį ª +èį ¬ +èį ® +æŁ ° +æł ī +æŁ ĺ +æł Ĭ +æŁ © +æŀ ° +æł Į +æŁ Ļ +æŀ µ +æŀ ³ +æŁ ŀ +æŁ Ŀ +æł Ģ +æŁ ¢ +æł İ +æŁ Ī +æŁ ģ +æŀ · +æŁ ½ +åī Į +éħ Ĭ +éĥ ¦ +çĶ Ń +çł Ĺ +çł ĺ +çł Ĵ +æĸ « +çł Ń +çł ľ +èĢ · +èĻ º +æ® Ĥ +æ® ĩ +æ® Ħ +è½ ± +è½ ² +è½ ³ +è½ ¶ +è½ ¸ +èĻ ¿ +æ¯ ĸ +è§ ĩ +å° ľ +åĵ IJ +çľ Ħ +çľ į +ðł ³ +ðł³ IJ +éĥ ¢ +çľ ĩ +çľ Ĭ +çľ Ī +ç¦ º +åĵ Ĥ +åĴ ´ +æĽ · +æĺ ´ +åĴ ¦ +åĵ ĵ +åĵ Ķ +çķ İ +åij ² +èĥ Ħ +çķ ĭ +çķ Ī +èĻ ¼ +èĻ » +çĽ ħ +åĴ £ +åĵ ķ +åī IJ +éĥ § +åĴ » +åĽ ¿ +åĴ ¿ +åĵ Į +åĵ Ļ +åĵ ļ +åĴ © +åĴ ¤ +åĵ Ŀ +åĵ ı +åĵ ŀ +å³ £ +ç½ ĺ +å³ Ĵ +å³ ¤ +å³ ĭ +è´ ¶ +éĴ ļ +éĴ ¡ +éĴ £ +éĴ ¤ +éĴ « +æ° ¡ +çī ¯ +éĥ ľ +ç§ ķ +ç§ Ń +ç« ½ +ç¬ Ī +ä¿ ¦ +ä¿ ¨ +ä¿ ħ +åı Ł +åŀ ¡ +çī ® +ä¿ £ +ä¿ ļ +çļ Ī +ä¿ Ł +éĢ ħ +å¾ ĩ +å¾ ī +èĪ ¢ +éĥ Ĺ +ä¿ İ +éĥ ¤ +çĪ ° +éĥ Ľ +çĵ ´ +èĥ ¨ +èĥ ª +èĥ Ľ +èĥ Ĥ +èĥ Ļ +èĥ į +èĥ Ĺ +è ĥĿ +æľ IJ +èĥ « +é¸ ¨ +åĮ į +çĭ ¨ +çĭ ¯ +é£ ij +çĭ © +çĭ ² +è¨ ĩ +éĢ Ħ +æĺ Ŀ +é¥ · +é¥ ¸ +é¥ ¹ +åŃ ª +å¨ Ī +åº ¥ +çĸ ¬ +çĸ £ +çĸ ¥ +çĸ Ń +åº ł +ç« ij +é£ Ĵ +éĹ ¼ +éĹ ¾ +éĹ ¿ +éĺ Ĥ +ç¾ ij +è¿ ¸ +ç± ¼ +éħ ĭ +çĤ » +çĥ Ģ +çĤ · +æ´ ± +æ´ ¹ +æ´ § +æ´ Į +æµ ĥ +æ´ ĩ +æ´ Ħ +æ´ Ļ +æ¶ İ +æ´ İ +æ´ « +æµ į +æ´ ® +æ´ µ +æµ Ĵ +æµ Ķ +æµ ķ +æ´ ³ +æģ ¸ +æģ ĵ +æģ ¹ +æģ « +æģ » +æģ Ĥ +æģ ª +æģ ½ +å® ¥ +æī ĥ +è¡ ² +è¡ ½ +è¡ ¿ +è¢ Ĥ +ç¥ ľ +ç¥ ĵ +ç¥ ļ +è¯ ® +ç¥ Ĺ +ç¥ ¢ +è¯ ° +è¯ ³ +é¸ © +æĺ ¶ +åĴ « +å¼ Ń +çī ģ +èĥ ¥ +éĻ Ł +å§ ® +å¨ Ĩ +å§ Ŀ +å§ £ +å§ ĺ +å§ ¹ +ç¾ ¿ +çĤ ± +çŁ ľ +ç» Ķ +éª ģ +éª ħ +ç» Ĺ +ç» Ľ +éª Ī +èĢ ĸ +æĮ Ī +çı ¥ +çı Ļ +é¡ ¼ +çı ° +çı © +çı § +çı £ +çı ŀ +çIJ ¤ +çı ² +æģ ļ +åŁ ķ +åŁ ĺ +åŁ Ļ +åŁ ļ +æĮ ¹ +èĢ Ĩ +èĢ Ħ +åŁ Ĵ +æį ĭ +è´ ½ +åŀ ¸ +æį ĥ +çĽ į +èį ¸ +èİ ³ +èİ ´ +èİ ª +èİ ł +èİ ľ +èİ ħ +èį ¼ +èİ © +èį ½ +èİ ¸ +èį » +èİ ¨ +é¸ ª +èİ ¼ +æł ² +æł ³ +æ¡ ¡ +æ¡ İ +æ¡ ¢ +æ¡ ¤ +æ¢ ĥ +æł Ŀ +æ¡ ķ +æ¡ ģ +æ¡ § +æ¡ ħ +æł Ł +æ¡ ī +æł © +éĢ ij +éĢ ĭ +å½ § +é¬ ² +è± ĩ +éħ IJ +éĢ ¦ +åİ Ŀ +åŃ ¬ +çł Ŀ +çł ¹ +çł § +çł · +çł Ł +çł ¼ +çł ¥ +çł £ +åī ŀ +çł » +è½ ¼ +è½ ¾ +è¾ Ĥ +é¸ « +è¶ ¸ +é¾ Ģ +é¸ ¬ +èĻ Ķ +çľ ¬ +åĶ Ľ +çľ Ļ +åĵ § +åĵ ½ +æĻ ģ +é¸ ® +è¶ µ +è¶ ¿ +çķ Ľ +èļ ¨ +èļ ľ +èļ į +èļ ĭ +èļ ¬ +èļ Ŀ +èļ § +åĶ ¢ +åľ Ħ +åĶ £ +åĶ ı +çĽ İ +åĶ ij +å´ Ĥ +å´ ĥ +ç½ ¡ +ç½ Ł +è§ Ĭ +èµ ħ +éĴ ² +éĴ µ +éĴ ¹ +éĴ º +éĴ ½ +éĴ ¼ +éĴ ¿ +éĵ Ģ +éĵ Ħ +éĵ Ĩ +éĵ Ī +éĵ ī +éĵ Ĭ +éĵ ĭ +éĵ Į +é ĵį +ä ¥ +ä¥ ½ +éĵ İ +æ° © +æ° ¤ +æ° ¦ +æ¯ ª +èĪ IJ +ç§ £ +ç§ « +çĽ ī +ç¬ Ħ +ç¬ ķ +ç¬ Ĭ +ç¬ ı +ç¬ Ĩ +ä¿ ¸ +ä¿ µ +åģ Į +ä¿ ³ +ä¿ ¶ +åĢ ¬ +åĢ ı +æģ ģ +åĢ Ń +ä¿ ¾ +åĢ ľ +éļ ¼ +éļ ½ +åĢ Į +åĢ ¥ +èĩ ¬ +éĥ « +åĢ ¨ +è¡ Ħ +é¢ Ģ +å¾ ķ +èĪ « +è¡ ¾ +èĥ ¯ +èĥ ± +èĥ ´ +èĥ Ń +èĦ į +èĥ ¼ +èĦ Ĵ +é¸ ± +é¸ ² +çĭ · +çĮ ģ +çĭ ³ +çĮ ĥ +çĭ º +éĢ ĸ +æ¡ Ģ +é¥ ½ +åĩ ĩ +æĮ Ľ +äº ³ +çĸ ³ +çĸ ´ +çĸ ¸ +çĸ ½ +çĹ Ī +çĸ ± +çĹ Ĥ +çĹ ī +è¡ ® +é¢ ĥ +æģ £ +æĹ Ĩ +æĹ Ħ +æĹ ĥ +éĺ ĥ +éĺ Ħ +è¨ ļ +éĺ Ĩ +æģ Ļ +ç² ij +çĥ ľ +çĥ © +çĥ Ĭ +åī ¡ +éĥ ¯ +çĥ ¬ +æ¶ ij +æµ ¯ +æ¶ ŀ +æ¶ Ł +å¨ ij +æ¶ ł +æµ ŀ +æ¶ ĵ +æµ ¥ +æ¶ Ķ +æµ ľ +æµ ł +æµ £ +æĤ ļ +æ ĤŃ +æĤ Ŀ +æĤ Ĵ +æĤ Į +æĤ Ľ +çª Ī +åī ľ +è¯ ¹ +è¯ ¼ +è¢ Ĵ +è¢ ¢ +è¯ ¿ +è° Ģ +è° Ĥ +è° Ħ +è° ĩ +å± IJ +å± Ļ +éĻ ¬ +åĭ IJ +å¥ ĺ +çī Ĥ +èļ © +éĻ ² +å¨ Į +å¨ ī +å¨ ² +å¨ ´ +å¨ £ +å¨ ĵ +å© Ģ +çķ ļ +éĢ ¡ +ç» ł +éª Ĭ +ç» ¡ +éª ĭ +ç» ¦ +ç» ¨ +éª İ +éĤ ķ +é¸ ¶ +å½ Ĺ +èĢ ľ +çĦ ĺ +èĪ Ĥ +çIJ ı +çIJ ĩ +éº ¸ +æı ¶ +åŁ ´ +åŁ ¯ +æį ¯ +æİ ³ +æİ ´ +åŁ ¸ +åŁ µ +èµ § +åŁ ¤ +æį Ń +éĢ µ +åŁ Ŀ +åł ĭ +åł į +æİ ¬ +é¸ · +æį ½ +æİ Ĭ +åł ī +æİ ¸ +æį © +æİ ® +æĤ « +åŁ Ń +åŁ ½ +æİ ĩ +æİ ¼ +èģ ĥ +èIJ ģ +èı ĺ +åł ĩ +èIJ ĺ +èIJ ĭ +èı ½ +èı ĸ +è IJľ +èIJ ¸ +èIJ ij +æ£ » +èı Ķ +èı Ł +èIJ ı +èı ¹ +èı ª +èı ħ +èı Ģ +èı ° +èı ¡ +æ¢ ¿ +æ¢ ı +è§ ĭ +æ¡ ´ +æ¡ · +æ£ ģ +æ¡ « +æ£ Ĥ +åķ ¬ +éĥ ¾ +æķ ķ +è± ī +éĦ Ħ +éħ ŀ +ç¡ İ +ç¡ Ń +ç¡ ĸ +ç¡ Ĺ +ç¡ IJ +ç¡ ĩ +ç¡ Į +é¸ ¸ +çĵ ł +åĮ ı +åİ © +æ® Ĵ +æ® ĵ +æ® į +èµ ī +éĽ © +è¾ Ħ +åł ij +çľ Ń +çľ ¦ +åķ § +æĻ ¡ +æĻ ¤ +çľ µ +åľ Ĭ +åĸ ı +åķ ī +åĭ ĸ +æĻ ŀ +åĶ µ +æĻ Ĺ +åķ Ń +çķ ¦ +è¶ º +åķ ® +è· Ħ +èļ ¶ +è ĽĦ +èĽ İ +èĽ Ĩ +èļ ° +åľ ī +èļ ± +èĽ ī +èĽ ı +èļ ´ +åķ ģ +åķ ķ +åĶ ¿ +åķ IJ +åĶ ¼ +åĶ · +åķ ĸ +åķ µ +åķ ¶ +åķ · +åĶ ³ +åĶ ° +åķ ľ +å¸ » +å´ ļ +å´ ¦ +å¸ ¼ +å´ ® +å´ ¤ +å´ Ĩ +èµ ĩ +èµ Ī +èµ Ĭ +éĵ ij +éĵ Ĵ +éĵ Ĺ +éĵ Ļ +éĵ Ł +éĵ ¡ +éĵ ¢ +éĵ £ +éĵ ¤ +éĵ § +éĵ ¨ +éĵ © +éĵ ª +éĵ « +éĵ ¯ +éĵ ° +éĵ ± +éĵ ³ +éĵ µ +éĵ · +çī ¾ +é¸ ¹ +ç§ ¾ +éĢ ¶ +ç¬ º +çŃ ĩ +ç¬ ¸ +ç¬ ª +ç¬ ® +ç¬ ł +ç¬ ¥ +ç¬ ¤ +ç¬ ³ +ç¬ ¾ +ç¬ ŀ +åģ ¾ +åģ ĥ +åģ ķ +åģ Ī +åĤ Ģ +åģ ¬ +åģ » +çļ ij +çļ İ +é¸ » +å¾ ľ +èĪ ¸ +èĪ » +èĪ ´ +èĪ · +é¾ Ľ +ç¿ İ +èĦ ¬ +èĦ ĺ +èĦ ² +åĮ IJ +çĮ Ĺ +çĮ ¡ +çĮ ŀ +æĸ Ľ +çĮ ķ +é¦ Ĺ +é¦ ĥ +é¦ Ħ +é¸ ¾ +åº ¹ +åº ¾ +çĹ Ķ +çĹ į +ç¿ Ĭ +æĹ Į +æĹ İ +è¢ ¤ +éĺ ĩ +éĺ Ī +éĺ ī +éĺ Ĭ +éĺ ĭ +éĺ į +éĺ ı +ç¾ Ł +ç² Ŀ +çĦ IJ +çĦ ĵ +çĦ Ĺ +æ· ħ +æ· ŀ +æ¸ İ +æ¶ ¿ +æ· ĸ +æĮ ² +æ· ł +æ¶ ¸ +æ¸ ij +æ· ¦ +æ· Ŀ +æ¶ ª +æ· Ļ +æ¶ « +æ¸ Į +æĤ » +æĤ ± +æ ĥĿ +æĥ ĺ +æĥ Ĩ +æĥ ļ +æĥ ĩ +æĥ ® +çª ķ +è° Į +æī Ī +çļ ² +è° ij +è£ Ĩ +è¢ · +è£ ī +è° Ĵ +è° Ķ +è° ķ +è° ĸ +è° Ĺ +è° Ļ +è° Ŀ +éĢ ¯ +éĥ ¿ +éļ Ī +ç² ľ +éļ į +éļ Ĺ +å© Ĭ +å¨ ¼ +å© ¢ +å© µ +èĥ ¬ +è¢ Ī +ç¿ Į +æģ ¿ +æ¬ ¸ +ç» « +éª IJ +ç» ¯ +ç» ± +éª Ĵ +ç» ² +éª ĵ +ç» ¶ +ç» º +ç» » +ç» ¾ +éª ĸ +ç¼ ģ +èĢ ł +çIJ « +çIJ µ +çIJ ¶ +çIJ ¥ +çIJ ¨ +çIJ ° +çIJ ® +çIJ ¯ +çIJ ¬ +çIJ ļ +è¾ ĩ +é¼ ĭ +æı ³ +åł ŀ +æIJ ½ +æı ¸ +æı ł +åł Ļ +è¶ Ħ +æı ĸ +é¢ ī +å¡ Ħ +æı ¿ +èĢ ĭ +æı Ħ +èĽ © +èĽ ° +å¡ Ĩ +æij Ĵ +æı Ĩ +æİ ¾ +èģ Ĵ +èij ij +èij ļ +éĿ ° +éĿ ¸ +èij ³ +èij º +èij ¸ +èIJ ¼ +èij ¶ +è ĴĮ +èij Ń +æ¥ ® +æ £¼ +æ¤ Ł +æ£ ¹ +æ¤ ¤ +æ£ ° +èµ į +æ¤ ĭ +æ¤ ģ +æ¤ ª +æ¤ IJ +é¹ ģ +éħ ¤ +éħ ¢ +éħ ¡ +é¹ Ĥ +æ® ļ +æ® Ľ +éĽ ± +è¾ ĭ +æ¤ ł +è¾ İ +çĿ Ħ +çĿ ĩ +çĿ ĥ +æĪ ¢ +åĸ ĭ +åĹ Ĵ +åĸ ĥ +åĸ ± +åĸ ¹ +æĻ · +åĸ Ī +è· ĸ +è· Ĺ +è· ŀ +è· ļ +è· İ +è· ı +è· Ĩ +èĽ ± +èĽ ² +èĽ Ń +èĽ ³ +èĽ IJ +èĽ Ķ +èĽ ŀ +èĽ ´ +èĽ ĺ +åĸ ģ +åĸ Ł +åķ ¾ +åĹ ĸ +åĸ ij +åĹ Ł +åĹ ŀ +åĸ Ļ +åµ ĺ +åµ ĸ +å´ ´ +éģ Ħ +è© Ī +åµ İ +å µ¬ +åµ Ľ +åµ ¯ +åµ Ŀ +åµ « +å¹ Ħ +åµ ĭ +èµ ķ +éĵ » +éĵ ¼ +éĵ ¿ +éĶ ĥ +éĶ Ĩ +éĶ ĩ +éĶ ī +éĶ ı +éĶ ij +éĶ Ĵ +éĶ Ķ +éĶ ķ +æİ £ +çŁ ¬ +æ° ° +æ¯ ³ +æ¯ ½ +çĬ Ĭ +çĬ Ħ +çĬ ĭ +é ¹Ħ +çĬ į +åµ ĩ +é» į +ç¨ ĥ +ç¨ Ĥ +çŃ ļ +çŃ µ +çŃ Į +åĤ £ +åĤ Ī +èĪ Ħ +çī į +åĤ ¥ +åĤ § +éģ ij +åĤ © +å¾ ¨ +åª Ń +çķ ² +å¼ ij +ç¿ ķ +é¹ Ĩ +èħ Ī +èħ ĵ +èħ Ĩ +èħ ´ +èħ ļ +èħ ± +é± ¿ +é² Ģ +é² Ĥ +çĮ ¢ +çĮ ¹ +çĮ ¥ +é£ ĵ +è§ ŀ +è§ ļ +çĮ ± +é¢ İ +é£ § +é¦ ĩ +é¦ Ĭ +äº µ +èĦ Ķ +è£ Ĵ +çĹ £ +çĹ ¨ +çĹ ¦ +çĹ ŀ +çĹ ¤ +çĹ § +èµ ĵ +ç« ¦ +çĵ ¿ +åķ » +é¢ ı +é¹ ĩ +éĺ ij +éĺ Ĵ +éĺ ķ +ç² ŀ +éģ Ĵ +åŃ ³ +çĦ ¯ +çĦ ľ +çĦ ± +é¹ Ī +æ¸ « +æ¹ ® +æ¹ İ +æ¹ ľ +æ¹ į +æ¹ « +æº ² +æ¹ Ł +æº Ĩ +æ¹ ² +æ¹ Ķ +æ¹ ī +æ¸ ¥ +æ» ģ +æĦ ł +æĥ º +æĦ ¦ +æĥ ´ +æĦ Ģ +æĦ İ +æĦ Ķ +åĸ ¾ +å¯ IJ +è° Ł +è£ ¢ +è£ İ +è£ ¥ +ç¥ ¾ +è° ł +è° ¡ +è° ¥ +è° § +åŃ ± +å¼ ¼ +å· ½ +éª ĺ +åª ª +å· ¯ +ç¿ ļ +çļ ´ +éª Ľ +ç¼ Ĥ +ç¼ ĥ +ç¼ Ħ +å½ ĺ +ç¼ ĩ +ç¼ Ī +ç¼ Į +ç¼ ij +ç¼ Ĵ +ç¼ Ĺ +é£ ¨ +èĢ ¢ +çij ģ +çij Ĺ +çij Ħ +éģ ¨ +éª ľ +éŁ « +é« ¡ +å¡ ¬ +éĦ ¢ +è¶ Ķ +è¶ ij +æij ħ +æij ģ +èľ ĩ +æIJ ĭ +æIJ ª +æIJ IJ +æIJ Ľ +æIJ ł +æij Ī +å½ Ģ +æ¯ Ĥ +æIJ ¦ +æIJ ¡ +èĵ ģ +æĪ ¡ +è ĵį +éĦ ŀ +èĵ IJ +èĵ ¦ +é¹ ĭ +èĴ ½ +èĵ ĸ +èĵ Ĭ +èĴ ¯ +èĵ Ł +èĵ ij +èĴ º +èĵ ł +èĴ Ł +èĴ ¡ +èĴ ¹ +èĴ ´ +èĴ Ĺ +èĵ ¥ +æ¥ Ķ +æ¥ Ĥ +æ¥ Ŀ +æ¥ « +æ¥ ¸ +æ¤ ´ +æ§ Į +æ¥ ¯ +çļ Ļ +æ¦ Ī +æ§ İ +æ¦ ī +æ¥ ¦ +æ¥ £ +æ¥ ¹ +æ¤ ½ +åī ½ +éħ © +èľ ĥ +ç¢ Ľ +ç¢ ĵ +ç¡ ¼ +ç¢ ī +ç¢ ļ +ç¢ ĩ +ç¢ ľ +é¹ Į +è¾ ı +é¾ ĥ +é¾ ħ +è¨ ¾ +ç² ² +çĿ ļ +åĹ ª +éŁ ª +åĹ · +åĹ ī +çĿ ¨ +çĿ ¢ +éĽ İ +çĿ ¥ +åĹ ij +åĹ « +åĹ ¬ +åĹ Ķ +åĹ Ŀ +æĪ ¥ +åĹ Ħ +çħ ¦ +æļ Ħ +éģ ¢ +æ ļĮ +è· ¬ +è· ¶ +è ·¸ +è· IJ +è· £ +è· ¹ +èĽ ¸ +èľ Ĭ +èľ į +èľ ī +èľ £ +çķ ¹ +èĽ ¹ +åĹ ¥ +åĹ ² +åĹ ³ +åĹ Į +åĹ į +åĹ IJ +åĹ ¤ +åĹ µ +ç½ ¨ +åµ Ĭ +åµ ´ +éª ° +éĶ Ĺ +éĶ Ľ +éĶ ľ +éĶ Ŀ +éĶ ŀ +éĶ Ł +éĶ ¢ +éĶ ¨ +éĶ © +éĶ Ń +éĶ ± +éĽ ī +æ° ² +çĬ ı +æŃ ĥ +ç¨ ŀ +ç¨ Ĺ +ç¨ Ķ +çŃ ł +çŃ ¢ +çŃ ® +çŃ ² +çī Ĵ +æķ « +å¾ Ń +æĦ Ĩ +èī Ħ +è§ İ +æ¯ ¹ +è² Ĭ +è² ħ +è² ī +é¢ Ķ +èħ ł +èħ © +èħ ¼ +èħ Ń +è ħ§ +å¡ į +åª µ +é² ħ +é² Ĩ +é² ĩ +é² Ī +é² ĭ +é² IJ +èĤ Ħ +é¹ IJ +é£ ķ +è§ ¥ +éģ Ľ +é¦ IJ +é¹ ij +äº ¶ +çĺ ĥ +çĹ ± +çĹ ¼ +çĹ ¿ +çĺ IJ +çĺ ģ +çĺ Ĩ +éº Ĥ +æŃ Ĩ +æĹ Ĵ +éĺ ĸ +éĺ Ĺ +ç¾ § +è± ¢ +ç² ³ +çĮ · +çħ ³ +çħ ¨ +çħ ħ +çħ Ĭ +çħ ¸ +çħ º +æ» Ł +æº ± +æº ĺ +æ¼ Ń +æ» ¢ +æº ¥ +æº ½ +è£ Ł +æº » +æº · +æ» Ĺ +æ» « +æº ´ +æ» ı +æ» ĥ +æ» ¦ +æº ı +æ» Ĥ +æ» ĵ +æº Ł +æ» ª +æĦ « +æħ Ĭ +é² İ +éª ŀ +çª ł +çª £ +è£ ± +è£ ¨ +è£ ¾ +è£ ° +ç¦ Ĭ +è° © +è° ª +åª ¾ +å« « +åª ² +å« Ĵ +å« Ķ +åª ¸ +ç¼ Ļ +ç¼ ľ +ç¼ Ľ +è¾ Ķ +éª Ŀ +ç¼ Ł +ç¼ ¡ +ç¼ ¢ +ç¼ £ +éª Ł +èĢ ¥ +çĴ Ī +çij Ń +çį Ĵ +è§ ı +æħ Ŀ +å« ł +åı Ĩ +æij ½ +å¢ ģ +æĴ Ĥ +æij ŀ +æĴ Ħ +ç¿ ¥ +è¸ ħ +æij Ń +å¢ ī +å¢ Ĵ +æ¦ ĸ +ç¶ ¦ +èĶ « +èĶ · +éĿ º +éĿ ¼ +éŀ ħ +éĿ ¿ +çĶ į +èĶ ¸ +èĶ Ł +èĶ º +æĪ ¬ +èķ ĸ +èĶ » +èĵ ¿ +æĸ ¡ +é¹ ķ +èĵ ¼ +æ¦ Ľ +æ¦ § +æ¦ « +æ¦ Ń +æ§ Ķ +æ¦ ± +æ§ ģ +æ§ ł +æ¦ · +åĥ ° +éħ ½ +éħ ¹ +ç¢ ¡ +ç¢ ´ +ç¢ £ +ç¢ ² +èĩ § +è± ¨ +æ® ¡ +éľ ģ +èľ ļ +é¾ ĩ +é¾ Ī +ä ģ +äģ ĸ +çĿ ½ +åĺ ŀ +åĺ Ī +åĺ Į +åĺ ģ +æļ Ŀ +è¸ Į +è¸ ī +èľ ŀ +èľ ¥ +èľ ® +èĿ Ī +èľ ´ +èľ ± +èľ © +èľ · +èľ ¿ +èŀ Ĥ +èľ ¢ +åĺ ¡ +é¹ Ĺ +åĺ £ +åĺ ¤ +åĺ ļ +åĹ ¾ +åĺ § +ç½ ´ +ç½ ± +å¹ Ķ +å¶ Ĥ +å¹ Ľ +èµ Ļ +ç½ Ĥ +éª · +éª ¶ +é¹ ĺ +éĶ ² +éĶ ´ +éĶ ¶ +éĶ · +éĶ ¸ +éĶ µ +éķ Ĥ +çĬ Ĵ +ç® IJ +ç® ¦ +ç® § +ç® ¸ +ç® ¬ +ç® ħ +ç® ª +ç® ľ +ç® ¢ +ç® ĵ +åĥ ĸ +åĦ Ĩ +åĥ ³ +åĥ Ń +åĬ ģ +åĥ ® +éŃ ĥ +éŃ Ĩ +çĿ ¾ +èī ĭ +éĦ ± +èĨ Ī +èĨ ij +é² ij +é² Ķ +é² ļ +é² Ľ +é² Ł +çį IJ +è§ « +éĽ Ĵ +å¤ ¤ +é¦ ij +éĬ ® +å¡ ¾ +çĺ Į +çĺ Ĭ +çĺ ĺ +çĺ Ļ +æĹ ĸ +èĨ Ĥ +éĺ ļ +éĦ ¯ +é² ŀ +ç² ¿ +ç² ¼ +ç³ ģ +æ§ Ĭ +é¹ ļ +çĨ ĺ +çĨ ¥ +æ½ ¢ +æ¼ ķ +æ» ¹ +æ¼ ¯ +æ¼ ¶ +æ½ ĭ +æ½ ´ +æ¼ ª +æ¼ ī +æ¼ © +æ¾ ī +æħ µ +æIJ ´ +çª ¨ +å¯ ¤ +ç¶ ® +è° ® +è¤ ¡ +è¤ Ļ +è¤ ĵ +è¤ Ľ +è¤ Ĭ +è° ¯ +è° ° +è° ² +å± £ +é¹ Ľ +å« ± +å« ĸ +å« ¦ +å« ļ +å «ĺ +é¼ IJ +çŀ Ģ +é¹ ľ +éª ł +ç¼ ¥ +ç¼ ¦ +ç¼ § +ç¼ ¨ +éª ¢ +ç¼ « +èĢ ¦ +èĢ § +çĴ ľ +çĴ İ +çĴ ģ +å¥ Ń +é« ¯ +é« « +æĴ · +æĴ ħ +èµ Ń +æĴ ¸ +éĭ Ĩ +æĴ Ļ +æĴ º +å¢ Ģ +èģ © +è§ IJ +éŀ ij +èķ Ļ +éŀ Ĵ +èķ Ī +èķ ¨ +èķ ¤ +èķ ŀ +èķ º +çŀ ¢ +èķ ĥ +èķ ² +èµ ľ +æ§ ¿ +æ¨ ¯ +æ§ Ń +æ¨ Ĺ +æ¨ ĺ +æ§ ² +éĨ Į +éĨ ħ +éĿ ¥ +éŃ ĩ +é¤ į +ç£ Ķ +ç£ Ļ +éľ Ī +è¾ ĺ +é¾ ī +é¾ Ĭ +è§ ij +çŀ Į +ç ŀĭ +çŀ ij +åĺ Ń +åĻ İ +åĻ ¶ +é¢ Ļ +æļ ¹ +åĻ ĺ +è¸ Ķ +è¸ Ŀ +è¸ Ł +è¸ Ĵ +è¸ ¬ +è¸ ® +è¸ ¯ +è¸ º +è¸ ŀ +èĿ ½ +èĿ ¾ +èĿ » +èĿ ° +èĿ ® +è ŀĭ +èĿ ĵ +èĿ £ +è Ŀ¼ +åĺ ¬ +é¢ ļ +åĻ į +åĻ Ļ +åĻ Į +åĻ Ķ +é¢ Ľ +å¹ ŀ +å¹ ¡ +å¶ Ļ +å¶ Ŀ +éª º +éķ Ĭ +éķ ī +éķ Į +éķ ı +éķ Ĵ +éķ ĵ +éķ Ķ +ç¨ · +ç® ´ +ç¯ ij +ç¯ ģ +ç¯ Į +çī ĸ +åĦ ĭ +èĻ ¢ +é¹ ŀ +èĨ ĺ +é² ł +é² ¡ +é² ¢ +é² £ +é² ¥ +é² § +é² © +çį Ĺ +çį ł +è§ ¯ +é¦ ĵ +é¦ Ķ +éº ¾ +å» Ľ +çĺ Ľ +çĺ ¼ +çĺ ¢ +çĺ ł +é½ ij +ç¾ ° +𥠻 +ð¥» Ĺ +ç³ Į +ç³ į +ç³ ħ +çĨ ľ +ç Ĩµ +æ¾ į +æ¾ Į +æ½ ¸ +æ½ ¦ +æ½ ² +éĭ Ī +æ½ Ł +æ½ º +å¯ ® +çª ³ +è° ³ +è¤ ´ +è¤ Ł +è¤ « +è° µ +çĨ ¨ +å± ¦ +åĭ ° +æĪ ® +èĿ ¥ +ç¼ ¬ +ç¼ ® +ç¼ ¯ +éª £ +çķ ¿ +èĢ © +èĢ ¨ +èĢ ª +çĴ Ł +éĿ Ľ +çĴ ł +çĴ ĺ +èģ ± +èŀ ¯ +é« » +é« Ń +é« ¹ +æĵ Ģ +çĶ ı +æĵ ŀ +ç¸ ł +ç£ ¬ +é¢ ŀ +èķ » +é¢ Ł +èĸ ¤ +èĸ ¨ +æª ł +èĸ ı +èĸ ® +èĸ ľ +èĸ ħ +æ¨ ¾ +æ© Ľ +æ© ĩ +æ¨ µ +æª İ +æ© ¹ +æ¨ ½ +æ¨ ¨ +æ© ¼ +å¢ ¼ +æ© IJ +ç¿ ® +éĨ IJ +éĨ į +éĨ ļ +ç£ ² +èµ Ŀ +æ® ª +éľ ı +éĮ ¾ +è¾ ļ +éģ ½ +æ° ħ +çŀ Ł +çŀ ł +çŀ ° +åļ Ħ +åļ Ĩ +åĻ ¤ +æļ ¾ +è¹ Ģ +è¸ µ +è¸ ½ +è¹ ī +è¹ ģ +èŀ ¨ +èŀ Ī +èŀ ħ +èŀ Ń +èŀ ł +èŀ Ł +åĻ ± +åĻ « +åĻ » +åĻ ¼ +ç½ ¹ +åľ ľ +ä ¦ +ä¦ ĥ +éķ Ĺ +éķ ĺ +éķ ļ +éķ Ľ +éķ Ŀ +éķ ŀ +éķ ł +æ° ĩ +æ° Ĩ +ç© ij +ç¯ Ŀ +ç¯ ¥ +ç¯ ¦ +ç¯ ª +ç¯ Ļ +çĽ ¥ +åĬ ĵ +ç¿ ± +éŃ ī +éŃ Ī +å¾ ¼ +æŃ Ļ +èĨ ¦ +èĨ Ļ +é² ® +é² ± +é² ³ +é² ´ +é² µ +é² · +é² » +çį ´ +çį Ń +çį ¬ +éĤ Ĥ +é¹ § +å» ¨ +èµ Ł +çĺ ° +å» ª +çĺ ¿ +çĺ µ +çĺ ´ +çĻ ĥ +çĺ ³ +éº ĩ +éº Ī +å ¬´ +å£ ħ +ç³ Ĺ +çĶ ij +çĩ İ +çĩ ł +çĩ Ķ +çĩ § +æ¿ ij +æ¿ ī +æ½ ŀ +æ¾ § +æ¾ ¹ +æ¾ ¥ +æ¾ ¶ +æ¿ Ĥ +è¤ ° +çª ¸ +å¬ ĸ +çĬ Ł +éļ ° +å¬ Ĺ +é¢ ¡ +ç¼ ± +ç¼ ² +ç¼ ³ +çĴ © +çĴ ª +èŀ « +æĵ ¤ +å£ ķ +è§ ³ +ç½ Ħ +æĵ ¢ +èĸ ¹ +éŀ ¡ +éŀ ¬ +èĸ · +èĹ ĵ +èĹ ģ +æª Ħ +æª © +æĩ ĭ +éĨ ¢ +ç¿ ³ +ç¤ ħ +ç£ ´ +é¹ © +é¾ ĭ +é¾ Į +è± ³ +å£ ij +é» » +åļ ı +åļ ħ +è¹ ij +è¹ Ĵ +è¹ Ĭ +è Ł¥ +èŀ ¬ +èŀ µ +çĸ ĥ +èŀ ³ +èŁ ij +åļ ĵ +ç½ ½ +ç½ ¾ +å¶ · +é» ľ +é» Ŀ +é« ģ +é« Ģ +éķ ¡ +éķ ¢ +éķ £ +éķ ¦ +éķ § +éķ © +éķ ª +éķ « +ç½ ħ +ç° Į +ç¯ ¾ +ç¯ ¼ +ç° ĸ +ç° ĭ +é¼ ¢ +åĦ ¡ +é¹ ª +é¼ ¾ +çļ ¤ +éŃ į +é¾ ł +ç¹ ĩ +è² ĺ +éĤ Ī +è² Ķ +èĩ Į +èĨ » +èĩ Ĩ +èĩ ĥ +é² ¼ +é² ½ +é³ Ģ +é³ ĥ +é³ ħ +é³ ĩ +é³ Ĭ +èŀ ½ +çĩ ® +é¹ « +ç³ ľ +ç¸ » +çĻ į +éº ĭ +æĩ ij +æ¿ ¡ +æ¿ ® +æ¿ ŀ +æ¿ ł +æ¿ ¯ +è¹ ĩ +è¬ ĩ +éĤ ĥ +è¥ ģ +æª Ĺ +æ ĵĺ +åŃ º +éļ ³ +å¬ · +èŁ Ĭ +é¹ ¬ +éį ª +éı Ĭ +é¬ Ī +é¬ ĥ +çŀ ½ +éŀ ¯ +éŀ ¨ +éŀ « +éŀ § +éŀ £ +èĹ ľ +èĹ ł +éĨ ª +è¹ Ļ +ç¤ ĵ +çĩ ¹ +é¤ ® +çŀ ¿ +æĽ Ľ +é¢ ¢ +èº ĩ +è¹ ļ +èŁ Ľ +èŁ ª +èŁ ł +èŁ ® +é¹ ® +é» ł +é» Ł +é« ħ +é« Ĥ +éķ ¬ +éķ Ń +éķ ¯ +é¦ ¥ +ç° Ł +ç° ª +é¼ ¬ +éĽ ł +èī Ł +é³ İ +é³ ı +é³ IJ +çĻ ŀ +çĻ Ķ +ç³ ¨ +è¹ © +éİ ı +éĤ ĭ +é¬ ı +æĶ ī +éŀ ² +éŀ ´ +èĹ ¿ +èĺ § +èĺ ħ +éĨ ® +éĨ ¯ +éħ ĥ +éľ ª +éľ Ń +éľ ¨ +é» ¼ +åļ ¯ +è¹ ° +è¹ ¶ +è¹ ½ +è¹ ¼ +è¹ ´ +è¹ ¾ +è¹ ¿ +èł ĸ +èł ĵ +èŁ ¾ +èł Ĭ +é» ¢ +é« ĭ +é« Į +éķ ² +ç± Ģ +é½ ģ +éŃ ij +èī ¨ +é³ ĵ +é³ Ķ +é³ ķ +é³ Ĺ +é³ Ļ +éı ĸ +ç¾ ¸ +㸠Ĩ +çĢ £ +çĢ Ľ +è¥ ¦ +è° ¶ +è¥ ŀ +éª ¥ +ç¼ µ +çĵ Ĵ +æĶ ĺ +èĺ © +èĺ ĸ +éĨ ´ +éľ ° +éħ Ĩ +çŁ į +èº ħ +é¼ į +å· ī +é» © +é» ¥ +é» ª +éķ ³ +éķ ´ +é» § +çº Ĥ +çĴ º +é¼ ¯ +èĩ ľ +é³ ľ +é³ Ŀ +é³ Ł +çį ¾ +åŃ Ģ +éª § +ç ĵĺ +é¼ Ļ +éĨ º +ç¤ ´ +é¢ ¦ +æĽ © +é³ ¢ +éº Ŀ +å¤ Ķ +çĪ Ŀ +çģ ı +ç¦ ³ +éIJ ¾ +ç¾ ¼ +èł ¡ +èĢ ± +é¹ ³ +æ° į +é¥ ķ +èº IJ +é« ij +éķ µ +ç© ° +é¥ Ķ +é¬ » +é¬ Ł +è¶ ± +æĶ « +æĶ ¥ +é¢ § +èº ľ +é¼ ¹ +çĻ ¯ +èł ² +èł ¹ +èº ŀ +è¡ ¢ +çģ ŀ +è¥ » +çº Ľ +é¬ £ +æĶ ® +åĽ Ķ +é¦ ķ +æĪ Ĩ +çĪ ¨ +é½ ī +äº į +å° ¢ +å½ ³ +åį ¬ +æ® ³ +ðł ϶ +æ¯ Į +éĤ ĺ +æĪ ĭ +åľ ¢ +æ° ķ +ä¼ ĭ +ä» Ŀ +åĨ ® +æ° ¿ +æ± Ī +æ° ¾ +å¿ ī +å® Ħ +ð¬£ Ļ +è® ± +æī ŀ +åľ ² +åľ « +èĬ ı +èĬ ĥ +æľ ³ +æľ ¸ +ð¨ Ļ +ð¨Ļ ¸ +éĤ ¨ +åIJ Ĵ +åIJ ĸ +å± ¼ +å± ¾ +è¾ ¿ +éĴ Ĩ +ä» ³ +ä¼ £ +ä¼ Ī +çĻ ¿ +çĶ ª +éĤ ł +çĬ ´ +åĨ ± +éĤ ¡ +ð¬ĩ ķ +æ± ĭ +ä ľ +äľ £ +è® » +𬣠ŀ +åŃ ĸ +ð¬ĺ ĵ +çº © +çİ Ĵ +çİ ĵ +çİ ĺ +çİ ļ +åĪ ¬ +ð«Ń Ł +åĿ ľ +åĿ ī +æī ½ +ð«Ń ¢ +åĿ ĭ +æī º +ã§ ij +æ¯ IJ +èĬ ° +èĬ £ +èĭ Ĭ +èĭ ī +èĬ ĺ +èĬ ´ +èĬ ł +ð« ĩ +ð«ĩ Ń +èĬ ¤ +æĿ ķ +æĿ Ļ +æĿ Ħ +æĿ § +æĿ © +å° ª +å° ¨ +è½ ª +ð«IJ Ħ +åĿ Ĵ +èĬ Ī +æĹ ´ +æĹ µ +åij Ļ +ã ķ +ãķ ® +å² į +ð« µ +𫵠· +å² ł +å² ľ +åij ĩ +åĨ ı +è§ ĥ +å² Ļ +ä¼ ¾ +ãij ĩ +ä¼ Ń +ä½ ĸ +ä¼ ² +ä½ ģ +é£ ı +çĭ ĥ +éĹ ¶ +æ± § +æ± « +𣲠ĺ +ð£² Ĺ +æ² Ħ +æ² ĺ +ð¬ĩ Ļ +æ± Ń +ã³ ĩ +æ² ĩ +å¿ ® +å¿ ³ +å¿ º +𬣠¡ +ç¥ ĥ +è¯ ĩ +éĤ ² +è¯ İ +è¯ IJ +å± ĥ +ð« ¸ +𫸠© +å² Ĭ +éĺ ½ +ä¢ º +éĺ ¼ +å¦ § +å¦ ĺ +ð¨ ļ +ð¨ļ ķ +çº ® +é© ² +ð«ĺ ľ +çº » +ð¬ĺ ĺ +ð«ĺ Ŀ +çº ¼ +çİ ¤ +çİ ŀ +çİ ± +çİ Ł +éĤ ½ +éĤ ¿ +åĿ ¥ +åĿ ° +åĿ ¬ +åĿ ½ +å¼ Ĩ +èĢ µ +ä¢ ¼ +ð¦ Ń +ð¦Ń ľ +èĮ ĭ +èĭ § +èĭ ¾ +èĭ ł +æŀ ħ +ãŃ İ +æŀ ĺ +æŀ į +çŁ ¼ +çŁ » +åĮ ¼ +𬨠Ĥ +ð¬Ģ © +ð¬Ģ ª +æĹ ¿ +æĺ Ħ +æĺ Ĵ +æĺ Ī +åĴ ī +åĴ ĩ +åĴ į +å² µ +å² ½ +å² ¨ +å² ŀ +å³ Ĥ +ã Ł +ãŁ ĥ +åĽ · +𬬠© +éĴ IJ +éĴ Ķ +éĴ ĸ +çī ¥ +ä½ ´ +åŀ Ī +ä¾ ģ +ä¾ ¹ +ä½ ¸ +ä½ º +éļ ¹ +ãij Ĭ +ä¾ Ĥ +ä½ ½ +ä¾ ĺ +éĥ Ī +èĪ ł +éĥ IJ +éĥ ĥ +æĶ ½ +èĤ Ń +èĤ ¸ +èĤ · +çĭ ī +çĭ Ŀ +é¥ ³ +å¿ ŀ +çĤ Į +çĤ Ĩ +æ³ Ļ +æ² º +æ³ Ĥ +æ³ ľ +æ³ ĥ +æ³ ĩ +æĢ Ĭ +å³ ĥ +ç© ¸ +ç¥ ĭ +ç¥ Ĭ +ð«į £ +𬣠³ +𬠩½ +é¸ ¤ +å¼ ¢ +å¼ ¨ +éĻ ij +𬮠¿ +éĻ İ +ð¬¯ Ģ +åį º +ä¹ ¸ +å¦ Ń +å§ Ī +ð« ° +ð«° Ľ +è¿ ³ +åı ķ +𬳠µ +é© µ +𬳠¶ +ä Į +äĮ ¹ +é© º +ð«ł Ĭ +ç» ĭ +ç» IJ +çł ī +èĢ Ķ +ãĽ ĥ +çİ ¶ +çı ĩ +çı ħ +ð¬į Ľ +çı ĭ +çİ ¹ +çı Į +çİ ¿ +éŁ ¨ +åŀ ļ +åŀ ¯ +åŀ Ļ +åŀ ² +åŁ ı +åŀ į +èĢ ĩ +é¿ į +åŀ İ +åŀ ´ +åŀ Ł +åŀ ŀ +æĮ ĵ +åŀ µ +åŀ ı +æĭ ¶ +èį ĸ +èį ģ +èį Ļ +èį Ľ +èĮ Ī +èĮ ½ +èį Ħ +èĮ º +ð¬ľ ¬ +èį ĵ +èĮ ³ +𦠰 +𦰠¡ +èĮ Ľ +èį Ń +ãŃ ķ +æŁ · +æŁ ĥ +æŁ Ĭ +æŀ ¹ +æł IJ +æŁ ĸ +éĥ ļ +åī ħ +ä´ ĵ +è¿ º +åİ ĸ +çł Ĩ +çł ij +çł Ħ +èĢ ı +å¥ ĵ +ä ¶ +ä¶ ® +è½ µ +è½ · +è½ ¹ +è½ º +æĺ º +𪠾 +𪾠¢ +æĺ ½ +çĽ · +åĴ ¡ +åĴ º +æĺ ³ +æĺ £ +æĺ ¤ +æĺ « +æĺ ¡ +åĴ ¥ +æĺ ª +èĻ · +èĻ ¸ +åĵ ĥ +å³ ĺ +èĢ ij +å³ Ľ +𪨠° +å³ Ĺ +å³ § +å¸ ¡ +éĴ ĺ +ð«ĵ § +éĴ ľ +𬬠® +𬬠± +ð¬¬ Ń +éĴ ª +éĴ ¬ +éĴ Ń +çŁ § +ç§ ¬ +ä¿ « +èĪ ģ +ä¿ ľ +ä¿ Ļ +ä¿ į +åŀ ķ +è¡ İ +èĪ £ +å¼ ĩ +ä¾ ´ +é¸ § +äı ¡ +èĥ ł +ð¦ ϶ +èĥ Ī +èĥ © +èĥ £ +æľ ı +é£ IJ +è¨ Ħ +é¥ » +åº ¤ +çĸ ¢ +çĤ £ +çĤ Ł +ã ¶ +ã¶ ² +æ´ Ń +æ´ ĺ +æ´ ĵ +æ´ ¿ +ã³ ļ +æ³ ļ +æµ Ī +æµ ī +æ´ ¸ +æ´ ij +æ´ ¢ +æ´ Ī +æ´ ļ +æ´ º +æ´ ¨ +æµ IJ +ã³ ĺ +æ´ ´ +æ´ £ +æģ Ķ +å® ¬ +çª Ģ +æī Ĥ +è¢ Ĩ +ç¥ ı +ç¥ IJ +ç¥ ķ +åı ļ +éĻ § +éĻ ŀ +å¨ Ģ +å§ ŀ +å§ ± +å§ ¤ +å§ ¶ +å§ ½ +æŀ ² +ç» ĸ +éª ĥ +ð¬ĺ ¡ +𬳠½ +ð¬ĺ © +ð«Ħ § +å½ ĸ +éª ī +æģ Ŀ +çı ª +çı Ľ +çı ¹ +çIJ Ĭ +çİ ¼ +çı ĸ +ðª Ł +ðªŁ Ŀ +çı ½ +çı ¦ +çı « +çı Ĵ +ð¬į ¤ +çı ¢ +çı ķ +çı Ŀ +ð«Ń ¼ +åŁ Ĺ +åŀ ¾ +åŀ º +åŁ Ĩ +åŀ ¿ +åŁ Į +åŁ ĩ +èİ ° +èĮ Ŀ +ð¬ľ ¯ +éĦ Ģ +èİ ¶ +èİ Ŀ +äĵ ĸ +èİ Ļ +æł » +æ¡ ł +ð¬ Ĥ +ð¬Ĥ © +æ¡ Ħ +æ¢ ł +æł ´ +æ¢ ´ +æł Ĵ +éħ İ +éħ ı +ð«ł Ĩ +çł µ +çł ł +çł « +çł ¬ +ç¡ ģ +æģ § +ç¿ ĥ +éĥ ª +ð¨ IJ +ð¨IJ Ī +è¾ Ģ +è¾ ģ +ð¬ Į +ð¬Į Ĺ +åī ķ +èµ Ģ +åĵ ¢ +æĻ ħ +æĻ Ĭ +åĶ Ŀ +åĵ ³ +åĵ ± +åĨ Ķ +æĻ Ķ +æĻ IJ +çķ ĸ +èļ Ħ +èļ Ĩ +ð« ij +ð«ij ¡ +å¸ ± +å´ ģ +å³ ¿ +𪨠¶ +å´ Ħ +å¸ ¨ +å ´Ģ +èµ Ĩ +𬠬¸ +éĴ · +𬬠» +𬬠¹ +𬬠¿ +ð¬Ń ģ +çľ ļ +çĶ ¡ +ç¬ « +åĢ » +åĢ ´ +èĦ © +åĢ ® +åĢ ķ +åĢ ŀ +ð« ¢ +ð«¢ ¸ +åĢ ĵ +åĢ § +è¡ ĥ +èĻ Ĵ +èĪ Ń +èĪ ¯ +èĪ ¥ +çĵ ŀ +é¬ ¯ +é¸ ° +èĦ İ +æľ ĵ +èĥ ² +èĻ ĵ +é± ½ +çĭ ´ +å³ ± +çĭ » +çľ ¢ +ð«Ĺ § +åĭ į +çĹ Ħ +çĸ ° +çĹ ĥ +ç« ĺ +ç¾ ĸ +ç¾ ĵ +æ¡ Ĭ +æķ ī +çĥ ł +çĥ Ķ +çĥ ¶ +çĥ » +ð¬Ĭ Ī +æ¶ į +æµ ¡ +æµ Ń +æµ ¬ +æ¶ Ħ +æ¶ ¢ +æ¶ IJ +æµ ° +æµ Ł +æµ Ľ +æµ ¼ +æµ ² +æ¶ ĺ +æĤ Ī +æĤ ĥ +æĤ ¢ +ð¬Ĵ Ī +å® § +çª ħ +çª Ĭ +çª İ +æī ħ +æī Ĩ +è¢ ª +è¢ Ĺ +è¢ ¯ +ç¥ § +éļ º +åł ² +çĸ į +𨠺 +ð¨º Ļ +éĻ ´ +ç ĥĿ +çł ® +ãĽ ļ +åĵ ¿ +ç¿ Ģ +ç¿ Ĥ +åī Ł +𬳠¿ +ð«Ħ ¨ +ç» ¤ +éª į +ð¬ĺ « +ä Ĥ +äĤ ® +çIJ İ +çı ¸ +çı µ +çIJ Ħ +çIJ Ī +çIJ Ģ +çı º +æİ Ń +åł İ +åł IJ +åŁ ¼ +æİ İ +åŁ « +åł Į +æĻ ¢ +ð« ® +ð«® ĥ +æİ ŀ +åŁ ª +å£ ¸ +ãĻ į +èģ į +èı Ŀ +èIJ ļ +èı ¥ +èİ ¿ +äĵ « +åĭ ļ +äĵ ¬ +èIJ Ĩ +èı Ĥ +èı į +èı ¼ +èIJ £ +äĵ ¨ +èı ī +äĵ Ľ +æ¢ ¼ +æ¢ ½ +æ¡ ² +æ¢ ¾ +æ¡ ¯ +æ¢ £ +æ¢ Į +æ¡ ¹ +æķ Ķ +åİ £ +ç¡ Ķ +é¿ İ +ç¡ Ļ +ç¡ ļ +ç¡ Ĭ +ç¡ į +åĭ Ķ +ä´ ķ +é¾ ģ +éĢ ´ +åĶ ª +åķ « +ç¿ Ī +ã « +ã« ° +æĻ Ļ +çķ ¤ +𬱠ĸ +è¶ ¼ +è· Ĥ +èĽ ĥ +èļ ² +ð¬Ł ½ +èļ º +åķ ´ +äİ ĥ +å´ § +å´ Ł +å´ ŀ +å´ Ĵ +å´ Į +å´ ¡ +éĵ ı +ð«ĵ ¯ +ð«Ł ¹ +éĵ ķ +ð«Ł ¼ +éĵ ĸ +éĵ ĺ +éĵ ļ +éĵ ŀ +éĵ ¥ +éĵ ´ +çī » +çī ¿ +ç¨ Ĩ +ç¬ ± +ç¬ ¯ +åģ ° +åģ ¡ +é¸ º +åģ Ń +åģ ² +åģ ģ +ã ¿ +ã¿ ł +éĦ ħ +åģ ĵ +å¾ Ľ +è¡ Ĵ +èĪ ³ +èĪ ² +é¸ ¼ +æĤ Ĩ +éĦ ĥ +çĵ » +ä Ŀ +äĿ Ļ +èĦ ¶ +èĦ ŀ +èĦ Ł +äı ² +é± ¾ +çĮ ĩ +çĮ Ĭ +çĮ Ħ +è§ ĸ +ðł ħ +ðłħ ¤ +åº ± +åº ¼ +åº ³ +çĹ ĵ +ä´ Ķ +ç« « +åł ĥ +éĺ Į +ç¾ Ŀ +ç¾ ķ +çĦ Ĩ +çĥ º +çĦ Į +æ· ı +ð¬ĩ ¹ +æ· Ł +æ· ľ +æ· ´ +æ· ¯ +æ¹ ´ +æ¶ ´ +ð¬į ¡ +ã ¥ +㥠Ħ +æĥ Ľ +æĥ Ķ +æĤ ° +æĥ Ļ +å¯ ģ +éĢ Ń +𬤠ĩ +ð«į ¯ +è¢ ¼ +è£ Ī +ç¥ ² +𬤠Ĭ +ð«į ² +è° ŀ +èī ´ +å¼ ¸ +å¼ ¶ +ð¬¯ İ +éļ ĥ +å© ŀ +å¨ µ +å© ¼ +åª ĸ +å© ³ +å© į +å© Į +å© « +å© ¤ +å© ĺ +å© ł +ð¬ĺ ¬ +ð¬ĺ Ń +𬴠Ĥ +ð«ĺ ¦ +ç» ¹ +ð«Ł ħ +ð¬ĺ ¯ +éª ķ +ð«ĺ § +çµ ľ +çı · +çIJ ² +çIJ ¡ +çIJ Ł +çIJ Ķ +çIJ Ń +åł ¾ +åł ¼ +æı ķ +ãĻ ĺ +åł § +åĸ Ĩ +åł ¨ +å¡ ħ +åł ł +çµ · +𪠣 +𪣠» +ð¡ İ +ð¡İ ļ +è ijľ +æĥ İ +èIJ ³ +èij Ļ +éĿ ¬ +èij ´ +èĴ ĩ +èĴ Ī +éĦ ļ +èĴ ī +èĵ ĩ +èIJ © +èij ° +èij İ +éĦ ij +èĴ İ +èij ĸ +èĴ Ħ +èIJ ¹ +æ£ ¤ +æ£ ½ +æ£ « +æ¤ ĵ +æ¤ ij +ð¬ ĥ +ð¬ĥ Ĭ +é¹ Ģ +æ¤ Ĩ +æ£ ĵ +æ£ ¬ +æ£ ª +æ¤ Ģ +æ¥ Ĺ +𬠷 +𬷠ķ +çĶ ¦ +éħ ¦ +è§ Į +å¥ ¡ +çļ ķ +ç¡ ª +æ¬ ¹ +è© Ł +ð«IJ IJ +è¾ Į +æ£ IJ +é¾ Ĥ +𬠹 +𬹠¼ +é» ¹ +çī ļ +çĿ İ +æĻ « +æĻ ª +æĻ ± +ð § +ð§ ¿ +ð§¿ ¹ +èĽ ij +çķ ¯ +æĸ Ŀ +åĸ ¤ +å´ ¶ +åµ ģ +ð« ¶ +ð«¶ ĩ +å´ ¾ +åµ ħ +å´ ¿ +åµ ļ +ç¿ Ļ +ð«ĸ ® +åľ Į +åľ IJ +èµ ij +èµ Ĵ +é¿ ı +éĵ ¹ +ð¬Ń Ĭ +éĵ ½ +𨱠ĩ +ð«ĵ ¶ +éĶ Ĭ +éĶ į +éĶ İ +ð¬Ń İ +éĶ ĵ +çĬ ĩ +é¢ ĭ +ç¨ Į +çŃ Ģ +çŃ ĺ +çŃ ľ +çŃ ¥ +çŃ ħ +åĤ ĥ +åĤ ī +ç¿ Ľ +åĤ Ĵ +åĤ ķ +èĪ ¾ +çķ ¬ +ð«ĸ ¯ +èĦ ¿ +èħ ĺ +ä IJ +äIJ ĥ +èħ Ļ +èħ Ĵ +ð¬± Ł +é² ĥ +çĮ ° +ð« Ľ +ð«Ľ Ń +çĮ ¯ +ã º +㺠Ħ +é¦ ī +åĩ ĵ +éĦ Ĺ +ð« · +ð«· · +å» ĭ +å» Ĩ +éĦ Į +ç² ¢ +éģ Ĩ +æĹ IJ +𬮠± +çĦ ŀ +ð¬Ĭ ¤ +æ¬ » +𣠸 +𣸠£ +æº ļ +æº ģ +æ¹ Ŀ +æ¸ ° +æ¹ ĵ +ã ´ +ã´ Ķ +æ¸ Ł +æº ł +æ¸ ¼ +æº ĩ +æ¹ £ +æ¹ ij +æº ŀ +æĦ IJ +æĦ ĥ +æķ © +çĶ ¯ +æ£ ¨ +æī Ĭ +è£ £ +ç¥ ¼ +å© » +åª Ĩ +åª ŀ +ãĽ ¹ +åª ĵ +åª Ĥ +åª Ħ +æ¯ µ +çŁ ŀ +𬴠ĥ +ð«ĺ ¨ +ç¼ Ĭ +ç¼ IJ +éª Ļ +çij ĥ +çij ĵ +çij ħ +çij Ĩ +ä´ ĸ +çij ĸ +çij Ŀ +çij Ķ +çij Ģ +𤠧 +𤧠Ľ +çij ³ +çij Ĥ +å¶ ħ +çij ij +éģ ĺ +é« ¢ +å¡ ¥ +åł ½ +èµ ª +æij Ľ +å¡ Ŀ +æIJ Ĵ +æIJ Į +èĴ ± +èĴ ¨ +èĵ ı +èĶ Ģ +èĵ ¢ +èĵ Ĥ +èĴ » +èĵ £ +æ¤ ¹ +æ¥ ª +æ¦ ĥ +æ¦ ħ +æ¥ Ĵ +æ¥ © +æ¦ ĩ +æ¤ ¸ +æ¥ Ļ +æŃ ħ +𬠪 +𬪠© +ç¢ ĥ +ç¢ ı +ð¬Ĵ Ķ +ç¢ Ī +äĥ ħ +ç¡ ¿ +éĦ ł +è¾ Ĵ +ð¬¨ İ +ð«IJ ĵ +é¾ Ĩ +è§ ľ +ä £ +ä£ ĺ +æļ ķ +é¹ į +ð« « +ð«« ĩ +㬠Ĭ +æļ ħ +è· ± +èľ IJ +èľ İ +åµ ² +èµ Ĺ +éª ± +éĶ ĸ +ð«ĵ ¹ +éĶ ĺ +éĶ ³ +éĶ § +éĶ ª +ð¬Ń ļ +éĶ « +éĶ ¬ +ð¬Ń Ľ +ç¨ ij +ç¨ Ļ +ä ħ +äħ Ł +ð¬ ķ +ð¬ķ Ĥ +çŃ » +çŃ ¼ +çŃ ¶ +çŃ ¦ +çŃ ¤ +åĤ º +é¹ İ +åĥ ĩ +èī ħ +èī ī +è° ¼ +è² Ĩ +èħ ½ +èħ ¨ +èħ ¯ +é² ī +é² Ĭ +é² Į +ä² Ł +𬶠ĭ +𬶠į +é² ı +éĽ Ĭ +çĮ º +é£ Ķ +è§ Ł +ð¦ Ŀ¼ +é¦ Į +è£ Ľ +å» Ĵ +çĺ ħ +éĦ ĺ +é¹ Ĵ +éĦ ľ +éº Ģ +éĦ £ +éĺ ĺ +ð«Ķ ¶ +çħ ģ +çħ ĥ +çħ ´ +çħ ĭ +çħ Ł +çħ ĵ +æ» ł +æº į +æº ¹ +æ» Ĩ +æ» ī +æº ¦ +æº µ +æ¼ · +æ» § +æ» ĺ +æ» į +æĦ Ń +æħ ¥ +æħ Ĩ +å¡ ± +ð« ĮĢ +è £¼ +ç¦ ĭ +ç¦ Ķ +ç¦ ĺ +ç¦ Ĵ +è° « +é¹ Ķ +ð«ĸ ³ +æĦ į +å« Ħ +åª ± +æĪ ¤ +åĭ ł +æĪ £ +ð«ĺ ª +ð«ĺ ¬ +ç¼ ŀ +èĢ ¤ +çij § +ð« ŀ +ð«ŀ © +çij ¨ +çij ± +çij · +çij ¢ +æĸ ł +æij ı +å¢ ķ +å¢ Ī +å¢ IJ +å¢ ĺ +æij ´ +éĬ İ +ð¡ IJ +ð¡IJ ĵ +å¢ ļ +æĴ ĸ +𪠤 +ðª¤ Ĺ +éĿ ½ +éŀ ģ +èĶ Į +èĶ Ī +èĵ ° +èĶ ¹ +èĶ Ĭ +åĺ ı +æ¦ ° +æ¦ ij +æ§ ļ +ð£ Ĺ +ð£Ĺ ĭ +æ§ ľ +æ¦ į +çĸ IJ +𬸠ĺ +éħ º +éħ ¾ +éħ ² +éħ ´ +ç¢ ¶ +äĥ İ +ð¬Ĵ Ĺ +ç¢ ¨ +ð¥ Ķ +ð¥Ķ ² +ç¢ ¹ +ç¢ ¥ +åĬ Ĥ +ð«ļ ĸ +ä´ Ĺ +å¤ ¥ +çŀ į +é¹ ĸ +ã¬ İ +è· ½ +èľ ¾ +å¹ ĸ +å¶ į +åľ Ļ +𨱠ı +éĶ º +éĶ ¼ +éĶ ½ +ð¬Ń ¤ +éĶ ¾ +éĶ ¿ +éķ ĥ +éķ Ħ +éķ ħ +é¦ Ŀ +é¹ Ļ +ç® ¨ +ç® ĸ +åĬ Ħ +åĥ ¬ +åĥ ¦ +åĥ Ķ +åĥ İ +æ§ ĥ +ãĻ ¦ +é² Ĵ +é² ķ +ð«ļ ķ +é² ĸ +é² Ĺ +é² ĺ +é² Ļ +𬶠IJ +𬶠ı +ð ©½ +𩽠¾ +å¤ IJ +çį į +é£ Ĺ +𬸠ļ +åĩ ĺ +å» ij +å» Ļ +çĺ Ĺ +çĺ ¥ +çĺ ķ +é² Ŀ +éĦ « +çĨ ĩ +æ¼ ¹ +æ¼ ĸ +æ½ Ĩ +æ¼ ¤ +æ½ © +æ¼ ¼ +æ¼ ´ +ã ½ +ã½ ı +æ¼ Ī +æ¼ ĭ +æ¼ » +æħ ¬ +çª ¬ +çª Ń +ã ® +ã® ¾ +𬤠Ŀ +è¤ ķ +ç¦ Ľ +ç¦ ļ +éļ © +å« ķ +å« Ń +å« ľ +å« ª +ð¬ ĻĤ +ã » +ã» ¬ +éº ¹ +çĴ Ĩ +æ¼ ¦ +åı ĩ +å¢ £ +å¢ ¦ +å¢ ¡ +åĬ IJ +èĸ ģ +èķ ° +èĶ ĥ +é¼ Ĵ +æ§ ± +é¹ Ŀ +ç£ ı +ç£ ī +æ® £ +æħ Ń +éľ ħ +æļ µ +æļ ² +æļ ¶ +è¸ ¦ +è¸ £ +äĹ ĸ +èĿ ĺ +èĿ ² +èĿ ¤ +åĻ ĩ +å ĻĤ +åĻ Ģ +ç½ ¶ +å¶ ² +å¶ ĵ +ãł ĩ +å¶ Ł +å¶ Ĵ +éķ Ĩ +éķ Ī +éķ ĭ +éķ İ +ð¬Ń © +éķ ķ +ç¨ ¹ +åĦ ĩ +çļ ŀ +çļ Ľ +ä´ ĺ +èī İ +èī ı +é¹ Ł +𩾠ĥ +é² ¦ +é² ª +é² ¬ +æ© ¥ +è§ Ń +é¹ ł +é¹ ¡ +ç³ ĩ +ç³ Ī +ç¿ ¦ +é¹ ¢ +é¹ £ +çĨ Ľ +æ½ ĸ +æ½ µ +ã µ +ãµ IJ +æ¾ Ĥ +æ¾ Ľ +çij ¬ +æ½ ½ +æ½ ¾ +æ½ ı +æĨ Ń +æĨ ķ +𬸠£ +æĪ Ń +è¤ ¯ +ç¦ ¤ +ð«į ½ +å« ½ +éģ ¹ +𬴠Ĭ +çĴ ¥ +çĴ ² +çĴ Ĵ +æĨ Ļ +æĵ IJ +éĦ ¹ +èĸ ³ +éŀ Ķ +é» ĩ +ð¬ ŀ +ð¬ŀ Ł +èķ Ĺ +èĸ ¢ +èķ ¹ +æ© ŀ +æ© ij +æ© ¦ +éĨ ij +è§ ± +ç£ ¡ +ð¥ ķ +ð¥ķ ¢ +ç£ ľ +è± ® +ð«Ł ¦ +ð¬º Ī +ð«ł ľ +é¹ ¾ +èĻ ¤ +æļ ¿ +æĽ Į +æĽ Ī +㬠ļ +è¹ ħ +è¸ ¶ +äĹ Ľ +èŀ Ĺ +çĸ ģ +ãł ĵ +å¹ ª +𪠩 +𪩠ĺ +å¶ ¦ +ð¬Ń ¬ +𨱠ij +ð¬Ń ¯ +é¦ ŀ +ç© Ħ +ç¯ ļ +ç¯ ¯ +ç° ī +é¼ ½ +è¡ ł +çĽ ¦ +èŀ £ +ç¸ ¢ +é² Ń +é² ¯ +é² ° +é² º +é² ¹ +ð«Ĺ ´ +äº ¸ +çĻ Ģ +çĺ Ń +𬸠¦ +ç¾ ± +ç³ Ĵ +çĩ ĭ +çĨ » +çĩ Ĭ +çĩ ļ +çĩ ı +æ¿ © +æ¿ ĭ +æ¾ ª +æ¾ ½ +æ¾ ´ +æ¾ Ń +æ¾ ¼ +æĨ · +æĨ º +æĩ Ķ +é» ī +å¬ Ľ +é¹ ¨ +ç¿ ¯ +ð«Ħ · +çĴ ± +𤠩½ +çĴ ¬ +çĴ ® +é« ½ +æĵ ¿ +èĸ ¿ +èĸ ¸ +æª ij +æ« Ĩ +æª ŀ +éĨ ¨ +ç ¹Ħ +ç£ ¹ +ç£ » +çŀ « +çŀ µ +è¹ IJ +èŁ ı +ã ĺ +ãĺ İ +ð¬Ń ³ +éķ ¤ +ð¬Ń ¶ +ð«Ķ į +éķ ¥ +éķ ¨ +ð¬Ń ¸ +ð¨± Ķ +ð¬Ń ¼ +ð«Ķ İ +çŁ ° +ç© Ļ +ç© ľ +ç© Ł +ç° ķ +ç° ĥ +ç° ı +åĦ ¦ +éŃ ĭ +æĸ ¶ +èī ļ +𬸠ª +è° ¿ +ä² ł +ð¬¶ Ł +é² ¾ +𬶠ł +é² ¿ +é³ ģ +é³ Ĥ +é³ Ī +é³ ī +çį ¯ +äĹ ª +é¦ ĺ +è¥ ķ +è¥ ļ +𬶠¨ +èŀ ± +çĶ ĵ +å¬ ¬ +å¬ ¥ +ð¦ Ī +ð¦Ī ¡ +ð«Ħ ¸ +çĵ Ģ +éĩ IJ +é¬ ¶ +çĪ ĩ +éŀ ³ +éŀ ® +ð¬Ł ģ +èĹ Ł +èĹ ¦ +èĹ ¨ +é¹ ² +æª « +é» ¡ +ç¤ ŀ +ç¤ Į +ð¥ ĸ +ð¥ĸ ¨ +è¹ ¢ +è¹ ľ +èŁ « +äĹ ´ +åļ ļ +é« ĥ +éķ ® +éķ ± +éħ Ĥ +é¦ § +ç° ł +ç° Ŀ +ç° ° +é¼ « +é¼ © +çļ ¦ +èĩ ij +ä² ¢ +é³ ij +é³ Ĵ +é¹ ± +é¹ ¯ +çĻ Ĺ +ð¦ Ĵ +ð¦Ĵ į +æĹ ŀ +ç¿ · +åĨ ģ +äİ ĸ +çĢ Ķ +çĢ į +çĢ Į +è¥ ľ +ä´ Ļ +ð¬Ļ Ĭ +åļ Ń +ã ° +ã° Ģ +é¬ · +éĨ Ń +è¹ ¯ +èł ĭ +ç¿ ¾ +é³ ĺ +åĦ ³ +åĦ ´ +é¼ Ĺ +ð¬¶ Ń +𩾠Į +é³ ļ +é³ Ľ +éº ij +éº ĸ +èł ĥ +å½ Ł +å¬ ¿ +é¬ Ĵ +èĺ ĺ +æ¬ Ĥ +é Ĩµ +é¢ ¥ +çĶ Ĺ +ð¨ Ł +ð¨Ł ł +å· ĩ +éħ ħ +é« İ +çĬ ¨ +𬶠® +ð¨ Ń +ð¨Ń ī +㸠Į +çĪ Ķ +çĢ ± +çĢ ¹ +çĢ ¼ +çĢ µ +è¥ « +åŃ ħ +éª ¦ +ð¬Ļ ĭ +èĢ ° +𤠫 +𤫠ī +çĵ ĸ +é¬ ĺ +è¶ ¯ +𬺠ĵ +ç½ į +é¼ ± +é³ ł +é³ ¡ +é³ £ +çĪ Ł +çĪ ļ +çģ Ī +éŁ Ĥ +ç³ µ +èĺ ¼ +ç¤ µ +é¹ ´ +èº Ķ +çļ Ń +é¾ ¢ +é³ ¤ +äº ¹ +ç± ¥ +é¼ · +ð«ļ Ń +çİ ĥ +éĨ ¾ +é½ ĩ +è§ ¿ +èł ¼ +× § +× ¤ +× Ľ +×ķ× ª +× ¡ +×Ļ× Ŀ +× ¦ +× Ĵ +× ĺ +×ķ× ¨ +× Ŀ +×ķ× ľ +× ĸ +๠Ĥ +ï º +ðŁ į +ðŁ IJ +×Ļ× ¨ +ï » +ðŁ ij +ðĿ IJ +ðŁ ı +ðŁ Ķ +ðŁ Į +ðŁ İ +ðŁ ĵ +× Ł +ðĿ ij +×ķ× ĵ +ï ¦ +Ġ× ķ +×ķ× ij +à¸Ń à¸ĩ +ðĿ ĺ +×Ļ× ª +ðĿ ķ +à¸Ĺ ีà¹Ī +Ø§Ø ¦ +ðŁ ¤ +×ķ× Ł +ر ÙĬ +×Ļ× ľ +ร ะ +า ย +ï ¯ +ï ® +า ม +â ĩ +ðŁ ¥ +ï Ń +ðĿ Ļ +×ķ× ł +á ½ +Ġ× Ľ +ðŁ ļ +â ļ +ï § +×ij ר +×Ļ× ł +á ´ +Ġ× Ĺ +á ¼ +ðĿ Ĺ +Ġ× ¢ +×Ļ× Ķ +ãģ£ ãģŁ +ãģĵ ãģ¨ +á ¸ +ÙĬ ÙĨ +ãģª ãģĦ +ا ع +ภ¨ +à¹Ī à¸ĩ +×Ļ× ĵ +×ŀ ש +á Ī +׳ ×Ļ +×Ļ× ij +ï ¥ +ðĿ ĵ +Ġ× Ļ +× ļ +ั à¸ĩ +â ĵ +ï ¤ +ĠاÙĦ Ø£ +า à¸ģ +à¹ī à¸Ļ +à¹Ģ ร +×ķ× Ŀ +á ¹ +ภ¶ +×Ļ× § +ภĭ +à¸Ħ ร +ภĺ +ั à¸ģ +ðŁ ķ +ÙĪ ÙĨ +à¸Ń ย +â Ĭ +ðĿ Ĵ +ĠاÙĦ ع +า à¸Ļ +×Ļ× Ł +ÙĦ ÙĬ +×Ļ× © +à¸Ľ ระ +à¹Ģ à¸Ľ +Ġ× ł +×ķ× ¡ +ภł +Ùħ ÙĨ +×ķ× ¢ +×ķ× ŀ +â Į +ðŁ § +à¹ĩ à¸Ļ +ภį +ã İ +á µ +ĠاÙĦ س +×ķ× § +ห ล +ðŁ ĩ +â ı +ðŁ ¦ +Ġ×Ķ ×ŀ +ÙĪ Ø§ +Ġ× ª +ר ×IJ +à¸Ń à¸Ļ +ภ© +à¹Ī ว +×ķ× ¦ +í Ĺ +ã Ħ +ï ¨ +ï ¹ +â İ +ï ² +ðĿ ļ +ð IJ +à¸Ħ ว +ห à¸Ļ +Ġ× ¨ +ب ÙĬ +ร à¹Į +ر ا +Ø´ ر +×ķ× Ĺ +×ķ× ¤ +×ķ× © +×ķ× Ĵ +í Ŀ +â Ľ +à¸ķ ิ +à¹Ģ à¸ģ +ï ³ +ï ± +à¸Ķ à¹ī +ë ¹ +ï ¬ +á ¿ +ðŁ Ľ +ðĿ ĸ +à¹Īา à¸ĩ +ู à¹ī +Ġ×Ķ ×IJ +ĠاÙĦ ØŃ +פ ר +ÙĪ Ùħ +à¹Ģ ล +í ĸ +×Ļ× ¢ +ì Ī +í ĵ +ðŁ ħ +á ł +à¸Ħว าม +à¸Ī ะ +׳ ×Ķ +Ġ× § +à¸ Ł +à¹ī à¸ĩ +ห ม +ت Ùħ +׾ ×Ļ +ÙĬ د +à¹Ī à¸Ļ +׊ר +ש ר +à¹Ģ à¸Ĺ +×ŀ ר +ë ĸ +ع ÙĦ +×ŀ ×¢ +â ² +׾ ×Ķ +Ġ× ¤ +à¸Ń à¸ģ +س ÙĦ +×Ļ× ŀ +ÙĤ ÙĬ +í İ +ت ØŃ +×Ļ× ¡ +×Ļ× Ĺ +í Ľ +ï ° +â ½ +á ī +á Ĭ +á ¨ +Ùĩ ا +Ġ׾ ×Ķ +×ķ× IJ +Ùħ ا +à¹īà¸Ń à¸ĩ +ر ب +ĠاÙĦ ج +×ŀ ×ĵ +Ùħ ÙĦ +ت ر +à¹Ģ à¸Ķ +×§ ר +í ħ +ì ¼ +ê ¿ +ã Ī +á IJ +ðŁ Ĺ +ê ¦ +á ĭ +ðĿ Ķ +à¹Ģà¸Ľ à¹ĩà¸Ļ +à¹ĥ ห +ม า +ว à¹Īา +ม ี +ี à¹ī +à¹Ħม à¹Ī +ÙĨ ÙĬ +Ø ¤ +ร า +×ķ ×Ļ +ãĤĪ ãģĨ +ิ à¸Ķ +×Ļ× ¤ +׊׾ +ÙĤ د +à¹Ģ ส +×Ļ× ĺ +à¸ģ ล +ר ׼ +×ķ× Ľ +×Ļ× Ľ +ë Ī +ë ĥ +ðŁ ĸ +á ħ +â ¼ +ã ī +à¹Ħ à¸Ķà¹ī +ת ×Ļ +×Ļ× IJ +ĠاÙĦ Ø¥ +à¸ł า +ร ิ +ÙĤ Ø© +ØŃ د +ê » +ì ± +ת ×Ĺ +ì º +â ĭ +á Ħ +á ¾ +â µ +â ¾ +ĠÙĪ Ø§ÙĦ +׳ ×ķ +Ù Ģ +ÙĬ ا +à¸ģ à¹ĩ +×ŀ ×Ķ +ãģĦ ãĤĭ +ع د +ĠاÙĦ ÙĨ +Ġ×Ķ ×© +Ø ¦ +ั à¹īà¸ĩ +ร ัà¸ļ +ÙĪ ÙĤ +ãģ§ ãģį +à¹Ģ à¸ŀ +׼ ׾ +×ĺ ר +ั à¸Ķ +à¸Ń า +ì ¢ +à¸Ń à¸ļ +à¸ķ ร +à¹Ģ à¸Ĭ +ì Ķ +ãģĹ ãģ¾ +ë ģ +ë ķ +ðŁ Ļ +â Ĵ +á ¶ +à¹ģ ล +ÙĨ ا +à¹ĥห à¹ī +à¹Ħ à¸Ľ +× £ +ั ว +า à¸ĩ +×ĵ ר +×ij ׾ +פ ×Ļ +Ġ× ĵ +ĠاÙĦ Ùģ +à¹Ģ à¸Ĥ +ש ×Ķ +×IJ ר +ë ¬ +ãģ« ãģª +ÑĢ Ð¾ +ว ิ +Ùħ ر +×IJ ת +Ùĥ ر +س ب +ÙĨ ت +ãģĹ ãģĦ +ا ج +à¸Ń รà¹Į +Ùĥ ÙĦ +س Ùħ +ส ิ +×Ļ× ¦ +ë Ŀ +í ľ +ì ī +á Ĩ +Ùĩ Ùħ +à¸Ļ ีà¹ī +ãģĤ ãĤĭ +ãģĦ ãģ¦ +س ÙĬ +׾ ×IJ +د ر +ãģ ļ +ÙĪ Ø¬ +ĠاÙĦ Ø® +ص ر +í ı +à¹īา à¸ĩ +ุ à¸Ķ +×ķ× ĺ +×ij ×¢ +í Ĩ +à¸Ĭ า +ร ม +ש ×ŀ +×ŀ ס +ê ´ +ì ´ +ë ľ +ì ¿ +ì © +ë » +â ¤ +ðŁ Ĩ +á Į +á ķ +ذ ا +à¸Ĺ ำ +à¸ķ à¹Ī +ĠاÙĦ ÙĤ +ÙĦ Ùĥ +ู à¹Ī +à¸Ħ ุ +ÙĬ Ùħ +׳ ×Ļ×Ŀ +ืà¹Ī à¸Ń +ÙĪ Ø¹ +ãĤ ĩ +ا ÙĤ +Ġ×ij ×¢ +à¹Ģ ม +ج Ùħ +á» « +ãģĵãģ¨ ãģĮ +ب د +×ķ× Ķ +ש ׾ +Ùĩ ر +à¹Ģ à¸Ļ +ãģ ¹ +í ĭ +ì » +ì ½ +ë Ń +ì Į +í Ģ +ë Į +ë º +ã Ĭ +à¹ĥ à¸Ļ +Ġ× Ĵ +๠Ĩ +à¸Ī าà¸ģ +ว ย +à¹ĥ à¸Ĭ +à¸ĩ าà¸Ļ +ĠاÙĦ Ø´ +ا ØŃ +à¹īา à¸Ļ +ืà¹Ī à¸Ńà¸ĩ +×IJ ×Ļ +ب ÙĦ +ãģ¨ æĢĿ +׳ ס +ãģ¾ ãģĽ +Ùĥ ÙĨ +×¢ ר +ĠاÙĦ د +ש ת +í ŀ +Ùħ س +ص ÙĦ +×ķ׳ ×Ķ +ار Ø© +ÙĦ Ùħ +ส ม +Ø£ ÙĨ +ת ר +×IJ ×ŀ +ع ب +Ø® ت +ãĤ ĥ +ì ¡ +ì £ +ив а +ส ั +ึ à¸ģ +ì ¸ +ë Ĩ +алÑĮ н +ì ³ +ì į +ê ¼ +ê ½ +ì ı +ã Į +ã ı +ï © +ê ª +á İ +Ġ× ĸ +à¸ģ ัà¸Ļ +×Ļ ×ķ +à¸Ħ à¸Ļ +׳ ×ķת +à¸ľ ูà¹ī +à¹ĥ à¸Ī +ãģĦ ãģŁ +Ùģ Ø± +×ĺ ×Ļ +צ ×Ļ +ãĤĤ ãģ® +ĠاÙĦ ص +ãģ¾ãģĽ ãĤĵ +د Ø© +×ij ×Ļ +ĠاÙĦ ر +Ġ×ŀ ×IJ +ส ำ +à¹Ģ ห +ع ر +ãģª ãģı +à¸ģร ะ +×ij ×ĵ +à¹Ģ à¸Ī +×Ļ× ļ +×Ĺ ×Ļ +ÙĬ ع +ש ×ij +ÙĨ Ø© +ÙĪ Ø¶ +ÙĦ Ùģ +ÙĢ ÙĢ +פ ×¢ +í Ī +×ŀ ×§ +ภIJ +ØŃ Ø© +ا ص +Ñĭв а +à¸Ħ ม +ว ั +à¸Ľ ล +ì Ł +í ļ +ë ´ +ë ij +ë ī +ë ĩ +ì ¨ +ë ± +ë İ +â ¬ +á ¥ +á Ĺ +á Ľ +á į +Å © +à¸Ķ ี +ô i +Ġ× ¡ +׾ ×ķ +á»Ŀ i +à¸Ħุ à¸ĵ +â y +à¸Ļ า +×Ĺ ×ĵ +×ĵ ×Ļ +ห า +ج ÙĦ +à¹Ģ ว +ãĤĩ ãģĨ +Ùħ Ø© +ĠاÙĦ Ùĥ +Ġ×Ķ ×¢ +ج ر +×ĸ ר +ا Ø· +׼ ת +×ķ׳ ×Ļ×Ŀ +ØŃ Ùħ +ê ¶ +ر Ùĥ +Ġ׾ ×¢ +×ķ× ĸ +ส ร +צ ׾ +Ø ¢ +ا ست +à¹Ī ม +Ø® ر +צ ×¢ +×Ļר ×ķת +اد Ø© +Ø´ ار +×ŀ ×Ĺ +í Ĵ +à¹Ģร ีย +×Ĺ ×§ +Ø§Ø « +ร à¸ĩ +à¹Ģ à¸ķ +à¸Ī ำ +ภĿ +à¹Īา ย +à¸Ħ ล +ÙĤ ÙĪ +иÑĩеÑģ к +à¸ĵ à¹Į +ั ย +Ùħ ع +ë ¨ +ë ¿ +ë ® +ï ´ +ì ¥ +ì « +ë µ +á ¡ +â į +ð ĵ +â ° +à¸Ĥ à¸Ńà¸ĩ +Ù ĭ +à¸ģ ัà¸ļ +ãģ® ãģ§ +à¹ī ว +à¸Ńย à¹Īาà¸ĩ +ãģ Ń +á»ĩ t +à¸ķ à¹īà¸Ńà¸ĩ +×ŀ ×Ļ +à¹ģ à¸ļ +×Ĵ ר +ÙĪ Ùģ +ÙĤ ÙĦ +à¸łà¸² à¸ŀ +ר ×Ļ +ล า +ÙĬ س +Ġ× ¦ +ÙĬ Ùģ +Ġ× ĺ +à¸ľ ล +á ng +ร ว +Ġ×ŀ ש +×IJ ×ķת +×ĸ ×Ķ +ู à¸ģ +à¸Ļ ัà¸ģ +اÙĨ ÙĬ +د ا +ãģ ³ +׼ ף +ãĤī ãĤĮ +ãĤĮ ãģ° +ת ×§ +ú c +ÙĪ Ø² +×Ļר ×Ķ +Ġn gh +án h +Ġ×ķ ×IJ +á» ħ +ส ุà¸Ķ +ë į° +ا ض +اÙĦ ÙĬ +ب ار +ع Ùħ +à¸ļ า +ت ج +à¸ŀ ร +×ķר ×Ķ +ả ng +Ø® ÙĦ +ภī +ắ c +ש ×Ļ×Ŀ +í Ķ +Ùģ Ø³ +×Ļ× Ĵ +п ÑĢ +ĠاÙĦ Ø« +س Ø· +ร ูà¹ī +ีà¹Ī ย +à¸Ń à¸Ķ +ãģª ãĤĬ +×Ĵ ×ĵ +ãģĦ ãģ¾ãģĹãģŁ +ס ×§ +Ø® ص +la ÅŁ +ен но +ب ØŃ +ส à¸Ļ +ภ® +ר×IJ ש +Ùħ ÙĪ +دÙĬ د +ษ า +×ķ× ļ +ãĥ§ ãĥ³ +à¸ķ ุ +Ġê µ +ĠÑģв о +צ ×ij +à¸Ń ม +à¸Ľ ร +ت ع +×Ķ ×ª +اÙħ ÙĦ +×ŀ ׳ +ç ¶ļ +ภ¤ +í į +ë ĺ +ë ¤ +ì ij +â ´ +ã ĭ +Ġب اÙĦ +á»ģ u +ĠاÙĦ ÙĦ +à¸ķ ัว +ذ Ùĩ +ึ à¸ĩ +à¹ĥà¸Ĭ à¹ī +á»ĵ ng +à¸Ļ ั +ม าà¸ģ +ãĥ Ł +×ŀ ×ķ +à¸Ĺ ย +á»Ļ i +Ạ± +ả o +à¹Ĥ à¸Ķ +×IJ ׾ +ส าม +ÙĪ Ø¨ +à¸Ĺ ุ +ย ัà¸ĩ +×¢ ת +×ķ׳ ×ķת +à¸Ĥ ึ +à¸Ĥึ à¹īà¸Ļ +à¸ģ à¹Ī +Ạ« +á»ij c +ãģĹ ãĤĩãģĨ +á»ĭ ch +Ġ×IJ ×ķת +Ġש ×IJ +׼ ×ķ׾ +á»Ļ c +ع Ø© +à¸Ĺ ี +à¹Ģ à¸Ń +Ùĥ ت +ãģ » +Ạ» +ìĹ ħ +à¸Ń à¸Ńà¸ģ +اÙĨ ت +à¹Ħ ร +Ġ×IJ ×Ĺר +Ø· ر +ÙĨ د +ื à¹īà¸Ń +Ø· ÙĦ +×IJ ×Ķ +uy ên +í ĸī +×ij ×Ķ +à¸Ħ à¹Ī +à¸Ĭ à¹Īว +ãģĤãĤĬ ãģ¾ãģĻ +ÙĬ ب +×§ ׾ +ãĥ Ļ +Ä © +س ر +า ว +ãĤ ± +à¸ļ ริ +ר ×Ĵ +á»ĥ u +ØŃ ت +×ķ×ŀ ×Ļ +ب ÙĨ +êµ IJ +ÄŁ u +ãģª ãĤĵ +×ij ×§ +Ġפ ר +ắ n +ØŃ ÙĦ +×ij ×Ĺ +ấ u +×ij ×ķ×ĵ +ãĥ ¯ +Ġ׾ ×§ +ั à¸į +à¸ŀ ิ +×Ĺ ×Ķ +×ĸ ׼ +ãĥ¼ãĥ ł +ÑĤ елÑĮ +×ŀ ×Ļ×ĵ +ÙĬ Ø® +Ạ³ +ت ص +à¸ĺ ิ +è¾ ¼ +ì ĵ +Ùĥ Ø© +ÙĤ ب +à¸Ħ à¹Į +à¹īา ย +à¸ĵ ะ +า ะ +ë Ĵ +ê ¾ +ë · +ì ĩ +ê º +ì ģ +ë Ģ +ì ¾ +ë ½ +ë ļ +ì Ń +ì İ +á ij +ë Ĺ +ê Ĵ +à ¡ +à ¬ +ðIJ Į +ã ĩ +ðĿ Ħ +Ġ׾ ×IJ +ãģ¨ ãģĦãģĨ +Ġn hi +×Ļ ×ķת +Ġש ×Ķ +à¹ģล à¹īว +Æ°á»Ľ c +à¸Ķà¹ī วย +à¸Ĺ าà¸ĩ +׳ ת +פ ת +à¹ģ à¸ķà¹Ī +ư ng +à¸Ńย ูà¹Ī +à¹ī ำ +Ġ×IJ ׾ +Ùĥ Ùħ +ấ p +ล à¸ĩ +ãģŁ ãĤģ +×Ĵ ׾ +ห ร +ĠÑĢ Ðµ +à¹Ģà¸Ĥ à¹īา +ÙĤ ر +Ġ×Ķ ×¡ +ÙĪ ÙĬ +สาม าร +สามาร à¸ĸ +Äĥ n +à¸Ń ี +פ ×ķ +×Ļ׳ ×ķ +ว ัà¸Ļ +ặ c +íķ Ļ +×ŀ ת +ê u +Ạ¹ +Ùģ ÙĬ +×ŀ צ +à¸Ħ า +ãģĿ ãģĨ +ãĢ ħ +ا ز +ا Ùĩ +ר ×Ļ×Ŀ +ấ n +ห าร +ạ t +ÙĨ Ùĩ +à¹Ģ à¸Ħร +ج Ùĩ +׼ ×Ļ +ắ t +à¸Ħ à¹īา +ر Ø© +ãĥ ı +Ùĥ ÙĪÙĨ +ứ ng +Ġìļ ° +ย à¹Į +à¹Īว à¸Ļ +à¸ģ ำ +Ø« ر +Ñģ и +ĠاÙĦ Ø· +Ġ×Ķ ×¦ +ĠØ · +ĠاÙĦ ÙĪ +ê¹ Į +ØŃ ÙĬ +ار ات +à¹Ģ à¸ĭ +ب ا +г ÑĢ +ร ี +ืà¸Ń à¸Ļ +ع ت +ÙĤ اÙĦ +د Ùħ +Ø ¡ +Ġ×ŀ ×§ +×ĵ ×Ļ×Ŀ +×¢ ׾ +ãģ Ĵ +ëĭ ĺ +×¢ ×Ķ +Ġìĸ ´ +Ñģ ÑĮ +ÙĤ Ø· +ãĥ Ľ +èĢĥ ãģĪ +à¹ģ à¸Ļ +ÙĪ Ø§Øª +â u +ĠìĤ¬ ëŀ +ห ว +ĠاÙĦØ£ Ùħ +Ġ×Ķ ×ŀש +ب ÙĪ +à¸Ĭ à¸Ļ +ãĤĵ ãģ§ãģĻ +ว à¸Ļ +à¸ģร รม +×ŀ ×ķ×ĵ +Ùĥ اÙĨ +×ķ× £ +ол ог +ت ÙĨ +à¸ķ à¹Į +ê² ĥ +ר ×ĺ +ừ ng +×ķ×ij ×Ķ +Ùħ ØŃ +ĠÐ § +פ ×Ĵ +ส à¸ĸ +ãģĭ ãĤĬ +ını z +à¹Ģ ย +ãĥ¼ ãĥ³ +ãģĬ ãĤĬ +פ ש +ิ à¸ķ +Ø· ÙĨ +×Ļת ×Ļ +×IJ ׳ +ç ek +ì ª +×ŀ ×ij +ศ า +ãĤ¹ ãĤ¿ +à¸ļ ุ +×ĵ ×ijר +ãģĦ ãģı +ส ะ +à¹Ģ หล +ิ à¸ĩ +à¸ŀ ัà¸Ļ +ãģĦ ãģŁãģł +ãĤĤ ãĤī +à¹ī ม +ãģĵãģ¨ãģĮ ãģ§ãģį +าร à¹Į +ุ à¸ĩ +í ij +ì ¯ +ë ¼ +í Ĥ +ì · +ê ¡ +á ı +á Ĵ +ðĿ ľ +á © +ðŁ Ħ +ðIJ ¤ +Ġש ׾ +Ġ×ŀ ×Ķ +à¹ģล ะ +Ġ׼ ׾ +Ạ½ +á»Ļ ng +ذ ÙĬ +л е +× ¥ +ãģª ãģ© +ĠÙĪ Ø£ +หà¸Ļ à¹īา +ãģ¾ ãģ§ +à¸ķà¹Ī à¸Ń +à¸Ĺ ัà¹īà¸ĩ +ãģł ãģij +à¹ģà¸ļ à¸ļ +à¹Ģร า +פ ׾ +ãģŁ ãģĦ +à¹Ģล ย +ãģ£ãģ¦ ãģĦãĤĭ +ế p +ึ à¹Īà¸ĩ +ê ´Ģ +ê³ Ħ +׼ ×ķ +à¹Ģร ืà¹Īà¸Ńà¸ĩ +×§ ×Ļ +êµ Ń +פ ס +ت ÙĬ +ãĥ Ħ +Ġ×Ķ ×Ĺ +г и +ר×IJ ׾ +×ŀ ׾ +ĠØ£ ÙĬ +Ġع ÙĦÙĬ +ãģĭ ãģ£ãģŁ +ש ×Ļ +д Ñĥ +×ŀ ף +׳ ×ĺ +׳ ×Ļת +mi ÅŁ +׼ ×Ŀ +Ġ×ij ר +Ġ׾ ×ij +ĠÐ Ľ +ç e +×ķ׳ ×Ļ +ãĤĪãģĨ ãģ« +פ ×ķר +ãĥ į +Ùĥ ÙĬ +×Ĺ ×ª +Ùģ ÙĦ +Ġ×Ķ ×§ +Ġ×Ķ ×ij +Ġ×ŀ ס +à¹Īา à¸Ļ +п еÑĢ +à¹Īา ว +Ġ×ij ×IJ +ĠÙĪ Ùĩ +à¸Ļ ำ +Ġ×ij ש +׳ ×§ +ãģ© ãģĨ +ש ×ķת +×ĵ ×Ķ +à¹Ģ à¸ļ +ÙĨ س +Ġìļ° ë¦¬ +ส à¹Īวà¸Ļ +ล ัà¸ĩ +ج ز +Ġ×Ĺ ×Ļ +Ùĥ ثر +ล ะ +Ùĩ د +ĠÙĪ Ø¨ +اÙĦ Ùħ +à¹ģ ม +Æ¡ i +Ġ×ij ×Ĺ +ữ a +à¹Ģà¸Ĺ ศ +à¸ķ ัà¹īà¸ĩ +ог да +׾ ×§ +د د +สร à¹īาà¸ĩ +à¸Ĭ ี +Ùģ Ø¶ +à¹ģ ห +uy á»ĩn +ร ัà¸ģ +á»ĩ m +ส า +פ ×§ +ีย à¸ĩ +à¸ķ à¹Īาà¸ĩ +à¸Ħร ัà¹īà¸ĩ +ØŃ ÙĤ +à¹Ģ à¸Ńà¸ĩ +ائ ÙĬ +×ĺ ×¢ +اÙĦ Ø© +ิ à¹Īม +ãĤ ½ +د Ùī +Ġר ×IJ +ãģ£ ãģ¨ +ãĥĥ ãĥĹ +ÙĬر Ø© +ê± ´ +×ŀ ×IJ +×ķ ×ķ +ب ع +ãģ ² +ร าย +×ĵ ×Ŀ +ت Ùģ +à¸ķ à¸ģ +ạ ng +ãĤĴ è¦ĭ +à¸Ĭ ั +Æ°á» Ł +Æ°á»Ł ng +ج ب +×ķ×ŀ ר +ĠìĤ¬ëŀ Į +ó ng +ร ั +Ġ×Ķ ×ĸ +ר צ +Ġ×Ĺ ×ĵ +ذ ÙĦÙĥ +×ķר ×Ļ +ãģ¡ ãĤĥ +Ùģ Ø¹ +Ġ׾ צ +á i +à¹ĩ à¸ļ +ãģ İ +à¸ģ ิ +ạ c +ë© ° +ãģª ãĤĭ +×ķ׾ ×Ŀ +à¹ģ à¸Ĺ +×ķ× ¥ +м еÑĤ +ü ÅŁ +ÑĢ Ñı +ภĴ +ÑģÑĤ оÑı +ع ÙĪØ¯ +Ùħ ار +Ø· Ø© +à¸ŀ ื +к ÑĢ +à¹ģ à¸ģ +à¹Ĥ รà¸ĩ +×ij ×Ļ×ĺ +ê² ł +×ķ׾ ×Ķ +ØŃ ر +ืà¹Ī à¸Ńà¸Ļ +×ķ×ij ר +׊ש +ãĥķãĤ ¡ +×ŀ ×ĺ +ú t +Ġd ön +ắ ng +ëł ĩ +ẳ ng +ว à¸ģ +ص د +Ø® Ø· +à¸Ń ั +ãĤı ãĤĮ +سÙĦ اÙħ +à¹Ģร à¹ĩ +×Ļש ×Ļ +ج اÙĦ +ãģij ãĤĭ +à¸Ĭา à¸ķิ +ÙĪØ§ ÙĤ +à¹Ĥ à¸Ļ +ãģ¦ ãģĹãģ¾ +اع Ø© +ãĤŃ ãĥ£ +à¸į า +ÙĦا ÙĤ +ิ à¸ģ +ĠÑģ ов +ÑĢаРº +×Ļ׳ ×Ļ +ü ÄŁ +Ã¼ÄŁ ü +×§ ×ij +à¹Ī à¸Ńà¸ĩ +Ġger çek +à¸Ĺ ั +ов аниÑı +×ŀ ׼ +س Ø© +×Ļ× £ +le ÅŁ +Ùħ ؤ +ĠìĿ ĺ +à¸IJ าà¸Ļ +ĠÑģ об +Ġêµ Ń +×¢ צ +з в +ส à¸ĩ +ز ÙĦ +ãģı ãĤĮ +и ÑĢÑĥ +ت Ø£ +п олн +ìĺ Ģ +ÙĨ Ø´ +׼ ×IJ +Ùħ Ø´ +à¸Ķ à¹Į +ÙĪ ÙĬÙĦ +à¹ģ à¸Ĥ +ãģ£ãģ¦ ãģĹãģ¾ +но ÑģÑĤ +в л +Ùħ ÙĤ +را ج +å¤ ī +ë Ľ +â ¸ +ì IJ +à » +á ļ +â » +ê Ļ +â § +ð Ĵ +ðĿ ĩ +Ġ×IJ ת +ĠÙĦ ÙĦ +ĠØ£ ÙĨ +Ġ×ķ ×Ķ +ãģ« ãģ¯ +Ġ×Ļ ×© +ت Ùĩ +ÃŃ nh +ÙĬ ات +Ġ×ij ×ŀ +à¸Ļั à¹īà¸Ļ +à¸Ļ à¹īำ +Ãł o +à¸ķ าม +ãģ® ãģ¯ +d ır +Ġn ghi +ặ t +×ŀ ×Ļ×Ŀ +ãģ¦ ãģĦãĤĭ +Ġ×ij ת +หร ืà¸Ń +Ġس ÙĬ +ãģª ãĤī +à¹Ĥà¸Ķ ย +ı yor +à¸Ńี à¸ģ +á»ĩ nh +Ñĭ м +à¸Ĺุ à¸ģ +Ġ׾ ×Ĺ +Ġ×Ķ ×¨ +Ġ×Ķ ×Ļ +à¸ŀ ระ +à¹Ģว ลา +ĠØ º +ẫ n +m Ä±ÅŁ +׼ ×Ķ +á»ij n +ãģ§ ãģĹãĤĩãģĨ +ãĥ ¢ +à¸Ľ ี +ס ×Ļ +ãģĵ ãĤį +Ġ׾ פ +ร à¸ĸ +ê¸ Ī +à¸ģ วà¹Īา +ë ¬´ +á»į ng +ãĤĵ ãģ§ +ãĤĪãģĨ ãģª +á»ĵ i +ãĤ ¬ +ส à¹Īà¸ĩ +×Ļ׳ ×Ķ +à¸ĸ ูà¸ģ +à¸Ī ัà¸Ķ +Ġ×Ķ ×Ĵ +ãĥ ľ +×ŀ ×ķת +ÙĪ Ùĥ +ëĭ ¨ +ĠØ « +ãģ® ãģĮ +à¹Ģห à¹ĩà¸Ļ +ع ا +à¸Ļ ิ +Å ŀ +à¸Ń ะ +ãģĪ ãĤĭ +Ø« ÙĦ +ØŃÙħ د +à¹Ģà¸ģ ิà¸Ķ +פ שר +פ ×Ķ +ม ิ +ئ ÙĬس +à¸Ĺำ à¹ĥหà¹ī +×¢ ×ĵ +ìĭ ¤ +à¸Ĭà¹Īว ย +ĠاÙĦÙħ ÙĨ +ز ÙĬ +ع ÙĬ +Ġ׼ ×IJ +ạ nh +á» ¹ +ãĤĵ ãģª +ส ู +צ ר +Æ°á»Ľ ng +×ķ ×ķ×Ķ +à¹Ĥ ล +ĠاÙĦ Ùĩ +ว า +หล าย +Ñī е +à¸Ĥ à¹īà¸Ń +à¹īà¸Ń ย +ب Ø· +ка Ñı +ĠØ ¢ +Ġи Ñģ +ĠاÙĦ غ +à¸ģ า +à¸Ļ à¹Īา +ÙĬ ÙĪ +×ij ×ķר +á»ħ n +ว à¸ĩ +×Ļ× ĸ +ì² Ń +н им +ëŁ ° +×Ĵ ×ķר +ص ØŃ +ÙĦ ÙĪ +×Ĺ ×ķת +ส ุ +رÙĬ ÙĤ +ס ×ĺ +Ġ×ŀ ×¢ +ãĥĨ ãĤ£ +à¸Ħ ิà¸Ķ +ãĤį ãģĨ +à¹Ħ ล +à¸Ļ à¹Į +á»ı i +ÑģÑĤÑĢ Ð¾ +ส à¸Ķ +ส าร +ÙĪÙĦ Ø© +ầ m +ร à¹Īว +รà¹Īว ม +ร ุ +ĠاÙĦس ÙĬ +ìĺ ģ +Ġ×ŀ ×ij +פ ×ĺ +à¸ķิ à¸Ķ +×ĺ ×Ļ×Ŀ +Ġë ¬´ +ÙĤد Ùħ +Ġdü ÅŁ +ائ ÙĦ +м Ñĭ +ØŃ س +ÙĪ Øµ +×Ļ×§ ×Ķ +ãģ§ãģ¯ ãģªãģĦ +à¹Ģ หม +оÑĢ ÑĤ +í Ĩµ +ãģ IJ +к ÑĢа +ีย ว +ع ار +ئ Ø© +íĥ Ģ +ãģ«ãģª ãĤĬ +ج Ø© +ÙĪÙĤ ع +ÑĮ Ñı +×ķצ ×Ķ +ש ×Ŀ +ب ÙĤ +Ġ×Ļ ×Ķ +ÙĬ Ø· +ım ız +д еÑĢж +×Ļש ר×IJ׾ +غ ÙĬر +ร à¸Ńà¸ĩ +à¹Ģรีย à¸Ļ +Ġ×Ķ ×ĺ +หม าย +Ùħ Ùĩ +اÙģ Ø© +Ġо ÑĢг +ÙĪ Ùī +ãĥ© ãĤ¤ +×ŀ ׳×Ķ +ĠÄij o +Ġг оÑĢ +اÙħ Ø© +æ¥ ½ +Ø« ÙĬر +à¸ģิ à¸Ī +á»ĵ n +ÙĨ ب +ÑĢÑĥ д +ìĹ Ī +Ġ×Ĺ ×ijר +ÑĢаР¶ +ạ ch +ت ÙĪ +à¹Ĥ ม +×ij ×Ļ×ij +Ġí Ĩµ +aca ģı +جÙĦ س +à¹Ģà¸Ľ ล +ว à¸Ķ +à¸Ń ล +ãģŁ ãĤĬ +à¸Ľ ัà¸į +Ġìķ Į +عر Ùģ +à¹Ħ à¸Ł +Ø£ Ø® +å¤ļ ãģĦ +à¸Ķ ัà¸ĩ +Ø´ Ùģ +ãģ£ãģ¦ ãģĦãģ¾ãģĻ +׼ ×ł×¡ +ÑĨ е +еÑģ п +Ùħ اÙħ +à¸ŀื à¹īà¸Ļ +иÑĩеÑģ ки +Ø® د +Ùĥ ÙĪÙħ +Ġ×Ķ ×¨×IJש +ت اب +é£Ł ãģ¹ +ื à¸Ļ +оÑĢ Ð¾ +Ġb öl +×ķ×Ĺ ×ĵ +دÙĬ ر +ắ m +د ع +ãģķ ãģĽ +à¸ĺ ร +à¸ĺร รม +ãģĭ ãĤĤ +å¤ļ ãģı +r ä +س ع +×Ļ׾ ×Ķ +ض ر +ĠاÙĦ شر +×ĸ ×ķר +×¢ ×ijר +ạ m +алÑĮ но +ر ÙĨ +اÙħ ج +׼ ×ļ +d ıģ +д ен +ض ا +ÙĦÙĬ Ùħ +Ġê·¸ 룬 +تÙħ اع +ار ÙĬØ® +à¹Ĥ à¸ķ +ĠÑģ ÑĢед +Ġ׳ ×ķס +ÙĤ بÙĦ +оÑĤ ов +le ÅŁtir +Ġм еÑģÑĤ +سÙĦ Ùħ +Ġ×¢ צ +ĠاÙĦس ÙĦ +еÑĤ ÑĮ +اب Ø© +н ак +สà¸ĸ าà¸Ļ +Ġ×ij ׳ +à¸ļ ัà¸Ļ +׼ ׳ +Ġö ÄŁ +ãģ¨ è¨Ģ +uy ến +di ÄŁ +áºŃ u +ÑĢ Ð°Ñģ +ãĤ· ãĥ§ãĥ³ +n ız +×ķ×ĵ ×Ķ +ت س +Ùħ اÙĦ +à¹Ģห à¸ķุ +ย ว +à¸ŀ ัà¸ģ +ãģĦ ãģªãģĦ +Ġк аÑĩ +ล à¹Į +ר׼ ת +ÅŁt ur +×ŀ ×ķס +ãģ ¥ +б ол +عÙħ اÙĦ +×ķר ת +ÑĨи он +ศ ึà¸ģ +ภı +ÑĢ ÐµÐ½ +اس ÙĬ +ائ ر +à¹Ĥ à¸Ľà¸£ +Ġse ç +غ ÙĬ +Ñį ÑĤ +ен н +ãģª ãģ® +×Ļש ×Ķ +×Ļפ ×ķר +ãģŁãĤģ ãģ« +ز Ø© +Ġç oc +ãĤ¯ ãĥª +ÑĪ ÐµÐ½ +ãĤı ãģij +رÙĬ د +ĠÑĢ Ð°ÑģÑģ +Ùĥ ات +ส à¸Ńà¸ļ +ce ÄŁi +ãĤ¿ ãĤ¤ +à¸ļ ร +ĠاÙĦ بر +׳ ×ķ×¢ +r ün +را ض +ศา ส +à¸ķ รà¹Į +ãģį ãģŁ +×ķ׾ ×ĵ +еÑĢ Ð¸ +íĹ ĺ +ắ p +ت عÙĦ +Ùĥ د +иÑĤелÑĮ но +Ø· Ùģ +Ġав ÑĤом +Ġ×ŀ צ +ÑĪи Ñħ +ات Ùģ +ĠÑħ оÑĤ +Ùİ Ø§ +ãģı ãĤĭ +×Ķ ×¤ +à¹Ĥ à¸Ĺ +à¹ģ à¸ŀ +à¹Ī à¸Ńย +ĠاÙĦÙħ Ø´ +à¸ģาร à¸ĵà¹Į +ани з +×Ķ ×ľ +ظ Ùħ +ย ุ +li ÄŁ +à¹Ħ à¸Ĥ +à¸ĸ ืà¸Ń +ö z +ãģij ãģ¦ +à¹Ģ à¸ľ +ุ ม +ãĥĹ ãĥ¬ +Ġ×Ķ×IJ ×Ĺר +خت ÙĦÙģ +à¸ İ +ÙĦا ØŃ +Ġdü zen +צ ×Ķ +س اء +×ķר ×ļ +×ķ×ĵ ×Ļ +ÑĢа ÑĦ +ÅŁt ır +ãģ« åħ¥ +ãģĪ ãģ° +ص ÙĪÙĦ +ĠÐľ оÑģ +ا Ùĩر +ãģ£ ãģ +ĠлÑİ Ð± +×Ļ×¢ ×Ķ +Ġ×Ķ×ŀ ×§ +สิ à¸Ĺ +สิà¸Ĺ à¸ĺิ +×Ļ׳ ×Ŀ +ÙĦا Ùģ +à¸ŀัà¸Ļ à¸ĺ +×ķ×IJ ×Ķ +ม ั +à¸Ĥ à¸ĵะ +д оÑĢ +ãģ¨ ãģª +à¸ģระ à¸Ĺ +ac ı +×ķ׾ ×ķ×Ĵ +Ñĥ ÑĪ +ãĥ¥ ãĥ¼ +ãĥ ¦ +Ùħ ست +Ġa ÅŁ +ש ×§ +פ ת×Ĺ +าย à¸Ļ +í ĩ +ë ¢ +ï · +í ī +ì µ +ì ¬ +ðĿ Ľ +ì Ĵ +ë Ļ +ê § +á ĸ +â ¨ +â ± +á ĺ +ð ĸ +à ł +á Ķ +ðIJ Ń +ữ ng +Å© ng +Ġ×Ķ ×ª +ĠاÙĦ ا +Ġ×ŀ ת +à¸ĸ ึà¸ĩ +ò n +á»ĭ nh +нÑĭ м +Ġc ả +à¸Ķ ู +Ġ à¹ģà¸ķà¹Ī +Ġ×ij ×Ķ +ó i +ãģ¨ ãģĹãģ¦ +ú ng +ĠØ ° +Ġ×Ķ ×ł +Ġب ÙĨ +ÙĦ اÙĦ +à¹Ħ à¸Ĺย +á»ĩ p +t ı +ม ัà¸Ļ +ằ ng +á»ij t +к ом +à¸ĭ ึà¹Īà¸ĩ +à¸Ħร ัà¸ļ +à¸ļ à¹īาà¸Ļ +ĠاÙĦ ÙĬ +l ü +ÙĪ Ø³ +ãģł ãģ£ãģŁ +à¹Ģ à¸ĩ +Ġê³ µ +н Ñĥ +ãĤĪ ãĤĬ +м Ñĥ +à¹Ģà¸Ĥ า +ãĤ Ģ +ни е +ãģ«ãģª ãĤĭ +áºŃ y +ĠÙĪ Ø§ +ëł ¤ +ש ×ķ +á p +×ĵ ×ķ +ãģ§ ãģĹãģŁ +ع ض +Ñģк ой +æĦŁ ãģĺ +ÑİÑĤ ÑģÑı +Ġ×Ļ ×Ľ×ķ׾ +ãĤĵ ãģł +в и +à¹Ģล à¹Īà¸Ļ +ìĿ´ ëĭ¤ +ĠÙĦ Ùĩ +à¸Ħ ืà¸Ń +ت Ùĥ +Ùħ ÙĥÙĨ +a ģı +׳ ×ĵ +ë¯ ¼ +à¹Ħ ว +สำ ห +สำห รัà¸ļ +Ñģл ед +t ır +ĠÙĦ ÙĬ +ĠاÙĦع ÙħÙĦ +×ij ×ķת +×ij ×Ļ×Ŀ +à¸Ħ ำ +à¹Ģà¸Ħร ืà¹Īà¸Ńà¸ĩ +lı ģı +ืà¸Ń à¸ĩ +ج د +íŀ Ī +ìĭ ¬ +×¢ ×ķת +ส ิà¸Ļ +Ñĩ и +ر ض +à¹Ģà¸Ľ ิà¸Ķ +à¸Ħ à¹Īา +ìĦ ł +ÙĪØ± Ø© +×§ ×ĺ +ìľ ł +ع ÙħÙĦ +×IJ ×Ļ×Ŀ +׾ ×Ļ×Ŀ +à¹ĥห à¸į +à¹ĥหà¸į à¹Ī +ừ a +á»į i +ãģ ¶ +ÃŃ ch +ãĥĩ ãĤ£ +×ķר ×Ļ×Ŀ +Ñģ о +ìķ ½ +ов а +Ñĩ аÑģÑĤ +à¹Ģà¸Ī à¹īา +п ÑĢо +Ġ×ŀ ×Ĺ +ãĥ İ +×ķ×Ļ ×ķת +Ġд е +ë§ Ī +ì§ ģ +×Ļפ ×Ķ +ĠاÙĦع اÙĦÙħ +ë¥ ´ +ר×IJ ×Ķ +uy á»ĥn +×¢ ×Ļ +ม ืà¸Ń +Ø¥ ÙĨ +ร ู +ĠØ ² +×Ļ ×ķ×Ŀ +à¸ķ à¹īà¸Ļ +ãģ¦ ãģĦãģ¾ãģĻ +Ùħ اÙĨ +ĠÐ ¥ +à¸Ľà¸£à¸° à¹Ģà¸Ĺศ +á» ³ +׾ ×ij +à¹Ģà¸Ķ à¹ĩ +ãģŁ ãģ¡ +à¸Ĺี ม +à¸Ļ ะ +ìĹ ° +Ġìł Ģ +ÙĦ Ùĩ +ợ i +ĠاÙĦ ز +د ار +ãĤ³ ãĥ³ +м ин +à¹ģห à¹Īà¸ĩ +à¸Ķ ัà¸ļ +׼ ר +ж а +íĸ Ī +×ŀ ×ĸ +ợ i +à¸Ķ า +Ġع بد +à¹ģ ร +×IJת ר +×¢ ׳×Ļ +à¹Ģ à¸Ħ +×ķצ ר +ì§Ģ ë§Į +ائ Ùħ +Ø£ س +uy á»ģn +Ġ×IJ ׳ +׊׳×ķ +×ĸ ×Ļ +ร à¹īาà¸Ļ +ĠÐł оÑģ +ĠÐłÐ¾Ñģ Ñģ +رب ÙĬØ© +t ür +ãĤĭ ãģĵãģ¨ +ظ ر +б Ñĭ +à¸Ĺีà¹Ī สุà¸Ķ +Ġצ ר +èĩª åĪĨ +л аÑģ +ĠÑı в +ĠÑıв лÑı +à¸ŀร à¹īà¸Ńม +à¸Ńา à¸Ī +à¸ļริ à¸ģาร +Ġç ı +ëį ĺ +ĠاÙĦÙħ ست +ت Ø´ +ש ×ķ×ij +ãĤ ´ +Ġyap ıl +ĠاÙĦ ذ +ุ à¹Īม +à¸ĸ à¹īา +ìĦ ¤ +ì° ¨ +в аÑĢ +à¹Ģà¸ŀ ิà¹Īม +Æ°á»Ľ i +Ùĥ س +à¸Ńย าà¸ģ +ãģ¦ ãĤĤ +Ġг од +ÙĬ ار +à¸ķ à¸Ńà¸Ļ +Ġиг ÑĢ +à¹Ħà¸Ķà¹ī รัà¸ļ +ĠاÙĦÙħ ر +ÙĤ ت +Ġë ĺ +Ġëĺ IJ +ẩ n +ãģĻãĤĭ ãģĵãģ¨ +×Ĵ ×Ŀ +Ġ×ij ×ij +ت د +ÙĪ Ø§Ø± +ãĤ ® +п ол +Ġм ог +تر Ùĥ +ÙĪ Ø« +Ġç ık +ا Ø© +à¹Ģà¸Ķ ียว +มี à¸Ħวาม +Ġ×ŀ ×Ĵ +ص Ùģ +ĠТ ак +Ġ׼ ת +×Ļ×ĵ ×Ļ +ов оÑĢ +ầ y +สิ à¹Īà¸ĩ +ب ت +ür ü +ÙĨ ج +หล ัà¸ģ +×Ļ×Ķ ×Ŀ +ÙĤ ص +з Ñĭ +×Ľ×ª ×ij +ư u +m ız +ĠìĦ ¸ +л ог +Ùħ ÙĬÙĦ +ÙĬ ج +íĴ Ī +à¸ŀ à¸ļ +ห ัว +з на +ר ×§ +à¹Ĥ ร +Ġ×ij ס +ĠBaÅŁ kan +ĠëĶ ° +à¸Ń ัà¸Ļ +ีà¹Īย ว +н еÑģ +à¹Ģà¸Ķ ิà¸Ļ +ÙĬ اÙĨ +×ķ׾ ×Ļ +ا خت +צ ×ķת +ãģĵ ãģĵ +ĠاÙĦ اÙĨ +ĠпÑĢо ÑĨ +ãģ¾ ãģł +׼ ס +ĠاÙĦ Ø¢ +ÙĬ ز +ĠاÙĦد ÙĪÙĦ +Ġíķĺ ëĤĺ +ض ع +ê» ĺ +ÅĽ wi +ย ิ +ãģ¡ãĤĥ ãĤĵ +ĠÙħ Ø´ +à¸ĺ ี +ãģ¨ ãģį +׳×Ļ ×ķת +Ġë ¯ +Ġë¯ ¸ +Ġs ı +ëĭĪ ê¹Į +Ġп л +غ ÙĦ +à¹ģ รà¸ĩ +ب ÙĬر +ãģĤãĤĬ ãģ¾ãģĽãĤĵ +ê· ¼ +Ġy üz +ĠdeÄŁ er +åł´ åIJĪ +á» ¡ +м аÑĤ +รา à¸Ĭ +ÙĪØ± ÙĬ +ж ен +ãģ¾ ãĤĬ +ãģ® ä¸Ń +×Ļ×ĵ ×¢ +à¸Ń ุ +à¸ļ à¸Ńล +à¸Ľà¸±à¸į หา +ز Ùħ +ÄŁ a +à¸Ń ืà¹Ī +à¸Ńืà¹Ī à¸Ļ +п л +Ġне обÑħодим +׼ ×ij +à¹Ģ ศ +קר ×Ķ +ì² ĺ +ëł ¨ +×ŀ×§ ×ķ×Ŀ +jÄħ c +Ùĩ ÙĦ +Ġ×¢ ×ij×ķ×ĵ +à¹Ħม à¹ī +à¸ģล ัà¸ļ +×ķ׼ ׾ +×§ ×ĵ +اÙĦ ÙĬØ© +ر Ùĩ +ãģij ãĤĮãģ° +ĠÙĨ Ù쨳 +ãĤ¢ ãĥ« +ìĹ Īëĭ¤ +×§ ×ķר +н еÑĢ +ب اب +ãĤ ¶ +سب ب +ÙĦ ÙĬÙĦ +ص ÙĨ +ص در +ế m +à¸Ĭà¹Īว à¸ĩ +ØŃ ÙĨ +Ġ×ij ×Ĵ +×ŀ ×ķ×¢ +׾ ×Ĺ +大 ãģį +ت ب +н еÑĤ +×Ļ×ij ×Ķ +б л +ãĥĹ ãĥª +اص Ø© +ãģ¤ ãģij +×Ļ×ŀ ×ķש +ãģĮ ãģĤ +ëĭ ´ +ãģĭãĤĤ ãģĹ +ãģĭãĤĤãģĹ ãĤĮ +ãģ¡ ãĤī +×ij ×ĺ +Ġba ÄŁ +×Ļ×Ĺ ×¡ +×ij ×ķ×¢ +ล ี +פע ×Ļ׾ +им и +g ÅĤ +Ġим е +خد اÙħ +×IJ ×Ļר +Ġy apt +ãģ¨ ãģĦ +à¸ĩ à¹Īาย +׾×Ļ ×ķ +ØŃد Ø« +را ÙĤ +ĠÄIJ i +اد ر +ãģĵãģ¨ ãĤĤ +×ij ×Ļר +Ġв з +ض اÙģ +ת ×ķ׼ +ÑĢ Ð¾Ð¼ +ر ات +à¹Ģà¸Ĺ à¹Īา +ãģĺ ãĤĥ +ãģĿ ãģĵ +اج تÙħاع +à¹īà¸Ń à¸Ļ +ÙĤ Ùħ +ë³ ¸ +Ä ŀ +ש ×Ļ×ķ +×ij ׳×Ļ +ìľĦ ìĽIJ +à¹ģ à¸Ī +×Ĺ ×ķר +دÙĬ ÙĨØ© +ت Ø· +ằ m +ò a +ย à¸Ńà¸Ķ +Ġëĭ ¹ +สุ à¸Ĥ +×ĵר ×ļ +د ÙĨ +س ÙĬÙĨ +ÙĪÙĤ Ùģ +ÑĨ Ñĭ +г оÑĤов +еж дÑĥ +à¸ŀ วà¸ģ +اÙĤ تص +اÙĤتص اد +cz ÄĻ +ni ÄĻ +ÑĢ ÐµÐ± +ØŃ ÙĪ +à¸Ĺ à¹Į +ãĤĪ ãģŃ +д ж +à¸ģล à¹Īาว +دÙĬ Ø« +ãĤ³ ãĥŁ +ÙĤ ÙĪÙħ +Ġت ØŃ +à¹Ģ à¸ķิ +اÙģ Ø¸ +à¸Ī ุ +رÙĬ اض +×ŀש ×ļ +à¹Ĥ ย +еÑĢ Ðµ +ãģ¿ ãģŁãģĦ +ìĿ´ ëĿ¼ +ĠاÙĦÙħ ÙĪ +ĠÑģÑĤ о +à¹Ģรà¹ĩ ว +Ġд еÑĤ +ĠÑģ дел +à¹Ģà¸Ĭ ืà¹Īà¸Ń +פ ׳×Ļ +ÙĪØ¶ ÙĪØ¹ +×ij ס +à¹ģ à¸Ķ +ó c +ริ ม +ÑĢаР´ +ìĪ ł +ãĥ¼ãĤ º +ãģ« ãģĬ +и но +פ ×Ļ׾ +à¸Ĭั à¹Īà¸Ļ +×Ĺ×ĵ ש +à¹Ģà¸Ļ ืà¹Īà¸Ńà¸ĩ +׳ ×Ļס +غ رب +ãĤ¸ ãĥ£ +ส ัà¸ĩ +à¹Ģ à¸Ĺีà¹Ī +à¹Ģà¸Ĺีà¹Ī ยว +ëŁ ¼ +à¹ģ à¸Ł +ãĥ¼ãĤ · +ãĥ¼ãĤ· ãĥ§ãĥ³ +Ġвоз мож +جÙħ ÙĪØ¹ +×ijר ×Ļ×Ŀ +ãĥĪ ãĥ© +ĠкаÑĩ еÑģÑĤв +Ø· ÙĬ +ÑĤ Ñı +צ ×ķ×¢ +ÄŁ ını +ع ÙĦÙī +ا ذ +ÙĪØ§ÙĤ ع +Ùħ ÙĪØ§ +ائ ÙĬÙĦ +к ол +á»ģ m +à¸ľà¸¥ ิà¸ķ +×Ļ׳ ×ĺר +س Ùĥ +ש ×Ļר +ศึà¸ģ ษา +à¸ļ ั +Ñĩ аÑģ +×ķפ ×Ķ +×Ļפ ×ķ׾ +ĠاÙĦس اب +رÙĬ ب +ĠاÙĦ بÙĬ +ãĤ¹ ãĥĨ +Ñĩ ен +à¹ģ à¸ľ +Ġ׳ ש +ز ÙĬد +ØŃ اد +ëį Ķ +رÙĪ Ø¹ +à¸Ĺุ à¸Ļ +ส มา +c zeÅĦ +×Ļ×ĵ ×Ķ +ãģ§ ãģĤ +Ġçoc uk +Ø® ب +à¸ļ าย +à¸Ľà¸£à¸° à¸Ĭา +×ŀש ׾ +ãģª ãģĭ +à¸ģ าย +ãĥģ ãĥ£ +аÑĢ Ð¸ +ĠÑĩ а +à¸Ķ ำ +à¸Ĺั à¹Īว +Ñĥ Ñħ +Ġö z +Ġì¢ ĭ +ج رÙĬ +ائ ÙĤ +à¸ł ัย +Ø· ار +د ارة +Ä© nh +Ø« ÙĨ +zell ik +اÙĦ ت +Ġg eli +ãĥķãĤ © +ол од +رب ع +שת ×ŀש +à¸ļร ร +íĿ ¬ +Ġü rün +Ġê·¸ ëłĩ +ศาส à¸ķรà¹Į +ãģ ľ +×Ļ×ij ׾ +ĠпÑĢед ÑģÑĤав +سط ÙĬÙĨ +ãĤĴ 使 +Ġпом оÑī +×ķ×§ ר +ãĥ¯ ãĥ¼ +Ġyö net +×Ļ×§ ר +à¸Ĥ า +еÑĢи ал +ØŃ Ùģ +Ġ×Ļ ×¦ +à¸Ĺ ิ +å£ ² +à¸Ļ à¸Ńà¸ģ +×ķ׼ ר +íĻ ľ +á»§ y +ĠاÙĦÙĤ ر +×Ļ×ij ×ķת +ÅĽ ni +Ùħ شار +ượ t +ĠÙĦ دÙĬ +ÑĤ ел +ĠØ¥ ÙĦÙĬ +عÙĦ ÙĪÙħ +ìķ ĺ +в иÑĤ +à¸Ħ ะ +yr ı +ãģ¨ ãģ£ãģ¦ +à¹Ģ à¸ī +à¸ĸ าม +ÙĤ ار +عÙĦ اÙħ +ặ ng +Ùħ ÙĴ +×Ļ×ŀ ת +سب Ø© +ãĤ¯ ãĥ© +×ķס ×£ +ĠпÑĢ Ð¸Ð½ +ãģĦ ãĤį +س اس +عت بر +วิ à¸Ĺย +วิà¸Ĺย า +س Ùĥر +ãĤ· ãĥ§ +ãģ ģ +ัà¸ģ ษ +×ij ×ķ×Ķ +ห ย +ãģ¾ ãĤĮ +ĠоÑĢг аниз +каз ал +ĠÑģв Ñıз +uy ết +ĠпÑĢо из +Ġ×§ ×ĺ +à¹ģà¸ģ à¹ī +п ÑĥÑģ +Ġê·¸ ê²ĥ +ëĬ IJ +л екÑģ +ãĥ¼ãĥ Ĺ +à¸ķ ำ +ת×Ĺ ×Ļ׾ +à¸Ńà¸ĩ à¸Ħà¹Į +Ạµ +׳ צ +Ø£ Ø´ +Ø´ Ùĩ +ย ะ +à¸ģ à¸İ +ĠاÙĦØ¥ سÙĦاÙħ +ед ÑĮ +ãģ² ãģ¨ +ëıĦ ë¡Ŀ +ãģ© ãģ® +Ñĥ в +еÑĩ ение +ĠاÙĦت ج +ãģ« è¡Į +Ġп озв +ãĤı ãĤĬ +ÙĦ اث +íķĺ ìĺĢ +Ġм аÑĢ +Ġkon uÅŁ +ãĥ¬ ãĤ¹ +ãĤĴ æĮģ +ĠоÑģ нов +×Ĺ ×ij +ÙĪØ¬ ÙĪØ¯ +פ ×ķף +в оÑĢ +Ġн ик +ãģĭ ãĤĭ +ÅŁtır ma +×Ļס ×ĺ +Ø£ ÙĦ +ห à¹Į +и она +лÑĮ н +Ġг оÑģ +ĠÐľÐ¾Ñģ к +ÑĢ Ð¾Ð± +×ķ×IJ ×Ļ +ãģĬãĤĬ ãģ¾ãģĻ +ãģ£ãģ ± +к л +à¸Ļ à¸Ķà¹Į +رÙĬ Ùģ +اس ب +ĠÑĢ ÐµÑĪ +Ġд ол +ãģ¹ ãģį +×Ļ×ij ×ķר +м еÑī +Ġна ÑĪ +à¹ģ à¸Ľà¸¥ +ÑĢ Ð¸ÑĤ +кÑĥ Ñģ +и ÑĢа +аÑĤ ÑĥÑĢ +ÙĪØ§ صÙĦ +à¹Ģà¸ľ ย +à¸Ń ำ +à¹Ģà¸ģ ิà¸Ļ +غ Ùħ +ãģĻ ãģİ +lı kl +ÅĦ sk +ê² ¬ +×Ļ׼ ×Ķ +׊ש×ij +ÙĪØ± ÙĬØ© +Ġд ейÑģÑĤв +×Ĺ׾ ×ĺ +Ġ׾ ×ŀ×¢ +צ׾ ×Ļ×Ĺ +еÑĩ а +Ùģ Ø§Ø¹ +×Ĵ ×Ļ×ĵ +áºŃ m +ÄĻ b +Ø´ ع +ãģı ãĤĬ +à¸ŀ ุ +ед еÑĢ +à¸Ĥ à¸Ļ +à¸Ħ าร +ĠболÑĮ ÑĪ +ãģı ãģªãĤĬ +à¸ĵ า +×ĵ ×ķ×Ĵ +Ġм н +ä¸Ĭ ãģĮ +ç¶ļ ãģį +ฤ ษ +ภĨ +Ø® ÙĬ +à¹Ģà¸Ĺ à¸ŀ +สั ม +à¹Ģส à¸Ļ +à¹Ģสà¸Ļ à¸Ń +ãĥ ´ +Ġи ÑģÑĤ +با شر +ĠÑĥ ÑĢов +×ŀ ×ķ×ĸ +ab ı +wa ż +×ķצ ×IJ×Ķ +ÑĤ веÑĢ +à¸ŀัà¸Ļà¸ĺ à¹Į +׳ ×Ĵ×ĵ +ãĤĭ ãģĵãģ¨ãģĮãģ§ãģį +ĠÑĤÑĢ ÐµÐ± +à¸ģร ุà¸ĩ +ØŃت اج +à¹Ģ à¸Ħล +ã Ĩ +ÄĻ tr +Ġszcz eg +Ġר ש +à¸Ĺ à¸ĺ +Ġн ек +Ġнек оÑĤоÑĢ +в ÑĪ +Ð ¬ +à¹Īว ย +ล ุ +б ÑĢÑı +หม ูà¹Ī +à¹ģ à¸ķà¸ģ +ר׼ ×Ļ×Ŀ +Ġí ĸī +ã i +Ùĥر Ø© +â Ń +í IJ +ã į +á ģ +â ® +â ¥ +ì ® +à ¿ +â ¿ +á Ĥ +á ¤ +â ł +í Ł +ðIJ į +ðIJ ° +ðĿ Ĩ +ðŁ Ī +Ġ×¢ ׾ +Ġع ÙĨ +ĠÙħ ع +Ġ×ĸ ×Ķ +ĠÙħ ا +Ġm Ãł +Ġd ụ +á»ĩ c +а Ñħ +s ı +íķĺ ê³ł +Ġ×ķ ×ij +ĠÐŁ о +×ķת ר +ĠÙĦ Ùħ +Ġ×ķ ׾ +ãģĹãģ¦ ãģĦãĤĭ +Ġ×ŀ ×Ļ +Ġب ÙĬÙĨ +з а +ĠÙĥ اÙĨ +Ġ×Ķ ×Ļ×Ķ +ëħ Ħ +×IJ ×ķ +д и +ĠпеÑĢ Ðµ +d ı +Ġ׾ ש +Ġש ×ŀ +ãģĮ ãģĤãĤĭ +ãģĦ ãģĦ +ÑĢ Ðµ +×§ ×ķ +и ли +м е +ÙĬ ت +ãģ§ ãģĤãĤĭ +Ġв о +à¹ĥ หม +à¹ĥหม à¹Ī +Ġש ×ij +Ġ à¹Ĥà¸Ķย +ÙĬ Ùĩ +ãģ§ãģĻ ãģĮ +ãģ¨ ãģ¯ +ר ×ķ +Ġ à¸ĭึà¹Īà¸ĩ +ãģ§ãģį ãĤĭ +м о +à¹Ģà¸ŀ ืà¹Īà¸Ń +צ ×ķ +×ĺ ×ķ +ìķ Ī +Ġh á»į +à¹Ģà¸ĩ ิà¸Ļ +ĠاÙĦ ب +Ġ มี +ë¬ ¼ +Ñģ е +ëĵ¤ ìĿ´ +Ġë§ IJ +Ġl Ỽ +a ÅĤ +×Ĺ ×ijר +Ġd á»± +ÙĬ Ø« +Ġth á»ĭ +à¸ģà¹Ī à¸Ńà¸Ļ +Ġ×ij ׼׾ +ãģ ¸ +ã썿ĢĿ ãģĦãģ¾ãģĻ +ả nh +ย า +Ùģ Ø§ +ส ี +à¸ķ า +ë² ķ +ãĥª ãĥ¼ +รา à¸Ħา +Ġ×ķ ׾×IJ +ãģ¨ ãģĵãĤį +à¹Ģล ืà¸Ń +di ÄŁi +ÙĪ Ø§ÙĨ +Ġ׾×Ķ ×ª +รว ม +פ ×Ļ×Ŀ +à¸ľ ม +ж и +c ı +ÑĢ Ð¾Ð´ +Ġkar ÅŁÄ± +×Ĵ ×ķ +ãģ« ãģ¤ +ãģ«ãģ¤ ãģĦãģ¦ +r Ãł +×Ļ×ķת ר +ĠìĨ Į +×§ ×Ķ +ÑģÑĤв о +ãģij ãģ© +g é +à¸Ķ à¹īาà¸Ļ +çļĦ ãģ« +ĠÙĬ ÙħÙĥÙĨ +ìĨ į +ÙĬ Ùĥ +à¹Ħว à¹ī +Ñģки й +ì m +Ġ׾×IJ ×Ĺר +à¸Ńา หาร +Ġà¹Ģ à¸ŀ +รา ะ +ล ูà¸ģ +ÑģÑĤ а +Ġìľ ł +ÙĤ ÙĪÙĦ +б оÑĢ +Ñģк ого +หล ัà¸ĩ +à¸Ĥ à¹Īาว +à¹Ģม ืà¸Ńà¸ĩ +ê° ģ +t Ãł +ÙĬ ÙĬÙĨ +عر ض +ë° © +Ġëı Ļ +Ġà¹Ģ à¸Ľ +Ġà¹Ģà¸Ľ à¹ĩà¸Ļ +ç i +li ÄŁi +ìĹIJ ê²Į +ãĤ¿ ãĥ¼ +Ġ׾ ת +פ ×ķת +à¸Ĥ à¸Ń +ر س +ìł IJ +à¸ľ à¹Īาà¸Ļ +ÑĦ и +ج ÙĨ +ì¢ ħ +Ġ×Ķ ×¤ +Ġn go +á»ĭ a +Ġtá» ķ +Ġê·¸ 리 +à¹Ģม ืà¹Īà¸Ń +ذ Ùĥر +ìĸ ij +ìĹ Ń +×ĺ ׾ +k ı +Ġع ÙħÙĦ +Ġع ÙĨد +à¸ĭ ืà¹īà¸Ń +Ġê± ° +в е +r ü +à¹Ģ à¸Ńา +ส à¹Į +à¸Ī à¸Ļ +ס ת +Ġgi ả +ãĤĭ ãģ¨ +à¸ģำ ลัà¸ĩ +н ей +à¸Ī ริ +à¸Īริ à¸ĩ +Ġë į +Ġëį Ķ +à¸Ħà¹Ī ะ +ì n +Ġsü re +Ġqu y +à¸ļ าà¸ĩ +åıĸ ãĤĬ +ר ×Ĺ +×ij ת +ãģĮ ãģĤãĤĬãģ¾ãģĻ +ר ש +ìĹIJ ëĬĶ +Ġ×IJ פשר +ay ı +ãģĮ ãĤī +ØŃ ب +ан Ñģ +س ÙĪ +ĠпÑĢ Ðµ +د ÙĪ +ãģ« ãĤĪ +à¹Ģà¸ģ ม +สู à¸ĩ +m akt +makt ad +maktad ır +Ġön em +×Ļ×ŀ ×Ļ×Ŀ +б о +ÙĪ ÙĬØ© +รู à¸Ľ +à¹Ĥล à¸ģ +Ùħ ÙĬع +ÑģÑĤ Ñĥп +à¹Ĥ à¸Ń +دÙĬ ÙĨ +ì¤ ij +ãģĹãģ ı +à¹Ģส ีย +в Ñĭ +Ùħ ت +íĺ Ħ +ãĥIJ ãĥ¼ +ا Ø´ +×§ ס +Ġtá» ¥ +ล à¸Ķ +Ùģ Ø© +í ijľ +ر ج +k ÅĤad +ĠÅŁ ey +ĠØ£ Ùħ +Ġà¹Ģ ม +Ġب ÙĦ +Ñģ каÑı +ãģ¨ ãģ® +Ġìĭ ¤ +ấ m +ห à¹īà¸Ńà¸ĩ +à¸Ĭ ม +d ü +Ġç ek +Ġê³ ł +×Ĵ ×ij +à¸Ĭี วิ +à¸Ĭีวิ à¸ķ +Ù쨶 ÙĦ +ภ¯ +ç ı +Ġب Ø´ +ĠÙĩ ÙĨا +ãģį ãģ¾ãģĹãģŁ +t ü +Ġìĺ ģ +ĠTür k +к ÑĤ +פר ס +ãģ¨ãģĦãģĨ ãģĵãģ¨ +í ĶĦ +à¹ģร à¸ģ +ר ×ķף +Ġar as +×ŀצ ×IJ +Ġtá» ī +س ا +à¸ŀ à¸Ń +ĠاÙĦÙħ ØŃ +ãĥ ¤ +ĠاÙĦ است +Ùģ ÙĨ +×Ļ×ŀ ×Ķ +ر ت +ãģ¨ ãĤĤ +Ġна Ñģ +п ÑĢи +Ġ×Ĺ ×ķ +и ла +ÙĬ Ø´ +Ġgö z +Ġ×ij ׳×Ļ +ım ı +ĠÑĤ еÑħ +Ġh á»Ļ +غ ر +к он +اØŃ ت +Ġ à¸ŀ +à¸Ń à¸Ńà¸Ļ +à¸Ńà¸Ńà¸Ļ à¹Ħล +à¸Ńà¸Ńà¸Ļà¹Ħล à¸Ļà¹Į +Ñħ о +Ñı в +à¹ģ สà¸Ķ +à¹ģสà¸Ķ à¸ĩ +à¹Ģà¸ŀ ียà¸ĩ +ÑĤ ов +ا ÙĬ +Ġ×Ķ ×ĵ +Ġ×ķ ׼ +ãĤī ãģĦ +×ķפ ף +Ġë ¶Ī +ล à¸Ńà¸ĩ +Ø· اÙĦ +Ġн и +ĠÙħ ست +ế c +Ġש ׼ +ĠëķĮ 문 +วัà¸Ļ à¸Ĺีà¹Ī +×Ļ׾ ×ĵ +ØŃ ا +е ÑĨ +Ġc ứ +×ĵ ×ķר +ĠÙħ ØŃ +ר׼ ×ij +بÙĬ ع +ни и +ĠاÙĦØ£ ÙĪÙĦ +à¸Ħว ร +ã썿ĢĿ ãģĨ +ĠС о +ائ ÙĬØ© +ر اء +оÑģ об +Ġب Ø£ÙĨ +×¢ ×ķ×ĵ +ĠÑĤ е +ãģĵ ãģĨ +ÑģÑĤ ÑĢа +ай н +Ġsö z +ت ÙĨا +à¸Ń ิ +ặ p +ĠìķĦ ëĭĪ +íķ Ń +Ġר×IJ ש +Ġ à¹Ħà¸Ķà¹ī +Ġ×Ĵ ×ĵ +Ġס פר +обÑī е +ĠÙĪ Ø¥ +ada ÅŁ +ãģ¡ ãĤĩ +×§ ×ķ׾ +ÑĢ ÐµÐ· +ĠdÃ¼ÅŁ ün +Ġ×ij ×IJ×ŀ +Ġìĸ´ ëĸ +ער ×ij +н ее +ĠÑģÑĤÑĢ Ð°Ð½ +س اÙĨ +yn ı +ĠاÙĦر ئÙĬس +ãģĹãģ ª +Ġ׳ ת +ãģ«ãģª ãģ£ãģŁ +g ü +åıĹ ãģij +׾ ת +ìł Ī +ëĬĶ ëį° +Ø® ÙĬر +à¸ķà¹īà¸Ńà¸ĩ à¸ģาร +ĠÙĦ Ø£ÙĨ +Ġch á»ĭ +ÙĪ Ø© +à¹ĥ ส +ë¶Ģ íĦ° +íķĺ ë©´ +ữ u +à¹Ģหม ืà¸Ńà¸Ļ +б еÑĢ +ĠìĿ´ ìļ© +ĠÑģ еб +wiÄĻ ks +Ġ׳ ×¢ +ÑĤ ÑĥÑĢ +Ġngh Ä© +ש ×ķ×ĺ +ti ÄŁi +Ġde ÄŁi +×IJ ×ij +Ġ×ŀ ×ŀ +ãĥĹ ãĥŃ +wa ÅĤ +à¸Ī ึà¸ĩ +Ø® دÙħ +×IJ ×Ŀ +Ä±ÅŁ ı +cz Äħ +ר ×ĵ +ĠÑĢ Ñĥб +خر Ùī +ãģ® æĸ¹ +Ġд енÑĮ +×Ĺ ×Ļ×Ŀ +еÑĤ е +ëĤ ľ +×IJ ×Ĵ +×¢ ×ķר +ë³ Ħ +åIJĮ ãģĺ +ãĤ ² +ר ×ļ +×ķש ×IJ +ìľ ¡ +ا Ø® +צ ×Ļ×Ķ +á»± a +ãģĪ ãģ¦ +ש×Ķ ×ķ +ан ÑĤ +ลา à¸Ķ +ин г +ë¡ ł +اع د +ÙĪ Ø³Ø· +Ġв оп +Ġвоп ÑĢоÑģ +Ùħ ÙĬÙĨ +à¸Ħ à¸ĩ +×Ļר ×Ļ×Ŀ +c ów +ê² © +Ġê·¸ 룰 +Ġì§ Ħ +Ġש ׾×Ķ +à¹Ģร ิà¹Īม +à¸Ĭ à¸Ńà¸ļ +д еÑĤ +ÑİÑī иÑħ +à¸ļ à¸Ńà¸ģ +æĢĿ ãģĦ +ع ÙĬد +ס ×ŀ +×Ĵ ×Ļ×¢ +צ ×ĵ +ب ات +ĠëͰ ëĿ¼ +à¸Ī ัà¸ĩ +ãģłãģij ãģ§ +×¢ ×Ļר +ĠÑĩ ел +ĠÑĩел ов +ĠÑĩелов ек +ãĥĥ ãĥģ +à¹Ģà¸ģ ีà¹Īยว +à¸Ķ ิ +Ġפ ×¢ +×Ļ×ŀ ×Ļ +ë° ĺ +Ø® ار +×ij ×Ļת +×¢ ×Ļ×Ŀ +ü yor +ãĤģ ãģ¦ +к лад +Ġ à¸Īาà¸ģ +à¹Ģà¸Ħ ย +ส à¸Ńà¸ĩ +à¹ģ à¸Ħà¹Ī +ẫ u +หà¸Ļ ัà¸ĩ +ש׾ ×ķ×Ŀ +اÙĨ ÙĬØ© +åĩº ä¼ļ +åĩºä¼ļ ãģĦ +à¸ł าย +à¸ļา à¸Ĺ +à¸Ĭา ว +mu ÅŁ +Ġ׾ק ×ij׾ +ãĤ· ãĥ£ +Ġİ ÅŁ +×Ĵ×ĵ ×ķ׾ +ج عÙĦ +ë³ Ģ +ยิ à¹Īà¸ĩ +à¸Ļ าย +à¸Ļ ีà¹Ī +วิ à¸ĺี +ãĤī ãģªãģĦ +ëł Ī +Ġ문 ìłľ +Ġ à¸ģ +à¸Ĺำ à¸ĩาà¸Ļ +à¹Ģว à¹ĩà¸ļ +ÑĦ е +楽 ãģĹ +สำ à¸Ħ +สำà¸Ħ ัà¸į +ر Ùħ +ãģķãĤĮ ãģ¦ +Ġоб ла +ר×IJ ×Ļ +หม à¸Ķ +ÙĨ ÙĬØ© +ли н +Ġe ÄŁ +it im +ëł ¹ +ص اÙĦ +ÅĽ l +à¸ľ ิà¸Ķ +ãĥŀ ãĥ³ +åħ¥ ãĤĮ +à¹Ģà¸ķ à¸Ńรà¹Į +ار ÙĬ +ĠÐ ¦ +d ür +ส วย +ë¦ ½ +رÙĥ Ø© +Ġh ã +×Ļת ×Ķ +à¸Ĥ à¸Ļา +à¸Ĥà¸Ļา à¸Ķ +à¸Īำ à¸Ļ +à¸Īำà¸Ļ วà¸Ļ +ש ×ķ×§ +Ġд ом +ì± ħ +ãģĭ ãģij +פ ×ķ׾ +à¸Ĭ าย +Ñģ моÑĤÑĢ +Ñģл Ñĥж +ש ×IJ׾ +кÑĢÑĭ ÑĤ +Ġìŀ ĺ +é«ĺ ãģĦ +ĠÑĢ Ñĥк +ÙĨ ص +д ав +ưỠ¡ +ưỡ ng +ر اÙħ +×Ļ׳ ×Ļ×Ŀ +ãĥ© ãĥ¼ +ëĦ ¤ +Ġت ع +l ke +好 ãģį +æĮģ ãģ¡ +Ġë§ İ +Ġy ük +ĠÑģоÑģÑĤ ав +енÑĤ ÑĢ +pe ÅĤ +à¹Ģà¸Ľà¸¥ ีà¹Īย +à¹Ģà¸Ľà¸¥à¸µà¹Īย à¸Ļ +íı ī +ãĤĦ ãģĻ +×Ĺ ×ĸ +×ijר ×Ķ +ë£ ¨ +ìĶ Ģ +بØŃ Ø« +à¹Ģà¸ķ à¹ĩ +ów i +ب Ùĩ +ãģį ãģ¾ãģĻ +Ġ×¢ ×ŀ +×Ĵ ×ķ׾ +ез д +ÙĬÙģ Ø© +สà¸Ļ à¹ĥà¸Ī +Ġת ׾ +Ñı Ñī +Ġس ÙĨ +ĠÙĪØ§ ØŃد +ĠÑģ м +lad ı +ı ld +×Ļר ת +ีย à¸Ļ +ת×Ĺ ×ª +Ġж из +à¸ŀ ั +à¸ŀั à¸Ĵ +à¸ŀัà¸Ĵ à¸Ļา +à¸Ĭ ิ +ا Ø®ÙĦ +ãģ£ãģ¦ ãģĦãģŁ +รั à¸IJ +ãĤģ ãĤĭ +à¹Ĥ à¸ģ +ĠT á»ķ +Ġh akk +ر Ùģ +ìł Ģ +Ñģ об +ãģª ãģijãĤĮãģ° +Ùĩ ÙĪ +Ġë² ķ +ãĤ Ĩ +ĠاÙĦس عÙĪØ¯ +Ġ×IJ תר +Ø§Ø º +Ġ׾ ×ĵ +à¹ģ à¸ķ +à¹ģà¸ķ à¹Īà¸ĩ +íĮ Į +Ñĥп иÑĤÑĮ +à¸ŀืà¹īà¸Ļ à¸Ĺีà¹Ī +×ij ת×Ļ +à¹ĩ à¸ģ +ÅĤ at +Ġê°ľ ìĿ¸ +ìłķ ë³´ +ÑĤ ал +Ġgü ven +Ġİ l +Ġê° ģ +Ġب ت +×ŀ ×ķ׳×Ķ +ĠاÙĦØŃ ÙĥÙĪÙħ +ÙĤ ات +à¹ģ à¸ģà¹Ī +ห าà¸ģ +н ÑĮ +à¸Ľ รัà¸ļ +มา à¸ĵ +Ġне Ñģк +ĠØ ¶ +สม ั +สมั à¸Ħร +ãģĮ ãģĤãĤĬ +м еÑģÑĤ +Ġ×IJ צ׾ +Ġкомп ани +ס ר +ÙĬÙħ Ø© +ĠÑħ оÑĢо +ĠÑħоÑĢо ÑĪ +Ġ×Ļ ×ķ×ĵ +ü s +×Ĵ ×Ļש +à¸ļ à¸Ĺ +تÙĨ ظ +ว าà¸ĩ +ม หา +Ġ׼ ×ķ׾ +à¸Ĥ à¹īาà¸ĩ +ë° ľ +г од +д ан +ãģĭãĤĤãģĹãĤĮ ãģ¾ãģĽãĤĵ +ãģĵ ãģ¡ãĤī +ãĥIJ ãĤ¤ +ece ÄŁi +دÙĬ دة +ÙĨ Ùī +Ġëĭ¤ ìĿĮ +ว ี +غ ا +ли з +à¹Ģà¸Ķ ิ +à¹Ģà¸Ķิ ม +ĠÙĬ ست +Ġy ılı +ko ÅĦ +ãģ§ãģĹãĤĩãģĨ ãģĭ +ãģĤ ãģª +ãģĤãģª ãģŁ +ÑĨ ен +ĠÙĪ Ø² +×IJ ×Ļש +à¹Ī à¸Ń +ر ØŃ +ê´ ij +ÑĢа ÑģÑĤ +Ġ×Ķ ×ľ +ãģĹãģ¦ ãĤĤ +×ŀר ׼ +×ŀר׼ ×ĸ +éģķ ãģĦ +ãģŁ ãģı +ĠÑģ Ñĥд +в еÑģÑĤи +ĠíķĦ ìļĶ +ãĥķ ãĤ§ +ÑĤелÑĮ но +à¹Ģà¸ŀ ืà¹Īà¸Ńà¸Ļ +ÅĤu ż +à¹Ģà¸Ķิà¸Ļ à¸Ĺาà¸ĩ +ש ×ķר +Ġ×ŀ ×ĵ +×ķ×¢ ׾ +ÙĦ اÙħ +à¹Ħ à¸ĭ +л ей +кÑĥ ÑĢ +Ạ¢ +à¸Ĺ าà¸Ļ +ì§ ij +ĠгоÑĢ Ð¾Ð´ +ר ס +׾ ×ķ×Ĵ +mas ını +Ġл ÑĥÑĩ +ล à¹Īา +ìļ ¸ +ש ×ĺ +ĠÐĺ н +í Ĥ¤ +ÙĪÙĦ ا +ìķ ł +ĠØ£ÙĬ ضا +Ùĥ ار +ĠاÙĦت ع +ส ูà¹Ī +ãĤ ¼ +×ij ×Ļ×IJ +ย à¸ģ +ĠØŃ ÙĤ +ر بÙĬ +ãģĺãĤĥ ãģªãģĦ +รัà¸ģ ษา +Ñħод иÑĤ +à¸ķ à¸Ńà¸ļ +׳ ×ĺ×Ļ +ĠاÙĦÙħ ج +تÙħ ع +ов аÑĤÑĮ +ÙĦ ÙĬÙĨ +×Ļ×ŀ ×ķת +Ġm ù +n ÄĻ +Ġد ÙĬ +׼ ש×Ļ×ķ +Ġhi ç +ë ijIJ +ÙĪ Ø§Ø¡ +ÙĪ Ø· +ĠاÙĦ بÙĦ +à¹ģม à¹ī +×§ ×ķת +ÙĪØ¬ د +å§ĭ ãĤģ +ÙĬ ئة +Ġë§ ¤ +ص بØŃ +פ ×IJ +г оÑĢ +ס ×Ķ +بÙĬ ÙĤ +ย าà¸ģ +Ġн ад +ÙĬ Ùij +Ġب ÙĪ +ס ×ķר +Ùħ ÙĥاÙĨ +ר ×ij +×Ĵ ×ĸ +צ ת +b ilit +л аг +ĠN go +×IJ ×ķר +à¸ķ à¸Ļ +íĬ ¹ +à¸Ĺีà¹Ī à¸Ķี +à¸Ľà¸£à¸° à¸Īำ +ов ание +ãģĦ ãģ¤ +ãĥĥãĤ¯ ãĤ¹ +åIJĪ ãĤı +åIJĪãĤı ãģĽ +×Ļ׳ ×ķ×Ļ +ạ y +Ø« ÙĤ +ĠпÑĢ Ð¾Ð± +ĠпÑĢоб лем +ÅŁ eh +ÅŁeh ir +ع ادة +اÙĨ ÙĪÙĨ +à¸ķัว à¹Ģà¸Ńà¸ĩ +ì¶ ķ +ı lan +б ан +ãĥ³ ãĥī +à¸Ī ี +Ġ×Ķש ׳×Ļ +п оÑĤ +×ķ׾ ×Ļ×Ŀ +ล ัà¸ļ +ĠÑį ÑĤи +×ij×§ ש +ë¹Ħ ìĬ¤ +à¸Ńยà¹Īาà¸ĩ à¹Ħร +×Ļ׾ ×Ļ +à¹ĥà¸Ĭ à¹Ī +ĠاÙĦ ÙĥÙĦ +ãĥļ ãĥ¼ãĤ¸ +ص Ø© +ÑĤи ÑĢ +ãĤĵ ãģ© +зÑĭ к +wy ż +Ùĩ ÙĬ +ĠÙħ ÙĦÙĬ +Ġвид е +ظ اÙħ +دا ÙĪÙĦ +×ŀ ת×Ļ +Ġs ık +à¹Ģà¸ķิ ม +ãĤ¢ ãĤ¤ +ка Ñħ +צ ×Ļ׾ +à¹Ģà¸Ĭ à¹Īà¸Ļ +м аг +маг аз +магаз ин +à¸Ľ ั +à¸Ľà¸± à¸Ī +Ġש ×Ļר×ķת +ีย ม +ãĥĸ ãĥ« +Ġد ÙĪÙĦ +קר ×Ļ×Ŀ +Ùĩ Ùı +ов о +Ġü ret +د ÙĪÙĨ +à¹ģà¸Ļ ว +à¹Ģà¸Ļ ืà¹īà¸Ń +ĠÑĦ оÑĤ +ãĥ ĺ +ãģ¤ ãģĭ +Ñı Ñģ +ĠíķĺëĤĺ ëĭĺ +ائ ع +Ġп лаÑĤ +ìĺ Ī +Ġdost ÄĻp +ÙĪØ¬ Ùĩ +Ġ×Ķ ×Ĺ×Ļ +׳ ×Ļ×§ +д ей +í ĽĦ +ı y +بØŃ ر +à¹Ģส ริม +Ġ׾ ×Ĵ +ذÙĩ ب +ج ÙĬÙĦ +رÙĥ ز +Ġë ħ +Ġëħ ¸ +פ×Ļ׾ ×ķ +ãģ¾ ãģļ +iri ÅŁ +ĠÙĥ ÙĬÙģ +Ġ×ij צ +Ġêµ IJ +ÑĢоÑģ Ñģ +ĠØ´ ÙĬ +Ġiç er +×Ĵ ×ķ×ij×Ķ +мен но +×¢ ×ij×Ļר +×ķ×ŀ ×Ķ +ãĤī ãģĹãģĦ +ãģ ¼ +Ñī ин +è²· ãģĦ +جÙħÙĪØ¹ Ø© +Ġdön em +Ġ×ij ×IJר +в еÑģÑĤ +×ķר ×ķת +س Ùģ +à¹ģà¸Ĺ à¸Ļ +Ġд окÑĥменÑĤ +Ġا ÙĬ +ج اÙĨ +צ×ķ×¢ ×Ļ +ĠоÑģ об +ĠاÙĦÙħ س +ÑĢаР± +à¸ł ู +à¸Ķ าว +л екÑĤ +ع ÙĤ +×ķ×ĵ ×ķת +Ġol u +Ġolu ÅŁtur +ãģ¾ ãģ¾ +ед ин +à¹Ģ à¸Ńà¸ģ +ãĤµ ãĤ¤ +ëĦ Ī +Ø· ÙĨÙĬ +Ø· ÙĤØ© +ĠÐł аз +ÙĦ Ùij +Ñĩ ем +Ġ׾ ×ĺ +สั à¹Īà¸ĩ +سر ائÙĬÙĦ +Ġפר ×ĺ×Ļ +д еÑģÑĮ +Ġ׳ ׼ +اÙĨ ب +ÙĬا Ø© +Ùħ بر +Ġk ı +à¸Ľ à¸ı +à¸Ľà¸ı ิ +à¸ļั à¸ķิ +׳ ת×Ļ +ìĨ ¡ +ر اب +à¹ĥ à¸ķ +à¹ĥà¸ķ à¹ī +×Ļ׳ ת +ÙĪ ÙĬر +Ġ×Ķ×ŀ ×Ļ +ей ÑĩаÑģ +×§ ×ķ×ij +در اس +ĠÙħ ÙĤ +رÙĬ ÙĨ +Ø® اص +ãģĬ éĩij +Ġج دا +ãģĨ ãģ¡ +ëħ ¸ +ır ım +æ§ ĺ +ãģ« å¯ +ãģ«å¯ ¾ +ÑĨ ев +Ġv ard +ĠÐIJ н +e ÄŁ +ÑģÑĤв енно +Ð ¨ +س د +à¸ģ ุ +à¹ģà¸ľ à¸Ļ +รูà¹ī ส +รูà¹īส ึà¸ģ +ات ØŃاد +Ñij ÑĤ +×Ĺ ×ķ×§ +ãģĻ ãģIJ +Ø· ÙĦاÙĤ +Ġ×§ ×ķ×ĵ +à¹ĥà¸Ĭ à¹īà¸ĩ +à¹ĥà¸Ĭà¹īà¸ĩ าà¸Ļ +ãĥ¼ãĤ ¿ +Ġs ür +ÑĢ Ð¾Ðº +ë³ ij +สมา à¸Ĭ +สมาà¸Ĭ ิà¸ģ +ãĥķ ãĥ¬ +è¾¼ ãģ¿ +ãĤ» ãĥ³ +Ġê°Ģ ì§Ģ +à¸ľ à¹īา +ÑįÑĤ омÑĥ +иÑĤ ел +à¸ł ั +ภij +ãĥĸ ãĥ© +×Ľ×ª ×ķ×ij +׳ ×Ŀ +ен нÑĭе +×¢ ×¨×Ľ×ª +Ġì Ĥ +ĠìĤ ´ +à¸Ĥ à¹īา +׳ ×ķס +ãĥ¬ ãĥĵ +ÑĢ ÐµÑģ +à¹Ģล à¸Ĥ +Ø« اÙĦ +ìĹ Ĩ +ĠÑĩ аÑģÑĤ +า ศ +ãĥª ãĤ¢ +u ç +×Ļ׼ ×ķת +ล à¹īาà¸Ļ +i ë +ãĤ¸ ãĤ§ +à¸Ī à¸Ń +ÙĪ ØŃد +×Ļצ ×ķ×ij +Ġ×ij ש׾ +ок о +ض Ø© +ذ ر +ĠÑĥ д +İ L +×ķצ ×Ļ×Ŀ +×ĸ ×ŀף +à¸Ľ à¸ģ +íķĻ êµIJ +س اÙħ +à¹Ħ à¸Ķ +ละ à¹Ģà¸Ń +ละà¹Ģà¸Ń ีย +ละà¹Ģà¸Ńีย à¸Ķ +ả y +аÑĨи он +ãĤ¹ ãĤ¯ +פ ×ķס +ร à¹Īาà¸ĩ +ен нÑĭй +ع ÙĨ +عÙĦ ÙĨ +ائ Ùģ +d ÄĻ +ؤ ÙĪÙĦ +׾×ķ ×ķ +Ġ×ij ש×ij +ä»Ĭ åĽŀ +ĠاÙĦج ÙĨ +د اد +wa Äĩ +ãĥª ãĥ³ +ĠìŀIJ ìĭł +اÙĨ ÙĬا +ãĥ¡ ãĥª +ÙĦ ÙĪÙĨ +à¸Ĺ à¹Īà¸Ńà¸ĩ +à¸Ĺà¹Īà¸Ńà¸ĩ à¹Ģà¸Ĺีà¹Īยว +اÙģ ÙĬ +Ġли ÑĪ +Ùħ ÙĬØ© +оÑĤ веÑĤ +Ñĩ ин +à Ĭ +ãĥ¡ ãĥ³ +å® Ł +éļĽ ãģ« +ĠÑĢаР¹ +ãĤ¦ ãĥ³ +×Ļר ×ķש +×Ļר×ķש ׾×Ļ×Ŀ +ม ะ +Ġar a +каз аÑĤÑĮ +à¸ķ ัà¸Ķ +ÑĥÑİ ÑĤ +Ġü st +×Ĵ ×ķ×ij +×Ĵ×ķ×ij ×ķת +mal ı +ег од +егод нÑı +اÙģ ÙĤ +à¸Ĭ à¹Īà¸Ńà¸ĩ +Ġö zellik +×Ļצ ×ķר +Ġmi ÄĻd +Ġili ÅŁ +Ġна Ñħод +×¢ ×ĸר +׾ ×Ľ×ª +ÙĨت اج +ĠÑģ ем +à¸Ī à¹Īาย +à¸ķร ว +à¸ķรว à¸Ī +פר ×ķ +à¸Ĥ ัà¸ļ +ãģ ŀ +Ġп ло +к олÑĮ +×ŀ×¢ ×ĺ +íķĺ ìĭľ +jÄħ ce +ÙĨ اÙĨ +ลี à¸ģ +н ÑĥÑĤ +Ġоб ÑĢаз +Ùĥ بر +ĠاÙĦÙĪ Ø·ÙĨ +ãģķãģĽ ãģ¦ +ÙĤ اء +×ŀ×ĵ ×Ļ׳ +y ü +פ ×Ļת +׳ ×ķף +ÙħÙĨ ظ +หà¸Ļ ัà¸ģ +ìŀ Ī +ãĤ« ãĥ¼ãĥī +ع ÙĨÙĬ +п од +ض اء +à¸Ļ à¸ķà¹Į +×ŀש פ +ว à¹Į +ר ×ķ×§ +ส ืà¹Īà¸Ń +פק ×Ļ×ĵ +ãģªãĤī ãģªãģĦ +ĠìŬ 룬 +ÙĦ ج +Ñī иÑĤ +ãĥĥ ãĤ· +ÙĦÙĬ س +ĠÙĦ Ùħا +ìł ij +×ij ×Ļף +ãĥģ ãĤ§ +Ġgü ç +Ġch ứ +×ķצ ×IJ +קר ×ij +à¹Ĥ à¸ŀ +оÑĩ но +סק ×Ļ +ש׾ ×Ŀ +صر Ùģ +ĠL Ãł +×¢ ×Ļת +á» · +à¹Ĥ à¸Ńà¸ģ +à¹Ĥà¸Ńà¸ģ า +à¹Ĥà¸Ńà¸ģา ส +Ġ×Ķ ×ĵ×ijר +à¸Ļั à¹Īà¸Ļ +ز ر +нак о +íļ į +ãĤĤ ãģ¡ +ãĤĤãģ¡ ãĤį +ãĤĤãģ¡ãĤį ãĤĵ +اÙħ ت +عد اد +и нÑĭ +ÅĤy w +à¸Ħ à¸ĵะ +à¸Ĺ ะ +kt ör +×Ļ×Ĺ ×Ķ +Ġм е +Ġме ÑģÑı +׳×Ķ ×Ĵ +ĠÑģ ÑĥÑīеÑģÑĤв +à¸Ļ ัà¸Ļ +ÑĦ ÑĦ +ек ÑĤив +عÙĦÙĪÙħ ات +б Ñĥд +à¸Ļัà¸ģ à¸ĩาà¸Ļ +หà¸Ļà¹īา à¸Ĺีà¹Ī +ÙĤÙĬ ÙĤ +ãĤ· ãĥ³ +ãģ« éĸ¢ +×IJר ×Ĵ +ĠпÑĢ Ð¾ÑĤ +ĠпÑĢоÑĤ ив +ĠìŀĪ ìĸ´ +ÙĤÙĬ ÙĤØ© +ìĹ ĩ +k ür +ãģ«ãģªãĤĬ ãģ¾ãģĹãģŁ +Ġде ÑıÑĤ +ĠдеÑıÑĤ елÑĮ +פ×ķר ×ĺ +à¸Ł à¹īา +à¹Ģ à¸ł +ĠавÑĤом аÑĤ +×ĸ ×Ļ×§ +Ġold uk +ع اÙħ +ĠÑĤ оÑĢ +yrı ca +ê Ì +ãĤŃ ãĥ³ãĤ° +ãģ« ãģ¨ãģ£ãģ¦ +à¹Ģà¸ī à¸ŀ +à¹Ģà¸īà¸ŀ าะ +ãģ¯ ãģļ +×ŀ ×IJ×Ļ +สะ à¸Ķ +สะà¸Ķ วà¸ģ +ìľ¼ ë©° +à¸ģ ี +ภ¬ +Ġ×¢ ×ķש +à¸łà¸² ษา +à¸Ĺ ัà¸Ļ +ac akt +acakt ır +اع دة +ĠÑĥÑģл Ñĥг +ס ר×ĺ +×ķ×ŀ ×ķת +×Ķ ×ķר +×ŀ ×ķ×ij +×ŀ×ķ×ij ף +سÙĬ اس +اتÙģ Ø§ÙĤ +×Ķ ×¦×ľ +Ùħؤ س +Ġp ó +Ġк ни +×Ļ׼ ×ķ׾ +à¹Ģหล ืà¸Ń +׼׾ ׼ +׳ ×ĸ +ÑĪи е +r ès +ĠاÙĦØŃ ÙĤ +лÑı ÑĢ +ห à¸į +หà¸į ิà¸ĩ +ר×Ĵ ×Ļש +à¹Ģส à¹īà¸Ļ +ש×ij ×ķף +ô tel +ап ÑĢ +апÑĢ Ð¸Ð¼ÐµÑĢ +اب ÙĦ +ĠÑĢаз виÑĤ +Ġп олÑĮз +ĠС еÑĢ +×ķ×ij ×Ļ +r óż +ìĭ Ń +ãĤ¯ ãĥĪ +ãģĹ ãĤĪãģĨ +à¸ģร ม +ØŃ ÙĥÙĪÙħ +à¹Ĥ à¸ļ +à¸Ĺ à¹īาย +ĠM á +ĠÑĤ Ñĭ +à¸Ħร ัว +ÑĢÑĥ б +ạ p +Ġm ÅĤ +ĠmÅĤ od +Ġgör Ã¼ÅŁ +Ġgeli ÅŁ +ươ i +×ŀש ×§ +ÙĢÙĢ ÙĢÙĢ +รา ว +ãģĹãģ £ +ãģĹãģ£ ãģĭãĤĬ +ĠÐļ он +Ġk ê +à¹Ĥà¸Ĺ ร +èIJ½ ãģ¡ +åĩº ãģ¦ +ล ัà¸ģษ +Ġ×Ĵ ×ij×ķ×Ķ +ãĥĻ ãĥ« +ê±° ëĤĺ +ë§ IJ +×Ļ׾ ×ĵ×Ļ×Ŀ +ĠëĦ Ī +×ŀר ×Ļ +ร ส +ãĥŃ ãĥ³ +и ло +ноÑģÑĤÑĮ Ñİ +×ĸר ×Ĺ +п он +Ġ×Ķש ׾ +ê²ł ìĬµëĭĪëĭ¤ +Ġki ÅŁ +ĠÐļ и +ว ร +د اع +ÅŁ im +ÙĨ Ùij +в аÑĤ +را Ùĥ +ب اÙĦ +ид е +Ġ×Ķ×ŀ ×Ĺ +ìĸ µ +تÙģ Ø§Ø¹ +Ø£ ت +ëĬ ĺ +ש ×Ļת +ست Ùħر +ĠÑĦ ак +ĠاÙĦØ£Ùħ رÙĬ +ëŀ ¨ +اس Ùħ +Ġa ÄŁ +Ġç ev +Ùĥ ÙĪØ± +ãģķ ãģ¾ +Ġç öz +Ġر س +Äħ da +สà¸Ļ ุ +ãģĹãģ¦ ãģıãĤĮ +н Ñİ +leÅŁ me +ãĤª ãĥ³ +ãģ¨ ãģªãĤĬ +ava ÅŁ +×ĺ ×Ļ×ij +ØŃ ض +×ķצ ×IJ×ķת +ÙĨ ÙħÙĪ +ı t +ĠÑħ а +ĠÑħа ÑĢак +ĠÑħаÑĢак ÑĤеÑĢ +Ġd ÅĤ +ãĥĹ ãĥ© +à¸Ĭ ุม +à¹Ī à¸Ńà¸Ļ +×ķ×ij ׾ +Ñģ ол +×ĵ ×Ĵ +аÑĢ Ð°ÑĤ +n ivers +Ġgerçek leÅŁtir +ĠاÙĦ ÙĦÙĬ +ระ ยะ +ĠÙħ ختÙĦÙģ +Ġgö nder +Ùģ Ø§Ø± +do ÄŁ +doÄŁ an +ص ÙĦاØŃ +Ġyay ın +ãĥĨ ãĥ³ +รว à¸Ī +×Ļ×Ĺ ×Ļ×ĵ +ünk ü +ÑĨи алÑĮн +à¸ļ ู +ม ุ +h ä +Ø® Ùģ +å¢ Ĺ +å¢Ĺ ãģĪ +еÑĩ но +ĠاÙĦس ÙĨ +à¸Ĥ าว +im di +Ð « +à¸Ļà¸Ńà¸ģ à¸Īาà¸ģ +à¸ļา ล +ת ש +Ġdüzen le +мÑĭ Ñģл +ãģı ãģª +ż u +Ġwsp óÅĤ +Ġн аз +ınd aki +تر Ø© +ÅŁ ek +Ġö d +ĠÙĪ Ùĥ +Ġпозв олÑı +Ġת ×ķ׼ +ÙħÙĨ تج +ë§ ī +ĠاÙĦØ« ÙĦاث +аÑĨи Ñİ +ÙĪØ± ÙĪ +Ñĭв аеÑĤ +خص ص +ĠاÙĦÙģ ÙĦ +ĠاÙĦÙģÙĦ سطÙĬÙĨ +Ø¥ جر +إجر اء +اÙĨت Ø® +اÙĨتخ اب +ار ÙĬØ© +×ķ Ö +Ø¢ ÙĨ +×ŀ×¢ ×ķת +Ġм ал +Ġ×IJ ×Ĺ +à¸Ĺ à¹īà¸Ńà¸ĩ +ze ÅĽ +Ġë§Į ëĵ¤ +رÙĬ ع +äºĭ ãĤĴ +à¸ļริ หาร +׾ ×ŀ×Ļ×ĵ +Ġм Ñĥж +ت رÙĪ +ĠباÙĦ Ø¥ +פ ×Ļ×§ +ز ÙħØ© +ĠÃ¶ÄŁ renc +ãĥ ¶ +اÙħ عة +×§×ij ×ķצ +×ŀ ׳×ķת +رÙĬ Ùħ +Ġо каз +ãģłãģij ãģ© +Ġh ız +Ġש ×IJת +ãĤ¢ ãĥ¼ +Ġmożli wo +ìĦ ¼ +ÙĪ Ø§Ø¨ +ог ÑĢаÑĦ +Ġعبد اÙĦ +ãĤĴ è¡Į +ب ÙĬÙĦ +Ġİ ç +ย าย +ĠÑĥ ÑĩаÑģÑĤ +ÑĦ еÑģÑģ +ÑĦеÑģÑģ иона +Ạ¤ +ÙĨ ÙĬÙĨ +عد ÙĦ +สร ร +دÙĬ ÙĦ +×ij ×Ļ×§ +czy ÅĤ +ÑĢом е +Ġм ед +ìĻ Ķ +ãĥ© ãĤ¤ãĥ³ +ĠÑĤ еп +еÑĢ ÑĮ +i ÄŁi +в ели +ÑĢи ÑģÑĤ +ס ×ķפ +×ŀ׾ ×Ĺ +ĠاÙĦØ¥ ÙĨ +Ġ׾×Ķ ×© +è¶Ĭ ãģĹ +ĠÑĢ Ñĭ +×ķ×IJ ר +رÙĩ اب +פ ×ķ×IJ×Ļ +ĠгоÑģ Ñĥд +ĠгоÑģÑĥд аÑĢ +ĠгоÑģÑĥдаÑĢ ÑģÑĤв +ĠاÙĦØ£Ùħ ÙĬر +Ùħ ج +à¹Ģหม าะ +ÑĢ ÐµÐ² +à¸Ĭี à¸ŀ +ãĥķ ãĥĪ +иÑĩ но +ĠاÙĦÙħ ؤ +Ġi ht +íħ ľ +د ÙĨÙĬ +ر ص +ла ÑģÑĤ +à¹Ģหล à¹Īา +ılı r +ร à¸ĵà¹Į +×ŀש ×Ļ×ļ +Ġd á»ĭ +Ø·Ùģ Ø§ÙĦ +×ĺ ×ķף +Ġ×ij ×Ļ׳ +ãģ¾ ãģ£ãģŁ +лож ениÑı +تØŃ ر +ب اØŃ +à¹Ģส ืà¹īà¸Ń +ãģĻ ãģĶ +lt ür +à¸ĩ าม +Ġt ü +ĠпÑĢ Ð¸Ð¼ +ĠпÑĢим ен +Ġhay at +ëĥ IJ +ëĭ Į +׳×Ļ ×ķ +вед ен +ìħ ¨ +à¸Ī ัย +à¸ģà¹Ī à¸Ń +Ġв од +оÑģÑĤ оÑı +н аÑĤ +à¹ģ หล +سÙħ ÙĬ +à¸Ķำ à¹Ģà¸Ļ +à¸Ķำà¹Ģà¸Ļ ิà¸Ļ +w ód +ö yle +ãĥĢ ãĤ¤ +ÑĪи й +меÑī ен +ãģĹãģ¾ ãģĨ +ãĥī ãĥ© +ÙĪØ¶ ØŃ +à¸Ńà¸Ļ ุ +ĠاÙĦ اجتÙħاع +laÅŁ ma +à¸Ħ à¸Ńà¸Ļ +×ŀר ×Ļ×Ŀ +ÙĨ اÙħج +שר ×ķת +اÙĦ Ø£ +Ġksi Äħż +Ġа н +ÑĢаР¹ +اÙĩر Ø© +×ŀ×ĵ ×Ķ +ä¸Ģ ç· +ä¸Ģç· Ĵ +ä¸Ģç·Ĵ ãģ« +ÑĢиÑĤ оÑĢ +d ıkl +à¹ģ à¸ĸ +à¹ģà¸Ĥ à¹Īà¸ĩ +екÑĤ оÑĢ +×ŀס ×¢ +ÑĢак ÑĤи +u ÄŁu +×ķ×ij ת +สู à¸ķร +ĠçalÄ±ÅŁ m +ĠçalÄ±ÅŁm alar +Ġа на +ãĥĽ ãĥ¼ãĥł +Ġböl üm +Ġب ص +ол оÑģ +ĠìķĬ ëĬĶ +à¹Ī ะ +ÙĪ ØªØ± +ä¹ Ĺ +ست خداÙħ +פ×Ļ ×Ļס +פ×Ļ×Ļס ×ij +פ×Ļ×Ļס×ij ×ķ×§ +Ġк ÑĢаÑģ +ли к +رÙĬ ØŃ +×ŀש ׾×Ķ +à¹Ģย ีà¹Īย +à¹Ģยีà¹Īย ม +в иÑģ +ом н +ÄŁ un +ãĥŃ ãĥ¼ãĥ³ +Ø£ تÙĬ +à¸ķร ี +çͳ ãģĹ +تÙħ ر +ìĹ ĪìĬµëĭĪëĭ¤ +ĠÙĪ ØºÙĬر +red ni +ĠاÙĦص Ùģ +Ġна ÑģÑĤоÑı +ĠнаÑģÑĤоÑı Ñī +à¸ķ รา +ĠÑĥÑģл ов +ĠÑĥÑģлов иÑı +ÑĨ еп +×Ķ ×Ĺ׾×ĺ +Ø· ÙĬع +ĠB akan +ĠاÙĦ رÙĪ +илÑĮ но +Ġм еÑĤ +à¸Ķ à¸Ńà¸ģ +ãģĭãĤī ãģªãģĦ +Ġпо ÑģÑĤоÑı +ĠпоÑģÑĤоÑı н +ĠÑĩ аÑģ +ü c +wr ó +б ÑĥÑĢ +ãĥIJ ãĥĥãĤ¯ +ãĥ©ãĥ³ ãĥī +Ġо гÑĢ +สั à¸į +สัà¸į à¸įา +มั à¹Īà¸Ļ +à¸Ħ à¸Ńม +al ık +Ġн ед +üm üz +ĠÅĽ wie +é rio +×Ļ×IJ ×Ķ +دÙħ ات +ı rl +ĠоÑĤ з +ĠоÑĤз Ñĭв +ä»ĺ ãģį +Ġkaż de +мин иÑģÑĤ +ãĤ° ãĥ« +ë° ĸ +ез н +اÙĦ Ùģ +Ġש ק׾ +Ùħ ض +ãĥĿ ãĥ¼ãĥĪ +ÙħÙĨ ت +ÙĤÙĬ اÙħ +Ø´ ÙĨ +×Ļר ×ķ×¢ +ãĤŃãĥ£ ãĥ³ +доÑĢ Ð¾Ð² +×ŀ ×Ļת×Ļ +ÙĪÙĦ ÙĪØ¬ +Ùĥ اÙģ +ĠÑĢаз лиÑĩ +иÑĤ еÑĤ +н олог +ลà¸ĩ à¸Ĺุà¸Ļ +Ġyak laÅŁ +ãĥ¬ ãĤ¤ +ê²ł ëĭ¤ +æ±Ĥ ãĤģ +رÙĪ Ùģ +Ġí Ĭ +ĠíĬ ¹ +ãģ£ ãģıãĤĬ +à¸Ħวาม à¸Ħิà¸Ķ +×Ķ ×Ļס×ĺ +Ø¥ ÙĤ +ãģ¦ ãģĦ +à¹Ĥ à¸Ĭ +ĠBü yük +ĠФ едеÑĢ +ÑĨи н +ÑĢов а +ĠاÙĦ اÙĤتصاد +Ġch á +à¸ĺ าà¸Ļ +ë¥ ł +à¹Ħ à¸ķ +ÃŃ pio +Ùĭ ا +Ġоб Ñıз +Ùĩ ج +Ġì¤ij ìļĶ +ãģ® ãģ§ãģ¯ãģªãģĦ +بار اة +ãĤ¤ ãĥ« +Ġн оÑĢм +á»ī nh +m ö +mö glich +ÑĨи п +ãĤ¢ ãĤ¯ +×Ķ ×Ļ +ÑĨи алÑĮно +ĠÅĽ wi +ت ÙĤ +ĠÑģÑĤо им +بÙĬ عÙĬ +Ġ׾ ש×ŀ +г лÑı +глÑı д +ãģ¦ ãģıãĤĮ +ÄĻd zi +à¸Ĥ ั +à¸Ĥั à¹īà¸Ļ +Ø· ÙĤ +ĠìĹ Ń +ãģ£ãģ¦ãģĹãģ¾ ãģĨ +ĠdeÄŁer l +ĠdeÄŁerl endir +Ġü lk +Ġмн ог +๠ĭ +ë¿ IJ +ĠУ кÑĢа +ÄŁ ini +Ġбез оп +Ġбезоп аÑģ +à¸Ńà¸Ńà¸ģ à¹ģà¸ļà¸ļ +Ø§Ø ¸ +ØŃد اث +л еÑĢ +×Ļ× ¥ +×Ļ׳×ĺר ׳×ĺ +lar ınız +ØŃÙĬ ØŃ +ż eli +à¸Ń ัà¸ĩ +à¸Ńัà¸ĩ à¸ģ +à¸Ńัà¸ĩà¸ģ ฤษ +ĠоÑĤ лиÑĩ +ั ส +ëŀ į +ож но +ãĤ¹ ãĥĿ +ĠÑħ оÑĩ +Ġк ап +еÑĩ ен +ØŃÙĦ Ø© +ÙĬا Ùĩ +на л +×ķצ ר×Ļ×Ŀ +Ġk ald +åĥ į +ĠاÙĦØ´ خص +Ġз на +Ġwz gl +ż ycz +ê° Ŀ +à¸ŀ ลัà¸ĩ +íģ ¼ +Ġö l +Ġb ụ +Ø´ Ùĩر +Ġз ам +Ġд ев +×Ļ×ĺ ת +تعÙĦ ÙĤ +ÙĪÙħ Ø© +ãĤĴ ä½ľ +ãģį ãģ¦ +í ĥĿ +ras ında +ãĤĴ æİ¢ +ĠÙħ باشر +راج ع +Ġв озд +ÙħØŃ ا +×ķש ר +ĠиÑģÑĤ оÑĢ +ม ัà¸ģ +t ıģ +Ø« ار +تر ÙĨت +à¹ģà¸Ĥ à¹ĩ +à¹ģà¸Ĥà¹ĩ à¸ĩ +п оÑĩ +Ġ×ij ×IJ×ķת +ë¯ Ģ +ëĿ¼ ëıĦ +à¸Ĭ ัà¸Ķ +ส à¸ķà¹Į +ãĥĭ ãĥĥãĤ¯ +ид енÑĤ +Ġг ÑĢÑĥпп +ت Ø® +Ạł +ย ืà¸Ļ +ย ัà¸Ļ +ó ry +T Ãľ +ãģĹ ãĤĥ +ĠпÑĢов ед +лÑı еÑĤ +Ùħ Ø® +ย à¸Ńม +×Ľ×ł×¡ ת +ĠاÙĦÙħ ÙĨت +Ġol mad +ר׼ ×ĸ×Ļ +Ġв ÑģÑĤÑĢ +ĠиÑģ Ñģлед +ÑĤвеÑĢ Ð¶ +بد ÙĪ +еÑĢ ÑĤ +ï» · +± ħ +สัม à¸ŀัà¸Ļà¸ĺà¹Į +ิ à¹Īà¸Ļ +צ ×Ļ×ij +wiÄĻ t +Ġì° ¸ +Ġz wiÄħz +سب ÙĪØ¹ +ãĥĥ ãĤ° +à¸Ľà¸¥ à¸Ńà¸Ķ +à¸Ľà¸¥à¸Ńà¸Ķ à¸łà¸±à¸¢ +ãĤĤ ãĤĬ +ÙĤد س +Ġspr z +Ġsprz eda +Ġist edi +Ġk hu +Ġд ен +Ġko ÅĦ +Ġ×ij ×Ĺ×Ļ +à¹Ģà¸Ĺ à¹īา +×ķס ×Ļ×£ +ãĥĭ ãĥ¥ãĥ¼ +ĠпÑĢед оÑģÑĤ +ĠпÑĢедоÑģÑĤ ав +à¹Ĥ à¸Ł +é v +ĠاÙĦص ØŃ +صØŃ اب +à¹Ģà¸Ī à¹ĩà¸ļ +вл ек +วั à¸ķ +à¸ĸ ุ +ãģĵãģ¨ãģĮãģ§ãģį ãģ¾ãģĻ +ÙĤÙĬ ÙĤÙĬ +×ķ׊ר +Ñĭ ÑĪ +ĠоÑĤ но +ĠоÑĤно ÑĪ +об илÑĮ +Ùģ ØŃ +ı nt +ınt ı +Ġ׾ ×ij×ĵ +í İĺìĿ´ì§Ģ +ãĥĬ ãĥ« +ĠÙħ ساء +×Ļ×ĺ ×ij +ÑĮ еÑĢ +ëĦ · +Ñĭ ÑĤа +ĠоÑĩ еÑĢ +à¸Ķ ืà¹Ī +à¸Ķืà¹Ī ม +ĠN gh +ت عب +ÙĦاÙĤ ات +×ķ׾×ķ×Ĵ ×Ļ×Ķ +ĠìĿ´ ê²ĥ +Ġ×Ķ ×ijר +ìľ µ +à¹Ģà¸Ħล ืà¹Īà¸Ńà¸Ļ +Ùĩ Ø© +à¸Īำ à¹Ģà¸Ľà¹ĩà¸Ļ +å¤ī ãģĪ +wi ÅĽcie +ch od +chod zÄħ +в ÑĢо +×ŀ×Ĺ ×Ļר +Ġy ı +Ġyı ll +ì¡ Į +à¹Ħ หว +ãģªãģı ãģª +Ġзав иÑģ +ĠìĺĪ ìĪĺ +Ùģ Ø° +á»§ ng +à¸ŀุ à¸Ĺà¸ĺ +з н +lay an +ãĤ ¡ +à¸ģà¹ĩ à¸ķาม +ĠsaÄŁ lam +ร à¸ĵ +ĠÑģ иÑĤ +ĠÑģиÑĤ Ñĥ +ĠاÙĦت ÙĨ +×Ķ ×ĸ +ĠØ· ÙĪÙĬÙĦ +ta ÅĤ +Ġgö rd +å¤ī ãĤı +ëĥ ¥ +à¸Ħà¹Ī à¸Ńย +×IJ ×ķ×ĺ +ëħ IJ +ãĥ©ãĥ³ ãĤ¹ +วั à¸Ĵ +วัà¸Ĵ à¸Ļ +Ġol uÅŁ +פע ×ķ׾ +Ġszczeg óÅĤ +à¸Ħา สิ +à¸Ħาสิ à¹Ĥà¸Ļ +pow ied +ĠÑĤ еб +หà¸Ļ à¹Īวย +Ġм ил +ØŃ Ùĥ +à¸Ĺ à¸Ķ +ĠмаÑĤ еÑĢиал +ÅĤ ow +à¹Ģà¸ģ ีย +ĠÑģов еÑĢ +ãĤ © +à¸Ľ ริ +Ġи Ñİ +наÑĩ ен +ÑĢен д +mu ÅŁtur +ĠпÑĢод Ñĥк +з д +Ñı ÑĤи +ÑıÑĤи Ñı +à¹Ģม ีย +رات ÙĬج +Ġam acı +ש ×ķ׾ +ש×ķ׾ ×Ĺ +สะ à¸Ńา +สะà¸Ńา à¸Ķ +פ×Ĵ ×¢ +عب Ø© +d ın +íħ Ķ +Ġ×ŀש ×Ĺ×§ +Ġfi yat +Ġз аÑı +ĠзаÑı в +à¹Ĥ หล +à¹Ĥหล à¸Ķ +à¸ģรุà¸ĩ à¹Ģà¸Ĺà¸ŀ +צ×Ļ ×Ļף +ìļ ± +Ùħ ب +Ùħب اد +land ır +Ġв еÑģÑĮ +Ġh ük +ĠÐĴ оз +ÑĩиÑĤ Ñĭва +ว ล +×ķצ ×¢ +à¸Ĥà¸ĵะ à¸Ĺีà¹Ī +ĠaÅŁ aģı +׾×IJ ×ķ×ŀ×Ļ +tr zym +Ã¤ÃŁ ig +owo ÅĽci +ãģĿ ãĤĤ +Ġroz wiÄħz +ĠgÅĤ ówn +м онÑĤ +×ŀ ×ķ×ŀ +ĠÑģÑĤ ан +ÙĦا ÙĤØ© +p rowad +prowad zi +ĠÑģоÑģÑĤ оÑı +×Ļ×IJ ×ķת +r ı +g ı +ãĥij ãĥij +Ġна лиÑĩ +×Ķ ×¦×¢ +Ġ׳ ×Ķ +à¸Ħ ัà¸ļ +ع راض +и ж +Ùĩ ائÙĬ +ãĤī ãģı +ож еÑĤ +Ġоб оÑĢ +ĠобоÑĢ Ñĥд +Ø£ سÙĦ +à¹ĩ à¸Ķ +ÑĢÑĥ ÑĤ +دÙĬ ÙħÙĤ +دÙĬÙħÙĤ را +Ġjest e +×ķ×ķ ×Ļר +×ij×ĵ ×Ļ×§ +деÑĢж ива +ãģĬ ãģı +ewn ÄĻtr +ewnÄĻtr zn +à¸ŀ ฤ +Ġ×IJ ×ķ×Ķ +ת×Ĺ ×ķש +Ġz ob +д Ñĥм +ĠÑģ Ñĭ +ÙĬر ا +ĠwiÄĻ ks +à¹ģà¸ķà¸ģ à¸ķà¹Īาà¸ĩ +lar aras +lararas ı +íĺ Ģ +ëī ´ +×ķ×Ĵ ׾ +ĠоÑĤ меÑĤ +ĠÑĢ Ð°Ð½ +ت ÙĥÙĦ +иÑĤелÑĮ н +à¸Ľà¸£à¸° วั +à¸Ľà¸£à¸°à¸§à¸± à¸ķิ +ìŀ ĸ +мож но +pie czeÅĦ +pieczeÅĦ st +ëª » +ìĬ ¨ +×ŀס ×ŀ +á» ¦ +ศ ิ +ศิ ล +ศิล à¸Ľ +ĠÅļ w +ãĥĥ ãĤ·ãĥ§ãĥ³ +unit Ãł +Ġmiesz ka +Ġmieszka ÅĦ +pr zed +przed si +przedsi ÄĻb +przedsiÄĻb ior +à¸Ľà¸£à¸° สิà¸Ĺà¸ĺิ +à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺิ à¸łà¸²à¸ŀ +ย à¹Ī +ìķ Ļ +รว à¸Ķ +รวà¸Ķ à¹Ģรà¹ĩว +å½ĵ ãģŁãĤĬ +äl le +Ñĥ еÑĤÑģÑı +ã n +ëł µ +th è +ãĤĴ åĪ©ç͍ +ì µľ +íĵ ¨ +à¸Ĺ ัà¸ļ +า à¸Ħม +ãģ ĩ +ëĤ Į +à¹Ģà¸Ľà¸¥ à¹Īา +â ¦ +ë ¾ +ê Ģ +ê ĩ +â ¡ +ðŁ Ł +ã IJ +â º +á Ń +á Ļ +á ĵ +á ² +ðĵ ı +á ¬ +â ¯ +ä ¨ +ê Ŀ +ê « +ð ij +ðĵ ĥ +ðĿ ħ +< unk + + + +Ġع ÙĦÙī +Ġm á»Ļt +Ġv Ỽi +Ġng ưá»Ŀi +ĠØ¥ ÙĦÙī +Ġnh ững +Ġth á»ĥ +Ġ×IJ ×ķ +Ġ×¢ ×Ŀ +ا Ùĭ +Ġ à¹ģละ +ĠÙĦ ا +Ġnh ư +ĠاÙĦت ÙĬ +Ġ×Ķ ×ķ×IJ +ĠÄij ến +ĠØ£ ÙĪ +Ġv á»ģ +ĠlÃł m +Ġs ẽ +Ġc Å©ng +Ġ ợ +ĠÄij ó +Ġnhi á»ģu +Ġt ại +Ġtr ên +Ġ×Ĵ ×Ŀ +Ġnh Ãł +Ġ׼ ×Ļ +Ġs á»± +ĠÄij ầu +Ġb á»ĭ +ĠÙĩ ذا +Ġnh ất +Ġph ải +Ġhi á»ĩn +Ġdụ ng +ĠÄij á»Ļng +ĠاÙĦÙĦ Ùĩ +ĠØ Į +ĠÙĥ ÙĦ +Ġvi á»ĩc +Ġn Äĥm +Ġth ì +Ġh á»įc +ĠÙĪ Øª +t é +Ġا ÙĨ +Ġt ôi +Ġ×IJ ׳×Ļ +Ġ׾ ×Ļ +Ġ×ŀ ×ķ +Ġng Ãły +Ġn Æ°á»Ľc +Ġ×Ķ ×Ļ×IJ +Ġ×IJ ×Ļ +Ġh Æ¡n +ĠÙĩ ذÙĩ +ĠÙĪ ÙĬ +ĠاÙĦ ذÙĬ +Ġ×ķ ×ŀ +Ġgi á +Ġnh ân +Ġch ÃŃnh +Ġm ình +ĠÐĿ а +Ġth ế +Ġ×Ļ ×ķתר +Ġ×IJ ×Ŀ +Ġn ên +Ġh ợ +Ġhợ p +Ġc òn +ĠÙĩ ÙĪ +Ġc Æ¡ +Ġr ất +ĠVi á»ĩt +Ġب عد +Ġש ×Ļ +Ġth á»Ŀi +Ġc ách +ĠÄij á»ĵng +Ġн о +Ġtr ưá»Ŀng +Ø Ł +ĠÄij á»ĭnh +ĠÄiji á»ģu +×Ļ ×Ļ×Ŀ +Ġth á»±c +n ın +Ġh ình +Ġn ói +Ġc ùng +Ġ×Ķ ×Ķ +ĠØ¥ ÙĨ +Ġ×IJ ×ij׾ +Ġnh ưng +Ġbi ết +Ġж е +Ġch úng +ĠÄij ang +Ġذ ÙĦÙĥ +Ġl ên +Ġkh ách +Ġn Ãło +Ġs á»Ń +Ġkh ác +Ġë° ı +Ġl ý +×Ļ ×Ļ +ĠÄij ây +Ġ׾ ×ŀ +Ġc ần +Ġtr ình +Ġph át +ãģ« ãĤĤ +п о +Ġn Äĥng +Ġb á»Ļ +Ġv ụ +ĠÄij á»Ļ +Ñĩ е +Ġnh áºŃn +Ġtr Æ°á»Ľc +Ġ×¢ ×ĵ +Ġh Ãłnh +ĠØ® ÙĦاÙĦ +Ġl ượng +Ġc ấp +Ġtá» ± +Ġv ì +Ġt ư +Ġch ất +Ġ׼ ×ŀ×ķ +Ġg ì +Ġש ׳ +Ġt ế +ת ×ķ +Ġnghi á»ĩp +Ġm ặt +ĠÙĥ Ùħا +Ġ×ij ×Ļף +Ġר ×§ +Ġth ấy +Ġmá y +ĠÙģ Ùī +Ġd ân +Ġ×IJ ×Ĺ×ĵ +Ġt âm +Ġ׼ ×ļ +Ġ׾ ×ķ +в о +Ġt ác +Ġto Ãłn +ĠÙĪ Ùħ +Ġk ết +Ġ หรืà¸Ń +ĠÙĪØ§ÙĦ Ùħ +ĠÄiji á»ĥm +Ġ×ĸ ×ķ +Ġ×ij ×ķ +׼ ×ķת +Ġh á»Ļi +Ġb ằng +ت Ùĩا +Ġ׼ ×ĵ×Ļ +Ġ×Ķ ×Ŀ +Ġxu ất +ĠÙĤ د +Ġb ảo +Ġt á»ijt +Ġt ình +ĠÙĩ ÙĬ +ĠÄij á»iji +Ġthi ết +Ġhi á»ĩu +Ġti ếp +Ġt ạo +ת ×Ķ +Ġch á»§ +o ÅĽÄĩ +Ġgi ú +Ġgiú p +Ġà ½ +Ġqu ả +Ġlo ại +Ġc ô +Ġà ´ +Ġô ng +Ġ×Ķ ×ķ +ĠاÙĦÙĬ ÙĪÙħ +ĠtÃŃ nh +г а +Ġph òng +Ġ Äĥn +Ġع اÙħ +Ġv á»ĭ +lar ını +r ÃŃa +Ġt Ỽi +ĠÄij ưá»Ŀng +Ġgi Ỽi +Ġb ản +Ġc ầu +Ġnhi ên +Ġb á»ĩnh +Ġth ưá»Ŀng +Ġ×IJ ×Ļף +ĠÄij á»ģ +Ġh á»ĩ +Ġ×Ļש ר×IJ׾ +Ġqu á +ĠÐĹ Ð° +ãģ® ãģ§ãģĻãģĮ +ĠÐŁ ÑĢи +Ġph ần +ĠÙĪ ÙĦا +ĠlỼ n +Ġtr á»ĭ +Ġcả m +Ġм о +Ġd ùng +ĠاÙĦ Ùī +ĠعÙĦÙĬ Ùĩ +ĠìŀĪ ìĬµëĭĪëĭ¤ +ÙĬ ÙĤ +ĠÙĤ بÙĦ +Ġho ặc +ĠØŃ ÙĬØ« +Ġ à¸Ĺีà¹Ī +Ġغ ÙĬر +ĠÄij ại +Ġsá»ij ng +нÑĭ ми +Ġth ức +Ġפ ×Ļ +ĠÄiji á»ĩn +ãģª ãģĭãģ£ãģŁ +Ġgi ải +Ġv ẫn +Ġи Ñħ +Ġö nce +Ġv áºŃy +Ġmu á»ijn +Ġ ảnh +à¹ĥà¸Ļ à¸ģาร +ĠQu á»ijc +Ġk ế +׳ ×IJ +Ġס ×Ļ +Ġy êu +ãģ® ãģĭ +ĠÄij ẹ +ĠÄijẹ p +Ġch ức +Ġy ıl +ĠTür kiye +d é +ĠÙĤ اÙĦ +Ġd á»ĭch +ĠolduÄŁ u +Ġch á»įn +Ġت Ùħ +หà¸Ļ ึà¹Īà¸ĩ +ãģķãĤĮ ãģŁ +Ġph áp +ìĽ Ķ +Ġti á»ģn +ãģĹ ãģ¾ãģĹãģŁ +Ġש ׾×IJ +ÙĦ Ø© +Ġ׾פ ׳×Ļ +Ġ×ij ×Ļת +ĠH Ãł +ĠØŃ ت +ĠØŃت Ùī +Ġ×¢ ×ķ×ĵ +Ġn ó +Ġth áng +à¹Ģลืà¸Ń à¸ģ +ר ×Ķ +Ġt Äĥng +Ġcá i +Ġtri á»ĥn +Ġ×IJ×ķת ×ķ +ìłģ ìĿ¸ +ĠC ông +Ġ׾×Ķ ×Ļ×ķת +Ġг ода +и Ñİ +Ġب عض +Ġ à¸ģาร +èī¯ ãģĦ +ÙĪ Øª +Ġli ên +ĠÐĿ о +ĠÐĿ е +çļĦ ãģª +ĠÙħ ت +ĠÑĤак же +ĠкоÑĤоÑĢ Ñĭе +Ġ×Ļ ×ĵ×Ļ +Ġtr á»įng +ãĤµ ãĤ¤ãĥĪ +ìłģ ìľ¼ë¡ľ +Ġt áºŃp +Ġש ׾×Ļ +íķĺ ê²Į +Ġt Ãłi +ĠÐ ¯ +Ġr á»ĵi +ا Ùĥ +Ġth ương +Ġ×Ķ ×ĸ×Ķ +ĠÙĪ ÙħÙĨ +à¸Ĺีà¹Ī มี +Ġcu á»Ļc +Ġbü yük +ãģ¨ ãģĭ +Ġ×ij ×Ļ×ķתר +Ġl ần +Ġgö re +Ġtr ợ +Ġ×ĺ ×ķ×ij +ÑĤÑĮ ÑģÑı +Ġth á»ijng +Ġ׼ ש +Ġti êu +Ġ×ŀ×IJ ×ķ×ĵ +Ø Ľ +k Äħ +Ġ à¹ĥà¸Ļ +Ġv ấn +Ġש ׾×ķ +ĠÄij á»ģu +Ùģ Øª +Ġê²ĥ ìĿ´ +Ġh óa +ĠاÙĦع اÙħ +ĠÙĬ ÙĪÙħ +к ой +Ġbi á»ĩt +ÑģÑĤ о +Ġ×Ķ ×Ļ×ķ +à¸Ĺีà¹Ī à¸Īะ +Ġ×ĵ ×Ļ +Ġ×IJ ×ļ +Ġá n +ص ÙĪØ± +Ġtr ÃŃ +ĠÐŁÑĢ Ð¾ +Ġl á»±c +ãģĹãģ¦ ãģĦãģ¾ãģĻ +Ġb Ãłi +Ġ×ĸ ×IJת +Ġb áo +à¸ļ à¸Ļ +ĠëĮĢ íķľ +Ġti ế +Ġtiế ng +Ġb ên +ãģķãĤĮ ãĤĭ +s ión +Ġt ìm +×¢ ×ķ +m é +ни Ñı +ãģ» ãģ© +Ġà¹Ģà¸ŀ ราะ +ب Ø© +Ġë¶ Ħ +Ġ×IJ ×ĸ +à¸Ĺ à¹Īาà¸Ļ +ת ×Ŀ +Ġth êm +Ġho ạt +y ı +×ĸ ×ķ +Ġgi á»Ŀ +Ġb án +à¸Ĥ าย +Ñĩ а +Ġ à¹Ĩ +ĠاÙĦÙħ ت +ĠоÑĩ енÑĮ +Ġb ất +Ġtr ẻ +ÑĤ ÑĢ +ĠØ£ ÙĨÙĩ +ĠØ« Ùħ +Ġ׼ ×ŀ×Ķ +Ġkh ó +Ġr ằng +ĠÙĪ ÙģÙĬ +ни й +Ġho Ãłn +t ó +Ġ×IJ שר +ĠìĥĿ ê°ģ +Ñģ а +Ġ׼ ×ijר +ĠÑįÑĤ ом +lar ının +Ġch ưa +з и +Ġd ẫn +ĠÐļ ак +ج ÙĪ +ĠбÑĭ ло +ĠÙĬ ت +n ı +ÅĤ am +ĠÙĪÙĩ ÙĪ +×ij ×ķ +п и +ר ת +Ġqu á»ijc +ж д +ĠÄij Æ¡n +Ùĥت ب +Ġm ắt +ระ à¸ļ +ระà¸ļ à¸ļ +ĠÙĥ اÙĨت +Ġth ân +สิà¸Ļ à¸Ħà¹īา +×Ĵ ×Ļ +Ġph ương +à¹Ħมà¹Ī à¹Ħà¸Ķà¹ī +ĠìĦ ± +ĠC ác +Ġ×Ķ×ŀ ×ķ +ĠÑĤ ем +Ġ×ĵ ×ķ +à¸Ńะ à¹Ħร +Ġv Äĥn +ãģª ãģ®ãģ§ +ĠN á»Ļi +Ġ×¢ ×ķ +ãĤīãĤĮ ãĤĭ +Ġs áng +Ġgö ster +ãģĵãģ¨ ãĤĴ +Ġtaraf ından +Ġм а +ĠпоÑģл е +Ġ׳ ×Ļת +Ġ׳×Ļת ף +Ġл еÑĤ +Ġ׾ ׳×ķ +Ñģ Ñģ +Ġ×Ļ ×ķ +п е +ĠÙĪ ÙĦÙĥ +ĠÙĪÙĦÙĥ ÙĨ +Ġngo Ãłi +ĠÄij á»ĭa +r zÄħd +dz iaÅĤ +ĠÙħ ر +иÑĤÑĮ ÑģÑı +Ġ×IJ×Ĺר ×Ļ +Ġ׾ ׼׾ +à¸Ĥ à¹īà¸Ńม +à¸Ĥà¹īà¸Ńม ูล +Ġб ол +Ġбол ее +جÙħ ع +л еÑĤ +Ġl á»ĭch +ĠÙħ Ø«ÙĦ +Ġ그리 ê³ł +Ġth ứ +ĠdeÄŁ il +ÙĪ ØŃ +Ġש׾ ×ļ +ĠÙħ ØŃÙħد +Ġn ếu +ĠÄij á»ķi +Ġv ừa +Ġm á»įi +Ġо ни +Ġl úc +ĠÙĬ ÙĥÙĪÙĨ +ì§ Ī +Ġש׾ ׳×ķ +ĠÐĶ Ð¾ +Ġש ׳×Ļ +ล ิ +×IJ פשר +Ġs ức +ê¶ Į +Ġ ứng +à¹Ħมà¹Ī มี +Ø·ÙĦ ب +ĠÑĩ ем +Ġch uyên +Ġth ÃŃch +Ġ×ķ ×Ļ +íķ © +ĠÙħ صر +д о +ĠÄij ất +Ġch ế +à¸Ĭ ืà¹Īà¸Ń +Ġìĭ ł +ĠØ¥ ذا +Ġر ئÙĬس +Ġש ×Ļש +Ġgiả m +Ñģ ка +lar ında +Ġs ợ +ĠtÃŃ ch +ĠÙĦ ÙĥÙĨ +Ġب Ùħ +×¢ ×ķ×ij +×¢×ķ×ij ×ĵ +ÅĤÄħ cz +ları na +Ġש ×Ŀ +ĠÙĦ ت +Ġש×Ķ ×ķ×IJ +t ów +Ġëĭ¤ 른 +ĠØ£ Ùĥثر +ãģ® ãģ§ãģĻ +׼ ×Ļ×Ŀ +ĠolduÄŁ unu +ãģĭ ãģª +ãĤĤ ãģĨ +ÙĬ ØŃ +Ġnh ìn +Ġngh á»ĩ +ãģ«ãģª ãģ£ãģ¦ +п а +Ġquy ết +ÙĦ ÙĤ +t á +Ġlu ôn +ĠÄij ặc +Ġ×IJ ר +Ġtu á»ķi +s ão +ìĻ ¸ +ر د +ĠبÙĩ ا +Ġ×Ķ×Ļ ×ķ×Ŀ +×ķ ×ķ×Ļ +ãģ§ãģĻ ãģŃ +ĠÑĤ ого +Ġth á»§ +ãģĹãģŁ ãģĦ +ر ÙĤ +Ġb ắt +г Ñĥ +Ġtá» Ń +ÑĪ Ð° +Ġ à¸Ľà¸µ +Ġ×Ķ×IJ ×Ŀ +íı ¬ +ż a +Ġ×IJת ×Ķ +Ġn á»Ļi +Ġph ÃŃ +ĠÅŁek ilde +Ġl á»Ŀi +d ıģı +Ġ׼×IJ ף +Ġt üm +Ġm ạnh +ĠM ỹ +ãģĿ ãĤĵãģª +Ġnh á»ı +ãģª ãģĮãĤī +Ġb ình +ı p +à¸ŀ า +ĠÄij ánh +ĠÙĪ ÙĦ +ר ×ķת +Ġ×IJ ×Ļ×ļ +Ġch uyá»ĥn +Ùĥ ا +ãĤĮ ãĤĭ +à¹ģม à¹Ī +ãĤĪ ãģı +ĠÙĪ ÙĤد +íĸ Īëĭ¤ +Ġn Æ¡i +ãģ«ãĤĪ ãģ£ãģ¦ +Ġvi ết +Ġà¹Ģà¸ŀ ืà¹Īà¸Ń +ëIJĺ ëĬĶ +اد ÙĬ +ĠÙģ Ø¥ÙĨ +ì¦ Ŀ +ĠÄij ặt +Ġh Æ°á»Ľng +Ġx ã +Ġönem li +ãģł ãģ¨ +Ġm ẹ +Ġ×ij ×Ļ +Ġ×ĵ ×ijר +Ġv áºŃt +ĠÄij ạo +Ġdá»± ng +ĠÑĤ ом +ĠÙģÙĬ Ùĩا +Ġج ÙħÙĬع +Ġthu áºŃt +st ÄĻp +Ġti ết +Ø´ ÙĬ +Ġе Ñīе +ãģĻãĤĭ ãģ¨ +ĠmÃł u +ĠÑįÑĤ ого +Ġv ô +ĠÐŃ ÑĤо +Ġth áºŃt +Ġn ữa +Ġbi ến +Ġn ữ +Ġ׾ ׼×Ŀ +×Ļ ×Ļף +Ġس ت +ĠÐŀ ÑĤ +Ġph ụ +ê¹Į ì§Ģ +Ġ׾ ×ļ +Ġk ỳ +à¹ĥ à¸Ħร +Ġg ây +ĠÙĦ ÙĦÙħ +Ġtụ c +ت ÙĬÙĨ +Ġtr ợ +Ġ׾ פ×Ļ +Ġb á»ij +ĠÐļ а +ĠÄij ình +ow Äħ +s ında +Ġkhi ến +s ız +Ġк огда +ס ׾ +ĠбÑĭ л +à¸Ļ à¹īà¸Ńย +обÑĢаР· +Ġê²ĥ ìĿ´ëĭ¤ +ëĵ¤ ìĿĢ +ãģ¸ ãģ® +Ġà¹Ģม ืà¹Īà¸Ń +Ġph ục +Ġ׊׾ק +Ġh ết +ĠÄij a +à¹Ģà¸Ķà¹ĩ à¸ģ +íĺ ķ +l ÃŃ +ê¸ ī +Ġع دد +ĠÄij á»ĵ +Ġg ần +Ġ×Ļ ×ķ×Ŀ +Ġs Ä© +ÑĢ Ñıд +Ġquy á»ģn +Ġ×IJ ׾×IJ +Ùĩ Ùħا +׳ ×Ļ×Ķ +׾ ×ķת +Ġ×Ķר ×ij×Ķ +Ġti ên +Ġal ın +Ġd á»ħ +人 ãģĮ +но Ñģ +л ÑģÑı +ĠÄij ưa +ส าว +иÑĢов ан +Ġ×ŀס פר +×Ĵ ף +Ġki ến +ĠÐ ¨ +p é +б Ñĥ +ов ой +б а +ĠØ¥ ÙĦا +×IJ ׾×Ļ +Ġx ây +Ġb ợi +Ġש ×ķ +人 ãģ® +×§ ×Ļ×Ŀ +à¹Ģà¸Ķ ืà¸Ńà¸Ļ +Ġkh á +Ġ×ķ ׾×Ķ +×ĵ ×ķת +Ġ×¢ ×ij×ķר +Ġبش ÙĥÙĦ +ĠÙĩÙĨا Ùĥ +ÑĤ ÑĢа +Ġ íķĺëĬĶ +ร à¸Ńà¸ļ +owa ÅĤ +h é +Ġdi á»ħn +Ġ×Ķ ×Ľ×ľ +ĠØ£ س +Ġch uyá»ĩn +ระ à¸Ķัà¸ļ +ĠNh ững +Ġ×IJ ×Ĺת +ĠØŃ ÙĪÙĦ +л ов +׳ ר +Ġ×ķ ׳ +Ġch Æ¡i +Ġiç inde +ÑģÑĤв Ñĥ +Ġph á»ij +ĠÑģ Ñĥ +ç§ģ ãģ¯ +Ġch ứng +Ġv á»±c +à¹ģ à¸Ń +Ġl áºŃp +Ġtừ ng +å°ij ãģĹ +ĠNg uy +ĠNguy á»ħn +ĠÙģÙĬ Ùĩ +Ġб а +×Ļ ×Ļת +Ġ×ľ×¢ ש×ķת +Ġ×ŀ ׼ +Ġnghi á»ĩm +Ġм ного +Ġе е +ëIJĺ ìĸ´ +Ġl ợi +Ġ׾ ׾×IJ +Ġ׼ ף +Ġch ÃŃ +ãģ§ ãģ® +×Ĺ ×ķ +ש ×ķ×Ŀ +Ġ×ŀ ר +ĠÐĶ Ð»Ñı +Å ģ +Ġ׼×IJ שר +ĠM á»Ļt +ĠÙĪØ§ÙĦ ت +ĠìĿ´ 룰 +ÅŁ a +Ġchi ến +Ġaras ında +Ġ×ij ×IJתר +ãģķãĤĮ ãģ¦ãģĦãĤĭ +Ø´ ÙĥÙĦ +Ġt ượng +Ġت ت +ĠC ó +Ġb á»ı +Ġtá»ī nh +Ġkh ÃŃ +ĠпÑĢ Ð¾ÑģÑĤ +ĠпÑĢоÑģÑĤ о +ĠÙĪ ÙĤاÙĦ +Ġgi áo +ĠN ếu +×IJ ×ŀר +×¢×ł×Ļ ×Ļף +íİ ¸ +Ùĩد Ùģ +ĠB á»Ļ +Ġb Ãłn +Ġng uyên +Ġgü zel +ส าย +ì² ľ +×ŀ ×ķר +Ġph ân +ס פק +×§ ×ij׾ +ĠاÙĦÙħ تØŃ +ĠاÙĦÙħتØŃ دة +ائ د +Ġ×IJ ×ŀר +Ġki ÅŁi +ì¤ Ģ +Ġtr uyá»ģn +ĠÙĦ Ùĩا +ĠÐľ а +à¸ļริ ษ +à¸ļริษ ั +à¸ļริษั à¸Ĺ +Ġש ׳×Ļ×Ŀ +Ġмен Ñı +ÅŁ e +Ġdi á»ĩn +Ġ×IJ׳ ×Ĺ׳×ķ +k ü +Ġc á»ķ +Ġm á»Ĺi +w ä +Ùħ ÙĬ +Ġhi á»ĥu +ëĭ ¬ +Ġ×Ķ ×Ĺ׾ +Ġt ên +Ġki á»ĩn +ÙĨ ÙĤÙĦ +Ġv á»ĩ +×ĵ ת +ĠÐłÐ¾ÑģÑģ ии +л Ñĥ +ĠاÙĦع ربÙĬØ© +ĠØ· رÙĬÙĤ +Ġ×Ķ×ij ×Ļת +Ñģ еÑĢ +Ġм не +ä u +Ġtri á»ĩu +ĠÄij á»§ +Ġר ×ij +ت ÙĩÙħ +à¸ĭ ี +Ġì§Ģ ê¸Ī +li ÅĽmy +د عÙħ +ãģł ãĤįãģĨ +Ñģки е +Ġh á»ıi +Ġ×§ ×ķ +ÑĢÑĥ Ñģ +ÙĨ ظر +ãģ® ãĤĤ +Ġ×Ķ ×Ľ×Ļ +ĠìĽ IJ +ÙĪ Ùĩ +ĠÙĪ Ùİ +ĠB ạn +п лаÑĤ +Ġ×ŀ ×ŀש +лÑİ Ð± +ĠнÑĥж но +Ġth ư +ãģ µ +ãģı ãĤīãģĦ +ر Ø´ +ר ×ķ×Ĺ +ĠÙĬ تÙħ +Ġצר ×Ļ×ļ +Ġph á +ม à¸Ńà¸ĩ +Ġ×ij×IJ ×ķפף +Ġcả nh +Ġíķľ ëĭ¤ +Ġ×Ķ×ŀ ת +à¸ķà¹Īาà¸ĩ à¹Ĩ +มี à¸ģาร +Ñģки Ñħ +ĠÐĴ Ñģе +Ġا ÙĪ +ج ÙĬ +ãģĵãģ¨ ãģ¯ +Ġd Ãłi +Ġh á»ĵ +èĩªåĪĨ ãģ® +à¹Ħ หà¸Ļ +ëĵ¤ ìĿĦ +ĠV Äĥn +Ġд аж +Ġдаж е +Ñĭ ми +лаÑģ ÑĮ +ÙĬ ÙĪÙĨ +ÙĨ ÙĪ +c ó +ãģĹãģ¦ ãģĦãģŁ +ãģł ãģĭãĤī +طاÙĦ ب +Ġc á»Ńa +п ÑĢоÑģ +ãģªãģ© ãģ® +รุ à¹Īà¸Ļ +Ġchi ếc +л Ñĭ +ĠÑıвлÑı еÑĤÑģÑı +Ġn á»ķi +ãģ® ãģĬ +Ġ×IJת ×Ŀ +ĠëķĮ문 ìĹIJ +à¸ģล าà¸ĩ +ĠbaÅŁ ka +ìĦ Ŀ +ĠÑĨ ел +Ùģ ÙĤ +ãģ«ãĤĪ ãĤĭ +ÙĤ ا +Ġçı kar +Ġcứ u +Ø· ا +Ġש ת +à¹Ĥ à¸Ħ +Ġ×ŀ ׾ +Ġ×Ķ ×¤×¨ +Ġг де +ĠØ® Ø· +åīį ãģ« +c jÄĻ +Ġ׊ש×ķ×ij +ר×Ĵ ×¢ +Ġkho ảng +ĠÄij á»Ŀi +ĠÐł е +Ġо на +Ġ×IJ ׳×ķ +ãģ® ãģ« +ĠاÙĦذ ÙĬÙĨ +кÑĥ п +ãĤµ ãĥ¼ãĥ +ãĤµãĥ¼ãĥ ĵ +ãĤµãĥ¼ãĥĵ ãĤ¹ +в ал +г е +Ġgi ữa +ĠKh ông +ĠâĹ ĭ +à¸ģล ุà¹Īม +ĠÙħÙĨ ذ +à¸Ń à¹Īาà¸Ļ +ĠÑģп оÑģоб +ĠÄij á»Ļi +Ġdi ÄŁer +Ġ à¸ĸà¹īา +Ùħ Ø«ÙĦ +Ġ×Ķ×IJ ×Ļ +Ġد ÙĪÙĨ +ÙĬر اÙĨ +Ñī и +بÙĨ اء +ĠØ¢ خر +ظ Ùĩر +Ġ×ij ׼ +ĠاÙĦÙħ ع +ãĥ Ĵ +Ġt ất +Ġm ục +ĠdoÄŁ ru +ãģŁ ãĤī +Ġס ×ķ +Ġx ác +ร à¸Ń +ĠcÄĥ n +Ġон л +Ġонл айн +Ġk ý +Ġch ân +Ġ à¹Ħมà¹Ī +اØŃ Ø© +r án +׳×Ļ ×Ļ×Ŀ +Ġ×ij ף +ĠÐ ĸ +à¸ķร à¸ĩ +д Ñĭ +Ġs ắc +ÙĦ ت +ãĥŃ ãĥ¼ +ĠÙĦ ÙĨ +Ġר ×ķ +Ġd Æ°á»Ľi +à¹Ģ à¸ĺ +à¹Ģà¸ĺ à¸Ń +e ÄŁi +Ġ×ķ ש +ĠÙĦ Ø£ +Ġg ặp +Ġc á»ij +ãģ¨ ãģ¦ãĤĤ +رÙĪ Ø³ +Ġ׾×Ķ ×Ļ +Ġë³ ¸ +ä¸Ĭ ãģĴ +Ġm ức +Ñħ а +Ġìŀ ¬ +à¸ī ัà¸Ļ +ÑĢÑĥ ж +Ġaç ık +ÙĪ Ø§ÙĦ +Ġ×ĸ ×ŀף +人 ãģ¯ +ع ÙĬÙĨ +Ñı Ñħ +Ġ×Ĵ×ĵ ×ķ׾ +ר ×ķ×ij +g ó +ëĿ¼ ê³ł +Ġark adaÅŁ +ÙĨ شر +Ġгод Ñĥ +ĠболÑĮ ÑĪе +ãģ¡ãĤĩ ãģ£ãģ¨ +Ġcâ u +Ġs át +íĶ ¼ +Ġti ến +íķ´ ìķ¼ +ĠÙĪ Ø£ÙĨ +à¸Ļ าà¸Ļ +Ġ×ij×IJ×ŀ צע +Ġ×ij×IJ×ŀצע ×ķת +Ġ׾ ר +Ġqu ản +ĠÙĪØ§ÙĦ Ø£ +Ġ×IJ×ķת ×Ķ +Ġìĸ´ëĸ ¤ +Ġê²ĥ ìĿĢ +ØŃس ÙĨ +Ġm ất +à¸Ħ ูà¹Ī +ãĥ¬ ãĥ¼ +ĠÐĶ Ð° +Ġol ması +Ġthu á»Ļc +׳ ×Ĺ +íĨ ł +Ġsö yle +ãģĿãģĨ ãģ§ãģĻ +Ġت ÙĥÙĪÙĨ +л ÑĥÑĩ +׾ ×Ļ×ļ +ĠØ£ ØŃد +ли ÑģÑĮ +ĠвÑģ его +Ġ×Ķר ×ij +Ġëª » +o ÄŁ +oÄŁ lu +ĠìĦ ł +Ġк аÑĢ +à¸łà¸² à¸Ħ +e ÅĦ +Ġ à¸ģà¹ĩ +Ġa ynı +Ġb Ãł +ãģªãĤĵ ãģ¦ +Ġ모 ëĵł +ÙĤر ار +ãģĹãģª ãģĦ +ĠÐĴ о +ĠÙĪÙĩ ÙĬ +ни ки +ãĤĮ ãģŁ +Ġchu ẩn +ר ×¢ +Ùģ Ø±ÙĬÙĤ +ãĤĴ åıĹãģij +ĠÄij úng +б е +׼ ×ķ×Ĺ +п Ñĥ +Ġ×ķ ×Ĵ×Ŀ +×ŀ ׳×Ļ +íĸ ¥ +צ ×Ļ×Ŀ +à¸ĭ ิ +Ùĩ ÙĨ +н ем +Ġ×ij×ij ×Ļת +ر ع +Ġ ส +ĠÄIJ Ãł +íķĺ ëĭ¤ +Ġ ấy +×Ĺ ×ķ×ĵ +×Ĺ×ķ×ĵ ש +ĠÑĩеÑĢ ÐµÐ· +Ñĥ л +ĠB ình +Ġê²ĥ ìĿĦ +Ġ×Ĵ ר +ä»ĺ ãģij +×Ĺ׾ ×§ +Ġت ÙĦÙĥ +à¹ĥส à¹Ī +sz Äħ +ÙĤ اÙħ +د ÙĪØ± +ĠÙģ ÙĤØ· +Ġh ữu +Ġмог ÑĥÑĤ +Ġg á»įi +Ġ×§ ר +à¸Īะ มี +ت ÙĤدÙħ +Ġع بر +Ġ׾×Ķ ×Ŀ +ĠÑģам о +ס ×ĵר +Ġc Ãłng +r ÃŃ +Ġìŀ ¥ +ëĵ¤ ìĿĺ +ĠÙĦ Ùĥ +п оÑĢÑĤ +Ġkh ả +ĠÑģеб Ñı +׳ ף +Ġد ÙĪØ± +Ġm ợ +Ġcâ y +Ġf ark +Ġfark lı +а ÑİÑĤ +Ġtr á»±c +wiÄĻks z +Ġthu á»ijc +Ġت ØŃت +ت ÙĦ +ов Ñĭе +ëĤ ł +Ġв ам +بÙĦ غ +Ġê°Ļ ìĿĢ +íĮ IJ +ÙĦ ب +Ġnas ıl +Ġод ин +м ан +ĠعÙĦÙĬ Ùĩا +б и +Ġפ ש×ķ×ĺ +×ijר ×Ļ +Ġש ׳×Ķ +Ġëı Ħ +ĠÄIJ ại +Ġ×IJ×ķת ×Ŀ +ĠاÙĦØŃ ر +Ġб о +à¸Ī ุà¸Ķ +Ġr õ +ĠdeÄŁi ÅŁ +Ġëĭ ¨ +ĠÑģлÑĥÑĩ а +ĠÑģлÑĥÑĩа е +Ġ×IJ׳ ש×Ļ×Ŀ +×ĵ ×£ +ש×ij ת +Ġש׾ ׼×Ŀ +Ġch ú +nik ów +Ġtan ı +Ġcá o +ĠÄij á +Ġ×IJ ×ĵ×Ŀ +Ġê° ķ +Ġnhi á»ĩm +Ġ׾ ס +Ġ×Ľ×ª ×ij +Ġ×Ķס פר +ĠÄij Äĥng +Ġë ijIJ +à¸ľ ิ +à¸ľà¸´ ว +ج ا +Ġê° IJ +ر Ø£ +ست خدÙħ +ãģ«ãģªãĤĬ ãģ¾ãģĻ +Ġtá» · +×ĺ ×ķר +г овоÑĢ +Ġв оÑģ +ĠÙħÙĨ Ùĩا +иÑĢов аÑĤÑĮ +ĠÄij ầy +׳ ×Ĵ +ĠÙħ ÙĪ +ĠÙħ ÙĪÙĤع +ר׼ ×Ļ +ت Ùı +ëª ¨ +Ġת ×ķ +ÙĬا Ùĭ +à¹ĥ à¸Ķ +ãĤĬ ãģ¾ãģĻ +à¸Ńยูà¹Ī à¹ĥà¸Ļ +ĠØ£ ÙĪÙĦ +ĠØ£ خرÙī +Ġc ư +ص ار +×ŀ׊ש×ij +б ÑĢа +ÅĦ ski +б ÑĢ +ĠÙĬ Ùı +à¸ģ ิà¸Ļ +Ġch á»ijng +Ùħ Ùı +Ġ à¸Ħืà¸Ń +Ġت ÙĨ +t ÃŃ +y Äĩ +Ġm ạng +Ùģ ÙĪ +Ġdü nya +×§ ר×IJ +Ġ×§ ׾ +ĠØŃ اÙĦ +c ÃŃa +Ġà¹Ģ รา +Ġר ×ķצ×Ķ +Ġá p +ë° ķ +ا ÙĤØ© +ни Ñİ +Ġ×IJ ׾×ķ +Ġ×ŀס ×ķ +ãģ§ãģ¯ ãģªãģı +Ġtr ả +Ġ×§ שר +mi ÅŁtir +Ġl ưu +Ġh á»Ĺ +ĠбÑĭ ли +Ġl ấy +عÙĦ Ùħ +Ġö zel +æ°Ĺ ãģĮ +Ġ×ĵ ר×ļ +Ùħ د +s ını +׳ ×ķש×IJ +r ów +Ñĩ еÑĢ +êµIJ ìľ¡ +ĠÐľ о +л ег +ĠV Ỽi +วัà¸Ļ à¸Ļีà¹ī +ÑİÑī ие +ãģĬ ãģĻ +ãģĬãģĻ ãģĻ +ãģĬãģĻãģĻ ãĤģ +ëı ħ +Ġ×Ļ×Ķ ×Ļ×Ķ +×ŀ ×ĺר +Ñı ми +Ġl á»±a +ĠÄij ấu +à¹Ģส ียà¸ĩ +Ġt ương +ëĵ ± +ĠÑģÑĤ аÑĢ +à¹ĥ à¸ļ +ว ัà¸Ķ +Ġİ stanbul +Ġ à¸Īะ +à¸ķ ลาà¸Ķ +Ġب ÙĬ +à¹ģà¸Ļ ะ +à¹ģà¸Ļะ à¸Ļำ +س اعد +Ġب Ø£ +Ġki á»ĥm +ØŃ سب +à¸Ĭั à¹īà¸Ļ +Ġ×ķ ×¢×ķ×ĵ +ов ÑĭÑħ +оÑģ нов +Ġtr Æ°á»Łng +צ ×ij×¢ +ĠÃŃ t +Ġk ỹ +cr é +Ñı м +êµ ° +ãģĮ ãģªãģĦ +ÙĬÙĦ Ø© +ãĥķ ãĤ£ +ر Ùī +ĠÙĬ جب +Ġ×IJ ×£ +Ġc á»±c +ãĤīãĤĮ ãģŁ +Ġ à¸ľà¸¹à¹ī +Ġ à¸Ń +lar ımız +Ġkad ın +Ġê·¸ ëŀĺ +Ġê·¸ëŀĺ ìĦľ +ĠëĺIJ ëĬĶ +ĠÄij ả +ĠÄijả m +Ġ×IJ ×ķ×ŀר +Ġy ếu +ci Äħ +ciÄħ g +Ġt á»ij +Ġש×IJ ׳×Ļ +Ġdz iaÅĤa +Ñī а +ĠÄij Ãłn +s ına +ãģĵãĤĮ ãģ¯ +Ġ×ij ׾×Ļ +Ġ×ij ×Ļשר×IJ׾ +л оÑģÑĮ +Ġgi ữ +ê° IJ +ÑĢ Ð¾Ð½ +تج ار +г лав +в ин +Ġh ạn +Ġyapı lan +ب س +Ġ à¸ŀรà¹īà¸Ńม +ê´Ģ 리 +mÄ±ÅŁ tır +b ü +r ück +ĠBaÅŁkan ı +ĠÙĦ ÙĬس +Ġs Æ¡ +à¸Īัà¸ĩ หว +à¸Īัà¸ĩหว ัà¸Ķ +د اء +Ġ×Ķ ×Ľ +v ÃŃ +ש ×IJר +Ġh Æ°á»Łng +Ġb óng +ĠCh ÃŃnh +Äħ c +à¹Ģà¸ģีà¹Īยว à¸ģัà¸ļ +Ġtá» © +Ġtứ c +ĠÑĨ веÑĤ +Ġt á»iji +ĠnghÄ© a +ÙĦا عب +د ÙĦ +Ġפע ×Ŀ +h ör +à¸Ĭ ุà¸Ķ +à¸ŀ ู +à¸ŀู à¸Ķ +п аÑģ +ĠÅŁ u +Ġt Æ°á»Łng +خار ج +Ġâ m +ĠинÑĤеÑĢ ÐµÑģ +ен нÑĭÑħ +×IJ ׳×Ļ +بد Ø£ +ëĿ¼ ëĬĶ +ì¹ ´ +æĸ¹ ãģĮ +ли в +Ġ à¸Ħà¸Ļ +ער ×ļ +à¸Ĥà¸Ńà¸ĩ à¸Ħุà¸ĵ +п ад +Ġc ạnh +ĠëĤ ¨ +ĠÄij âu +Ġbi á»ĥu +ãĤĤ ãģĤãĤĭ +׾ ×Ĵ +Ġ สำหรัà¸ļ +Ġxu á»ijng +ס ×ķ +Ġذ ات +ĠÐľ е +ع اÙĦÙħ +×IJ ס +ب ÙĬØ© +Ø´ ا +и ем +ĠNg ưá»Ŀi +íĺ ij +Ñģл ов +Ġп а +Ġm ẫu +ĠпÑĢоÑĨ еÑģÑģ +ĠNh Ãł +пÑĢо из +пÑĢоиз вод +à¸łà¸²à¸¢ à¹ĥà¸Ļ +Ġ à¸ļาà¸Ĺ +×ŀ ׳×ķ +ĠоÑĢг ан +רצ ×ķ +×ķ×ŀ ×Ļ×Ŀ +Ġyaz ı +Ġd ù +ãĥ¬ ãĥ³ +ÙĪÙĦ ÙĬ +ย ู +Ġtr ò +à¹Ģà¸ŀ ลà¸ĩ +Ġ×ŀ ׾×IJ +à¸ķ ล +à¸ķล à¸Ńà¸Ķ +ĠÄij ạt +Ġ×Ĺ×ĵ ש +p óÅĤ +Ġ×ŀ ×ĵ×Ļ +ujÄħ c +×ŀ׳×Ķ ×ľ +Ġש×ij ×ķ +Ġ×Ķ×ŀש פ×ĺ +Ġ×IJ ׾×Ķ +ĠÙĪ Ø°ÙĦÙĥ +à¹Ģà¸ŀ ราะ +ĠÄijo Ãłn +Ġíķ¨ ê»ĺ +Ġd ục +Ø´ ت +Ġ ula +Ġula ÅŁ +Ġqu ý +Ġ×Ķ ×Ĵ×ĵ×ķ׾ +à¸ķัà¹īà¸ĩ à¹ģà¸ķà¹Ī +Ġש ר +Ø´ Ùĩد +׳ ש×Ļ×Ŀ +à¸ŀ ล +رÙĪ Ø§ +ãĤĮ ãģ¦ +Ġн иÑħ +Ġдел а +ãģ§ãģį ãģªãģĦ +ÅĤo ż +×IJ ×Ĺר +ì ½Ķ +ãĤ¢ ãĥĥãĥĹ +د Ù쨹 +Ġti á»ĩn +Ġkh á»ı +Ġkhá»ı e +ĠاÙĦع اÙħØ© +ãģ« ãģĤãĤĭ +ĠÄij á»Ļc +ì¡ ± +Ġc ụ +й ÑĤе +Ġзак он +ĠпÑĢо екÑĤ +ìĸ ¸ +ÙĦ ØŃ +ĠçalÄ±ÅŁ ma +ãĤĴ ãģĻãĤĭ +Ñħ и +ع اد +Ġ׳ ×ŀצ×IJ +Ġר ×Ļ +à¸Ńà¸Ńà¸ģ มา +ĠT ôi +Ġth ần +ĠÙĬ ا +ล าย +Ġав ÑĤо +Ġsı ra +ĠÙĥ Ø«ÙĬر +Ùħ ÙĬز +ĠاÙĦع ÙĦÙħ +æĸ¹ ãģ¯ +×ķ×¢ ×ĵ +Ġобла ÑģÑĤи +×Ļ׾ ×Ļ×Ŀ +ãģĮ åĩº +à¸ĺ ุ +à¸ĺุ ร +à¸ĺุร à¸ģิà¸Ī +ÙĤت ÙĦ +ר×IJ ×ķ +Ġng u +Ġngu á»ĵn +Ġ มา +Ġпл ан +t ório +Ġcu á»iji +Ñģк ом +ĠاÙĦÙħ اض +ĠاÙĦÙħاض ÙĬ +Ġ×ij×¢ ׾ +Ġר ×ij×Ļ×Ŀ +Ġlu áºŃn +Ùĥ ÙĪ +à¸Ĺัà¹īà¸ĩ หมà¸Ķ +в ан +Ġtho ại +à¹Ħ à¸Ń +б иÑĢ +ĠاÙĦ ض +ت ا +ĠÑĢ Ð¾Ð´ +ĠV Ãł +×ŀ ×Ļף +ĠбÑĭ ла +к ами +ĠÐĶ Ðµ +t ık +קר ×Ļ +ĠeÄŁ itim +ĠÙĥ بÙĬر +ب Ùĥ +ĠÙĦ ÙĪ +в ой +Ġ ãģĵãģ® +ĠÑĤ ÑĢÑĥд +my ÅĽl +Ġs ư +à¸ŀ ีà¹Ī +Ġ à¹ģลà¹īว +×¢ ×§ +Ġ×Ĺ×ijר ת +ระ หว +ระหว à¹Īาà¸ĩ +×Ļ ×Ļ×Ķ +ĠاÙĦÙĨ اس +ün ü +Ġ׾ ×ŀ×Ķ +Ġch ương +ĠH á»ĵ +ار ت +ãĤĪãģĨ ãģ§ãģĻ +l á +×§×Ļ ×Ļ×Ŀ +æľ¬ å½ĵ +æľ¬å½ĵ ãģ« +ãģĵãĤĵ ãģª +Ñģ ов +Ġ×ķ ×Ĺ +à¹Ģà¸ģ à¹ĩà¸ļ +Ġк ÑĤо +à¹Ĥร à¸Ħ +ĠØ´ رÙĥØ© +ع زÙĬ +عزÙĬ ز +Ø·ÙĦ ÙĤ +п ÑĥÑģÑĤ +Ùģ ØªØŃ +ëŀ Ģ +Ġhã y +ض Ùħ +ë¦ ° +åł´åIJĪ ãģ¯ +ãĤª ãĥ¼ +Ġh ắn +Ġ×IJ ×ij×Ļ×ij +Ġש׾×Ķ ×Ŀ +Ġ×Ķ×Ļ ×Ļת×Ķ +ĠاÙĦد ÙĪÙĦØ© +ĠاÙĦ ÙĪÙĤ +ĠاÙĦÙĪÙĤ ت +ãģĤ ãģ¾ãĤĬ +Ġta ÅŁÄ± +İ N +×¢ סק +ãģ¦ ãģĦãģŁ +Ġtá»ķ ng +ĠاÙĦØ¥ ÙĨس +ĠاÙĦØ¥ÙĨس اÙĨ +ÑĢ ÐµÑĪ +Ġg ái +ĠÑĨ ен +ĠÙģ ÙĤد +Ùħ ات +ãģķãĤĵ ãģ® +Ġph ù +×ĺ ×Ķ +ĠÙĪØ§ÙĦ تÙĬ +Ġب Ùĥ +ìĿ´ ëĤĺ +к Ñģ +Ùħ ÙĬر +Ġv ùng +ĠاÙĦØ´ عب +ĠNh ưng +ãĥĢ ãĥ¼ +Ġ×Ĺ×Ļ ×Ļ×Ŀ +ĠØ´ خص +×§ ×ķ×ĵ +ê² Ģ +×¢ ש +×¢ ×ķ׾×Ŀ +צ ×ķר +ع ÙĤد +ĠiÅŁ lem +Ġ×Ķ×ij ×IJ +Ġd ưỡng +à¸Ł รี +Ġph ÃŃa +ãģ®ä¸Ń ãģ§ +Ġп и +Ġng Ãłnh +ним а +ĠÙĩ ÙĦ +Ġ×ķ ×IJת +ĠÄij áng +é quipe +ĠÑįÑĤ оÑĤ +Ġgö rev +ë§ ¤ +Ġqu ân +å¼ķ ãģį +æĻĤ ãģ« +Ġب Ùħا +×ŀ ×Ļת +Ġü lke +Ġ×ŀ×§ ×ķ×Ŀ +×ij ף +æ°Ĺ æĮģãģ¡ +Ġë§İ ìĿĢ +Ġyük sek +ÑĨ енÑĤÑĢ +ĠÙħ جÙĦس +ç§ģ ãģ® +ÙĤد ر +Ġë¶Ģ ë¶Ħ +Ġì° ¨ +خر ج +ãģĭ ãģªãĤĬ +ë³´ ëĭ¤ +Ġ×ŀ ×Ļ×ĵ×¢ +peÅĤ ni +Ġx á»Ń +ìĹIJìĦľ ëĬĶ +ĠباÙĦ Ùħ +ĠÙĪ Ùħا +ĠÑįÑĤ ой +ب ÙĬÙĨ +n ü +ØŃ ز +ØŃز ب +ĠÑĢабоÑĤ а +ĠNh áºŃt +ÙĦ اء +Ġëĵ ¤ +Ġëĵ¤ ìĸ´ +ãĤĦãģĻ ãģĦ +×Ĺ×ĸ ×§ +Ġ×Ķ×Ĺ ×ijר×Ķ +п иÑĤ +ãģĭãĤī ãģ® +Ġë§IJ ìĶĢ +Ġפ ×ķ +ÙĦ Ùİ +à¹Ģà¸ķà¹ĩ ม +ĠÐļ о +Ġm ówi +Ġt ÃŃn +ר×Ĵ ש +פר ×§ +Ġtr ạng +ĠÐŀ н +×Ĺ ×ķ×¥ +ĠعÙĨد Ùħا +Ġب ر +使 ãģĦ +Ġr á»Ļng +ëĮĢ ë¡ľ +íĪ ¬ +Ġktóry ch +в ид +ลูà¸ģ à¸Ħà¹īา +Ġmog Äħ +Ġש ×Ĺ +×ij ×Ĺר +ãĥĸ ãĥŃãĤ° +ĠTh Ãłnh +Ġ×Ķ ×¨×Ļ +ĠÑģÑĤ аÑĤÑĮ +ĠH á»Ļi +à¸ļ à¹īาà¸ĩ +çī¹ ãģ« +ĠÄIJ ức +èĢħ ãģ® +×¢ ×ŀ×ķ×ĵ +×ĺר ×Ķ +Ð ¥ +ĠÙħ Ùħا +Ġe ÅŁ +ĠнеобÑħодим о +ник ов +Ġüzer inde +a ÅĤa +Ġchá»ĭ u +ĠاÙĦ دÙĬÙĨ +أخ بار +ĠÄij au +ãģĮ å¤ļãģĦ +jÄħ cych +د Ø®ÙĦ +ları nd +larınd an +Ġs ẻ +à¸ŀิ à¹Ģศ +à¸ŀิà¹Ģศ ษ +ת ף +t ıģı +Ġlu áºŃt +ĠÅŀ e +ãĤ« ãĥ¼ +ãģ® ãģĤãĤĭ +Ġ×Ķ×IJ תר +ĠاÙĦØ¢ ÙĨ +ıld ı +Ġá o +ĠнаÑĩ ал +Ġvi á»ĩn +Ġ×ij×¢ ×ķ׾×Ŀ +з наÑĩ +×Ļ×ĺ ×Ķ +к ам +ĠÐĺ з +à¹Ģà¸Ĥ ียà¸Ļ +à¸Ļ à¹īà¸Ńà¸ĩ +ÑĤ ÑĢо +à¹Ģ à¸Ł +Ġжиз ни +Ġ สà¹Īวà¸Ļ +Ġv áºŃn +Ġê´Ģ 볨 +Ġl âu +ס ×ĺר +×§ ש +س ÙĬر +Ġ×IJ×ķת ×Ļ +Ġm ôi +ائ ب +Ġо ÑģÑĤа +Ġm ón +Ġ×ij ×ŀ×§×ķ×Ŀ +Ġد اخÙĦ +Ġ×IJ ×ķר +Ġв аÑģ +Ùĥ Ø´Ùģ +ìĺ ¨ +à¸ĸ à¹Īาย +Ġkullan ıl +Ġt ô +ãģ« ãĤĪãĤĬ +ĠëĺIJ íķľ +Ġ×¢×ij×ķ×ĵ ×Ķ +Ġri ê +Ġriê ng +Ġyak ın +ز ا +Å » +×IJ ×ķ׼׾ +شار Ùĥ +Ġб еÑģ +× ´ +Ġا بÙĨ +ĠTá»ķ ng +ÙĨ ظ +ÅĽwi ad +ãĤµ ãĥ¼ +ห าย +ĠG ün +Ġhakk ında +à¹Ģà¸Ĥà¹īา มา +ز ÙĨ +ĠÐł о +Ġbi á»ĥn +ãģ© ãģĵ +Ùģ Ø¹ÙĦ +ز ع +פר ×ĺ +Ġ×Ķ ×Ł +Ø£ ÙĩÙĦ +Ġth ất +ØŃ ÙħÙĦ +Ñĩ Ñĥ +ĠìĤ¬ ìĭ¤ +ì° ¸ +ĠìľĦ íķ´ +ÙĪ Ø¸ +ĠÐŁ од +Ġkho ản +ÑĤ ен +ĠÙģ Ø§ÙĦ +Ñģ ад +à¸Ļ à¸Ńà¸Ļ +ĠاÙĦسعÙĪØ¯ ÙĬØ© +" ØĮ +ĠاÙĦ ÙĴ +ãĤī ãģļ +Ġto án +Ġch ắc +׼ ×Ļר +m éd +méd ia +ز ÙĪ +Ġyan ı +פ ׳×Ļ×Ŀ +ØŃ ظ +Ġб еÑģп +ĠбеÑģп лаÑĤ +ĠбеÑģплаÑĤ но +ĠØ£ ÙħاÙħ +à¸Ń าย +à¸Ńาย ุ +ר שת +Ġg á»ĵ +Ġgá»ĵ m +Ġu á»ijng +ص ب +k ır +ãĥij ãĥ¼ +Ġ׾×ĵ עת +Ġк ÑĥпиÑĤÑĮ +׾ ×ķ×Ĺ +ÙĪØ¶ ع +ÙĤÙĬ Ùħ +à¸Ľ า +ж ив +à¸Ķ ิà¸Ļ +×IJ ×ķפ +à¹Ģล à¹ĩà¸ģ +ãĥĥ ãĥī +иÑĩеÑģки Ñħ +ĠCh á»§ +кÑĢ Ð°Ñģ +ÙĪ ØµÙĦ +p ÅĤat +м оÑĢ +Ġ×Ķ×IJ ×ķ +à¸Ń ิà¸Ļ +Ġíķľ êµŃ +гÑĢ Ðµ +Ġìłľ ê³µ +ì° ½ +Ġê°ľìĿ¸ ìłķë³´ +Ġngh á»ĭ +à¸ĭ า +ØŃس اب +Ġby ÅĤa +ÙħÙĦ Ùĥ +иÑĩеÑģки е +Ġb ác +ض ØŃ +ê¸ ¸ +ש ×ŀ×¢ +Ġìĸ´ëĸ » +Ġìĸ´ëĸ» ê²Į +ìĽ Į +ات Ùĩ +à¹Ĥรà¸ĩ à¹ģ +à¹Ĥรà¸ĩà¹ģ รม +خد ÙħØ© +ĠÐł а +׼×ķ׾ ×Ŀ +×ŀש ×Ĺ×§ +ĠÙĪ ÙĥاÙĨ +ס ×ķ×£ +ĠاÙĦØŃÙĥÙĪÙħ Ø© +Ġ×ij ×ĺ +Ġtr áºŃn +Ġ×Ķ×¢ ×ķ׾×Ŀ +ĠÃŃ ch +t Äħ +ש×ŀ ×ķ +Ġ×Ķר×IJש ×ķף +Ġíķĺ ê³ł +ãģķ ãĤī +ãģķãĤī ãģ« +ãģ« ãģĹãģ¦ +Ġ à¸ľà¸¡ +ãģ® ãĤĪãģĨãģª +ĠÙĪ ÙĤت +ãĥį ãĥĥãĥĪ +ÙĦ عب +ÙĪ Ø´ +ìĺ ¬ +Ġ หาà¸ģ +Ġm iaÅĤ +à¸Ĺ à¸Ńà¸ĩ +иÑĤ а +ا صر +ил ÑģÑı +з е +à¸Ľà¸£à¸° มาà¸ĵ +ãģĿãĤĮ ãģ¯ +Ġb ır +Ġbır ak +صÙĨ اع +Ð ® +Ø´ عر +Ġ׳ ×Ĵ×ĵ +Ġب سبب +ãĥĿ ãĤ¤ +ãĥĿãĤ¤ ãĥ³ãĥĪ +ĠاÙĦج ÙĪ +ĠнеÑģк олÑĮко +Ġki ếm +Ùģ Ùİ +Ġض د +×ij×Ļ×ĺ ×ķ×Ĺ +تاب ع +ÙĨ ز +ĠB ản +Ġaç ıkl +Ġaçıkl ama +Ġ à¸Ħุà¸ĵ +à¸Ĺ า +ÅĤ ów +Ø· ب +ÙĨ ØŃÙĨ +Ġ×ŀ×§ ×ķר +Ġİ s +Ġдом а +Ġ วัà¸Ļ +Ġd Ãłnh +Ñı н +ми ÑĢ +Ġm ô +ĠvÃł ng +ص اب +s ının +à¸Ħ ืà¸Ļ +Ø® بر +×ĸ׼ ×ķ +Ġ×ŀ ש×Ķ×ķ +m ü +Ġкомпани и +Ġ×Ķ×¢ ×Ļר +ĠÙĥ ÙĪ +ÙĤÙĦ ب +ĠlỼ p +и ки +׳ ×ij +à¹Ĥ à¸Ħร +à¹Ĥà¸Ħร à¸ĩ +à¹Ĥà¸Ħรà¸ĩ à¸ģาร +×ŀ×ķ×¢ ×ĵ +ÑıÑĤ ÑģÑı +หลัà¸ĩ à¸Īาà¸ģ +ени Ñİ +Ġש ×¢ +Ġb Æ°á»Ľc +ãĥ¡ ãĥ¼ãĥ« +ãĤĦ ãĤĬ +Ġ×Ļ×ķ×ĵ ×¢ +Ġê´Ģ íķľ +ĠاÙĦØ£ Ùħر +Ġböl ge +ĠÑģв ой +ÙĦ س +Ġ×ŀ×Ļ ×ķ×Ĺ×ĵ +ĠëĤ´ ìļ© +ĠØ£ جÙĦ +ĠÄIJ ông +Ġ×ŀ ×ł×ª +Ġìĭľ ê°Ħ +Ùĥ Ùİ +ãģ¨ãģĦãģĨ ãģ®ãģ¯ +Ġnale ży +تÙĨظ ÙĬÙħ +ĠÑģозд а +Ġph é +Ġphé p +ãģ§ãģį ãģ¾ãģĻ +Ġع ÙĦÙħ +大ãģį ãģª +ãĤ² ãĥ¼ãĥł +í ħĮ +Ġ׼×ķ׾ ׾ +ĠинÑĤеÑĢ Ð½ÐµÑĤ +ĠT ừ +ãģ¨ ãģªãĤĭ +ز اÙĦ +Ġktóry m +Ġnh é +ìĪ ľ +н ев +д еÑĢ +ãĤ¢ ãĥĹãĥª +i á»ĩu +×ij ×Ļ׾ +Ġت س +ĠÄIJ ây +ĠاÙĦØ® اصة +Ġà¹Ģ à¸Ĭ +Ġà¹Ģà¸Ĭ à¹Īà¸Ļ +ص اد +Ġd ạng +س عر +Ġש ×Ļ×ŀ×ķש +×Ĵ ×Ļ×Ŀ +ãģĮãģĤ ãģ£ãģŁ +п ÑĢов +пÑĢов од +Ġ×IJ ×Ļ׳×ķ +Ġ׾ ר×IJ +Ġ׾ר×IJ ×ķת +ĠØ£ Ù쨶ÙĦ +ĠØŃ ÙĦ +ĠØ£ بÙĪ +ê° ķ +Ġì§ ij +ãģ® ãĤĪãģĨãģ« +Ġפ ׳×Ļ +ס ×Ļ×Ŀ +ĠÙĪÙĩ ذا +Ġka ç +Ġé én +Ġê± ´ +ë° Ķ +Ñĥ з +à¸Ĥà¸Ńà¸ĩ à¹Ģรา +i ÅĤ +ĠÐľ Ñĭ +Ġch ết +ĠاÙĦØ« اÙĨÙĬ +×IJ ×§ +Ġ×ķ ×¢×ľ +ĠاÙĦØ· ب +×ij×ĺ ×Ĺ +Ġج دÙĬدة +Ġع دÙħ +ع ز +สิà¹Īà¸ĩ à¸Ĺีà¹Ī +ãģĻ ãĤĮãģ° +ĠÄij ô +ì£ ł +د ÙĤ +н омÑĥ +Ġk á»ĥ +ãĤ¢ ãĥ³ +å¤ļãģı ãģ® +à¸Ľà¸£à¸° à¸ģ +à¸Ľà¸£à¸°à¸ģ à¸Ńà¸ļ +פע×Ļ׾ ×ķת +ĠÑģÑĤ ол +may ı +ãģ¤ ãģĦ +Ġyılı nda +Ġ à¸Īึà¸ĩ +koÅĦ cz +ĠTh ông +Ġак ÑĤив +н ÑģÑĤ +нÑģÑĤ ÑĢÑĥ +ĠÃĸ z +Ġת ×ŀ×Ļ×ĵ +ĠÙĥ ÙĨت +Ñģ иÑģÑĤем +pr és +prés ent +Ġn â +Ġnâ ng +gÅĤ os +ĠÙĪØ² ÙĬر +ØŃ صÙĦ +Ġиме еÑĤ +ØŃ رÙĥØ© +à¸ŀ à¹Īà¸Ń +ãĤĴ ãģĬ +Ġاست خداÙħ +×IJ×Ļר ×ķ×¢ +ä»ĸ ãģ® +Ġש×Ķ ×Ŀ +ãģĹãģŁ ãĤī +ש×ŀ ×Ļ +Ñģ ла +m ı +Ġbaz ı +Ġíķĺ ì§Ģë§Į +×ĵ ׾ +Ġyapt ıģı +ãĥĬ ãĥ¼ +׾ ×Ļ׾×Ķ +ãģ¨ãģĦ ãģ£ãģŁ +änd ig +ĠÅŁ a +ĠÙģÙĬ Ùħا +иÑĤ елÑı +×ŀ ×ķש +à¸Ĥ à¸Ńà¸ļ +l ük +Ġh á»ĵi +Ġëª ħ +ĠاÙĦÙĥ Ø«ÙĬر +צ ×IJ +Ġhaz ır +طر Ùģ +ا ÙĬا +ĠÄij ôi +ен д +ÙĦ غ +×Ĺ ×ĸ×ķר +ĠвÑģ ег +ĠвÑģег да +ëIJĺ ê³ł +×ĵ ×ķ×ĵ +ан а +د ÙĪÙĦØ© +Ġho ạch +ع ÙĦا +عÙĦا ج +Ġ×ķ ×¢×ĵ +×Ķ ×Ŀ +ки й +ÙĦ ÙIJ +Ġ×¢ ׾×Ļ×ķ +ÑİÑī ий +Ġng á»§ +صÙĨ ع +ĠاÙĦع راÙĤ +à¸ķà¹Īà¸Ń à¹Ħà¸Ľ +ãģŁãģı ãģķãĤĵ +Ġph ạm +ÙĦ اÙĨ +ات Ùĩا +Ġbö yle +تÙĨ ÙģÙĬ +تÙĨÙģÙĬ ذ +Ġש×Ķ ×Ļ×IJ +Ñģ Ñĥ +ย าว +Ġש ×ķ׳×Ļ×Ŀ +Ġ×ŀ ×ķ׾ +ĠÑģ ил +Ġ×IJ×Ĺר ×Ļ×Ŀ +Ġph á»§ +ÙĤØ· ع +ĠTh á»§ +à¸Ľà¸£à¸°à¹Ģà¸Ĺศ à¹Ħà¸Ĺย +ÙĨ ÙĤ +ĠÄijo ạn +Ġب Ø¥ +п ÑĢедел +×ķת ×ķ +Ġy arı +пÑĢ Ðµ +ĠczÄĻ ÅĽci +ØŃ ÙĥÙħ +×ķ׳ ×Ļת +פע ׾ +ãĤĴ ãģĹãģ¦ +Ġktó rzy +׾ ×Ŀ +ĠÄIJi á»ģu +ĠкоÑĤоÑĢ Ð°Ñı +ĠìĿ´ ìĥģ +ãģĤ ãģ£ãģŁ +Ġ×ŀ×ĵ ×ķ×ijר +פ ×ķ×¢×ľ +d ım +éĢļ ãĤĬ +ĠбÑĥд ÑĥÑĤ +à¹Ģวà¹ĩà¸ļ à¹Ħà¸ĭ +à¹Ģวà¹ĩà¸ļà¹Ħà¸ĭ à¸ķà¹Į +ا خر +×Ĺ ×Ļ׾ +Ġ×Ļ ×ľ +Ġ×Ļ׾ ×ĵ×Ļ×Ŀ +×Ĺ ×Ļפ +×Ĺ×Ļפ ×ķש +Ġd òng +Ġש ×ĸ×Ķ +ÑĮ е +ãģĤ ãģ¨ +ìŀIJ ê°Ģ +×IJ ×ĵ +Ġü z +Ġüz ere +ظ ÙĦ +Ġ×IJ ×ķ׾×Ļ +Ġ×ij ×Ļ×ķ×Ŀ +ÙĦ ات +Ġm ê +ì¹ ¨ +تØŃ د +تØŃد Ø« +ĠØ® اصة +Ġب رÙĨ +ĠبرÙĨ اÙħج +ĠH Ãłn +×Ĺ ×¡ +ĠÙĪ ÙĦÙħ +×¢ ×Ŀ +Ġm ı +à¸Ł ัà¸ĩ +ש ×¢×Ķ +ÙĪÙģ ÙĤ +ס ×ij×Ļר +алÑĮ нÑĭй +×Ĺש ×ķ×ij +Ġn Ãłng +ë³ ¼ +ĠкоÑĤоÑĢ ÑĭÑħ +Ġ×Ĺ ×ķ×§ +t ör +ĠлÑĥÑĩ ÑĪе +ãĥij ãĥ³ +ลà¹Īา สุà¸Ķ +Ġج دÙĬد +ÙĬد Ø© +à¸Ĺ รà¸ĩ +ãĤĪãĤĬ ãĤĤ +ÙĦ ÙĦ +ãĤĤ ãģ£ãģ¨ +ש×ĺ ×Ĺ +Ġ×ķ ×IJ×Ļ +Ġgi á»ijng +Ø¥ ضاÙģ +×§ ת +ë§ Ŀ +Ġzosta ÅĤ +ÑĢ Ð¾Ð· +×Ļפ ×Ļ×Ŀ +Ġ׼׾ ׾ +ת×ķ׼ ף +dıģ ını +ÙĤ سÙħ +ĠÑģ ÑĩиÑĤ +ĠÑģÑĩиÑĤ а +×ĺ ×ķת +Ġ ưu +ĠØ¢ ÙĦ +Ġм ом +Ġмом енÑĤ +ĠاÙĦتع ÙĦÙĬÙħ +×¢×ľ ×ķת +Ġch ữa +Ġy ön +Ġtr Ãł +ĠØŃ ÙĬÙĨ +à¸ĭ ั +ĠC á +×¢ ×ĸ +ĠاÙĦØ£ ÙħÙĨ +c ÃŃ +Ġv á»ijn +Ġ à¸Ļาย +об ÑĢа +×§ ×IJ +Ġthi ếu +ãĥŀ ãĥ¼ +ส วà¸Ļ +Ġg á»Ń +Ġgá»Ń i +Ġê ¹ +Ġê¹ Ģ +Ġthi á»ĩn +ÙĤ ع +w ÄĻ +Ġн ам +ÑĤ ол +Ġs ân +ס ×ķ×Ĵ +Ġgeç ir +ÑĤ он +ев а +ĠÙĪ Ø¶Ø¹ +Ġع شر +Ñģ ло +à¸Ī ัà¸ļ +ãĤ· ãĥ¼ +ãĤĤ ãģĤãĤĬãģ¾ãģĻ +Ġv ẻ +ĠÄIJ á»ĥ +ر Ù쨹 +ĠاÙĦØ£ÙĪÙĦ Ùī +ÑĤ аÑĢ +ãģªãģı ãģ¦ +Ùħ Ùİ +qu ÃŃ +×¢×ł×Ļ ×Ļ׳ +г ен +Ġh ôm +à¸Ī า +Ġnh Ỽ +ĠاÙĦع ربÙĬ +×IJ ף +Ġl á»Ļ +Ġje ÅĽli +à¹Ģà¸Ĺà¹Īา à¸Ļัà¹īà¸Ļ +ĠØ£ÙĨ Ùĩا +Ġt uy +Ġtuy á»ĩt +Ġت ص +Ġتص ÙĨÙĬ +ĠتصÙĨÙĬ Ùģ +Ġê·¸ëŁ¬ ëĤĺ +о ÑĨен +à¸ģิà¸Ī à¸ģรรม +ãĤĦ ãģ£ãģ¦ +Ġkh á»ıi +Ġl á»ĩ +ĠاÙĦÙħج تÙħع +à¸Ńาà¸Ī à¸Īะ +à¸Īะ à¹Ģà¸Ľà¹ĩà¸Ļ +ов Ñĭй +ר ×Ŀ +ร à¹īà¸Ńà¸Ļ +ש ×ŀש +人 ãģ« +Ġüzer ine +פר ×Ļ +du ÄŁu +Ñĩ ик +Ġmù a +Ġ×ŀת ×ķ×ļ +Ġc áºŃp +Ġت ارÙĬØ® +×ij׾ ת×Ļ +Ġì¢ Ģ +ÙĦ ع +ب اÙĨ +Ġch út +Ġ×Ķ×ĸ ×ŀף +n ée +ĠLi ên +ĠÙĦÙĦ Ø£ +ØŃد ÙĪØ¯ +Ġ×¢ ׼ש×Ļ×ķ +в оз +Ġyapt ı +Ġоб о +à¹ĥหà¹ī à¸ģัà¸ļ +Ġ×ij×Ķ ×Ŀ +ãģı ãģ¦ +ر أس +ĠÑģÑĢед ÑģÑĤв +ĠB Ãłi +ãģĵãģ¨ ãģ« +ĠìĤ¬ íļĮ +Ġ모 ëijIJ +×ij ×IJ +Ġtr ắng +ĠاÙĦبÙĦ د +ĠHo Ãłng +ли бо +ĠдÑĢÑĥг иÑħ +İ R +Ñĥм а +ĠJe ÅĽli +ãĤĤ ãģĹ +Ġv òng +Ġ×IJתר ×Ļ×Ŀ +ĠÄij á»įc +Ġв оÑĤ +ãģł ãģĮ +ë° ° +à¸Ķู à¹ģล +Ġ×ŀ ׼׾ +ìĹIJ ëıĦ +г аз +Ġ׳×ķס פ×Ļ×Ŀ +ãģĵãģ¨ ãģ§ +Ġت ÙĪ +ãģ§ ãģĤãĤĬ +à¸Ļั à¹Īà¸ĩ +ĠможеÑĤ е +sz ÄĻ +ãģ® ãģł +ĠÙħÙĨ Ùĩ +Ġb á»ķ +Ġb üt +Ġbüt ün +ë³´ ê³ł +Ġch á»ĵng +à¹ģà¸Ī à¹īà¸ĩ +ĠV ì +ĠØŃ ر +Ġgi ản +ĠÙħ دÙĬÙĨØ© +تط بÙĬÙĤ +à¸Ī ิ +æĹ¥ ãģ® +б ил +à¸ģ à¸Ńà¸ĩ +ê³ ³ +ĠØ£ Ùħا +ìĨ IJ +Ġtr ái +ĠвÑģ ем +Ġس ÙĨØ© +ĠÑģай ÑĤ +Ġг оÑĤов +п Ñĭ +ĠëIJ ł +ĠاÙĦØ® Ø· +ĠاÙĦرئÙĬس ÙĬØ© +Ġíķ ©ëĭĪëĭ¤ +ĠìķĦëĭĪ ëĿ¼ +ĠìĿ´ ëłĩ +ĠìĿ´ëłĩ ê²Į +) ØĮ +h ält +ĠØ£ Ùħر +Ġع Ùħر +à¸ģà¹ĩ à¸Īะ +Ġ à¸Ĺำà¹ĥหà¹ī +Ġc ân +Ġ×ij ׾ +Ġ×ij׾ ×ij×ĵ +פ סק +ĠÙĬ ÙĤÙĪÙĦ +н ÑĥÑĤÑĮ +à¹ģ à¸Ħ +Ġ×§ צת +Ġn ằm +Ġh òa +bilit Ãł +ĠìĹĨ ëĭ¤ +Ġ׼ פ×Ļ +ÑĢ Ð¾Ð¶ +лаг а +Ġ×Ķש ×Ļ +ĠNgo Ãłi +ĠÙĪ Ø¬ +ĠÙĪØ¬ ÙĪØ¯ +ĠìľĦ íķľ +Ġus ÅĤug +Ġtu ần +d ź +×ŀ ×ķף +ĠاÙĦع دÙĬد +Ġch ẳng +สุà¸Ĥ à¸łà¸²à¸ŀ +Ġ×ij ×ĵר×ļ +ĠÑģеб е +ĠìŀĪ ìĿĦ +ĠاÙĦØŃ اÙĦ +Ġd á +Ġc ưá»Ŀi +Ġnghi ên +ie ÅĦ +ĠD ương +ï¼ ħ +Ø´ د +ãģĦãģ¤ ãĤĤ +ĠвÑĭб оÑĢ +Ġc á»Ļng +ש ×Ļ׳×ķ×Ļ +Ġch ạy +Ġ×ij×¢ ׾×Ļ +اخ بار +íķĺ ë©° +ż Äħ +ج از +Ġ׳ ר×IJ×Ķ +ศ ู +ศู à¸Ļ +ศูà¸Ļ ยà¹Į +×Ĵ ×¢ +Ġ×¢ ×ĵ×Ļ +Ġ×¢×ĵ×Ļ ×Ļף +بر ا +ÑĨи й +ĠÄIJ á»ĵng +ÙĤ اÙĨÙĪÙĨ +ĠÄij ứng +ãģĹãģŁ ãĤĬ +Ġ×Ĺ×Ļ ×Ļ +Ġë IJľ +ĠëIJľ ëĭ¤ +Ġм еждÑĥ +à¸ŀวà¸ģ à¹Ģà¸Ĥา +ĠB ắc +ล ำ +ë° ± +ĠíĻ ķ +มาà¸ģ ม +มาà¸ģม าย +бан к +à¸Ńา à¸ģาร +Ġh Ãł +Ġ׾ ׳ +à¸Ń à¸Ń +Ġë°Ķ ë¡ľ +л ом +m ática +ĠØŃ د +اب ت +à¸Ĺีà¹Ī à¸Ļีà¹Ī +Ġco ÅĽ +ÙģÙĬ دÙĬ +ÙģÙĬدÙĬ ÙĪ +ĠмеÑģÑĤ о +Ġph út +มาà¸ģ à¸ģวà¹Īา +×IJ פ +ب ÙIJ +ĠPh ú +ì± Ħ +ĠÙĪ Ø³ÙĦÙħ +à¸Īี à¸Ļ +поÑĤ ÑĢеб +Ġ×Ĺ×ĵ ש×ķת +Ø´ ÙĪ +Ġעצ ×ŀ×ķ +ĠعÙħÙĦ ÙĬØ© +à¸Ħุà¸ĵ à¸łà¸²à¸ŀ +ãģ¾ãģĻ ãģĮ +دع ÙĪ +طر ÙĤ +à¹Ħมà¹Ī à¸ķà¹īà¸Ńà¸ĩ +ë² Ķ +ìĬ ¹ +Ġk ÃŃch +ĠìĹĨ ëĬĶ +ĠÑĤ ам +ĠÙĨ ØŃÙĪ +ĠاÙĦÙĤ اÙĨÙĪÙĨ +×Ĺ ×ķ×Ŀ +Ġk ız +Ġ×ĵ ×Ļף +ĠвÑĢем ени +ãģ£ãģŁ ãĤĬ +ĠØ´ Ùĩر +ĠìĦľ ë¹ĦìĬ¤ +×¢ ש×Ķ +Ġgi ác +ĠاÙĦسÙĦ اÙħ +Ġ×IJ ש +ĠполÑĥÑĩ а +à¸Īัà¸Ķ à¸ģาร +к оÑĢ +Ġ×Ķ×ĺ ×ķ×ij +ราย à¸ģาร +주 ìĿĺ +à¹ģà¸ķà¹Ī ละ +Ġê·¸ëŁ° ëį° +à¸Ĺีà¹Ī à¹Ģà¸Ľà¹ĩà¸Ļ +Ġת ×ķ×ļ +بÙĬ اÙĨ +Ð Ļ +oÅĽci Äħ +ÑĤ ок +ĠÃ Ķ +ĠÃĶ ng +à¹Ħมà¹Ī à¹ĥà¸Ĭà¹Ī +ãģ¿ ãģ¦ +ÐŁ о +ĠЧ ÑĤо +íĻ © +×ĺ ×ij×¢ +меÑĤ ÑĢ +Ġ×ij ×ŀ×Ķ +Ġ×ij×ŀ×Ķ ×ľ +Ġ×ij×ŀ×Ķ׾ ×ļ +Ñĩ ÑĮ +×§ ש×Ķ +з нак +знак ом +uj ÄĻ +×Ļצ ר +ĠاÙĦÙħ ÙĦÙĥ +ı yla +×IJ×ŀ ת +à¸Ľ ิà¸Ķ +×IJ ×Ĺ×ĵ +ر اد +Ġm áºŃt +ëĭ¤ ëĬĶ +Ġl ạnh +ש׾ ×ķש +ØŃ دÙĬØ« +ت ز +å¹´ ãģ® +Ġк ваÑĢ +ĠкваÑĢ ÑĤиÑĢ +ä½ľ ãĤĬ +رÙĪ Ø¨ +ов ан +ĠТ е +à¸Īำ à¸ģ +à¸Īำà¸ģ ัà¸Ķ +ب اط +×Ĵ ת +Ġм аÑĪ +ĠмаÑĪ Ð¸Ð½ +×Ļצ ×Ķ +ãģ» ãģ¨ +ãģ»ãģ¨ ãĤĵãģ© +ÃŃ do +ĠÑı зÑĭк +à¸ļ ิà¸Ļ +สà¸ĸาà¸Ļ à¸Ĺีà¹Ī +ĠìĹ ´ +ãĤ¦ ãĤ§ +Ġc Ãł +п ан +åı£ ãĤ³ãĥŁ +Ġر د +اÙĤ ت +ĠÙĥ ب +ĠÙĥب ÙĬرة +ÑģÑĤ ал +ש×ŀ ×Ĺ +pos ición +ĠÙħÙĦÙĬ ÙĪÙĨ +ĠìĿ´ ìķ¼ +ĠìĿ´ìķ¼ ê¸° +Ġh út +ĠÅĽw iat +Ġë°© ë²ķ +ĠÑģв еÑĤ +Ġвиде о +ĠاÙĦÙĨ ظاÙħ +Ġtr á»Ŀi +ĠëĮĢ íķ´ìĦľ +ר ×ŀת +ت داÙĪÙĦ +×ķר ×ĵ +ת ×ŀ +ת×ŀ ×ķ׳×ķת +Ġ×ŀ ף +Ġдв а +Ġ×Ķ×§ ×ķ +æĹ¥ ãģ« +Ġ×Ķ×Ĵ ×Ļ×¢ +à¹Ģà¸ŀิà¹Īม à¹Ģà¸ķิม +Ùħار س +Ġê²ĥ ìŀħëĭĪëĭ¤ +ãģªãģĦ ãģ¨ +Ġnhi á»ĩt +ëIJ ©ëĭĪëĭ¤ +Ġ×ij׳ ×ķש×IJ +Ġê°Ģ ìŀ¥ +Ġv ợ +ĠÄij óng +צ×Ļ׾ ×ķ×Ŀ +ê´Ģ ê³Ħ +в аÑı +×IJ ×Ļ×ĸ +×IJ×Ļ×ĸ ×Ķ +ĠÙĨ ظاÙħ +ÙħØŃ اÙ쨏 +Ġt ải +기 ëıĦ +à¸Ľà¸±à¸Ī à¸Īุ +à¸Ľà¸±à¸Īà¸Īุ à¸ļัà¸Ļ +׼ ×ĵ×ķר +ĠìķĦ ìĿ´ +׼׳ ×Ļס +à¹Ģ à¸ķร +à¹Ģà¸ķร ียม +Ġngo ại +ĠدÙĪÙĦ ار +Ġr ẻ +Ġkh Äĥn +عد د +Ø´ عب +czy Äĩ +ĠاÙĦ Ùĥر +ĠÑĩеловек а +ĠÙĪ Ø¥ÙĨ +×IJ ×ĺ +Ġth Æ¡ +ĠاÙĦ رÙĬاض +оп ÑĢедел +опÑĢедел ен +×Ķ ×ŀש×ļ +ĠÐĿ ово +з Ñĭва +ĠاÙĦدÙĪÙĦ ÙĬ +ĠÄij áp +Ġк ÑĢед +ĠкÑĢед иÑĤ +ов ого +Ġm ôn +à¸Ľà¸£à¸° à¹Ĥย +à¸Ľà¸£à¸°à¹Ĥย à¸Ĭà¸Ļ +à¸Ľà¸£à¸°à¹Ĥยà¸Ĭà¸Ļ à¹Į +ÑģÑĤ е +ĠTh á»ĭ +د ÙĬØ© +×ŀצ ×ķ +Ùģ Ø§Øª +×§ ×ĵ×Ŀ +ìĿ´ëĿ¼ ê³ł +ÙĪ Ø® +Ġ×Ĺ ×ĸ +ĠÑĦоÑĤ о +׾ ×Ļת +ت Ùİ +ÙĪ Ø¨Ø± +й ÑĤи +ĠÃ¶ÄŁ ren +Ġ×Ķ×ĸ ×ķ +Ġv á»įng +ÙĤÙĪ Ø© +ĠT ây +ĠÐĿ и +Ġש ×ķ×ij +ãģ¨è¨Ģ ãĤıãĤĮ +ãģ© ãĤĵãģª +׊צ×Ļ +ï½ ľ +Ġ×ķ×Ķ ×ķ×IJ +ä¸Ģ ãģ¤ +ĠÑģÑĤо иÑĤ +ni Äħ +×ĺר ×Ļ +ĠдеÑĤ ей +нÑı ÑĤÑĮ +ĠÑģдел аÑĤÑĮ +Ġë§İ ìĿ´ +ä½ķ ãģĭ +ãģĽ ãĤĭ +à¹Ħ หม +à¸ķิà¸Ķ à¸ķà¹Īà¸Ń +Ġ×ij ת×Ĺ +Ġ×ijת×Ĺ ×ķ×Ŀ +ìĻ Ħ +ì§Ģ ëĬĶ +ÑģÑĤ аÑĤ +ÑıÑģ н +ü b +Ġth ả +Ġ×ij×IJ×ŀ ת +Ġt uyến +×ĵ ×Ļר×Ķ +Ġ×IJ ×Ļש×Ļ +×ĸ׼ ר +ãģ° ãģĭãĤĬ +Ġx ét +׼ ×Ļ×ķ +׼×Ļ×ķ ×ķף +diÄŁ ini +ĠاÙĦÙħ ÙĪØ¶ÙĪØ¹ +Ġh áºŃu +à¸Īาà¸ģ à¸ģาร +×ijס ×Ļס +Ġ×ŀ×Ĵ ×Ļ×¢ +×ij ×Ļ×¢ +ĠÙĪ Ø¬Ùĩ +à¹ģà¸Ķ à¸ĩ +à¸Ļ าà¸ĩ +ĠÅŀ a +ì ¡´ +ë¡ Ģ +à¸ķ ะ +Ġ×Ķ×Ĺ×Ļ ×Ļ×Ŀ +Ùģ ÙĬد +ãģ§ãģĻ ãģĭãĤī +ê· ľ +ź ni +ĠлÑİ Ð´ÐµÐ¹ +Ġyüz de +ıy orum +ĠاÙĦ بØŃر +e ño +п аÑĢ +ÙĬ ÙĤØ© +об ÑĢ +ר ×ķ×ļ +ت ÙĪÙĤع +ĠاÙĦØ´ ÙĬØ® +åĪĿ ãĤģãģ¦ +ĠÑĤ елеÑĦ +ĠÑĤелеÑĦ он +Ġth ôi +Ġ×Ļ׼×ķ׾ ×Ļ×Ŀ +ĠÅŁ irk +ĠÅŁirk et +Ġìļ°ë¦¬ ê°Ģ +ĠÄij ông +Ġת ×ķ×ĵ×Ķ +ÑģмоÑĤÑĢ ÐµÑĤÑĮ +ĠÙĦ ÙĩÙħ +Ġ׾ ׼ +ĠN ó +ĠØŃ اÙĦØ© +ãģĦ ãģij +קר ×ķ +az ı +ãĤ³ ãĥ¼ +ĠÙĦÙĦ ت +s ınız +ĠH ải +기 ìĪł +ยัà¸ĩ à¹Ħมà¹Ī +ëĭ¤ ê³ł +פ ×Ĺ +Ġ׾×Ĵ ×ij×Ļ +Ġع ÙĨÙĩ +Ġк аз +Ġказ ино +ب ÙĪØ± +ÑĦ еÑĢ +Ġê°Ļ ìĿ´ +تس جÙĬÙĦ +ĠاÙĦÙħ رÙĥز +ĠTh ái +д аÑĤÑĮ +×ŀ×Ļ ×Ļ׾ +Ġpay laÅŁ +ãģ¤ ãģ® +à¹Ģร ืà¸Ń +n ça +׳ ×ķ×Ĺ +Ġ×IJ פ×Ļ׾×ķ +ãģ¨ èĢĥãģĪ +ãģ¨ãģĹãģ¦ ãģ¯ +à¹Ģà¸Ī à¸Ń +×ŀ פ +Ġg iriÅŁ +л иÑĤ +ÑĤ елÑı +Ñij н +æ°Ĺ ãģ« +Ġg ó +Ġgó p +åĪĩ ãĤĬ +Ġ×Ķ ×Ĺ×ĵש +ж ал +Ġ×ĵ עת +éģķ ãģĨ +à¹Ģà¸Ĥà¹īา à¹Ħà¸Ľ +Ġס ר×ĺ +e ña +æĸ° ãģĹãģĦ +ر Ùİ +ĠÐIJ ÑĢ +Ġph ản +à¸Īะ à¹Ħà¸Ķà¹ī +Ġ×ijצ ×ķר×Ķ +Ø´ اÙĩ +شاÙĩ د +ÙĪØ± د +à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ à¸Īาà¸ģ +или ÑģÑĮ +à¹ģละ à¸ģาร +Ġ×Ķ ×ĸ׼ +Ġ×Ķ×ĸ׼ ×ķ×Ļ×ķת +ei ÃŁ +ãĥ ¨ +ìĥ Ī +ĠÃĩ a +Æ ¯ +ש ×Ĵ +ÙĬÙĨ Ø© +ร à¹īà¸Ńà¸ĩ +ãĤµ ãĥ³ +ÑĢоÑģÑģ ий +ÑĢоÑģÑģий Ñģк +a ÄŁa +ĠнаÑĩ ина +Ġص ÙĦÙī +à¸Ĺุà¸ģ à¸Ħà¸Ļ +íļĮ ìĤ¬ +Ġли ÑĨ +Ø´ ÙĬر +ĠØ´ÙĬ Ø¡ +ÙĬÙĨ ا +Ġפ ×Ĺ×ķת +Ġiçer is +Ġiçeris inde +ĠØ£ ØŃÙħد +Ġże by +ì´ Ŀ +Ġп оказ +Ġи менно +หà¸Ļัà¸ĩ ส +หà¸Ļัà¸ĩส ืà¸Ń +ĠÑĤÑĢ Ðµ +สัà¸ĩ à¸Ħม +Ø¥ ÙIJ +ãģĮ å¿ħè¦ģ +ÙĬÙij Ø© +פ צ +íĭ ° +ĠÙħ جاÙĦ +׳ פש +к ан +×Ĺ ×ķפ +×Ĺ×ķפ ש +ì²ĺ ëŁ¼ +ов аÑı +з ов +Ġh ạ +Ġdzi ÄĻki +×Ļר ×ķ +Ġ׾ ×ŀצ +Ġ׾×ŀצ ×ķ×IJ +×Ļ×ĵ ×ķ +Ġs ợ +Ġ׾×Ķ ×Ĵ×Ļ×¢ +×§ ×ij×¢ +Ġchi á»ģu +ãĥŀ ãĤ¤ +Ġd Ãłng +à¹ģà¸Ł à¸Ļ +Ġü ye +×Ļ׳ ×Ĵ +à¹Ģรีย à¸ģ +ç§ģ ãģĮ +th é +ĠÑĦ илÑĮ +ĠÑĦилÑĮ м +ĠNg Ãły +Ġж ен +Ġжен Ñīин +ج ÙĬد +n ç +à¸Ľ รา +×Ļ×ŀ ×ķ +Ġn á»ģn +×IJ ×ķ׾×Ŀ +Ġвозмож ноÑģÑĤÑĮ +Ġëĭ¤ ìĭľ +è¦ĭ ãģŁ +à¸ĸ à¸Ļ +à¸ĸà¸Ļ à¸Ļ +mız ı +ĠÙħ جÙħÙĪØ¹Ø© +c jÄħ +ĠÐł Ф +à¸ģำ หà¸Ļ +à¸ģำหà¸Ļ à¸Ķ +ĠìŬ 기 +land ı +ни ÑĨ +ÑģÑĤв е +Ġ×ĵ ×ijר×Ļ×Ŀ +Ġsk ÅĤad +ãĤĬ ãģ¾ãģĹãģŁ +ĠоÑĤ кÑĢÑĭÑĤ +нÑı ÑĤ +ĠÑģво ей +à¸Ī ิà¸ķ +ĠкаÑĩеÑģÑĤв е +Ġet tiÄŁi +ìĤ¬ íķŃ +ĠاÙĦÙĬ ÙħÙĨ +иÑĩеÑģки й +ë¸ Į +Ġ×ij×IJר ×¥ +Ġا سÙħ +Ġиз веÑģÑĤ +r ão +Ġatt ivitÃł +à¹Ģà¸Ľà¹ĩà¸Ļ à¸ģาร +ĠاÙĦد Ùĥت +ĠاÙĦدÙĥت ÙĪØ± +ĠÙĪØ§ØŃد Ø© +ĠÑģ ÑĩеÑĤ +ĠпÑĢ Ð¸Ñĩ +ĠпÑĢиÑĩ ин +ĠÙĪØ² ارة +Ġh uyá»ĩn +ĠÙĥ تاب +à¹ģà¸Ļ à¹Īà¸Ļ +à¹ģà¸Ļà¹Īà¸Ļ à¸Ńà¸Ļ +Ġgün ü +г ÑĢÑĥз +ĠاÙĦØ® اص +Ġgör ül +׾ ×ŀ×ĵ +Ġìłķ ëıĦ +×ķ×ij ×Ļ׾ +Ġ×ŀ×§ צ×ķ×¢×Ļ +ĠоÑģоб енно +à¸Ľà¸£à¸° à¸ģา +à¸Ľà¸£à¸°à¸ģา ศ +aca ģını +ë¶ ģ +à¸łà¸¹ มิ +ĠÑį лекÑĤ +ĠÑįлекÑĤ ÑĢо +Ġ×§ ש×Ķ +سÙĦ Ø· +à¸Ĭà¸Ļ ะ +×¢ ×Ļ׾ +ĠЧ е +à¹ģà¸Ļ à¹Ī +lı ÄŁ +lıģ ın +Ġ×ŀ×¢ ×¨×Ľ×ª +好ãģį ãģª +มาà¸ģ à¸Ĥึà¹īà¸Ļ +×ŀ×¢ ×ijר +ĠاÙĦÙħ غرب +ĠпеÑĢ Ð¸ +ĠпеÑĢи од +Ġnh ạc +ا ÙĪÙĬ +ĠÙĪ Ø¹ÙĦÙī +أخ ذ +ĠC ô +תר ×ij×ķת +×Ĵ ×Ķ +Ġktóre j +×IJ ×Ļת +×ij ×ķ×IJ +д елÑĮ +รี วิ +รีวิ ว +ж Ñĥ +Ġ×ij×Ĺ ×ķ +еÑĪ ÑĮ +ĠØ£ ÙĦÙģ +ĠاÙĦÙĪ Ø·ÙĨÙĬ +ĠاÙĦÙħÙĨ Ø·ÙĤØ© +nÄħ Äĩ +Ġthi ên +иÑĩеÑģк ой +ĠاÙĦÙħ ÙĦ +Ġع Ùħ +ס פר +Ġnh óm +ÙĪØµ Ùģ +ĠCh úng +Ġر ÙĤÙħ +ãģ¾ãģĹãģŁ ãģĮ +al ité +ล ม +ĠëĤ´ ê°Ģ +׾ק ×ķ×Ĺ +ĠS Æ¡n +pos ição +mi ÄĻ +Ġtr ánh +ĠÄIJ á»Ļ +׼ ×Ĺ +ãģĤ ãģ£ãģ¦ +à¸Ńย à¹Īา +Ġ×ŀ×Ĺ ×Ļר +Ġ×Ķ ×Ļת×Ķ +à¸Ľ à¹Īา +à¸Ńืà¹Īà¸Ļ à¹Ĩ +Ø´ ÙĤ +×ł×¡ ×Ļ +ë¦ ¼ +ãģ¦ãģĹãģ¾ ãģĨ +Ġ×ŀ צ×ij +ãģ« åĩº +ÙħÙĪØ§ Ø·ÙĨ +ยัà¸ĩ มี +алÑĮ нÑĭе +san ız +Ø¥ سرائÙĬÙĦ +ĠvÃł i +ì¤ Ħ +ã썿ĢĿ ãģ£ãģ¦ +×Ļ ×ķ׳×Ļ +çĶŁ ãģį +Ġs âu +Ñĩ иÑģÑĤ +Ġl á»ħ +ĠGi á +à¸Ńุ à¸Ľ +à¸Ńà¸¸à¸Ľ à¸ģร +à¸Ńà¸¸à¸Ľà¸ģร à¸ĵà¹Į +Ġnh ẹ +r ö +ס ×ĺ×Ļ +ãģķãĤĵ ãģĮ +Ġd ầu +ع Ùİ +ت را +×Ĵ×ĵ ׾ +Ġtécn ica +׼ ׳×Ļ×Ŀ +תק ש +תקש ×ķרת +Ġн его +ét ait +Ġm á»ģm +Ñģ еÑĤ +Ġnh áºŃt +Ġ×ŀ ×¢×ľ +Ġ×Ķ×¢ ×ij×ķ×ĵ +Ġ×Ķ×¢×ij×ķ×ĵ ×Ķ +Ġ×Ĵ ×Ļ׾ +ãģ¯ ãģªãģĦ +ائ ØŃ +Ġз деÑģÑĮ +×IJ ×Ļ׳×ĺר +Ùħ ÙIJ +Ġ×Ļ ×Ĺ×ĵ +ر اÙģ +ì²ĺ 리 +×ĵ ×¢×ķת +ì¹ ľ +ĠТ о +ĠTh ế +ì¶ © +Ġ׳׼ ×ķף +عÙĬ Ø´ +ни з +Ġج اÙĨب +×ŀ×§ צ×ķ×¢ +à¹Ĥ à¸ĭ +Ñģ ÑĥÑĤ +ìĸ´ ìļĶ +ãĤĴè¦ĭ ãģ¦ +ار د +Ġaç ıl +ĠاÙĦØŃ ÙĬاة +à¸ģà¹ĩ à¹Ħà¸Ķà¹ī +ãģĿãĤĮ ãĤĴ +عض ÙĪ +Ġг ÑĢаж +ĠгÑĢаж дан +à¸Īะ à¸ķà¹īà¸Ńà¸ĩ +ĠìĿ´ 룬 +ĠìĿ´ë٬ íķľ +Ġtr ách +ÙĨ Ùİ +Ġkı sa +Ã Ķ +ÑĪ ÐºÐ° +ãģ® äºº +ĠÐŁ оÑģ +ĠÐŁÐ¾Ñģ ле +Ñĥ лÑĮ +ÙĪØ§ جÙĩ +ÙĤ رب +à¸Ľà¸ıิ à¸ļัà¸ķิ +ê° Ļ +Ġ×ŀ ׳ +ĠÑģво и +بر اÙħج +Ġر ÙĪ +пÑĢ Ð¾Ð´ +пÑĢод аж +Ġby ÅĤy +วั ย +Ġgör ün +ĠÃ Ī +ÑİÑī им +ĠÑĤак ой +Ùģ ÙĪØ± +ĠÙģ Ø¹ÙĦ +Ġб ел +ëIJ ł +er ÃŃa +ĠÑģво Ñİ +Ġl ã +Ġlã nh +à¹Ģà¸ŀืà¹Īà¸Ń à¹ĥหà¹ī +ÙĤ ÙĨ +تط ÙĪÙĬر +Ġsay ı +ĠÑģ ейÑĩаÑģ +Ġ×IJ×Ĺר ת +×§ ×ķפ×Ķ +×§×ķר ס +Ġس Ùħ +Ġ×ĺ ×Ļפ×ķ׾ +ìĿ´ëĿ¼ ëĬĶ +دراس Ø© +èµ· ãģĵ +×Ĺ ×Ļ׳ +×Ĺ×Ļ׳ ×ķ×ļ +×ĵ ×§ +Ġë§ ŀ +Ġком анд +ĠÐij о +Ġиг ÑĢÑĭ +à¸ļ ี +ĠØ£ Ùİ +в ен +ĠاÙĦج دÙĬد +ĠÙĦ Ø¥ +Ġ×ķ×IJ ׳×Ļ +Ġ×Ķס ×Ļ +иÑĩеÑģк ого +رÙĪ ØŃ +à¸ģาร ศึà¸ģษา +ĠTr ưá»Ŀng +иг ÑĢа +ıl ması +Ġм аÑģÑģ +ãģ¨ãģį ãģ« +à¸Ĺีà¹Ī à¸ľà¹Īาà¸Ļ +à¸Ĺีà¹Īà¸ľà¹Īาà¸Ļ มา +ĠاÙĦساب ÙĤ +Ġ×ŀ×¢ ×ĺ +в аÑĤÑĮ +m Ã¼ÅŁ +Ġ׾ ׼×ļ +Ġt á»ĭch +Ùģ ÙĩÙħ +تد رÙĬب +Ø´ Ùĥ +Ġ×ij ×ŀ×Ļ +Ġ×ij×ŀ×Ļ ×ķ×Ĺ×ĵ +ÙĤØ· اع +ãģª ãģĹ +×ķצ ×Ļ×IJ +ĠÙĪ Ø³ÙĬ +з Ñĥ +Ġy at +Ġyat ırım +ë§ İ +Ġth ắng +ãģĬ 客 +ãģĬ客 æ§ĺ +ĠThi ên +ãģ«å¯¾ ãģĹãģ¦ +ÑĢ Ð¸Ñģ +ÙĨت ائ +ÙĨتائ ج +Ġ×ŀ שר +Ġ×ŀשר ×ĵ +Ġتع اÙĦ +ĠتعاÙĦ Ùī +ש ׳×Ļ +Ùĩ اÙħ +×IJ׳ ש×Ļ×Ŀ +Ġżyc ia +ĠÑĢÑĥб лей +ÙĬ ض +Ġkat ıl +ĠÙħ ÙĪØ¶ÙĪØ¹ +Ġvard ır +ĠÙħÙĨ Ø·ÙĤØ© +ĠTr ần +Ġв еÑģ +ü p +Ùħ ÙĪÙĨ +ÑĪ Ð»Ð¸ +Ġn óng +Ø® ÙĦÙģ +ĠС ÑĤа +Ġд оÑĢ +ĠдоÑĢ Ð¾Ð³ +ĠwÅĤa ÅĽnie +eÄŁ in +Ġhi á»ĥm +ĠС ам +ê»ĺ ìĦľ +ĠÑĦ а +ãģ» ãģĨ +ãģ»ãģĨ ãģĮ +×ķפ ×Ļ×¢ +ê° Ī +د ÙĪÙĦ +Ġthu ê +Ġch á»Ĺ +Ġëĭ¹ ìĭł +ãģij ãĤĮ +ãģijãĤĮ ãģ© +ë³´ íĺ¸ +ãģķãĤĮ ãģ¦ãģĦãģ¾ãģĻ +Ġнад о +ĠìĤ¬ëŀĮ ëĵ¤ +à¹Ģà¸Ĥ à¸ķ +สม ัย +z ÅĤ +ت ÙĪØ± +Ġש ת×Ļ +v ê +Ġ×ijת ×ķ×ļ +à¸Ĭ ัย +ãģĦ ãģ£ãģŁ +ìĿ ij +Ġt ầ +Ġtầ ng +ש ׼ר +Ġê¸ Ģ +Ġ×Ķש ׳×Ķ +Ġا ÙĨÙĩ +ç«ĭ ãģ¡ +r és +füh ren +ر ØŃÙħ +ê· ¹ +ĠâĢ « +Ġsu ất +à¸Ł ิ +ÙĬ Ùĩا +ĠاÙĦ اتØŃاد +Ġt uyá»ĥn +ãģ¾ ãĤĭ +Ġm ại +Ġng ân +ãĤ° ãĥ© +欲 ãģĹãģĦ +س ار +ãĤĤãģ® ãģ§ãģĻ +ки е +Ġseç im +åħ¥ ãĤĬ +ãģªãģ© ãĤĴ +ÑĤ ÑĢи +ĠÑģп еÑĨ +ĠØ£ د +Ġод но +ÑĪ ÐµÐ» +ãĥĩ ãĥ¼ãĤ¿ +ãĤ· ãĤ¹ãĥĨ +ãĤ·ãĤ¹ãĥĨ ãĥł +è¡Į ãģį +ã썿ĢĿ ãģ£ãģŁ +à¹Ģà¸ģิà¸Ķ à¸Ĥึà¹īà¸Ļ +ĠÑĤ ож +ĠÑĤож е +Ġs ạch +ĠÑģ ÑĢок +Ġкли енÑĤ +ĠÙħØ´ رÙĪØ¹ +Ġalt ında +Ġì ·¨ +ä¸Ń ãģ® +ãģķãģĽ ãĤĭ +ãģĻ ãģ¹ +ãģĻãģ¹ ãģ¦ +ê°ľ ë°ľ +ĠÄij êm +ãģªãģĦ ãģ®ãģ§ +ì² ł +×¢ ×ij×ĵ +Ġd ấu +à¸Ħà¸Ļ à¸Ĺีà¹Ī +ĠC ách +تع ÙĦÙĬÙħ +Ġh ại +ãĤ» ãĥķãĥ¬ +ĠÙĨÙ쨳 Ùĩ +ĠíĨµ íķ´ +ÑĪ Ð»Ð¾ +Ġнап ÑĢав +ĠнапÑĢав лен +ÑĢÑĥ Ñĩ +íĶ Į +Ġ×ijר ×Ļ×IJ +ãģ® ãģ¿ +ãģ«ãģĬ ãģĦãģ¦ +×ij ׳ק +ãĤ¨ ãĥ³ +Ø«ÙĦ اث +Ġm ỹ +ĠÑģай ÑĤе +Ġе мÑĥ +ت غÙĬ +تغÙĬ ÙĬر +خص ÙĪØµ +ÑĤе ли +Ġ×ķ׾ ׼ף +פע ×Ŀ +Ġпо ÑįÑĤомÑĥ +ر اÙĨ +иÑĤел ей +пиÑģ ан +×¢ ×¥ +ĠìĤ¬ ìĹħ +Ùħ ز +جÙħ ÙĬع +ë©´ ìĦľ +à¸ľà¸¥à¸´à¸ķ à¸łà¸± +à¸ľà¸¥à¸´à¸ķà¸łà¸± à¸ĵ +à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵ à¸ij +à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵà¸ij à¹Į +ĠпÑĢ Ð¸Ð¼ÐµÑĢ +ãĤŃ ãĥ¼ +l â +Ġch Äĥm +缮 ãģ® +ãģĦ ãģĭ +ãģ¨è¨Ģ ãģĨ +×ĸ ×ķ×Ĵ +Ġ×ij ×ĵ×Ļ +Ġ×ij×ĵ×Ļ ×ķ×§ +ãģĬ åºĹ +à¸ķà¸Ńà¸Ļ à¸Ļีà¹ī +Ġph á»iji +п ÑĤ +สà¸Ļ าม +Ø· ÙĪ +ص اØŃ +صاØŃ ب +ĠD ü +ĠDü nya +Ġп ока +п ал +ĠÄij ảo +ĠاÙĦÙģ ÙĪØ± +ĠاÙĦÙģÙĪØ± Ùĥس +Ġmá u +кÑĢ ÐµÐ¿ +ĠاÙĦس اعة +ĠгоÑĢ Ð¾Ð´Ð° +Ùģ ØµÙĦ +ай ÑĤе +Ġд ог +Ġдог овоÑĢ +ĠØ¥ ذ +Ġ×ij׼׾ ׾ +ÙĬ تÙĩ +×Ĵ ×ijר +Ġbir ç +Ġbirç ok +문 íĻĶ +ãģĿãģĨ ãģª +را ØŃ +ĠÙħ رة +ĠденÑĮ ги +f ä +à¸Ĥà¹īา ว +ĠÑģов ÑĢем +ĠÑģовÑĢем енн +׾×Ĺ ×¥ +èī¯ ãģı +ĠÙģ Ø£ +Ġ×ķ ×ĸ×Ķ +Ġз ани +Ġзани ма +Ġê°Ģì§Ģ ê³ł +Ġh Æ¡i +ãģªãģ® ãģĭ +ãĥĨ ãĥ¬ãĥĵ +Ġר ×ij×ķת +à¸ķ ี +Ġ×ijש ×ł×ª +ĠT ại +Ġthu áºŃn +Ñģ ел +Ñij м +dzi Äĩ +ĠÑģ ка +ĠÑģка Ñĩ +ĠÑģкаÑĩ аÑĤÑĮ +×ķ×ŀ ×ķ +г ла +Ġмин ÑĥÑĤ +åĩº ãģĻ +Ġ×Ĺ×Ļ ×Ļ×ij +Ġת ×Ĵ×ķ×ij×Ķ +à¸£à¸¹à¸Ľ à¹ģà¸ļà¸ļ +ни ÑĨа +Ġİ n +ĠØ£ ع +Ġض ÙħÙĨ +Ùħ ثاÙĦ +ĠyaÅŁ an +ĠìŰ 구 +ĠL ê +ש׾ ×Ĺ +ãģı ãģªãĤĭ +ìĹĨ ìĿ´ +ĠÑĤ ÑĢи +ĠÑĩаÑģÑĤ о +Ġоб ÑĢаÑĤ +п ло +د Ø® +دخ ÙĪÙĦ +س Ùĩ +à¸Ń าà¸ģ +à¸Ńาà¸ģ าศ +Ġ׼ ×ĸ×Ķ +Ġ×Ķ×¢ סק +ĠاÙĦØ£ ÙĨ +å¹´ ãģ« +×¢ ש×ķ +Ġש ×¢×ķת +Ġm Ãłn +×IJר ×Ļ +sı yla +Ù쨱 ÙĤ +ни Ñħ +Ġت ست +è¦ĭ ãģ¦ +ØŃا ÙĪÙĦ +×IJ ×Ļ׼×ķת +ĠbaÅŁ ladı +st Äħ +stÄħ pi +à¸Ĺีà¹Ī à¹Ģรา +ÙĤر ر +ج اب +Ġ×ijר ×ķר +à¹Ģà¸Ĥà¹īา à¹ĥà¸Ī +×ŀ׊קר +al ım +Ġס ×Ļפ×ķר +ãģ§ãģĤ ãĤĮãģ° +Ġש×ŀ ×ķר×ķת +Ġ×ķ ×ŀ×Ķ +ãģĵ ãģĿ +id ée +ä¸ĭ ãģķãģĦ +تÙĨا ÙĪÙĦ +Ġ ลà¹īาà¸Ļ +Ġìļ°ë¦¬ ëĬĶ +اÙĨ ا +ÑģÑĤ ой +б оÑĤ +ĠyaÅŁ am +kö y +Ø¥ ÙĦ +ÑĢ Ñĭв +기 ìĹħ +Ġ×Ķ×ŀ ×ĵ +Ġ×Ķ×ŀ×ĵ ×Ļ׳×Ķ +د ب +×¢ ×Ļ׳×Ļ +×ŀ ת×Ĺ +Ġפ ר×Ļ +ãĥĭ ãĥ¼ +اÙħ ÙĬ +Ġnh ằm +ãĤĮ ãģªãģĦ +ت عرÙģ +Ġë§Ī ìĿĮ +ìĵ ° +Ġh ấp +ר×Ĵ ×Ļ׾ +ب Ùİ +Ġr Äĥng +gl Äħd +ĠÑģиÑģÑĤем Ñĭ +Ġkh óa +ãģ§ãģĻ ãĤĪãģŃ +大ãģį ãģı +기 를 +Ġké o +ÙĪ Ø¡ +ج اÙħ +جاÙħ ع +Ġ×¢ ×Ļצ×ķ×ij +t éri +Ġת ש +Ġ×IJ ×ij×Ļ +ĠCh ương +à¸ļริ à¹Ģว +à¸ļริà¹Ģว à¸ĵ +ãģ¤ ãģı +Ġ×Ĺ ×ķ׾ +עת ×Ļ×ĵ +ש ×Ļ×ŀ×Ķ +ëĤ ¨ +Ġש×IJ ×Ļף +ĠÙĪØ§ÙĦ Ø¥ +ÑĦ а +Ġkh ám +Ġ×ĺ ×ķ×ij×Ķ +ĠвÑĭ Ñģ +ĠвÑĭÑģ око +ĠاÙĦØŃ دÙĬØ« +人 ãĤĤ +d Ã¼ÄŁÃ¼ +×Ļ×Ĺ ×ķ×ĵ +تع ÙĦÙĬ +تعÙĦÙĬ ÙĤ +l ö +تØŃ دÙĬد +н его +ĠÑĥд об +Ġ׾ ×ŀ×Ļ +Ġר ×ķצ×Ļ×Ŀ +Ġج اء +Ġ×ij ×ĸ×ŀף +à¸Ľà¸ģ à¸ķิ +é«ĺ ãģı +à¸Ľà¸¥ า +Ġart ık +Ġbug ün +×§ ׳×Ļ +Ġkho á +ĠÙħ رÙĥز +ĠìŀIJ 기 +در جة +×ŀש ר×ĵ +Ġgi ấy +Ġch óng +×§ פ +ÙĬب Ø© +ĠczÄĻ sto +в али +Ùĥ ب +ìŁ ģ +ส à¸ļาย +à¸Ľà¸£à¸°à¸Ĭา à¸Ĭà¸Ļ +×Ĵ ×ķ×£ +ëŁ ī +ãģ® ãģĵãģ¨ +ล à¸Ń +Ġngh á»ī +åŃIJ ãģ© +åŃIJãģ© ãĤĤ +à¹Ħà¸Ķ à¹īà¸Ńย +à¹Ħà¸Ķà¹īà¸Ńย à¹Īาà¸ĩ +×ĵ ×¢ +ĠاÙĦت Ùī +ĠÑģов еÑĤ +Ġqual itÃł +åĩº ãģĹ +ĠÑĢÑĥк ов +ĠÑĢÑĥков од +ราย ละà¹Ģà¸Ńียà¸Ķ +ãģªãģĭ ãģªãģĭ +기 ê´Ģ +Ġ×Ĺ ×ķש +Ġ×Ĺ×ķש ×ij +л оÑĤ +à¸Ļะ à¸Ħรัà¸ļ +×§×ij ×ķצ×Ķ +Ġth ái +Ġש ×ij×Ķ +ĠÑĪ ÐºÐ¾Ð» +ĠÙĦ ÙĥÙĦ +à¹ĥà¸Ļ à¸Ĭà¹Īวà¸ĩ +ĠÙħ ÙĥاÙĨ +ë ķĮ +Ġc ải +ĠCh ÃŃ +ÑĥÑĩ а +ìĿ µ +Ġx ảy +à¸Ĭà¸Ļ ิà¸Ķ +Ġc áºŃu +к ÑĢов +ss é +ĠÙĨ ÙĪØ¹ +ĠТ а +Ø® Ùħس +פ×ķס ×ĺ +Ġm ắc +ĠÄij em +à¸ģาร à¹ĥà¸Ĭà¹ī +ר ×ķס +ĠÐĽ е +Ġth á»Ń +รà¹Īาà¸ĩ à¸ģาย +üz ü +æĹ¥æľ¬ ãģ® +ê³¼ ìłķ +ש ×Ļ×IJ +ĠìŀĪ ê³ł +×ij ×ķ׾ +ìķ ħ +ĠÙĪØ§ÙĦ ا +ĠÐĽ и +ĠвÑģ Ñij +Ġużytk ow +×Ĺ ×ķ׾ +ر Ù쨶 +Ġson uç +ãģĦ ãģ¾ãģĽãĤĵ +ìĤ¬ ìĹħ +ëĪ Ħ +ÑĤ ек +Ġud ziaÅĤ +л ез +Ġ×Ķ×Ļ ×Ļת×Ļ +ãĤīãĤĮ ãģ¦ +Ùħس ؤÙĪÙĦ +ر ار +ÑĤ ан +ĠÄij Ãło +Ġר ×ķ×ij +Ġ×ijש×ij ×Ļ׾ +ä»ĬåĽŀ ãģ¯ +ãĤ¸ ãĥ¥ +Ġ×¢ ×ijר +ãģĽ ãģ¦ +п олÑĮ +ak lı +Ġk ÃŃnh +د ت +лож ение +ĠاÙĦÙħ ص +ĠاÙĦÙħص رÙĬ +à¸Īริà¸ĩ à¹Ĩ +ĠاÙĦشر ÙĥØ© +ĠÄij á»ı +ãĥĽ ãĥĨ +ãĥĽãĥĨ ãĥ« +Ñį кон +Ñįкон ом +ĠÙĪ Ø¹ÙĨ +Ġת ׳ +Ġ×ª×ł ×IJ×Ļ +ĠاÙĦدÙĪÙĦ ÙĬØ© +Ġì§Ģ ìĹŃ +ãģ§ãģĻ ãģĭ +Ġв аÑĢи +ĠваÑĢи анÑĤ +ĠاÙĦع رب +ел а +Ġt Æ°á»Ľng +sk Äħ +Ġm ặc +ส ัà¸ģ +ãĥĵ ãĥ¼ +Ġ×ij ×Ĵ׾ +Ġ×ij×Ĵ׾ ׾ +ãĥķãĤ¡ ãĥ³ +×ij ×Ļצ +×ij×Ļצ ×ķ×¢ +ли ÑģÑĤ +à¸Ł ุ +à¸Łà¸¸ à¸ķ +à¸Łà¸¸à¸ķ à¸ļà¸Ńล +à¸Ŀ à¹Īาย +ìŀIJ ìĿĺ +Ġس ÙĪÙģ +Ġש ×Ķת +Ġê± ¸ +×¢ ×ij×ķ×ĵ +ãģĻãĤĭ ãģĵãģ¨ãģĮ +ĠÑĩа ÑģÑĤÑĮ +ãĤ¢ ãĥ¡ãĥª +ãĤ¢ãĥ¡ãĥª ãĤ« +Ġtak ım +Ġs Ỽ +ĠsỼ m +שר ×Ķ +è¨Ģ ãģĨ +л ан +ì» ¤ +׼ ׳×Ķ +ÙĪÙģ ÙĬ +íĹ Ī +lu ÄŁu +ĠëĮĢ íķ´ +Ġ׾×ij ×Ļת +Ġ×Ķר×IJש ×ķ׳×Ķ +ص Ùħ +Ġsö yled +Ġsöyled i +à¸Ľ าà¸ģ +Ġard ından +ãģĪ ãģŁ +à¸Ĺัà¹Īว à¹Ħà¸Ľ +Ġ׳×ķס ×£ +б олÑĮ +ãĤĵãģ§ãģĻ ãģijãģ© +ĠлиÑĪ ÑĮ +Ġ×ij ×IJ×Ļ +ĠбÑĭ ÑģÑĤÑĢо +ส ัà¸Ļ +Ġ×ij פ׳×Ļ +л еÑĩ +ĠاÙĦØ® بر +Ġsó c +Ġth ú +Ġп ÑıÑĤ +ãģĬ é¡ĺ +ãģĬé¡ĺ ãģĦ +ÑĤ ин +ãģ«ãģ¤ãģĦãģ¦ ãģ¯ +פ ף +Ġдв ÑĥÑħ +à¸į ีà¹Ī +à¸įีà¹Ī à¸Ľ +à¸įีà¹Īà¸Ľ ุ +à¸įีà¹Īà¸Ľà¸¸ à¹Īà¸Ļ +оп еÑĢ +ĠاÙĦب شر +ĠاÙĦÙħ اÙĦ +ıyor uz +تØŃ ÙħÙĬÙĦ +à¸ģ ะ +éĸĵ ãģ« +×Ĺ ×ķש +ĠNg uyên +ãģĦãģ¦ ãģĦãĤĭ +дÑĥ ÑĪ +ש פע +ÑĪ Ñĥ +å®Ł éļĽãģ« +ĠÑĢай он +ĠCh á»ī +ÙĨ صر +Ġìļ ´ +Ġìļ´ ìĺģ +Ġ×Ķ×ĵ ×Ļף +ØŃد د +ر ز +ĠاÙĦد Ùħ +ĠPh áp +ÑĤ ÑģÑı +è¦ĭ ãģĪ +Ġti á»ĥu +Ġs á»Ńa +а ÑİÑĤÑģÑı +ĠB á +Ġ×ķ ׼׾ +Ð ĸ +ÑĪ Ð¸Ð¼ +ìĿ´ ëĬĶ +л ев +d ık +Ġprés ente +Ġara ç +صد ÙĤ +Ġпом ог +ĠاÙĦشر ÙĤ +ĠÙĪØ§ÙĦ ذÙĬ +رÙĬ ا +×ij ׳×ķת +Ġng á»ĵi +ר ×ķפ +ר×ķפ ×IJ +Ġth ấp +ãĤĦ ãģ¯ +ãĤĦãģ¯ ãĤĬ +ĠاÙĦج دÙĬدة +éĿŀ常 ãģ« +ÙĬÙĦ ÙĬ +ìª ½ +تع اÙħÙĦ +ãģł ã썿ĢĿãģĦãģ¾ãģĻ +Ùħ Ùħ +иÑĤе ли +ãĤµãĤ¤ ãĤº +اد ات +ĠاÙĦÙħ اÙĦÙĬØ© +Ùĥات ب +к ли +веÑĢ Ñħ +ни Ñĩ +Ġ×ľ×¢ ×ij×ķ×ĵ +׾ ×Ļ×Ķ +ØŃ Ùİ +ãĤ¤ ãĥĻ +ãĤ¤ãĥĻ ãĥ³ãĥĪ +Ġת ×Ĵ×ķ×ij×ķת +ÑĦ он +ĠдÑĢÑĥг ие +×IJ ×ĸ×ķר +Ġper ò +ìķ ŀ +åĢŁ ãĤĬ +ר צ×Ļ +×IJ ×ĸ +алÑĮ нÑĭÑħ +Ġê²ĥ ìľ¼ë¡ľ +ĠпÑĢав о +ĠاÙĦØ£ رض +à¹Ģà¸Ĺ à¸Ħ +à¹Ģà¸Ĺà¸Ħ à¹Ĥà¸Ļ +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļ à¹Ĥล +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥล ย +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥลย ี +צ ר×Ļ +ĠÐļ Ñĥ +ıl ma +決 ãĤģ +ا ÙĪ +Ġ×ĵ ×§×ķת +à¸Ħร ู +ĠÙħست ÙĪÙī +à¸Ľ à¹īà¸Ńà¸ĩ +à¸Ľà¹īà¸Ńà¸ĩ à¸ģัà¸Ļ +×ĵ ×ķ×ŀ×Ķ +ĠÑģ егоднÑı +س ÙĪÙĤ +ר×Ĺ ×ķ×ij +ĠØ¥ دارة +Ñħ ож +éģİ ãģİ +à¸Ħ à¸Ń +нÑĥ л +×ķ׼ ×Ķ +ÙĪ Ø§ÙģÙĤ +׼׾ ׾ +Ġ×Ķ ×ĵ×ķ +Ġl Ä©nh +Ġkh ảo +×IJ×ŀ צע +ë¨ ¸ +Ġ׼ ×Ļצ +Ġ׼×Ļצ ×ĵ +Ġдолж нÑĭ +หว ัà¸ĩ +ãĥĩ ãĤ¶ +ãĥĩãĤ¶ ãĤ¤ãĥ³ +Ġng á»Ŀ +ä¸Ń ãģ« +à¸ģลัà¸ļ มา +جÙħ اÙĦ +à¸Ķัà¸ĩ à¸ģลà¹Īาว +س ÙĥÙĨ +س ÙĨ +Ġözellik le +з еÑĢ +rz ÄĻ +×ŀ ×ķר×Ķ +Ġl ạ +×ŀ ×Ļ׳×Ļ +ר ×Ļת +ãģĿãĤĮ ãģĮ +ãģĭ ãĤĮ +ĠÙĬÙħÙĥÙĨ Ùĥ +öff entlich +г ан +ĠاÙĦØŃ ÙĦ +ĠmiÄĻd zy +ĠÑĩа ÑģÑĤи +ujÄħ cy +ĠbaÄŁ lı +ĠiliÅŁ ki +Ùģ Ø§Ø¡ +ãĥª ãĥ³ãĤ° +Ġhã ng +ĠконÑĤ ÑĢ +ĠконÑĤÑĢ Ð¾Ð» +к оп +ש ×Ļ×¢ +ש×Ļ×¢ ×ķר +ĠÐĴ аÑĪ +Ġ×Ķ ×ª×§ +ÙħÙĨ ع +ĠpolÃŃt ico +Ġг олов +ĠØ¥ ÙĬ +Ø¥ ÙĨتاج +à¸ļ ิ +Ġг овоÑĢ +ĠговоÑĢ Ð¸ÑĤ +Ġph á»ķ +ĠÑģем ÑĮ +ãģ¯ ãģĤãĤĬãģ¾ãģĽãĤĵ +ĠÙĪ Ø§Ø³Øª +×ŀש פ×ĺ +з ем +×ŀ×ĵ ×ijר +Ġíģ ° +ĠìĿ´ ë²Ī +ê°Ģ ëĬĶ +Ġì§Ģ ìĽIJ +Ġca ÅĤy +Ġgeli ÅŁtir +Ñģк ое +pos é +Ġkh ô +à¸ķิà¸Ķ à¸ķาม +miss ão +Ġ׾ ×ŀר +Ġ׾×ŀר ×ķת +Ġb ó +à¸ķรวà¸Ī สà¸Ńà¸ļ +Ġngh á»ģ +Ġб из +Ġбиз неÑģ +ÑģÑĤ еÑĢ +ÙĪ Ùİ +楽 ãģĹãģ +楽ãģĹãģ ¿ +ãģĵãĤĮ ãģĭãĤī +wiÄħ zan +ส à¸Ńà¸Ļ +Ùħ ÙĪØ± +׳×ĵ ׾ +Ġ×Ķ×IJ ×ĵ×Ŀ +Ġм олод +ØŃ Ùħا +ØŃÙħا ÙĬØ© +ÑģÑĤ ÑĢан +Ġbu á»ķi +ת×Ļ ×Ļ×Ŀ +abile ceÄŁi +L İ +à¹Ģย à¸Ńะ +à¸Ī ร +س ÙĥاÙĨ +à¸Ļ ัà¸Ķ +Ġm ấy +ĠÐij а +s ÅĤaw +ĠÙģ ÙĦا +ĠкоÑĤоÑĢ Ð¾Ð¹ +Ġпло Ñī +ĠплоÑī ад +ãĤĤ ãģĤãĤĬ +sz czÄĻ +×Ļפ ×ķ +ש×ŀ ת +owa ÅĤa +Ġn ông +צ×ij ×IJ +ĠìŀĪ ìĹĪ +ãģ¾ ãģ¨ +ãģ¾ãģ¨ ãĤģ +ÙĤÙĪ Ø§Øª +ãģ¿ ãĤĵãģª +Ġ׼ ×ŀ×¢×ĺ +Ġx úc +ï¼ Ĩ +r ÄĻ +rÄĻ cz +×ĵ ×ŀ×Ļ +Ġt áºŃn +à¸Ķ วà¸ĩ +ê²½ ìłľ +п ÑĥÑĤ +Ø£ ربع +Ġ×ŀ שת×ŀש +ãĤ¿ãĤ¤ ãĥĹ +Ġìłľ ê°Ģ +Ġ׾ ׼ף +ĠобÑĢаз ом +ÙĬÙĥ ا +w ÅĤ +wÅĤ asn +ĠاÙĦÙĪØ·ÙĨ ÙĬØ© +بÙĬ ب +×ŀ ׾×Ļ +к ÑĢаÑĤ +기 ìĹIJ +ÙĤ اد +ĠÙĦ دÙī +à¸Ħวาม รูà¹ī +×ŀ×ĵ×Ļ׳ ×Ļ×ķת +ê² ¨ +Ġíĺ Ħìŀ¬ +ש ת×Ļ +м ол +Ġmá i +à¸ŀิ ม +à¸ŀิม à¸ŀ +à¸ŀิมà¸ŀ à¹Į +หล วà¸ĩ +Ġx uyên +×Ĺ ×¡×¨ +رÙĪ ÙĨ +ãģĿãģĨ ãģĦãģĨ +ãģĿãĤĮ ãģŀ +ãģĿãĤĮãģŀ ãĤĮ +Ġ׼ ש×Ķ +ÐŁ ÑĢав +×ŀ×ij צע +ع رب +Ġbü yü +פ×Ļת ×ķ×Ĺ +à¸Ī à¸ļ +ĠØ£ Ùĥبر +שר ת +×ŀ׼ ש×Ļר +ĠÙĪ Ùħع +ãģ® ãģŁãĤģãģ« +à¸Ļ ัà¸ļ +ì° ° +ãĥª ãĥķãĤ© +ãĥªãĥķãĤ© ãĥ¼ãĥł +Ġc ưá»Ŀng +ĠìłĢ íĿ¬ +ÙħÙĨظ ÙħØ© +Ġhiç bir +ãģ§ãģ¯ ãģĤãĤĬãģ¾ãģĽãĤĵ +ร à¸Ńย +ëIJľ ëĭ¤ +ãģĻãģIJ ãģ« +к ла +Ġürün ler +Ġki á»ĥu +ĠëĤĺ ëĬĶ +ÑĤ ки +Ñģ им +Ġchá»ī nh +ãĤĤ ãģªãģĦ +ศ รี +æĽ¿ ãģĪ +ta ÅŁ +Ġب ÙĥÙĦ +Ġ×ķ ×Ļש +vis ão +ä¼ Ŀ +ä¼Ŀ ãģĪ +ÙĦ د +׾ ×Ļ×ŀ +׾×Ļ×ŀ ×ķ×ĵ +t ória +د Ùij +اÙħ ر +Ġê·¸ëłĩ ê²Į +Ġmateria ÅĤ +à¸Ĺ รา +à¸Ĺรา à¸ļ +ã쮿ĸ¹ ãģĮ +ãģ¦ ãģįãģŁ +ض غ +ضغ Ø· +ĠÙĬ عÙĨÙĬ +ел о +×IJ×Ķ ×ij×Ķ +×¢ ×ŀ +ÅŁ ık +ìŀIJ ëĬĶ +ãĤ¿ ãĥ³ +Ġb áºŃt +×ŀשפ ×Ĺ×Ķ +к ÑĢи +б ли +สั à¸ķ +สัà¸ķ วà¹Į +ĠسÙĨ ÙĪØ§Øª +ĠPh ương +ãģ¦ãģĹãģ¾ ãģ£ãģŁ +ãģª ãģľ +Ġ×ij×IJ ×ķ +Ġc án +س جÙĦ +Ġl ẽ +ãĤ± ãĥ¼ãĤ¹ +Ġ×§ ×Ļ×ij׾ +à¸ļà¸Ĺ à¸Ħวาม +Ġ×ķ ׼ף +ĠпÑĢедÑģÑĤав лен +Ġn á»iji +Ġcoment ário +ени ем +Ġtá» ı +l Ãł +Ġש×Ķ ×Ļ×Ķ +Ñģл ав +ĠاÙĦ ÙĪÙĦا +ĠاÙĦÙĪÙĦا ÙĬات +ÙĦج ÙĨØ© +×§×ķר ×IJ +бÑĭ ÑĤ +Ġì ¦ +Ġì¦ ī +ãģ§ãģĻ ãģĹ +หรืà¸Ń à¹Ħมà¹Ī +за ÑīиÑĤ +ÙģÙĦ سطÙĬÙĨ +Ġmi á»ħn +à¹Ģย à¹ĩà¸Ļ +ĠçalÄ±ÅŁ an +×Ļ×Ĵ ×Ķ +ĠE ÄŁ +ĠEÄŁ itim +ãĥĥãĤ· ãĥ¥ +Ġоп Ñĭ +ĠопÑĭ ÑĤ +ر غ +رغ ب +ĠÑģво иÑħ +à¸Ľà¸£à¸° à¸ķ +à¸Ľà¸£à¸°à¸ķ ู +Ġ×ŀ×IJ ×ĵ +׼ ×ķ׳×Ļ×Ŀ +à¸Ļ ี +ĠвÑĭ Ñħод +ãģ®ä¸Ń ãģ« +פ ׾×IJ +ĠÙĪ ÙĦÙĬس +פ×ķר ס +פ×ķרס ×Ŀ +Ùħ سÙĦÙħ +Ġng ôi +×ĵ ×ŀ×ķת +ãĤĴ使 ãģ£ãģ¦ +ĠпомоÑī ÑĮÑİ +Ø£ سر +бл ок +ÙĤ Ùĩ +ãģĹãģ¾ ãģĦ +ãģ¨ ãģĹãģŁ +Ġп еÑģ +ãĥī ãĥ« +×Ĺ ×Ŀ +ãģĹãģª ãģĮãĤī +ĠÐŁ ÑĢед +ãĥģãĤ§ ãĥĥãĤ¯ +å¼· ãģĦ +ש ×Ļר×ķת +д аеÑĤ +×Ļ×ij ×ķ +Ġgen ç +ил аÑģ +илаÑģ ÑĮ +ĠبÙĦ د +æĤ ª +æĤª ãģĦ +Ġ×ŀ שת +æ§ĺ ãĢħ +æ§ĺãĢħ ãģª +à¸ĺรรม à¸Ĭาà¸ķิ +ĠÙĥ اÙħÙĦ +ĠاÙĦس Ùħ +×ij×ĺ ×Ļ×Ĺ +c á +g ência +ãĤ¹ãĤ¿ ãĥ¼ +à¸Ĺำ à¸ģาร +×Ļ׾ ת +Ġ×Ļ ×ķצ×IJ +w ój +à¸ļุ à¸Ħ +à¸ļุà¸Ħ à¸Ħล +ع تÙħ +عتÙħ د +ãģĿãĤĮ ãģ« +ĠاÙĦت ارÙĬØ® +ÙĤر اء +Ġyönet im +×§ שר +ĠÑģп оÑĢÑĤ +Ġר×IJש ×ķף +Ġseñ al +Ġch ắn +çĦ¡ ãģĦ +ĠдоÑģÑĤ аÑĤ +ĠдоÑģÑĤаÑĤ оÑĩно +Ġá gua +à¸ģร à¸ĵ +à¸ģรà¸ĵ ี +Ġ×ŀש ×ķ +Ġtr ải +ë² Į +ujÄħ cych +Ù쨱 د +à¹ĥ à¸ģล +à¹ĥà¸ģล à¹ī +ãĤĭ ãģ®ãģ¯ +ר×ķ ×ķ×Ĺ +ÙĨ Ùĥ +ĠاÙĦÙĨ ÙĤ +ãģ®ãģ§ ãģĹãĤĩãģĨ +ãģ®ãģ§ãģĹãĤĩãģĨ ãģĭ +Ùħ عرÙģ +ÙħعرÙģ Ø© +ÑĥÑī е +Ġ×ij×¢ ×Ļקר +ت صÙĦ +Ġ×Ķ×IJ ר +Ġ×Ķ×IJר ×¥ +ĠÅŀ i +à¸Ĥา à¸Ķ +íŀ ĺ +ãģªãĤĵ ãģ¨ +ĠìĤ¬ëŀ ij +l Ã¼ÄŁÃ¼ +ب اء +ĠاÙĦØ¢ خر +Ġfam ÃŃlia +ĠTh áng +Ñī ениÑı +ãĤ¯ ãĥŃ +ĠTh ứ +æĽ¸ ãģį +ен ной +ìŀ ¡ +бл аг +благ о +п ов +à¹ģ ว +à¸ĩ à¸Ħà¹Į +à¸Ńัà¸Ļ à¸Ķัà¸ļ +ãģĤ ãģĴ +ร à¹īาย +ün ün +Ġ×Ļ׼×ķ׾ ×Ķ +з он +ĠÐľ и +маÑĤ еÑĢиал +Ġë³´ ë©´ +ØŃÙģ Ø¸ +ê Ìģ +ãģ« ãģĻãĤĭ +Ġת ×IJ +Ġ×Ķס ×ķ +ĠÑģÑĤ оÑĢ +ĠÑģÑĤоÑĢ Ð¾Ð½ +ãĥĪ ãĥĥãĥĹ +ÅĤo ÅĽÄĩ +ëħ ¼ +ëĵ Ŀ +ĠÙĪØ§ÙĦ ع +ì¶ Ķ +Ġ×Ļצ ×IJ +ĠÑĢаз дел +алÑĮ наÑı +×IJ׳ ש×Ļ +spo ÅĤ +spoÅĤ ec +spoÅĤec zn +Ø¥ عÙĦ +إعÙĦ اÙĨ +ÙĤÙĪ Ùī +íķĺë©´ ìĦľ +تط ÙĪØ± +Ġsi êu +Ỽ t +д ви +дви ж +Ġqu ần +k ıl +ĠпÑĢи зна +ĠH ã +ĠHã y +ĠباÙĦ ت +man ın +ãĤ« ãĥ« +Ġk á»· +×§ ׾×Ļ +ëIJĺ ì§Ģ +تعÙĦ Ùħ +ìĭľ ìĦ¤ +ìĭ ¶ +íĺ ¼ +Ùĥ ÙĬÙģ +売 ãĤĬ +วิ à¸Ĭา +б ал +ĠØ£ ØŃ +Ġдолж ен +รา à¸ĩ +ราà¸ĩ วั +ราà¸ĩวั ล +Ùħ اء +ج ار +Å ļ +Ġ×ŀ×IJ ×ĸ +ר ×ŀ×Ķ +ãģĭãĤĤãģĹãĤĮ ãģªãģĦ +ét ude +czÄħ c +Ġg ór +×ł×¡ ×Ķ +Ùħ ÙĬد +ĠÐŁ еÑĢе +Ø£ خر +ãģĿãģ® å¾Į +à¹Ģà¸Ķียว à¸ģัà¸Ļ +×ŀ ×Ĵ×ķ +×ŀ×Ĵ×ķ ×ķף +д ов +mas ına +×¢ ׳×Ķ +ãĤ± ãĥĥãĥĪ +ס ×¢ +סע ×Ļ×£ +ĠT ư +Ġt óc +íĻľ ëıĻ +ĠÐŀ д +ĠÐŀд нако +Ġdol ayı +ؤ Ùĥد +ê³Ħ íļį +׾ ר +в еÑĩ +Ġkh ợi +Ġth á»§y +×ĵ ף +ร à¸ģ +à¸ļั à¸ķร +à¹Ģà¸ģ à¹Īา +ĠاÙĦØ« اÙĦ +ĠاÙĦثاÙĦ Ø« +Ġpod rá +ער ×Ļ +ÙĨج اØŃ +Ġkh ắc +ì¸ ¡ +İ M +ãĤ» ãĥĥãĥĪ +ż enia +Ġ׾×Ĺ ×ijר +er Ãł +ì ´Ī +Ġkü ç +Ġküç ük +ات ÙĩÙħ +à¸ĭ à¹Į +Ùħشار ÙĥØ© +ĠاÙĦ بط +Ġd ây +ен нÑĭм +à¸Ĺีà¹Ī à¹Ħมà¹Ī +ÙĤ Ùİ +Ġv ượt +Ġtr ì +Ġwp ÅĤyw +A Åŀ +з о +ĠاÙĦس ÙĬد +à¸Ĺะ à¹Ģล +ĠÑģодеÑĢж а +ع Ø·ÙĬ +ĠاÙĦع ÙĨ +èĢħ ãģĮ +à¹Ģ หà¸Ļ +à¹Ģหà¸Ļ ืà¸Ń +Ġb ÃŃ +Ġüzer inden +ĠV Å© +Ġnu ôi +ÙĨ Ùħ +алÑĮ ного +×¢ ×Ļף +ØŃ ضر +ĠоÑĤ дел +ëª ĩ +ìķ ¡ +ĠÙĦدÙĬ Ùĩ +ìĻ ľ +Ġse ktör +Ġвозмож но +ĠÐĶ Ð¶ +Ġh ô +äºĭ ãģĮ +иÑĢов ание +алÑĮ ной +Ġ미 êµŃ +ر ØŃÙĦ +ĠÑįк Ñģ +пÑĢав лÑı +Ġnh á»Ŀ +ĠÄij ẩ +ĠÄijẩ y +Ùģ Ùĥر +ĠÙĪØ£ ضاÙģ +ãĥIJ ãĤ¹ +ת×ķ׼ ׳×Ļת +ÑĤел ей +ĠØ¥ÙĦÙĬ Ùĩ +ãģ¨è¨Ģ ãģ£ãģ¦ +Ġдв е +Ġch ấp +ĠL ö +à¸Ħล ิ +à¸Ħลิ à¸Ľ +Ġس ÙĪØ± +ĠسÙĪØ± ÙĬا +×ŀ×Ĺ ×ķ +st ä +д об +Ġni á»ĩm +ãģ® å¤§ +פר×ķ ×Ļ×§ +פר×ķ×Ļ×§ ×ĺ +ĠCh âu +Ġ×ŀ×Ķ ×Ŀ +Ñģк им +ĠполÑĥÑĩ иÑĤÑĮ +ÙĬ ÙĪÙħ +Ø« ÙĪØ± +פ×ķ׾ ×Ļ×ĺ +פ×ķ׾×Ļ×ĺ ×Ļ +ĠмеÑģÑı ÑĨ +åħ¨ ãģ¦ +ĠاÙĦÙħ جÙĦس +ĠاÙĦت اÙĦÙĬ +Ġ׊ר +åIJij ãģij +׼ ×ŀ×Ķ +б ед +Ø£ عض +أعض اء +ÙĪÙĦ د +วà¹Īา à¸Īะ +Ġb ánh +à¸Ļิ ย +à¸Ļิย ม +à¸Ľà¸£à¸° à¸ģัà¸Ļ +ÑģÑĤав иÑĤÑĮ +à¸ŀ à¸Ļัà¸Ļ +ĠÑį ÑĦÑĦ +ĠÑįÑĦÑĦ екÑĤив +Ġав ÑĤоÑĢ +ĠÄIJ Äĥng +Ġth Æ°á»Łng +ãĤĴ æĦŁãģĺ +à¸ģัà¸ļ à¸ģาร +å¾Į ãģ« +Ġya ÄŁ +ست اÙĨ +Ġli á»ģn +ãģĦ ãģ¾ +i êu +à¹Ĥà¸Ķ à¸Ļ +ĠÙĦ ذÙĦÙĥ +à¹Ĥรà¸ĩ à¹Ģรียà¸Ļ +צ ×Ļ×Ĵ +ĠاÙĦÙħ عÙĦÙĪÙħات +ç§ģ ãģŁãģ¡ +à¸Ĺีà¹Ī à¸Ħุà¸ĵ +ãģ«ãģª ãģ£ãģ¦ãģĦãĤĭ +×ŀ×ĵ ×Ļ׳×Ķ +ס ׼×Ŀ +Ġв не +à¸ŀ à¸Ļัà¸ģà¸ĩาà¸Ļ +ÑĢ ÐµÐ¹ +à¹Ģà¸Īà¹īา หà¸Ļà¹īาà¸Ĺีà¹Ī +ĠHi á»ĩn +Ġméd ico +ĠتØŃ ÙĤÙĬÙĤ +ÑĮ ÑĤе +miÅŁ ti +ÙĤÙĬ ادة +ãĤı ãģĭãĤĬ +มา à¸Īาà¸ģ +ëħ Ģ +ãģ«éĸ¢ ãģĻãĤĭ +×IJר×Ĵ ×ķף +m ètre +Ġעצ ×ŀ×Ļ +ĠCh úa +รูà¹ī à¸Ī +รูà¹īà¸Ī ัà¸ģ +ì£ Ħ +ëĭ µ +à¹ģà¸Ĺ à¹ī +Ġgeç en +Ġlan ça +ĠاÙĦ بØŃØ« +×ĵ ×ŀ×ķ +ãģ¯ ãģĺ +ãģ¯ãģĺ ãĤģ +Ġdön Ã¼ÅŁ +è¿ij ãģı +à¹Ģส ม +à¹Ģสม à¸Ń +ëĿ ½ +Ġü ç +á» ŀ +ÑĪ Ð°Ñı +à¸Ĺ ร +ØŃ ÙĤÙĬÙĤØ© +à¸Ĥà¸Ńà¸ĩ à¸ģาร +Ġ무 ìĹĩ +Ġ×Ķ ×Ľ×¨ +ĠاÙĦص ÙĬÙĨ +ĠлÑİ Ð´Ð¸ +à¸ķ าย +ب ÙĪÙĦ +Ġvi êm +Ġthi á»ĩu +à¸ģ à¸Ķ +Ġ׾ ×ĵ×ijר +פ ׳×Ķ +×IJר ×ij×¢ +س Ùī +ĠاÙĦسÙĬ اس +ĠاÙĦسÙĬاس ÙĬØ© +yd ı +ÙĪØŃØ¯ Ø© +ĠдеÑıÑĤелÑĮ ноÑģÑĤи +Ġ×ķ×Ķ ×ŀ +п еÑĩ +пеÑĩ аÑĤ +иÑĢов аниÑı +ĠÑģ ог +ĠÑģог лаÑģ +Ġ׼ ×ĵ +Ġ׼×ĵ ×IJ×Ļ +ĠиÑģполÑĮзов аÑĤÑĮ +ס פ×ķר×ĺ +Ġil çe +exp érience +ĠTh á»Ŀi +İ K +à¹Ħà¸Ł à¸Łà¹īา +ëĵ¤ ìĹIJê²Į +à¸Ľà¸£à¸° à¹Ģà¸ł +à¸Ľà¸£à¸°à¹Ģà¸ł à¸Ĺ +Ġmü mk +Ġmümk ün +Ġ×IJ×ķת ׳×ķ +ìĦ± ìĿĦ +ĠìĿ´ ìľł +زÙĬ ارة +Ġolduk ça +r ób +ĠØ£ ÙĨا +Ġ×Ķ ×ij×Ļ +Ñģ ен +×¢ ×Ļקר +×Ļ×ĵ ×ķ×¢ +d zÄħ +Ùħ عÙĦÙĪÙħات +Ø´ اب +Ġpar ça +à¸Ļะ à¸Ħะ +ب اس +ĠÑĤоÑĢ Ð³ +ĠÑĤоÑĢг ов +Ġ×Ĺ ×ĵר +׼ ר×ĺ +׼ר×ĺ ×Ļס +ĠA yrıca +ÃªÌ £ +ìľ ¨ +ĠÑĤак ие +Ġ×ŀצ ×ķ×Ļ +ãĥ©ãĥ³ ãĤŃãĥ³ãĤ° +ש×Ļ×ķ ×ķ×§ +åīį ãģ® +ĠB ảo +Ñī Ñĥ +æĹ© ãģı +ĠPh òng +à¸ŀระ ราà¸Ĭ +פ ×Ĺ×ķת +Ġг л +Ġгл аз +à¸Ĺ à¹Īา +Ġd ạy +ÑĢ Ð¾ÑģÑĤ +à¹Ĥà¸Ķย à¹Ģà¸īà¸ŀาะ +Ġqu áºŃn +Ġ×Ĺ×ijר ×ķת +m ême +mÄ±ÅŁ tı +ĠاÙĦت داÙĪÙĦ +Ġn ạn +Ġ×Ķ ×ĵ×Ļ +ĠاÙĦØ· رÙĬÙĤ +×Ĵ ×ķת +Ġ×Ķ ×ĵר×ļ +ujÄħ ce +Ġch ữ +ãĤĤãģ® ãģ® +ë° Ľ +ãģķãĤĵ ãģ¯ +Ġyard ım +ĠاÙĦع Ùħ +Ġì§Ħ íĸī +Ġ×Ļ ×Ĺ +Ġ×Ļ×Ĺ ×¡×Ļ +ĠاÙĦÙħ دÙĬÙĨØ© +Ġc ú +à¸ģี ฬ +à¸ģีฬ า +Ġni ên +mis ión +׳×Ļס ×Ļ +׳×Ļס×Ļ ×ķף +Ġвоз ÑĢаÑģÑĤ +Ġ×¢×ķש ×Ķ +ĠÙħ دÙĬر +Ñı ÑģÑĮ +ØŃ جÙħ +íĻĺ ê²½ +ĠاÙĦØ£ خرÙī +u ÃŁer +ĠاÙĦعاÙĦÙħ ÙĬØ© +ĠNg á»įc +êµIJ íļĮ +ä¸Ĭ ãģ§ +×Ļ×Ķ ×ķ×ĵ +×Ļ×Ķ×ķ×ĵ ×Ļ×Ŀ +Ùħس اعدة +Ġжиз нÑĮ +ĠпоÑĤ омÑĥ +ĠاÙĦÙħ ÙħÙĦ +ĠاÙĦÙħÙħÙĦ ÙĥØ© +ĠG ör +ر ÙIJ +×ŀ×§ ×ķ×ŀ×ķת +åĩºæĿ¥ ãĤĭ +ÑĦ ÑĤ +ĠìĿ´ ìłľ +ĠÑĢ ÐµÐ¼ +ĠÑĢем онÑĤ +ת ×ķ×ļ +æĻĤ ãģ¯ +ãĤīãĤĮ ãģªãģĦ +alt ı +å®¶ ãģ® +ĠاÙĦØ¥ عÙĦاÙħ +리 ëĬĶ +ãģĭãĤī ãģ¯ +ĠH ạ +ãģĤ ãģ® +×ĵ×Ļ ×ķף +رÙĬ س +Ġsoci etÃł +ĠاÙĦÙĥ بÙĬر +Ġ×ij ×ŀס +Ġ×ij×ŀס ×Ĵר +Ġ×ij×ŀס×Ĵר ת +ĠìŀĪ ìľ¼ë©° +Ġn ặng +Ùĩ Ùī +ĠB Ãł +×ŀר ×ķ +Ġj ÄĻ +ĠjÄĻ zy +ĠjÄĻzy k +Ġ׼ ×ŀ×ķ×ijף +×¢ ׾×Ķ +à¸Ĺีà¹Ī à¹Ħà¸Ķà¹ī +ãģ¾ ãģĹãĤĩãģĨ +×ŀס פר +Т Ðŀ +سÙĬاس Ø© +Ġкажд Ñĭй +ë² ł +t ım +y á»ĩn +ร ีà¹Ī +ĠдеÑĤ Ñģк +วิà¸ĺี à¸ģาร +m ówi +×ĺ×¢ ×Ŀ +×Ķצ׾ ×Ĺ×Ķ +ض ÙĬÙģ +ĠÑħоÑĤ Ñı +ãĤĵãģ§ ãģĦãĤĭ +à¸Ħา à¸Ķ +à¸Ħร à¸ļ +Ġк ÑĥÑĢÑģ +ĠbaÅŁ arı +×ijר ×ķ +ÙĬع Ø© +ĠÐĿ Ñĥ +à¸Ħวาม à¹Ģà¸Ľà¹ĩà¸Ļ +Ġ׾ ×ŀש׾ +Ġì¢ĭ ìĿĢ +Ùħؤس س +Ùħؤسس ات +Ġpréc is +Ġth ảo +à¸ģà¹ĩ à¸Ħืà¸Ń +Ġש ׼׾ +führ ung +ãģĦ ãģ§ +à¹ģละ มี +à¸ģà¹ĩ มี +Ġש ש +м ел +Ġкни г +ĠباÙĦ ÙĨ +ĠباÙĦÙĨ سبة +Ġald ı +ÑĤ ай +Ġ×Ĺ×ĵ ש×Ļ×Ŀ +å®Ł ãģ¯ +ع ÙĪØ§ +ĠìĿĺ 미 +из м +ÑĢабоÑĤ аÑĤÑĮ +Ùģ Øµ +Ġ×ij׳ ×ķסף +ãģ¨ãģĹãģ¦ ãĤĤ +à¹Ģà¸Ľà¹ĩà¸Ļ à¸Ĺีà¹Ī +ĠÑģлед ÑĥеÑĤ +èĢĥãģĪ ãģ¦ +Ġ׼ ×Ļ×ķ×Ŀ +ÑģÑĤ Ñĭ +׼׾׼ ׾×Ļ +æµģ ãĤĮ +ãĤĴ ãģ¤ãģij +Ñĩ аÑĤ +×Ļ׼ ×ķף +×Ļר ×Ļ +ları yla +ãĤ¤ ãĥ¡ +ãĤ¤ãĥ¡ ãĥ¼ãĤ¸ +׳×ĸ ×§ +Ġci ò +Ġs ın +Ġsın ır +à¸Ļ à¸Ħร +к аÑĤ +Ġl á»Ĺi +ëŀ Į +تÙģ Ø§Øµ +تÙģØ§Øµ ÙĬÙĦ +ëĨ ĵ +ĠÙħ ض +il miÅŁ +بار Ùĥ +ÐĿ Ðĺ +Ġth ẩm +Ġ×IJ×ķת ×ļ +ĠпÑĢин им +ĠпÑĢиним а +Ġyö nt +Ġyönt em +Ġ×ŀ×§ ×ij׾ +Ġktó rego +ê· Ģ +شر Ùģ +د اÙħ +ãģĦãĤį ãģĦãĤį +ĠAl ém +Ġgör ü +Ġgörü nt +Ġgörünt ü +د س +ÑĪ ÐºÐ¸ +г ÑĢад +Ġl ạc +Ġs ữa +ãĤīãĤĮ ãģ¾ãģĻ +o Ãłi +Ñī ен +ãģĭ ãģªãģĦ +Ġп оп +Ġпоп Ñĥ +ĠпопÑĥ лÑıÑĢ +ĠاÙĦÙħ ÙĪÙĤع +rä g +ï¼ ¡ +íķ Ħ +ãĤĴè¦ĭ ãĤĭ +اÙħ ا +ĠاÙĦØŃ رب +ĠÐŁ а +Ġ׾ ×IJתר +Ġt á»ijc +×ij ׾×Ķ +ر ئÙĬس +в Ñĥ +ÙĬ دÙĬ +каз ан +Ġ׊ש×ij×ķף +h ôtel +×¢ ×ķ׳×Ķ +ب ÙĨÙĬ +×ŀ ×ķ׾ +Ġд нÑı +éĽ£ ãģĹãģĦ +вед ениÑı +Ġ×ķ ×ŀת +н апÑĢимеÑĢ +ÙĤ ابÙĦ +Ġrésult at +ĠÑĢазвиÑĤ иÑı +ر Ùij +ìłĦ 문 +ĠاÙĦÙħ زÙĬد +ĠìľĦ íķ´ìĦľ +ëĨ į +íĻ ķ +ĠThi ết +íĮ ¨ +malı dır +Ġcz ÅĤ +ĠczÅĤ owie +ĠczÅĤowie k +ĠÙĦ بÙĨ +ĠÙĦبÙĨ اÙĨ +üs ü +ãģªãĤĵ ãģł +Ġżyc ie +ĠÑħоÑĢоÑĪ Ð¾ +æĸ¹ ãģ« +ëĭ¤ ë©´ +иÑĩеÑģ каÑı +ער ×Ļ׼ +ער×Ļ׼ ת +ãģ¾ãģĽãĤĵ ãģ§ãģĹãģŁ +ĠÑģоб ой +Ġg á»Ĺ +Ġдел аÑĤÑĮ +da Äĩ +аÑĢ Ð° +róż ni +à¹Ģล ีà¹ī +à¹Ģลีà¹ī ย +à¹Ģลีà¹īย à¸ĩ +à¸Ŀ าà¸ģ +Ġت ÙĤ +ĠتÙĤ دÙĬ +ĠتÙĤدÙĬ Ùħ +หà¸Ļ ุà¹Īม +Ġmü cade +Ġmücade le +ì§Ģ 를 +ãĤ¤ ãĤ¹ +ĠØ£ ساس +jÄħce go +ĠÅŁ eh +н ÑĤеÑĢ +ÑĨи Ñİ +ï» » +ÑİÑī его +à¹Ĥà¸Ľà¸£ à¹ģ +à¹Ĥà¸Ľà¸£à¹ģ à¸ģรม +Ġmie Äĩ +ØŃÙĥÙĪÙħ Ø© +ãģ§ãģĹãģŁ ãģĮ +×Ļס ×Ķ +ãĤĤãģ® ãĤĴ +Ġ×ŀ ×IJת +สุà¸Ķ à¸Ĺà¹īาย +Ġc Å© +ÙĨ سب +ĠпÑĢ Ð¾Ñĩ +Ġд ней +ĠÑįÑĤи Ñħ +׾ ×ŀת +нÑı Ñı +Ñį к +Ġì§Ģ ëĤľ +มหา วิà¸Ĺยา +มหาวิà¸Ĺยา ล +มหาวิà¸Ĺยาล ัย +d ão +ĠMá y +ĠêµŃ ê°Ģ +à¸ļุ รี +×Ĵ ×Ļ׾ +ĠÑĤÑĭ ÑģÑı +ĠÑĤÑĭÑģÑı Ñĩ +Ùģ Ùĥ +ĠÐĺ Ñģ +è¡Į ãĤıãĤĮ +פר ×ĵ +ãģ¤ ãģį +à¸Ħร à¸Ńà¸ļ +à¸Ħรà¸Ńà¸ļ à¸Ħรัว +à¸Ĥึà¹īà¸Ļ มา +ä»ĬæĹ¥ ãģ¯ +ĠìĤ¬ëŀĮ ìĿ´ +עצ ×ŀ×Ķ +п оÑĢ +ĠK ỳ +Ġ Æ¡n +Ġth Äĥm +Ùģ Ø§ÙĤ +ãģļ ãģ« +Ġ׾ קר +Ġ׾קר ×ķ×IJ +اÙģ ÙĬØ© +Ùħ ÙİØ§ +г аÑĢ +ص ÙĦا +صÙĦا Ø© +Ġ×ŀ ×ĸ×Ķ +lı ģını +Ġ×IJ ×Ļ׳×Ķ +к ÑĢо +Ġng ươi +Ġв ним +Ġвним ание +jÄħ cy +ÙĢÙĢÙĢÙĢ ÙĢ +Ñģ Ñħод +ãģªãĤĵ ãģĭ +×ŀ ×Ļ׾ +Ġ×Ķ×IJ ×Ĺ +ãĤı ãģªãģĦ +ع سÙĥر +ĠìĦ¸ ê³Ħ +ĠÑĩ его +ĠÑģÑĢед ÑģÑĤва +ĠÐł аÑģ +ãģª ãģģ +ÙĨ Ù쨳 +ר×Ļ ×ķף +Ñģ Ñĥд +ĠìĿ¸ ê°Ħ +ĠاÙĦÙħ ÙĤبÙĦ +ÙĨ عÙħ +تÙĪ Ù쨱 +ש ×ij×¢ +ı lm +ılm Ä±ÅŁ +Ġ×ľ×ª ת +تص Ùģ +×Ķפ ×ķ×ļ +à¹ĥà¸Ļ à¸Ľà¸µ +ìĿ´ ê³ł +Ùģ ÙĪØ² +à¸ľà¸¥ à¸ĩาà¸Ļ +ĠGi áo +à¸ļà¸Ńà¸ģ วà¹Īา +Ġd Ä±ÅŁ +ĠdÄ±ÅŁ ında +ì£ ½ +Ġdzie ÅĦ +к ÑĨии +и ÑĨе +ãģ® ä¸Ģ +ع Ø´ +пÑĢ ÐµÑģÑģ +หà¸Ļ à¹Īà¸Ńย +ลัà¸ģษ à¸ĵะ +Ġpossibilit Ãł +à¹Ħà¸Ķà¹īรัà¸ļ à¸ģาร +หย ุà¸Ķ +Ġphi ên +çĶŁ ãģ¾ãĤĮ +Ø· ÙĪÙĦ +ÑĦ ин +f ür +ØŃ ÙĬاة +íĸ ĪìĬµëĭĪëĭ¤ +׼ ׳×ķת +à¸Ľà¸£à¸° ส +à¸Ľà¸£à¸°à¸ª à¸ļ +à¸Ľà¸£à¸°à¸ªà¸ļ à¸ģารà¸ĵà¹Į +ëIJĺ ìĹĪ +Ġkaż dy +Ġl uyá»ĩn +ĠоÑĢганиз аÑĨии +å°ij ãģªãģı +ÑģÑĤÑĢо ен +Ġtécn ico +×§ ×Ķ׾ +Ġ×ķ×IJ ×Ĺ +ĠعÙĦÙĬ Ùĥ +Ñī ение +Ġ×Ķ ×Ļ׾×ĵ×Ļ×Ŀ +ÙĪØ³ ائÙĦ +Ġ×ķ ×Ķת +تÙħ ÙĬز +ĠÑģ казал +Ġпол и +Ġ×Ķ×ŀ ס +ÙĦÙij Ùİ +Ùħؤس سة +Ġ×ŀ ×Ļ×ĵ +ãģ£ ãģ¡ +ĠëĦĪ ë¬´ +à¸ŀ ี +Ġt ặng +Ġt ấn +ר ש×Ŀ +Ġméd ica +Ġ×¢ ×ķ×ŀ +Ġ×¢×ķ×ŀ ×ĵ +ÑĦ оÑĢ +Ùħر Ø© +Ġvat anda +Ġvatanda ÅŁ +Ġдел о +à¸Ļ ม +ãģ¨ åIJĮãģĺ +Ùģ Ùī +Ñģ оÑĢ +Ġ×Ķס ר×ĺ +Ġép oca +ìłķ ì±ħ +ĠÑģвÑıз ан +ض رب +ĠÙĦ ÙĨا +Ġuży wa +ĠاÙĦج ÙĬØ´ +Ñİ ÑĢ +×ijס ×ķ×£ +Ġм Ñĥ +ĠмÑĥ зÑĭк +bilit é +Ġma ç +س Ùİ +ت ÙĦÙĥ +ãģ ¬ +ÙĬ ÙĦا +ÑĪ Ð»Ð° +ÙĢÙĢ ÙĢ +Ġод ной +зв ан +ĠÑģ ÑĢаз +ĠÑģÑĢаз Ñĥ +ÙĨ ظÙħ +را Ùĩ +ĠÙĦÙĩ ذا +׼ ×ķר +Ġ×Ķש ×ij×ķ×¢ +Ġ×Ķש ת +ĠQu ảng +ãĥ« ãĥ¼ +ãģĪ ãģªãģĦ +×ĺ ×IJ +Ġmi á»ģn +ĠPh áºŃt +ĠاÙĦس ÙĪÙĤ +Ä Ĥ +ĠاÙĦج Ùħع +ĠاÙĦجÙħع Ø© +ÑİÑī ей +a ÅĤem +عت ÙĤد +Ø£ ÙĦÙħ +Ñģ ке +ĠìĿ´ íķ´ +ÙĨس Ø® +è¨Ģ ãģĦ +д обав +سب ÙĤ +×¢×ķר ר +ÑĤи п +ãģĿãģĵ ãģ§ +vis ión +عÙĪØ¯ Ø© +ë¨ ¹ +×ŀ ×ĸר×Ĺ +ĠØ¥ ØŃ +Ġ׾×ij ×Ļף +Ġ׾צ ×IJת +Ġyard ı +Ġyardı mc +Ġyardımc ı +İ Z +×§ פ×Ķ +tr é +liÄŁ ini +клÑİÑĩ а +Ġüret im +Ġa yrı +ĠkiÅŁ iler +à¸Ħ à¹īà¸Ļ +à¸Ħà¹īà¸Ļ หา +ĠS á»± +Ġ׼ ס +Ġ×Ľ×¡ ×£ +ĠÑĤак иÑħ +ĠXu ân +Ġл ег +Ġлег ко +Ø«ÙĤ اÙ쨩 +ÐĿ Ðŀ +ãĤ¹ãĤ¿ ãĥĥ +ãĤ¹ãĤ¿ãĥĥ ãĥķ +åIJĪ ãģĦ +Ġ×Ķש ×Ļ×ŀ×ķש +man ız +ĠÐĴ аÑģ +g ün +ìľĦìĽIJ íļĮ +Ġwsp óln +ĠÑģв ое +í ĥģ +à¹Ģà¸Ļ ีย +ÙĪØ¨ Ø© +в Ñıз +ı dır +ëIJĺ ìĹĪëĭ¤ +ĠdeÄŁi ÅŁtir +ãĤĭ ãģĵãģ¨ãģĮ +Ġ×Ĺ×ĵ ש×Ķ +ãĤīãĤĮ ãģ¦ãģĦãĤĭ +×Ĺ×Ļ ×Ļ×ij +ĠÐļ аÑĢ +׳×Ļת ×ķ×Ĺ +Ġ×§×ĺ ף +ר ×ĸ +ÙĪ Øº +èªŃ ãģ¿ +Ġت ÙĤÙĪÙħ +ĠÙĥ اÙĦ +à¸Ŀ ึà¸ģ +Ġë°ľ ìĥĿ +ológ ico +ر اع +à¹ģà¸ģà¹ī à¹Ħà¸Ĥ +ĠÑĢабоÑĤ Ñĥ +ÙĨÙij Ùİ +à¸Ńยูà¹Ī à¸Ĺีà¹Ī +ĠاÙĦØ« اÙĨÙĬØ© +ĠNh ân +Ñħ ваÑĤ +ö ne +Ġع دة +à¹ģ สà¸ĩ +ÑĤ оп +пÑĥÑģ ка +شر اء +ĠÐļ ом +Ġפע ×ķ׾×Ķ +ìĤ¬ ìĿ´ +ìĤ¬ìĿ´ íĬ¸ +è¡Į ãģ£ãģ¦ +Ġ×Ķ ×Ķת +ĠÑģÑĤ оÑĢо +ĠÑģÑĤоÑĢо нÑĭ +در س +à¸ĭ ู +à¸ķà¹Ī ำ +ĠØ£ بÙĬ +под об +ãģ« ãģ¦ +ار تÙģØ§Ø¹ +ĠÙħ ؤ +ик ов +ge führt +มืà¸Ń à¸ĸืà¸Ń +ĠÙĦ ÙĤد +ĠØ£ÙĨ Ùij +سÙĬ طر +ãģ¾ãģļ ãģ¯ +ס ×ĵ +Ñģк олÑĮко +ãģ¿ãģŁãģĦ ãģª +×ĵר ×Ĵ +×¢ ×Ļ×ĵ +à¹ĥหà¹ī à¸ļริà¸ģาร +ĠÐĶ Ð¸ +×ij×¢ ×Ļ×ķת +Ġ×Ķ×Ĺ ×ķ +пиÑģ ÑĮ +ĠاÙĦØ® ÙĦ +б ав +Ġİ lk +ĠاÙĦØ® Ùħ +ĠاÙĦØ®Ùħ ÙĬس +ĠÙĬ ÙĤÙĪÙħ +æĻĤ ãģ® +ĠsÅĤ ow +ĠØ£ ÙĩÙħ +Ø®ÙĦ ÙĤ +ĠØ£ صبØŃ +Ġchứ a +Ġth ác +Ùģ Ø§ÙĦ +Ġch á»Ŀ +ĠاÙĦØ® ار +ĠاÙĦخار ج +ĠاÙĦخارج ÙĬØ© +Ø· ائر +Ġt Ãł +ĠtÃł u +à¸ģล à¹īà¸Ńà¸ĩ +ĠاÙĦÙħر Ø£ +ĠاÙĦÙħرأ Ø© +åħ¨ ãģı +ĠÃĸ n +çļĦ ãģ«ãģ¯ +Ġpiè ce +×Ĵ ×Ļ×ij +ĠاÙĦ ÙĪØ§ÙĤع +ä»Ĭ ãģ® +ĠاÙĦÙħ ÙĤ +cz nÄħ +Ù쨹 اÙĦ +ен ного +ĠÑĦак ÑĤ +ìĭł ì²Ń +ĠÐŀ ни +ĠاÙĦبÙĦ اد +ов иÑĩ +ëı Į +ÑĦ ÑĥнкÑĨи +Ġìĸ´ ëĬIJ +ãĥķãĤ© ãĥ¼ +d ÃŃ +ил оÑģÑĮ +Ùħ Ùī +ĠاÙĦØ£ÙħرÙĬ Ùĥ +ĠاÙĦØ£ÙħرÙĬÙĥ ÙĬØ© +×ĺ ×Ļפ×ķ׾ +íĶĦ ë¡ľê·¸ +íĶĦë¡ľê·¸ ëŀ¨ +Ġש ×ķ׳×ķת +Ø´ ÙħÙĦ +ĠпаÑĢ Ð° +Ġ×Ķ×Ĺ ×ķ×§ +ÙĪØ² ارة +ãģ¨ ãģĻãĤĭ +Ġqu ảng +ĠaÄŁ ır +ĠاÙĦÙĦ ج +ĠاÙĦÙĦج ÙĨØ© +ê¸ ´ +ĠT ân +ج ÙħÙĦ +д ол +à¹ģà¸ŀ à¸Ĺย +à¹ģà¸ŀà¸Ĺย à¹Į +Ġר×IJ ש×Ļ +Ñī ей +Ġçev re +Ġкомп лекÑģ +Ġ×ij ×ŀש×ļ +Ġalt ın +ĠØ£ عÙħاÙĦ +ĠÑģво его +ãĤĪ ãģĦ +×Ĺ׾ ×Ļ×ĺ +×ŀ׳ ×¢ +Ġר ×ij×Ķ +ĠØ£ÙĬضا Ùĭ +×ĸ ׾ +ĠاÙĦسÙĬ اسÙĬ +æĢĿ ãģĨ +קר ×§ +קרק ×¢ +ĠاÙĦÙģ Ø±ÙĬÙĤ +б иÑĤ +×§ ׳×Ķ +ĠØ¥ ÙĨÙĩ +ĠÐĴ ам +Ðł Ðŀ +ãĥĪ ãĥª +å¿ħè¦ģ ãģª +Ġch âu +ç¶ļ ãģij +Ġçöz üm +gÅĤ ow +ع ÙĤÙĦ +売 ãĤĭ +i ết +à¸Ĭิ à¹īà¸Ļ +ĠØŃÙĤ ÙĪÙĤ +Ø·ÙĦ ع +ĠÄij en +ĠÙĥ اÙ쨩 +ãģ® ãģĶ +Ġë ¬ +Ġë¬ ¼ +Ġ물 ë¡ł +Ġرس ÙĪÙĦ +з ам +зам ен +Ġkullan ıcı +×¢ ×ķ׾ +èī² ãĢħ +ÑĪи ÑĢ +Ġ׊ש +Ġwy gl +Ġwygl Äħda +ש ×Ļ×ŀ×ķש +å¿ĺ ãĤĮ +×¢ ×Ļצ×ķ×ij +ĠاÙĦس ÙĪØ±ÙĬ +å°ij ãģªãģĦ +Ġпо иÑģк +สำ à¸Ļัà¸ģà¸ĩาà¸Ļ +Ġ×ŀצ ×ĵ +Ġmü ÅŁ +ĠmÃ¼ÅŁ ter +ĠmÃ¼ÅŁter i +ĠÙħÙĨ ÙĩÙħ +à¸ķำ à¹ģ +à¸ķำà¹ģ หà¸Ļ +à¸ķำà¹ģหà¸Ļ à¹Īà¸ĩ +ÅĽ mie +Ġש ×ł×ª +Ġ×Ķ ×¤×Ļ +פר ש +×¢×ijר ×Ļת +สà¸Ļ ัà¸ļ +สà¸Ļัà¸ļ สà¸Ļุ +สà¸Ļัà¸ļสà¸Ļุ à¸Ļ +è¨Ģ ãģ£ãģ¦ +à¸ģาร à¸Īัà¸Ķ +ĠMo że +из аÑĨии +ứ t +ĠÙĪØ¨ عد +ĠdeÄŁ ild +ĠdeÄŁild ir +Ġת ×ŀ +Ġ×ŀ×ŀ ׳×ķ +話 ãĤĴ +ĠÑĨ ена +Ġth úc +×Ļ×ŀ ×ķף +ĠB áo +ãĤĴ åıĸãĤĬ +å®ī ãģĦ +Ġ×¢×ķש ×Ļ×Ŀ +èĩªåĪĨ ãģĮ +l ée +ãĤĭ ãģ®ãģ§ +иÑĢÑĥ еÑĤ +ãģ¦ ãĤĭ +ست ر +ĠاÙĦØŃ ÙĬ +×Ļ׾ ×ķת +Ġ×Ĺ ×ij +ÙĤر Ø£ +تÙħ ÙĥÙĨ +س ائÙĦ +prü f +ãģĭ ãģijãģ¦ +ĠÑģоб ÑģÑĤвенно +ĠìľĦ íķĺìŬ +׾ ×Ļ×ĺ +ãģĮ å¤ļãģı +ÙĬت Ùĩا +ç«ĭ ãģ¦ +ม à¸Ńà¸ļ +ìĭľ ìŀ¥ +оÑĢ Ð° +Ġs avaÅŁ +×ĺ×Ļ×ij ×Ļ +×ij ׳×ķ +Ùħا ذا +기 ê°Ħ +ãģªãģ© ãģ§ +Ġ×ŀ ת×Ĺ×Ļ׾ +Ġnhi á»ħ +Ġnhiá»ħ m +ка ÑĢ +каÑĢ ÑĤ +Ġ׾×Ķ ×©×ª×ŀש +׳ ×Ļ×Ĺ +اد ÙĬØ© +ราย à¸ĩาà¸Ļ +Ġprzy kÅĤad +Ñī ий +ØŃض ÙĪØ± +Ġh ôn +à Ŀ +ת ×ķצ×IJ×ķת +راب Ø· +Ġb ếp +ĠполÑĥÑĩ и +åĩºä¼ļãģĦ ç³» +à¸Ľà¸¥ à¹Īà¸Ńย +ĠاÙĦØ´ باب +اÙĩ ÙĦ +ä»Ĭ ãģ¾ãģ§ +رج ع +ãĤ¶ ãĥ¼ +ÙĤ Ùģ +ĠGro ÃŁ +ĠíļĮ ìĽIJ +اج ر +Ġ×ij×ŀ קר×Ķ +Ġseg urança +fü hl +ãģ¦ ãģĦãģı +หม à¸Ń +ĠкоÑĤоÑĢ Ð¾Ð¼ +ĠN Äĥm +ĠdÅĤ ugo +ÙħÙĨ ØŃ +ש×ķ ×ķ×Ļ +ĠØ£ÙĬ اÙħ +ส à¸łà¸²à¸ŀ +r zÄħ +شر Ùĥات +ãĤĴ èĢĥãģĪ +д аÑĢ +à¸Ľà¸£à¸° à¸Ĭุม +Ġ×ķ×IJ ×ĸ +i á»ĩn +Ġt ươi +ש ×Ļ×Ĺ +à¸Ń à¹Īà¸Ńà¸Ļ +æĽ¸ ãģĦãģ¦ +Ġng ữ +×ij×Ļ×ĺ ×Ĺ +×ij×Ļ×ĺ×Ĺ ×ķף +Ġs ẵ +Ġsẵ n +ì§Ģ ëıĦ +ĠпÑĢ ÐµÐ¿ +ĠпÑĢеп аÑĢаÑĤ +Ġна ÑĥÑĩ +ĠÃľ nivers +ĠÃľnivers ites +ĠÃľniversites i +Ġ×Ĵ×ĵ ×ķ׾×Ķ +Ġ×Ķ ×ł×ª +Ġ×Ķ×ł×ª ×ij×¢ +ãģ§ãģĤ ãģ£ãģŁ +Ġmies iÄħ +ĠmiesiÄħ c +г ÑĢам +гÑĢам м +Ġبش Ø£ÙĨ +ĠÑħ ÑĢ +×§ ×Ļ×ĵ +×§×Ļ×ĵ ×ķ×Ŀ +Ø´ Ùĥر +Ġ á»ķ +Ġá»ķ n +ãģĮãģĤ ãģ£ãģ¦ +ãģķãĤĮ ãģ¾ãģĻ +Ġ×Ĺ ×ķ×ĵ +Ġ×Ĺ×ķ×ĵ ש×Ļ×Ŀ +ÙħÙĪØ§ جÙĩ +ÙħÙĪØ§Ø¬Ùĩ Ø© +أش خاص +ب غ +à¹Ģรียà¸Ļ รูà¹ī +ãģĹãģ¦ ãģĦãģı +Ġs ạn +å¿ħ ãģļ +׳ ×Ļ×Ĵ +׳×Ļ×Ĵ ×ķ×ĵ +باÙĦ غ +׊ש×ŀ +×Ĺש×ŀ ׾ +Ġnap raw +Ġnapraw dÄĻ +Ø´Ùĩ اد +×IJ ×ķ×Ķ +×IJ×ķ×Ķ ×ij +и ÑĨÑĭ +Ġ×Ķ ×¨×Ľ×ij +ëŀ ij +Ġת ×¢ +Ġ×Ķ ×Ļש +Ġ×Ķ×Ļש ר×IJ +Ġ×Ķ×Ļשר×IJ ׾×Ļ +Ø£ ÙħÙĨ +ÑİÑī аÑı +sk ór +LER İ +Ġ×Ķ×IJ×Ĺר ×ķף +×¢ ׳ק +ĠÙĪ ÙĥÙĦ +ãģĵãģĵ ãģ§ +Ġqu án +liÄŁ in +à¸ģà¸İ หมาย +Ø· Ùħ +Ø£ جÙĩ +أجÙĩ زة +ĠEr doÄŁan +ãģ§ ãģĬ +Ġв ÑĢа +ĠвÑĢа Ñĩ +ĠPh ó +à¸Ĭั à¹Īว +à¸Ĭัà¹Īว à¹Ĥม +à¸Ĭัà¹Īวà¹Ĥม à¸ĩ +Ġph úc +×Ļפ ×ķת +×¢×Ļ ×ķף +Ġduż o +ãĥģ ãĥ¼ãĥł +ĠÙĬ Ùİ +Ġзад аÑĩ +Ġ×Ĵ×ij×ķ×Ķ ×Ķ +Ġ׼ ׼׾ +лож ен +ét at +Ġng Äĥn +èµ· ãģį +ĠTi ến +ص عب +Ġexperi ência +Ø® Ùħ +à¸ģาร à¸Ĺำà¸ĩาà¸Ļ +س ÙĬد +ĠD á»± +ĠкоÑĤоÑĢ Ð¾Ð³Ð¾ +lad ıģı +Ġkh á»ķ +Ġê³Ħ ìĨį +Ñī ик +สà¹Īวà¸Ļ à¸ķัว +з оÑĢ +ÙĨ Ùı +Ġ à¸Ķัà¸ĩ +Ġà¸Ķัà¸ĩ à¸Ļัà¹īà¸Ļ +Ġc ấu +ĠÄij á»ijc +о ÑĦ +ĠاÙĦØ£ عÙħاÙĦ +ãģªãģı ãģ¦ãĤĤ +×ķ׼ ×Ļ×Ŀ +à¹ģ à¸Ľ +ĠB ên +ãĥ¯ ãĥ³ +Ġgi ám +ĠÅŀ u +Ġd áng +ع ÙĦÙĬ +à¹Ģà¸ģ ษ +à¹Ģà¸ģษ à¸ķร +ÙĪØ¬ ب +н нÑĭе +ÙĤ ضاء +à¸Ħว à¸ļ +à¸Ħวà¸ļ à¸Ħุ +à¸Ħวà¸ļà¸Ħุ ม +ãģ¤ ãģ¤ +ĠVi á»ĩc +×ŀ×ij ×ĺ +ש×Ļת ×ķ×£ +Ġв едÑĮ +k aza +kaza ÅĤ +à¸ķำ รวà¸Ī +ãĤ¿ ãĥ« +Ġпов Ñĭ +ĠповÑĭ ÑĪен +ĠS ợ +ĠìĦ¤ ëªħ +ĠÃĩ ünkü +ìĥĿ íĻľ +Ö ¾ +ãĤĮ ãģ¦ãģĦãĤĭ +Ġ×ij ר×IJש +ר ×ķ×Ĵ +Ġо ÑĦи +ĠоÑĦи ÑĨиалÑĮн +ĠÑĥ ÑģÑĤанов +ĠÑĥÑģÑĤанов лен +ĠاÙĦÙħ صر +ĠاÙĦÙħصر ÙĬØ© +ĠÐŁÐ¾ ÑįÑĤомÑĥ +ÙĨ صÙģ +ĠÙĪØ§ÙĦ ÙĨ +Ġh Ãłi +à¸Ħ ิ +ĠApr ès +ì³ IJ +à¹Ģà¸ĭ ีย +×ĵ ×ŀ×Ķ +activ ité +à¸Ħิà¸Ķ วà¹Īา +ÑĤ ÑĢен +à¹Ģ ฮ +ãĥı ãĤ¤ +ãģĮ å¢ĹãģĪ +ен наÑı +Ġìĺ¤ ëĬĺ +ãĥ¢ ãĥ³ +Ġкон еÑĩно +ĠÙħÙĤ ابÙĦ +cl é +Ġh ü +Ġth ẳng +ìłģ ìĿ´ +ĠÐIJ лекÑģ +ĠÐIJлекÑģ ан +ĠÐIJлекÑģан дÑĢ +ãĥŀãĥ³ ãĤ·ãĥ§ãĥ³ +ãģ²ãģ¨ ãģ¤ +ãģª ãģĬ +à¹Ģà¸Īà¹īา à¸Ĥà¸Ńà¸ĩ +ëĵľ 리 +Ø´ اء +ĠsaÄŁ lık +ĠÅŁ imdi +×Ļ×IJ ׾ +تأ Ø«ÙĬر +Ø£ سب +أسب اب +ĠвÑĭполн ен +л ок +ש ×Ļ×ij×Ķ +Ġl ắm +ĠTr Æ°á»Ľc +Ġ×Ķ×¢ ׾ +리 를 +ĠÑĢ ÐµÐ¶ +ĠÑĢеж им +int é +inté gr +×Ĵ ׳×Ļ +ĠاÙĦØ´ عر +Ġmil hões +Ġpeque ño +ãĤ³ ãĥ¼ãĤ¹ +×ķ׼ ×Ĺ +à¹Ģà¸Ĭ à¹īา +شر ÙĤ +Ġh ương +รัà¸IJ à¸ļาล +à¸ģล าย +à¸ģลาย à¹Ģà¸Ľà¹ĩà¸Ļ +Ġпод Ñħод +תש ×ķ×ij×Ķ +ãģıãģª ãģ£ãģ¦ +ĠاÙĦØ£Ùħ Ùħ +ĠH á»įc +ĠwspóÅĤ pr +ĠwspóÅĤpr ac +Ñĩ Ñĥв +ÑĩÑĥв ÑģÑĤв +ÃŃst ico +à¹Ģà¸ģ าะ +ìĽ Ģ +Ġназ ад +ãĤĭ ãĤĪãģĨãģ« +ĠС Ш +ĠСШ ÐIJ +м он +ĠAs ÃŃ +×ķר ×Ĵ +полн ен +×ŀס ׾ +×ŀ×¡×ľ ×ķ׾ +à¹Ģลืà¸Ń à¸Ķ +à¹Ģริà¹Īม à¸ķà¹īà¸Ļ +ĠاÙĦØ¥ Ùħ +ĠاÙĦØ¥Ùħ ارات +צ×Ķ ×¨ +ãĥ¡ãĥª ãĥĥãĥĪ +ĠпоÑĤ ом +в из +ĠÙģ ØªØ±Ø© +å¾Į ãģ® +ÐĿ ÐIJ +×ŀס ר +ÙĬر ÙĬ +pr é +Ġte ÅŁek +ĠteÅŁek kür +Ġöd eme +د اÙĨ +ãģ¾ ãģĹãģ¦ +缮 ãģ« +ĠÑĤ еÑĩение +l ard +lard ır +à¹Ģรา à¸Īะ +ס פ×Ļ +ĠÙĪÙĥ ذÙĦÙĥ +Ġh át +Ġt á»Ļc +à¸Ħุ ย +Ġb ức +ØŃ ÙĬÙĨ +èģŀ ãģĦãģ¦ +Ùħؤ شر +ĠNh ư +Ġмен ее +ละ à¸Ħร +Ñģ ин +ĠÑĢ ÐµÐº +ĠÑĢек л +ĠÑĢекл ам +ĠÙģ ÙĩÙĪ +Ġ׾ ×ĸ +×Ļ׳ ×ķת +ĠÅŁ art +ÑģÑĤав ка +Ġíı¬ íķ¨ +ãģ«è¡Į ãģı +ï¼ Ŀ +ĠпозволÑı еÑĤ +Ġת×ķ׼ ׾×ķ +ов ал +صÙĦ Ø© +Ġ׾ש ׳×ķת +ĠÐĺ гÑĢ +ÙħÙĨتج ات +Ġsat Ä±ÅŁ +Ñģ ко +ĠاÙĦØ«ÙĦاث اء +Ġ×Ķ×ĵ×ijר ×Ļ×Ŀ +ãģĹãģ¾ ãģĹãĤĩãģĨ +بÙĤ Ùī +åĬĽ ãĤĴ +ĠÃĩ ok +ãĥģ ãĥ¥ +à¹Ģà¸Ĭ ืà¹īà¸Ń +ยุ à¸Ħ +ศา ล +Ġ×§×ķ×ĵ ×Ŀ +×ĸר ×Ļ×Ŀ +ãģ® åł´åIJĪ +ĠìķĬ ìķĺ +ãģĤãĤĬãģ¾ãģĻ ãģĮ +×IJ שר +è¡Į ãģı +ãģ» ãģĭ +æ°Ĺ ãģ«ãģªãĤĭ +й деÑĤ +íķĺìĺĢ ëĭ¤ +ستÙħر ار +ĠÐŁÑĢ Ðµ +ĠÑģ боÑĢ +ĠìķĦ 무 +ç§ģ ãĤĤ +ع ص +Ġн иÑĩ +ĠниÑĩ его +ĠпÑĢи ем +×§ ×ķ×ŀ +ĠìĪĺ ëıĦ +Ġì ¡´ +Ġì¡´ ìŀ¬ +ĠØ£ Ø«ÙĨ +ĠأثÙĨ اء +ĠÙĪØ§ÙĦ ØŃ +ãģĮ ãģ§ãģįãĤĭ +Ġת ×Ķ +Ġת×Ķ ×Ļ×Ķ +ר ף +ĠÑģвÑıз и +×Ĵ שת +Ñģп екÑĤ +ס ×ij×Ļ×ij +ס×ij×Ļ×ij ×Ķ +ĠíķĦìļĶ íķľ +ت خصص +Ġж ив +Ġжив оÑĤ +ĠMay ıs +تع ا +تعا ÙĪÙĨ +ĠعÙĨ Ùĩا +ów ki +ĠاÙĦÙģÙĦسطÙĬÙĨ ÙĬ +ãģłãģijãģ§ ãģªãģı +ìĿ¸ ì§Ģ +ĠاÙĦس ÙĪØ¯ +ĠاÙĦسÙĪØ¯ اÙĨ +إجراء ات +Ġkö tü +Ġ×Ļ ×ª×¨ +×Ĵ ×Ļש×Ķ +Ġצ ×ķר×ļ +รà¸ĸ ย +รà¸ĸย à¸Ļà¸ķà¹Į +Ñħ оÑĤ +Ðł ÐIJ +ÙĪ Ø·ÙĨ +Ġsay ısı +ס ×Ĺר +Ùħ ÙĪÙĦ +ãĤĴæĮģ ãģ£ãģ¦ +ع اÙĨ +Ġt á»Ļi +ĠвÑĭ ÑĪе +Ġt ầm +ãĥĪ ãĥ¬ +×Ļצ ×ķ +ม ุม +س ÙĪØ¯ +ìłĦ ìŀIJ +ãĤµ ãĥŃãĥ³ +ìĤ° ìĹħ +ĠоÑģнов ан +Ø® Ù쨶 +רצ ×Ķ +بÙĬ ض +×ķÖ ¹ +ס×Ļ ×Ļ×¢ +Ġש ×IJ×Ļ +ĠاÙĦÙĤر Ø¢ÙĨ +ĠТак же +×ŀש ×ŀ×¢×ķת +س ÙĩÙĦ +Ġ×Ķ ×ł×Ķ +ãĤĴ ãģĹãģ¦ãģĦãĤĭ +×Ļ ×Ļס +×Ķ ×ķ×IJ +ĠB ÃŃ +Ġмал о +ĠëͰëĿ¼ ìĦľ +Ġר ×Ĺ×ij +ãģĮ é«ĺãģĦ +ÙĪ Ø§Ø³ +ìĤ ¼ +׳ ×¢ +ãģ£ ãģ¡ãĤĥ +ĠT üm +à¸Ńีà¸ģ à¸Ķà¹īวย +ãģĹãģ¦ ãģıãģłãģķãģĦ +ÙĨØ´ اط +ãĥĹ ãĥ©ãĥ³ +али ÑģÑĮ +×ĵ ×ľ×ª +Ġwc zeÅĽ +ĠwczeÅĽ niej +ĠÑįÑĤ им +Ġthá»ĭ t +à¸ļ ัà¸į +à¸ļัà¸į à¸Ĭี +ãģļ ãģ£ãģ¨ +ÑĢ Ð¸Ð½ +Ġswo jÄħ +íķĺëĬĶ ëį° +Ġë§Įëĵ¤ ìĸ´ +تش Ùĥ +تشÙĥ ÙĬÙĦ +ائ Ùĩ +Ġ׾פ ×Ĺ×ķת +ãĥĭ ãĥ¥ +ãĥĭãĥ¥ ãĥ¼ãĤ¹ +׼×IJ ף +ãģ§ãģį ãģŁ +зв он +Ġsta ÅĤ +×Ĺ×ijר ת×Ļ +ĠØ£ عÙĦÙĨ +à¹ģà¸ļà¸ļ à¸Ļีà¹ī +بد Ø¡ +ãĤģ ãģŁ +Ġ×ŀש ×ŀ×¢×ķת +Ġ×ŀש×ŀ×¢×ķת ×Ļ +ör ü +Ġh ạnh +z ähl +ĠL ý +Ġ×ij ×Ķת +Ġ×ij×Ķת ×IJ×Ŀ +б аÑĢ +ì¦ Ī +ä»ĬåĽŀ ãģ® +Ġy ü +Ġyü ks +Ġyüks el +ãĤ½ ãĥ¼ +ãģĤ ãĤĮ +ת ׾×ŀ×Ļ×ĵ +ãģ¤ ãģª +×ij ׳×Ļ×Ŀ +Ġx ếp +ĠмÑĥж Ñĩин +ĠاÙĦÙĥ تاب +׼ ×ŀ×ķת +Ġç e +Ġçe ÅŁ +ĠçeÅŁ it +ĠçeÅŁit li +×ĵ ×Ļר×ķת +à¸ļุ à¸į +ĠاÙĦØ¥ ÙĦÙĥ +ĠاÙĦØ¥ÙĦÙĥ ترÙĪ +ĠاÙĦØ¥ÙĦÙĥترÙĪ ÙĨÙĬ +ĠباÙĦØ¥ ض +ĠباÙĦإض اÙ쨩 +Ġyö nel +Ġyönel ik +mys ÅĤ +à¸Ķà¹īวย à¸ģาร +à¸ģาร à¸Ĺำ +ов Ñĭм +Ø£ زÙħØ© +æİ¢ ãģĹ +íļ ¨ +Ġ×ķ×IJ ×Ŀ +Ġnghi êm +ÑĪ Ð¸Ð½ +ка л +Ġcrian ças +èĩªåĪĨ ãģ§ +Ġн ай +Ġнай ÑĤи +ĠS á»ij +ĠÃ¶ÄŁrenc iler +ãĥ¶ æľĪ +Ñģ ан +ĠJ á +ĠkonuÅŁ ma +شر Ø· +ëĪ Ī +ar rière +ضر ÙĪØ±Ø© +ãĥĶ ãĥ³ +×¢ שר +аÑĢ ÑĮ +جÙħ اع +Ġdé co +Ġ×Ļ×Ķ ×ķ×ĵ×Ļ +à¸ŀ ลาà¸Ķ +ĠÙĬ ÙĥÙĨ +Ġج اÙħعة +Ø· بÙĤ +Ġbo ÅŁ +×ķ ×ķ×IJ +×ŀ×ĵ ×¢ +×§×ij×ķצ ת +פ ×Ļר +jÄħc ym +ÙħØ´ ا +Ùħشا ÙĥÙĦ +צ פ×ķף +Ø¥ ست +×ŀ׼ ר +سÙħ ع +Ġкак ой +ÑĤ воÑĢ +ØŃ ج +Ù쨱 ض +пÑĢав лен +Ġник ак +Ġmi á»ĩ +Ġmiá»ĩ ng +ü ÃŁ +иÑĢов ал +׾ ×ŀ×ķת +次 ãģ® +ÙĦ Ø· +à¸ķ ัà¸Ļ +×Ķ ×ª×Ĺ×Ļ׾ +Ġfoto ÄŁ +ĠfotoÄŁ raf +طر ØŃ +à¸Ńà¸Ńà¸ģ à¹Ħà¸Ľ +Ġy ên +Ġп ок +Ġпок Ñĥп +ĠпокÑĥп а +ÑĨ Ñĥ +Ġкомп ÑĮÑİ +ĠкомпÑĮÑİ ÑĤеÑĢ +ĠاÙĦÙĥ رÙĬÙħ +تص Ùħ +تصÙħ ÙĬÙħ +Ġоказ а +Ġzar ówn +Ġzarówn o +ëĮĢ ì¶ľ +ãĤ»ãĥ³ ãĤ¿ãĥ¼ +Ġjako ÅĽci +æĤ © +æĤ© ãģ¿ +Ø£ÙĨ ÙĪ +Ø£ÙĨÙĪ Ø§Ø¹ +ë¹ ł +Ġìłķ ë§IJ +Ġk ẻ +ĠÑģай ÑĤа +Ġ×Ķ ×¢×¨×ij +Ùĩ ز +pres ión +ĠÑģÑĤ ен +ãģ£ãģ¦ ãĤĭ +Ġhız lı +Ðļ ÐIJ +×ŀשפ ×Ĺת +ĠÙĨ Ùĩا +ĠÙĨÙĩا ÙĬØ© +ãģ¾ ãģĦ +о ÑħÑĢан +ร à¹īà¸Ńย +ล ึà¸ģ +ĠÙĪØ¨ اÙĦ +ãĤĤãģ® ãģĮ +ר׼ ×Ļ×ij +ãĤ¤ ãĥ¤ +س ؤ +سؤ اÙĦ +ĠÙĦØ£ÙĨ Ùĩ +ĠkonuÅŁ tu +Ðļ ÑĥпиÑĤÑĮ +Ġש×IJת ×Ķ +ĠÙĪØ§ÙĦ س +Ġmożliwo ÅĽci +Ġpró b +ëĶ ° +ãģ© ãĤĮ +ĠÐľ ин +ĠоÑĢганиз м +ãģ«å¯¾ ãģĻãĤĭ +ĠPr é +Ġpriv é +ch è +ãģĦãģŁãģł ãģį +สà¸Ļุ à¸ģ +ajÄħ ce +ĠD zi +ĠDzi ÄĻki +ÅĤat w +r än +rän k +æĿ¥ ãģŁ +Ġ×Ķ×Ļ×Ķ ×ķ×ĵ×Ļ +ãĤ¬ ãĥ¼ +ĠÑĢаР´ +ĠÑĢад и +к ÑĤив +Ø£ Ùĩد +Ø£Ùĩد اÙģ +ש ×IJ×Ļר +ãģ¦ ãģĦãģªãģĦ +Ġfr üh +Ġок ол +Ġокол о +Ġreg ião +ĠÑĩиÑģ ле +Ġpon iew +Ġponiew aż +ìĦ¼ íĦ° +Ġb ầu +Ġê · +Ġê· ľ +Ġê·ľ ìłķ +ĠH òa +ĠÑĤ оÑĤ +ãĤĤ å¤ļãģĦ +ĠاÙĦإسÙĦاÙħ ÙĬØ© +ãģĭ ãģĦ +Ñį н +ĠÑĥказ ан +ĠÑĤак ое +ï¼ ³ +ëĮĢ íķĻ +Ġgen iÅŁ +ĠاÙĦØ® ÙĬ +ĠاÙĦØ®ÙĬ ارات +ãĤĴè¡Į ãģĨ +ש ×ŀ×Ķ +ĠLÃł m +ÙĪÙĨ ÙĬ +Ġ×IJ ׾×Ļ×ķ +Ä ĺ +à¹Ħมà¹Ī สามารà¸ĸ +人 ãģ¨ +بر ز +×Ļס ×ķ×ĵ +×Ĵ ׾×Ļ +ĠÙĬ ÙĨا +ĠÙĬÙĨا ÙĬر +ĠкаÑĢÑĤ ин +Ġt ôn +à¹Ģ à¸ģร +à¸Ħ à¸Ķี +Ġ׾×IJ ×ķר×ļ +ãĤĤãĤī ãģĨ +ãģĭ ãģĭãĤĭ +ани и +Ġara ÅŁtırma +ÙĦاØŃ ظ +ãģĦ ãĤĦ +ĠT Ãłi +Ġ à¸Ļà¸Ńà¸ģà¸Īาà¸ģ +Ġà¸Ļà¸Ńà¸ģà¸Īาà¸ģ à¸Ļีà¹ī +ĠÄIJ ảng +ãģ£ãģ¦ ãģįãģŁ +Ġà¸ĭึà¹Īà¸ĩ à¹Ģà¸Ľà¹ĩà¸Ļ +Ġt ả +Ġmożliwo ÅĽÄĩ +ĠS ản +Ġİ ki +Ġc ắt +س Ø£ÙĦ +Ġbak ım +Ø´ ب +à¸ķ ีà¹ī +à¸ŀ ยาย +à¸ŀยาย าม +สั à¸Ľ +à¸ªà¸±à¸Ľ à¸Ķา +à¸ªà¸±à¸Ľà¸Ķา หà¹Į +ë° Ģ +еÑĢ Ñĭ +Ġc ánh +Ġthu ế +ت بع +ãģ«åħ¥ ãĤĮ +Ñİ ÑģÑĮ +íļĮ ìĿĺ +ç°¡ åį +ç°¡åį ĺ +ç°¡åįĺ ãģ« +Ġtr úc +ĠاÙĦÙĥ ÙĪÙĬ +ĠاÙĦÙĥÙĪÙĬ ت +ãĤıãģij ãģ§ãģĻ +ĠÑģв об +ĠÑģвоб од +ĠÑĥÑĩаÑģÑĤ ник +สิ à¹īà¸Ļ +ĠпÑĢо ÑĦеÑģÑģиона +ĠпÑĢоÑĦеÑģÑģиона лÑĮн +Ñģп оÑĢ +×Ĺ ×ķ×ij×Ķ +Ùħع ÙĨÙī +ĠاÙĦÙģ ØªØ±Ø© +สูà¸ĩ สุà¸Ķ +ãĤı ãģļ +ĠÄij è +ĠÄijè n +æ¯Ķ ãģ¹ +า à¸ĺิ +Ġmoż emy +à¹ģ à¸ĭ +à¸Īะ à¹Ħมà¹Ī +Ġs ắp +Ðļ Ðŀ +Ġprá ctica +ÙĪÙĥ اÙĦØ© +è¾¼ ãĤĵãģ§ +ológ ica +Ġе Ñī +ĠеÑī Ñij +تع دÙĬÙĦ +ĠØ£ Ùĥد +Ġצר ×Ļ׼ +Ġצר×Ļ׼ ×Ļ×Ŀ +Ø« Ùħ +Ġк ÑĢÑĥ +ĠкÑĢÑĥ п +×ij×Ļ×§ ×ķרת +Ġì¡° ê¸Ī +ãģ¨ãģį ãģ¯ +Ġb ạc +ĠÑĢаÑģ пол +ĠÑĢаÑģпол ож +ĠÑĢаÑģполож ен +ز ÙĬÙĨ +ĠÐļ ÑĢоме +ĠاÙĦÙĨ ظر +×Ķ ×ķ×ĵ +ĠاÙĦس بت +ã썿ĢĿ ãģĦ +Ġpa ÅĦst +ĠpaÅĦst w +ĠÙĦÙĬ ست +ĠбÑĥд Ñĥ +à¸Ĺัà¸Ļ à¸Ĺี +ร าม +ØŃ صÙĪÙĦ +ãģĹãģ¦ãģıãĤĮ ãĤĭ +ĠاÙĦØ¥ سرائÙĬÙĦ +ĠاÙĦإسرائÙĬÙĦ ÙĬ +ãģĵãĤĮ ãģ¾ãģ§ +ìĤ¬ 를 +Ġs ürü +à¹Ģว à¸Ńรà¹Į +à¹Ģà¸ĭ à¸Ńรà¹Į +Ġutilis é +ĠÑģиÑģÑĤем а +Ġdw ó +Ġdwó ch +Ġpróp rio +Ġëĵ± ìĿĦ +arr êt +ĠЧ а +×IJ×ŀ ׳×ķת +عار ض +à¹Ģà¸ģม สà¹Į +Ġ׾×Ķ ×ij×Ļף +Ġ׾ ×ij×Ĺ +Ġ׾×ij×Ĺ ×ķר +สา à¸Ĥา +ĠÐľÐ¾Ñģк ве +ب عد +ĠاÙĦÙĤر ار +ĠÄIJ á»ĭa +Ġ×Ĺ ×Ĵ +Ùģ ØªØ± +ÙĪÙĨ Ø© +Ġ×Ķ×ĸ ×IJת +å¸Ĥ ãģ® +ãģ» ãģĹãģĦ +Ġ×ij×¢ ×Ļר +ĠÑĤеп еÑĢÑĮ +ìĬµ ëĭĪê¹Į +à¹Ħม à¹Īว +à¹Ħมà¹Īว à¹Īา +à¹Ħมà¹Īวà¹Īา à¸Īะ +×ŀ ×IJ×Ķ +æĥħ åł± +æĥħåł± ãĤĴ +غ ÙĨ +Ġпо Ñı +ĠпоÑı ви +éģİ ãģĶ +تش غ +تشغ ÙĬÙĦ +в ел +Ġ×Ĺ ×ŀ +ãģ¨ãģªãĤĬ ãģ¾ãģĻ +Ġra ÄŁ +ĠraÄŁ men +ãģĭ ãģ©ãģĨ +ãģĭãģ©ãģĨ ãģĭ +ен ко +ì§Ģ ê³ł +Ġ×IJ׾ ×Ļ×Ķ +ĠØ£ ÙĦ +à¸Īำ หà¸Ļ +à¸Īำหà¸Ļ à¹Īาย +nız ı +Ġ׾ק ×Ĺת +Ø£ ÙĩÙħ +Ø£ÙĩÙħ ÙĬØ© +ت غÙĬر +ש ×Ĺר +ס×ķפ ר +×ĵ ×Ļר +èī¯ ãģĭãģ£ãģŁ +×ŀ׾×Ĺ ×ŀ×Ķ +ÑģÑĤв ие +ÑĤ ÑĢаÑĤ +ĠاÙĦØ£ Ø® +ĠاÙĦأخ ÙĬرة +ĠاÙĦØŃ صÙĪÙĦ +Ġcréd ito +צ ×Ļ×¢ +ãĥ¬ ãĥĻãĥ« +بر ÙĬ +ëIJ IJ +ãģł ãģ£ãģ¦ +Ġreal tÃł +س Ù쨱 +×ķ׳ ×ķ +×Ĵ ×ķ×ĵ +×Ĵ×ķ×ĵ ׾ +ฮ า +ãģĹãģ¦ ãģĬãĤĬãģ¾ãģĻ +Ġg Ãł +Ġ׾×ij צע +å¼ķ è¶ĬãģĹ +Ġ×ŀ ×Ļ׾×Ļ +Ġ×ŀ×Ļ׾×Ļ ×ķף +Ùħ در +Ùħدر سة +פ ×ķ×ĺ +à¸Ļà¹īำ มัà¸Ļ +ëģ Ŀ +ع Ùĥس +ĠÙĤ ض +ĠÑĢÑĭ б +خط Ø· +×ŀ×ķס ×ĵ +Ġ׼׾ ׾×Ļ +ĠкоÑĤоÑĢ Ð¾Ðµ +צ×Ļ ×ķף +ĠмеÑģÑĤ а +ãģĭ ãģ¤ +г ÑĢÑĥпп +׾ ×Ļ׾ +ת ×ķ×IJר +ë³µ ì§Ģ +à¹ģà¸ľ à¹Īà¸Ļ +Ġ×ij×¢ ת +æĻĤéĸĵ ãĤĴ +ï¼ £ +ãģ¨ãģĦãģĨãģĵãģ¨ ãģ§ +Ġ׾×Ķ ×§ +Ġ׾ ×ĸ×Ķ +ĠìłĢ ëĬĶ +ĠاÙĦØ¥ رÙĩاب +ĠìŀĪëĬĶ ëį° +ĠÑĤ огда +Ġ×Ķ ×¦×Ļ +×ķ׾ ×ĺ +Ġר פ×ķ×IJ×Ļ +ãģĵãģ¨ ãģ§ãģĻ +ĠÄij ÃŃch +ØŃ ÙĬا +Ġ×Ķ×ŀש ×Ĺ×§ +ãģľ ãģ² +Ġ×ŀ×IJ פשר +ãģ¿ ãģ¾ãģĹãģŁ +ĠاÙĦØ£ÙħÙĬر ÙĥÙĬ +Ùħج تÙħع +Ġس اب +Ġساب ÙĤ +׼ ×Ļ׾ +Ạ¾ +ãĥª ãĤ¹ãĥĪ +Ġì ĥ +Ġìĥ Ī +ĠìĥĪ ë¡ľ +ĠìĥĪë¡ľ ìļ´ +ĠD á»ĭch +à¹Ģหมาะ สม +ĠاÙĦÙĨ بÙĬ +׾ ׾ +ÙĨ ع +Ðĵ лав +Ðĵлав наÑı +Ùħر ض +Ġ×ķ ×ĵ +ت ÙĤÙĬ +تÙĤÙĬ ÙĬÙħ +Ġb ảng +ĠÙģ ÙĤاÙĦ +×¢ ×ŀ×Ļ +д ÑĢа +Ġsu á»ijt +سر عة +Ġc á»Ń +Ġ×Ķ ×Ļ×Ĺ×Ļ×ĵ +سع ÙĬد +à¸Ńา à¸Ĭีà¸ŀ +Ġس ÙĪØ§Ø¡ +ãĤ½ ãĥķãĥĪ +Ġл иÑĩно +ĠÐļ оÑĢ +اÙĩ تÙħ +اÙĩتÙħ اÙħ +à¸Ń à¸Ķี +à¸Ńà¸Ķี à¸ķ +ãģIJ ãĤīãģĦ +Ġiht iya +Ġihtiya ç +ãģ¾ãģ§ ãģ® +ìĭľ ìĬ¤ +ìĭľìĬ¤ íħľ +ÑĢÑĥ ÑĪ +ãĤĦ ãģ£ãģ± +ãĤĦãģ£ãģ± ãĤĬ +к еÑĢ +Ġ ży +Ġży w +кл он +Ġl ượt +à ¾ +да Ñĩи +tür k +غ ÙĪ +ĠигÑĢ Ð¾Ðº +Ġph ê +Ġש ×¢×ľ +ĠاÙĦÙħ دÙĨÙĬ +ĠìŬ룬 ë¶Ħ +ער ×Ļ×Ŀ +Ñħод ÑıÑĤ +Ġx ứ +ÐĹ Ð° +ĠÙģ Ø±Øµ +à¸Īะ à¸Ĺำà¹ĥหà¹ī +íģ ´ +×¢ ×ij×ķר +à¹Ģหลà¹Īา à¸Ļีà¹ī +èĢĥãģĪ ãĤĭ +ÑĢ ÐµÑģÑĤ +н нÑĭй +Ġc ầm +دا Ø®ÙĦ +ĠÙħÙĦÙĬ ار +ĠÐIJ л +ĠвÑĢем ен +à¸Ĭà¹Īวย à¹ĥหà¹ī +ר×Ļ ×ķת +ëĵ ¯ +飲 ãģ¿ +׳ ׾ +שת ×£ +ĠاÙĦسعÙĪØ¯ ÙĬ +u ÃŁ +ìĿ¸ ëį° +ĠìĿ¼ ë°ĺ +ÅĤ ÄĻ +Ġm á»iji +×ŀ ×Ļ׳ +ĠاÙĦØ£ Ø·Ù쨧ÙĦ +Ġçı kan +é cole +×§ ×Ļש +×§×Ļש ×ķר +ĠоÑģ ÑĥÑīеÑģÑĤв +ĠоÑģÑĥÑīеÑģÑĤв лÑı +×ij ×IJר +à¹Ħà¸Ľ à¸Ķà¹īวย +Ġ×¢ ×ķ׾×Ķ +à¸ģà¹ĩ à¹Ħมà¹Ī +ãĥ¢ ãĥĩ +ãĥ¢ãĥĩ ãĥ« +تØŃ ÙĪÙĦ +Ġод ного +ת×Ĺ×Ļ׾ ת +Ġت Ø® +Ġch cia +Ġchcia ÅĤ +ãĥIJ ãĥ³ +èĢħ ãģ¯ +ĠÙħ ØŃÙĦ +Ñģл ож +Ñģлож н +Ġt ÄĻ +Ġçı kt +Ġçıkt ı +ĠC Æ¡ +à¹Ħà¸Ķà¹ī à¹Ģลย +ır ken +à¹Ģà¸Ĥà¹īา สูà¹Ī +ÙħØŃ Ùĥ +ÙħØŃÙĥ ÙħØ© +à¸Ħุ à¹īม +à¸Ļà¹Īา à¸Īะ +лÑİ Ð´ +де ÑģÑı +деÑģÑı ÑĤ +ĠлÑİб ой +تØŃر ÙĬر +צע ×ĵ +Ġе Ñij +ĠاÙĦØŃ ÙĥÙħ +Ġص باØŃ +à¹Ģà¸ļ à¸Ńรà¹Į +Ġróż nych +ги б +ĠÑģ оÑĤ +ĠÑģоÑĤ ÑĢÑĥд +ĠÑģоÑĤÑĢÑĥд ник +ĠобÑĬ ем +פ ×ĺר +ãģĻãģĶ ãģı +ãģ«éĸ¢ ãģĹãģ¦ +в ол +Ø« ÙħاÙĨ +Ġd ần +æĬ ľ +æĬľ ãģij +Ġ×¢ ש +Ġעש ×ķ×Ļ +ס ×ķף +ãģªãģ® ãģ§ãģĻ +ãģ¯ ãģ©ãģĨ +×ŀ×¢ ר×ij +ï¼ ° +Ùħ صر +ÙħÙĨ اسب +ÙħÙĨاسب Ø© +ä¸Ĭ ãģ® +×IJ×Ļש ×ķר +ĠìĦ¤ ì¹ĺ +×ŀ×ĵ×Ļ׳ ×ķת +×ŀר ת +ãĤĭ ãģ®ãģĮ +د Ùİ +ĠاÙĦشر Ùĥات +ìĭľ ê°Ħ +ĠÑĢеÑĪ ÐµÐ½Ð¸Ðµ +ãģĻãĤĭ ãģ®ãģ¯ +ĠìŀIJìĭł ìĿĺ +׾ ×ŀ×ķ +ãģ¨ãģĵãĤį ãģ§ +Ġ×§ צר +Ġmã i +Ġkü ltür +ãĥ©ãĤ¤ ãĥĸ +à¸ľà¸¹à¹ī หà¸įิà¸ĩ +æĻĤéĸĵ ãģĮ +клÑİÑĩ и +diÄŁ iniz +มาà¸ģ à¹Ĩ +تØŃ ÙħÙĦ +Ġh ạt +ãĤ¦ ãĤ£ +п ле +×ŀ ׾×IJ +ÅĤ ó +Ġg á»ijc +Ġ×IJ ×ķ×ĵ×ķת +หว าà¸Ļ +ĠاÙĦ ÙĪØ² +ĠاÙĦÙĪØ² راء +ëĵ¤ ê³¼ +Ġص ØŃ +ĠصØŃ ÙĬÙ쨩 +Ġм м +تد Ø®ÙĦ +Ġpersön lich +Ġز ÙĬ +ĠزÙĬ ادة +ãĤ· ãĤ¢ +Ġng ắn +à¸Ħล ิà¸ģ +Ġs ông +Ġtü ket +Ñį ÑĦÑĦ +ÑįÑĦÑĦ екÑĤ +ש ×Ļ×ij +Ġا عت +ت ض +تض ÙħÙĨ +ĠاÙĦÙħØ´ رÙĪØ¹ +Ġprodu ção +ĠпÑĢимен Ñı +ни ÑĨÑĭ +주 ëĬĶ +ر Ùı +Ġm Æ¡ +Ġhayat ı +ëŁ ½ +Ġü cret +Ġyan ında +Ġpr ática +×ij×Ļ×§ ×ķר +Ãľ N +Ñģ оÑĤ +ãĤıãģij ãģ§ +Ġдол го +ת ׼×ķ +ĠìķĦ ëĭĮ +ë į°ìĿ´ +Ġç iz +Ġcho Äĩ +Ġ×Ķ ×Ļת +Ġ×Ķ×Ļת ר +Ġso át +׼ ×ij×ĵ +à¹Ģล à¹Īา +Ġд еÑĢ +ĠдеÑĢ ÐµÐ² +ãĤĴ åħ¥ãĤĮ +×Ĺ ×ķס +×Ĺ×ķס ר +ج ÙĬÙĨ +t ón +onn é +Ġпол ноÑģÑĤÑĮÑİ +人 ãģŁãģ¡ +Ġpr êt +ëł ¸ +Ġdéc embre +cı lar +Ġת ת +Ġê²½ìļ° ìĹIJëĬĶ +ÙĪ Ø¹Ø¯ +è¦ĭ ãĤĭ +วิ à¸Īัย +ë ¶Ī +ز ÙĪØ§ +زÙĪØ§ ج +d ì +ãģ§ãģĻ ãĤĪ +Ġвод о +ĠÙĬ ÙĪØ¬Ø¯ +Ñģ оÑģÑĤоÑı +Ðŀ С +ĠÄIJ ó +׊פש +Ġצ ×Ļ×ij×ķר +ĠاÙĦÙĤ Ø· +ĠاÙĦÙĤØ· اع +Ġиме ÑİÑĤ +Ġph áºŃn +×Ľ×¡ פ×Ļ +полн иÑĤелÑĮ +éĻIJ ãĤĬ +ĠÑģ ÑĢав +ĠÑģÑĢав н +ÙħاÙĦ Ùĥ +×ĵר ×ķ×Ŀ +çļĨ ãģķãĤĵ +ØŃÙĤ ÙĤ +à¹ģหล à¹Īà¸ĩ +ĠاÙĦر سÙħÙĬ +оÑĩ ки +×ĺ ×ij×Ĺ +Ġcan lı +Ġ׾ ׾ +Ġ׾׾ ×ŀ×ķ×ĵ +×ŀ×ij ×ķ +ת ׼ +×ª×Ľ ׳×Ļת +ĠاÙĦÙħ شار +ĠاÙĦÙħشار ÙĥØ© +İ Åŀ +ĠسÙĬ اسÙĬ +в олÑĮ +ĠÑģ пÑĢав +æĿ¥ ãģ¦ +פ×ķר ×ķ×Ŀ +สำ à¹Ģรà¹ĩ +สำà¹Ģรà¹ĩ à¸Ī +ĠÅŁ öyle +Ġzosta ÅĤa +ĠH ü +ר ×ķש +د ÙĦÙĬÙĦ +ÑĢи д +ש ף +×ŀ×§ ×ķר +ĠÑĥ Ñĩ +ĠÑĥÑĩ еб +ĠÑį ÑĤа +ков а +à¸ķà¸Ļ à¹Ģà¸Ńà¸ĩ +ÙĨ ÙIJ +à¸Ńีà¸ģ à¸Ħรัà¹īà¸ĩ +ระ à¸ļุ +Ġd ữ +ĠاÙĦØŃ اÙĦÙĬ +׼ ×ķ׼ +׼×ķ׼ ×ij +Ġ×ŀ×IJ שר +Ġtr ụ +ÑĤел ем +Ġв ли +Ġвли Ñı +Ġש×IJת ×Ŀ +Ġuw ag +Ġuwag ÄĻ +×ĺ ×Ļת +×IJ ×ĵ×Ŀ +à¸Ķ ุ +Ġ×Ķ×IJ ׾×Ķ +Ġkar Ä±ÅŁ +ĠÄIJ á»iji +да ÑİÑĤ +ãģªãģ® ãģ« +Äħ cych +à¹Ģà¸Ļ à¹īà¸Ļ +ãģĹãģ¦ ãģĹãģ¾ãģĨ +int érieur +ĠfÃŃs ica +ĠÐŁ ол +ãģĹãģ ķ +à¸Ĺำ à¹Ħม +ĠL âm +ĠاÙĦÙħ سÙĦÙħ +ĠاÙĦÙħسÙĦÙħ ÙĬÙĨ +ص ØŃØ© +ìĹ Ħ +à¹Ģà¸Ķà¹ĩ à¸Ķ +ĠÑĥ ÑĩеÑĤ +â Ìģ +Ġب ÙĦا +ĠاÙĦاجتÙħاع ÙĬ +פרס ×Ŀ +ãĥķ ãĥ© +ĠÐļ огда +mie ÅĽci +ĠبÙĬÙĨ Ùħا +Ġ×ŀ×IJ ×ŀר×Ļ×Ŀ +Ġ×ij×IJ ×ĸ×ķר +×ķש ×Ļ×Ŀ +ĠÑģдел а +entr ée +à¹Ģ à¸Ħà¹īา +Ñĥг л +ĠاÙĦÙģ ÙĨÙĬ +ĠÐĴ оÑĤ +à¸Ĺีà¹Ī มา +×ķצ ×Ĵ +ÙĤد رة +Ġëª © +Ġ목 ìłģ +íıī ê°Ģ +ĠاÙĦØ£ ربع +ĠاÙĦأربع اء +פס ×Ļ×§ +ĠÑıвлÑı ÑİÑĤÑģÑı +ب ÙĪÙĨ +ì° ¾ +×ŀ×¢ ר׼ +×ŀ×¢×¨×Ľ ×ķת +ãĤ· ãĤ§ +ĠباÙĦ Ø£ +íĸĪ ëįĺ +ĠاÙĦبر ÙĨاÙħج +ĠاÙĦØ£ ØŃد +Ġm Å© +ĠmÅ© i +п аÑĤ +ب Ø« +ĠÑĨ енÑĭ +Ġ×ijת ׾ +è¨Ģ ãĤıãĤĮ +ĠاÙĦÙħ جاÙĦ +ĠìĦ¸ ìĥģ +Ġ×Ĵ ×ķפ +ĠнаÑĪ ÐµÐ¹ +Ġкомп аниÑı +б ин +öl ü +×Ļ ×Ļ×ĺ +Ġ×ŀס פ×Ļ×§ +ยัà¸ĩ à¸Ħà¸ĩ +ĠЧ и +Ġан ÑĤи +ĠÑģÑĢед и +สà¹Īวà¸Ļ à¹ĥหà¸įà¹Ī +оÑĩ ка +íĬ¹ ë³Ħ +ว à¹Īาà¸ĩ +гоÑĢ Ð¾Ð´ +با Ùĥ +à¹Ģส ีà¹Īย +à¹Ģสีà¹Īย à¸ĩ +ãĤĤãĤī ãģĦ +×§ ×ķ×Ŀ +ãģĽ ãģļ +ĠاÙĦÙĤ اÙĩرة +Ġ×ij ׼×ļ +Ùħشار ÙĬع +باØŃ Ø« +Ġпо Ñĩ +ĠпоÑĩ ÑĤи +ĠÑĦоÑĢм а +S İ +Ġ×ŀצ ×Ļ×¢ +ล ื +ลื ม +ĠÑĤ еÑĢ +ĠÑĤеÑĢ ÑĢиÑĤоÑĢ +ĠÑĤеÑĢÑĢиÑĤоÑĢ Ð¸Ð¸ +Ġв меÑģÑĤ +ĠвмеÑģÑĤ е +dıkl arı +op ération +à¹Ĥ ห +ص دÙĬ +صدÙĬ ÙĤ +íĸī ìłķ +تج ا +تجا ÙĪØ² +Ġsu ç +Ġar ty +Ġarty ku +Ġartyku ÅĤ +ãĤ·ãĥ§ ãĥĥãĥĹ +ש פ +שפ ×Ļ×¢ +Ġ×Ķש ×Ļר×ķת +à¹ģà¸ĸ ม +ë¸ Ķ +Ġuk ÅĤad +Ġ×ķ ׼×Ļ +หล าà¸ģ +หลาà¸ģ หลาย +æĸ¹ ãĤĤ +Ġpodr óż +ĠE ÄŁer +Ġком наÑĤ +ĠÑģам ÑĭÑħ +Ġв кÑĥÑģ +б еж +Ġ×ij ×§×ķ +æİĽ ãģij +ãģ¿ ãĤĭãģ¨ +ĠiliÅŁ kin +ĠÙĬ عÙħÙĦ +Ġпод аÑĢ +Ġyaz ılı +ãĤĴ å¾Ĺ +Ġwyst ÄĻp +à¸Ĺีà¹Ī à¹ĥà¸Ĭà¹ī +ØŃاد Ø« +ÙĪ ÙĬد +кÑĥ лÑĮÑĤ +кÑĥлÑĮÑĤ ÑĥÑĢ +à¸ģาร à¹ģà¸Ĥà¹Īà¸ĩ +à¸ģารà¹ģà¸Ĥà¹Īà¸ĩ à¸Ĥ +à¸ģารà¹ģà¸Ĥà¹Īà¸ĩà¸Ĥ ัà¸Ļ +ÙħÙĪ Ø¸ +ÙħÙĪØ¸ Ùģ +ÙĬÙħ ÙĬ +ãĤĵãģ§ãģĻ ãģĮ +diÄŁ im +diÄŁim iz +ĠÐŁ еÑĢ +ĠÐŁÐµÑĢ Ð² +Ġm ão +ĠÑģ ез +ĠÑģез он +Ġ×Ķ×ŀ ×¢ +Ùħ جÙħÙĪØ¹Ø© +ĠинÑĦоÑĢм аÑĨии +i ếc +ã ng +ĠÄij ấy +ãģĶ ç´ +ãģĶç´ ¹ +ãģĶç´¹ ä»ĭ +Ġad ım +à¹Ħ หล +Ġп ÑĢакÑĤи +ĠпÑĢакÑĤи Ñĩ +ĠпÑĢакÑĤиÑĩ еÑģ +ĠпÑĢакÑĤиÑĩеÑģ ки +ĠاÙĦÙĨ Ù쨳 +ĠÑĢабоÑĤ е +ÙĦÙĬ Ùģ +ĠاÙĦجÙĨ ÙĪØ¨ +Ġвод Ñĭ +ì¹ Ļ +Ġм иÑĢа +ĠÄij ừng +ĠпÑĢоÑĤив о +ĠÑģÑĤÑĢан Ñĭ +ล ู +ìĤ ¶ +kre ÅĽl +Ġbul und +Ġbulund uÄŁu +à¹ģ สà¸Ļ +ãĤ± ãĤ¢ +ת×Ĺ ×ķ×ŀ×Ļ +ר׼ ×Ķ +Ġ׾ק ×ķ×Ĺ +Ġ׾ק×ķ×Ĺ ×ķת +Ġ×Ľ×ª ×ķ×ijת +ĠÙĦ ÙĥÙħ +ب شر +Ġr Ãłng +Ġ×ŀ×Ķ ×ŀ +Ġ×IJ×Ĺר ×ķת +Ġб он +Ġбон ÑĥÑģ +ï½ Ĺ +à¹ģ ยà¸ģ +ãģĤãģªãģŁ ãģ® +ĠÑĥÑĩаÑģÑĤ ие +ĠE yl +ĠEyl ül +ĠçalÄ±ÅŁmalar ı +Ø® طر +ìĿ ½ +à¸ģาร à¹ĥà¸Ĭà¹īà¸ĩาà¸Ļ +Ġана лиз +תק ×ij׾ +ни ем +Ġİ ns +Ġİns an +ĠبÙĪ Ø§Ø³ +ĠبÙĪØ§Ø³ طة +Ġ׳ ×Ľ×ł×¡ +Ġ×Ķ×ŀ ×Ļ×ĵ×¢ +Ġç o +Ġço ÄŁu +á» ĺ +ĠêµŃ 민 +ãĤĤ ãģĦãģĦ +Ġ׼ ׾×Ļ +ĠÑģÑĢед не +g ÅĤo +gÅĤo ÅĽ +Ġneg ó +Ġnegó cio +ĠÑĢ ÐµÐ³Ð¸ÑģÑĤ +ĠÑĢегиÑģÑĤ ÑĢа +ĠÑĢегиÑģÑĤÑĢа ÑĨии +Ġtr á»ĵng +ĠпÑĢ Ñı +ĠпÑĢÑı мо +ëłĪ ìĿ´ +Ġk ém +к ле +à¸Ļำ มา +ĠÑĦ ин +ĠÑĦин анÑģ +ĠÑĦинанÑģ ов +Ġki á»ĩm +ยัà¸ĩ à¹Ħ +ยัà¸ĩà¹Ħ à¸ĩ +ย ิà¸ĩ +à¹Ĥ à¸Ľ +ĠполÑĥÑĩ ил +×Ļ×ĸ ×Ŀ +à¹ģละ à¸Ħวาม +Ġво обÑīе +ص ÙĬر +ãĥı ãĥ³ +ĠاÙĦÙĤ اد +ĠاÙĦÙĤاد Ùħ +Ġب دÙĪÙĨ +ع ظÙħ +ת ׳×ķ×¢ +×ª×ł×ķ×¢ ×Ķ +Ø£ ÙħÙĦ +ãģķ ãģĪ +ÑĤ ем +ÑĤем пеÑĢ +ÑĤемпеÑĢ Ð°ÑĤÑĥÑĢ +Ġ׾ ×Ļצ×ķר +Ġr ÄĻk +ر سÙĦ +ìŀIJ 를 +Ġ×Ļצ ×Ļרת +ÙĨ بÙĬ +Ñĩ наÑı +تØŃ ÙĦÙĬÙĦ +Ġм ик +Ġмик ÑĢо +ĠS öz +Ġfor ça +Ñģ он +ĠاÙĦع را +ĠاÙĦعرا ÙĤÙĬ +ĠH á»ĵng +ãģĻãĤĭ ãģŁãĤģãģ« +à¸Ĺีà¹Ī à¸Ńยูà¹Ī +Ġ×ķ×IJ ×£ +ص ÙĬد +ĠìķĬ ê³ł +ร ัà¸ĩ +ĠاÙĦت ÙĪØ§ØµÙĦ +à¹Ģม à¸ķร +Ñĥ ÑģÑĤÑĢой +ÑĥÑģÑĤÑĢой ÑģÑĤв +m ıyor +Ġبا سÙħ +Ġ×ķ ׼×ķ +ĠG ül +á» IJ +Ãī tat +غ اÙĦ +Ø¥ ÙĨØ´ +Ø¥ÙĨØ´ اء +T İ +à¸Ĥà¹īา ม +Ġtro ch +Ġtroch ÄĻ +Ø¥ ص +إص ابة +ĠØ« اÙĨÙĬ +ĠاÙĦص ØŃØ© +Ġ×ĸ×Ķ ×ķ +jÄħ cej +ãĥĢ ãĥ³ +ìĿ¸ ìĿ´ +Ġв олоÑģ +ëIJĺ ë©´ +Ġzak ÅĤad +ãģĻ ãģĵãģ¨ +以ä¸Ĭ ãģ® +Ġ×Ķ×ŀ×§ ×ķ×Ŀ +ÙħØ´ اÙĩ +ÙħشاÙĩ دة +Ñĩ ив +ب Ø´ +ย à¹īาย +Ġsür dür +ĠN ẵ +ĠNẵ ng +ĠигÑĢ Ð°ÑĤÑĮ +Ġê·¸ëŁ¬ ë©´ +ãĥķ ãĥ« +ล à¹Īะ +Ġtend rá +Ġb Ãły +à¹Ģà¸Ľà¹ĩà¸Ļ à¸ľà¸¹à¹ī +Ġok o +Ġoko ÅĤo +w ÅĤa +wÅĤa ÅĽci +wÅĤaÅĽci w +æĢĿ ãĤı +ĠYa ÅŁ +ĠB á»ĩnh +íı Ń +بÙĬ د +קר ף +à¹Ģศ ร +à¹Ģศร ษ +à¹Ģศรษ à¸IJ +à¹Ģศรษà¸IJ à¸ģิà¸Ī +ĠاÙĦØ£ ÙĪØ±ÙĪ +ĠاÙĦØ£ÙĪØ±ÙĪ Ø¨ÙĬ +fl äche +ä¹Ĺ ãĤĬ +Ġb á»ģn +Ùĩ ب +æľĢ ãĤĤ +Ġsa ç +à¸Ńำ à¹Ģà¸ł +à¸Ńำà¹Ģà¸ł à¸Ń +ĠØ£ ج +ĠاÙĦد اخÙĦ +ĠاÙĦداخÙĦ ÙĬØ© +×ĺ ×ķ×ij +ãĤĤ ãģªãģı +Ġли ÑĨа +à¹ģลà¹īว à¸ģà¹ĩ +×ĸ׼ ×Ļר +Ġqu Ãł +ĠÙĥ ذÙĦÙĥ +صØŃ Ùģ +ĠÃĤ u +ÙĪØ¨ ا +à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļ à¹ģà¸Ľà¸¥ +à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļà¹ģà¸Ľà¸¥ à¸ĩ +à¸ķัว à¸Ńยà¹Īาà¸ĩ +Ġráp ida +Ġtas ar +Ġtasar ım +ĠعÙĦÙĬ ÙĩÙħ +ס ×ķ׾ +c ılı +cılı k +Ġر غÙħ +ìĭľ íĤ¤ +Ġ×IJ׾ ×§ +Ġ×IJ׾ק ×ĺר +Ġ×IJ׾ק×ĺר ×ķ׳×Ļ +à¹ģà¸ļ à¹Īà¸ĩ +Ġh ạng +ãģ£ãģ¦ ãģıãĤĮ +ĠÙĨ تÙĬ +ĠÙĨتÙĬ جة +ıkl ı +غ اÙĨ +à¸Ĥà¹īà¸Ń à¸Ħวาม +à¸Ľà¸¥ าย +ĠØ£ Ùħس +à¸Ĺีà¹Ī à¹Ģà¸ģีà¹Īยว +à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยว à¸Ĥ +à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยวà¸Ĥ à¹īà¸Ńà¸ĩ +Ġdé fin +Ġdéfin i +ÙģÙĨ اد +ÙģÙĨاد ÙĤ +à¹Ħà¸Ķà¹ī วà¹Īา +ãģªãģĦ ãĤĪãģĨãģ« +Ġpróp ria +ĠPh át +ãĤĦãģĻ ãģı +สวย à¸ĩาม +ê³ł ìļĶ +Ñı еÑĤ +ãģĭãĤĤãģĹãĤĮãģ¾ãģĽãĤĵ ãģĮ +تر جÙħ +ĠкÑĢаÑģ ив +Ġ×ŀ ר×IJש +д еж +ĠÙĬ ÙĪÙĨ +ĠÙĬÙĪÙĨ ÙĬÙĪ +Ñģк оÑĢ +ĠKas ım +ê³Ħ ìķ½ +к оÑģ +Ġна ÑĢÑĥ +ĠнаÑĢÑĥ ÑĪен +Ġdu że +acc ès +Ġh á»ĵng +Ġv Å© +ãģĦãģŁ ãģĹãģ¾ãģĻ +Ġ×ĺ ×Ļ +Ġ×ĺ×Ļ ×ķ׾ +lıkl arı +Ġqu ê +ëħ¸ ëıĻ +ìķ Ķ +CI ÃĵN +Ġt ắc +press ão +ĠìŀĪ ìľ¼ +สิà¸Ĺà¸ĺิ à¹Į +íĥ Ħ +Ġ×Ķ×ŀ ×ŀש׾×Ķ +å¬ī ãģĹãģĦ +ĠÄIJ ặc +ÙĨ زÙĦ +ĠдÑĢÑĥг ой +д ÑĥÑĤ +ìĪ Ļ +Ġth ụ +à¹Ģส ร +à¹Ģสร à¹ĩ +à¹Ģสรà¹ĩ à¸Ī +Ġto plant +Ġtoplant ı +×IJ×ŀ ף +×ķ׾ ת +п омн +Ġyo ÄŁun +ÅĦsk iego +ì° © +ĠØ« ÙĦاث +ĠØ«ÙĦاث Ø© +Ġl ắng +ë¦ ´ +ราà¸Ĭ à¸ģาร +ĠÑģлов а +á» Ĩ +à¸Ķี à¸ģวà¹Īา +ãģĶãģĸ ãģĦãģ¾ãģĻ +Ġд из +Ġдиз айн +fé rence +lıkl ar +ãģªãĤĵ ãģ§ãģĻ +ajÄħ cy +Ġëĭ¤ ìĸij +Ġëĭ¤ìĸij íķľ +×§ ×Ļר +ØŃ ار +ส ูà¹ī +Ġz ro +Ġzro bi +Ġzrobi Äĩ +×ŀ ×Ļ׼×Ķ +à¸Ĭà¹Īวย à¹Ģหลืà¸Ń +ĠÑįÑĤ Ñĥ +ë´ ī +楽 ãģĹãģĦ +س ÙĪØ± +íķĺ ê±°ëĤĺ +Ùħؤ تÙħر +Ġpoc zÄħ +ĠpoczÄħ tk +ĠpoczÄħtk u +Ġع ربÙĬ +اÙĦØ£ ر +اÙĦأر دÙĨ +à¸Ķ ร +Åĵ uvre +ĠÙĪÙĥ اÙĨت +ĠÅĽ redni +Ø® ضر +Ġch uyến +н ÑĤ +ĠìķĮ ê³ł +Ġv á»Ŀi +Ġ×ij ×Ļ×ĵ×Ļ +×ŀ×ĵ ×ķ×ijר +ÙĪ Ù쨱 +ÙĬ Ø¡ +׳ ×Ľ×¡ +ĠÐĽ а +л он +Ġx ấu +Ùģ ÙĬÙĨ +Ġfé vrier +ĠÐŀ на +ĠV á»ģ +ĠÅŁey ler +ĠполÑĥÑĩ ен +з ад +Ġn ét +à¹Ħà¸Ľ ยัà¸ĩ +×Ĺש×ij ×ķ +à¸ļัà¸Ļ à¸Ĺ +à¸ļัà¸Ļà¸Ĺ ึà¸ģ +Ġgerçek leÅŁ +иÑĩеÑģк ое +ìĪĺ ê°Ģ +Ø« بت +ãģ¤ ãģ¾ãĤĬ +ĠÑĥÑģловиÑı Ñħ +ëĭ¤ ê°Ģ +ราย à¹Ħà¸Ķà¹ī +׼×IJ ×ij +à¹Ĥà¸Ľà¸£ à¹Ĥม +à¹Ĥà¸Ľà¸£à¹Ĥม à¸Ĭัà¹Īà¸Ļ +j ähr +jähr ige +×§ ׳×Ļ×Ŀ +×ŀ ×ķ×§ +×ŀ×ķ×§ ×ĵ +ãģ«è¡Į ãģ£ãģ¦ +Ø¢ ÙĦ +вед ение +Ġ׾ ×Ľ×ª×ķ×ij +جÙħ Ùĩ +جÙħÙĩ ÙĪØ±ÙĬØ© +à¸ī à¸ļ +à¸īà¸ļ ัà¸ļ +ĠC òn +à¸ľ สม +ãģªãģ© ãģĮ +×IJ×Ķ ×ij +ĠдейÑģÑĤв иÑı +y ız +à¹Ħมà¹Ī à¹Ģà¸Ħย +ج ÙĪØ² +×Ķ×Ĺ׾×ĺ ×Ķ +f ällt +ãĥĵ ãĤ¸ +ãĥĵãĤ¸ ãĥį +ãĥĵãĤ¸ãĥį ãĤ¹ +Ġ×IJ ×Ļ׳×Ŀ +ĠнаÑħод иÑĤÑģÑı +Ġdzi ÅĽ +ست Ø·ÙĬع +׾ ×Ļף +Ø® ÙĦاÙģ +Ùĩ ÙIJ +Ġatr ás +íĺ ģ +ãĤĴ ãģĶ +Ġ×Ķ×ŀ ×ķצר +ĠBakan lıģı +ÑİÑī ее +ÙħÙĨ اط +ÙħÙĨاط ÙĤ +Ùģ Ø¯ +à¸Ļำ à¹Ħà¸Ľ +Ġв аж +Ġваж но +Ġm ạch +׼ ׳×ķ +بع Ø« +lan ması +Ġa yr +Ġayr ıl +ìĤ¬ íļĮ +d ÃŃa +p ÅĤyw +اÙħ ÙĬØ© +íĺ ľ +×IJ׳ ×Ĵ׾ +×IJ׳×Ĵ׾ ×Ļת +ĠìŀĪëĭ¤ ëĬĶ +Ġس اعة +ĠëĤĺ íĥĢ +b ö +à¸Ħ ัà¸Ļ +ĠdziaÅĤ ania +Ø© Ùĭ +Ġng Å© +׳צ ×Ĺ +ãģ¯ ãģĤãĤĭ +ĠyaÅŁ ında +st ück +car acter +caracter ÃŃsticas +Ġr á»Ńa +ĠÙħختÙĦÙģ Ø© +ãģ«ãģĬ ãģijãĤĭ +à¹ģà¸ŀ à¸ĩ +วิ à¹Īà¸ĩ +ת פ×ķ +سا ÙĩÙħ +使 ãģĨ +Ùĥ رÙĬ +×IJ פ×Ļ +........ ....... +ĠÑĤак им +×Ļ׼ ×ķ×Ļ +Ø´ بÙĩ +ج ÙĬر +ãģĿãģ® ãģ¾ãģ¾ +ac jÄĻ +ĠاÙĦت رÙĥ +ĠاÙĦترÙĥ ÙĬ +ĠпÑĢав илÑĮно +Ġت عÙħÙĦ +à¸ģล à¹īา +Ġbi ên +Ġ×ij׳×Ļ ×Ļת +Ġкл Ñĥб +Ġ×ŀ ש×Ķ +в ÑĪий +ãģĵãģ¨ãģĮãģ§ãģį ãĤĭ +à¸ŀัà¸Ļà¸ĺ ุ +à¸ŀัà¸Ļà¸ĺุ à¹Į +ר ×ķ×Ŀ +ĠاÙĦÙģ Ø±ÙĨ +ĠاÙĦÙ쨱ÙĨ سÙĬ +à¹Ģà¸Ľà¹ĩà¸Ļ à¸Ħà¸Ļ +ãģĹãģ¦ ãģĬãĤĬ +Ġth ầy +ãĤĵ ãģłãģijãģ© +ìĶ ¨ +Ùħ دÙĨ +ت ÙĪÙĨ +ĠмеÑĤ ал +ĠмеÑĤал л +Ġin ÃŃcio +à¸Ńà¸Ńà¸ģ à¸Īาà¸ģ +ëĴ ¤ +Ġcu á»ijn +Ġbu á»Ļc +ÙĨ سÙĬ +ä cht +×ŀ ×Ļ׳×Ļ×Ŀ +ãģķ ãģ¦ +ãģĮ ãģ§ãģį +ÑĬ ем +Ġtá i +ĠЧ ÑĤ +ĠЧÑĤ обÑĭ +à¸Ľà¸¥ ูà¸ģ +à¸Ĭุม à¸Ĭà¸Ļ +н Ñģкий +Ġv ững +Ġ×Ķ ×ľ×ij +ë le +Ġש ×¢×ijר +в аÑĤÑĮÑģÑı +б ой +ع ÙĪÙĨ +à¹ģà¸Ķ à¸Ļ +Ġספר ×Ļ×Ŀ +Ġt uyên +Ġnhi êu +ĠQu ý +Ġh uyết +ãĤı ãģĭãĤīãģªãģĦ +Ġ×ŀ ׼ף +Ġ×Ķ ×§×ľ +Ġ׾×IJ ×ķר +ĠÄIJi á»ĩn +Ø´ ؤ +شؤ ÙĪÙĨ +Ġ×ŀ׊פש +ĠпоÑģÑĤоÑıн но +×ŀ ×Ļר +ìħ Ķ +Ðŀ Ñģ +ÐŀÑģ нов +×ĸ ×Ļת +ĠH á +ĠÑĩаÑģ ов +×IJ ×ķ׾×Ļ +Ġm át +Ø® رÙĪ +خرÙĪ Ø¬ +ÙĤ ضا +ÙĤضا ÙĬا +à¹Ģà¸Ľ à¸Ńรà¹Į +ĠÙĬ ÙĪÙĦ +ĠÙĬÙĪÙĦ ÙĬÙĪ +à¹Ĥà¸Ĺ ษ +׳ פ׾ +ת ×ķש +ת×ķש ×ij×Ļ +Ġv ários +×ŀ ר×IJ×Ķ +ëĿ¼ ìĿ´ +ÙĨ غ +×ij צע +г он +ĠÄIJ ược +ع Ùı +пÑĥÑģ к +ĠÙĪØ§ÙĦ Ùģ +üc ü +×Ļ×§ ×Ļ×Ŀ +Ġس بÙĬÙĦ +׾×ij ף +ĠاÙĦÙĤ رÙĨ +ס ×ķת +ĠQu áºŃn +ãģĵãĤĮ ãģĮ +ãĥĸ ãĥ©ãĥ³ãĥī +×Ĵ ×ŀר +Ġwarto ÅĽci +ĠÙĪØ¨ ÙĬÙĨ +Ġd ạ +ÐIJ в +ÐIJв ÑĤо +Ġol acaktır +à¸Ļ à¸Ĺà¹Į +Ùħ طار +Ġ×¢ ×§×ij +Ġת פ +ãģĹãģ¦ ãģĦãģ¦ +צ ×ŀ×Ĺ +à¸Ī à¸Ńà¸ĩ +Ġö de +ìį ¨ +ÙĨ اس +調 ãģ¹ +ĠогÑĢ Ð¾Ð¼Ð½ +ë³´ íĹĺ +×ĺ ×§ +×ĺ×§ ס×ĺ +ĠbaÅŁ v +ĠbaÅŁv uru +Ġpom ys +Ġpomys ÅĤ +ãģ« ä¹Ĺ +Ġש ׼ף +ĠاÙĦÙħس ؤÙĪÙĦ +Ġз ан +Ġзан ÑıÑĤ +Ġd ương +ãĥĹãĥ¬ ãĤ¤ +ล à¸ļ +ÑĤи ка +ĠAr alık +Ġнед о +Ġm á»Ļ +Ġor an +Ġoran ı +Ġktó r +Ġktór Äħ +Ġ×Ķ×IJ×Ĺר ×ķ׳×ķת +ائ ÙĨ +ÅĦ s +ÅĦs ka +åĽ½ ãģ® +×ŀ ×ĺ×Ļ +ĠвопÑĢоÑģ Ñĭ +à¸Ńà¸ĩà¸Ħà¹Į à¸ģร +×ŀ ×ķצ×IJ +Ġpó ź +Ġpóź niej +ש×ŀ ×IJ׾ +Ġk aps +Ġkaps am +Ġkapsam ında +Ġmá quina +ĠÅĽwie cie +Ġho Ãłng +Ġöz gü +×Ĵ×ķר ×Ŀ +ãģĤ ãģŁãĤĬ +à¸ķัà¸Ķ สิà¸Ļ +à¸ķัà¸Ķสิà¸Ļ à¹ĥà¸Ī +б ÑĢи +ãģ«ãģªãĤĭ ãģ¨ +ت ÙĥÙĪÙĨ +Ġ×ķ×Ķ ×Ļ×IJ +Ġchi ếu +ÑģÑĤан ав +ÑģÑĤанав ли +ÑģÑĤанавли ва +×ŀ ×ķ×Ĵ +c ité +ĠK örper +Ġש ×Ĵ×Ŀ +ع ظ +عظ ÙĬÙħ +Ġ×Ķ×IJ ×Ļש×Ļ +Ġmat ière +ĠÙģ ÙĪÙĤ +Ġk to +Ġkto ÅĽ +à¸Ļ à¹Ĥย +à¸Ļà¹Ĥย à¸ļาย +å¾ħ ãģ¡ +à¹Ģม à¸Ļ +à¹Ģมà¸Ļ ู +A ÃĩÃĥO +Ġt ù +Ġtù y +ãĥĪ ãĥ³ +ĠоÑĤ каз +Ġ×ŀ ×ķצר +ül ü +ãģķãĤĵ ãģ« +Ġ×Ĺ ×ķ×ij +קר ×Ļ×IJ×Ķ +ĠاÙĦØ® دÙħات +ĠÙĦÙħ دة +ر ؤ +رؤ ÙĬØ© +ãĤĴè¦ĭ ãģ¤ãģij +à¸Ł า +Ġréuss i +à¸Ļัà¸ģ à¹Ģรียà¸Ļ +ĠÑĩиÑģ л +à¸ģาร à¹Ģลà¹Īà¸Ļ +Ġhaz ırl +Ġhazırl an +ĠпеÑĢв Ñĭй +ли м +ĠоÑĤзÑĭв Ñĭ +Ġwy jÄħ +ĠwyjÄħ tk +ĠØ£ ÙĤÙĦ +ס ×ļ +Ġê²° ìłķ +Ġ׾×ŀ×¢ ש×Ķ +Ġl ắp +à¹ģà¸ļ ร +à¹ģà¸ļร à¸Ļà¸Ķà¹Į +วà¹Īา à¹Ģà¸Ľà¹ĩà¸Ļ +Ġب دا +Ġبدا ÙĬØ© +ãģ¨ãģĦãģĨ ãģ®ãģĮ +иÑĩеÑģк им +à¸ģาร à¸ŀัà¸Ĵà¸Ļา +Ġb Ãło +Ġmia ÅĤa +y waÄĩ +ĠMär z +ĠÙĨ سبة +Ġéconom ique +×ĸ ×ŀ +×ĸ×ŀ ׳×Ļ×Ŀ +æŃ¢ ãĤģ +Ġt á»§ +íķĺ ìĭł +Ġkażde go +stra ÃŁe +à¸Ĭ ีà¹ī +à¹Ģ à¸ļา +ÑĢеÑģ ÑĥÑĢÑģ +ев ой +Ø´ باب +à¸ķà¹Īาà¸ĩ à¸Ľà¸£à¸°à¹Ģà¸Ĺศ +Ġ×IJ ×Ļש +Ġ×IJ×Ļש ×Ļת +×Ļ ×ķפ +×Ļ×ķפ ×Ļ +ĠìļĶ êµ¬ +ì¡° ìĤ¬ +ãģ£ãģŁ ãĤī +׾ ×Ļ×§ +миниÑģÑĤ ÑĢ +ãĤĤãģ® ãģ¯ +Ġl ương +Ġна и +Ġнаи бол +Ġнаибол ее +íİ ĺ +à¹ģà¸ŀ à¹ī +ãĤŃ ãĥ¥ +ĠкоÑĤоÑĢ Ñĭм +à¹ģà¸Ĺ à¸ĩ +à¹ģà¸Ĺà¸ĩ à¸ļà¸Ńล +Ġ׳ ×Ļ×Ķ +Ġ׳×Ļ×Ķ ×ķ׾ +âĤ ª +ĠGi ải +ĠиÑģполÑĮзов а +ëł¥ ìĿĦ +ãģĹãģĭ ãĤĤ +à¸ģà¹ĩ à¸ķà¹īà¸Ńà¸ĩ +ĠÑĢ ÐµÐ± +ĠÑĢеб ен +ĠÑĢебен ка +ت ÙĪØ§ØµÙĦ +ãĤ°ãĥ« ãĥ¼ãĥĹ +ãĤĦ ãĤī +à¹Ģà¸Ľà¸´à¸Ķ à¸ķัว +б ÑĢо +ë°ĸ ìĹIJ +ÙĨ ÙİØ§ +×Ķ ×Ĵ +×Ķ×Ĵ ׳×Ķ +à¸Ĺ รั +à¸Ĺรั à¸ŀ +à¸Ĺรัà¸ŀ ยà¹Į +Ġkh á»iji +עצ ×ŀ×ķ +бол езн +Ġë°Ľ ìķĦ +ม à¸Ļ +มà¸Ļ ุ +มà¸Ļุ ษ +มà¸Ļุษ ยà¹Į +âĹ Ĩ +×ŀ צ׾×Ļ×Ĺ +Ñıв ление +Ùħ Ø·ÙĦ +ÙħØ·ÙĦ ÙĪØ¨ +Ø® اÙĦÙģ +ت ÙĪÙĤÙģ +ãģ§ãģį ãģ¾ãģĽãĤĵ +оÑģÑĤ ей +м еÑĩа +기 ëĬĶ +תש ×¢ +ص ÙĬب +Ġ×ij×¢ ×ķ×ĵ +à¸Ĥà¸Ńà¸ĩ à¹Ģà¸Ĥา +ÑĤÑı ж +ĠÑĥ пÑĢав +ĠÑĥпÑĢав лениÑı +Ġgén ér +Ġth ÃŃ +פ ×ļ +Ġر Ùħض +ĠرÙħض اÙĨ +Ġtr uyá»ĩn +Ø¥ عداد +ãĤµ ãĥĿãĥ¼ãĥĪ +Ġпол но +Ø® اÙħ +ÐŁ еÑĤ +ÐŁÐµÑĤ еÑĢ +ÐŁÐµÑĤеÑĢ Ð±ÑĥÑĢ +ÐŁÐµÑĤеÑĢбÑĥÑĢ Ð³ +ÙħÙĨت دÙī +ãģķãĤĮ ãģ¾ãģĹãģŁ +ĠëĮĢ íķĺìŬ +à¸ľà¸¹à¹ī à¸Ĺีà¹Ī +Ġ×ŀ×IJ ×ķ +׾ ׳×ĵ +оÑĩ нÑĭе +ĠнаÑĩ ала +Ġ׾ ×Ļ׾×ĵ×Ļ×Ŀ +ов ое +ãģĻãĤĭãģĵãģ¨ ãģ§ +ĠاÙĦÙĨ Ùģ +ĠاÙĦÙĨÙģ Ø· +ìŀĪ ëĬĶ +غ ÙĨÙĬ +פ ×ĵ +ãĤ ¾ +ĠCr é +ãģ© ãģ¡ãĤī +Ø« اÙĨ +ÑĢаб аÑĤ +ÑĢабаÑĤ Ñĭва +Ġê°Ļ ëĭ¤ +à¸Ī ั +à¸Īั à¸ģร +Ġch ụ +Ġchụ p +Ġм аÑģÑĤ +ĠмаÑģÑĤ еÑĢ +Ġn ắm +ĠÑģÑĤ али +Ġ×Ķ×IJ ×Ļר×ķ×¢ +ãĤ½ ãĥ³ +åĪĨ ãģĭãĤĬ +Ø· بع +بد ا +gr áfico +г еÑĢ +à¸Ķำà¹Ģà¸Ļิà¸Ļ à¸ģาร +Ġsal dır +Ġsaldır ı +в ÑĪиÑħ +ãģĭãģ£ãģŁ ãģ§ãģĻ +Ġyapı yor +ĠاÙĦÙģ Øª +צר פת +з доÑĢов +×ij×¢ ׾ +Ġ×IJ ×ŀ×Ļת×Ļ +Ġоб Ñĭ +ĠобÑĭ Ñĩ +ĠобÑĭÑĩ но +Ġ׾ ×ķ×ŀר +ت ÙĥÙĨ +تÙĥÙĨ ÙĪÙĦÙĪØ¬ +تÙĥÙĨÙĪÙĦÙĪØ¬ ÙĬا +Ġhakk ı +ĠÑĢаР² +ĠÑĢав но +رÙĬ Ùĥ +Ġ×ij ×ŀ×Ļ×ĵ +Ġ×ij×ŀ×Ļ×ĵ ×Ķ +à¹ģà¸ģ à¹īว +Ġìĸ ĺ +Ġìĸĺ 기 +ãģĹãģ¦ ãģĦãģ¾ãģĹãģŁ +Ġkı sm +Ġkısm ı +ê± ¸ +åĨħ ãģ® +ì§ ķ +à¹Ģหมืà¸Ńà¸Ļ à¸ģัà¸Ļ +ĠÙģ ÙIJ +ĠÙģÙIJ ÙĬ +ÙĤ اعدة +Ġmoż esz +Ùħ صاÙĦ +ÙħصاÙĦ ØŃ +ãģ¾ãģŁ ãģ¯ +б ег +Ġs ıc +Ġsıc ak +Ñĩ иÑģ +ÑĩиÑģ лен +Ġн ог +ãĥģãĥ£ ãĥ³ +ãĥ« ãĥī +Ġgi ó +Ġs ını +Ġsını f +ив аÑĤÑĮ +Ġqu ên +Ġì łģ +Ġìłģ ìļ© +ĠJo ão +Ùģ Ø§Ø¯ +ĠGl ück +à¸Ĺ à¸Ńà¸Ķ +Ġg ói +ï¼ Ĭ +Ġdé tail +ĠدÙĬ سÙħ +ĠدÙĬسÙħ بر +ë¡ľ ìĦľ +×ŀ ×ķ×Ĺ +à¹Ħ ฮ +ĠоÑĤ д +ĠоÑĤд ÑĭÑħ +Ġkh uyến +à¸Ħ à¸Ńย +Ġج ÙĨÙĬ +ĠجÙĨÙĬ Ùĩ +ĠاÙĦد ÙģØ§Ø¹ +à¸Ļà¹īำ หà¸Ļัà¸ģ +ĠìĤ¬ëŀĮ ëĵ¤ìĿ´ +Ġth ừa +ĠÃ¶ÄŁrenc i +ĠпомоÑī и +ĠczÄĻ ÅĽÄĩ +ש ×ĺר +ĠN hi +ĠNhi á»ģu +׳ צ×Ļ +ĠнаÑĪ ÐµÐ¼ +ĠkarÅŁÄ± laÅŁ +Ġ×Ķש ׳×Ļ×Ŀ +ĠÄIJ ưá»Ŀng +Ġtr ú +ĠÑĢазлиÑĩ нÑĭÑħ +ĠاÙĦØ´ Ùĩر +Ġ×ľ×¢ ×ķ׾×Ŀ +ØŃ جر +ĠÄij á»ķ +ĠìĿĺ íķ´ +à¸ļ à¹Īà¸Ńย +Ġ×Ķ ×Ļ׾×ĵ +ãģ¨ãģª ãģ£ãģŁ +Ġ×Ĺ×ķ ×ķת +Ġש×Ļר×ķת ×Ļ +Äħ cy +س رÙĬ +K İ +פ ׳×ķ +ÑģÑĤÑĢÑĥк ÑĤÑĥÑĢ +ÑĤ ÑĢÑĥд +Ġ×Ķ ×§×¨ +Ġ×Ķקר ×ķ×ij +Ġth áºŃm +èģŀ ãģį +ÙĤÙĪ ÙĬ +клÑİÑĩ ен +ÑĤе Ñħ +ÑĤеÑħ нолог +è¡Į ãģ£ãģŁ +Ġ×ķ×IJ ×Ļף +ĠÅŁek lin +ĠÅŁeklin de +r ô +ÑĢ Ð¾Ð³ +Ġнов Ñĭе +Ġס ×ij×Ļ×ij +Ġtecn ologÃŃa +ס ׼ +×¡×Ľ ×ķ×Ŀ +ĠÅŀ ub +ĠÅŀub at +Ġ×Ķ×ŀ ׾×IJ +Ġwy pos +Ġwypos aż +ãģ¯ ä½ķ +ãĤ¬ ãĥ³ +ê° ĸ +Ġкак ие +Ġçocuk lar +Ġ׾צ ×ĵ +Ġkay ıt +ĠмеÑģÑĤ е +Ùħ دÙĬÙĨØ© +Ġ׼ ×Ĵ +Ġ׼×Ĵ ×ķף +ãģĹãģ¦ ãĤĭ +ĠÙħا ÙĬÙĪ +ãģ£ãģ¦ãģĹãģ¾ ãģ£ãģŁ +ĠпÑĢогÑĢамм Ñĭ +à¹ģล à¸Ļà¸Ķà¹Į +ãĥ¯ ãĤ¤ +ער ×ķ×¥ +Ñģ ид +ĠB öyle +Ġì²ĺ ìĿĮ +Ġת פק×Ļ×ĵ +ĠTr ên +íĥ Ī +ĠÐłÐ¾ÑģÑģ ий +ĠÐłÐ¾ÑģÑģий Ñģкой +Ġs Ãłn +Ġrè gle +ĠyaklaÅŁ ık +à¹Ģล ิà¸ģ +Ġد ائÙħ +Ġ×ķ ×Ĵ +اب ر +Ġb è +ĠاÙĦ ÙĤدÙħ +ĠÑĢеÑĪ ÐµÐ½Ð¸Ñı +hi ên +ÑĤи к +Ä Ħ +à¸ļรร ยาà¸ģ +à¸ļรรยาà¸ģ าศ +רצ ×ķף +åĭķ ãģį +ĠGä ste +Ġ기 본 +ĠÙĬ عرÙģ +ĠS á»Ń +gÅĤ ÄĻb +à¹Ģà¸Ń ส +×IJ×ŀ ×Ļף +Ġп Ñĥнк +ĠпÑĥнк ÑĤ +Ġ×Ļ×ķ×ĵ ×¢×Ļ×Ŀ +ãĤ« ãĥ©ãĥ¼ +Ġ×ijס ×ĵר +Ġbu á»ĵn +й ÑĤ +йÑĤ еÑģÑĮ +ãĤĴ æ±ĤãĤģ +Ġ×IJת ׼×Ŀ +Ġ모 르 +ظ رÙĪÙģ +Ñĩ еÑģÑĤво +ìĸ´ ìĦľ +Ġод на +Ġkap ı +Ġëħ¸ ëł¥ +ĠKü che +ĠاÙĦت Ø´ +Ø· ÙĬب +ĠíĬ¹ íŀĪ +ĠвÑĭп ÑĥÑģ +ĠвÑĭпÑĥÑģ к +×ĵ ת×Ļ +Ġu ÄŁ +ĠuÄŁ ra +ائ Ùĩا +Ġtho át +ãģª ãĤĤãģ® +Ñij ÑĢ +기 ê°Ģ +ĠgeliÅŁ me +تØŃ ÙĤ +تØŃÙĤ ÙĤ +Ġоп аÑģ +б ÑĢоÑģ +ห ุ +หุ à¹īà¸Ļ +ì¼ Ģ +ãĤ¹ ãĥŀ +ãĤ¹ãĥŀ ãĥĽ +Ø£ Ù쨱 +Ø£Ù쨱 اد +ĠTh á»±c +Ġth ắ +ãĥªãĥ³ ãĤ¯ +Ġni á»ģm +ĠHö he +عÙħ ار +ÙĥÙĪØ± ÙĪÙĨ +ÙĥÙĪØ±ÙĪÙĨ ا +ĠÄIJ ến +ĠÑģам ом +ĠÑĤ еле +ĠÄijo án +à¸Ħวามà¸Ħิà¸Ķ à¹Ģหà¹ĩà¸Ļ +Ġд иÑģк +Ø£ Ø·Ù쨧ÙĦ +ม ารà¹Į +à¸Ĺ หาร +à¸Ĺ à¸Ļ +Ġب عÙĬد +ĠاÙĦÙĩ ÙĨد +åĩº ãģĹãģ¦ +Ġkar de +Ġkarde ÅŁ +×Ķ×Ļס×ĺ ×ķר +×Ķ×Ļס×ĺ×ķר ×Ļ×Ķ +éģ¸ ãģ³ +ع اÙħÙĦ +à¸Ĥ ยาย +Ġtü rl +Ġtürl ü +ĠìĿ¼ ìĿ´ +Ġmaté ria +Ġ׼׾ ×ķ×ŀר +ãĥģãĥ£ ãĥ¼ +جÙħ اعة +ĠÑģво им +Ø¥ÙĤ اÙħØ© +ä¾ĭ ãģĪãģ° +س اب +Ø¢ خر +ÙĤ دÙĬر +×IJ×ŀ ×Ļ +ìĸ » +Ġ׳×ķס פת +ĠÐĴ лад +ĠÐĴлад им +ĠÐĴладим иÑĢ +Ġest ará +ãģĵãģĨ ãģĦãģĨ +ãĤĴ 使ç͍ +มา à¸ķร +มาà¸ķร à¸IJาà¸Ļ +ãģ£ãģ ½ +Ġn ú +Ġnú i +ย าà¸ĩ +ĠاÙĦج ÙĨس +Ġüst ün +ëľ » +ãĤ» ãĥ« +ãģ¦ãģĦ ãģįãģ¾ãģĻ +Ġ×Ĺ ×ķ×ĸ +Ġ×Ĺ×ķ×ĸ ר +ĠÐĵ лав +à¹Ĥà¸Ĭ à¸Ħ +íı IJ +ÙĨت ظر +Ġ×Ĵ ×ij×Ļ +ع ÙĤب +int ér +intér êt +×ŀ פ×Ĵ +×ŀפ×Ĵ ש +Ġth ù +اÙģ Øª +Ġ×ŀש פ +Ġ×ŀשפ ×ĺ×Ļ +ĠÙħ ÙĪØ§ÙĤع +è¦ ļ +è¦ļ ãģĪ +×ĵ ×Ļף +à¹Ģรืà¹Īà¸Ńà¸ĩ ราว +ãģ¾ ãģĤ +Ġgh ế +иÑĢÑĥ ÑİÑĤ +à¸ģ ว +à¸ģว à¹īาà¸ĩ +Ġпов еÑĢ +ĠповеÑĢ Ñħ +ĠповеÑĢÑħ ноÑģÑĤ +׳ ×ĵר +Ġкон ÑĨе +Ġдолж на +Ġ×Ļש ×Ļר +acaģı z +ìĹ Ķ +Ġn ÃŃvel +Ġö r +Ġör nek +Ùĥ Ùģ +ĠФедеÑĢ Ð°ÑĨии +Ġ구 ìĦ± +หัว à¹ĥà¸Ī +ĠV áºŃy +м ед +мед и +меди ÑĨин +медиÑĨин Ñģк +از ÙĬ +×Ĵ×ij ×ķ׾ +ÑĦ ÑĢ +Ġzus ätzlich +à¸ģ à¸ģ +ĠاÙĦاÙĤتصاد ÙĬØ© +Ġh è +lu ÄŁun +ج Ùİ +à¹Ħà¸Ł ลà¹Į +ÄIJ T +ãģĿãģ® ä»ĸ +à¸Ĺิ à¹īà¸ĩ +ĠاÙĦØ£ ÙĪ +ر سÙħ +æ°Ĺ ãģ¥ +ìĿ´ ë©° +ÑĮ ев +ص Ø· +ĠاÙĦاست Ø« +ĠاÙĦاستث Ùħار +à¸Ńา à¸Ħาร +ĠÑĤоÑĩ но +ĠV ân +à¸Ń ร +à¸Ńร à¹Īà¸Ńย +ĠاÙĦس ÙĨØ© +Ġc Æ°á»Ľi +×Ļ×Ķ ×Ł +íį ¼ +話 ãģĹ +âĹ ĭ +ĠìķĬ ìĿĢ +ãĥ¡ ãĥ¼ãĤ +ãĥ¡ãĥ¼ãĤ « +ãĥ¡ãĥ¼ãĤ« ãĥ¼ +ĠÑĤеп ло +å½¼ ãĤī +Ġİ z +Ġİz mir +íĻ į +Ġr ượ +Ġrượ u +æĢĿãģĦ åĩº +ĠPh ạm +Ġchá u +צ×Ļ ×ķת +ĠìĿ¼ 본 +ìĤ¬ ëĬĶ +ĠÑģозд ан +Ġar acı +Ġ×¢ ר +Ġער ×Ļ׼×Ķ +ĠíķĺëĤĺëĭĺ ìĿĺ +dzi ÅĤ +à¸Ľà¸£à¸° à¸ĺาà¸Ļ +Ġser ÃŃa +ĠìŀĪ ëıĦë¡Ŀ +در ج +íķľëĭ¤ ëĬĶ +à¸Ńา à¸Ĺ +à¸Ńาà¸Ĺ ิà¸ķ +à¸Ńาà¸Ĺิà¸ķ ยà¹Į +ÑĤелÑĮ нÑĭй +ĠØ® دÙħات +×ŀ׳ ×ĺ +Ġl ược +ĠS Ãłi +ĠÙĪ Ø§Ø¶ +ĠÙĪØ§Ø¶ ØŃ +غ از +ĠdoÄŁ al +Ġ×ijש ×Ŀ +Ġд лин +ĠØ¥ طار +Ġ×ijס פר +ãĤĴ ä¸İ +ãĤĴä¸İ ãģĪ +Ġë²ķ ë¥ł +ĠÑĥ вели +ĠÑĥвели Ñĩи +ส à¹Ħà¸ķ +สà¹Ħà¸ķ ลà¹Į +à¹Ħ à¸ģล +×ij׊ף +ĠìĿ´ íĽĦ +Ġm unic +Ġmunic ÃŃpio +تÙħ Ø«ÙĦ +ĠÄij áo +H ôtel +Ġl á»Ńa +ĠÄij ẳng +Ñĩ ки +Ø´ رÙĪ +شرÙĪ Ø· +ĠìĿ´ 를 +ÙĬ Ùĭا +×ŀ׾ ×ļ +×ŀ×Ķ ×Ļר×ķת +ĠобÑıз аÑĤелÑĮ +ĠобÑıзаÑĤелÑĮ но +é nergie +Ġmud ança +Ġm ụ +Ġmụ n +Ġn º +ĠاÙĦت عا +ĠاÙĦتعا ÙĪÙĨ +ĠاÙĦاجتÙħاع ÙĬØ© +Ġп лаÑģÑĤ +Ġëĵ± ìĿĺ +ãĥIJãĤ¤ ãĤ¯ +Ùĩج ÙĪÙħ +ĠSa úde +Ġì¤ijìļĶ íķľ +Ġ×Ķצ ×Ļ×ij×ķר +תק ף +ĠاÙĦعاÙĦÙħ ÙĬ +ĠболÑĮÑĪ Ð¾Ð¹ +ĠÙĥ ÙĦÙħ +ĠÙĥÙĦÙħ Ø© +ãģ®ãģ§ãģ¯ãģªãģĦ ãģ§ãģĹãĤĩãģĨãģĭ +ĠÙħ باراة +Ġש×IJ ׳ +Ġש×IJ׳ ×Ĺ׳×ķ +ãĤ¹ãĤ¿ ãĤ¤ãĥ« +ĠSa ÄŁ +ĠSaÄŁ lık +Ġh ư +׳ ×Ĺ×Ķ +Ġ×ij קר×ij +Ø· عÙħ +ห ิà¸Ļ +à¸Ĺุà¸ģ วัà¸Ļ +à¸Ħรัà¹īà¸ĩ à¸Ĺีà¹Ī +ĠlÃł nh +Ġdonn é +ãģĽ ãģĦ +جز ÙĬرة +доÑĢ Ð¾Ð¶ +ì¼ ľ +تÙĨظ ÙĬÙģ +ãĥģ ãĥ§ +Ġald ıģı +ج اج +ĠÑĤ омÑĥ +à¸Ľ ิ +Ġ×ijר שת +ãģıãģªãĤĬ ãģ¾ãģĻ +ĠпÑĢин ÑĨип +Ġ׊׾×ķ +ëı ¼ +×ķ×Ĵ ש +س س +à¸Ľ ู +Ġh ầu +æĦŁãģĺ ãĤĭ +ï¼ ´ +د ÙĪØ§ +ĠÑģм ог +scri ção +Ġth áºŃn +Ġר ×ķ×IJ×Ķ +обÑĢаж ен +ĠاÙĦتج ارÙĬØ© +Ø· بÙĬع +jÄħc Äħ +íĸī ìľĦ +Ġнов Ñĭй +Ġ×ŀ ×Ĺ×ĵש +æĮ¯ ãĤĬ +gu é +Ġ×IJ ×Ļר×ķ×¢ +Ġ×IJ×Ļר×ķ×¢ ×Ļ×Ŀ +ĠاÙĦ ذÙĩب +×ĵ ×IJ +ت اÙĨ +ãģł ãģĹ +à¸Ńั à¸ķรา +à¹Ĥ à¸Ī +بÙĦ اد +×Ķ×Ļ ×Ļ׳×ķ +ĠÑģп е +ĠÑģпе ÑĨиалÑĮно +ĠÅĽwi ata +ãĤĵãģ§ãģĻ ãĤĪ +شر ÙĥØ© +ĠpÅĤ yt +Ġsitu é +Ġ׼×IJ ׾×Ķ +ס ×ijר +Ġkaż d +Ġkażd ym +ãĤĴæĮģ ãģ¤ +׾×Ķ ×ľ +׾×Ķ׾ ף +ĠwÅĤ as +ĠwÅĤas ne +ĠsaÄŁ lan +×ŀ×¢ ׾×Ķ +ĠاÙĦا ÙĪÙĦ +ìĹIJìĦľ ëıĦ +×IJ×Ļר ×ķפ×Ķ +تÙĤ ÙĨÙĬØ© +Ùħ ائ +Ùħائ Ø© +Ġcompañ ÃŃa +Ġsü rek +Ġsürek li +ĠиÑģ кÑĥÑģ +ĠиÑģкÑĥÑģ ÑģÑĤв +ĠB ürger +ת ×Ĺר +ת×Ĺר ×ķת +à¸ŀรà¹īà¸Ńม à¸ģัà¸ļ +Ø´ Ùħ +à¸ĸืà¸Ń วà¹Īา +è¾¼ ãĤĢ +ä¼ij ãģ¿ +ĠاÙĦØ£ ب +ĠÑģÑĤоим оÑģÑĤÑĮ +ĠпÑĢав а +may ın +ห วย +ĠاÙĦØ· بÙĬعÙĬ +à¸Ĺีà¹Ī à¸ŀัà¸ģ +ĠEst á +Ñĭва ÑİÑĤ +ب سÙĬ +بسÙĬ Ø· +Ġ×ij×¢ ×ijר +åı¯èĥ½ ãģ§ãģĻ +Ġ×ĵ ×ķ׾ +Ġ×ĵ×ķ׾ ר +Ùĩ ÙİØ§ +воÑĢ Ð¾ÑĤ +ãģ¦ ãģĦãģ¾ãģĹãģŁ +à¹Ĥà¸Ĺร ศ +à¹Ĥà¸Ĺรศ ั +à¹Ĥà¸Ĺรศั à¸ŀ +à¹Ĥà¸Ĺรศัà¸ŀ à¸Ĺà¹Į +Ġ×§ ׳ +ĠاÙĦØ« ÙĨ +ĠاÙĦØ«ÙĨ ائÙĬØ© +Ġco ût +à¸ķิà¸Ķ à¸ķัà¹īà¸ĩ +Ġö rg +Ġörg üt +ĠاÙĦØ® ÙĦÙĬ +ĠاÙĦØ®ÙĦÙĬ ج +Ġb á»įn +×ķ׾×ķ×Ĵ ×Ļ +ëŀ ľ +ĠÐij олÑĮ +ĠÐijолÑĮ ÑĪ +×Ĵ ×ijר×Ļ×Ŀ +ÙĤ ÙĬد +×ij×Ļ×ĺ ×ķ×Ļ +æīĵ ãģ¡ +Ġol muÅŁ +f äh +fäh ig +ล าà¸Ļ +ĠÙĤ طر +ש פ×Ķ +èªŃ ãĤĵãģ§ +à¸Ĥ วา +Ġchi ếm +ãĤ¤ãĥ³ ãĤ¿ +ãĤ¤ãĥ³ãĤ¿ ãĥ¼ãĥ +ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥ į +ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥį ãĥĥãĥĪ +Ġ׾ש×ŀ ×ķר +Ġت رÙĥ +ĠترÙĥ ÙĬا +ר ×ķ×ĺ +ã썿ĢĿ ãģĦãģ¾ãģĹãģŁ +ĠاÙĦت ÙĤ +Ġd ư +ãģ¦ãģıãĤĮ ãĤĭ +ãģĹãģŁ ãģĵãģ¨ +Ġróż ne +ĠاÙĦØ· ÙģÙĦ +ĠPost é +Ġ×ŀש ×ķ×Ŀ +Ñį ÑĢ +ĠÑĢабоÑĤ аеÑĤ +ãĤ· ãĥª +ãĤ·ãĥª ãĥ¼ãĤº +Ġ×ij×Ķ ×Ĺ׾×ĺ +×§×Ķ ×Ļ׾×Ķ +ãĤ« ãĥ¡ +ãĤ«ãĥ¡ ãĥ© +ï¼ ¯ +ĠìĤ¬ ìĿ´ +Ġk ì +Ġth Æ°á»Ľc +ض بط +ÙĤب ÙĪÙĦ +åĪ¥ ãģ® +Ġparticul ière +ĠÑģво ем +Ġ×¢ סק +Ġעסק ×Ļ×Ŀ +×ij×Ĺ ×Ļר×ķת +×ij ×Ļ׳×ķ +à¸ĭ à¸Ń +Ġ×¢ ×ķ×ijר +ãģłãģ£ãģŁ ãģ®ãģ§ +ıld ıģı +Ùħ دار +Ùħدار س +주 ìĭľ +à¸Ńา ศ +à¸Ńาศ ัย +Ġt ấm +à¸ŀิ à¸Ī +à¸ŀิà¸Ī าร +à¸ŀิà¸Īาร à¸ĵา +ÑĤелÑĮ нÑĭе +Ñģк ÑĥÑİ +Ðľ Ðĺ +à¹Ģà¸ģ า +à¹Ģà¸ģา หล +à¹Ģà¸ģาหล ี +×ĵ ×Ĺ +à¹Ģà¸Ĭ ิà¸ĩ +Ġد ÙĤÙĬÙĤØ© +íķĻ ìĥĿ +Ġש×IJ ׾×Ķ +Ġcontr ôle +Ġsit uação +à¸Ĥà¸Ńà¸ĩ à¸ľà¸¹à¹ī +ÙĨ Ø·ÙĤ +ê³¼ íķĻ +หลาย à¸Ħà¸Ļ +Ġn ắng +ÙĤ Ùı +ì¡° ê±´ +Ñ ķ +ãĥĥ ãģ¨ +×ŀ ×Ļ׾×Ķ +Gr ün +×Ļ ×Ļ×¢ +×Ļ×Ļ×¢ ×ķ×¥ +×ŀ׳ ׼ +ë ŃIJ +×ŀ×¢ ×ŀ×ĵ +สำ à¸Ļัà¸ģ +ج دد +à¸Ħ ัà¸Ķ +Ġ×Ķ×ŀש פ +Ġ×Ķ×ŀשפ ×Ĺ×Ķ +×ŀש ק׾ +ÙĦ Ùı +Ġty tu +Ġtytu ÅĤ +ÑĪ ÐµÐ¹ +ĠìĿ¼ ë¶Ģ +ÑĪ ÐµÐ½Ð¸Ðµ +Ġph óng +ĠìĹŃ ìĤ¬ +ãĤ« ãĥ³ +Ġtú i +ĠÙĨ ÙĪÙģ +ĠÙĨÙĪÙģ Ùħبر +gr ün +ĠاÙĦØ´ ÙħاÙĦ +ÅĽwi adc +ÅĽwiadc zenie +ער ×Ķ +Ġ×¢ ×ķ×ij +Ġ×¢×ķ×ij ×ĵ×Ļ×Ŀ +×ĵ×ķ×Ĵ ×ŀ×IJ +ä»Ĭ ãģ¯ +Ġv ão +ĠТ ем +Ñģ илÑĮ +Ġch ợ +Ùħ را +Ùħرا ÙĤب +à¹Ħมà¹Ī รูà¹ī +Ġر ائع +×IJ׳ ×Ĺ׳×ķ +สà¹Īà¸ĩ à¹Ģสริม +צ ×Ĺ +ĠìŀĪìĸ´ ìĦľ +Ġkur ulu +Ġkurulu ÅŁ +ĠÃĸ zellik +ĠÃĸzellik le +Ġת ×Ļ×§ +Ġgh é +Ġspr zÄĻ +ĠsprzÄĻ t +ער ×ķת +را ØŃØ© +ãģ£ ãģį +ãģ£ãģį ãĤĬ +ĠìķĦ ëŀĺ +stit uição +Ġдолж но +×Ķ ×¨×© +×Ķרש ×ŀ×Ķ +×Ķ׾ ×ļ +ãģ¡ ãģª +ãģ¡ãģª ãģ¿ +ãģ¡ãģªãģ¿ ãģ« +פ ×Ĺ×ĵ +ĠاÙĦج ÙħÙĬع +×ij×¢ ׾×Ļ +Ġtr ùng +Ġפ ת×Ĺ +×ŀ׾×Ĺ ×ŀת +ãĥĨ ãĥ¼ãĥ +ãĥĨãĥ¼ãĥ ŀ +Ùħ تاب +Ùħتاب عة +Ġ모 ìĬµ +ÙĬ ص +åIJĪ ãģĨ +ĠY ap +ĠYap ı +ĠÑģ казаÑĤÑĮ +ëª ° +à¸Ĺีà¹Ī สำà¸Ħัà¸į +ĠìĹĨ ìĬµëĭĪëĭ¤ +Ġnh ắc +Ġülk eler +Ġмног ие +íķĺ ìħ¨ +มาà¸ģ à¸Ĺีà¹Īสุà¸Ķ +à¸ģ à¹īา +à¸ģà¹īา ว +Ġİ yi +л еж +леж а +ãĤ¸ ãĥ§ +à¸Ĺั à¸ŀ +ا ÙĪØ± +Ġ×Ĺ×ijר ×Ļ +Ġ׾ ש×Ŀ +ì² « +ĠT á»Ń +×ŀ ×ķ׳×Ļ +ÙĤ ÙĪØ¯ +à¸ģระ à¹Ģà¸Ľ +à¸ģระà¹Ģà¸Ľ à¹ĭ +à¸ģระà¹Ģà¸Ľà¹ĭ า +ĠпÑĢоблем Ñĭ +Ġaç ıs +Ġaçıs ından +Ġ×Ķ×ŀ ׼ +ĠÙħع ظÙħ +ÙĤÙĬ اس +ĠпÑĢод олж +ĠпÑĢодолж а +Ġver diÄŁi +ĠпÑĢед меÑĤ +ãģĦãģ¾ãģĻ ãģĮ +ĠëͰ 른 +ĠاÙĦ ÙĤÙĬاÙħ +ĠØ¥ÙĦÙĬ Ùĩا +Т ÐIJ +п оз +ãĤ· ãĥ¥ +ä¸ĬãģĮ ãĤĬ +à¹Ģà¸Ķิม à¸ŀัà¸Ļ +à¸ģุ ล +ØŃر ÙĬØ© +×§×ij×ķצ ×ķת +ë¯ ¿ +ĠاÙĦÙħ ÙĨا +ĠاÙĦÙħÙĨا Ø·ÙĤ +ĠвÑĭп ол +ĠвÑĭпол нÑı +ãĥĭ ãĤ¢ +Ġê²° êµŃ +×Ĺ ×ķ×ŀ +×Ĺ×ķ×ŀ ר×Ļ×Ŀ +ĠУкÑĢа инÑĭ +ห à¸Ńม +ר ×Ļס +ĠÑħоÑĤ ел +ĠобÑĢаз ованиÑı +Ġkh ẳng +Ġm ưa +Ġgör me +Ġgüç lü +سع Ùī +มัà¹Īà¸Ļ à¹ĥà¸Ī +íķĺ ê²łìĬµëĭĪëĭ¤ +Ġпол Ñĥ +Ġfün f +ã썿ĢĿ ãģ£ãģ¦ãģĦãģ¾ãģĻ +Ġê·¸ê²ĥ ìĿĢ +ĠdÃ¼ÅŁÃ¼n ce +ìŀ ł +ĠH Æ°á»Ľng +ĠTi á»ĥu +Ġç ift +ãģij ãģ° +à¸Īà¸Ļ à¸ĸึà¸ĩ +à¸Ĺำ à¹Ħà¸Ķà¹ī +ĠìŀIJ ì²´ +Ġd õ +Ġdõ i +à¸Ī ัà¸Ļ +à¸Īัà¸Ļ à¸Ĺ +à¸Īัà¸Ļà¸Ĺ รà¹Į +ece ÄŁini +׳×ķ×¢ ר +غ ار +ĠاÙĦØ£ÙħرÙĬ ÙĥÙĬ +داع Ø´ +ĠбезопаÑģ ноÑģÑĤи +Ġб Ñİ +ĠбÑİ Ð´Ð¶ +ĠбÑİдж еÑĤ +ãĥĬ ãĤ¤ +à¸ŀà¸ļ วà¹Īา +da ÄŁ +×IJ ×ķפף +íĹ Į +ãĥĢãĤ¤ ãĤ¨ +ãĥĢãĤ¤ãĤ¨ ãĥĥãĥĪ +ĠëĮĢ íĨµ +ĠëĮĢíĨµ ëł¹ +D İ +Ø£ ØŃداث +ĠA ÄŁ +ĠAÄŁ ust +ĠAÄŁust os +ØŃÙĦ ÙĪÙĦ +Ġw ÅĽ +ĠwÅĽ ród +ĠÑģо оÑĤвеÑĤ +ĠÑģооÑĤвеÑĤ ÑģÑĤв +ĠÑģооÑĤвеÑĤÑģÑĤв ии +ĠLu áºŃt +Ġ׼׾ פ×Ļ +Ġв еÑī +ĠвеÑī еÑģÑĤв +×§ ×Ļ×¥ +ĠبÙĩ ذا +عا Ø´ +à¹Ģà¸Ľà¹ĩà¸Ļ à¹Ģรืà¹Īà¸Ńà¸ĩ +Т Ðķ +Ġ×ij×IJ ×Ļ׳×ĺר׳×ĺ +س عد +Ġ×Ķ×ĺ ×Ļפ×ķ׾ +פ ×Ļס +à¸ĩà¹Īาย à¹Ĩ +ĠGer ät +׾ ×Ļ×ĵ×Ķ +ĠÑĢ Ð¸Ñģк +׾ק ×Ĺ +н наÑı +ר ×Ļ×ĵ +п ÑĢакÑĤи +пÑĢакÑĤи к +à¸Ĥัà¹īà¸Ļ à¸ķà¸Ńà¸Ļ +à¸Ļà¹Īา รัà¸ģ +larınız ı +à¸Ńà¸Ļุ à¸įา +à¸Ńà¸Ļุà¸įา à¸ķ +ĠzdjÄĻ cia +Ġb ây +Ñģ ÑĢ +ÑģÑĢ Ð¾Ñĩ +ãĥĭ ãĥ³ãĤ° +Ġö ner +Ġöner i +Ġнов ÑĭÑħ +دع ÙĪØ© +Ġg ắn +ĠاÙĦÙĦ بÙĨ +ĠاÙĦÙĦبÙĨ اÙĨÙĬ +ãĥĨãĤ£ ãĥ¼ +Ġص ØŃÙĬØŃ +ем ÑĭÑħ +çĸ² ãĤĮ +ĠпÑĢо иÑģ +ĠпÑĢоиÑģ ÑħодиÑĤ +ส à¸ķิ +ĠT ết +Ġ×Ķ׾ ׾×ķ +à¹Ģรืà¹Īà¸Ńà¸ĩ à¸Ļีà¹ī +×ŀ×ij ׳×Ķ +Ġconte údo +Ġا خت +Ġاخت ÙĬار +Ùħ سÙĦ +ÙħسÙĦ سÙĦ +ëı Ī +Ġ׾ ×Ļ×ĵ +à¸ŀิ à¸ĺี +ĠÑģов Ñģ +ĠÑģовÑģ ем +ãģĮãģĤãĤĬ ãģ¾ãģĹãģŁ +Ġsó ng +Ø¥ صÙĦاØŃ +ë§ ģ +Ùģ ÙĬر +ĠJe żeli +ìłľ ëıĦ +d ÅĤug +ìĥģ ìĿĦ +Ġc áºŃn +Ġhá»į p +Ø£ ست +أست اذ +Ġ×ŀ ×Ļש×Ķ +Ġ×ŀ×Ļש×Ķ ×ķ +Ġd Ãły +Ġch Ãłng +ãģ¡ãĤĥãĤĵ ãģ¨ +ĠÄij ám +Ġsw ój +Ġpoder á +ĠоÑĤлиÑĩ а +Ġpéri ode +ünd ig +×ĺ×¢ ף +ÑģÑĤÑĢо иÑĤелÑĮ +ר ת×Ļ +Ġ×Ļ×Ķ ×Ļ×ķ +׾ ס +ĠاÙĦÙħÙĨ زÙĦ +à¸Ļิ à¹īว +иÑĦ ика +иÑĦика ÑĨи +ðŁĺ ī +Ġad ına +ãĢĤãĢĤ ãĢĤ +×IJ ×Ļף +ס ×Ļר +ĠÙĬ عد +çŃĶ ãģĪ +اÙĦ جز +اÙĦجز ائر +енÑĮ к +ร ห +รห ัส +ĠTürk çe +ê¾ ¸ +Ġ×Ļ ×ķ׼׾ +Ġש ×ķ׳×Ķ +Ġ×ij×ŀ צ×ij +ĠдейÑģÑĤв иÑĤелÑĮно +ĠبأÙĨ Ùĩ +×ŀ×§ ×ĵ +Ġ×Ķש ×§ +Ø®ÙĬ ارات +Ġf ı +Ġfı rs +Ġfırs at +ëij ĺ +ĠìĦľ ìļ¸ +Ġ×Ķ×Ĵ ×ķ×£ +ر عا +رعا ÙĬØ© +ĠK ết +к Ñģи +ĠÑĥÑģлÑĥг и +ноÑģÑĤ ей +ìļ´ ëıĻ +ĠобÑĬ Ñı +ĠобÑĬÑı вл +н еж +×Ķפ ×ļ +Ġ×ij×¢ ×Ļ׳×Ļ +ëĨ Ĵ +ĠпÑĢоÑĨ ед +ĠпÑĢоÑĨед ÑĥÑĢ +Ġiht iy +Ġihtiy acı +Ġë°Ķ ëŀį +Ġë°Ķëŀį ëĭĪëĭ¤ +à¸ģล ัว +ĠÑģл ожно +×§×Ļ ×Ļ×ŀת +ĠÄIJ ình +ĠÙħ ÙĦÙģ +Ġà¹Ĥà¸Ķย มี +Ġkat kı +تØŃ ÙĪÙĬÙĦ +à¹Ħ à¸ŀ +ĠH á»į +ñ e +Ġдо Ñħод +Ġtho ải +íķĺìŬ ìķ¼ +ãĤ¹ãĥĿ ãĥ¼ãĥ +ãĤ¹ãĥĿãĥ¼ãĥ Ħ +ĠG òn +Ġk è +Ġkè m +é̲ ãĤģ +ãĤ¹ ãĥ¼ãĥ +ãĤ¹ãĥ¼ãĥ ij +ãĤ¹ãĥ¼ãĥij ãĥ¼ +ĠgiÃł u +ĠØ¥ عادة +Ġ׾ ×ķ×§ +Ġ׾×ķ×§ ×Ĺ +ĠÑħоÑĩ еÑĤ +×ĺ ׾×ķ×ķ +×ĺ׾×ķ×ķ ×Ļ×ĸ +×ĺ׾×ķ×ķ×Ļ×ĸ ×Ļ×Ķ +Ġth uyết +ãģĿãĤĮ ãģ§ +Ġvard ı +à¹Ħร à¹ī +ع بد +ĠRep ública +ãĥ¼ãĤ¿ ãĥ¼ +Ġ×ŀ×IJ ×ķת +à¹Ħà¸Ľ à¹ģลà¹īว +Ġyapıl acak +ãĤ¹ãĤ¿ ãĥ¼ãĥĪ +ãģ» ãģ¼ +Ġko ÅŁ +ĠмаÑĤ еÑĢи +Ġsiè cle +ĠاÙĦÙħ ختÙĦÙģ +ĠاÙĦÙħختÙĦÙģ Ø© +Ġ׾ק ר×IJ +Ġ׾קר×IJ ת +Ġ×Ķפ ×ķ×¢×ľ +Ġt òa +Ġr Æ¡i +åij¨ ãĤĬ +à¸Ŀ à¸Ļ +j ÅĽÄĩ +ĠìķĬ ìĿĦ +اÙĨت ÙĤاÙĦ +ëĸ ł +ив аеÑĤ +ãĥĪ ãĥ« +ĠاÙĦÙģÙĦسطÙĬÙĨ ÙĬØ© +à¸ģลà¹Īาว วà¹Īา +ا Ùĥت +ĠÃĸ l +ĠÑĢе ÑĪи +ĠÑĢеÑĪи л +Ġ׳×ķס פ×ķת +Ġìłķ ì¹ĺ +вл еÑĩен +Ùħر ØŃÙĦØ© +Ġcome ça +Ġy ık +ìĤ ´ +à¸ĺ à¸Ļา +à¸ĺà¸Ļา à¸Ħาร +à¸Ńà¸Ļ า +à¸Ńà¸Ļา à¸Ħ +à¸Ńà¸Ļาà¸Ħ à¸ķ +Ġpeque ña +ä»ķ äºĭãĤĴ +Ġب ذÙĦÙĥ +Ġнов ого +ãģĹãģ¦ ãģĦãģªãģĦ +ĠاÙĦÙħ ÙĬاÙĩ +à¸ģà¹ĩ à¹Ģà¸Ľà¹ĩà¸Ļ +Ġж ÑĥÑĢ +ĠжÑĥÑĢ Ð½Ð°Ð» +в еÑģ +خت ار +Ġ매 ìļ° +ĠM ã +ĠавÑĤомаÑĤ Ñĭ +ضع Ùģ +ĠاÙĦÙģ Ùĥر +ãģ§ãģĻ ãģ®ãģ§ +ãĥ¡ãĥ³ ãĥIJãĥ¼ +Ġк ÑĢÑĥг +ĠاÙĦسÙĦ طة +à¸Ħรัà¹īà¸ĩ à¹ģรà¸ģ +à¸ģระà¸Ĺ รว +à¸ģระà¸Ĺรว à¸ĩ +ÑĨ ов +éķ· ãģĦ +大ãģį ãģĦ +Ġgeç miÅŁ +ìĦ± ìĿ´ +Ġצר ×Ļ׼×Ķ +Ġм оÑī +ĠмоÑī н +Ġ×§ ×Ļש +Ġ×§×Ļש ×ķר×Ļ×Ŀ +ĠNas ıl +г ÑĢан +Ġ×ŀ ×ķצר×Ļ×Ŀ +Ġ×ŀס ×ķ×Ĵ +Ġy ür +Ġyür üt +Ġ׾׊צ×ķ +×ķÖ ¼ +ĠìŀĪ ìĹĪëĭ¤ +Ġter ör +ĠTh ương +ĠÙĪ ÙĬÙħ +ĠÙĪÙĬÙħ ÙĥÙĨ +ج ÙĪÙĨ +ĠÙĪØºÙĬر Ùĩا +×ŀ פ×ķ +×Ĵ×ķר ×ŀ×Ļ×Ŀ +׼×ij ×Ļש +ĠاÙĦÙĦ غ +ĠاÙĦÙĦغ Ø© +شر Ùĥ +ĠاÙĦر اب +ĠاÙĦراب ع +ĠпÑĢ ÐµÐº +ĠпÑĢек ÑĢаÑģ +ĠпÑĢекÑĢаÑģ н +Ġenerg ÃŃa +×§×ĵ ×ŀ×Ļ +ãģıãģª ãģ£ãģŁ +ĠÄij ứ +ĠÄijứ a +Serv i +Servi ço +Ġkald ır +åĥį ãģį +Ġод еж +Ġодеж д +물 ìĿĦ +ãģĿãģĨ ãģ§ +ãģĮãģĤ ãĤĮãģ° +ìĻ ķ +צ×ĵ ×§ +Ġart ır +Ġile ti +Ġileti ÅŁim +ãĤĪãģĨ ãģ§ +ãĥĪ ãĥ¼ +ãĤ¢ ãĥĭ +ãĤ¢ãĥĭ ãĥ¡ +×ĺ×Ļ ×Ļ׾ +ãĥķ ãĥªãĥ¼ +ãĥĿ ãĥ³ +ÐŁÑĢ Ð¾ +Ġع اÙĦÙĬØ© +ĠÃ¶ÄŁ ret +ĠÃ¶ÄŁret men +ĠкаÑĩеÑģÑĤв а +Ġ×Ķ×ĺ ×ij×¢ +Ġзна Ñİ +ãģ¦ ãģıãĤĭ +Ġm ừng +ÙħÙĪ Øª +ש ×ķ×ŀר +×Ĺ׾ ×ij +Ġwzgl ÄĻ +ĠwzglÄĻ du +ë²Ī 째 +Ġtá» ĵ +Ġtá»ĵ n +ãĥ¯ãĥ¼ ãĤ¯ +Ġpo życz +Ġpożycz k +×Ļ ×ķצר×Ļ×Ŀ +Ùĥر Ùħ +Ġг аÑĢ +ĠгаÑĢ Ð°Ð½ +ĠгаÑĢан ÑĤи +ล à¹īาà¸ĩ +Ġìĺģ íĻĶ +×ĺ ×Ļס +Ġth ẻ +ĠìŀĪëĭ¤ ê³ł +اÙĦت ز +اÙĦتز اÙħ +Ġна ÑĪи +is ée +ãģĵãĤĮ ãĤĴ +Ġm ẽ +ض ÙĦ +بÙĪ Øª +Ġ׼ ׼×Ķ +h ợ +ĠاÙĦس ÙĪØ±ÙĬØ© +Ġ×ľ×¢ ×ķ×ŀ +Ġ×ľ×¢×ķ×ŀ ת +ĠbaÅŁ ar +ĠbaÅŁar ılı +е ÑģÑĤÑĮ +à¸Ħร ี +à¸Ħรี ม +ĠìłĦ ì²´ +ĠسÙĬ ÙĥÙĪÙĨ +Ġ×ŀ×ĵ ×ķ×¢ +ĠëķĮ문 ìĿ´ëĭ¤ +Ġc ứng +ger ät +Ġм иÑĢ +ĠмиÑĢ Ðµ +ĠÙĥÙĬÙģ ÙĬØ© +Ġפר ×ĺ×Ļ×Ŀ +Ġgo ÅĽci +иÑĤ еÑģÑĮ +ÑĥÑĪ ÐºÐ¸ +ؤ ÙħÙĨ +Ġ×IJ ׼ף +ĠاÙĦر جÙĦ +Ġl á»įc +à¹Ģรีย à¸ģวà¹Īา +ãģĵãģ® ãĤĪãģĨãģª +ë§Į íģ¼ +Ġп еÑĩ +ÙĪÙĦ ات +ĠÃľ ye +liÄŁ inde +à¸Ħะ à¹ģà¸Ļ +à¸Ħะà¹ģà¸Ļ à¸Ļ +ãĤĭãģĵãģ¨ ãģ¯ +วิ à¹Ģà¸Ħร +วิà¹Ģà¸Ħร าะ +วิà¹Ģà¸Ħราะ หà¹Į +Ġвозмож ноÑģÑĤи +ĠاÙĦÙĨ ساء +ãĥīãĥ© ãĥŀ +Ġgü c +Ġgüc ü +Ġt ưá»Ŀng +Ġacomp aña +ãĤ¤ ãĥ© +×§ צ×ij +ĠY ö +ĠYö net +ĠYönet im +สัม à¸ľ +à¸ªà¸±à¸¡à¸ľ ัส +à¸Ļ าม +ĠÄij ợi +à¹ģหà¹Īà¸ĩ à¸Ĭาà¸ķิ +ãģĿãĤĮ ãģ§ãĤĤ +ät ig +ת ×ķ×Ŀ +ĠbaÅŁ lat +ĠвÑģ ей +ת ×Ļ×§ +ת×Ļ×§ ×ķף +ĠNg ô +ĠGesch ä +ĠGeschä fts +Ø£ Ùħ +Ø£Ùħ راض +à¹Ģà¸Ĺ à¸Ħà¸Ļ +à¹Ģà¸Ĺà¸Ħà¸Ļ ิ +à¹Ģà¸Ĺà¸Ħà¸Ļิ à¸Ħ +Ġм енÑĮ +ĠменÑĮ ÑĪе +Ġöl ç +Ġölç ü +ĠÙĬ جعÙĦ +ĠÄij ỡ +ש ×Ļ׾ +ש×Ļ׾ ×ķ×ij +ĠGr Ã¶ÃŁe +ĠÙĩ اتÙģ +รà¹īาà¸Ļ à¸Ńาหาร +×Ķ׾ ×Ļ׼ +×Ķ׾×Ļ׼ ×Ļ +иÑĢÑĥ ÑİÑī +èĭ¥ ãģĦ +ĠÃĸ zel +ãģĦãģŁ ãĤī +à¸Ħำ à¸ĸาม +Ġzosta ÅĤy +Ġ×Ķס ×Ļפ×ķר +×Ķ ×ķ׾ +×Ķ×ķ׾ ×ļ +à¹Ģà¸Ĭà¹Īà¸Ļ à¸ģัà¸Ļ +à¹Ĥ à¸Ĩ +à¹Ĥà¸Ĩ ษ +à¹Ĥà¸Ĩษ à¸ĵา +×IJר צ×ķת +×Ĵר פ×Ļ +Ġao ût +ĠÙĬ رÙĬد +ت ÙĪØ¬ +تÙĪØ¬ ÙĬÙĩ +ĠÑįÑĤ ап +ãĤ¹ãĤ¿ ãĥ³ +Ġkr ó +Ġkró tk +ãĤĴ使 ãģĨ +ì ·¨ +éĸ¢ ãĤı +à¸Ķà¹īวย à¸Ħวาม +à¸Ļำ à¹Ģสà¸Ļà¸Ń +Ġa yrıca +à¸Ī à¹īาà¸ĩ +ĠÑĦоÑĤ огÑĢаÑĦ +Ġв еÑĩ +ĠвеÑĩ еÑĢ +åĩº ãģĹãģŁ +ĠÐ¥ о +Ġ×ŀ ר×Ĵ×Ļש +à¹ĥหà¹ī à¹Ģà¸Ľà¹ĩà¸Ļ +ãĤĴ 缮 +ãĤĴ缮 æĮĩ +׾ ×ŀ×Ļ×Ŀ +nÄħ ÅĤ +ĠÑģÑĤ анд +ĠÑģÑĤанд аÑĢÑĤ +ĠSü d +ĠT âm +اخت بار +à¹Ģà¸ģ à¸Ńรà¹Į +Ùħس رØŃ +Ġbi á»ĩn +ب Ùı +Ġص اÙĦ +ĠصاÙĦ ØŃ +ĠPh ụ +íľ ´ +ãĥ¬ãĥĵ ãĥ¥ãĥ¼ +Ġbụ ng +Ġrég ime +ĠØ£ Ø´Ùĩر +ĠÑĢабоÑĤ ник +à¸Ŀ ัà¸Ļ +اع تÙħ +اعتÙħ اد +Ġзам еÑĤ +ãģ¾ ãģ£ãģ¦ +Ġch ặt +æĿ¥ ãĤĭ +ĠاÙĦÙĤ ÙĪØ§Øª +ãģ«åħ¥ ãģ£ãģ¦ +تØŃ اÙĦÙģ +Ùħ زÙĬد +ĠÙĬ صÙĦ +ìĹ ¼ +à¹Ģà¸Ĭ à¹ĩ +à¹Ģà¸Ĭà¹ĩ à¸Ħ +Ġk á»ĭ +Ġká»ĭ p +ĠìķĦ ì§ģ +×IJ׳ ×Ĵ +Ġобла ÑģÑĤÑĮ +Ġpomoc Äħ +Ġ×ķ ש׾ +ëĵł ì§Ģ +ĠGi ám +ĠSt ück +Ġchá y +ĠëĤĺ ìĺ¤ +ש ×Ļ×ĺת +×ŀ×ĵ ר +×ŀ×ĵר ×Ļ×ļ +Ġsüre ç +к ва +×ij׾ ×Ļ×Ŀ +×Ķ ×ª×Ļ +×Ķת×Ļ ×Ļ×Ĺס +ÙĤب اÙĦ +Ġס ×ķ×Ĵ +Ġס×ķ×Ĵ ×Ļ +ÑģÑĤ олÑĮ +ä½ķ ãĤĤ +×ĸ׼ ×ķר +è²· ãģĨ +å®ī ãģı +à¸Ħรัà¹īà¸ĩ à¸Ļีà¹ī +kö p +ĠÑģеÑĢ Ð²Ð¸Ñģ +оÑĩ нÑĭÑħ +ê±° ëŀĺ +تأ Ùĥ +تأÙĥ ÙĬد +×ĵ ׾ק +Ġпо Ñĩем +ĠпоÑĩем Ñĥ +пиÑģ аÑĤÑĮ +×ij שר +ĠH Ãłng +ĠT ìm +Ġtr ừ +ãĤ» ãĥĥãĤ¯ãĤ¹ +×ķ׳ ×Ĵ +mız da +п Ñģи +ĠìŀĪ ê¸° +Ġr út +ز اÙĨ +تÙĨ ÙĪØ¹ +ÙħÙĤ ا +ÙħÙĤا ÙĪÙħØ© +Ġ׾צ ×ķר×ļ +Ġ×ij ×Ļר×ķש׾×Ļ×Ŀ +ãĥ´ ãĤ£ +eb ile +ebile ceÄŁi +ãĥ¦ ãĥ¼ãĤ +ãĥ¦ãĥ¼ãĤ ¶ +ãĥ¦ãĥ¼ãĤ¶ ãĥ¼ +ãĤĴä½ľ ãĤĭ +Ñģ меÑĢ +ÑģмеÑĢ ÑĤ +Ġì§ ģ +Ġì§ģ ìłij +ĠÐŁ аÑĢ +ØŃ اض +ØŃاض ر +Ùħ ÙĥاÙģ +ÙħÙĥاÙģ ØŃØ© +ล ิà¸Ļ +ãģ¦ ãģįãģ¦ +ÑĢоÑģ л +ĠÄ°ÅŁ te +ÙĤص ÙĬر +Ġ×ij×Ĵ ×Ļ׾ +Ġ×ŀת ×IJ×Ļ×Ŀ +Ġ×Ķ ×Ĺ×ĵ +Ġ×Ķ×Ĺ×ĵ ש×Ķ +ר ×ķ×¢ +Ġprodukt ów +ĠÙħ صدر +не ÑĨ +ĠاÙĦعÙħÙĦ ات +Ġçık ma +Ġد بÙĬ +×§ ×Ļף +ת ×IJר +ת×IJר ×Ļ×ļ +׳×Ļ ×Ļ×ĵ +صر اع +l ève +צ ×Ļר +à¸Ķ ัà¸Ļ +à¹ĥหà¹ī à¹Ħà¸Ķà¹ī +ãĤ¿ãĤ¤ ãĥł +Ġgi ảng +С ÐŁ +ĠاÙĦÙħ ØŃÙĦ +ĠاÙĦÙħØŃÙĦ ÙĬØ© +ĠT ất +׾ ×ķ×ĺ +h á»ķ +Ġam éric +Ġaméric ain +Ġ×ijש׾ ×ij +Ġ׾×IJ ×ķ×ŀ×Ļ +Ġpe ça +ĠÑĢаз нÑĭÑħ +ãģĦãĤĭ ãģ¨ +ãĥĩ ãĥ³ +ס קר +Ġ×Ķ×ŀ×Ĺ ×Ļר +ãģ¨ãģĦãģĨ ãĤĤãģ® +رت بط +ĠиÑģÑĤ оÑĩ +ĠиÑģÑĤоÑĩ ник +สมัà¸Ħร สมาà¸Ĭิà¸ģ +Ġ à¸Ĺัà¹īà¸ĩ +Ġà¸Ĺัà¹īà¸ĩ à¸Ļีà¹ī +ĠT áºŃp +ãģ£ãģ¦ ãģĦãģĨ +ĠاÙĦÙĪ ØµÙĪÙĦ +Ġdéc ada +Ġо ÑĦоÑĢм +ĠоÑĦоÑĢм лен +สำหรัà¸ļ à¸ģาร +Ġog óln +ãģĨãģ¡ ãģ« +Ġvá rias +ãģĻãģİ ãĤĭ +ÙĪ Ùĩا +à¹Ĥà¸Ľà¸£ à¸Ķ +ĠÐłÐ¾ÑģÑģ иÑı +人 ãĢħ +ãģĹãģ¦ ãģįãģŁ +Ġsı rasında +Ġng ôn +س ÙĨØ© +تÙħ تع +×ŀ׼ ×ij×Ļ +Ġnh ấn +×¢ ×ŀ×Ļ×ĵ +á» ¨ +ж иÑĤÑĮ +ãĤī ãģĽ +gr áf +gráf ica +ĠÙĤ ÙĪÙĦ +ĠÙĤÙĪÙĦ Ùĩ +ëĭ¨ ì²´ +ห à¹īา +หà¹īา ม +使 ãģ£ãģ¦ +ת ×Ļ×ij +ת×Ļ×ij ת +i á»ĥu +à¹ģ à¸Ĭม +à¹ģà¸Ĭม à¸Ľ +à¹ģà¸Ĭà¸¡à¸Ľ à¹Į +Ạ¬ +ĠëĤĺ ëĿ¼ +ĠÙħباشر Ø© +Ġtr Äĥm +سÙĥ ÙĪ +ĠاÙĦذ Ùī +Ġbi ç +Ġbiç im +ت راجع +Ġоб еÑģп +ĠобеÑģп еÑĩ +ĠобеÑģпеÑĩ ива +Ġвозд ÑĥÑħ +Ñĭв аÑĤÑĮ +ÙĦ ØŃÙĤ +ĠMü dü +ĠMüdü rl +ĠMüdürl Ã¼ÄŁÃ¼ +Ġyapt ır +Ġפר ס +Ġפרס ×ķ×Ŀ +Ø· ÙĪØ± +ÑģÑĤв оваÑĤÑĮ +ìŀ¥ ìĿĦ +à¸Ĺีà¹Īà¸Ķี à¸Ĺีà¹Īสุà¸Ķ +à¸Ńั ล +ÑĢ Ñİ +Ùħست ÙĤبÙĦ +Ñģл ÑĥÑĪ +ÑģлÑĥÑĪ Ð° +èªį ãĤģ +Ġ׾ ×Ļ×ŀ +Ġ׾×Ļ×ŀ ×ķ×ĵ×Ļ +ת ש×ķ×ij +תש×ķ×ij ×ķת +ĠgerçekleÅŁtir il +ĠاÙĦ اتÙ쨧ÙĤ +ĠÑĥÑĢов не +ĠÑĤ ÑĢав +Ġ×Ķ×ŀ ×ķף +ØŃÙģ Ø§Ø¸ +ĠÙħ ÙIJ +ĠÙħÙIJ ÙĨ +ĠÙħÙIJÙĨ ÙĴ +Ġdem ás +×ŀ×ķ×ĸ ×Ļ×§×Ķ +ש ×Ļ×Ĺ×Ķ +Ġb ú +алÑĮ нÑĭм +ãĤı ãģŁ +ãĤıãģŁ ãģĹ +ĠاÙĦÙħÙĪ Ø§Ø¯ +ת ׼׳ +×ª×Ľ×ł ×ķף +ãĥŃ ãĥĥãĤ¯ +hi ếu +ĠÑĥ ме +ÙħØŃا ÙĪÙĦØ© +×IJ ×ķשר +Ġкон кÑĥÑĢ +ĠконкÑĥÑĢ Ñģ +Ġ×ŀ ×ij×Ĺ +Ġ×ŀ×ij×Ĺ ×Ļ×ł×ª +Ġan lam +Ġanlam ı +Ġli á»ĩt +Ġв Ñħод +ĠH ình +ĠÙĨ ÙĬ +ĠÙĨÙĬ ÙĪØ² +ãĤ¸ãĥ£ ãĥ¼ +×ij ×Ļ×¥ +ÑĤелÑĮ нÑĭÑħ +à¸Ĺุà¸ģ à¸Ńยà¹Īาà¸ĩ +ĠkiÅŁ inin +Ø£ Ùĥثر +ĠиÑģÑĤоÑĢ Ð¸Ð¸ +Ġë³Ģ íĻĶ +פ׾ ס×ĺ +×¤×ľ×¡×ĺ ×Ļ׳×Ļ +ĠÑģ еÑĤ +ĠÑģеÑĤ и +dıģ ımız +íķĺ ëıĦë¡Ŀ +×Ķ ×¨ +×Ķר ×ij×Ķ +ãģĻãĤĭãģĵãģ¨ ãģ¯ +Ġphi ếu +تØŃ سÙĬÙĨ +ĠÅĽ rod +ĠÅĽrod ow +ĠÅĽrodow isk +ĠÑĢаÑģ Ñħод +بر ÙĬد +Ġر ÙĬ +ĠرÙĬ اÙĦ +Ġ×ķ ׼×ļ +ì§Ģ ìļĶ +׼ ×ŀ×ķ +Ġ×¢×ľ ×Ļ×Ķ×Ŀ +f ÃŃcio +Ġkar arı +tıģ ını +ĠС ов +ĠСов еÑĤ +ãģĬéĩij ãĤĴ +м еждÑĥ +междÑĥ на +междÑĥна ÑĢод +междÑĥнаÑĢод н +Ġm á»Ŀi +ĠاÙĦØ¥ ÙĬر +ĠاÙĦØ¥ÙĬر اÙĨÙĬ +ĠاÙĦرÙĪ Ø³ÙĬ +ص ÙĨد +صÙĨد ÙĪÙĤ +ĠاÙĦØ¥ÙĨ ترÙĨت +Ġt ắm +ĠÑĤак ого +Ġ×ij ׾×ķ×Ĵ +Ġü crets +Ġücrets iz +×Ĺ×ĸ ×Ļר +ìĸ´ ìķ¼ +ĠPh ần +ï¼ ľ +Ġ×ĺ ×ij×¢ +Ġ×ĺ×ij×¢ ×Ļ +×IJ×ŀ ×IJ +اÙĤ ÙĦ +Ġcondi ções +ÙĤات ÙĦ +ĠÑĢезÑĥлÑĮÑĤаÑĤ е +ĠÑģво ими +צ×ij ×Ļ×¢ +gé ni +Ġz es +Ġzes po +Ġzespo ÅĤ +ÑĪ Ð¸Ð² +Ġפר×ĺ×Ļ ×ķת +Ùħست Ø´Ùģ +ÙħستشÙģ Ùī +شر ع +Ġko ÅĽci +Ġ×Ķ×IJ ×Ļ׳×ĺר׳×ĺ +ĠЧ еÑĢ +поÑĩ ÑĤ +Ġactiv ités +çŁ¥ ãģ£ãģ¦ +Ġ×ij ×ĸ×Ķ +Ġyüz den +ãģªãĤĬ ãģ¾ãģĽãĤĵ +Ġíĺ ¹ +Ġíĺ¹ ìĿĢ +Ġ×ŀש ׳×Ķ +ĠÐĴ еÑĢ +Ġ×ij×IJ×ķת ×ķ +éĿ¢ çϽ +éĿ¢çϽ ãģĦ +شر ØŃ +gr ünde +Ùģ Ø´ +Ù쨴 ÙĦ +Ġsé jour +ë´ IJ +Ġr ôle +Ø´ عار +ем Ñĭе +ĠاÙĦج سÙħ +алÑĮ ное +Ġìĥģ íĥľ +ï¼ ¤ +ë¯Ģ ë¡ľ +ĠÙĨ ÙĤØ· +ĠÙĨÙĤØ· Ø© +ãģĿãģĨ ãģł +ãģĻãĤĭ ãģ®ãģĮ +ห ู +Ġnh á»ĭ +Ġeconóm ica +ס×ĺ ×ķ×ĵ +ס×ĺ×ķ×ĵ ׳×ĺ +มี à¹Ĥà¸Ńà¸ģาส +Ġgest ão +รูà¹ī วà¹Īา +Ġlo ạt +ĠاÙĦÙħ Ùı +ĠاÙĦØŃ ÙħÙĦ +ĠاÙĦعÙħÙĦ ÙĬØ© +Ġê²ĥ ëıĦ +ĠÐľÐ¾Ñģк ва +×§×ĺ ×ķר +Ġпод ÑĢоб +ĠподÑĢоб н +Ġl ưng +ت Ù쨳 +تÙ쨳 ÙĬر +ĠاÙĦ بع +ĠاÙĦبع ض +ئ ت +Ðķ ÐĿ +ìŰ 구 +à¹ĥหà¹ī à¸Ħุà¸ĵ +ãģĤãĤĬ ãģ¾ãģĹãģŁ +Ġbir ka +Ġbirka ç +Ġİ sl +Ġİsl am +çĹĽ ãģ¿ +Ġh ảo +Ġм аÑı +ĠiÅŁ çi +ש × +×©× ģ +à¸ģาร à¹Ģมืà¸Ńà¸ĩ +×ķ×Ķ ×¨ +Ġch ó +ëĨ Ģ +Ġyan lı +Ġyanlı ÅŁ +幸 ãģĽ +×IJר×Ĵ ×ķ׳×Ļ +à¸Ńาà¸Ī าร +à¸Ńาà¸Īาร ยà¹Į +ĠинÑĦоÑĢм аÑĨиÑİ +Ðĵ Ðŀ +׳ ×Ĺש +ĠìķĮ ìķĦ +ĠÑħаÑĢакÑĤеÑĢ Ð¸ÑģÑĤ +ĠÑħаÑĢакÑĤеÑĢиÑģÑĤ ик +à¸Ħุà¸ĵ สามารà¸ĸ +è¦ĭ ãģĪãĤĭ +à¸Ĭัà¸Ķ à¹Ģà¸Ī +à¸Ĭัà¸Ķà¹Ģà¸Ī à¸Ļ +ĠdziaÅĤ al +ĠdziaÅĤal noÅĽci +à¹Ĥà¸ŀ สà¸ķà¹Į +ĠÐļ ол +ĠÙģ ÙĩÙĬ +Ġ×ŀ פ׳×Ļ +Ġ×Ķ×§ שר +Ùħر Ùĥ +ÙħرÙĥ ز +Ġho á +Ġа пп +Ġапп аÑĢаÑĤ +Ġp ami +Ġpami ÄĻ +ĠpamiÄĻ ta +Ġç ünkü +×ĵ ×ķף +ãģ¯ ãģĵãģ¡ãĤī +ĠM Ãł +ĠÙĬ ÙĤدÙħ +ĠпÑĢ ÐµÐ· +ĠпÑĢез иденÑĤ +à¸Ńุ à¸ķ +à¸Ńุà¸ķ สา +à¸Ńุà¸ķสา ห +à¸Ńุà¸ķสาห à¸ģรรม +ì§Ģ ìĽIJ +Ġ×IJפשר ×ķת +sch üt +schüt z +ĠTi ên +Ġsay ılı +ĠгÑĢÑĥпп Ñĭ +оÑĩ нÑĭй +Ġ×ľ×¢ ×ŀ×ķ×ĵ +Ġwr zeÅĽ +ĠwrzeÅĽ nia +ĠÄIJ ầu +à¹Ģà¸Ĥà¹īา รà¹Īวม +nız da +Ø®ÙĬ ص +Ġgü nc +Ġgünc el +ĠÙĦÙĩ ذÙĩ +ĠÙĬ عتبر +lé gi +ãĤı ãģĭãĤĭ +Ġr ừng +ظ Ùĩ +ظÙĩ ÙĪØ± +Ġ×ŀ×ij ×Ļף +Ġ기 íĥĢ +åĪĩ ãĤĮ +lan mÄ±ÅŁ +à¸Ĺีà¹Ī มีà¸Ħวาม +Ġh á»ģ +ت ÙĪØ¬Ùĩ +ĠاÙĦØ¥ دارة +Ġú til +ס פ×ķ +à¸Ħวาม รัà¸ģ +à¹Ĥ ฮ +Ġпол иÑĤ +ĠполиÑĤ ик +Ġsat ın +ĠÅŀ imdi +×ŀ ×ķר×Ļ×Ŀ +ìķĺ ëĭ¤ +×Ĺ ×ķ×ķ +×Ĺ×ķ×ķ ×Ļ×Ķ +à¸Ħà¸Ńม à¸ŀิ +à¸Ħà¸Ńมà¸ŀิ ว +à¸Ħà¸Ńมà¸ŀิว à¹Ģà¸ķà¸Ńรà¹Į +Ġا ذا +تخ اذ +ãĤ¨ ãĥ« +Ġpossibilit é +ยืà¸Ļ ยัà¸Ļ +Ġü nivers +Ġünivers ite +ĠاÙĦد ÙĪØ±ÙĬ +ĠìķĬëĬĶ ëĭ¤ +ĠìĦľ ë¡ľ +ØŃ اÙĦ +Ġë ¨ +Ġë¨ ¼ +Ġ먼 ìłĢ +à¸Ĺีà¹Ī à¸ĸูà¸ģ +ì§ ľ +Ġsk óry +лÑĮ ÑĨ +à¹ĥà¸Ĭà¹ī à¹Ģวลา +×ij×§ שת +Ġذ ÙĪ +æĹ¥ ãĢħ +ĠкоÑĤоÑĢ ÑĥÑİ +ĠÑĥÑĢов енÑĮ +ê¹ ¨ +à¹Ħ à¸Ĺ +ãĤµ ãĥĹãĥª +ãĤ¸ ãĥ§ãĥ³ +ãģĻ ãģ¹ãģį +ĠG ór +ãĥĪ ãĤ¤ +ãĥĪãĤ¤ ãĥ¬ +ĠyaÅŁ ama +Ġdá»ĭ p +Ġb ữa +à¸ĭ ุ +Ġöl üm +ãģ£ãģ¦ ãģıãĤĭ +à¸ģาร à¸Ħà¹īา +ש ער +ĠÑĤип а +Ġг еÑĢ +ĠгеÑĢ Ð¾ +רק ×¢ +Ġu waż +Ġuważ a +ש×ŀ ף +Ġhast alık +ãĤıãĤĮ ãĤĭ +ba ÅŁÄ± +Ñĩ ÑĤо +Ġ×ij ×ŀר׼×ĸ +Ġìļ°ë¦¬ ìĿĺ +ĠÙĥاÙĨ ÙĪØ§ +ĠØ£ بر +Ġأبر ÙĬÙĦ +ì¸ µ +à¹Ħà¸Ĥ à¹Ī +ĠÙĪ ÙĦÙĪ +à¸Ĺ ัว +à¸Ĺัว รà¹Į +ĠÙĪØ£ Ùĥد +à¸Ĭ วà¸Ļ +׾ ×ķ×§ +æį ¨ +æį¨ ãģ¦ +Ġİç in +p éri +Ġy al +Ġyal nız +ÑĮÑı н +Ġg ắng +à¸ģà¹ĩ ยัà¸ĩ +ĠУкÑĢа ин +ĠÑģ ами +ĠпÑĢовед ен +à¸ķà¸ģ à¹ģà¸ķà¹Īà¸ĩ +ĠQu ân +é paration +ĠbaÅŁ ında +Ġzn ale +Ġznale ź +Ġznaleź Äĩ +ãĤ± ãĥ¼ +ãĥİ ãĥ¼ +à¸ĸูà¸ģ à¸ķà¹īà¸Ńà¸ĩ +ëª ¸ +Ġëı Į +ĠëıĮ ìķĦ +ĠSch üler +Ġпод гоÑĤов +ĠподгоÑĤов к +ع رÙĪ +عرÙĪ Ø¶ +la ÅŁtır +ĠÑģоÑģÑĤав лÑıеÑĤ +ĠпÑĢоиз вод +ĠпÑĢоизвод ÑģÑĤва +ĠоÑģнов е +ĠØ´ ÙħاÙĦ +à¸ģร ี +ĠgörÃ¼ÅŁ me +оÑĩ ек +Ġ×Ĺ×ijר ×Ļ×Ŀ +ÙħØ® اط +Ùħخاط ر +ï¼ Ń +ר פ×IJ +ĠM ẹ +ยà¸Ńม รัà¸ļ +Ġv ết +Ø® ذ +ĠاÙĦت Ø· +ĠاÙĦتط بÙĬÙĤ +à¸Ļ ึà¸ģ +Ġ×Ķ ×Ľ×ł×¡×ª +ĠогÑĢ Ð°Ð½Ð¸ +ĠогÑĢани Ñĩен +ĠÃĩ alÄ±ÅŁ +ĠاÙĦÙħÙĨت دÙī +à¸Īำà¸Ļวà¸Ļ มาà¸ģ +ĠÑĤоÑĢ ÑĢ +ĠÑĤоÑĢÑĢ ÐµÐ½ÑĤ +ĠìĤ´ ìķĦ +à¸ŀลัà¸ĩ à¸ĩาà¸Ļ +à¸Ĭ ัà¸Ļ +ĠÐIJн дÑĢ +Ġréalis é +×ŀש ×IJ +à¹ģ à¸Ĭ +à¹ģà¸Ĭ รà¹Į +Ġб ог +มา à¹ģลà¹īว +ĠاÙĦÙĨ ار +Ġolmad ıģı +×ĵ ×¢×Ķ +ĠÑĥ веÑĢ +ĠÑĥвеÑĢ ÐµÐ½ +ãĤĭ ãĤĤãģ® +Ø£ د +أد ÙĪØ§Øª +Ġ×Ķ×ĸ ×ķ×Ĵ +Ø¥ عÙĦاÙħ +h á»ı +ĠNä he +ĠÑĤ еÑģÑĤ +Ġ×ŀ ×ķ׼ר +Ġë¬¸ìłľ ê°Ģ +ת ×ķצ×IJ×Ķ +m ó +mó vel +ĠاÙĦتج ارة +Ġмног иÑħ +обÑī а +Ġ×¢ סק×Ļ +ĠEdu cação +×§ ש×Ļ×Ŀ +é tabl +établ issement +Ġд еле +иÑĢÑĥ еÑĤÑģÑı +Ø¢ ثار +Ġ×Ķ×ŀ ר׼×ĸ×Ļ +ãĥIJ ãĥ« +ĠвÑģÑĤÑĢ ÐµÑĩ +ãģĴ ãĤĭ +Ġci Äħ +ĠciÄħ gu +ÙĬ ست +à¸łà¸² ว +à¸łà¸²à¸§ ะ +Ø£ Ùħر +Ġо жи +Ġожи да +Ġ á»§y +ãĥŀ ãĥ« +ر اس +оÑĩ ной +ת ×Ĵ×ķ×ij×ķת +تع رÙĬÙģ +ĠÑģо ÑĨиалÑĮно +ãĤĴ éĸĭ +ĠиÑģÑģлед ова +Ġd ú +Ġdú vida +Ġsk ÅĤ +ĠskÅĤ ada +Ġhä ufig +ĠвÑĭб ÑĢ +ĠвÑĭбÑĢ Ð°ÑĤÑĮ +ãģ®ãģ§ãģ¯ãģªãģĦ ãģĭ +ĠÑģ илÑĮно +ÑĤвеÑĢж ден +ר פ +רפ ×ķ×IJ×Ķ +æĢĿ ãģĦãģ¾ãģĻ +ØŃر ص +ש×ķת ×£ +Ùħس جد +à¹Ĥà¸Ĭ วà¹Į +ем ÑģÑı +в ÑĪие +Ġм л +Ġмл н +Ġ׾×Ķ ×ij×Ļ×IJ +ĠÙĬ تعÙĦÙĤ +à¸ķ ูà¹ī +Ġп ÑĢаз +ĠпÑĢаз д +ĠпÑĢазд ник +Ġн ем +Ġнем ного +Ġs Ãłng +تÙĨ سÙĬ +تÙĨسÙĬ ÙĤ +Ġtá» Ŀ +Ġмед и +ãģ« æĪ +ã쫿Π» +à¸Ħว à¹īา +ãģĭ ãģijãĤĭ +×ij׾ ×ķת +ĠÑįк Ñģп +ĠÑįкÑģп еÑĢÑĤ +Ġдев ÑĥÑĪ +ĠдевÑĥÑĪ Ðº +ĠØŃ ص +ÙĨØ´ Ø£ +ãģĮãģĤãĤĭ ãģ®ãģ§ +Ġت راÙħ +ĠتراÙħ ب +أس ÙĪØ§ÙĤ +Ġ׾פ ׳×ķת +Ġا ï»· +ãģ« ãģı +ãģ«ãģı ãģĦ +ĠØ£ عÙĦÙī +Ġ׾×Ķ ×ŀש×Ļ×ļ +rä u +ש×ŀ ×Ļ×Ŀ +åĪĨ ãģij +ãģĻ ãģ§ +ãģĻãģ§ ãģ« +×Ķ׾ ׼×Ķ +×Ĺ׾ ×Ļ×£ +Ġì ±ħ +Ġì±ħ ìŀĦ +à¹Ģà¸Ī ริ +à¹Ģà¸Īริ à¸į +éģĬ ãģ³ +ج سد +สา à¸ĺ +สาà¸ĺ าร +สาà¸ĺาร à¸ĵ +Ġbas ın +ÑĢаР³ +г ад +Ġho ÅŁ +íķ µ +×ij×Ĺ ×Ļר×Ķ +×ŀס ×ļ +Ġìłľ íĴĪ +تÙħ ÙĪÙĬÙĦ +ĠL ưu +ë¡ľ ë¶ĢíĦ° +Ġп об +Ġпоб ед +ÙħÙĨ ذ +常 ãģ« +ÙĤ س +ĠاÙĦÙħ صدر +ĠÙĪØ§ÙĦ است +Ġkh ắp +ĠاÙĦج اÙĨب +Ġng uyá»ĩn +éĸĵ éģķãģĦ +ĠÑģÑĤ ÑĢа +ĠÑģÑĤÑĢа Ñħ +ĠÑģÑĤÑĢаÑħ ов +รี à¸ļ +Ġx ương +Ġì° ¾ +Ġì°¾ ìķĦ +Ġng ại +г ал +à¸ĭ ีà¹Ī +Ġ×ij פ×Ļ×Ļס×ij×ķ×§ +Ц енÑĤÑĢ +Ġaval iação +Ġeconóm ico +×ĸ ף +ĠÐľ ак +Ġinter és +à¸ģล ิà¹Īà¸Ļ +ÑģÑĤÑĮ Ñİ +ĠÄij ương +å¼· ãģı +ĠKh ách +à¹Ģà¸Ļืà¹īà¸Ń หา +ĠYaz ı +è²· ãģ£ãģ¦ +Ðł Ðķ +à¹Ģà¸ŀิà¹Īม à¸Ĥึà¹īà¸Ļ +สม à¸ļู +สมà¸ļู รà¸ĵà¹Į +Ġм иÑĢов +×Ĵ ׳×Ļ×Ŀ +ĠÄij ức +à¸Ń ารà¹Į +ص اص +ãģĬ ãĤĪ +ãģĬãĤĪ ãģ³ +ÃªÌ ī +ĠاÙĦÙħؤ تÙħر +ĠاÙĦÙħر ØŃÙĦØ© +สà¸Ńà¸ļ à¸ĸาม +Ġà¸Īาà¸ģ à¸Ļัà¹īà¸Ļ +Ġت عد +ãģĿãģ® ãģŁãĤģ +Ġkh áng +à¸Ļ ิà¸Ķ +ãĥĬ ãĥ³ +ëĦ¤ ìļĶ +ĠاÙĦ اØŃت +ĠاÙĦاØŃت ÙĦاÙĦ +ìļ ķ +Ġмод ели +ĠпÑĢоÑĨ енÑĤ +à¸ŀวà¸ģ à¹Ģรา +Ġ×Ķצ ×ĵ +Ġ×Ķצ×ĵ ×ĵ×Ļ×Ŀ +ständ e +׳ ×Ĵר +Ġdot yc +Ġdotyc zÄħ +ĠdotyczÄħ ce +ĠÅĽ wiÄĻt +×ŀר ×Ķ +ãģĻãģĶ ãģĦ +ãĥĩãĤ£ ãĥ³ãĤ° +à¸ģาร สรà¹īาà¸ĩ +ë Ĥ¬ +Ġì°¸ ìŬ +Ñģ Ñħ +ÑģÑħ ем +ÙħÙĪ Ø³ +Ġn ấu +Ġ׾×ŀ×¢ ׾×Ķ +à¹Ģà¸Ľ à¹īา +à¹Ģà¸Ľà¹īา หมาย +Ġmù i +ائ ز +íĽ Ī +×Ĺ×ij ×ķר×Ķ +à¸ľà¸¹à¹ī à¹ĥà¸Ĭà¹ī +Ġpa ź +Ġpaź dzi +Ġpaździ ern +Ġpaździern ika +ลà¸ĩ à¹Ħà¸Ľ +ÙĤ اع +Ġch áºŃm +Ġözellik leri +ĠÄIJ o +ĠÄIJo Ãłn +ж ение +Ġh ẳ +Ġhẳ n +ĠaÅŁ k +ï½ į +ãĥij ãĤ¹ +×Ķ×ķר ×IJ×ķת +ĠÅ » +ĠÅ» y +×ŀ×ĸ ׾ +ĠÑĥ кÑĢа +ĠÑĥкÑĢа ин +à¹Ģà¸Ĭ ิ +à¹Ģà¸Ĭิ à¸į +Ðł Ðĺ +ĠzwiÄħz ku +×Ķ×Ĺ׾×ĺ ת +ãĤĵãģ§ãģĻ ãĤĪãģŃ +ãģ¦ ãģĬãĤĬ +лож иÑĤÑĮ +×ŀ ×ķ׳×Ļ×Ŀ +ฮ ิ +ì° ¬ +ĠاÙĦÙħØ´ ترÙĥ +ĠdÃ¼ÅŁ ük +аг енÑĤ +ĠاÙĦØ£ سبÙĪØ¹ +ĠÙĤ رÙĬب +ин д +инд ив +индив ид +индивид Ñĥ +индивидÑĥ алÑĮн +för der +Ġseç en +Ġseçen ek +Ġét ant +ĠлÑİб им +каз ÑĭваеÑĤ +ว ิà¸Ļ +Ġ×Ķ×ij ×IJ×Ļ×Ŀ +Ġд ов +Ġдов олÑĮ +ĠдоволÑĮ но +×¢×ĵ ×Ļ×£ +Ġok re +Ġokre ÅĽ +ĠokreÅĽ lon +Ġت رÙĬد +à¹Ģมืà¹Īà¸Ń วัà¸Ļà¸Ĺีà¹Ī +ãĤĪ ãģĭãģ£ãģŁ +Cum h +Cumh ur +Cumhur ba +Cumhurba ÅŁ +CumhurbaÅŁ kan +CumhurbaÅŁkan ı +Ġn ợ +à¸ľà¸¹à¹ī à¹Ģลà¹Īà¸Ļ +Ġcompl ète +à¹Ģà¸ŀ ศ +د ÙIJ +Ġdü z +Ġdüz ey +ãģ§ãģĤãĤĭ ãģĵãģ¨ +ext érieur +× ³ +Ġinform ação +ãĤ¯ãĥª ãĥĭãĥĥãĤ¯ +ĠPub li +ĠPubli é +ר ×ķ×ĵ +à¸Ħวาม à¸Ľà¸¥à¸Ńà¸Ķà¸łà¸±à¸¢ +ĠØ£ÙĬ ض +ĠØ£ÙĬض Ùĭا +ت سبب +ãģ¤ ãĤĤãĤĬ +из ма +à¸Ĥึà¹īà¸Ļ à¹Ħà¸Ľ +Ùĥ ÙIJ +ÙĦ ÙĪÙħ +Ġש צר +Ġשצר ×Ļ×ļ +ãģ¯ ãĤĤãģ¡ãĤįãĤĵ +Ġк ан +Ġкан ал +ãģ«ãģª ãģ£ãģ¦ãģĦãģ¾ãģĻ +ĠاÙĦØ£ Ùĥثر +ت اØŃ +ÙĨت Ùĩ +ÙĨتÙĩ اء +ا ÙĪÙĬØ© +ĠBug ün +н Ñģкого +à¸Ķ à¹Īวà¸Ļ +é volution +ãģ£ãģ¦ ãģĦãģ¾ãģĹãģŁ +ãĤ ħ +ĠV ương +à¸łà¸²à¸ŀ ย +à¸łà¸²à¸ŀย à¸Ļ +à¸łà¸²à¸ŀยà¸Ļ à¸ķรà¹Į +Ġ×Ķ ×¦×ľ×Ļ×Ĺ +ĠاÙĦإسÙĦاÙħ ÙĬ +ÙĦÙĬ ب +Ġed ição +ÑģÑĤÑĢ ÐµÐ» +Ġkh úc +ÙĨÙħÙĪ Ø° +ÙĨÙħÙĪØ° ج +׾ צ×Ķ +ÑģÑĤав ил +à¸ĸ า +สรà¹īาà¸ĩ à¸Ħวาม +ãģĦ ãģ£ãģ± +ãģĦãģ£ãģ± ãģĦ +ÑģÑĤав лен +ĠاÙĦ ÙĤدس +Ġng ược +ب Ø® +ส หร +สหร ั +สหรั à¸IJ +ĠØ£ غ +Ġأغ سط +Ġأغسط س +ãģĨ ãģ¾ +ãģĨãģ¾ ãģı +ĠêµŃ ìłľ +ØŃض ار +Ġd ừng +æĬ¼ ãģĹ +ت ÙĪØ§ +تÙĪØ§ جد +ש×ŀ ×Ĺ×Ķ +ãģı ãĤĵ +Ġ×ij×¢ צ +Ġ×ijעצ ×Ŀ +×ŀ ׳×Ļ×ķת +×ķ ×Ļ×ĵ +×ķ×Ļ×ĵ ×IJ×ķ +à¸Ĭ ิà¸ĩ +Ġprac ÄĻ +Ġз аÑĤ +ĠзаÑĤ ем +ĠìŀIJ ìľł +Ġì¤ Ģ +Ġì¤Ģ ë¹Ħ +Ġb áºŃ +ĠbáºŃ c +Ġ×Ķ×ŀ צ×ij +ĠÙĤ ÙĬÙħØ© +à¹Ģà¸Ń à¹Ģà¸Ĭ +à¹Ģà¸Ńà¹Ģà¸Ĭ ีย +Ġperch è +ĠاÙĦع سÙĥر +ĠاÙĦعسÙĥر ÙĬØ© +ج ÙĬب +ëŀ µ +Ùħ Ùĩر +ÙħÙĩر جاÙĨ +Ùħ راÙĥ +ÙħراÙĥ ز +Ġод нако +à¸Ķี à¹Ĩ +Ġצ פ×ķ +Ġkullan ılan +Ġк ино +ãĥĨãĤ£ ãĥ³ãĤ° +ĠGi Ỽi +ت ÙĪØ² +تÙĪØ² ÙĬع +ย ิà¸Ļ +ยิà¸Ļ à¸Ķี +Ġc Åĵur +ĠiÅŁ aret +Ġ×ij×¢ ×ĸר +Ġ×ij×¢×ĸר ת +Ġп аÑĨи +ĠпаÑĨи енÑĤ +ãģ¿ãģŁãģĦ ãģ§ãģĻ +в ез +ли на +од е +Ġ×IJ×ķת ף +dıģ ınız +ĠÐIJ в +ĠÐIJв ÑĤоÑĢ +ï¼ ® +ĠC ần +ĠاÙĦا Ø® +ĠاÙĦاخ بار +Ġê±° ìĿĺ +Ġat enção +Ġgeld iÄŁi +ãĤª ãĤ¹ +ãĤªãĤ¹ ãĤ¹ +ãĤªãĤ¹ãĤ¹ ãĥ¡ +ев Ñĭе +кÑĢÑĭ л +à¹Ģà¸Ĭ ียà¸ĩ +à¹Ģà¸Ĭียà¸ĩ à¹ĥหมà¹Ī +Ġmar ço +ĠاÙĦÙħ ادة +Ġг ол +Ġsprzeda ży +Ġíķ´ ê²° +ĠÐķ го +ê¹ Ģ +Ġ׾ק×ij׾ ת +ĠاÙĦÙģ ÙĨاÙĨ +Ġcomunic ación +à¹Ģสà¹īà¸Ļ à¸Ĺาà¸ĩ +íĺ ¹ +à¸Ĭ ำ +à¸Ĭำ ระ +Ġ׼ ×IJ×ŀ +Ġ׼×IJ×ŀ ×ķר +à¸Ĭ à¹Īาà¸ĩ +ز Ùĩر +Ġklient ów +ива ÑİÑĤ +ан г +׳ ×ļ +Ġg á»įn +Ãľ R +ìĺģ ìĥģ +Ġغ زة +ìĿĮ ìĿĦ +Ġbez po +Ġbezpo ÅĽ +ĠbezpoÅĽ redni +ĠاÙĦÙħ ÙĪØ§ +ĠاÙĦÙħÙĪØ§ Ø·ÙĨ +ĠاÙĦÙħÙĪØ§Ø·ÙĨ ÙĬÙĨ +ãĤĮ ãģ¾ãģĻ +ĠмаÑĤ Ñĩ +×IJ ×ķף +Ġر سÙħÙĬ +ĠÑįк он +ĠÑįкон ом +ĠÑįконом иÑĩеÑģк +ãĥľ ãĥ¼ +Ġд иÑĢ +ĠдиÑĢ ÐµÐºÑĤоÑĢ +ĠÑģк оÑĢо +à¸ļ ำ +à¸ļำ ร +à¸ļำร ุà¸ĩ +ĠÑĦ ÑĥÑĤ +ĠÑĦÑĥÑĤ бол +Ġ×IJ ×Ļ׾ +Ġì¤ij êµŃ +ìľ ¤ +eÄŁ e +à¹Ħ à¸ģà¹Ī +tra î +traî n +ĠÑĤ ÑĢÑĥб +à¹Ģà¸ļ ื +à¹Ģà¸ļื à¹īà¸Ńà¸ĩ +à¹ģม à¸Ļ +ĠتØŃ دÙĬØ« +Ġ׼ עת +ØŃ اسب +lı ÄŁa +×§×Ļ ×Ļ×ŀ×Ļ×Ŀ +оÑģÑĤ ÑĮÑİ +à¸Ŀ ั +à¸Ŀั à¹Īà¸ĩ +Ø´ غÙĦ +ìĽ ¹ +Ġкажд ого +Ġbölüm ü +หà¸Ļ ี +Ġistedi ÄŁi +Ġtr ưng +ãĥ Į +ฮ à¸Ń +Ø£ÙĨ Ø´ +Ø£ÙĨØ´ طة +ĠاÙĦÙħ سÙĬ +ĠاÙĦÙħسÙĬ ØŃ +ลัà¸ģษ à¸ĵà¹Į +Ġn á»Ńa +à¸Ĺีà¹Ī à¸ķà¹īà¸Ńà¸ĩà¸ģาร +ÑĪ ÐµÐº +л Ñij +Ġש ×Ļ×Ķ +Ġש×Ļ×Ķ ×Ļ×Ķ +Ġkhu ôn +ĠÑĤÑĢеб ованиÑı +Ġ×ľ×¢ ×ĸ×ķר +ĠاÙĦع Ùħر +ราà¸Ħา à¸ĸูà¸ģ +ÙĩÙı ÙħÙĴ +ü st +üst ü +Ġден ег +Ġn ạ +à¸Ĥà¸Ļ ม +Ġбл аг +Ġблаг од +Ġблагод аÑĢ +ĠблагодаÑĢ Ñı +Ø¥ سÙĦاÙħ +à¸Ļิ ว +çŁ¥ ãĤīãģªãģĦ +Ø« ÙĤØ© +Ġг олоÑģ +×IJ×ķר ×Ĺ +Ġtr ứng +Ġод ном +ĠkoÅĦ cu +Ġ×ķ רק +Wi ÄĻ +WiÄĻ cej +Ġ×IJ ×Ļ׼×ķת +Ġ×IJ×Ļ׼×ķת ×Ļ +Ñģ оÑģ +Ġje żeli +以ä¸ĭ ãģ® +å°ı ãģķ +å°ıãģķ ãģª +олог ии +Ġоб ÑģлÑĥж +ĠобÑģлÑĥж ива +Ùĥت ابة +Ġê´Ģ ìĭ¬ +×¢ ש×Ļר +Ġaras ındaki +ĠÑĢай она +ÙĪØ§ جب +Ġ×ij×Ĺ×Ļ ×Ļ +íķ´ ì£¼ +Ġg óc +ай л +ĠT ình +æļ® ãĤī +æļ®ãĤī ãģĹ +æĻĤ ãģ«ãģ¯ +ĠгоÑĢод е +Ġ׼×IJ ×Ļ׾ +Ġ׼×IJ×Ļ׾ ×ķ +ĠC á»Ļng +ãģ©ãģĨ ãģĹãģ¦ãĤĤ +×Ĺ ×ķ×£ +تØŃ رÙĥ +ĠÑģлов ам +à¸Īะ à¸Ĭà¹Īวย +ĠاÙĦÙħست ÙĤبÙĦ +ÙĤ ض +ÙĤض ÙĬ +×ijס ×ķפ +×ijס×ķפ ×ķ +iÄĻ Äĩ +ĠY ıl +Ø´ ÙĬØ® +à¸Ħุà¸ĵ à¸Īะ +ש×ŀ ×ķת +Ġت عرض +Ġanál ise +ĠÑģоб иÑĢа +à¹Ģà¸ŀ à¸Ĭ +à¹Ģà¸ŀà¸Ĭ ร +Ġв ели +Ġвели к +สั à¹īà¸Ļ +Ġpop ulação +รà¹Īวม à¸ģัà¸Ļ +×Ĺ ×ŀ +×Ĺ×ŀ ×Ļש×Ļ +ס ×Ļס +åĨħ ãģ§ +Ġsob Äħ +ĠY ay +ĠYay ın +ãĥ¡ ãĥĭãĥ¥ãĥ¼ +ĠпÑĢедоÑģÑĤав лÑı +ãģł ã썿ĢĿãģĨ +Ġê³ł ê°Ŀ +Ġод ним +à¹ĥà¸Ļ à¹Ģรืà¹Īà¸Ńà¸ĩ +Ġs á»ķ +ĠÐĹ Ð´ÐµÑģÑĮ +Ġизмен ениÑı +ĠìĿ¼ ìĿĦ +ãģªãģ® ãģł +клад Ñĭва +ÑĢ Ð¼Ð° +Ġ×ķ×ij ׼׾ +تأ ÙħÙĬÙĨ +ĠпÑĢи ÑıÑĤ +ĠпÑĢиÑıÑĤ н +Ùħ Ùħار +ÙħÙħار سة +ãģ¨ãģª ãģ£ãģ¦ +Ġج ÙħÙĬÙĦ +Ġì§ Ī +Ġì§Ī 문 +Ġquest ão +i é +ié ndo +หà¹īà¸Ńà¸ĩ à¸ŀัà¸ģ +ãĥij ãĥ¼ãĥĪ +ÑĤвеÑĢж да +н Ñģкой +з ал +มุ à¹Īà¸ĩ +á» Ĭ +Ġ×Ķ×IJ×Ĺר ×ķ׳×Ķ +ĠTh ư +주 민 +ĠاÙĦع ب +év én +évén ement +ÙĤÙĪ Ø§Ø¹Ø¯ +د Ùı +ĠìķĬ ìĬµëĭĪëĭ¤ +Ġë³´ 기 +Ġyapıl ması +à¹Ģร าà¸ģ +à¹Ģราà¸ģ à¹ĩ +ØŃ ذر +ÙĤ صر +ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĹãģŁ +Ġà¹Ģà¸Ľà¹ĩà¸Ļ à¸ķà¹īà¸Ļ +ãģ¨ ãģ« +ãģ¨ãģ« ãģĭ +ãģ¨ãģ«ãģĭ ãģı +н ÑĨе +зв Ñĥк +ãģĹãĤĪãģĨ ãģ¨ +ĠاÙĦصØŃ ÙĬØ© +Ġש×Ķ ×Ļ×ķ +ĠDi ÄŁer +ÙĤÙĦ ÙĤ +ãĤ¸ãĥ£ ãĥ³ +Ġr á»Ŀi +Ġл еÑĩ +ĠлеÑĩ ениÑı +تب اد +تباد ÙĦ +צ פ×Ķ +à¸Ħวาม à¹Ģหà¹ĩà¸Ļ +ĠØ´ ب +Ġشب ÙĥØ© +ר ×Ļ×§ +Ùħ عد +Ùħعد ات +dıģ ında +Ġ×ijש ׳×Ļ×Ŀ +Ġ×Ķ ×Ļשר×IJ׾ +Ġ×Ķ×Ļשר×IJ׾ ×Ļת +Ġsı nav +׳צ ×Ļ×Ĵ +วัà¸ķ à¸ĸุ +ĠاÙĦبر ÙĦÙħ +ĠاÙĦبرÙĦÙħ اÙĨ +t ivitÃł +ãĤĵãģł ãĤįãģĨ +×§×Ļ ×Ļ×ŀ +ÙĦÙĬ Ùĥ +ĠÄij ò +ĠÄijò i +ĠÐĺн ÑĤеÑĢ +ĠÐĺнÑĤеÑĢ Ð½ÐµÑĤ +ãģ«ãģ¨ãģ£ãģ¦ ãģ¯ +ãģ£ ãģĵ +×§ ×ķס +ست ØŃÙĤ +æķĻ ãģĪãģ¦ +ãĥĢ ãĥ¡ +ĠÙħÙĨ زÙĦ +à¹Ģà¸ĭ à¹ĩà¸Ļ +使 ãģĪãĤĭ +è¦ĭ ç©į +è¦ĭç©į ãĤĤãĤĬ +Ø£ Ùģ +Ø£Ùģ Ùĥار +Ġиг ÑĢов +ĠигÑĢов Ñĭе +Ġm ÄĻż +ĠmÄĻż czy +ĠmÄĻżczy zn +ĠاÙĦØŃ ÙĤÙĬÙĤÙĬ +ع بر +׼×ķ׾ ׳×ķ +íĿ ¥ +×ŀ×IJ ×ķ×Ĺר +خت ص +ãĥŀ ãĥŀ +Ġ×IJ×Ĺ ×ķ×ĸ +í ĮĢ +Ġr á»iji +Ġв ÑĤоÑĢ +ĠвÑĤоÑĢ Ð¾Ð¹ +Ġl ẫn +пÑĢ Ð¾Ð¼ +пÑĢом ÑĭÑĪ +пÑĢомÑĭÑĪ Ð»ÐµÐ½ +пÑĢомÑĭÑĪлен н +ĠоÑĤноÑĪ ÐµÐ½Ð¸Ñı +Ġs ứ +Ġм обилÑĮ +ĠмобилÑĮ н +ĠÑįÑĤ омÑĥ +Ġt ạp +ĠìĤ¬ ê±´ +ĠìķĮ 볤 +Ùĥ Ùı +ÙĥÙı ÙħÙĴ +Ġ×§ ×ķר×Ķ +ĠÑĦ иÑĢ +ĠÑĦиÑĢ Ð¼ +Ġsık ıntı +׳ ׼ +׳׼ ×ķף +ÙĪÙĦÙĪØ¬ ÙĬ +ØŃ اÙĨ +Ġlo ạn +Ġ×IJ׾ ×£ +Ġm ắn +abh äng +abhäng ig +ĠÑĥÑĢов нÑı +Ġ׾×ij×ĵ ×ķ×§ +ÙĬ ÙħÙĨ +lay ın +Ġh ải +Ġзав од +ĠìķĦ 주 +สà¸ĸ า +สà¸ĸา à¸ļัà¸Ļ +Ġgüven lik +à¹Ģà¸Ķ à¹Īà¸Ļ +×ij×ĵ ×§ +Ġë Ī +ĠëĪ Ħ +ĠëĪĦ 구 +éĩįè¦ģ ãģª +รà¸Ńà¸ĩ รัà¸ļ +sch lie +schlie ÃŁen +Ġìĸ ¼ +Ġìĸ¼ ë§Ī +Ġìĸ¼ë§Ī ëĤĺ +ÑĤи ки +íķľëĭ¤ ê³ł +ãģłãģ£ãģŁ ãĤī +Ġ×Ķ ×Ļ×ĺ×ij +ãģªãģijãĤĮãģ° ãģªãĤīãģªãģĦ +â Ì +Ã¢Ì £ +Ġph ạt +ak Ä±ÅŁ +ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĻ +à¹Ģà¸ĭ à¹ĩ +ĠС егоднÑı +Ġinsan ların +Ġdévelop pe +ת פר +תפר ×Ļ×ĺ +اÙĨت شار +ê° ij +Fran çois +Ø£ÙĦ ع +Ø£ÙĦع اب +ãĤĴ è¶ħ +ãĤĴè¶ħ ãģĪ +Ġê°Ļ ìĬµëĭĪëĭ¤ +ãĤ³ ãĥ¬ +ĠмеÑģÑı ÑĨев +íĮ ħ +ĠاÙĦج اÙħعة +ìĿ¸ íĦ° +ìĿ¸íĦ° ëĦ· +×ĵר ×ķש +ĠÙĪØ£ شار +ĠпÑĢав ила +ãģĿãģĵ ãģ« +×Ĺ ×ŀ×ĵ +à¹Ģหà¸ķุ à¸ģารà¸ĵà¹Į +Ġê²½ íĹĺ +ãģ¶ ãĤĬ +׾ ש +׾ש ×ķף +à¹Ģ à¸ĸ +ĠDo ÄŁu +ĠиÑģполÑĮзов ание +Ġçoc uÄŁu +магазин е +ĠÄiji á»ĥn +Ġas lı +Ġaslı nda +Ġdoen ça +Ġس اع +Ġساع ات +ĠиÑģполÑĮзов аниÑı +ר ×ķצ×Ļ×Ŀ +ĠзнаÑĩ иÑĤ +ĠÑĢаР¼ +ĠÑĢам каÑħ +ê±° 리 +Ġп ÑĭÑĤа +ãĥģ ãĥ³ +Ġпо Ñģк +ĠпоÑģк олÑĮ +ĠпоÑģколÑĮ кÑĥ +Ø¥ بر +إبر اÙĩ +إبراÙĩ ÙĬÙħ +ĠÑĤÑĢ ÐµÑħ +ĠGen ç +س ÙĪÙģ +Ġve ÃŃculo +ĠNg ân +ĠоÑĩеÑĢ ÐµÐ´ÑĮ +à¸Ħร ึà¹Īà¸ĩ +×IJ ×ij×Ļ +à¸ķ à¹īม +ãĤĴè¡Į ãģĦ +ĠاÙĦساب ÙĤØ© +на ÑĨи +наÑĨи она +наÑĨиона лÑĮн +Ġgest ión +ت ÙĤد +ĠاÙĦبÙĬ اÙĨ +ĠاÙĦبÙĬاÙĨ ات +ĠاÙĦ اÙĨتخاب +ĠاÙĦاÙĨتخاب ات +à¹Ģà¸Ĭ à¹Īา +×ĵ ×IJ×Ĵ +Ġ׾×Ĵ ×ŀר×Ļ +Ġت ØŃتاج +Ġth ôn +à¸ķ à¹īà¸Ńà¸Ļ +à¸ķà¹īà¸Ńà¸Ļ รัà¸ļ +女 ãģ® +女ãģ® åŃIJ +Ġth ợ +Ø· ØŃÙĨ +ารà¹Į à¸Ķ +ת ×ŀ×Ļ×ĵ +ĠÑģам Ñĭм +Ġìĭľ íĸī +Ø¥ صد +إصد ار +ĠNgh á»ĩ +ìķ ķ +س ئ +سئ ÙĦ +à¸Ń าร +à¸Ńาร ม +à¸Ńารม à¸ĵà¹Į +à¹ģ ฮ +׳×ĺ ׾ +Ġì¢ĭ ìķĦ +×ķ׾ ׾ +Ġ×ij ×Ľ×ª×ij +ãĤ« ãĥ© +צע ×Ļר×Ļ×Ŀ +تعب ÙĬر +Ġ×ŀ קר×Ķ +ĠÑĦак ÑĤоÑĢ +Ġت ÙħاÙħ +ĠتÙħاÙħ ا +ëį ķ +Ġv ưá»Ŀ +Ġvưá»Ŀ n +Ġd Ä±ÅŁÄ± +ãģĦ ãģ¡ +Ġ׾ק ׳×ķת +ĠاÙĦع ÙĦاÙĤات +п Ñĥб +пÑĥб ли +Ø¥ ÙĬÙħ +Ø¥ÙĬÙħ اÙĨ +à¸Ńำ à¸Ļา +à¸Ńำà¸Ļา à¸Ī +åIJ« ãģ¾ãĤĮ +ãĤĭ ãģŁãĤģãģ« +ס ×Ĵ +ס×Ĵ ׳×ķף +تØŃ دÙĬ +Ġaup rès +ĠاÙĦج Ùĩا +ĠاÙĦجÙĩا ز +Ġ×ŀ ת×Ĺת +ен нÑĥÑİ +Ġз им +à¸ģา à¹ģà¸Ł +Ġ×ijת ×ķר +Ġngh è +Ġnghè o +ĠÐĽ Ñİ +ĠÐĽÑİ Ð± +תק צ×Ļ×ij +×ŀ×¢ ש×Ķ +ĠاÙĦبÙĬ ت +צ ×Ļפ +ĠобÑıз ан +ĠM á»Ĺi +ĠТ ÑĥÑĢ +ĠÙĪØ¨ اÙĦت +ĠÙĪØ¨Ø§ÙĦت اÙĦÙĬ +Ġdéc ision +Ġب د +Ġبد أت +Ġc ục +Ġb ask +Ġbask ı +Ġhat ırl +Ġhatırl a +å°ı ãģķãģĦ +Ġgerçek ten +à¸ľ ัà¸ģ +åı¯èĥ½ ãģª +×ŀ×IJ ס +Ġcr ÃŃtica +ĠìĿĺ ìĽIJ +عÙĤ ÙĪØ¯ +×ĺ ׼׳ +×ĺ׼׳ ×ķ׾×ķ×Ĵ×Ļ×Ķ +è¨Ģ ãģĪãģ° +ĠÙĤ ÙĨا +ĠÙĤÙĨا Ø© +ĠìĿ´ê²ĥ ìĿĢ +ت صر +à¸Ł ัà¸Ļ +ĠÑĢе ÑĨеп +ĠÑĢеÑĨеп ÑĤ +ĠبÙĨ Ù쨳 +ÑĢо ÑĪ +ĠмаÑĢ ÑĤа +Ġson ras +Ġsonras ı +×ķ×ij ש +ãĥª ãĤ¹ãĤ¯ +ĠFranç ais +á» ļ +ê° Ķ +Ġ×Ķ×ijר ×Ļת +פ ×Ļצ +פ×Ļצ ×ķ×Ļ +ĠÙĦÙħا ذا +ĠÐļи ев +ĠÑģ мÑĭÑģл +ê¸Ī ìľµ +ãĤ·ãĥ£ ãĥ« +ãĥ© ãĤ¤ãĥĪ +ìĽ ĥ +×ŀ ×Ĺר +ãĨ į +Ġkullan ım +Ġ×IJצ׾ ׳×ķ +Ġt Ãłn +ãĥı ãĥ¼ +ãģ¨ ãģ¨ãĤĤ +ãģ¨ãģ¨ãĤĤ ãģ« +ÑĢ ÐµÐ³ +ÑĢег и +ÑĢеги он +ãģªãģı ãģªãĤĭ +Ġch ảy +Ġج ÙĩØ© +ÅĦsk iej +à¸Ńี à¹Ģม +à¸Ńีà¹Ģม ล +ãģį ãģ£ãģ¨ +ĠìĺĪ ìĤ° +Ġkit abı +Ġedu cação +Ġbul uÅŁ +олог иÑı +Ġкон кÑĢ +ĠконкÑĢ ÐµÑĤ +×Ĵ ×Ļר +ĠпÑĢед лаг +ĠпÑĢедлаг аеÑĤ +ĠY ên +Ġíķľ ë²Ī +Ġ×ŀ ר׼×ĸ×Ļ +à¹Ģà¸Ľà¸´à¸Ķ à¹Ģà¸ľà¸¢ +ÑĤвеÑĢ Ð´ +ĠH á»ĩ +ĠÐĵ ÑĢ +à¸Ŀ à¹īา +×Ķ ×©×§ +×Ķשק ×¢×Ķ +Ġна Ñĥк +ìłIJ ìĿĦ +Ġн елÑĮ +ĠнелÑĮ з +ĠнелÑĮз Ñı +г ин +ĠB öl +ĠBöl ge +Ġв ла +Ġвла ÑģÑĤи +à¹Ģà¸Ļ à¹ĩ +à¹Ģà¸Ļà¹ĩ à¸ķ +ê³ ¨ +Ġö ld +Ġöld ür +׼׳ ×¢ +ĠاÙĦÙĩ ÙĬئة +ت ارÙĬØ® +ĠÐij ÑĢ +ĠÑģ мож +ĠÑģмож еÑĤе +ĠL úc +à¹Ħà¸Ľ à¸ĸึà¸ĩ +ĠBakan ı +Ġerklä rt +ĠÐIJ на +Ġsc ène +åķı ãģĦ +åķıãģĦ åIJĪãĤıãģĽ +ÙħÙĩ ÙĨد +ÙħÙĩÙĨد س +Ġн азвание +ив аниÑı +ãĤĴ å¤īãģĪ +ä»ĺãģį åIJĪ +ãĥij ãĤ½ +ãĥijãĤ½ ãĤ³ãĥ³ +æĺİ ãĤī +æĺİãĤī ãģĭ +à¹Ģà¸Ńà¸ģ สาร +à¹Ģà¸ģิà¸Ļ à¹Ħà¸Ľ +л еп +ãģĹãģŁ ãĤĤãģ® +ĠC âm +ĠCâm ara +×§×ķ׾ ׳×ķ×¢ +Ġ×ij×Ĵ ×Ļף +Ġoc zy +Ġoczy wiÅĽcie +att ivitÃł +ãĥĵ ãĥ¥ãĥ¼ +Ġeduc ación +İ YE +ê¹Į ìļĶ +ãĤ¨ ãĥªãĤ¢ +н еÑģÑĤи +Ġm óg +Ġmóg ÅĤ +Ġ×§×ĺ ׳×Ļ×Ŀ +ĠPr ä +Ġ×ľ×¢ ×ij×ķר +بÙĨ Ùī +з ол +зол оÑĤ +Ġwn ÄĻtr +ĠwnÄĻtr z +Ġconstr ução +รัà¸ļ รà¸Ńà¸ĩ +س جÙĨ +Ġ×§ ×ķ׳ +ס ×Ļפ×ķר +ĠÙħ دÙī +رض Ùī +п лав +ï¼ ¥ +Ġil a +Ġila ç +ãĤĭ ãģ¹ãģį +ĠÙħ ÙĪÙĤÙģ +à¸ģร ุ +à¸ģรุ à¸ĵา +chodzÄħ c +ĠÑĤÑĭ Ñģ +Ðķ вÑĢо +ĠÙĬ ØŃدث +ãĥ¡ ãĤ¤ãĥ³ +ĠاÙĦص ØŃÙĬ +ĠÐĶ Ð°Ð½ +دع اء +ãĤ´ ãĥ¼ãĥ« +ש ×ł×ª×Ļ +×©×ł×ª×Ļ ×Ļ×Ŀ +à¸Ķà¹īวย à¸ģัà¸Ļ +Ġol acaģı +Ġ×ij ×ŀ×Ĺ×Ļר +×Ķ ×§ +×Ķ×§ ×ŀת +ãĥ¢ ãĥİ +ĠçalÄ±ÅŁ tı +Ġjó venes +ãģĦãģı ãĤī +ĠÙħ عدÙĦ +ĠC Å©ng +ĠSeg ún +Ġdönem de +Ġ׾ ×Ļ×ĵ×Ļ +ãģį ãģ¡ +ãģįãģ¡ ãĤĵ +ãģįãģ¡ãĤĵ ãģ¨ +Ù쨱 ÙĨس +Ù쨱ÙĨس ا +åIJij ãģį +Ġcamp aña +ĠÑģам оÑģÑĤоÑı +ĠÑģамоÑģÑĤоÑı ÑĤелÑĮно +á» Ģ +ÙĤ ÙĪØ§ +س ÙĦاØŃ +à¸ģระ à¹ģ +à¸ģระà¹ģ ส +ĠполÑĮз Ñĥ +n qu +nqu ête +รà¹Īวม à¸ģัà¸ļ +ëĬIJ ëĥIJ +à¸Ĺีม à¸Ĭาà¸ķิ +Ġyıll ık +ìĬ ¬ +ĠØ£ صØŃاب +ill é +Ġdó la +Ġdóla res +Ġк ож +Ġкож и +ล à¹īà¸Ń +à¹Ģรีย à¸ļร +à¹Ģรียà¸ļร à¹īà¸Ńย +à¹Ģà¸ŀ ิ +à¹Ģà¸ŀิ à¹Īà¸ĩ +ÑĢиÑĤоÑĢ Ð¸ +Ġí ijľ +Ġíijľ íĺĦ +ĠпеÑĢ ÐµÐ² +ĠпеÑĢев од +פ×Ĵ ×Ļ×¢×Ķ +ĠdeÄŁerlendir me +Ùģ Ø§Ø¦ +ĠвÑĭ год +ınız ı +×ķ׼ ×Ļ×Ĺ +ĠдоÑģÑĤ иг +Ġng Ãłn +æĢĿ ãģ£ãģŁ +ĠÐķ ÑģÑĤÑĮ +ĠاÙĦر غÙħ +ĠzwiÄħz ane +رب Ø· +à¸Ļ ึà¸ĩ +Ġ׾×Ĺ ×ķ×§ +Ġszczeg óln +Ġszczególn ie +Ġبا ستخداÙħ +ĠfÃŃs ico +×¢ ס +עס ×ķ×§ +سÙĦ ÙĪÙĥ +Ġا ØŃد +Ñĩ ÑijÑĤ +×ĸ׼ ×Ķ +Ġl á»ĩnh +ĠÙĪ ØŃت +ĠÙĪØŃØª Ùī +à¸Ħวาม สามารà¸ĸ +à¸Ńยูà¹Ī à¹ģลà¹īว +à¸ģาร à¹Ģà¸Ķิà¸Ļà¸Ĺาà¸ĩ +تخ ذ +צ×Ļ ×ķ×ĵ +ĠاÙĦØ£ س +ĠاÙĦأس ÙĩÙħ +Ġt á»ĩ +ãģ£ãģ¦ ãģĦãģ¦ +สร ุ +สรุ à¸Ľ +Ġком ÑĦ +ĠкомÑĦ оÑĢÑĤ +ìĺ¤ ëĬĶ +ĠÑĢаз в +ĠÑĢазв ива +л анд +h änge +ĠبÙĨ سبة +à¹Ģà¸Ĥ ียว +עצ ×Ŀ +Ġ׾ ×ľ×Ľ×ª +Ñģо ÑĨиалÑĮн +Ġëĭ¤ìĿĮ ê³¼ +Ġרש ×ķ×ŀ +×ŀר ×Ĺ×ij +س ÙĤØ· +Ġalan ı +ĠÄij á»ĩ +é£Łãģ¹ ãĤĭ +à¸Ķ ึà¸ĩ +Ġgegen über +ĠبÙĩ ذÙĩ +à¸ĸืà¸Ń à¹Ģà¸Ľà¹ĩà¸Ļ +ëķ ħ +à¸Ħà¸Ļ à¹Ħà¸Ĺย +ãĤ¢ ãĤ¦ +ãĤ¢ãĤ¦ ãĥĪ +ศ ัà¸ģ +ศัà¸ģ à¸Ķิ +ศัà¸ģà¸Ķิ à¹Į +ÙĤÙĪ Ø§ÙĨ +ÙĤÙĪØ§ÙĨ ÙĬÙĨ +Ġhá»Ļ p +ãģªãģıãģª ãģ£ãģ¦ +Ġ×IJ ×ŀ׳ +Ġ×IJ×ŀ׳ ×Ŀ +à¹Ģà¸ķ ืà¸Ńà¸Ļ +ĠзавиÑģ им +ĠзавиÑģим оÑģÑĤи +ת ×Ļ×IJ +ת×Ļ×IJ ×ķר +å§ĭãĤģ ãģŁ +Ġng á»į +Ġngá»į t +íĴ į +ê³¼ ìŀ¥ +Ġb ại +ãģ§ãģį ãģ¦ +Ġcomeç ar +à¸Ľà¸£ าà¸ģ +à¸Ľà¸£à¸²à¸ģ à¸ı +Ġгод Ñĭ +м еÑģ +ĠاÙĦÙħست ÙĪÙī +ĠÑģам Ñĭе +л леÑĢ +ãģ£ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĻ +ãģ¨ãģ® ãģĵãģ¨ +bi ó +à¸ģล à¹Īà¸Ńà¸ĩ +ĠاÙĦز ÙĪØ¬ +ãģ«è¡Į ãģ£ãģŁ +à¸Ħà¹Ī à¸Ńà¸Ļ +à¸Ħà¹Īà¸Ńà¸Ļ à¸Ĥà¹īาà¸ĩ +ĠbaÄŁ l +ĠbaÄŁl ant +ĠbaÄŁlant ı +確 ãģĭ +確ãģĭ ãģ« +ãĥľ ãĥ¼ãĥ« +çµĤ ãĤıãĤĬ +ש ×ŀר +à¸Ĺีà¹Ī สามารà¸ĸ +ÙĦ زÙħ +д аеÑĤÑģÑı +รัà¸ļ à¸Ľà¸£à¸° +รัà¸ļà¸Ľà¸£à¸° à¸Ĺาà¸Ļ +å¤ī ãĤıãĤĬ +ï¼ ¢ +ĠìĺĪìĪĺ ëĭĺ +ãĤĪãģĨ ãģ¨ +มัà¸ģ à¸Īะ +ĠH ương +ÙĨ Ù쨰 +×ŀ×ĵ ×ĵ +ĠìĿ¸ ìłķ +Ñħод иÑĤÑĮ +ĠзавиÑģ иÑĤ +×ķ×ĵ ×Ļ×¢ +ãģĵãģ¨ãģĮ ãģĤãĤĬãģ¾ãģĻ +ع راÙĤ +سط ØŃ +à¸ģำ à¹Ħร +ëĵ¤ ëıĦ +×Ļצ ×Ļר×Ķ +ãģĨ ãģĵãģ¨ +ÙĦا ØŃÙĤ +ãģĦ ãĤĮãģ° +ĠиÑģполÑĮз ÑĥÑİÑĤ +ĠB ợi +Ġשק׾ ×Ļ×Ŀ +ÑĨи кл +ÐIJ Ðŀ +Ġ×ijש ׳×Ķ +ÙĨØ´ Ø· +Ġש ×Ļ׳×ķ×Ļ +Ġש×Ļ׳×ķ×Ļ ×Ļ×Ŀ +Ġpobl ación +ĠH ưng +ระ ว +ระว ัà¸ĩ +رÙĬاض Ø© +ر صد +تÙĤ ÙĦÙĬ +تÙĤÙĦÙĬ د +Ġülk em +Ġülkem iz +à¸Ĭ ะ +ãĤ¯ãĥª ãĥ¼ãĥł +èģŀ ãģĦãģŁ +Ġwa ż +Ġważ ne +ê±° ëĵł +ê±°ëĵł ìļĶ +×ŀ×IJ ×ij×§ +×Ĺ×ĵ ש×ķת +ĠW roc +ĠWroc ÅĤaw +ĠKü ltür +s ist +sist ência +×¢×ĸר ×Ķ +Ġg ương +รà¹īาà¸Ļ à¸Ħà¹īา +ĠÙĪØ£ ÙĪØ¶ØŃ +ánd ose +ãĤ· ãĥ¼ãĥ³ +×IJ׳ ר×Ĵ +×IJ׳ר×Ĵ ×Ļ×Ķ +ãģªãģĦ ãģ§ãģĻ +Ġkh á»§ng +Ġ문 ìĦľ +Ġ×ij ×ĵ×ijר +×ĵ ×Ļ×ķ +×ĵ×Ļ×ķ ×ķ×Ĺ +Ġré gl +ÙħÙĪ Ø§Ø¯ +об оÑĢ +обоÑĢ Ð¾ÑĤ +Ġ×Ķ ×ij׾ +Ġ×Ķ×ij׾ ×ķ×Ĵ +ØŃ اÙħ +ĠاÙĦع اص +ĠاÙĦعاص ÙħØ© +пеÑĢ Ð°ÑĤоÑĢ +ت Ø®ÙĦ +تخÙĦ ص +ãģŁãģł ãģĹ +ت سÙħ +à¹Ĥรà¸ĩ à¸ŀ +à¹Ĥรà¸ĩà¸ŀ ยา +à¹Ĥรà¸ĩà¸ŀยา à¸ļาล +ĠY ük +ĠYük sek +Ġש ׳×Ļת +Ġש׳×Ļת ף +liÄŁ e +Ġפ ת +Ġפת ×ķ×Ĺ +Ġbe ÄŁ +ĠbeÄŁ en +Ġ×ŀ ×ķר +Ġ×ŀ×ķר ׼×ij +Ġرس اÙĦØ© +íĨµ ìĭł +Ġaval ia +Ġavalia ções +Ġman h +Ġmanh ã +Ġìķ ŀ +Ġìķŀ ìľ¼ë¡ľ +ÙĤ تر +ÙĤتر ØŃ +à¹Ģà¸ģ ืà¸Ń +à¹Ģà¸ģืà¸Ń à¸ļ +Ġpropos é +Ø£ Ùħا +Ø£Ùħا ÙĥÙĨ +ĠÐŀ Ðŀ +ĠÐŀÐŀ Ðŀ +ÙħÙĤ ار +ÙħÙĤار ÙĨØ© +ëĦ IJ +ãģĦãģŁãģł ãģı +ÙĤ ÙĬÙĦ +Ġна ÑĪиÑħ +ãĤ« ãĥĥãĥĹ +×Ĺ׾ ת +Ġëĭ¤ ë§Į +à¸Ĺัà¹Īว à¹Ĥลà¸ģ +ãĥį ãĤ¿ +ØŃس اس +ãģ«ãģª ãĤĮ +ج ائ +جائ زة +é change +é conom +économ ie +Т Ðĺ +סת ׼׾ +à¸Ĺัà¹īà¸ĩ สà¸Ńà¸ĩ +ĠاÙĦØ® اÙħ +ĠاÙĦخاÙħ س +×§ ×ĺ×¢ +au waż +à¸ľà¸¹à¹ī à¸Ĭาย +à¹ģà¸Ľà¸¥ à¸ģ +åIJĮæĻĤ ãģ« +зн аниÑı +ãģĦãģŁãģł ãģįãģ¾ãģĹãģŁ +Ġ×ŀ×ij ׾×Ļ +à¸Ĥà¸Ń à¹ĥหà¹ī +ĠاÙĦت ربÙĬØ© +Ġdécou vert +Ġżyc iu +apr ès +Ġy ab +Ġyab anc +Ġyabanc ı +ĠbaÅŁ layan +ìĹĪ ëįĺ +Ġhes abı +Ġë§Į ìķ½ +ë§ Īëĭ¤ +ĠTh ánh +ãĥ´ ãĤ¡ +à¸Ľà¸£à¸±à¸ļ à¸Ľà¸£ +à¸Ľà¸£à¸±à¸ļà¸Ľà¸£ ุà¸ĩ +ĠM ặc +à¹Ģหà¸ķุ à¸ľà¸¥ +ĠÐij ез +Ġcapac itÃł +ÅĤe ÅĽ +ĠпÑĢе им +ĠпÑĢеим ÑĥÑīеÑģÑĤв +ĠÅļ wiÄĻt +Ġpubli é +×ŀ×¢ צ×ij +Ùħشار Ùĥات +à¸łà¸² ษ +à¸łà¸²à¸© ี +Ġdeux ième +ĠÙħØŃ اÙ쨏 +ĠÙħØŃاÙ쨏 Ø© +ĠSch ön +ï½ ¤ +Ġ×Ķ ×ij×¢ +Ġ×Ķ×ij×¢ ×Ļ×Ķ +ĠÙĪØ§ÙĦ ÙĦÙĩ +è¨Ģ ãģ£ãģŁ +à¸ķ à¹īาà¸Ļ +วร รà¸ĵ +à¸Ĺิ ศ +ĠbaÅŁ ına +Ġmog ÄĻ +ש ×Ļפ×ķר +ĠÙĪ Ø¹Ø¯ +ĠÙĪØ¹Ø¯ Ùħ +Ġhistó rico +Ġk ısı +ĠìĿ´ ê²Į +ĠPol ÃŃtica +ĠÑģиÑĤÑĥ аÑĨии +ĠkoÅĦ ca +×ij×ĵ ×Ļ×§×Ķ +ĠاÙĦسÙĬ ارات +ãģªãĤī ãģ° +ãĤµ ãĥ© +ãĤĭãģĵãģ¨ãģĮãģ§ãģį ãĤĭ +Ġdecis ão +×ķ ×ķ×ĵ +lä ss +läss ig +Ġ׾ ×Ļשר×IJ׾ +ĠÙĬ أتÙĬ +ר ×ķ×ĸ +ö ÄŁ +Ã¶ÄŁ ret +Ã¶ÄŁret im +Ġд ек +Ġдек аб +Ġдекаб ÑĢÑı +Ġש ×Ĺ×ķר +ãģ¦ãģıãĤĮ ãģŁ +عب ارة +Ġélect rique +ĠاÙĦتÙĨ ÙħÙĬØ© +جر Ùī +ĠìĪĺ íĸī +à¸Ĺ ู +ĠÑĢе алÑĮно +Ñģп оÑģоб +à¸Ħล à¹īาย +Ġس عÙĪØ¯ +ön ü +ĠÙģ ÙħÙĨ +تÙĥ ÙĪ +تÙĥÙĪ ÙĬÙĨ +ĠкаÑĩ еÑģÑĤво +ĠконÑĤ ак +ĠконÑĤак ÑĤ +Ġsöz leÅŁme +à¸Ń à¹īาà¸ĩ +Ġت ÙĪÙģ +ĠتÙĪÙģ ÙĬر +×Ķ×ĸ ×ĵ +×Ķ×ĸ×ĵ ×ŀ׳×ķת +ĠØ·ÙĪÙĬÙĦ Ø© +Ġtér mino +Ġ×IJ ×Ļפ×Ķ +ãĥĵ ãĥ« +ส à¹Ĥม +สà¹Ĥม สร +ĠاÙĦ اث +ĠاÙĦاث ÙĨÙĬÙĨ +ев иÑĩ +Ġopin ión +à¸Ľ วà¸Ķ +åı¤ ãģĦ +ร à¹Īา +ĠB iaÅĤ +ĠÑģÑĤ ал +ĠÑģÑĤал о +ó logo +ĠìķĦ ëĭĪëĭ¤ +Ġ×IJ ×Ļת +Ġ×IJ×Ļת ×ķ +à¹Ģหà¹ĩà¸Ļ วà¹Īา +à¸ļ ารà¹Į +çĦ ¼ +çĦ¼ ãģį +ĠìĿ´ìļ© ìŀIJ +ĠнекоÑĤоÑĢ Ñĭе +ks z +ksz taÅĤ +ksztaÅĤ c +ãĤŃãĥ£ ãĥĥãĤ· +ãĤŃãĥ£ãĥĥãĤ· ãĥ³ãĤ° +Ġro ÅĽ +ĠroÅĽ lin +ÑĢаж а +×ij׳×Ļ ×Ļ×Ķ +à¸Ľà¸£ สิ +à¸Ľà¸£à¸ªà¸´ à¸ķ +Ġgörd ü +×ŀ׳×Ķ ×Ļ×Ĵ +å¤īãĤı ãģ£ãģ¦ +Ġ×IJ ×Ķ +Ġ×IJ×Ķ ×ijת×Ļ +à¹Ģร à¹Īà¸ĩ +Ġön ünde +Ġê·¸ ëĥ¥ +пол иÑĤ +полиÑĤ иÑĩеÑģк +ãĥ¡ ãĥĩãĤ£ +ãĥ¡ãĥĩãĤ£ ãĤ¢ +ĠDet ay +ĠDetay lı +ĠاÙĦصÙģ ØŃØ© +à¸ģาร à¹Ģà¸ĩิà¸Ļ +Ġìµľ ê·¼ +׼ ש׾ +ï¼ © +вÑĪ ÐµÐ³Ð¾ +íķĺ ìĭ¤ +ĠÐŃ ÑĤ +ĠÐŃÑĤ оÑĤ +ส ื +สื à¸ļ +Ġng ừng +ĠдокÑĥменÑĤ ов +дав аÑĤÑĮ +ĠاÙĦشخص ÙĬØ© +Ġצ ×¢×Ļר +در Ùĥ +س ØŃب +à¹Ħมà¹Ī à¸Ħà¹Īà¸Ńย +Ġ×Ķ×ŀ×§ ×ķ×ŀ×Ļ +สัà¹Īà¸ĩ à¸ĭืà¹īà¸Ń +Ġê·¸ê²ĥ ìĿĦ +ãģĤãĤĭ ãģĦ +ãģĤãĤĭãģĦ ãģ¯ +×IJ×ķ×ĺ ×ķ×ij +×IJ×ķ×ĺ×ķ×ij ×ķס +к ÑĨион +ĠÐľ ожно +ãģı ãģł +ãģıãģł ãģķ +ĠинÑĦоÑĢм аÑĨиÑı +ï» Ł +Ġìŀij ìĹħ +Ġ×Ļ ×ķסף +Ø¥ دارة +ĠاÙĦØŃ اج +×ł×¡ ×Ļ×¢×Ķ +из аÑĨиÑı +×IJ׾ ×ij +×IJ׾×ij ×ķ×Ŀ +п ед +Ġ×§×ĺ ׳×Ķ +ĠÙĨÙ쨳 Ùĩا +ĠMinist ério +Ġп ен +Ġпен Ñģи +ãĥIJ ãĥ©ãĥ³ãĤ¹ +Ġ×Ķת ×ķר×Ķ +Ġt ạm +ĠìĹŃ ìĭľ +ï½ ¡ +Ġth á»± +Ġ ısı +ì» ¨ +ãģĹãģ£ãģĭãĤĬ ãģ¨ +Ġx ưa +Ġc ặp +×Ĺ ×Ļ×ij×ķר +วัà¸Ĵà¸Ļ à¸ĺรรม +st är +stär ke +ĠÑģам Ñĭй +p isa +pisa Äĩ +ĠoluÅŁ an +ĠاÙĦØ¥ ÙħاÙħ +ĠcÄĥ ng +Ġgü nl +Ġgünl ük +Ġ׳ש ×IJר +Ġkhi á»ĥn +ç¶ļ ãģijãĤĭ +stit ución +Ġcapac ité +Ġj aki +Ġjaki ÅĽ +вÑĪ Ð¸Ñģ +вÑĪиÑģ ÑĮ +פע×ķ׾ ×ķת +ĠØŃ ÙĬات +ĠØŃÙĬات Ùĩ +Ġник огда +ÐĽ Ь +Ġ×Ķ×¢ ×ķ×ij +Ġ×Ķ×¢×ķ×ij ×ĵ×Ķ +Ġch Ãło +หลาย à¹Ĩ +ĠÑı н +ĠÑıн ваÑĢ +ĠÑıнваÑĢ Ñı +à¸Īำà¹Ģà¸Ľà¹ĩà¸Ļ à¸ķà¹īà¸Ńà¸ĩ +Ġhö her +ãģķãĤĮãģ¦ ãģĦãģŁ +สà¸ĩ สั +สà¸ĩสั ย +ĠاÙĦ اس +ĠاÙĦاس ÙĦاÙħ +ĠاÙĦØ´ Ùħس +สà¸ĸาà¸Ļ ี +ãĤ¯ãĥ© ãĤ¹ +à¸ŀร ร +à¸ŀรร à¸Ħ +p õ +põ e +Ġpor ém +à¸Ľà¸£à¸° สà¸ĩ +à¸Ľà¸£à¸°à¸ªà¸ĩ à¸Ħà¹Į +powied zie +powiedzie Äĩ +Ġмог Ñĥ +Ġж ел +Ġжел ез +ĠاÙĦØ« ÙĤ +ĠاÙĦØ«ÙĤ اÙģÙĬ +ĠпÑĢав ило +Ġgdy ż +פש ×ķ×ĺ +ÑĢабоÑĤ ка +ĠÙĥ رة +Ø´ دد +Ùħار Ùĥ +Ùħ ÙĥØ© +Ġпод пиÑģ +×ĺ×ķ ×ķ×Ĺ +ĠÅĽ c +ĠÅĽc ian +Ġر جاÙĦ +Ġ×ª×ľ ×ķ×Ļ +и ÑĪ +иÑĪ ÑĮ +Ġmé dec +Ġmédec in +ëįĶ ëĿ¼ëıĦ +ĠÑĤеб Ñı +Ġ׾×Ķ ×ķס×Ļ×£ +ãģĬ 話 +Ġà¹ģà¸ķà¹Ī à¸ģà¹ĩ +د اÙģ +داÙģ Ø¹ +ĠC ùng +ãĥ»ãĥ» ãĥ»ãĥ» +ê¶ ģ +Ġdeber ÃŃa +หà¸Ļà¹Īวย à¸ĩาà¸Ļ +Ġva ÌĢ +Ġעצ ×ŀ +Ġעצ×ŀ ×Ŀ +à¹Ģà¸Ĭืà¹Īà¸Ń วà¹Īา +שק ×¢ +Ġ×Ķ ×Ľ×ķ׾ +Ġ×Ķ׼×ķ׾ ׾ +ни бÑĥд +нибÑĥд ÑĮ +ĠëĦĪ íĿ¬ +Ġоб ÑĢаÑī +ĠобÑĢаÑī а +Ġ×¢×ij×ķ×ĵ ת +ĠاÙĦÙħÙĨت خب +ıy ord +ıyord u +ÙĪ Ø° +×Ĺש ×Ļ×ij×ķת +Ġ×Ķ×¢ ×Ļ×§ +Ġ×Ķ×¢×Ļ×§ ר×Ļ +ì¢ Į +ยุ à¹Ĥร +ยุà¹Ĥร à¸Ľ +Ġа пÑĢ +ĠапÑĢ ÐµÐ»Ñı +sz ed +szed ÅĤ +д он +à¹Ģà¸ķิ à¸ļ +à¹Ģà¸ķิà¸ļ à¹Ĥà¸ķ +кол о +Ġkażde j +å¸ ° +帰 ãĤĬ +Ġмил ли +Ġмилли он +ç¾İåij³ ãģĹãģĦ +ت ÙĤار +تÙĤار ÙĬر +ĠìĿ´ 루 +ĠìĿ´ë£¨ ìĸ´ +Ġsprzeda ż +×Ķ ×ķצ×IJ×ķת +ãĤ¢ãĤ¯ ãĤ» +ãĤ¢ãĤ¯ãĤ» ãĤ¹ +ר ×ķ×¥ +ĠгоÑģÑĥдаÑĢÑģÑĤв енн +Ø£ ØŃÙĥ +Ø£ØŃÙĥ اÙħ +ĠoluÅŁ u +ĠA ç +ĠAç ık +ãĤ¸ ãĥ¼ +ç´ł æĻ´ +ç´łæĻ´ ãĤīãģĹãģĦ +Ġ×ijש×ij ×ķ×¢ +ب ذ +بذ ÙĦ +สา à¹Ģหà¸ķุ +Ġpoz osta +Ġpozosta ÅĤ +ØŃر Ùħ +Ġimport ância +leÅŁtir me +Ġд ÑĢев +Ġmó vil +ĠA ynı +Ġна лог +Ġналог ов +Ġ×Ĺ ×Ļפ×Ķ +ĠÑĦоÑĢм Ñĥ +à¸Ĺà¸Ķ สà¸Ńà¸ļ +ĠksiÄħż ki +Ġma ÅĤe +Ùħس Ø£ÙĦ +ÙħسأÙĦ Ø© +ï¼¾ ï¼¾ +ç ãeste +év iter +Ġкон ÑģÑĤÑĢÑĥк +ĠконÑģÑĤÑĢÑĥк ÑĨи +ï¾ ŀ +Ġת×ķ׼ ׳ +ãĤ¹ãĥĪ ãĥ¬ãĤ¹ +ĠاÙĦاÙĤتصاد ÙĬ +×ŀ×ĵ ×Ļ +Ġw ÅĤad +ĠwÅĤad z +Ø® ÙĪÙģ +ĠмаÑĤеÑĢиал ов +ãģ¨ãģ£ãģ¦ ãĤĤ +Ġznaj du +Ġznajdu jÄħ +Ùģ Ø¦Ø© +ãģ©ãģ® ãĤĪãģĨãģª +æĬij ãģĪ +׳ ×Ĺ׾ +Ġdü ny +Ġdüny an +Ġdünyan ın +гÑĢ Ð°Ð½Ð¸ +гÑĢани Ñĩ +Ġ×Ķש׾ ×Ļש×Ļ +Ġ×Ķ×IJ ש +åıĬ ãģ³ +ìĭŃ ìĭľ +ìĭŃìĭľ ìĺ¤ +Ġдол л +Ġдолл аÑĢ +Ġпов ÑĤоÑĢ +Ġ×Ĺ ×Ļ׳×Ŀ +ת פת×Ĺ +Ñĥв ели +Ñĥвели Ñĩен +ãĤ« ãĥª +raw id +rawid ÅĤow +×ķ ×ķ׾ +ãĥŁ ãĥ¥ +ì½ ĺ +ĠBy ÅĤ +Ðľ ÐIJ +ع ÙIJ +ĠÑģовеÑĢ ÑĪ +ĠÑģовеÑĢÑĪ ÐµÐ½Ð½Ð¾ +Ġм ой +Ġ×ķ׾×IJ ×Ĺר +æħ £ +æħ£ ãĤĮ +ØŃ اÙ쨏 +Ġ무 ë£Į +à¸Ħà¸ĵะ à¸ģรรม +à¸Ħà¸ĵะà¸ģรรม à¸ģาร +Ġìĸ´ ëĶĶ +Ġdif eren +Ġdiferen ça +ĠاÙĦØ£ ساس +ĠاÙĦأساس ÙĬØ© +Ġ׾×IJ×Ĺר ×ķ׳×Ķ +ê· ł +Ġ×Ķש׳×Ļ ×Ļ×Ķ +ìľĦìĽIJ ìŀ¥ +ลุ à¸ģ +ç iler +Ġ×Ķ×IJ ׾×ķ +èģŀ ãģı +Ġ×ķ×IJ פ×Ļ׾×ķ +ĠÑĢе ализ +ĠÑĢеализ аÑĨи +ระยะ à¹Ģวลา +Ġجدا Ùĭ +تب اع +Ġveh ÃŃculo +Ġдол г +à¸Ľà¸£à¸´ มาà¸ĵ +ì¦ IJ +Ġ׾ ×ŀ×§×ķ×Ŀ +ĠìĤ¬ ì§Ħ +à¸Ĭ à¹īา +Ġ×ŀ×¢ ×ķ׾×Ķ +Ġgö rm +Ġgörm ek +ĠÙĪÙĩ ذÙĩ +пеÑĢ Ð² +пеÑĢв ÑĭÑħ +ê·¸ ëŀĺ +ĠاÙĦبر ÙĬØ· +ĠاÙĦبرÙĬØ· اÙĨÙĬ +ĠиÑİ Ð½Ñı +ĠÐĵ оÑĢ +Ġ׾ ש׾×Ŀ +ÐIJ ÐĿ +Ġназ наÑĩен +о оÑĢ +ооÑĢ Ñĥж +Ġöz elli +Ġözelli ÄŁi +Ġни же +ç¶ļ ãģijãģ¦ +Ġа ÑĢенд +Ġkat ılı +Ġkatılı m +ĠØ¥ Ø·ÙĦاÙĤ +ĠÙĪØ¥ ذا +Ġок ÑĤÑı +ĠокÑĤÑı бÑĢÑı +à¹Ĥà¸ķ ๠+à¹Ĥà¸ķ๠Ĭ +à¹Ĥà¸ķà¹Ĭ ะ +Ġolduk ları +Ùħ ÙĪÙĤع +ëĤ © +ã썿ĢĿ ãģ£ãģ¦ãģĦãĤĭ +Ġש ×Ļ׼×ķ׾ +วา à¸Ķ +س ÙĬÙĦ +à¸Ĥ วั +à¸Ĥวั à¸į +تØŃ ÙĥÙħ +ì ĤŃ +Ġconna ît +׳ פת×Ĺ +Ġch ặ +Ġchặ n +ĠÙħ ØŃÙħ +ĠÙħØŃÙħ ÙĪØ¯ +ãģ ´ +ĠпÑĢодÑĥк ÑĨии +зд ÑĢав +ãģĶ è¦ +ãģĶè¦ § +×IJ×ij ×IJ +Ġvé ritable +ĠØ· ÙģÙĦ +ãĥĪãĥ© ãĥĸãĥ« +ê³ ¡ +Ġת ×ŀ×ķ׳×Ķ +Ġki ên +ĠÙĤ ادر +Ø¥ÙĤ ÙĦÙĬÙħ +ĠпÑĢед пÑĢи +ĠпÑĢедпÑĢи ÑıÑĤиÑı +Ġb Äĥng +Ġay ında +Ġg ấp +еÑħ ал +Ġgi Ãłnh +Ġд ав +Ġдав но +ìĺĢ ëĭ¤ +à¸Ļัà¸ģ à¹Ģà¸ķ +à¸Ļัà¸ģà¹Ģà¸ķ ะ +Ùħست شار +ست راتÙĬج +ستراتÙĬج ÙĬ +رÙħ ز +Ġt Ä©nh +ë¡ Ń +ĠÑĩ еÑĤ +ĠÑĩеÑĤ Ñĭ +ĠÑĩеÑĤÑĭ ÑĢе +ĠEnt ão +Ġص غ +Ġصغ ÙĬرة +×ij×Ļ×ĺ ×ķ׾ +خط ÙĪØ· +ĠÑĢазвиÑĤ ие +Ġamacı yla +à¸Ĺี วี +Ġо ÑģÑĤ +ĠоÑģÑĤ алÑĮн +ש×ķ׾׊ף +Ġ׼ ׳×Ļס +Ġ׼׳×Ļס ×Ķ +Ġd áºŃy +ĠyaÅŁ ayan +Ġ×ŀ×Ķ ×ķ×ķ×Ķ +ĠÑĥ Ñģи +ĠÑĥÑģи ли +×ŀ פ×Ļ +ĠпÑĢовед ениÑı +Ġر ب +Ġرب Ùħا +ĠاÙĦØ£ ÙĪØ³Ø· +Ġìľł ì§Ģ +Ġprac ownik +Ġpracownik ów +×ŀס ×ķרת +ÙĤار ب +à¸Ħวาม รูà¹īสึà¸ģ +à¹ģหล ะ +ĠاÙĦÙĨ ÙĤد +Ġ×IJ׾ פ×Ļ +Ùħس ئ +Ùħسئ ÙĪÙĦ +ев ÑĭÑħ +клÑİÑĩ ениÑı +×ij ×Ļ׳ +×ij×Ļ׳ ×Ļ×Ķ×Ŀ +ש ×ķ×IJ×Ķ +ĠÅŁ ark +ĠÅŁark ı +Ġsü rec +Ġsürec in +à¹Ģà¸Ħร à¸Ķ +à¹Ģà¸Ħรà¸Ķ ิà¸ķ +ãĥIJ ãĥ¬ +ĠØ´ Ø£ÙĨ +à¹Ģà¸Ńา à¹Ħวà¹ī +niÄĻ cie +רצ ×Ĺ +ĠaÅŁ ama +׳ פ×Ĵ×¢ +Ġth á»Ŀ +Ġkhu ẩn +diÄŁ inde +ÑıÑī иÑħ +ãĥĺ ãĥ« +Ġüber h +Ġüberh aupt +ĠÑĤÑĢеб ова +ĠdÅĤ ugi +×ĺ ×Ļף +à¸Ĥà¸Ļาà¸Ķ à¹ĥหà¸įà¹Ī +ĠاÙĦØ£ Ùĩ +ĠاÙĦØ£Ùĩ ÙĦÙĬ +ĠMü d +ĠMüd ürü +Ġ×Ļ×Ķ ×ķ×ĵ×Ķ +Ñĭв аеÑĤÑģÑı +س اط +×Ķת ׳×Ķ×Ĵ +×Ķ×ª×ł×Ķ×Ĵ ×ķת +à¸ģาร à¸ľà¸¥à¸´à¸ķ +íĴ Ģ +สà¸ĸาà¸Ļ à¸ģารà¸ĵà¹Į +Ġо ÑĦ +ĠоÑĦ иÑģ +ĠÙĦ عبة +Ġstron ÄĻ +Ġר×IJ ×ķ×Ļ +×Ĺ ×ij׾ +ĠÑĢÑĭ н +ĠÑĢÑĭн ке +Ġ׾×ŀ×¢ ף +اس ÙĦ +ห ัà¸Ļ +Ġ×IJ ×Ĺ×Ļ +ĠпÑĢод ол +ê°Ģ ìŀħ +Ġ×ijר ×Ĺ +Ġ×ijר×Ĺ ×ij×Ļ +дж еÑĢ +Ġ׾ ×Ĺ׾ +Ġ׾×Ĺ׾ ×ķ×ĺ +Ġ׾×Ĺ׾×ķ×ĺ ×Ļף +ศาส à¸Ļา +ãĤ¢ãĤ¤ ãĥĨ +ãĤ¢ãĤ¤ãĥĨ ãĥł +Ġפר ×ķפ +جز اء +ล à¸Ńย +Ġc iaÅĤa +Ġgi ết +ĠзнаÑĩ иÑĤелÑĮно +Ġolmad ıģ +Ġolmadıģ ını +н д +нд екÑģ +تأ Ùĥد +Ġìĸ ¸ +Ġìĸ¸ ìłľ +ay dın +ãĥī ãĥ¬ãĤ¹ +Ġs ắt +Ġíĺ¸ íħĶ +Ġë¶ ģ +Ġë¶ģ íķľ +ãĥij ãĤ¤ +Ġ×ŀש×Ĺ×§ ×Ļ +à¸Ħà¸Ļ à¸Ńืà¹Īà¸Ļ +Ġиз гоÑĤов +ĠизгоÑĤов лен +à¹Ģà¸ģีย ร +à¹Ģà¸ģียร à¸ķิ +תק שר +ĠÑĢаÑģ ÑĩеÑĤ +ส à¹Ģà¸ķ +Ġl änger +ĠiÅŁ let +ĠiÅŁlet me +Ġع ÙĦÙĬÙĨ +ĠعÙĦÙĬÙĨ ا +é lection +ĠاÙĦغ ربÙĬØ© +íĭ Ģ +ãĤĤãĤī ãģĪ +Ġкни ги +Ø£ سÙħ +أسÙħ اء +Ġth á»ı +Ġthá»ı a +หà¸Ļ ู +Ġ×ł×¢ ש×Ķ +à¸łà¸²à¸¢ à¹ĥà¸ķà¹ī +à¸ŀื à¸Ĭ +رÙĬ Ø· +Ùģ ÙĪØ¶ +ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸ ãģĦãģ¾ãģĹãģŁ +ש ×ĵ×Ķ +Ġng á»±c +ĠÑģеÑĢ ÑĮ +ĠÑģеÑĢÑĮ езн +T ôi +Ġfiyat ları +ĠвÑģ Ñİ +ĠC ódigo +Ġ×Ķש ×IJ +Ġ×Ķש×IJ ׾×Ķ +ĠP ública +Ø¥ Ø® +إخ ÙĪØ§ÙĨ +ĠзаÑıв ил +ãĥ¦ ãĥ¼ +ר×IJ ×Ļת +vol ución +Ġsz ko +Ġszko ÅĤy +جرÙĬ دة +Ġpens é +ìī ¬ +ĠBüyük ÅŁehir +ĠØ£Ùħ رÙĬ +ĠØ£ÙħرÙĬ ÙĥÙĬ +à¸Ļัà¸ģ ศึà¸ģษา +Ġtod av +Ġtodav ÃŃa +ĠС ан +ĠСан кÑĤ +íķĺ ìŀIJ +ØŃÙĪ Ø§ÙĦ +׼ ×ķשר +à¹Ģลย à¸Ħรัà¸ļ +Ġal gu +Ġalgu ém +Ùģ Ø² +Ġçek il +Ġ×ĵ ר׼×Ļ×Ŀ +ãĥIJ ãĥ© +à¸ģà¹ĩ สามารà¸ĸ +สà¹Īวà¸Ļ ลà¸Ķ +íı ° +ĠP úb +ĠPúb lico +à¹ģà¸Ļว à¸Ĺาà¸ĩ +×IJת ×Ĵר +Ø´ اش +شاش Ø© +ci ÅĽni +ĠÃľ rün +ÙĦÙĪ ØŃ +ĠاÙĦ بÙĨ +ĠاÙĦبÙĨ Ùĥ +ì¡° ì¹ĺ +Ġorganiz ación +ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸ ãģĦãģ¾ãģĻ +s ätze +ĠÑģем ей +ÙĤ صد +ÑģÑĤв еннÑĭе +Ġpréc éd +Ġprécéd ent +à¸ģรุà¸ĩà¹Ģà¸Ĺà¸ŀ ฯ +ãģ¨è¨Ģ ãģĦ +×ij׳×Ļ ×Ļף +ĠØŃ ÙĪ +ĠØŃÙĪ Ø§ÙĦÙĬ +סק ס +ĠsaÄŁlam ak +Ġ׾ צ×Ļ×Ļף +×§×ĵ ש +Ġ×Ķ×ŀ ×¢×¨×Ľ×ª +Ġ׾×Ķ ×¢×ij×Ļר +Ġg ünd +Ġgünd em +ĠнаÑĪ ÐµÐ³Ð¾ +à¹ĥà¸Ļ à¸ŀืà¹īà¸Ļà¸Ĺีà¹Ī +à¹Ģà¸Ħร ืà¸Ń +à¹Ģà¸Ħรืà¸Ń à¸Ĥ +à¹Ģà¸Ħรืà¸Ńà¸Ĥ à¹Īาย +ظ اÙĩرة +ÙħÙĨ ظÙħ +ÙħÙĨظÙħ ات +Ùħت از +追 ãģĦ +dı kt +dıkt an +ĠëįĶ ìļ± +ĠÐĿ апÑĢимеÑĢ +tw ór +×ŀ×ķ×¢ צ×Ķ +Ùĥ ÙĪÙĥ +Ð © +×ŀ×ĺ פ׾ +ó lica +訪 ãĤĮ +ĠëĮĢ ë¶Ģ +ĠëĮĢë¶Ģ ë¶Ħ +ãĤ¯ãĥª ãĥĥãĤ¯ +ãĤĴ éģ¸ +ãĤĴéģ¸ ãģ¶ +Ġpow sta +Ġpowsta ÅĤ +Ġraz ón +×ij ×ķ×Ĺר +ĠÑģообÑī ил +Ġ×§ ×ij×ķ×¢ +r êt +à¸Ķี à¸Ĥึà¹īà¸Ļ +×ŀס ×¢×ĵ +×ŀסע×ĵ ×ķת +ĠÃĸ sterreich +Ġ׳ ×Ĺש×ij +Ùħباد رة +ì´ ī +×Ĵ ׳×ĺ×Ļ +ä¿¡ ãģĺ +du ÄŁ +duÄŁ unu +Ġph ú +ĠاÙĦØ£ Ø®ÙĬر +Ġت عتبر +landır ıl +ãģ¨ãģ¯ ãģĦ +ãģ¨ãģ¯ãģĦ ãģĪ +ĠاÙĦ Ø·ÙĦ +ĠاÙĦØ·ÙĦ اب +ĠN º +éģ¿ ãģij +اÙĦ Ùħع +اÙĦÙħع رÙĪÙģ +ส à¸łà¸² +éĽ¢ ãĤĮ +ĠпомоÑī ÑĮ +Ġзна еÑĤ +ãĥĹãĥ¬ ãĤ¼ +ãĥĹãĥ¬ãĤ¼ ãĥ³ãĥĪ +Ġsup érieur +Ġש׾ ×Ļש×Ļ +ĠاÙĦÙĨ ÙĪØ¹ +ãĤĵãģ§ãģĻ ãģŃ +à¸Ńà¸ļ รม +Ġgi á»įng +Ġwzgl ÄĻd +ĠاÙĦÙģ ÙĤر +è rent +Ġ×ŀ×IJ ×Ĺ +Ġ×ŀ×IJ×Ĺ ×ķר×Ļ +×Ĵ ×Ĵ +×Ļ ×Ļ×ij +ÙħÙĦ اب +ÙħÙĦاب س +Ġhük ü +Ġhükü met +Ġ×ŀ×Ĵ ×Ļ×ij +ĠÐŀ Ñĩ +ĠÐŀÑĩ енÑĮ +æĹ© ãģĦ +Ġconstr ucción +Ġth ượng +ï¼ ĭ +Ġcor ação +à¹Ģหล à¹ĩà¸ģ +ĠBaÅŁ b +ĠBaÅŁb akan +éĢ£ ãĤĮ +ãģĻãĤĭ ãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ +ĠÙĤ اÙħت +Ġا Ùĥثر +ÙģØ§Ø¹ ÙĦ +ĠÑĦ оÑĢ +ĠÑĦоÑĢ Ñĥм +غ ذÙĬ +ĠiÅŁ le +ĠiÅŁle ml +ĠiÅŁleml eri +ĠìĤ¬ëŀĮ ìĿĢ +Ġìŀij ìĦ± +Ġë§Ī 볨 +Ùħ جÙĦس +หม ู +д в +дв иг +двиг а +à¹Ģสีย à¸Ĭีวิà¸ķ +×Ķת פת×Ĺ +×Ķתפת×Ĺ ×ķת +ĠмеÑĤ ÑĢо +ĠÑģ енÑĤ +ĠÑģенÑĤ Ñı +ĠÑģенÑĤÑı бÑĢÑı +ê³ § +Ġ׾ פע +Ġ×ľ×¤×¢ ×ŀ×Ļ×Ŀ +à¹Ģà¸ļ ีย +詳 ãģĹãģı +çķ° ãģªãĤĭ +Ġİl çe +ĠAt at +ĠAtat ür +ĠAtatür k +รุ à¹Īà¸ĩ +Ġkald ı +Ġ주 ìŀ¥ +Ġprés ence +Ġн аб +Ġнаб лÑİ +ĠнаблÑİ Ð´Ð° +ĠÑģам ого +×Ĵ ×ķש +×ŀ×ĺ ×ķפ +×ŀ×ĺ×ķפ ׾ +ĠвÑĭб иÑĢа +ĠìŀIJ 리 +åĪĨ ãģĭãĤīãģªãģĦ +Ġз Ñĥб +Ġש׼ ×ijר +Ġد ائ +Ġدائ Ùħا +ĠпаÑĢ ÑĤи +ï¼ ² +ĠاÙĬ ضا +ĠÑħ оз +ĠÑħоз Ñı +ĠÑħозÑı й +ĠÑħозÑıй ÑģÑĤв +ĠاÙĦØ£ ج +ĠاÙĦأج ÙĨب +ĠاÙĦأجÙĨب ÙĬØ© +ĠÐĹ Ð½Ð° +ĠAp ós +ĠÑį неÑĢ +ĠÑįнеÑĢ Ð³Ð¸ +Ġy ans +Ġyans ı +ĠJust i +ĠJusti ça +Ġpré vu +ม วล +ìŀ¥ ëĭĺ +à¸ģระ à¸ļ +à¸ģระà¸ļ วà¸Ļ +à¸ģระà¸ļวà¸Ļ à¸ģาร +×ŀ ×ŀ +×ŀ×ŀ ×ķצע +Ġh ẹ +Ġhẹ n +зд ание +Ġak ÅŁ +ĠakÅŁ am +×ĺ ×ķפ +Ġgere kt +Ġgerekt i +Ġgerekti ÄŁini +Ġnar z +Ġnarz ÄĻdzi +é po +épo que +ĠTh ần +Ġwys oko +Ġwysoko ÅĽci +à¸ľà¸¹à¹ī à¸Ľ +à¸ľà¸¹à¹īà¸Ľ à¹Īวย +ĠÙĬ بدÙĪ +ÑĤелÑĮ ного +Ġвз глÑıд +Ġjed nÄħ +ĠìĿĺ 견 +Ġ à¸Ĥà¸ĵะà¸Ĺีà¹Ī +פ ×Ļ×ĵ +ìĥģ ëĭ´ +Ġm ỡ +×Ķ ×ŀ׾ +×Ķ×ŀ׾ צ×ķת +ĠÑģоÑģÑĤ о +ĠÑģоÑģÑĤо иÑĤ +Ġав и +Ġави а +ĠL änder +تص ÙĪÙĬر +×ŀ×ĵ ×Ļ×Ķ +ìłĪ ì°¨ +ãģ¨ ãĤĬ +ãģ¨ãĤĬ ãģĤ +ãģ¨ãĤĬãģĤ ãģĪ +ãģ¨ãĤĬãģĤãģĪ ãģļ +ĠÑĢ Ñıд +ĠÑĢÑıд ом +ĠNh ất +ĠاÙĦÙĥ اÙħÙĦ +×Ĺ׾ ׾ +ĠGi ấy +צ ×ĺר +צ×ĺר ×£ +Ġ׾×ij ×ĺ׾ +Ġим еÑĤÑĮ +ס×ŀ ×ķ×ļ +Ġparticip ação +íķľëĭ¤ ë©´ +ÙħÙĨت دÙĬ +ÙħÙĨتدÙĬ ات +ĠeÄŁ len +g änge +رب ØŃ +ãĤ® ãĥ£ +ĠاÙĦر ÙĤÙħ +à¸ĭ à¹īำ +ĠH óa +×ŀר ×Ĺ×§ +ØŃÙħ اÙħ +بÙĪ Ùĥ +ĠArt ÃŃculo +ãĥĦ ãĤ¢ãĥ¼ +×Ķפ ׼×Ķ +×Ĺ׾ ×ķף +ĠпеÑĢе Ñħод +len miÅŁ +زر اعة +Ġseñ or +ãģ£ãģ¦ ãģįãģ¦ +Ø¥ Ø´ +إش ارة +Ġpod ÃŃa +ĠÃľ lke +н ÑģкаÑı +Ġadapt é +Ġdüzen len +Ġdüzenlen en +ĠÑģÑĤ ала +ĠÙĬ ØŃتاج +Ġn ier +Ġnier uch +Ġnieruch omo +Ġnieruchomo ÅĽci +ãģĵãģ¨ãģĮ ãģĤãĤĭ +ยà¸Ńà¸Ķ à¹Ģยีà¹Īยม +ĠÙħ ج +ĠÙħج اÙĨÙĬ +Ġз аб +Ġзаб ол +Ġзабол ев +Ġзаболев аниÑı +ĠÅĽ ro +ĠÅĽro dk +ĠÅĽrodk ów +Ġ×Ķ ×ľ×IJ×ķ×ŀ×Ļ +Ġdok ÅĤad +ĠdokÅĤad nie +ãģŁãģı ãģªãģĦ +ãģ¯ãģļ ãģ§ãģĻ +ã썿ĢĿ ãģ£ãģ¦ãģĦãģŁ +é cran +ìĹħ ì²´ +trzym aÅĤ +ÑģÑĤв еннÑĭй +ĠNot ÃŃc +ĠNotÃŃc ias +Ùħ رÙĬ +ÙħرÙĬ ض +æ°Ĺ è» +æ°Ĺè» ½ +æ°Ĺ軽 ãģ« +ëĵ £ +Ġ×ĵ ×ķ×IJר +Ġ׾ ×ŀ׳ +Ġ׾×ŀ׳ ×ķ×¢ +ĠçalÄ±ÅŁ ıyor +ĠÅŁ idd +ĠÅŁidd et +ĠM ặt +Ġate ÅŁ +ĠполÑĥÑĩ ениÑı +à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ มืà¸Ń +Ġgrö ÃŁer +د ائ +دائ رة +Ġbul un +Ġbulun maktadır +à¹Ģห ร +à¹Ģหร ีย +à¹Ģหรีย à¸į +à¸Ļัà¸ģ à¸Ĺà¹Īà¸Ńà¸ĩà¹Ģà¸Ĺีà¹Īยว +Ġalan ında +ĠÑĥ зна +Ġл еÑĩение +売 ãĤĮ +Ġçev ir +Ġdeste ÄŁi +ĠheiÃŁ t +âĸ ² +ØŃ Ø· +à¸Ħำ à¸ķà¸Ńà¸ļ +ãĤªãĥ³ ãĥ©ãĤ¤ãĥ³ +Ġ×ij×Ĺ×Ļ ×Ļ×Ŀ +ãĥ¦ ãĥĭ +Ġdüzenle me +Ġmodal itÃł +سر Ø· +سرط اÙĨ +×ŀ׼ ×ķף +ĠданнÑĭ й +تر ت +ترت ÙĬب +à¸ļาà¸ĩ à¸Ħà¸Ļ +ĠÄIJ á»ĭnh +ม ูล +มูล à¸Ħà¹Īา +ÙĨ ÙĤص +à¸ģาร รัà¸ģษา +ĠÑĦ он +ĠÑĦон д +ãĤĪãģĨ ãģ«ãģªãģ£ãģŁ +Ùħع اÙĦ +ÙħعاÙĦ جة +ĠOs man +ĠOsman lı +иÑĩеÑģк ом +à¸Ńยาà¸ģ à¸Īะ +ãģķãģ¾ ãģĸ +ãģķãģ¾ãģĸ ãģ¾ +ãģķãģ¾ãģĸãģ¾ ãģª +Ġת ×ķ׼׾ +×¢ צ×ij +ĠاÙĦع سÙĥ +ĠاÙĦعسÙĥ رÙĬ +Ġvé hic +Ġvéhic ule +Ġ×Ļצ ×Ĺ×§ +ĠاÙĦÙĪ ØŃ +ĠاÙĦÙĪØŃ ÙĬد +ĠاÙĦع دÙĪ +ĠQu ản +Ġê³µ ëıĻ +بد ÙĦ +ĠÄij ảng +Ġm á»ĩnh +Ġnie zb +Ġniezb ÄĻ +ĠniezbÄĻ dn +Ġyayın lan +обÑī и +Ġgö tür +צ פ +צפ ×ķ×Ļ +ĠÙĦÙĬ بÙĬ +ĠÙĦÙĬبÙĬ ا +ØŃ ÙĪØ§ +Ġд об +Ġдоб ÑĢо +иÑĢÑĥ ем +ĠاÙĦØŃÙĥÙĪÙħ ÙĬØ© +m Ã¤ÃŁig +Ġed ición +влек аÑĤелÑĮ +влекаÑĤелÑĮ н +Ġת ש׾×ķ×Ŀ +Ġ×Ķש ×ķ׳×Ļ×Ŀ +มิ à¸ĸุ +มิà¸ĸุ à¸Ļ +มิà¸ĸุà¸Ļ ายà¸Ļ +é£Łãģ¹ ãģ¦ +ĠìĪĺ ì§ij +ס ×ij×Ļ +ĠиÑİ Ð»Ñı +Ġà¹Ħà¸Ķà¹ī à¹ģà¸ģà¹Ī +׾×Ĺ ×Ŀ +tr ä +trä gt +ãģĿãĤĤ ãģĿãĤĤ +ÐĿ Ðķ +Ġв нÑĥÑĤ +ĠвнÑĥÑĤ ÑĢи +ãģ¨ ä¸Ģç·Ĵãģ« +ãĤ« ãĥķãĤ§ +Ġ×ij×Ĺ ×ĵר +×Ĺ ×ŀש +ãĤ¨ ãĥį +ãĤ¨ãĥį ãĥ« +ãĤ¨ãĥįãĥ« ãĤ® +ãĤ¨ãĥįãĥ«ãĤ® ãĥ¼ +à¸Ĥà¸Ńà¸ĩ à¸ķัวà¹Ģà¸Ńà¸ĩ +بÙĤ اء +פס ×Ļ׼ +פס×Ļ׼ ×ķ׾×ķ×Ĵ +ãĥ¡ ãĥĥ +ãĥ¡ãĥĥ ãĤ» +ãĥ¡ãĥĥãĤ» ãĥ¼ãĤ¸ +ÙĦ ÙĤب +A Äŀ +שק ×Ļ×¢ +ÙĤ ساÙħ +×ĵ×ķ×Ĵ ×ŀ×Ķ +æ·± ãģĦ +íĸĪ ëĬĶëį° +ĠrozwiÄħz anie +à¸Ļัà¹Īà¸Ļ à¹Ģà¸Ńà¸ĩ +×Ļצ ×ij +Ġtr ông +à¹ĥà¸Ĭà¹ī à¸ļริà¸ģาร +ĠاÙĦÙħÙĪ Ø³Ùħ +ĠдеÑĤ и +ãģĹãģĭ ãģªãģĦ +ס ×Ļף +Ġréfé rence +à¹ģห à¹īà¸ĩ +ãĤĤãĤī ãģ£ãģŁ +Ġ׾ ר׼ +Ġ׾ר׼ ×ķש +شع ÙĪØ± +ĠÐij ог +Ġlaz ım +Ġ×Ļש ׳×Ŀ +Ġп аÑĢÑĤ +ĠпаÑĢÑĤ неÑĢ +ĠÑĥ ника +ĠÑĥника лÑĮн +Ġmaté riel +×ŀר ×§ +Ġph ưá»Ŀng +Ġз ай +Ġзай м +Ùģ ÙĤد +Univers itÃł +×¢ ר׼×Ļ×Ŀ +Ġba ño +Ġн оÑı +ĠноÑı бÑĢÑı +à¸Ľ à¹īาย +Ġt ats +Ġtats äch +Ġtatsäch lich +ĠÑĤÑĢ ÐµÑĤÑĮ +Ñį м +ãĥĻ ãĥ¼ãĤ¹ +Ġnh á»±a +ìĬ¤ íģ¬ +ĠعبداÙĦ ÙĦÙĩ +Ġת ×ķר×Ķ +أش ÙĬ +أشÙĬ اء +ĠÙĦÙĦ غا +ĠÙĦÙĦغا ÙĬØ© +Ùħ ÙĪØ§ÙĤ +ÙħÙĪØ§ÙĤ Ùģ +ĠgÅĤówn a +Ġart Ä±ÅŁ +Ġ×ŀ×§ ×ķ×ŀ×Ļ +ãĤ¯ãĥ© ãĥĸ +Ġس ÙĪÙī +ĠìŬ ìĦ± +اس ر +اسر ائÙĬÙĦ +Ġ׳ ×Ľ×ª×ij +ย à¹īà¸Ńà¸Ļ +Ġdeber á +Ġph ẫu +ÑİÑī ем +ĠÙĦدÙĬ ÙĨا +×ŀ×ĺ ×Ķ +Ġ׳ ×ķ׾×ĵ +ĠвÑģÑĤÑĢ ÐµÑĩа +ãĤīãĤĮ ãģ¦ãģĦãģ¾ãģĻ +ĠcaÅĤ ej +ย ึ +ยึ à¸Ķ +поÑĤ ен +поÑĤен ÑĨи +Ġл иÑĤ +ĠлиÑĤ еÑĢ +ĠлиÑĤеÑĢ Ð°ÑĤÑĥÑĢ +Ġкажд ом +ĠíĮ IJ +ĠíĮIJ ëĭ¨ +à¸Ī ู +Ġpres ença +ãģªãĤĵ ãģ§ +Ùħ ÙĬاÙĩ +ин ÑĦоÑĢм +инÑĦоÑĢм аÑĨион +инÑĦоÑĢмаÑĨион н +ĠìŀIJ ìŰ +ר׼ ש +Ġöd ül +ç¶ļ ãģı +Ġп Ñģ +ĠпÑģ иÑħ +ĠпÑģиÑħ олог +ت ذÙĥر +Ġìŀħ ìŀ¥ +ล à¸Ķà¹Į +ìĦł ê±° +ãģ£ãģ¦ ãģĬãĤĬãģ¾ãģĻ +Ġ×Ļ ×¢ +Ġ×Ļ×¢ ×§×ij +ĠاÙĦØ· عاÙħ +ãĥĨ ãĤ¹ãĥĪ +ĠTu ấn +Ġparticip ación +×ŀ×ķ×ŀ ×Ĺ×Ķ +×Ĵר ס×Ķ +ĠاÙĦتÙĨ ÙģÙĬ +ĠاÙĦتÙĨÙģÙĬ ذÙĬ +ĠбезопаÑģ н +ge f +gef ähr +Ø´ ÙĪØ± +Ġmy ÅĽli +ÙĪØ§ Ø´ÙĨ +ÙĪØ§Ø´ÙĨ Ø·ÙĨ +׳×ķס ×¢ +Ùĥ Ùĩ +ÙĥÙĩ رب +ÙĥÙĩرب اء +Ġmus iaÅĤ +ìĭ ¸ +ãĥĸãĥ© ãĥĥãĤ¯ +Ġcré é +ÙĨÙĩ ار +owo ÅĽÄĩ +ÙħØŃا ÙĥÙħ +ĠwÅĤa ÅĽ +ĠwÅĤaÅĽ c +ĠwÅĤaÅĽc iciel +ĠÙĬ ؤ +ĠÙĬؤ دÙĬ +×ŀ×¢ ×ķ׳ +×IJ ×ij׾ +خط Ø£ +ĠÑħ олод +×ĸ ×ķ׾ +ãģĵãĤĮ ãĤī +ãģĵãĤĮãĤī ãģ® +Ġbás ica +ฤ à¸Ķ +ฤà¸Ķ ูà¸ģ +ฤà¸Ķูà¸ģ า +ฤà¸Ķูà¸ģา ล +èIJ½ãģ¡ çĿĢ +ãģªãģĦ ãģĵãģ¨ +ص ÙĪÙħ +ÙĨج ØŃ +׳ק ×ķ×ĵ +׳ק×ķ×ĵ ת +кл аÑģÑģ +íķĺìĭľ ëĬĶ +ëĦ ĺ +Ġש×IJ ×Ļ׳×ķ +ĠС ейÑĩаÑģ +may acaģı +Ġyap ılır +Ġcategor ÃŃa +عب اد +ĠТ еп +ĠТеп еÑĢÑĮ +×Ķ×Ļס×ĺ ×ķר×Ļ +h ế +ãĤ³ ãĥ¼ãĥī +Ġcabe ça +ج Ùħا +جÙħا Ùĩ +جÙħاÙĩ ÙĬر +ä½İ ãģĦ +ĠÑĤоваÑĢ Ð¾Ð² +à¸Ĭาว à¸ļà¹īาà¸Ļ +ĠÑģÑĤан ов +ĠÑģÑĤанов иÑĤÑģÑı +ĠавÑĤом обилÑĮ +ĠÑģлÑĥÑĩ ай +à¸Ńั à¸ŀ +ĠG iriÅŁ +ĠìĿ¼ ëĭ¨ +ĠпÑĢ Ð¾Ñģ +ĠпÑĢоÑģ моÑĤÑĢ +ãģªãģıãģª ãģ£ãģŁ +มี à¸Ľà¸±à¸įหา +ïº İ +éc oute +ĠÙħ ÙĪØ¬ÙĪØ¯ +Ġس رÙĬع +ĠÙĪÙĩ ÙĨا +ĠÙĪÙĩÙĨا Ùĥ +à¸Ħุà¸ĵ สม +à¸Ħุà¸ĵสม à¸ļัà¸ķิ +Ġìļ° ìĦł +à¸ŀระ à¸ŀุà¸Ĺà¸ĺ +好 ãģ¿ +ظ ÙĦÙħ +Ġм акÑģ +ĠмакÑģ ималÑĮ +ĠмакÑģималÑĮ но +ãĥª ãĤ¢ãĥ« +à¹ģมà¹ī วà¹Īา +ĠاÙĦØŃ ÙĪØ§Ø± +ãĥĹãĥ© ãĤ¹ +Ġع ÙĦاÙĤØ© +Ġíĸī ëıĻ +Ġgönder il +Ġl ãi +ĠsaÄŁ lıkl +ĠsaÄŁlıkl ı +ĠÑĪ Ð°Ð³ +Ġ×ij×IJר ×Ķ +prowadzi Äĩ +ãģĦãģı ãģ¤ãģĭ +Ġبت ارÙĬØ® +Ġ×ij×IJ×ķת ×Ķ +Ġmó c +ĠÐľ не +ãĥĹãĥ¬ ãĥ¼ +×IJ ×ĸר×Ĺ +åł´åIJĪ ãģ«ãģ¯ +使 ãģĪ +à¹Ģร ืà¸Ńà¸Ļ +ĠÐŁ еÑĤ +ĠÐŁÐµÑĤ ÑĢ +ãģ«åħ¥ ãĤĭ +Ùħ ادة +à¹Ģà¸ĩ ืà¹Īà¸Ńà¸Ļ +à¹Ģà¸ĩืà¹Īà¸Ńà¸Ļ à¹Ħà¸Ĥ +ĠÑģоÑģÑĤоÑı ние +ôn ica +ĠÑĦ ев +ĠÑĦев ÑĢа +ĠÑĦевÑĢа лÑı +Ġ×ķ ×ĸ +Ġ×ķ×ĸ ×IJת +à¸Ħร ิ +à¸Ħริ ส +ĠÐķ Ñīе +ãģ£ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĹãģŁ +ĠпÑĢав иÑĤелÑĮ +ĠпÑĢавиÑĤелÑĮ ÑģÑĤв +Ġtä glich +Ġëĭ¹ ìĭľ +×ŀ×ķ×¢ ×ŀ×ĵ +Ġдв оÑĢ +æī ķ +æīķ ãģĦ +ĠÑģÑĤан еÑĤ +Ġвозд ейÑģÑĤв +ĠвоздейÑģÑĤв и +Ġf ête +à¹Ģส า +תק ×ķ×ķ×Ķ +Ġu yar +Ġuyar ı +à¸ģลัà¸ļ à¹Ħà¸Ľ +Ġgi ưá»Ŀng +Ġв а +Ġва ÑĪи +ĠÄij áºŃu +ĠSpa ÃŁ +ĠìķĦ ë§Ī +à¹Ħà¸Ķà¹ī à¸ĩà¹Īาย +Ġ×Ķ×ŀ ×ijקש +æĸ° ãģŁ +æĸ°ãģŁ ãģª +ılı yor +пл ан +Ġ×Ķ×ijר ×Ļ×IJ×ķת +ĠaÄŁ rı +Ġsay gı +建 ãģ¦ +Ġnaj wyż +Ġnajwyż sz +سÙĬاس ات +ãģĬ å¾Ĺ +ĠاÙĦع ÙĦÙĬ +ĠاÙĦعÙĦÙĬ ا +Ġcoraz ón +ì¹ĺ ë£Į +หัว à¸Ĥà¹īà¸Ń +Ġب ØŃÙĬ +ĠبØŃÙĬ Ø« +зв езд +بÙĪ Ø§Ø¨Ø© +ÐĽ Ðĺ +ÙĦا زÙħ +Ġroz p +Ġrozp oc +Ġrozpoc zÄĻ +触 ãĤĮ +ĠاÙĦج ÙħÙĩ +ĠاÙĦجÙħÙĩ ÙĪØ± +Ġsp ÄĻd +ĠspÄĻd z +วิà¸Ĺยา ศาสà¸ķรà¹Į +ив аеÑĤÑģÑı +Ġдан ной +Ġreprés ente +ĠÄij á»ĭch +Ġ×¢×ŀ ×ķ×§ +à¸Ńัà¸Ļ à¸ķร +à¸Ńัà¸Ļà¸ķร าย +Ġestr atég +Ġestratég ia +pad ÅĤ +Ġв полн +Ġвполн е +ĠпÑĢедоÑģÑĤав лен +×Ĺ׾ ×ķ×§ +×Ĺ׾×ķ×§ ת +ãĤ¢ ãĥĬ +ĠاÙĦغ ذ +ĠاÙĦغذ ائÙĬ +ĠÑĥ зн +ĠÑĥзн аÑĤÑĮ +à¸ĭ à¹īาย +å½ĵ ãģ¦ +ØŃÙĬ اء +Ġbás ico +×§×ķ×ij ×¢ +ĠاÙĦÙħ باراة +ĠاÙĦÙĩ اتÙģ +Ġ׼ ׳×Ĵ×ĵ +à¸Ľà¸£à¸° หย +à¸Ľà¸£à¸°à¸«à¸¢ ัà¸Ķ +Ðļ ак +à¸Ĺีà¹Ī à¸Ļà¹Īา +à¸Ĺีà¹Īà¸Ļà¹Īา สà¸Ļà¹ĥà¸Ī +ãģ¾ ãģģ +ï½ ¢ +Ñģк оп +Ġson rasında +Ġur zÄħd +ĠurzÄħd zenia +׼×ķ ×ķ׳ +׼×ķ×ķ׳ ת +Ġ׾×Ķת ×ŀ×ķ×ĵ +Ġ׾×Ķת×ŀ×ķ×ĵ ×ĵ +ĠÑģ ли +ĠÑģли ÑĪ +ĠÑģлиÑĪ ÐºÐ¾Ð¼ +ĠÑģÑĤ Ñĥд +ĠÑģÑĤÑĥд енÑĤ +Ġ×Ķ ×ķ×ĵ +Ġ×Ķ×ķ×ĵ ×¢×Ķ +ë¹Ħ ìļ© +à¸Ńยาà¸ģ à¹ĥหà¹ī +Ġb á»ģ +ยุ à¸Ĺà¸ĺ +Ðĺ ÐĿ +س ائر +Ø£ صÙĪÙĦ +ĠاÙĦغ رÙģ +ãģĵãģ¨ãĤĤ ãģĤãĤĬãģ¾ãģĻ +è¾¼ ãģ¾ãĤĮ +ĠاÙĦساب ع +Ġc á»§ +ãģĦãģŁãģł ãģĦãģŁ +ì§ ĵ +ìĤ¬ 무 +powied ź +تÙģ Ùĥ +تÙģÙĥ ÙĬر +иÑĢов ки +ĠíĨµ íķ´ìĦľ +ãĤ¨ ãĤ¹ãĥĨ +ĠдеÑıÑĤелÑĮ ноÑģÑĤÑĮ +ĠданнÑĭ м +Ġ×¢ ×ķר +Ġ×¢×ķר ׼×Ļ +×ķ×ĵ עת +Ġhayat ını +Ġb Äħd +ĠbÄħd ź +obs ÅĤug +à¹Ģà¸ŀียà¸ĩ à¹ģà¸Ħà¹Ī +à¸ĭ à¹Īา +è²ł ãģij +ĠÑģÑĤÑĢ ÐµÐ¼ +ĠÄij á»īnh +ĠÐł ÑĥÑģ +ĠN ữ +Ġ׾×Ķש ×Ļ×Ĵ +Ġjed noc +Ġjednoc ze +Ġjednocze ÅĽnie +Ġ×Ķ×Ĵ ×ij×ķ×Ķ +أخ ÙĦاÙĤ +ĠнаÑģ ел +ĠнаÑģел ениÑı +ĠÙĬ ÙĨب +ĠÙĬÙĨب غÙĬ +ãģĮ ãģĭ +ãģĮãģĭ ãģĭ +×Ĵ עת +Ðŀ Ðł +ĠналиÑĩ ии +Ġë§Ī ì§Ģ +Ġë§Īì§Ģ ë§ī +Ġíĸī ìĤ¬ +Ġtre ÅĽci +Ġê°Ģ ì¹ĺ +ì¦ ĺ +Ġана лог +×Ķצע ת +в лад +влад е +ĠÑģдел ал +Ġ׳ ×Ĵ×Ļש +Ġ׳×Ĵ×Ļש ×ķת +полн ение +à¸Ĩ à¹Īา +ĠD ön +׼׾׼ ׾×Ķ +×ŀ×ĸ ×Ĵ +Ùħ Ùģ +ÙħÙģ Ùĩ +ÙħÙģÙĩ ÙĪÙħ +×Ķ ×ĵ +×Ķ×ĵ פס +×Ķ×ĵפס ×Ķ +ãģĻãģİ ãģ¦ +Ġг ÑĢ +ĠгÑĢ Ð½ +×ŀ×ĺ ×ķס +Ġ기 ìĸµ +ï¾ Ł +ĠpÅĤ yn +ĠGr ünde +ĠBü cher +Ġwed ÅĤug +ãģ¾ãģł ãģ¾ãģł +Ġ׳×Ķ ×ĵר +ĠÙĬست Ø·ÙĬع +ĠHi á»ĩp +ãĤŃãĥ£ãĥ³ ãĥļ +ãĤŃãĥ£ãĥ³ãĥļ ãĥ¼ãĥ³ +Ġth á»ķ +Ġeuropé enne +à¸ļ ัà¸ĩ +à¸ļัà¸ĩ à¸Ħัà¸ļ +ĠszczegóÅĤ owo +׳ שק +ãĥķ ãĥ©ãĥ³ãĤ¹ +×ŀ×ķ×ŀ ×Ĺ×Ļ +Ġcom ún +Ġç arp +ØŃت ÙĬا +ØŃتÙĬا ج +ØŃتÙĬاج ات +ëĭ´ ëĭ¹ +ä½ķ 度 +ä½ķ度 ãĤĤ +×ĵ ×ij×§ +ãģį ãĤĮ +ãģįãĤĮ ãģĦ +Ġк ам +Ġкам еÑĢ +ĠespecÃŃf ico +Ġtel éfono +à¸ķัà¹īà¸ĩ à¸Ńยูà¹Ī +I Åŀ +ãģ© ãĤĵãģ© +ãģ©ãĤĵãģ© ãĤĵ +עצ ×ŀ×IJ×Ļ +à¸Ķัà¸ĩ à¸Ļีà¹ī +ĠÑĦоÑĢм иÑĢов +ĠÑĦоÑĢмиÑĢов а +×ķ×ŀ ×ij +Ġkullan ımı +Ðľ Ðŀ +×¢ ש×Ļ +עש×Ļ ×Ļ×Ķ +Ġön lem +à¹Ģà¸Ń à¹ĩ +à¹Ģà¸Ńà¹ĩ ม +×ŀשק ×Ļ×¢ +ר ×Ļ×Ĺ +à¸Ĥ ัà¸Ķ +ĠíĻ ľ +ĠíĻľ ìļ© +à¸ĭ ะ +ãĤĪãģĨ ãģ«ãģªãĤĬãģ¾ãģĹãģŁ +ĠÑĢаÑģ пÑĢ +ĠÑĢаÑģпÑĢ Ð¾ÑģÑĤ +ĠÑĢаÑģпÑĢоÑģÑĤ ÑĢан +ĠÑĢаÑģпÑĢоÑģÑĤÑĢан ен +׼×Ļ ×ķף +ÙĤب ض +تص رÙĬØŃ +تصرÙĬØŃ ات +Ġо ÑĢи +ĠоÑĢи г +ĠоÑĢиг ина +ĠоÑĢигина л +ĠاÙĦع اÙĦÙĬ +à¹ģหà¹Īà¸ĩ à¸Ļีà¹ī +ãĥķãĤ¡ ãĥ¼ +ãģ¦ãģĦ ãģį +ãģ¦ãģĦãģį ãģŁãģĦ +פ תר +פתר ×ķ׳×ķת +Ġ×ij ×Ļ×Ĺ +Ġ×ij×Ļ×Ĺ ×ĵ +Ġod by +Ġodby ÅĤ +ĠоÑĩеÑĢ ÐµÐ´ +Ġtr ương +ãĤŃ ãĥ³ +×ŀ ×ķפ +×ŀ×ķפ ×¢ +ëĵľ 립 +ëĵľë¦½ ëĭĪëĭ¤ +à¸ŀืà¹īà¸Ļ à¸IJาà¸Ļ +ìŀIJ 격 +ĠVi á»ĩn +ĠDes pués +Ġ×IJ׾ ×Ļ׳×ķ +Ġdur ée +íĩ ´ +Ġmü zik +i ếu +ĠÑĢаз меÑīен +Ġк Ñĥд +ĠкÑĥд а +غ ض +غض ب +ĠTamb ém +à¸Īัà¸Ķ สà¹Īà¸ĩ +à¸ģาร à¹ģสà¸Ķà¸ĩ +onom ÃŃa +Ġан г +Ġанг ли +Ġангли й +Ġанглий Ñģк +Ġzn al +Ġznal az +Ġznalaz ÅĤ +תר ×Ĵ +תר×Ĵ ×ķ×Ŀ +ĠÑģ нов +ĠÑģнов а +ĠÑĩаÑģ а +Ġcommun auté +ĠespecÃŃf ica +ĠL á»ĭch +Ġli é +Ùģ Ø¬Ø± +à¹Ģà¸ģ à¹Īà¸ĩ +ع اÙĦ +عاÙĦ ج +Ø£ÙĨ ظ +Ø£ÙĨظ ÙħØ© +ES İ +ĠاÙĦØŃ دÙĬد +à¸ŀระ à¸Ńà¸ĩà¸Ħà¹Į +Ġפר שת +Ġдв иж +Ġдвиж ениÑı +ĠاÙĦج ارÙĬ +à¸ĺาà¸Ļ ี +неÑģ ен +ĠاÙĦÙĨ ÙĩائÙĬ +Ġб еÑĢ +ĠбеÑĢ ÐµÐ¼ +ĠбеÑĢем енн +Ġdépart ement +à¹Ģà¸Ĺ ีย +à¹Ģà¸Ĺีย à¸ļ +ĠÐľ аÑĢи +ĠнекоÑĤоÑĢ ÑĭÑħ +об еÑģп +обеÑģп еÑĩен +×Ĺ ×ķ×ĸ +×Ĺ×ķ×ĸ ×Ķ +ÙĨت ج +à¸Īะ à¹Ħà¸Ķà¹īรัà¸ļ +á» ° +Ġél éments +ع Ø· +عط اء +Ġt ắt +i á»ĩm +ÑİÑīиÑħ ÑģÑı +ãģĹãģ ° +ãģĹãģ° ãĤīãģı +Ġпом ожеÑĤ +à¸Ĥà¸ĵะ à¸Ļีà¹ī +Ġ×¢ שר×ķת +éģķ ãģ£ãģ¦ +ĠпÑĢ Ð¾Ð³ +ĠпÑĢог н +ĠпÑĢогн оз +Ġt ÅĤ +ĠtÅĤ um +ĠtÅĤum acz +T ür +Tür kiye +ãģį ãģ£ +ãģįãģ£ ãģĭãģij +Ġ×Ķ׳ ×ķ׼ +Ġ×Ķ׳×ķ׼ ×Ĺ×Ļ +ĠìĥĿ ìĤ° +ĠÑĦоÑĢм Ñĭ +ç¾İ ãģĹãģĦ +à¸Ľà¸£ ึà¸ģ +à¸Ľà¸£à¸¶à¸ģ ษา +Ġlum ière +ãĤª ãĥ¼ãĥĹ +ãĤªãĥ¼ãĥĹ ãĥ³ +à¸Ľ ืà¸Ļ +วั สà¸Ķ +วัสà¸Ķ ุ +еÑĢÑĤ в +ÙĥÙĦ Ùģ +ï½ £ +à¸ĺรรม à¸Ķา +׳ ×ĺר +ĠпÑĢедÑģÑĤав лÑıеÑĤ +Ġanál isis +Ġb ãi +با ÙĤÙĬ +à¸Ľà¸£à¸° à¹Ģà¸Ķ +à¸Ľà¸£à¸°à¹Ģà¸Ķ à¹ĩà¸Ļ +ĠÑģлÑĥÑĩ аÑı +ĠÑģлÑĥÑĩаÑı Ñħ +ÐĽ ÐIJ +สัà¸ĩ à¹Ģà¸ģ +สัà¸ĩà¹Ģà¸ģ à¸ķ +Ġprz ec +Ġprzec ież +Ùħ صÙĦ +ÙħصÙĦ ØŃØ© +ש×ķ×§ ×ķ׾×ĵ +ĠобоÑĢÑĥд ованиÑı +Ġtr waÅĤ +رÙĪ Ùħ +ìķĪ ëĤ´ +ĠNgh á»ĭ +Ø® Ø´ +à¸ļา à¸Ħาร +à¸ļาà¸Ħาร à¹Īา +Ġоп ÑĨион +ĠÑģозд аниÑı +ãĤ³ ãĤ¹ãĥĪ +Ġ×Ķ×¢ ׾×Ļ +Ġ×Ķ×¢×ľ×Ļ ×ķף +lä uft +ãĥĻ ãĤ¹ãĥĪ +Ġr ê +Ġrê ve +×IJ ×ij×Ļ×ij +×Ļ ×Ļ×ļ +ë¶ Ļ +ãĤ¤ãĥ³ ãĥī +ÅĤo ży +ÅĤoży Äĩ +ع ائÙĦ +عائÙĦ Ø© +Ø£ ÙĪØ± +Ø£ÙĪØ± اÙĤ +à¸Ĺà¹īà¸Ńà¸ĩ à¸ĸ +à¸Ĺà¹īà¸Ńà¸ĩà¸ĸ ิà¹Īà¸Ļ +Ġä hn +Ġähn lich +ãĥŁ ãĥĭ +à¸ľ ู +à¸ľà¸¹ à¹īà¸Ļ +à¸ľà¸¹à¹īà¸Ļ ำ +ĠмаÑĤеÑĢиал Ñĭ +Ġкап иÑĤ +ĠкапиÑĤ ал +ï¼ ¦ +Ġseç il +Ġh ứng +Ġintéress ant +ãģ£ãģ¦ ãģĦãģı +Ġe ÄŁer +ëIJĺ ìĹĪìĬµëĭĪëĭ¤ +Ġan laÅŁma +ãģĶ åĪ©ç͍ +Ġ×ij ×ĸ׼ +Ġ×ij×ĸ׼ ×ķת +ëĿ¼ ë©´ +ĠÙĬ ÙĪØ³ +ĠÙĬÙĪØ³ Ùģ +أسÙĦ ØŃØ© +ĠGef ühl +ĠноÑĢм алÑĮн +ãĥĻ ãĥ³ +ãģķãĤĮ ãĤĭãģĵãģ¨ +ĠÐij еÑģ +ãģ¨ãģĦ ãģĪãģ° +ĠÙħ ÙĩÙħ +ĠÙħÙĩÙħ Ø© +ãģ§ãģĹãĤĩãģĨ ãģŃ +ĠêµŃ ëĤ´ +à¹Ģม à¹ĩà¸Ķ +×ŀ×ij קר +ĠاÙĦد ÙĨÙĬ +ĠاÙĦدÙĨÙĬ ا +à¸Ĭ ู +к ÑĢÑĥÑĤ +Ġtho áng +Ġ׳ ×ĵר +Ġ׳×ĵר ש +ĠÑĢаÑģÑģ казал +ĠAu ÃŁerdem +פ ×IJר +פ×IJר ×§ +Ġ×ŀש×Ĺ×§ ×Ļ×Ŀ +צ ר׼×Ļ×Ŀ +×ŀ×ĵ ×ķ +×ŀ×ĵ×ķ ×Ļ×§ +èĭ¦ ãģĹ +ĠÑģ иг +ĠÑģиг нал +ĠM á»įi +Ġtr ữ +Ġnast ÄĻp +ĠnastÄĻp nie +Ġì¶Ķ ì§Ħ +ĠاÙĦÙģ ÙĨد +ĠاÙĦÙģÙĨد ÙĤ +koÅĦ czyÅĤ +ส ีà¹Ī +×§ ×Ļ×ij +×§×Ļ×ij ×ķ×¥ +ĠнÑĥж нÑĭ +大 åĪĩ +大åĪĩ ãģª +æıĽ ãģĪ +ת ×ķס +ת×ķס פת +ãģ£ãģ¦ ãģĦãģªãģĦ +Ġм Ñı +ĠмÑı г +ĠмÑıг к +Ġjak ie +Ġjakie ÅĽ +à¸ķำ à¸ļ +à¸ķำà¸ļ ล +ĠìŀĪ ì§Ģ +×ij×ĺ ×IJ +ĠоÑĤлиÑĩ но +ÙĤ ÙIJ +ĠавÑĤом об +ĠавÑĤомоб и +ĠавÑĤомоби лÑı +دÙĬÙħÙĤرا Ø·ÙĬ +ĠاÙĦ ÙĪØ§ +ĠاÙĦÙĪØ§ ØŃد +Ġس ÙĪØ±ÙĬØ© +Ø£ غÙĦ +أغÙĦ ب +ĠÑįк ÑĢан +ãĥĹ ãĥ©ãĤ¤ +Ġjeste ÅĽ +ãĥIJ ãĥª +Ġ×Ķ×IJ ×ķ×ķ×Ļר +ائ Ùĥ +à¸Ńยà¹Īาà¸ĩ ยิà¹Īà¸ĩ +ÑĢ ÐµÐºÑĤ +Ġum o +Ġumo ż +Ġumoż li +Ġumożli w +Ġumożliw ia +Ġnäch ste +ĠìŀĪ ì§Ģë§Į +ĠпÑĢед н +ĠпÑĢедн аз +ĠпÑĢедназ наÑĩен +Ġma çı +Ġp omi +Ġpomi ÄĻd +ĠpomiÄĻd zy +ĠاÙĦÙĦ ÙĤاء +à¹Ģà¸Ķ à¸Ńะ +Ġнов оÑģÑĤи +×ŀ׊׾×Ķ +رÙĬاض ÙĬ +à¸Ķ à¸Ļ +à¸Ķà¸Ļ à¸ķรี +ب صر +ìĬ¤ íĥĢ +scri pción +Ġnap isa +Ġnapisa ÅĤ +Ġ׳ש ×ŀ×¢ +ĠاÙĦÙħØŃ ÙĦÙĬ +Ġhi á»ĥn +×IJ ×Ĺ +×IJ׊ר×IJ×Ļ +Ġг ÑĢаниÑĨ +æīĭ ç¶ļãģį +Ùĥ سب +Ġà¹ģà¸ķà¹Ī à¸ĸà¹īา +à¸Ķาว à¸Ļà¹Į +à¸Ķาวà¸Ļà¹Į à¹Ĥหลà¸Ķ +ãĤĭãģĵãģ¨ãģĮãģ§ãģį ãģ¾ãģĻ +åŁºæľ¬ çļĦãģ« +ÙĪÙĦ اد +rä ume +د ÙģØ§Ø¹ +×Ļצ ×¢ +ĠO czy +ĠOczy wiÅĽcie +ĠÅ ģ +ĠÅģ a +اÙĦÙĬ اب +اÙĦÙĬاب اÙĨ +áºł I +ĠBir liÄŁi +×Ķ ×ķצ +×Ķ×ķצ ×IJת +ĠÄij ua +Ġê·¸ëŁ¬ ëĭĪê¹Į +Ġréal ité +ع ÙĦاÙĤات +J este +Jeste ÅĽ +Ġмн ож +Ġмнож еÑģÑĤво +ï¼ « +ãĥĹãĥŃ ãĤ¸ãĤ§ +ãĥĹãĥŃãĤ¸ãĤ§ ãĤ¯ãĥĪ +ĠÑĦ л +ظ ÙĨ +×Ĵ׾ ×Ĵ׾ +ĠmÅĤod zie +ĠmÅĤodzie ż +à¸Ļà¹īำ à¸ķา +à¸Ļà¹īำà¸ķา ล +ÐĽ Ðķ +×ij ×ķ×ĺ +Ġ׾×Ķ ×Ĵ×Ļ×ĵ +ãģĵãģ¨ãĤĤ ãģĤãĤĭ +ز اد +×ŀ×Ļ×ĵ ×¢ +ĠgÅĤówn ie +ãĥı ãĤ¦ +ãĥıãĤ¦ ãĤ¹ +б ел +Ġét ape +ðŁĺ Ģ +Ġмод елÑĮ +a ģını +ש ×Ĺ×§ +ש×Ĺ×§ ף +Ġni ño +à¸Ĭ à¹īาà¸ĩ +à¹Ģล ีย +ĠÑĦоÑĢм е +ĠاÙĦØ´ رÙĬÙģ +ĠÑĥд аÑĢ +arr iv +arriv ée +Ġmies iÄĻ +ĠmiesiÄĻ cy +ØŃ رÙĥ +ØŃرÙĥ ات +ĠDi á»ħn +ÐĿ Ы +ãģ¾ãģ£ãģŁ ãģı +Ġ×Ļ ×¨×ķ×§ +еÑģÑĤ еÑģÑĤв +еÑģÑĤеÑģÑĤв енн +Ġê·¸ ëŁ¼ +ĠاÙĦÙħ تÙĪ +ĠاÙĦÙħتÙĪ Ø³Ø· +Ġbéné fic +Ġbénéfic ie +Ġwy bra +Ġwybra Äĩ +ĠاÙĦز ÙħÙĨ +ĠпÑĢин Ñı +ĠпÑĢинÑı л +Ù쨱 ØŃ +Ġk sz +Ġksz taÅĤ +ĠksztaÅĤ t +ק׾ ×ĺ +×ij×ĵ×Ļ×§ ת +Ġgi ấ +Ġgiấ c +Ġpropriet Ãł +деÑĢж ан +ĠKö ln +ĠGü zel +×Ļפ ×ķ×Ļ +ĠCu á»Ļc +ÑįÑĤ аж +تر ÙĥÙĬ +ترÙĥÙĬ ز +лож ений +Ġп Ñĥ +ĠпÑĥ ÑĤи +اخت ÙĦاÙģ +åĩºãģ¦ ãģıãĤĭ +à¸ļุ à¸ģ +âĿ ¤ +ÑĦ ан +פש ×ĺ +à¸ļัà¸Ļ à¹Ģà¸Ĺ +à¸ļัà¸Ļà¹Ģà¸Ĺ ิà¸ĩ +ĠاÙĦس اد +ĠاÙĦساد س +ĠاÙĦÙĤ ÙĪÙħ +ĠاÙĦÙĤÙĪÙħ ÙĬ +Ġyönet ici +Ùĩ ÙĪØ§Øª +ÙĩÙĪØ§Øª Ùģ +Ġrespons ável +Ġпод деÑĢжива +ĠاÙĦسÙĦ Ø· +ĠاÙĦسÙĦØ· ات +ãģĹãģ¦ ãģĬãģı +ãĥļ ãĥĥãĥĪ +à¸Ľ ุà¹Īม +Ġogl Äħda +ÙĨا ÙĤ +ÙĨاÙĤ Ø´ +à¸Ħà¸Ńà¸Ļ à¹Ĥà¸Ķ +ĠMü sl +ĠMüsl ü +ĠMüslü man +ĠMo ż +ĠMoż na +Ġnum érique +Ġv á»ı +ĠسÙĬ تÙħ +Ġyer leÅŁ +монÑĤ аж +Ġgo ût +ãģ¦ ãģĬãĤĬãģ¾ãģĻ +ĠKh ánh +Ġе дин +Ġедин ÑģÑĤв +اÙĨ Ø®Ùģ +اÙĨØ®Ùģ Ø§Ø¶ +ìĭľ íĹĺ +Ġl ặng +ĠÑĢ Ð¾Ð»ÑĮ +à¸ķัว à¹ģà¸Ĺà¸Ļ +à¸Ħà¹Īา à¹ĥà¸Ĭà¹ī +à¸Ħà¹Īาà¹ĥà¸Ĭà¹ī à¸Īà¹Īาย +Ġver füg +Ġverfüg bar +ìĻĶ ëĭ¤ +ãģĦ ãģļ +ãģĦãģļ ãĤĮ +ĠиÑģÑģлед ованиÑı +меÑī а +×Ķ ×Ĺ +×Ķ×Ĺ ×ĸר +à¹ģà¸Ł à¸Ĭัà¹Īà¸Ļ +ت صرÙģ +Ø¥ رÙĩاب +Ġexerc ÃŃcio +Ġé lev +Ġélev é +สัà¸įà¸įา à¸ĵ +Ãĸ Z +ãĥĹ ãĥŃãĤ° +ãĥĹãĥŃãĤ° ãĥ© +ãĥĹãĥŃãĤ°ãĥ© ãĥł +Ġw ewnÄĻtrzn +Ġhen üz +é£Ľ ãģ³ +à¹Ģà¸Ķ à¸Ńรà¹Į +Ñģ Ñĥж +ÑģÑĥж ден +شع ÙĪØ¨ +ãģ²ãģ¨ ãĤĬ +Ġwy ÅĤÄħ +ĠwyÅĤÄħ cznie +Ġпло Ñħо +ÐĶ Ðķ +Ạ¦ +Ù쨹 اÙĦÙĬ +ÙģØ¹Ø§ÙĦÙĬ ات +ĠاÙĦع شر +ÑģÑĤÑĥп ил +Ġy arg +Ġyarg ı +нÑİ Ñİ +×ķ×IJ ×ij +Ġu ç +Ġuç ak +ë² ½ +تÙĪ ÙĤÙĬ +تÙĪÙĤÙĬ ع +Ġì¤ij ìĭ¬ +׳×Ļ×ķ ×ķ×ĺ +Ø£ ÙĥÙĦ +ç½® ãģĦãģ¦ +éłĤ ãģį +Ġ×Ķת ×ij +Ġ×Ķת×ij ×Ļ×¢×Ķ +Ġdür fen +Ùħ ÙĤاÙĦ +ÙħÙĤاÙĦ ات +Ġز ÙħÙĨ +à¸ŀฤ ศ +à¸ŀฤศ à¸Ī +à¸ŀฤศà¸Ī ิà¸ģ +à¸ŀฤศà¸Īิà¸ģ ายà¸Ļ +ĠнеÑģк олÑĮ +ĠнеÑģколÑĮ ки +ĠнеÑģколÑĮки Ñħ +Ġcrian ça +มิ à¸ķร +×ŀ׼ ×Ļר×ķת +à¸ģาร à¸ļริหาร +Ġtélé charg +Ġ×IJ×ķ×Ķ ×ijת +ĠBü ro +ä½ľ ãģ£ãģŁ +ĠKi ÅŁi +ç¾İåij³ ãģĹ +à¹Ģลย à¸Ħà¹Īะ +à¸ŀà¸ļ à¸ģัà¸ļ +à¸Ī à¹īา +Ġç er +Ġçer ç +Ġçerç eve +ãĤĴä½ľ ãģ£ãģ¦ +ĠпеÑĢв ÑĥÑİ +×ŀצ ר×Ļ×Ŀ +×IJ׾ ×ķ×Ķ +×IJ׾×ķ×Ķ ×Ļ×Ŀ +Ġagr é +Ġagré able +Ġay ır +İL İ +ãĤ ¥ +Ġíĺ Ħ +ĠíĺĦ ìĭ¤ +ثاÙĦ Ø« +ת ×ĸ +ת×ĸ ×ķ׳×Ķ +ãģ¨ãģĦ ãģ£ãģ¦ +ãģ¨ãģĦãģ£ãģ¦ ãĤĤ +Ġا بÙĪ +ĠÑģоб ак +é£Łãģ¹ ãģŁ +Ġдан ном +à¹Ģล ิ +à¹Ģลิ ศ +Ġí ļ +Ġíļ ¨ +Ġíļ¨ ê³¼ +ãĤĤãĤī ãģĪãĤĭ +׳ צ׾ +ÑĦ ик +ÑĦик Ñģ +Ġjeste ÅĽmy +ת×Ĺ×ķש ×Ķ +à¹Ħมà¹Ī à¸Ħวร +ĠØŃ سÙĬÙĨ +à¸ģาร ลà¸ĩà¸Ĺุà¸Ļ +ë´ ¤ +ĠÐĺ менно +à¸ļ à¸Ńรà¹Į +à¸ļà¸Ńรà¹Į à¸Ķ +ĠC ảnh +ìĦľ ë¹ĦìĬ¤ +Ġпол ов +Ġполов ин +Ġзам еÑĩа +ãģĦãĤį ãĤĵãģª +Ġ×ij ×Ļ×§ +Ġ×ij×Ļ×§ ש +л ÑĥÑĪ +ãĤĴ è¿İ +ãĤĴè¿İ ãģĪ +جرÙĬ ÙħØ© +Ġt ây +ĠاÙĦÙĨ ÙĪ +ĠاÙĦÙĨÙĪ ÙĪÙĬ +ÃĤ N +ì¿ ł +หà¸Ļ าว +Ġ×ij׊ש×ij×ķף +ز ار +à¸Ķ าร +à¸Ķาร า +ĠÅĽ l +ĠÅĽl ub +มีà¸Ħวาม สุà¸Ĥ +Ġn hu +Ġnhu áºŃn +ÙħØŃ طة +à¹Ģสืà¹īà¸Ń à¸ľà¹īา +ĠТ олÑĮко +ĠÙĥ س +ĠÙĥس ارة +ÙħØ´ رÙĪØ¹ +niÄĻ cia +×¢ ׼ש×Ļ×ķ +ت ÙĦÙģ +تÙĦÙģ Ø²ÙĬ +تÙĦÙ쨲ÙĬ ÙĪÙĨ +Ġl Æ°á»Ľi +ĠÐľÐ¾Ñģк вÑĭ +Ġré serve +Ġan laÅŁ +ĠanlaÅŁ ıl +Ġed eceÄŁi +รà¸Ńà¸ĩ à¹Ģà¸Ĺà¹īา +Ġب Ø· +Ġبط رÙĬ +ĠبطرÙĬ ÙĤØ© +ãģ¦ãģĹãģ¾ ãģ£ãģ¦ +ãĤĤãĤī ãģ£ãģ¦ +بر ج +æ± ļ +æ±ļ ãĤĮ +Ġch oc +Ġchoc ia +Ġchocia ż +Ġzob ac +Ġzobac zyÄĩ +пÑĢ Ñı +пÑĢÑı жен +ĠÑĨ иÑĦ +ĠÑĨиÑĦ ÑĢ +Ġм ам +Ġвз ÑıÑĤÑĮ +Ġch ạm +ج سÙħ +ØŃÙħ اس +à¹Ģล à¹Īม +à¸ŀิ ษ +×Ķפ ׼×ķ +à¸Ĭà¹Īà¸Ńà¸ĩ à¸Ĺาà¸ĩ +Ġв ек +Ġвек а +Æ¡ Ìģ +Æ¡Ìģ i +ĠTi á»ģn +Ġtr ầm +мÑĭ ÑĪ +мÑĭÑĪ Ð» +ĠÑĤ Ñĥ +ĠÑĤÑĥ ÑĢиÑģÑĤ +Ġch c +Ġchc Äħ +Ġав г +Ġавг ÑĥÑģÑĤ +ĠавгÑĥÑģÑĤ а +ס ×IJ×ķת +Ġר ×Ĵ׾ +à¸ľà¸¥ à¸ģระà¸Ĺ +à¸ľà¸¥à¸ģระà¸Ĺ à¸ļ +å¤īãĤı ãĤĭ +Ġ×Ķ×IJ×Ĺר ×ķ׳×Ļ×Ŀ +سÙģ ÙĬر +ĠÑĩа Ñīе +ãģĦ ãĤī +ãģĦãĤī ãģ£ +ãģĦãĤīãģ£ ãģĹãĤĥ +×ķ×ŀ ׳×Ļ×Ŀ +Ġart tır +ĠCh á»ĭ +Ġì¡° ì§ģ +ĠÑĥÑģп еÑħ +Ġ×¢ ×ķס +Ġ×¢×ķס ×§ +ĠìĥĿ ëªħ +ÑĨ иÑĤ +Ġreg ión +Ðŀ ÐĿ +ĠdoÄŁ um +ĠyaÅŁ ad +ĠyaÅŁad ıģı +à¸Ĺà¸Ķ ลà¸Ńà¸ĩ +Ġgöz ü +ש ×Ļר×Ķ +дÑĥм ал +Ġda ģı +Ġdaģı t +à¸Ĺีม à¸ĩาà¸Ļ +Ġti á»ģm +ĠاÙĦÙĥ بر +ĠاÙĦÙĥبر Ùī +ì¹ Ń +ĠGü nc +ĠGünc elle +ĠGüncelle me +ê¹ Ĭ +ĠобоÑĢÑĥд ование +ĠÑĢеÑĪ Ð° +á» ¤ +Ġп иÑĤ +ĠпиÑĤ аниÑı +à¹Ģรีย à¸ļ +×Ľ×ª ×Ļ×ij×Ķ +Ġп он +Ġпон ÑĢав +ĠпонÑĢав и +Ġ×Ķ ×ķ׾×ĵ +Ġ×Ķ×ķ׾×ĵ ת +Ġê² ģ +Ġê²ģ ëĭĪëĭ¤ +ĠпеÑĢв ой +ãĥ©ãĤ¤ ãĥķ +ĠÅŁi ir +kr ÄĻ +krÄĻ c +Ġthi á»ĥu +à¹Ģลย à¸Ĺี +à¹Ģลยà¸Ĺี à¹Ģà¸Ķียว +×ĺ×¢ ׳×ķת +ائ ÙĩÙħ +Ġ×IJ ס×ķר +ĠплаÑĤ еж +تر دد +Ġmożli we +Ġkh Ỽ +ĠkhỼ p +تÙģØ§Ø¹ ÙĦ +ĠÑĪ ÐºÐ¾Ð»ÑĮ +ĠÑĪколÑĮ н +ĠÙĤ صة +Ġmét ier +nÄĻ ÅĤa +หล à¹Īà¸Ń +Ġ á»§ng +Ġprz egl +Ġprzegl Äħd +ĠاÙĦÙħ تعÙĦ +ĠاÙĦÙħتعÙĦ ÙĤØ© +ĠÑģÑĭ н +Ġв олн +ãĥĩ ãĥ¼ãĥĪ +ĠÐŃ ÑĤи +Ġк ÑĢоме +à¸Ħ ารà¹Į +׳ק ×ķ×ĵ×Ķ +Ġ׾ש×ŀ ×ķ×¢ +Ġ×ĸ ×ķ׼ר +ï¼ § +ÙĬ ÙİØ§ +Ġgi á»ıi +åĥį ãģı +ĠÑģ ни +ĠÑģни жен +à¹ģà¸Ķ à¸Ķ +รุ à¸Ļ +รุà¸Ļ à¹ģรà¸ĩ +Ġhi á»ĩp +ograf ÃŃa +à¹Ģà¸Ī à¸Ńรà¹Į +Ġдв иг +Ġдвиг аÑĤ +ĠдвигаÑĤ ел +Ġü y +Ġüy eler +Ġüyeler i +Ġб Ñĥк +ĠбÑĥк в +ãĤĤ å¤ļãģı +Ġthi á»ĩt +ĠPa ÃŃs +ĠØ· بÙĬعÙĬ +à¹ģà¸Ī à¸ģ +ĠاÙĦص ØŃÙĬØŃ +Ġapp ré +Ġappré ci +Ġdecis ión +Ġë°ĺ ëĵľ +Ġë°ĺëĵľ ìĭľ +ĠÑĤеб е +ãĤ· ãĥ¼ãĤº +ãĤ·ãĥ¼ãĤº ãĥ³ +Ġд алÑĮн +ĠìĬ ¤ +ĠìĬ¤ ìĬ¤ +ĠìĬ¤ìĬ¤ ë¡ľ +ĠTh á»ĥ +Ġkar ÅŁ +ĠkarÅŁ ıs +ĠkarÅŁÄ±s ında +ĠK ön +ĠKön ig +ив ание +×ij ×ķצע +г лаÑģ +Ġtw ó +Ġtwó rc +à¸Ľà¸ģ à¸Ħร +à¸Ľà¸ģà¸Ħร à¸Ńà¸ĩ +ĠG ÅĤ +ĠGÅĤ ówn +ĠUnter stüt +ĠUnterstüt zung +Ġд ÑĥÑħ +ĠдÑĥÑħ ов +Ø£ ÙħاÙĨ +×Ĺש ש +ت ظ +تظ اÙĩر +ĠлÑİб ом +à¸ķ าร +à¸ķาร าà¸ĩ +Ġkr ól +Ø£ ØŃدث +ì¡Į ëĭ¤ +Ðļ ÑĥÑĢÑģ +ãĥĥ ãĥĦ +×ŀ×§ ×ķ×ij׾ +ĠÑģимв ол +Ġdés orm +Ġdésorm ais +w üns +wüns che +Ñĥ ни +Ñĥни ÑĨип +ÑĥниÑĨип алÑĮн +หลัà¸ģ สูà¸ķร +ÙĨت شر +Ġа л +Ġал к +Ġалк ог +Ġалког ол +ĠÑĥ ÑĩиÑĤÑĭва +à¸ģำ à¸ģัà¸ļ +Ġ׾ פע×ķ׾ +ĠìŰ ê²° +s Äħd +ĠاÙĦØ£ ÙĬ +ĠاÙĦØ£ÙĬ اÙħ +غÙĬ اب +Ġна ÑĢ +ĠнаÑĢ ÐºÐ¾ +×ŀ×ķ×ĵ ×¢ +ĠÑģеÑĢ Ð¸Ð¸ +пиÑģ Ñĭва +สิ ว +ç¶ļ ãģĦãģ¦ +çͳãģĹ è¾¼ãģ¿ +Ġ׾ ×Ĵר +Ġ׾×Ĵר ×ķ×Ŀ +Ġд ем +Ġдем о +Ġë³´ ëĤ´ +تÙĩ دÙĬد +ĠÙħØ´ ÙĬرا +Ġdu y +Ġduy á»ĩt +ĠwiÄĻks ze +Ùħع اÙĬ +ÙħعاÙĬ ÙĬر +ĠG da +ĠGda ÅĦsk +Ġr ah +Ġrah ats +Ġrahats ız +ר ×ķצ×Ķ +l ös +lös ung +ĠТак им +ÑĪ ÐµÐ´ +ÑĪед ÑĪ +ع زÙĦ +Ġרש ×Ļ×ŀת +Ġ׾×Ķ ×Ļ׼ +Ġ׾×Ķ×Ļ׼ ×ł×¡ +Ġп ÑĥÑĤ +ĠпÑĥÑĤ еÑĪ +ĠпÑĥÑĤеÑĪ ÐµÑģÑĤв +Ġnot ÃŃcia +Ġal Ä±ÅŁ +ĠalÄ±ÅŁ ver +ĠalÄ±ÅŁver iÅŁ +ĠwÅĤ os +ĠwÅĤos ów +Ġب غ +Ġبغ داد +Ġver öffent +Ġveröffent licht +ĠKh á +Ġt án +ëIJĺ 기 +Ġë°© 문 +Ùģ ÙĬÙĦ +à¹Ģà¸ģิà¸Ķ à¸Īาà¸ģ +åı¯ æĦĽ +åı¯æĦĽ ãģĦ +à¸ĸ ุà¸ĩ +Ġz ewnÄĻtrzn +à¸łà¸²à¸©à¸² à¸Ńัà¸ĩà¸ģฤษ +Ġmá xima +Ġul us +Ġulus lararası +Ġ׳×Ķ ×ł +à¸Ĥà¹Īาว สาร +ĠìĿĺ ìĤ¬ +à¹Ģหล ืà¸Ńà¸ĩ +Ġد ÙĤ +ĠدÙĤ ائÙĤ +สืà¹Īà¸Ń สาร +ë¨ ¼ +ĠÑģоÑģÑĤоÑı нии +สมา à¸Ħม +á» Ĥ +ĠÐľÐ¾Ñģ ков +ĠÐľÐ¾Ñģков Ñģк +×ŀס ×ķ×Ĵ׾ +ãģĭ ãģĭãĤĬ +ĠTr uyá»ģn +à¹ģà¸Ĥà¹ĩà¸ĩ à¹ģรà¸ĩ +×ŀ×Ĺ ×ĸ×Ļ×§ +à¹Ĥà¸ģ à¹ī +ÙĬس ر +ìĶ © +×IJ ×ķ×§ +×IJ×ķ×§ ×ĺ +×IJ×ķ×§×ĺ ×ķ×ijר +Ġprox imité +ÙħÙĨ Ùĩج +ĠاÙĦج ز +ĠاÙĦجز ائ +ĠاÙĦجزائ رÙĬ +ĠÄIJi á»ĥm +Ġден еж +Ġденеж н +ÙģØŃ ص +Ùģ Ø¦ +ĠÐij Ñĥд +×Ĵ×Ļ×ĵ ×ķ׾ +ĠÐĴ едÑĮ +عÙĦ اÙħØ© +Ġ×IJ×Ĺר ×ķ׳×ķת +ãģĦãģŁãģł ãģĦãģ¦ +سÙĦ ØŃ +ØŃ ÙĦÙħ +ز ÙĪØ§Ø± +Ùĥ سر +×ĺ קס +Ġб ан +Ġбан ков +ĠпÑĢ Ð¾Ð¶ +ĠпÑĢож ива +li wo +liwo ÅĽci +ĠTi ếp +ĠاÙĦÙħÙĨ اسب +ĠاÙĦØ® ÙĬار +ãģĬ ãģĭ +ãģĬãģĭ ãģĴ +à¸Ķà¸Ńà¸ģ à¹Ħมà¹ī +ä mp +ämp fe +à¸ķัà¹īà¸ĩ à¹ĥà¸Ī +Ġза ÑīиÑĤ +ĠзаÑīиÑĤ Ñĭ +ĠTh ưá»Ŀng +Ġص Ùģ +ĠصÙģ ØŃØ© +×Ĺ×ķר ×£ +ãĥIJ ãĥĥãĤ° +Ġ×ĵ ×Ļ×Ĵ +Ġ×ĵ×Ļ×Ĵ ×Ļ×ĺ +Ġ×ĵ×Ļ×Ĵ×Ļ×ĺ ׾×Ļ +Ġ×Ķ×Ĺ ×ķ׾×Ļ×Ŀ +в еÑī +веÑī а +Ġк ÑĥлÑĮÑĤ +ĠкÑĥлÑĮÑĤ Ñĥ +ĠкÑĥлÑĮÑĤÑĥ ÑĢÑĭ +ĠاÙĦاÙĨ ترÙĨت +Ġhö ch +Ġhöch st +Ġíĺ ķ +Ġíĺķ íĥľ +Ġв ой +Ġвой нÑĭ +ÐĽ Ðŀ +ìĭł ìļ© +Ġ×ŀ×ij ×ķס +Ġ×ŀ×ij×ķס ס +×ŀ׳ ×Ļ×¢ +Ġfiyat ı +ĠÑģл Ñĥж +ĠÑģлÑĥж бÑĭ +à¸Ĺั ศ +à¸Ĺัศ à¸Ļ +ãģĵãģ¨ãģĮ å¤ļãģĦ +Ġ×Ķ×ŀש ת +Ġ×Ķ×ŀשת ×ŀש +å¯Ħ ãģĽ +×ŀש׾ ×ķ×Ĺ +æĻĤ çĤ¹ +æĻĤçĤ¹ ãģ§ +à¸ŀร ี +à¸ŀรี à¹Ģมีย +à¸ŀรีà¹Ģมีย รà¹Į +à¸ŀรีà¹Ģมียรà¹Į ลีà¸ģ +Ġdiffic olt +Ġdifficolt Ãł +ãĥ¬ ãĤ¹ãĥĪ +ãĥ¬ãĤ¹ãĥĪ ãĥ©ãĥ³ +สม à¹Ģà¸Ķà¹ĩ +สมà¹Ģà¸Ķà¹ĩ à¸Ī +Ġж ид +Ġжид к +Ġzu peÅĤ +ĠzupeÅĤ nie +ĠÙħ جر +ĠÙħجر د +ãģĮ å§ĭ +ãģĮå§ĭ ãģ¾ +ãĤŃãĥ£ ãĥ© +Ġ×IJ ×ķ×ķ×Ļר +ãģĬ äºĴ +ãģĬäºĴ ãģĦ +Ġpot rÃł +ĠPa ÅĦst +ĠPaÅĦst wo +Ġب ÙĬاÙĨ +ĠبÙĬاÙĨ ات +Ġин огда +ĠÑĢ Ð° +ĠÑĢа ÑģÑĤв +ĠÑĢаÑģÑĤв оÑĢ +Ġ×ĸ ×ŀ׳ +ยิ à¹īม +Ä Ĩ +ãģ¾ ãģķ +ãģ¾ãģķ ãģ« +ãĥķãĤ¡ ãĤ¤ãĥ« +Ġgörd Ã¼ÄŁÃ¼ +สà¸ĩ à¸Ħร +สà¸ĩà¸Ħร าม +ĠArk adaÅŁ +ĠrozwiÄħz ania +×ŀ ×ķ×ĺ +pi ÄĻ +piÄĻ t +ص غر +ส ย +สย าม +ãĤĨ ãģ£ãģıãĤĬ +Ġtr ần +Ġeconom ÃŃa +Ġgeh ören +ãĤ·ãĥ§ ãĥ¼ +ĠsÅĤ ucha +à¸ŀà¸Ń à¹ĥà¸Ī +ĠоÑĤмеÑĤ ил +ÙĨت ÙĤÙĦ +Ġprop ósito +ĠваÑĪ ÐµÐ³Ð¾ +Ġnh ắn +à¹ģà¸ĸ ว +Ġком иÑģ +ĠкомиÑģ Ñģи +waż nie +Ġy avaÅŁ +×ŀ ×Ļ×§ +×ŀ×Ļ×§ ×ķ×Ŀ +ש×IJ׾ ת +Ġyıll arda +ĠÐ ® +ĠЮ ÑĢ +×ł×¡ ×Ļ×ij×ķת +ת צ +תצ ×ķ×Ĵ +Ġод нÑĥ +Ġ à¸Ńยà¹Īาà¸ĩà¹Ħร +Ġà¸Ńยà¹Īาà¸ĩà¹Ħร à¸ģà¹ĩà¸ķาม +ëģ ¼ +à¹Ħล à¹Ī +تس ÙĦÙĬÙħ +بÙĦ اغ +Ġì ī +Ġìī ½ +Ġìī½ ê²Į +ãĥļ ãĥ³ +зв ÑĥÑĩ +ĠW äh +ĠWäh rend +Ġ×Ļ ×Ļת +Ġ×Ļ×Ļת ׼ף +Ġkh uyên +Ġv ẽ +Ġа меÑĢ +ĠамеÑĢ Ð¸Ðº +ĠамеÑĢик ан +ĠамеÑĢикан Ñģк +ع جب +ãĥĽãĥ¼ãĥł ãĥļãĥ¼ãĤ¸ +Ġник ÑĤо +ĠÙĤ Ùİ +ĠÙĤÙİ Ø§ÙĦ +ĠÙĤÙİØ§ÙĦ Ùİ +ÐIJ ÐĹ +Ùħ جÙħÙĪØ¹ +ÙħجÙħÙĪØ¹ ات +Ġnecess itÃł +Ġpob li +Ġpobli żu +Ġph ấn +ĠСо обÑī +ÙħÙĤ اط +ÙħÙĤاط ع +Ġ×Ķצ ×ķר×ļ +la ÅŁtırma +ว ิà¸Ķ +วิà¸Ķ ี +วิà¸Ķี à¹Ĥà¸Ń +Ġ그리 ìĬ¤ +Ġ그리ìĬ¤ ëıĦ +ãĤ¿ãĤ¤ ãĥŁ +ãĤ¿ãĤ¤ãĥŁ ãĥ³ãĤ° +×§×ĺ ×Ĵ×ķר +×§×ĺ×Ĵ×ķר ×Ļ×Ķ +Ġ×Ĺ ×ķפ +Ġ×Ĺ×ķפ ש×Ļ +Ø£ جر +Ġим ени +ĠÑĢан ее +à¹Ģà¸ŀืà¹Īà¸Ńà¸Ļ à¹Ĩ +ĠJes ús +Ñģо един +Ñģоедин ен +Ġר ×Ĺ×ķ×§ +à¹Ĥà¸ļ รา +à¹Ĥà¸ļรา à¸ĵ +ĠH Æ¡n +Ġth áºŃp +تع ÙĬÙĬÙĨ +Ġtart Ä±ÅŁ +ĠtartÄ±ÅŁ ma +ĠGes pr +ĠGespr äch +תר ×ķפ +תר×ķפ ×ķת +Ġcat égorie +Ġоказ Ñĭва +ĠналиÑĩ ие +Ġprésent é +Ġk ull +Ġkull and +Ġkulland ı +Ġü nl +Ġünl ü +ĠÙģ Ùĥرة +из аÑĤоÑĢ +×IJ ×ķ׳ +×IJ×ķ׳ ×Ļ×ij +×IJ×ķ׳×Ļ×ij רס +×IJ×ķ׳×Ļ×ijרס ×Ļ×ĺת +ĠÑĢаÑģÑģ маÑĤ +ĠÑĢаÑģÑģмаÑĤ ÑĢ +ĠÑĢаÑģÑģмаÑĤÑĢ Ð¸Ð²Ð° +تÙĥÙĦ Ùħ +Ùĥت رÙĪ +ÙĥترÙĪ ÙĨÙĬ +ĠÑģо ÑĩеÑĤ +ĠÑģоÑĩеÑĤ а +ãĤĴè¦ĭ ãģĽ +Ġng ừa +ĠÐł еÑģп +ĠÐłÐµÑģп Ñĥб +ĠÐłÐµÑģпÑĥб лик +ãĤ¦ ãĤ© +ãĤ¦ãĤ© ãĥ¼ +ĠÐľ еждÑĥ +ĠìŀĪ ê²Į +Ġm â +ĠìļĶ ì²Ń +ض ار +ลุ à¹īà¸Ļ +ëĮĢ íķĻêµIJ +×ĸ ×Ļ׼ +×ĸ×Ļ׼ ר×ķף +ãĤ¹ ãĥļ +ãĤ¹ãĥļ ãĥ¼ãĤ¹ +ĠкÑĢаÑģ оÑĤ +ï¼ ¨ +ê¼ Ń +ãĤĴ éĽĨ +ãĤĴéĽĨ ãĤģ +ë° Ŀ +Ġ×Ķ׳ ×IJ +Ġ×Ķ׳×IJ ש×Ŀ +Ġê°Ģ ìļ´ +Ġê°Ģìļ´ ëį° +تÙĥÙĦ Ù쨩 +ĠØŃ ÙĤÙĬÙĤÙĬ +Ġh alk +Ġhalk ın +ÑİÑī ÑĥÑİ +ĠÑģп ин +סר×ĺ ף +ĠпеÑĢв ого +Ġпол ож +Ġполож иÑĤелÑĮн +Ġд л +Ġдл иÑĤелÑĮн +ĠV Ä©nh +ê´ ´ +ĠÑģÑĭ ÑĢ +ĠíĨµ íķĺìŬ +ë³ij ìĽIJ +à¹Ĥรà¸ĩ à¸ĩาà¸Ļ +รัà¸ļ à¸ľà¸´à¸Ķ +รัà¸ļà¸ľà¸´à¸Ķ à¸Ĭà¸Ńà¸ļ +تج ÙĨب +s ÅĤ +sÅĤ uch +ãĤ¢ãĥ« ãĥIJ +ãĤ¢ãĥ«ãĥIJ ãĥł +ëī´ ìĬ¤ +Ġpat ië +Ġpatië nt +Ġìĺ ¤í +Ġìĺ¤í ŀ +Ġìĺ¤íŀ Ī +Ġìĺ¤íŀĪ ëł¤ +ĠDer ne +ĠDerne ÄŁi +wró ci +wróci Äĩ +Ġоб Ñī +ĠобÑī еÑģÑĤв +ĠобÑīеÑģÑĤв енно +ĠêµIJ ìĪĺ +tıģ ımız +Ġ×Ķ×ŀש ×Ļ×ij +k örper +Ġпозв ол +Ġпозвол иÑĤ +ĠChi ến +أخ ÙĪ +ĠAy dın +à¸Ķà¹īาà¸Ļ ล +à¸Ķà¹īาà¸Ļล à¹Īาà¸ĩ +Ġdr u +Ġdru ż +Ġdruż yn +Ġë°ľ íijľ +ĠTh ảo +جÙĩ اد +à¸ģระà¸Ĺ ูà¹ī +Ġк ÑĢов +ĠкÑĢов и +Ġiçer ik +Ġnad zie +Ġnadzie jÄĻ +ĠС моÑĤÑĢ +Ġph ức +ج تÙħاع +جتÙħاع ÙĬØ© +ком пон +компон енÑĤ +Ġб ил +Ġбил еÑĤ +ãĥIJ ãĥ³ãĥī +ĠPol ÃŃcia +اÙĦ تÙĩ +اÙĦتÙĩ اب +ØŃر Ùģ +ت خط +تخط ÙĬØ· +ãĤ³ ãĥ¼ãĥ +ãĤ³ãĥ¼ãĥ Ĵ +ãĤ³ãĥ¼ãĥĴ ãĥ¼ +・・ ï½¥ +à¸ĭ à¸Ńย +Ġcréd it +è²· ãģ£ãģŁ +ĠпоÑĢ Ñıд +ĠпоÑĢÑıд ке +Ġph ó +Ġw ida +Ġwida Äĩ +جر ائÙħ +à¸ľ ี +ĠbÄĻd ÄĻ +Ġ×ŀ פת×Ĺ +ãĥij ãĥ¼ãĥ +ãĥijãĥ¼ãĥ Ĩ +ãĥijãĥ¼ãĥĨ ãĤ£ +ãĥijãĥ¼ãĥĨãĤ£ ãĥ¼ +ĠKa ż +ĠKaż dy +ĠнеобÑħодим оÑģÑĤи +à¸Ł à¸Ńรà¹Į +à¸Łà¸Ńรà¹Į ม +Ġмал ÑĭÑĪ +Ġпл оÑĤ +ĠÑĥ ÑģÑĤÑĢой +ĠÑĥÑģÑĤÑĢой ÑģÑĤва +à¸ĸ à¸Ńà¸Ļ +ĠoluÅŁtur ul +ĠÅĽwi ad +ĠÅĽwiad om +Ùħع Ùĩد +ĠпÑĢоиз веден +Æ ł +ר ×Ļש +Ùħست Ø« +Ùħستث Ùħر +׳×Ļ ×Ļר +pa ñ +Ġ; -) +Ġë°ľ 견 +Ġgör üyor +Ùħؤ ÙĦÙģ +ĠÄIJ á»ģ +ĠاÙĦÙĨ ÙĪØ§Ø¨ +×Ĺ×§ ×Ļר×Ķ +Ġm á»ıi +è¿° ãģ¹ +ÐĿ ик +ìŀĸ ìķĦ +ìŀĸìķĦ ìļĶ +prowadzi ÅĤ +l óg +lóg ica +פס ×ĺ +פס×ĺ ×Ļ×ij׾ +Ġ×ŀ ×ĵ×Ķ +Ġ×ŀ×ĵ×Ķ ×Ļ×Ŀ +ãģĵãģĵ ãģ¾ãģ§ +×Ķ ×ª×Ĺ +×Ķת׊׾×Ķ +Ġפ ×ķס +Ġפ×ķס ×ĺ×Ļ×Ŀ +Ġн ев +Ġнев оз +Ġневоз можно +ĠdostÄĻp ny +Ġغ اÙĦ +ĠغاÙĦ ب +Ġbez pieczeÅĦst +ĠbezpieczeÅĦst wa +åĪĨ ãģĭãĤĭ +ĠF ührung +à¸ģ ีà¹ī +gem Ã¤ÃŁ +à¸Ĭà¹Īวà¸ĩ à¹Ģวลา +Ġìļ°ë¦¬ ëĤĺ +Ġìļ°ë¦¬ëĤĺ ëĿ¼ +ãģ¥ ãģıãĤĬ +ĠاÙĦÙħ سÙĦ +ĠاÙĦÙħسÙĦ ØŃØ© +Ġlibert é +клÑİÑĩ ение +Ġzam ów +Ġzamów ienia +รà¸ĸ à¹Ħà¸Ł +Ø£ ÙģÙĦ +Ø£ÙģÙĦ اÙħ +Ùħ راج +Ùħراج عة +Ġë¹Ħ êµIJ +ĠاÙĦت اب +ĠاÙĦتاب عة +Ġë§Į ëĤĺ +Ġб Ñĥм +ĠбÑĥм аг +Ġgé nero +Ġìŀĺ 못 +×ŀ פ×ķר×ĺ +è²·ãģĦ çī© +ĠÙĦدÙĬ Ùĥ +Ġ×ľ×¢ ×Ļת +Ġ×ľ×¢×Ļת ×Ļ×Ŀ +ĠsÅĤ ab +ĠпÑĢедÑģÑĤав лÑı +ãĤ¿ ãĤ¤ãĥĪ +ãĤ¿ãĤ¤ãĥĪ ãĥ« +Ùħ ص +Ùħص Ø·Ùģ +ÙħصطÙģ Ùī +Ġdifficult é +ãĥĨãĤ£ ãĥĸ +Ġpew noÅĽci +ĠpewnoÅĽci Äħ +Ġ무 ìĬ¨ +Ø¥ رس +إرس اÙĦ +Ġд алÑĮ +ĠдалÑĮ ÑĪе +Ġ׾ ×ł×¡ +Ġ×ľ×ł×¡ ×ķת +หมูà¹Ī à¸ļà¹īาà¸Ļ +×ŀס×ŀ ׼×Ļ +أسÙĦ ÙĪØ¨ +Ġzw ÅĤ +ĠzwÅĤ as +ĠzwÅĤas zc +ĠzwÅĤaszc za +ĠпÑĢ ÐµÐ¶ +ĠпÑĢеж де +ĠоÑĢганиз аÑĨиÑı +Ġdön emin +Ġdönemin de +Ġ Ủ +ĠỦ y +ä¸ĭ ãģĴ +ĠпоÑģлед ние +Ġgü ne +Ġgüne ÅŁ +Ġ×IJ ×ĸר +Ġ×IJ×ĸר ×Ĺ×Ļ +ãģ§ãģĤ ãĤįãģĨ +ĠÙĨ ÙĤ +ĠÙĨÙĤ اط +æŃ£ ãģĹãģĦ +ĠÑĢ ÐµÐ³ +ĠÑĢег иона +ĠFör der +ê²½ ìĺģ +dıkl ar +dıklar ını +trzym aÄĩ +أش Ùĥ +أشÙĥ اÙĦ +×Ķת ×IJ +×Ķת×IJ ×ŀ×Ķ +à¸Ĺำà¹ĥหà¹ī à¹Ģà¸ģิà¸Ķ +ĠGeb ä +ĠGebä ude +ĠСеÑĢ Ð³ +ĠСеÑĢг ей +Ġз доÑĢов +ĠздоÑĢов ÑĮÑı +Ġr ãi +ĠпÑĢед ÑĥÑģ +ĠпÑĢедÑĥÑģ моÑĤÑĢ +ĠпÑĢедÑĥÑģмоÑĤÑĢ ÐµÐ½ +Ġ×Ķצ ×Ļ×ij +Ġ×Ķצ×Ļ×ij ×ķר×Ļ +Ġdés ir +Ġн оÑĩ +ĠноÑĩ ÑĮ +möglich keiten +Ġ×IJ×Ĺר ×ķ׳×Ļ×Ŀ +Ġsoir ée +ĠNh áºŃn +Ù ª +à¸Ľà¸£à¸°à¸§à¸±à¸ķิ ศาสà¸ķรà¹Į +êµIJ íĨµ +ĠØ£ Ø®ÙĬ +Ġdé cid +Ġdécid é +Ġwy ja +Ġwyja ÅĽni +Ġ สิ +Ġสิ à¸ĩ +Ġสิà¸ĩ หา +Ġสิà¸ĩหา à¸Ħม +à¹ģ à¸Ńรà¹Į +หà¸Ļà¹īา à¸Īà¸Ń +ס תר +Ġê ¶ +Ġê¶ Į +Ġê¶Į 리 +pl ätze +ب Ø·ÙĦ +ê±´ ìĦ¤ +Ġ×IJ ×Ļ×ŀ×Ļ +Ġ×IJ×Ļ×ŀ×Ļ ×Ļ׾ +ãģ ½ +تر اث +×IJ׾ ×Ļ×ŀ×ķת +Ġdispon ÃŃveis +Ġz ale +Ġzale ży +à¸Ľà¸£à¸°à¸Ĭา สัมà¸ŀัà¸Ļà¸ĺà¹Į +ĠÅļw iat +Ġpor ówn +Ġporówn a +Ġ׾×ĺ ×ķ×ijת +×Ķ×ĸ ×ŀ׳×Ķ +Ġ×Ľ×ª ×ķצ×IJ×Ķ +Ġ×ij ק׾ +Ġ×ijק׾ ×ķת +ĠоÑĤ кÑĢ +ĠоÑĤкÑĢ Ñĭва +ãĥij ãĥ¯ãĥ¼ +ë¿IJ ë§Į +Ġв ÑģÑı +ĠвÑģÑı к +ãģ¨ãģª ãģ£ãģ¦ãģĦãĤĭ +Ġgi áºŃn +Ġок ÑĢÑĥ +ĠокÑĢÑĥ жа +ĠокÑĢÑĥжа ÑİÑī +ĠUnivers ität +ĠÑĢ Ð¾Ð¶ +ĠÑĢож д +ĠÑĢожд ениÑı +Ø® ÙĬÙĦ +Ġкомпани й +ĠÑĢазлиÑĩ нÑĭе +ĠЦ ена +׳×Ļ ×ķ×ĸ +׳×Ļ×ķ×ĸ ׾ +׳×Ļ×ķ×ĸ׾ ×ĺר +Ġê³µ ê°Ħ +Ġê°ľ ëħIJ +landır ma +ĠÑĥдал ен +à¸ŀัà¸ģ à¸ľ +à¸ŀัà¸ģà¸ľ à¹Īà¸Ńà¸Ļ +Ġprote cción +Ġb ÅĤ +ĠbÅĤ ÄĻd +Ã Ī +Ġíĸī ë³µ +ĠÅŁ ü +ĠÅŁÃ¼ phe +Ġí Ķ +ĠíĶ ¼ +Ġíͼ íķ´ +Ġëĭ¤ 르 +à¹Ħมà¹Ī à¹Ģà¸ģิà¸Ļ +ãģ¿ ãģª +ãģ¿ãģª ãģķãĤĵ +ĠпоÑĤ ÑĢеб +ĠпоÑĤÑĢеб иÑĤел +ĠاÙĦÙĥÙĦ اÙħ +ìķĦ ë²Ħ +ìķĦë²Ħ ì§Ģ +ãĤĴ使 ãģ£ãģŁ +Ġbụ i +ĠпоÑĤ еÑĢ +ĠпоÑĤеÑĢ Ñı +ĠØ¢ ÙĦاÙģ +ĠнаÑģÑĤоÑıÑī ее +ãģıãģªãĤĬ ãģ¾ãģĹãģŁ +clus ão +ãĤ³ ãĥĶãĥ¼ +צ פ×Ļ +צפ×Ļ ×Ļ×Ķ +Ø® ÙĦا +Ø®ÙĦا ص +ล à¹īำ +ãĥ¯ ãĤ¤ãĥ³ +Ġมี à¸Ļา +Ġมีà¸Ļา à¸Ħม +Ø´ خص +شخص ÙĬات +Ġ×ĸ ×§ +Ġ×ĸ×§ ×ķ×§ +×Ļ ×Ļצ +×Ļ×Ļצ ×Ĵ +èĢĥãģĪ æĸ¹ +Ġürün ü +ĠиÑģп ол +ĠиÑģпол ни +Ġcompañ ero +×§ צ×Ķ +×ŀ×¢ ׳×Ļ×§ +Ùħ ØŃÙħد +Ġc ámara +Ġп ед +Ġпед аг +Ġпедаг ог +м аÑĢ +маÑĢ Ðº +×Ķת ׳×Ĵ×ĵ +ĠìĨĮ ê°ľ +Ġcom unitÃł +ê³ ¤ +ĠNg Ãłi +สà¸ĩ à¸ļ +ĠmieszkaÅĦ ców +ĠÙĨ ÙĩائÙĬ +iv ité +Ġи де +Ġиде алÑĮн +ĠØ£ سبÙĪØ¹ +Ġ×Ļ ×¢×ľ +Ġ׾ ר×IJש +Ġ׾ר×IJש ×ķ׳×Ķ +ĠзапиÑģ и +ĠкоÑĢ Ð¿ÑĥÑģ +วà¸ĩ ศ +วà¸ĩศ à¹Į +ĠÐĶ Ð¼ +ĠÐĶм иÑĤ +ĠÐĶмиÑĤ ÑĢ +Ġkön nt +Ġböl ges +Ġbölges inde +׼ ×Ļ׼ +׼×Ļ׼ ר +ĠاÙĦØ¥ Ø«ÙĨ +ĠاÙĦإثÙĨ ÙĬÙĨ +Ġng á»Ļ +ì¹ ł +د راج +Ġu da +Ġuda ÅĤo +ìº IJ +بر ÙĨاÙħج +ĠÑģÑĥд еб +ĠÑģÑĥдеб н +Ġzun ächst +ĠEduc ación +ãģ¨ãģª ãģ£ãģ¦ãģĦãģ¾ãģĻ +Ġ×Ķ×IJ ×ŀ×Ļת×Ļ +Ġİ nt +Ġİnt ernet +ĠcaÅĤ ego +ãĥĹãĥª ãĥ³ +Ø¥ بد +إبد اع +ĠпоÑĢ ÑĤал +à¹Ĥà¸ķ à¹ī +Ġ×Ķ×§ ש×ķר +пл од +ĠÙħ د +ĠÙħد رÙĬد +×ŀסע ×ĵ×Ķ +ĠØ´ÙĬ ئ +ĠØ´ÙĬئ ا +à¸ģà¹Īà¸Ń สรà¹īาà¸ĩ +Ġì°¸ ê³ł +à¹Ģà¸Ĺ ร +à¹Ģà¸Ĺร à¸Ķ +Ġ×ij×ŀ קר×Ļ×Ŀ +Ġb ât +Ġbât iment +åij¼ ãģ³ +ç´ł æķµ +ç´łæķµ ãģª +przedsiÄĻbior st +przedsiÄĻbiorst w +Ġ×ł×ª ×ķ׳×Ļ×Ŀ +×Ĺ׾ ×ķ×Ŀ +ร วย +Ùħ ÙĪØ¶ÙĪØ¹ +ĠÑģоб ÑĢан +вед ÑĥÑī +ĠÑĤе аÑĤ +ĠÑĤеаÑĤ ÑĢ +m eye +meye ceÄŁi +Ġpien iÄħ +ĠpieniÄħ d +ĠpieniÄħd ze +ÑĢез иденÑĤ +ØŃ صر +ìĺ ¥ +à¹Ģย ืà¸Ńà¸Ļ +ĠÑĥ ни +ĠÑĥни веÑĢ +ĠÑĥнивеÑĢ Ñģ +ĠÑĥнивеÑĢÑģ иÑĤеÑĤ +ĠاÙĦر ØŃ +ĠاÙĦرØŃ ÙħÙĨ +ĠÑĤеÑħ нолог +ĠÑĤеÑħнолог ии +ìĹIJ ëĦĪ +ìĹIJëĦĪ ì§Ģ +Ġíķ Ń +ĠíķŃ ìĥģ +à¸ĺ า +à¸ĺา à¸ķุ +ĠEspañ ol +×ĵ×Ĵ ש +Ġêµ ī +Ġêµī ìŀ¥ +Ġêµīìŀ¥ íŀĪ +ĠÅĤ at +ĠÅĤat wo +Ġk á»ĭch +Ø¥ ز +إز اÙĦØ© +ĠдейÑģÑĤв ие +ĠsaÄŁ layan +สุà¸Ķ ยà¸Ńà¸Ķ +Ġzosta Äĩ +Ġdispon ÃŃvel +ïº į +ver ständ +verständ lich +tw or +twor zyÄĩ +ع جز +à¹Ģà¸Ĥ à¹īม +ยà¹Ī à¸Ńม +Ġstrat ég +Ġstratég ie +à¸ľà¸¥ à¹Ħมà¹ī +Ġê°ģ ì¢ħ +ĠÙħ ÙĪØ§ +ĠÙħÙĪØ§ ض +ĠÙħÙĪØ§Ø¶ ÙĬع +اØŃ تج +اØŃتج اج +Ġ Ấ +ĠẤ n +×ŀ ×ŀש׾×Ķ +ĠÅŁek il +×ŀ ×Ĺ׾ +×ŀ×Ĺ׾ ×ķת +Ġ à¸ĺ +Ġà¸ĺ ัà¸Ļ +Ġà¸ĺัà¸Ļ วา +Ġà¸ĺัà¸Ļวา à¸Ħม +Ġìĭ¤ ìłľ +Ġìĭ¤ìłľ ë¡ľ +ì¤ij ìķĻ +ëįĶ ëĿ¼ +ĠÑĪ Ð¸ÑĢ +ĠÑĪиÑĢ Ð¾ÐºÐ¾ +Ġsol ución +วาà¸ĩ à¹ģà¸ľà¸Ļ +×IJ×ķ×ĺ ×ķ×ŀ +×IJ×ķ×ĺ×ķ×ŀ ×ĺ×Ļ +ĠÑĢ ÐµÑģÑĤ +ĠÑĢеÑģÑĤ оÑĢ +ĠÑĢеÑģÑĤоÑĢ Ð°Ð½ +ëį ¸ +ÑĤ ÑĢад +ÑĤÑĢад и +ÑĤÑĢади ÑĨион +ÑĤÑĢадиÑĨион н +มะ à¹Ģรà¹ĩ +มะà¹Ģรà¹ĩ à¸ĩ +à¹Ĥ ส +Ġol masını +×ŀ×ķס ר +ĠоÑĤноÑĪ ÐµÐ½Ð¸Ð¸ +Ġê°ĢëĬ¥ ìĦ± +Ġy uk +Ġyuk arı +ìĨ Ķ +ĠÑģ ÑĦ +ĠÑģÑĦ еÑĢе +Ġ×§ ×ķפ +ãĤ± ãĥ¼ãĤ +ãĤ±ãĥ¼ãĤ Ń +âĢķ âĢķ +ĠاÙĦØ£ ÙĦÙħ +ĠاÙĦØ£ÙĦÙħ اÙĨÙĬ +Ả N +ת×ķ׼ ׳×Ļ×ķת +ĠÑģÑĥÑīеÑģÑĤв ÑĥеÑĤ +æĪij ãĢħ +ĠاÙĦص ادر +ĠTr á»įng +Ġа д +Ġад миниÑģÑĤ +ĠадминиÑģÑĤ ÑĢа +ĠадминиÑģÑĤÑĢа ÑĨи +ĠдÑĢÑĥг ими +Ñģп еÑĪ +عÙĦاÙħ ات +Ġа б +Ġаб Ñģол +ĠабÑģол ÑİÑĤ +ĠабÑģолÑİÑĤ но +ฤ à¸Ķู +é tr +étr anger +нÑı ÑĤи +нÑıÑĤи е +×¢ ×ķ׳ +×¢×ķ׳ ש +ĠÙĤ ائ +ĠÙĤائ ÙĦا +Ġм аÑģ +ĠмаÑģ ло +ãĥī ãĤ¤ +ãĥīãĤ¤ ãĥĦ +å¿ħè¦ģ ãģĮãģĤãĤĬãģ¾ãģĻ +×ŀ×ķ×ĸ ×Ļ×IJ +×ŀ×ķ×ĸ×Ļ×IJ ×ķף +ĠNgo ại +Ġkê nh +à¸ģาร à¸Ńà¸Ńà¸ģà¹ģà¸ļà¸ļ +×ŀ פק +×ŀפק ×ĵ +ÙħÙĨ از +ÙħÙĨاز ÙĦ +ë· ° +íĹ ¤ +ÙħÙĩ ارات +Ġpropri été +פ×Ĵ ×Ļש×Ķ +Ñĩ ÑĢ +ÑĩÑĢ ÐµÐ¶ +ÑĩÑĢеж ден +×Ķ ×ķצ×IJ×Ķ +ØŃÙĥ ÙĬÙħ +ĠíĻ Ī +ĠíĻĪ íİĺìĿ´ì§Ģ +åİ ³ +åݳ ãģĹãģĦ +×¢ ×ŀ×ĵ×Ķ +ĠAu ÃŁen +سÙĪ Ø¡ +ë¹ Ī +ĠÙĪ Ø® +ĠÙĪØ® اصة +ин ÑĤеÑĢ +инÑĤеÑĢ ÐµÑģ +èĩ´ ãģĹãģ¾ãģĻ +Ġhük üm +à¹Ħà¸Ĥ มัà¸Ļ +Ġdav ran +Ġdavran Ä±ÅŁ +à¹Ģà¸ķ ียà¸ĩ +в ÑĢем +вÑĢем енно +à¹Ģà¸Ĺศ à¸ģา +à¹Ģà¸Ĺศà¸ģา ล +å¼ķ ãģ£ +å¼ķãģ£ è¶ĬãģĹ +×IJר ×ķ×Ĺ +×IJר×ķ×Ĺ ×ª +à¹Ģ วิ +à¹Ģวิ รà¹Į +à¸Ńยà¹Īาà¸ĩ รวà¸Ķà¹Ģรà¹ĩว +ĠìŬ íĸī +ĠÑĢан ÑĮ +ĠÑĢанÑĮ ÑĪе +Ġzob ow +Ġzobow iÄħ +ĠzobowiÄħ z +Ġ×ķ׼ ×ŀ×ķ×ijף +ĠاÙĦÙħ Ùĩ +ĠاÙĦÙħÙĩ ÙĨÙĬ +ãĤ¢ ãĤ¸ +ãĤ¢ãĤ¸ ãĤ¢ +ë°© ìĨ¡ +à¸Ńà¸Ńà¸ģ à¸ģำลัà¸ĩ +à¸Ńà¸Ńà¸ģà¸ģำลัà¸ĩ à¸ģาย +am éli +améli orer +å½ĵãģŁãĤĬ åīį +Ġreg elm +Ġregelm Ã¤ÃŁig +ãģĬ åĭ +ãģĬåĭ § +ãģĬåĭ§ ãĤģ +Ġm ưá»Ŀi +بر Ùħج +ĠNat ürlich +ĠD Å©ng +ĠاÙĦر جاÙĦ +Ġthé p +Ġol muÅŁtur +×ŀ×ķס ×Ļ×§×Ķ +f älle +주 íĥĿ +ĠاÙĦÙģ Ø±Øµ +Ġnaj wiÄĻks +ĠnajwiÄĻks zy +Ġça ÄŁ +ĠçaÄŁ rı +ì¸ ł +ĠvÃŃ ct +ĠvÃŃct ima +ĠÑģовеÑĢ ÑĪен +×Ķ×Ļ ×Ļת×Ļ +à¹Ģà¸Ķ ี +à¹Ģà¸Ķี à¹ĭ +à¹Ģà¸Ķีà¹ĭ ยว +ü yü +Ġд оп +Ġдоп олн +Ġдополн иÑĤелÑĮно +à¹ģà¸ķà¸ģà¸ķà¹Īาà¸ĩ à¸ģัà¸Ļ +Ġá l +Ġál bum +à¸Ľà¸£à¸°à¸Īำ à¸Ľà¸µ +ĠÑĦ едеÑĢ +ĠÑĦедеÑĢ Ð°Ð»ÑĮн +Ġobs ÅĤ +ĠobsÅĤ ugi +à¹Ģร ืà¹Ī +à¹Ģรืà¹Ī à¸Ńย +à¹Ģรืà¹Īà¸Ńย à¹Ĩ +ëģ Į +Ġngh ìn +ĠBaÅŁkan lıģı +تأ سÙĬ +تأسÙĬ س +Ġ×ij×ij ×ķקר +Ġ×¢×ij×ķ×ĵ ×ķת +Ġبص ÙĪØ±Ø© +ãĤıãģij ãģ§ãģ¯ãģªãģĦ +führ er +ãĤ¹ ãĤŃ +ãĤ¹ãĤŃ ãĥ« +ĠاÙĦÙĤ ض +ĠاÙĦÙĤض ÙĬØ© +Ġдолж ноÑģÑĤ +ÙģØ§Ø± ÙĤ +Ġcomeç ou +Ġorganis é +Ġxu ân +ĠÑģообÑī аеÑĤ +ĠпÑĢи д +ĠпÑĢид еÑĤÑģÑı +TÃľ RK +ãĥ¬ ãĥ¼ãĤ·ãĥ§ãĥ³ +Kh ông +است Ùģ +استÙģ Ø§Ø¯Ø© +ä¸ĬãģĮ ãģ£ãģ¦ +Ġum ie +Ġumie jÄĻ +ĠumiejÄĻ tn +ĠumiejÄĻtn oÅĽci +ëĤ ¸ +à¹Ģà¸Ļ à¸Ńรà¹Į +×ĵ×ķ ×ķ×Ĺ +ÃŃs imo +I ÃĬ +IÃĬ N +Ġalcan ç +Ġ à¸ķุ +Ġà¸ķุ ลา +Ġà¸ķุลา à¸Ħม +ש׾ ×ĺ×ķף +Ġél è +Ġélè ves +ĠÄij u +ĠÄiju á»ķi +ĠØ£ Ùģ +ĠØ£Ùģ Ø±ÙĬ +ĠØ£Ù쨱ÙĬ ÙĤÙĬ +ĠØ£Ù쨱ÙĬÙĤÙĬ ا +ãĤĴæİ¢ ãģĻ +ĠпÑĢед ложениÑı +ج اد +ĠÑħоÑĤ ÑĮ +Ñģ ал +Ñģал он +à¸Ľà¸£à¸° à¹Ģม +à¸Ľà¸£à¸°à¹Ģม ิà¸Ļ +ãĤŃ ãĥĥãĥģ +ãĤŃãĥĥãĥģ ãĥ³ +×ij×ĵ×Ļ×§ ×ķת +Ġch ù +Ġchù a +ÐĴ иде +ÐĴиде о +иÑĢов ка +ĠÑħоÑĤ иÑĤе +Ġspéc ifique +รส à¸Ĭาà¸ķิ +è¾¼ ãĤĵãģł +伸 ãģ³ +×Ķצ׾ ×Ĺת +ãģ©ãģ® ãĤĪãģĨãģ« +سع ادة +Ġл ид +Ġлид еÑĢ +ม à¸ĩ +มà¸ĩ à¸Ħล +ØŃ اÙħÙĦ +หล ุà¸Ķ +à¸Ńยà¹Īาà¸ĩ à¸ķà¹Īà¸Ń +à¸Ńยà¹Īาà¸ĩà¸ķà¹Īà¸Ń à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ +ãģķãģĽãģ¦ éłĤ +تس ÙĪÙĬ +تسÙĪÙĬ ÙĤ +ĠaÅŁaģı d +ĠaÅŁaģıd aki +ĠÑĨ елÑĮ +ĠÑĨелÑĮ Ñİ +ĠAra ÅŁtırma +à¸Ĥัà¸ļ รà¸ĸ +Ùĩ ذÙĩ +ลà¸ĩ à¸Ĺะ +ลà¸ĩà¸Ĺะ à¹Ģà¸ļ +ลà¸ĩà¸Ĺะà¹Ģà¸ļ ียà¸Ļ +تÙĥ اÙħÙĦ +Ġc io +Ġcio è +ãģ¦ ãģĬãģı +ĠاÙĦصØŃ ÙģÙĬ +ĠíĬ¹ ìłķ +полн иÑĤÑĮ +ãĤĵ ãģĺãĤĥãģªãģĦ +ãĤĵãģĺãĤĥãģªãģĦ ãģĭ +ĠاÙĦج Ùĩ +ĠاÙĦجÙĩ ات +ĠÑĥÑģпеÑĪ Ð½Ð¾ +Ġв ок +Ġвок ÑĢÑĥг +ĠÑģиÑĤÑĥ аÑĨиÑı +Ġ×Ķ×IJ ×ŀר +Ġ×Ķ×IJ×ŀר ×Ļ×§ +Ġ×Ķ×IJ×ŀר×Ļ×§ ×IJ×Ļ +×ŀ ×Ĵ×ĸ +×ŀ×Ĵ×ĸ ×Ļף +Ġак ÑĤÑĥ +ĠакÑĤÑĥ алÑĮн +é ta +éta is +Ġmog ÅĤa +ĠÑĤоÑĩ ки +Ġ×ŀ×Ķ ×ŀ×¢ +Ġ×ŀ×Ķ×ŀ×¢ ×¨×Ľ×ª +มี à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺà¸´à¸łà¸²à¸ŀ +×Ļר ×Ļ×ĵ×Ķ +×Ĵר ×ŀ׳ +×Ĵר×ŀ׳ ×Ļ×Ķ +Ġг лав +Ġглав ное +Ġ미 ëŀĺ +Ġ׳׼ ×ķ׳×Ķ +ĠÙĪ Ø·ÙĨÙĬ +op port +opport unitÃł +Ġh á»§y +ĠÙĦ تØŃ +ĠÙĦتØŃ ÙĤÙĬÙĤ +Ġó rg +Ġórg ão +ãĤ¹ ãĥĶ +ãĤ¹ãĥĶ ãĥ¼ãĥī +Ġön ü +Ġönü ne +Ùħع اÙħÙĦ +ש×ŀ ×Ļר×Ķ +ĠвеÑģÑĮ ма +ĠwiÄĻks zo +ĠwiÄĻkszo ÅĽÄĩ +Ġاست راتÙĬج +ĠاستراتÙĬج ÙĬØ© +ĠÙģ Ø¥ +ĠÙ쨥 ذا +à¹Ģà¸Ĭืà¹Īà¸Ń ม +à¹Ģà¸Ĭืà¹Īà¸Ńม à¸ķà¹Īà¸Ń +Ġ׾ פר +Ġ׾פר ×ĺ×Ļ×Ŀ +Ùħض ÙĬ +ĠGer çek +Ġçocuk ların +ÙĪØ« ائÙĤ +ĠÙħساء Ùĭ +Ġunterstüt zt +Ġpré st +Ġprést amo +ĠÐłÐ°Ð· меÑĢ +ĠÅŁ eker +Ġsé culo +×ij×Ķ ×Ļר +Ø´Ùĩ ÙĪØ± +Ġ à¸Ńีà¸ģ +Ġà¸Ńีà¸ģ à¸Ĺัà¹īà¸ĩ +Ġlleg ó +à¸¨à¸´à¸¥à¸Ľ ะ +æĪij ãģĮ +æĪijãģĮ å®¶ +ع ÙĤÙĪ +عÙĤÙĪ Ø¨Ø§Øª +ĠF älle +Ġs ÅĤuż +ĠsÅĤuż b +ĠاÙĦØŃÙĤ ÙĪÙĤ +Ġпл иÑĤ +Ġи ноÑģÑĤ +ĠиноÑģÑĤ ÑĢан +ĠиноÑģÑĤÑĢан н +à¹ĥà¸Ļ à¸Ĥà¸ĵะà¸Ĺีà¹Ī +ãĤ« ãĥĨ +ãĤ«ãĥĨ ãĤ´ +ãĤ«ãĥĨãĤ´ ãĥª +à¸Ńิ ส +à¸Ńิส ระ +à¹Ģà¸ľà¸¢ à¹ģ +à¹Ģà¸ľà¸¢à¹ģ à¸ŀร +à¹Ģà¸ľà¸¢à¹ģà¸ŀร à¹Ī +ãģĬ ãģĦ +ãģĬãģĦ ãģĹãģĦ +است ÙĤÙĦ +استÙĤÙĦ اÙĦ +تØŃ ض +تØŃض ÙĬر +åĬ© ãģij +Ùħر اÙģÙĤ +Ġ×ĵ ×ķר +Ġ×ĵ×ķר ש +×ŀת×Ļ ×Ļ×Ĺס +ס ×Ļ׼ +ס×Ļ׼ ×ķ×Ŀ +íĮĮ íĬ¸ +Ġwy ÅĽ +ĠwyÅĽ w +ĠwyÅĽw iet +ĠwyÅĽwiet l +ĠاÙĦاÙĨ ساÙĨ +ĠStra ÃŁen +ï¼ ¬ +ãģ« åŁº +ãģ«åŁº ãģ¥ +Ġcap ÃŃtulo +ลุ ย +Ġ×Ķ×ŀ×§ צ×ķ×¢×Ļ +ãģĤãĤĭ ç¨ĭ度 +á» ¢ +ĠاÙĦ ÙĦا +ĠاÙĦÙĦا زÙħØ© +æķĻ ãģĪ +Ġרש ×IJ×Ļ +з ав +зав иÑģ +завиÑģ им +à¸Ľà¸±à¸Ī à¸Īัย +à¹Ģà¸ĭ ล +à¹Ģà¸ĭล ลà¹Į +Ġdiffé rence +ĠAlt ın +Ġк ÑĢай +ĠкÑĢай не +Ġз ло +Ġgün ümüz +Ġн аÑĤÑĥÑĢ +ĠнаÑĤÑĥÑĢ Ð°Ð»ÑĮн +×Ĵ×ķ׾ ש×Ļ×Ŀ +Ġк аÑĤегоÑĢ +ĠкаÑĤегоÑĢ Ð¸Ð¸ +Ġз нак +à¸ģà¹Īà¸Ńà¸Ļ หà¸Ļà¹īา +à¸ģà¹Īà¸Ńà¸Ļหà¸Ļà¹īา à¸Ļีà¹ī +ĠÙħÙĨ ت +ĠÙħÙĨت خب +ãĥĽ ãĥ¼ãĥ« +Ġе вÑĢо +ส ว +สว ม +ĠìľĦ ìĽIJ +ĠìľĦìĽIJ ëĭĺ +ĠاÙĦØŃ ÙĪØ« +ĠاÙĦØŃÙĪØ« ÙĬ +ĠÑģодеÑĢж иÑĤ +ãĥķãĤ¡ ãĥĥãĤ·ãĥ§ãĥ³ +Ġ à¸ģัà¸Ļ +Ġà¸ģัà¸Ļ ย +Ġà¸ģัà¸Ļย ายà¸Ļ +ãĤª ãĥª +ãĤªãĥª ãĤ¸ +ãĤªãĥªãĤ¸ ãĥĬãĥ« +Ġб ÑĢенд +ãĤĴæĮģ ãģ£ãģ¦ãģĦãĤĭ +Ġinvers ión +Ġê° ĸ +Ġê°ĸ ê³ł +Ġnov itÃł +ê´Ģ ê´ij +Ġà¸ŀ ฤษ +Ġà¸ŀฤษ à¸łà¸² +Ġà¸ŀà¸¤à¸©à¸łà¸² à¸Ħม +×ķר ×Ĺ×Ļ×Ŀ +׼׾ ×ķ׾ +Ġng ạc +×Ļ ×Ļש +×Ļ×Ļש ×ķ×ij +f äll +fäll ig +ĠÑĤÑĢеб ÑĥеÑĤÑģÑı +Ġcar á +Ġcará cter +Ġprinc ÃŃpio +ĠÅĤ az +ĠÅĤaz ien +ĠÅĤazien k +Ġgi ãn +ÑģÑĤÑĢа ива +Ùħس اب +Ùħساب ÙĤØ© +à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ à¸Ķืà¹Īม +ترÙĥ ÙĬب +vol ução +ĠÐŁ оÑĩ +ĠÐŁÐ¾Ñĩ ем +ĠÐŁÐ¾Ñĩем Ñĥ +казал оÑģÑĮ +ĠпÑĢимен ениÑı +à¹Ģà¸Ĺ ียม +íĮ Ķ +à¸Ĥà¹īà¸Ń à¹Ģสà¸Ļà¸Ń +à¸Ľà¸±à¸į à¸įา +Ġоб ÑĥÑĩ +ĠобÑĥÑĩ ениÑı +ĠÑģеÑĢ Ð¸ +ĠÑģеÑĢи ал +Ġingl és +ĠÙĦ Ùĥرة +Ġ×ĺ ׾ +Ġ×ĺ׾ פ×ķף +Ġìł ij +Ġìłij ê·¼ +×IJ ×ķ×Ĵ +×IJ×ķ×Ĵ ×ķס +×IJ×ķ×Ĵ×ķס ×ĺ +ĠболÑĮÑĪ Ð¾Ðµ +ĠÐļон еÑĩно +×¢×Ļת ×ķ׳ +×¢×Ļת×ķ׳ ×IJ×Ļ +Ġкноп к +Ġз н +Ġзн аÑĤÑĮ +ĠÄij á»± +ĠÄijá»± ng +вл аж +влаж н +×ŀ ×Ļ×ĺ×ij +ãĤ¬ ãĤ¤ +ãĤ¬ãĤ¤ ãĥī +........ .. +Ġà¸ģ ุม +Ġà¸ģุม à¸łà¸²à¸ŀ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀ ัà¸Ļ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļ à¸ĺ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļà¸ĺ à¹Į +be z +bez pieczeÅĦst +bezpieczeÅĦst w +ãĥijãĥij æ´» +ع اط +عاط Ùģ +ĠÄij áºŃm +Ġз ÑĢ +ĠзÑĢ ÐµÐ½Ð¸Ñı +Ġbor ç +Ġнед ел +Ġнедел Ñİ +Ġh á»ı +Ġhá»ı ng +ìŀ¥ ìķł +ìŀ¥ìķł ìĿ¸ +ĠاÙĦع ÙĦاÙĤØ© +Ġíģ ¬ +Ġíģ¬ ê²Į +à¹Ħร à¹Ī +à¸ļา à¸Ķ +à¸ļาà¸Ķ à¹Ģà¸Īà¹ĩà¸ļ +à¸Ŀ รั +à¸Ŀรั à¹Īà¸ĩ +à¸Ŀรัà¹Īà¸ĩ à¹Ģศ +à¸Ŀรัà¹Īà¸ĩà¹Ģศ ส +ר ×¢×Ļ +רע×Ļ ×ķ׳×ķת +Ġë Į +ĠëĮ ĵ +ĠëĮĵ ê¸Ģ +Ġnaj b +Ġnajb li +Ġnajbli ż +Ġnajbliż sz +ĠиÑģполÑĮз ÑĥеÑĤÑģÑı +Ġcient ÃŃf +ĠcientÃŃf ico +×¢ ×ŀ×§ +Ġg ợi +Ø´ ØŃÙĨ +ĠÅĽ m +ĠÅĽm ier +ĠÅĽmier ci +à¸Ħาสิà¹Ĥà¸Ļ à¸Ńà¸Ńà¸Ļà¹Ħลà¸Ļà¹Į +×Ĺש×ij ת×Ļ +Ġn ingu +Ġningu ém +è¾¼ ãĤģ +ãģ · +ĠÑĥ г +ĠÑĥг ол +ï½ ° +פת ×Ļ×Ĺ +פת×Ļ×Ĺ ×ª +Ġ×Ķר×IJש ×ķ׳×Ļ×Ŀ +p ósito +ãĤŃ ãĥ¬ãĤ¤ +ãģ© ãģĵãĤį +à¹Ģà¸Ĺà¹Īา à¹Ħ +à¹Ģà¸Ĺà¹Īาà¹Ħ หร +à¹Ģà¸Ĺà¹Īาà¹Ħหร à¹Ī +ĠинÑĤеÑĢ ÑĮеÑĢ +ĠØŃ اج +ĠØŃاج Ø© +สี à¸Ĥาว +ìĸ ¼ +Ġn á»Ļ +Ġná»Ļ p +ĠÃŃ nd +ĠÃŃnd ice +สำ รวà¸Ī +Ġкажд ой +Ġhot éis +Ġnast ÄĻ +ĠnastÄĻ pn +Ġ×Ķ×§ ×ķ×ĵ +Ġ×Ķ×§×ķ×ĵ ×Ŀ +פ ×ķפ +פ×ķפ ×ķ׾ +פ×ķפ×ķ׾ ר×Ļ +вÑĪ ÐµÐ¹ +ãĤ·ãĥ³ ãĥĹ +ãĤ·ãĥ³ãĥĹ ãĥ« +ĠzdjÄĻ Äĩ +ĠгÑĢÑĥпп а +Ġпом еÑī +ĠпомеÑī ениÑı +ãģ©ãģĨ ãģĦãģĨ +ĠиÑģп ÑĭÑĤа +Ġog ÅĤ +ĠogÅĤ os +ĠogÅĤos zen +ĠogÅĤoszen i +สรà¹īาà¸ĩ สรร +สรà¹īาà¸ĩสรร à¸Ħà¹Į +à¸ŀร รà¸ĵ +Ġçık Ä±ÅŁ +ĠÑĩаÑģÑĤ ноÑģÑĤи +Ġ×ķ ×Ļ×ķתר +ç¶ļãģį ãĤĴ +ç¶ļãģįãĤĴ èªŃ +ç¶ļãģįãĤĴèªŃ ãĤĢ +à¸ģร ั +à¸ģรั ม +г ÑĢаÑĦ +Ġв лад +Ġвлад елÑĮ +ĠвладелÑĮ ÑĨ +Ġistedi ÄŁ +ĠistediÄŁ iniz +×ij׾ ×¢ +×ij×ľ×¢ ×ĵ×Ļ +ÙħÙĪ Ø§Ùģ +ÙħÙĪØ§Ùģ ÙĤØ© +Ġ×Ļ ×ķר +Ġ×Ļ×ķר ×§ +ãĤ«ãĥ¼ãĥī ãĥŃãĥ¼ãĥ³ +ĠاÙĦÙħØ´ ÙĥÙĦ +ĠاÙĦÙħØ´ÙĥÙĦ Ø© +ĠêµŃ íļĮ +ס פ×ĺ +ספ×ĺ ×ŀ +ספ×ĺ×ŀ ×ijר +Ġìĸ´ ëłµ +Ùĥ اÙħ +ÙĥاÙħ ÙĬرا +sch lü +schlü sse +ĠØ« ÙĨ +ĠØ«ÙĨ ائÙĬ +ìī ½ +ĠÐŀ Ñģоб +ĠÐŀÑģоб енно +Ġин веÑģÑĤи +ĠинвеÑģÑĤи ÑĨи +اØŃ تÙħ +اØŃتÙħ اÙĦ +E Äŀ +EÄŀ İ +íķĺ ê²łëĭ¤ +Ġ×IJ ×ijר×Ķ +Ġ×IJ×ijר×Ķ ×Ŀ +Ġ×ij×Ĺ ×Ļ׳×Ŀ +Ø£ ÙĪØ¶ +Ø£ÙĪØ¶ اع +Ġdé l +Ġdél ai +Ġ×IJ×ķ×Ķ ×ij×Ļ×Ŀ +ĠÑģо Ñħ +ĠÑģоÑħ ÑĢ +ĠÑģоÑħÑĢ Ð°Ð½Ð¸ +ĠдоÑģÑĤ иж +ĠдоÑģÑĤиж ени +สิà¹Īà¸ĩ à¹ģ +สิà¹Īà¸ĩà¹ģ วà¸Ķ +สิà¹Īà¸ĩà¹ģวà¸Ķ ล +สิà¹Īà¸ĩà¹ģวà¸Ķล à¹īà¸Ńม +ĠاÙĦÙħ باشر +ĠÑĦ иг +ĠÑĦиг ÑĥÑĢ +мож ем +׾×ŀ×Ļ×ĵ ×Ķ +Ġcin é +Ġciné ma +Ġb ada +Ġbada ÅĦ +جب ÙĩØ© +Ġд еп +Ġдеп ÑĥÑĤ +ĠдепÑĥÑĤ аÑĤ +Ġdist ância +ĠاÙĦÙħ عار +ĠاÙĦÙħعار ضة +thè se +ü nc +ünc ü +Ġдан ного +ĠBel gi +ĠBelgi ë +Ġ×ij ×ij×§ +Ġ×ij×ij×§ ש×Ķ +ย à¹Īาà¸Ļ +Ġsol ução +Ġ×Ķצ ×ĺר +Ġ×Ķצ×ĺר פ×ķ +ĠØ£ÙĨ ØŃ +ĠØ£ÙĨØŃ اء +Ġد ÙħØ´ +ĠدÙħØ´ ÙĤ +มั à¹ī +มัà¹ī ย +Ùħ غرب +است عÙħاÙĦ +ĠS ÅĤow +ĠëıĻ ìĭľ +ĠëıĻìĭľ ìĹIJ +ĠÑģ оÑģ +ĠÑģоÑģ ед +ì²Ń ìĨĮ +ì²ŃìĨĮ ëħĦ +Ġг ÑĢаÑĦ +ĠгÑĢаÑĦ ик +Ġìŀij ìĿĢ +Ġyet i +Ġyeti ÅŁtir +ĠìĿ´ê²ĥ ìĿ´ +ห à¹Īาà¸ĩ +Ø¥ ÙħÙĥاÙĨ +Ø¥ÙħÙĥاÙĨ ÙĬØ© +است عراض +ÙħØ® در +ĠÑĩ ÑĥÑĤÑĮ +Ùħ دÙĬر +ÙħدÙĬر ÙĬØ© +Ġà¹Ģม ษ +Ġà¹Ģมษ ายà¸Ļ +Ġм еÑħ +ĠмеÑħ аниз +ĠмеÑħаниз м +ĠÑģ Ñĥм +ĠÑģÑĥм мÑĥ +Ġv ö +Ġvö ll +Ġvöll ig +Ġд ÑĢÑĥз +ĠдÑĢÑĥз ÑĮÑı +ãĤĴåĪ©ç͍ ãģĹãģ¦ +à¸ļรร à¸Īุ +po życz +×ŀש ׼ +×ŀש׼ ×ł×ª +×ŀ×©×Ľ×ł×ª ×IJ +Ġeuropé en +Ġpropri é +Ġproprié taire +Ġkh ấu +ãģĦãģŁãģł ãģijãĤĭ +Ġtec rü +Ġtecrü be +×Ķ ×ij +×Ķ×ij ׳×Ķ +Ġcu Ì +ĠcuÌ ī +ĠcuÌī a +×IJ ×ķ×ķ +×IJ×ķ×ķ ×Ļר×Ķ +Ġ׼×ķ׾ ×ķ +U lus +Ulus lararası +Ġ׳ ×ķת +Ġ׳×ķת ף +ãģ« åIJij +ãģ«åIJij ãģijãģ¦ +ë¹ Ľ +à¸Ĺ ัà¸ģษ +à¸Ĺัà¸ģษ ะ +س ÙĤÙĪ +سÙĤÙĪ Ø· +Ġв н +Ġвн еÑĪ +ĠвнеÑĪ Ð½Ðµ +Ġur z +Ġurz ÄĻd +Ġá mb +Ġámb ito +à¸Ń à¸ĺิ +à¸Ńà¸ĺิ à¸ļาย +Ġ ÅĤad +ĠÅĤad n +ê±´ ì¶ķ +wód zt +wództ w +Ġquest ões +Ġש ×§ +Ġשק ×Ļ×ij׾ +Ġmiejsc owoÅĽci +Ġв ал +Ġвал ÑİÑĤ +hä user +หà¸Ļ à¸Ńà¸ĩ +ãģ¨ åħ± +ãģ¨åħ± ãģ« +ãĥı ãĥ¼ãĥī +Ġê°ľ ìµľ +ĠоÑģнов ном +Ġм ÑıÑģ +اع ت +اعت ÙĤاÙĦ +สà¸ĸ ิ +สà¸ĸิ à¸ķิ +N gu +Ngu á»ĵn +ĠÙħ جÙĦ +ĠÙħجÙĦ Ø© +à¹ģà¸Ĥ à¸Ļ +ĠاÙĦÙĦÙĬ بÙĬ +פע×Ļ׾ ×ķ×Ļ×ķת +Ġ×Ķר פ×ķ×IJ×Ļ +פר ×ķפ +פר×ķפ ×Ļ׾ +×§ ׾×IJ +ק׾×IJ ס×Ļ +Ùĥت Ø´Ùģ +ãģ«ãģª ãģ£ãģ¦ãģĹãģ¾ãģĨ +à¹Ģà¸Ħล à¹ĩà¸Ķ +à¹Ģà¸Ħลà¹ĩà¸Ķ ลัà¸ļ +Ġì» ´ +Ġì»´ íĵ¨ +Ġì»´íĵ¨ íĦ° +Ġ×Ĺ×Ļ ×ķ×ij×Ļ +Ġnä m +Ġnäm lich +åij¼ ãģ° +åij¼ãģ° ãĤĮ +ĠÑĢ Ð¾Ð» +ĠÑĢол и +Ġspécial isé +à¸Ļ วัà¸ķ +à¸Ļวัà¸ķ à¸ģรรม +ÙĨص ÙĪØµ +пеÑĢ ÐµÐ´ +пеÑĢед аÑĩ +thè que +Ġר×IJ ×Ļת×Ļ +ãĥĢ ãĤ¦ãĥ³ +ãĤı ãģĭ +ãĤıãģĭ ãģ£ãģ¦ +беÑĢ ÐµÐ¶ +ĠÑģ ек +ĠÑģек ÑĢ +ĠÑģекÑĢ ÐµÑĤ +ĠпоÑģÑĤоÑıн н +à¸Ĥà¸Ļ สà¹Īà¸ĩ +Ġm ük +Ġmük em +Ġmükem mel +еÑĤ еÑģÑĮ +ĠاÙĦسÙĨ ÙĪØ§Øª +ĠìłĦ íĺĢ +Ġ×Ķ×ŀ×§ ×ķר×Ļ +Ġmü d +Ġmüd ah +Ġmüdah ale +Ġwy b +Ġwyb ór +Ġtend ência +Ø¥ دار +إدار ÙĬØ© +Ġunterstüt zen +ת ×ijר +ת×ijר ר +Ġdi á +Ġdiá logo +ĠÃĸ nce +ĠÃĸnce ki +ãĤ¹ãĥĿ ãĥĥãĥĪ +ëĦ £ +ĠG eli +ĠGeli ÅŁ +ãĤĴ éĢļ +ãĤĴéĢļ ãģĹãģ¦ +ĠFuÃŁ ball +Ġsal ari +Ġsalari é +ĠпÑĢодÑĥк ÑĤов +صÙģ ÙĤØ© +รว à¸ļ +รวà¸ļ รวม +à¹ĥà¸Ļ à¸IJาà¸Ļ +à¹ĥà¸Ļà¸IJาà¸Ļ ะ +Ġkay na +Ġkayna ģı +Ġìŀij íĴĪ +ĠвÑĭ ÑĢаж +ĠвÑĭÑĢаж ен +ĠÑģÑĤ еп +ĠÑģÑĤеп ени +ĠاÙĦÙħ ÙĪØ¬ÙĪØ¯ +ĠاÙĦÙħÙĪØ¬ÙĪØ¯ Ø© +ล à¹īม +Ġnaj czÄĻ +ĠnajczÄĻ ÅĽcie +ĠnajczÄĻÅĽcie j +Ġz wy +Ġzwy k +Ġzwyk ÅĤ +Ġê·¸ëłĩ ì§Ģ +à¸ģระ à¸Ī +à¸ģระà¸Ī าย +Ġëĭ µ +Ġëĭµ ë³Ģ +ĠÑĢе ак +ĠÑĢеак ÑĨи +ĠÅĽwie ż +ĠÑģÑĤоим оÑģÑĤи +ÙħÙĨ اÙĤ +ÙħÙĨاÙĤ Ø´ +ÙħÙĨاÙĤØ´ Ø© +ĠÑħоÑĩ Ñĥ +ãĥľ ãĥ¼ãĥī +Ġróż nic +Ġк ÑĢÑĭ +ĠкÑĢÑĭ ÑĪ +âľ ĵ +ãĤ³ãĥ³ ãĥĨãĥ³ +ãĤ³ãĥ³ãĥĨãĥ³ ãĥĦ +ĠпÑĢед поÑĩ +×ŀר ×ij×Ļת +ĠØ´ Ùĥ +ĠØ´Ùĥ را +Ġд ал +Ġдал ек +Ġдалек о +بر ÙĬØ· +برÙĬØ· اÙĨÙĬا +ع ÙĨا +عÙĨا ÙĬØ© +ĠÑĢаÑģÑģ каз +ĠÑĢаÑģÑģказ Ñĭва +Ø£ ÙĦÙĪ +Ø£ÙĦÙĪ Ø§ÙĨ +æĮģ ãģ£ãģ¦ +æĮģãģ£ãģ¦ ãģĦ +Ùħباد ئ +×Ķ ×¢×ijר +×Ķ×¢×ijר ת +Ġyay ı +Ġyayı ml +Ġyayıml a +m át +mát icos +à¸ģ ัà¸ĩ +à¸ģัà¸ĩ วล +Ġ׾ פת +Ġ×ľ×¤×ª ×ķ×Ĺ +à¸ŀฤ à¸ķิ +à¸ŀฤà¸ķิ à¸ģรรม +í Ĥ¬ +Ġок ÑĢÑĥг +Ġ×ŀצ ×ķ×ķ×Ķ +ÐĽ ени +ÐĽÐµÐ½Ð¸ н +ĠTri á»ģu +ãĤ³ãĥŁ ãĥ¥ +ãĤ³ãĥŁãĥ¥ ãĥĭ +ãĤ³ãĥŁãĥ¥ãĥĭ ãĤ± +ãĤ³ãĥŁãĥ¥ãĥĭãĤ± ãĥ¼ãĤ·ãĥ§ãĥ³ +Ùĥ ÙĨÙĬ +ÙĥÙĨÙĬ سة +ãĤĴ ä¸Ńå¿ĥ +ãĤĴä¸Ńå¿ĥ ãģ« +ĠmiÄĻd z +ĠmiÄĻdz yn +ĠmiÄĻdzyn ar +ĠmiÄĻdzynar od +ĠmiÄĻdzynarod ow +ÙĦ ÙĨ +ÙĦÙĨ دا +بر Ø´ +برش ÙĦÙĪÙĨ +برشÙĦÙĪÙĨ Ø© +à¸ģระ à¸ķุ +à¸ģระà¸ķุ à¹īà¸Ļ +Ġg ı +Ġgı da +à¸Ľà¸£à¸° à¸Ĺัà¸ļ +à¸Ľà¸£à¸°à¸Ĺัà¸ļ à¹ĥà¸Ī +Ġë¶Ī 구 +Ġë¶Ī구 íķĺê³ł +ĠÙĨ Ø· +ĠÙĨØ· اÙĤ +ĠÐľ ожеÑĤ +Pr äs +Präs ident +ĠÑģк оÑĢ +ĠÑģкоÑĢ Ð¾ÑģÑĤÑĮ +Ġ×Ķ×ij ×ķקר +еÑħ аÑĤÑĮ +Ġg ạo +Ġש×IJ ×Ļ׳×Ŀ +Ġ×ij׳ ×ķ×Ĵ +Ġ×ij׳×ķ×Ĵ ×¢ +Ġо пиÑģание +Ġucz ni +Ġuczni ów +à¹Ģà¸Ń à¹ĩà¸Ļ +Ġت Ø´ +Ġتش رÙĬÙĨ +Ġnh ãn +ë¹ ¨ +Ġcaract ère +×¢ ׾×Ļ +×¢×ľ×Ļ ×Ļ×Ķ +楽ãģĹ ãĤģãĤĭ +ĠÑģ аÑħ +ĠÑģаÑħ аÑĢ +дÑĥм аÑĤÑĮ +ĠÐĴоз можно +ص ÙĬاÙĨ +صÙĬاÙĨ Ø© +öm ür +ส ล +สล à¹ĩ +สลà¹ĩ à¸Ń +สลà¹ĩà¸Ń à¸ķ +ë¡ ¯ +Ġth ói +gr Ã¶ÃŁe +Ġksi ÄĻ +ĠksiÄĻ g +ĠÑĢ Ð¾Ð¼ +ĠÑĢом ан +ÙĤ اسÙħ +×ŀ×ij ×ķ×Ĵ +×ŀ×ij×ķ×Ĵ ר×Ļ×Ŀ +bes ch +besch äft +beschäft ig +×Ķצע ×Ķ +ĠÃģ rea +ĠзаÑıв к +Ä ¹ +ĠлÑİб ого +Ġ ม +Ġม à¸ģร +Ġมà¸ģร าà¸Ħม +ÑĦ из +ÑĦиз иÑĩеÑģк +ин ÑĦ +инÑĦ ек +инÑĦек ÑĨи +اÙĦ Ø· +اÙĦØ· ائÙģ +Ġкол л +Ġколл екÑĤив +ез жа +Ġس بØŃ +ĠسبØŃ اÙĨ +ĠسبØŃاÙĨ Ùĩ +sch lä +schlä ge +Ġд и +Ġди аг +Ġдиаг ноÑģÑĤ +ĠоÑĤмеÑĤ иÑĤÑĮ +Т Ь +ĠاÙĦ در +ĠاÙĦدر اسÙĬ +עצ ×ŀ +עצ×ŀ ×IJ×ķת +Ġdém arch +Ġdémarch e +Ġ×ĺ ×ķ×¢ +Ġ×ĺ×ķ×¢ ף +Ġfuncion ários +á» µ +׾ ׼×IJ +׾׼×IJ ×ķר×Ķ +à¸ĭ à¹Ī +à¸ĭà¹Ī à¸Ńม +ĠÑĩ Ñĥв +ĠÑĩÑĥв ÑģÑĤво +âĸ ¼ +п ÑĥÑī +пÑĥÑī ен +Ġм еÑĢ +ĠмеÑĢ Ð¾Ð¿ +ĠмеÑĢоп ÑĢи +ĠмеÑĢопÑĢи ÑıÑĤиÑı +Ġu çu +Ġuçu ÅŁ +ãĤĴåĪ©ç͍ ãģĻãĤĭ +a ÄŁ +aÄŁ lı +ìĺĪ ìĪł +à¹ģ ยà¹Ī +ĠاÙĦÙĥ Ùħ +ĠاÙĦÙĥÙħ بÙĬ +ĠاÙĦÙĥÙħبÙĬ ÙĪØªØ± +ت ÙĪÙĬ +تÙĪÙĬ تر +à¹Ģà¸Ĭ ีà¹Īยว +à¹Ģà¸Ĭีà¹Īยว à¸Ĭา +à¹Ģà¸Ĭีà¹Īยวà¸Ĭา à¸į +á» Ķ +Ġhi ếm +ذا Ùĥرة +Ġ×Ķ×ŀ×Ļ ×ķ×Ĺ×ĵ +ĠìĪ ľ +ĠìĪľ ê°Ħ +ĠK ı +ĠKı sa +Ġgele ceÄŁi +пÑĢо ÑĦеÑģÑģиона +пÑĢоÑĦеÑģÑģиона л +Ġog ó +Ġogó le +ĠgÅĤ ów +ĠgÅĤów ne +ĠÑģÑĤ илÑĮ +×IJ פ׾ +×IJפ׾ ×Ļ×§ +×IJפ׾×Ļ×§ צ×Ļ×Ķ +สม ารà¹Į +สมารà¹Į à¸Ĺ +สมารà¹Įà¸Ĺ à¹Ĥà¸Ł +สมารà¹Įà¸Ĺà¹Ĥà¸Ł à¸Ļ +Ġth ánh +ÐŁ од +ÐŁÐ¾Ð´ ÑĢоб +ÐŁÐ¾Ð´ÑĢоб нее +ĠاÙĦت ÙĪÙĨ +ĠاÙĦتÙĪÙĨ سÙĬ +Ġbah çe +à¹ģà¸ģà¹ī à¸Ľà¸±à¸įหา +é ducation +eu rop +europ ä +europä ische +ĠK si +ĠKsi ÄĻ +ĠëĦ ĺ +ĠëĦĺ ìĸ´ +Ġv üc +Ġvüc ud +Ġyay g +Ġyayg ın +Ġnie kt +Ġniekt óry +Ġniektóry ch +ãģŃ ãģĩ +Ġк аж +Ġкаж еÑĤÑģÑı +к аж +каж еÑĤ +ĠاÙĦ دÙĬÙħÙĤرا +ĠاÙĦدÙĬÙħÙĤرا Ø· +ĠاÙĦدÙĬÙħÙĤراط ÙĬØ© +æŃ © +æŃ© ãģĦãģ¦ +Ġv az +Ġvaz ge +Ġvazge ç +Ġмин ималÑĮ +ĠминималÑĮ н +ãĥij ãĤ¿ +ãĥijãĤ¿ ãĥ¼ãĥ³ +Ġë Ĭ +ĠëĬ IJ +ĠëĬIJ ëĤĮ +ãģ¡ ãĤĩãģĨ +ãģ¡ãĤĩãģĨ ãģ© +Ġ à¸ģร +Ġà¸ģร à¸ģà¸İ +Ġà¸ģรà¸ģà¸İ าà¸Ħม +تج دÙĬد +ĠØ´ اÙħÙĦ +หลัà¸ģ à¸IJาà¸Ļ +ĠмаÑĢ ÑĪ +ĠмаÑĢÑĪ ÑĢÑĥÑĤ +Ġv ÃŃt +ĠvÃŃt ima +Ġquiz á +ay gı +×ĵ×ijר ×Ļ×ķ +Ġиз д +Ġизд ели +Ġиздели Ñı +п ла +пла Ñĩ +плаÑĩ ива +ä»» ãģĽ +Ġéquip é +ä¹ħ ãģĹãģ +ä¹ħãģĹãģ ¶ +ä¹ħãģĹãģ¶ ãĤĬ +Ġк аÑĤ +ĠкаÑĤ ал +ĠкаÑĤал ог +ส à¹īม +ĠÑĢ ÐµÐ¹ +ĠÑĢей ÑĤ +ĠÑĢейÑĤ инг +Ġth uyá»ģn +ĠاÙĦÙħ ÙĤدس +esp ère +ãģ«åħ¥ ãģ£ãģŁ +หมาย à¹Ģลà¸Ĥ +ת×Ĺ×ķש ת +à¸Ļ à¹Īะ +Ġpe ÅĤ +ĠpeÅĤ ne +Ġpé rd +Ġpérd ida +หม วà¸Ķ +หมวà¸Ķ หมูà¹Ī +иÑĩеÑģк ÑĥÑİ +çµĤ ãĤı +çµĤãĤı ãģ£ãģŁ +Ġ×Ĵ ×ķ×Ĵ׾ +à¸Ĺำ à¸Ħวาม +à¸Ĺำà¸Ħวาม สะà¸Ńาà¸Ķ +Hot éis +Ġз аÑĢ +ĠзаÑĢ ÐµÐ³Ð¸ÑģÑĤ +ĠзаÑĢегиÑģÑĤ ÑĢи +ĠзаÑĢегиÑģÑĤÑĢи ÑĢова +ĠÑģ обÑĭÑĤи +ĠÑģобÑĭÑĤи Ñı +Ġ×ĸ ׼×IJ +ÙħÙĨظ ÙĪÙħØ© +Ġ×Ķ×ŀ צ +Ġ×Ķ×ŀצ ×Ļ×IJ×ķת +Ùħ ÙĥÙĪÙĨ +ÙħÙĥÙĪÙĨ ات +ä¸ĬãģĮ ãĤĭ +Ġm ÄĻ +ĠmÄĻ sk +หรืà¸Ń à¹Ģà¸Ľà¸¥à¹Īา +ëĤ ® +Ġnok tas +Ġnoktas ı +ĠболÑĮÑĪ Ð¸Ð¼ +ĠлÑĥÑĩ ÑĪиÑħ +Ø´Ùĩ ÙĬد +à¸Ńำ à¸Ļ +à¸Ńำà¸Ļ วย +à¸Ńำà¸Ļวย à¸Ħวาม +à¸Ńำà¸Ļวยà¸Ħวาม สะà¸Ķวà¸ģ +Ġе в +Ġев ÑĢ +ĠевÑĢ Ð¾Ð¿ +ĠевÑĢоп ей +à¸ī าย +ìĦ Ń +Ùħ Ù쨧 +ÙħÙ쨧 ÙĪØ¶ +ÙħÙ쨧ÙĪØ¶ ات +ë¹ Į +赤 ãģ¡ãĤĥãĤĵ +ĠÑĥдал оÑģÑĮ +ĠÐ¥ оÑĤ +ĠХоÑĤ Ñı +przedsiÄĻbior c +ĠH ôm +íķĺìĺĢ ìĬµëĭĪëĭ¤ +Ġн аг +Ġнаг ÑĢÑĥз +ĠнагÑĢÑĥз к +Ġ×ij×Ļ׳ ׾×IJ×ķ×ŀ×Ļ +Ġê°ĢëĬ¥ íķľ +ĠH ữu +à¸Ń ุà¸Ķ +à¸Ńุà¸Ķ ม +ת ×ķפ +ת×ķפ ×¢×Ķ +Ġmi ÅĤo +ĠmiÅĤo ÅĽci +ksi Äħż +ksiÄħż ka +ĠاÙĦÙĦ عبة +à¸ī าà¸ģ +สะ สม +×ŀ תר +×ŀתר ×Ĺש +Ġlég ère +Ġ׾צ פ +Ġ׾צפ ×Ļ×Ķ +ĠиÑģÑĤоÑĢ Ð¸Ñı +Ġ ãĥĪãĥ© +ĠãĥĪãĥ© ãĥĥãĤ¯ +ĠãĥĪãĥ©ãĥĥãĤ¯ ãĥIJãĥĥãĤ¯ +Ġк а +Ġка ÑĦе +×ŀס×ŀ ×ļ +Ġc üm +Ġcüm le +à¹Ģà¸Ħลืà¹Īà¸Ńà¸Ļ à¹Ħหว +ãģĬ ãģĿ +ãģĬãģĿ ãĤīãģı +ìŀIJ ëıĻ +ìŀIJëıĻ ì°¨ +à¸Ńั à¸ķ +à¸Ńัà¸ķ à¹Ĥà¸Ļ +à¸Ńัà¸ķà¹Ĥà¸Ļ มั +à¸Ńัà¸ķà¹Ĥà¸Ļมั à¸ķิ +ĠÅŁ ik +ĠÅŁik ay +ĠÅŁikay et +extr ême +kr ä +krä fte +ëĤ Ļ +íķ ij +ì² Ļ +íĺ Ī +ì° į +âĻ ¡ +ìŀ Ķ +ë¢ ° +íĿ Ķ +íĿ IJ +âĩ Ĵ +ë§ Ľ +ìĬ Ī +á» Ĵ +ìĺ µ +âĹ İ +í Ĥ¨ +ê¿ Ī +ìĪ ¨ +ìĽ ¨ +ë§ ¥ +ï½ Ģ +ï¼ ª +Ạ¨ +ãħ İ +Ñ Ĺ +ìĦ ¬ +ì¹ ¼ +ï¼ ¶ +ìĽ ł +ëŁ ´ +Å ĥ +ëĤ ¼ +ëĭ IJ +âĢ ¹ +ë¦ Ń +ì§ IJ +âĢ ¤ +à ħ +ëľ ¨ +íĦ ¸ +íľ ĺ +ê² ģ +ë´ ħ +à ĺ +ëŃ Ķ +ëĺ ij +âĹ ĩ +ìĹ ĺ +ï» ´ +ë§ ¹ +ï¾ Ŀ +ìĬ · +íĥ ķ +ï¼ ł +ì» ´ +ëł Į +ì½ ľ +ï» ¹ +ãħ ł +ì¡ ¸ +ëħ ¹ +âĤ º +âĸ ¶ +íĥ IJ +êµ ´ +íij ¸ +Ñ Ķ +íĶ ½ +Ð ħ +ë° ¤ +Ô ģ +ì² ¨ +ì¶ ĺ +ë² Ĺ +ë© ¸ +ï¼ » +ï¼ ½ +ï¼ · +ì° Į +à Ĵ +íı ´ +ìĵ ¸ +ì´ Į +ëģ Ķ +ëĶ © +ëĩ Į +ë© Ģ +ë² ¨ +ï¼ µ +ë§ ¡ +ëĭ « +ภ¿ +ãģ ± +ìĩ ¼ +ìº ł +ë® ¤ +ê± ± +ì» ¬ +âĦ ĥ +ëĶ ± +ëĥ Ī +ìĭ ± +íĻ Ī +ëŀ IJ +ìħ Ģ +ìł ł +Ð Ĩ +ëł ī +ï½ ħ +ï½ ı +íĻ Ģ +ëĽ ° +á» ® +í Ĥ¹ +ê½ ĥ +ï» ¤ +ïº Ķ +êº ¼ +ìķ ī +âĻ ¦ +ï½ ģ +ìĵ ´ +ãĢ ī +ì° ® +ì¤ ĺ +á» ª +ëģ Ħ +ëIJ ¨ +ìķ Į +íĿ ĺ +íħ IJ +ãĢ Ī +ê² ª +ëĭ ¥ +ê² ¼ +á» Į +ë§ ¨ +ëģ Ĭ +ë² ¤ +ëij Ķ +íĿ ¡ +á» ¬ +ë¬ ĺ +ãģ ī +ëŀ « +íĶ Ī +í ħį +ìŀ ĥ +ï½ ī +ìģ ľ +âĸ ½ +ë¬ » +âĸ ³ +ï¼ ¸ +ìģ ĺ +ì¶ ° +ìĬ ´ +ìķ ± +ìĩ Ħ +Ạ® +ï´ ¿ +ï´ ¾ +âĤ ½ +ëĦ ĵ +ë£ © +ì³ ¤ +ê´ ľ +Ã Ļ +á» ľ +ï¿ £ +ëĵ Ń +ë© ĺ +ê» ´ +ëł ´ +Ð ĥ +ë¬ µ +ì§ Ŀ +ãģ º +ðŁĺ Ĥ +ëŀ ¬ +ìł Ĭ +ê´ Ħ +ìŀ Ĭ +íŀ Į +ìĦ ¯ +âĪ Ģ +âĸ ¡ +ëĢ Į +ëŀ Ļ +ï½ ĥ +Ạ¶ +ï¾ Ħ +ïº ĺ +ë¹ ¼ +à Į +âĸ · +ê¸ į +ë© ĭ +ãģ ĥ +ìĺ Ĩ +ìĺ ® +ëª ¬ +ë¡ ¤ +ëł ¬ +ëĬ ¦ +âĸ ª +ì¼ ĵ +ìľ Ī +ì§ § +ï½ ½ +ëĥ ī +ï¾ Į +ëĺ IJ +ï¼ ĥ +á» Ħ +ì´ ¬ +ì¶ ¤ +ï¼ ¹ +ï» Ń +âĤ « +ï½ ĩ +ìĺ · +ëĸ ¨ +âī « +ë¦ ¿ +âľ ¨ +Ù ± +ì¯ ¤ +ê¹ Ķ +ðŁĺ Ĭ +ìĪ « +ê³ ± +êµ ³ +ï½ ĭ +ภĮ +Ä ł +ëĶ ¸ +ë° ij +ìħ ĭ +íİ ´ +âľ ħ +íĥ ij +ëĪ ĩ +íı ¼ +ðŁĺ į +ìĺ Ľ +ï» £ +Ñ ĺ +ì© Į +ë¦ ħ +ìĿ į +ï½ ¸ +ëį ľ +ãģ ħ +íİ ¼ +ëĭ Ŀ +ë¿ Į +ì¼ ° +ìĭ « +ë° ¥ +íĽ Į +ì¨ Į +ë¹ Ļ +ï½ İ +ë´ Ħ +ìĦ ¹ +ï½ ² +ìĮ ĵ +Ò ij +ë° į +ëł Ģ +íĨ ¤ +ï½ ¯ +ë¤ Ħ +ê½ ¤ +ï½ Ĵ +ìķ ¨ +ï½ ¼ +ê¹ IJ +íģ IJ +âĦ ĸ +ë§ º +ïº ® +ëħ ģ +ê² ¸ +ï» ł +íĬ ľ +Å ¹ +ë¥ Ń +ëĪ ī +ï½ Ķ +íĮ ¬ +ìŀ ĩ +ï ¬ģ +ï» ¨ +ëij ¥ +ëŀ Ħ +Ù ¬ +íĭ ´ +ìŀ ī +Ú ¾ +ìĽ ħ +ï» ® +ëĭ ī +âī ª +âĹ Ħ +ëĪ Į +íĽ ¼ +ì¤ į +Å ¸ +ì¤ ¬ +ì¾ Į +ï½ ĵ +ï¾ Ĭ +ðŁı » +ï¾ ī +Ð ģ +íĺ IJ +ï¾ Ļ +ê¼ ¬ +íŀ IJ +âĢ ¥ +ëŁ Ń +ë§ ŀ +ìĥ ¤ +ïº Ĵ +íĭ ± +ë½ ij +à ķ +âĪ ļ +ëĤ Ħ +ê¹ Ŀ +ëĨ Ī +Ạº +ìħ Ī +ìĮ į +âĢ ¡ +ï¼ ± +ìģ ¨ +âĺ º +ëĴ · +ìĺ ³ +ðŁij į +ëª ½ +ëĤ Ń +ïº Ń +ë© Ī +á» Ī +íķ Ģ +ëĭ Ļ +ë¦ ĩ +ìķ ¤ +ìį ¼ +ãĥ µ +Ñ £ +ìľ Ĺ +â ŃIJ +ï¾ ĺ +íĹ ¬ +ê¾ ¼ +ìķ Ĺ +ï» Į +ê± · +ëħ ķ +ë¡ ± +ìķ Ĭ +ï¾ Ģ +ìĩ ł +íĮ © +ïº ª +ë§ Ļ +ï¼ ¿ +ê¿ Ķ +íİ ľ +ë£ ¸ +íĶ Ķ +ï» ³ +ëı ķ +ìĭ ¼ +á» İ +ë§ ĺ +ì¢ ĭ +íĨ ¡ +ï½ ± +íĿ ij +á» ¸ +ì¦ Į +ì¹ ¸ +ëŃ ĺ +ï¾ Ĺ +ï» ĭ +íĬ Ģ +ë¥ Ļ +ì½ © +ëģ Ĺ +ëį ´ +ìħ ľ + ¸ +ë» IJ +ìĥ µ +ê² IJ +ëĵ ¬ +ë£ ° +ãħ ĭ +ìĹ ī +á» ĸ +ëĦ Į +ï½ ¶ +ë´ ĩ +ëĤ ³ +ãĤ ľ +ëĸ » +íİ Ģ +ëį © +íķ ¸ +à · +ê¼ ¼ +ëĶ ľ +ë° ´ +ë© į +âĹ ¯ +ìĹ ij +ìĻ ¼ +ïº ij +ë¶ ķ +ë¡ ¬ +ï½ Į +íĨ ¨ +ïº ´ +ëł ĺ +ê° ¤ +ìĪ ² +Ñ ĵ +ìħ ī +ï» ĵ +ëĪ Ķ +ëį § +âĢ ¼ +ï» ² +ê° ± +ê¿ Ģ +ëĭ · +Ạ¸ +Ạª +Æ Ĵ +ëį ¤ +ìĪ Ń +ï½ Ĥ +ï½ Ī +Å ł +ë£ ¬ +Ñ µ +ëĸ ¡ +ëĥ Ħ +ìĦ ° +ëĵ Ī +ï¾ ĥ +ëĩ ¨ +ï½ IJ +êµ ½ +ìĹ ½ +ëĤ Ģ +ë¬ ¶ +ï½ · +ìı Ł +íĺ Ķ +ê¼ Ī +ëģ Ī +ì¥ IJ +ïº Ĺ +Ä Į +ëĪ ł +ëĸ ¼ +íĢ ´ +âī ¥ +ëĭ Ń +ì± Ļ +ê» ı +ë© ¤ +ìĥ ĺ +ëį ® +ë£ ¡ +ìĤ ½ +ãĪ ľ +Ä ¨ +âĢ § +ï½ º +Ä £ +ì¦ ī +ï¼ ¼ +Û © +âĪ Ļ +ë° ı +ë¹ ħ +ðŁĺ Ľ +íĪ ´ +ðŁĴ ķ +ãĢ Ĵ +ìŀ ĺ +ïº ¤ +ï½ ĸ +ë© ľ +ë² ¼ +ëĿ Ħ +ëļ ľ +ï» ĺ +ìĥ Į +ï½ Ħ +ì© Ķ +ï½ Ļ +ïº © +Û ŀ +âĺ İ +ìł ¤ +ëIJ © +Å Ŀ +âŀ ¡ +ï» § +Ð ı +ì« ĵ +ê³ ½ +É ij +ãĥ ² +ëĤ « +ë¦ ī +ì¢ ģ +ë° Ń +ðŁĺ ģ +ë¹ µ +ì² © +ì» µ +ðŁĺ ĺ +ë± ħ +âī Ī +ë¹ ļ +ï» ľ +ðŁĻ ı +íģ ° +ìĦ ŀ +ï¾ ļ +ìĺ ¹ +ë¼ Ī +ëĤ ¯ +ëŀ © +íļ ¡ +ï½ ķ +íĥ ĵ +ëĿ ł +ê³ ģ +ëĵ Ģ +ìĹ ł +ï¼ º +ë§ ij +ëĭ ¿ +ì¿ ¨ +ãİ ¡ +Ð Ĭ +íĦ ± +Å ¨ +ïº ³ +ï¾ ı +âĭ ħ +ê¼ ´ +âī ¤ +íĮ ģ +Î © +ê¶ ¤ +ìĪ į +âľ ¿ +ì½ ¤ +ëĪ ħ +íĨ ± +ãħ ľ +áIJ ħ +Å Ĵ +ðŁij ī +ï» ¦ +Ð ª +ë¥ ľ +íķ « +ï¾ ĭ +âĻ « +ê¹ ľ +ë° ¸ +ëĶ ĺ +íĿ ī +ï¾ ģ +ï¾ Ľ +ëł Ľ +ê² ¹ +ì¿ ¼ +ï» ¬ +âŀ ¤ +ðŁĻ ģ +ïº ł +ëĨ ¨ +ë¯ ¹ +ê¸ ĭ +ë» Ķ +ê¹ ĥ +ëij ij +íĭ ¸ +íİ Ļ +âŀ ĸ +ãĥ ½ +ì§ ļ +ï½ ¬ +ï» ¥ +íĮ ½ +âĢ Ĵ +ì ĮĢ +ìŃ ī +ëļ ± +ãĤ ŀ +íĭ Ī +ãĤ IJ +ëī ĺ +Î £ +ê³ ° +ë¹ Ĺ +ï¾ İ +ðŁĺ Ń +íĿ ł +ìĹ ¿ +ê° ļ +ì¤ Į +ë§ µ +ï½ ³ +ãģ ¢ +ï» Ĺ +âī ¦ +Ú ¤ +ë łģ +ê¼ ½ +ï» « +âī § +ì´ Ľ +ìł Ŀ +Ạ° +âĻ £ +ìº ĺ +âĪ ĩ +ê² ī +ë° Ł +ï» Ķ +íĸ ĩ +âĸ Ĵ +ðŁij ı +à ŀ +ðŁĺ Ĩ +ïº ¼ +âĿ Ĺ +ìº Ķ +ì¹ © +ëĸ ¤ +ëĥ ħ +âĶ ľ +ï½ » +Î Ķ +áĥ ¦ +ìŀ İ +âĺ Ģ +âĪ ¼ +ðŁĶ ¥ +ë° Į +ìł ĸ +íĹ Ľ +Î ķ +ïº ĥ +ë¶ ī +âĪ ŀ +íĥ Ń +à ĭ +âģ Ħ +ãħ ĩ +ëĦ ¥ +ëĭ ® +ëł · +íĮ Ŀ +ìº ¡ +ë· Ķ +ì© į +íĤ ´ +ëļ « +âĵ Ĵ +íķ į +âĻ Ĥ +ï¾ Ĩ +âĨ © +ìį © +ïº ķ +íĿ Ļ +Ñ ľ +íĤ · +íĿ ° +íĥ ± +ëķ IJ +ï¾ Ĵ +× ĥ +ëĮ Ħ +ìĺ ´ +ìķ µ +ê¹ ¥ +ëŀ Ń +ìª ¼ +ãİ Ŀ +ðŁĺ ħ +ëı ĭ +ëª « +ïº ¸ +ë® ¬ +ë² ħ +ëij ł +ìħ ° +ì» · +ëĶ ª +ëħ Ķ +ãħ ¡ +ìĶ » +íķ ı +ëį ± +ïº ¨ +ï¾ į +ï½ µ +ì¢ Ģ +íİ Į +ï» ° +ïº £ +Æ £ +ðŁ¤ £ +ï· º +ëĤ ļ +âĭ Ĩ +ë³ į +ðŁĺ Ħ +ìĸ Ģ +ìĻ ł +ëĨ Ķ +íĹ ¨ +ï» Ľ +ï» Ŀ +á» ¶ +ìĸ ĺ +ìİ Ħ +Ú Ĩ +ï» ŀ +ëĢ IJ +ê² Ķ +ï» µ +âĹ ¦ +íļ Ł +ê¹ ģ +ê° ĵ +ëĶ ´ +ìı ĺ +ëļ Ŀ +á» ł +ëŀ ´ +ëĦ ī +âĺ ŀ +ï½ ĺ +Å ½ +ë¦ İ +âĸ ¬ +ëŃ ī +âĩ Ľ +ìį ¬ +ïº Ł +Ë ľ +ë¶ ĵ +ìĽ ° +Å ľ +ëŃ ĩ +á» ² +Ë ļ +ëķ Ģ +âĺ ij +ðŁı ¼ +ìĸ ½ +âĮ Ĵ +Ð İ +É ¾ +íĮ ¡ +ï¾ ħ +ìŀ Ń +ï½ ¨ +ì¹ « +ìľ Į +Ò Ľ +êµ ¿ +ëĭ ¦ +âĶ Ķ +ï¾ ij +ì§ ĸ +ìº Ħ +ãĢ ĥ +Ê ¼ +ê² Ł +ï½ § +Ä ¢ +íİ ł +ë§ · +ê° ĩ +ìĭ ¹ +ðŁĴ ¦ +ï¾ ľ +ëĬ Ļ +ë² ¡ +Å ¿ +ðŁĺ ĭ +ðŁĴ ª +ì¿ Ħ +ë© ķ +ìŃ ¤ +ëĬ Ħ +ðŁĮ ¸ +ãĤ Ŀ +Ç İ +ï½ ļ +Ä Ĺ +ëģ ĵ +ê¶ IJ +áµ ī +ãĥ Ĥ +ê» į +ðŁĺ ¦ +ãĢ Ŀ +ðŁ¤ Ĺ +Ñ Ł +ìĹ İ +âľ Į +ìī IJ +à Ĩ +íĹ IJ +ðŁİ ī +Î ij +ï½ Ń +ðŁĴ Ļ +ìĽ ¬ +íĢ ĺ +ï» ¢ +ðŁĺ İ +íij ¼ +íĿ © +ï» Ħ +íħ Ģ +ëł IJ +ì¥ ¬ +Ð ĭ +ìĥ · +ëľ ¬ +ðŁĺ ĥ +ëĦ ¬ +ë¥ ¨ +ìĽ į +ï½ Ĩ +ï½ ´ +ãĥ ħ +à ı +ï» ª +âĻ ł +ëĬ ¬ +ë± Ģ +ë° ĭ +ìĥ Ģ +ï½ ¾ +ëĤ ± +ì» ¸ +ðŁĴ ĸ +ðŁij Į +Ñ ŀ +ì§ ± +Ë Ĩ +ðŁĵ ļ +âŃ ķ +ï¬ Ĥ +ï» ¡ +ëij ¬ +íĪ ¼ +âĸ ¸ +ê° ¯ +ê¹ ħ +ï½ ® +ëĺ ¥ +Ä ¡ +íĮ Ł +Ð Į +ìĨ Ł +ïº ĵ +ï» ¼ +à Ľ +ãĥ ¾ +ëĮ ĵ +íĴ ĭ +ìķ ĵ +ï½ ¹ +ëĤ ¡ +ðŁij ĩ +Ạ¼ +ãĢ Ł +ðŁĮ Ł +íĥ ł +ãĢ Ĩ +âĢ Ł +ë¸ IJ +ðŁĮ ¹ +ìł ¼ +ðŁĵ Į +ìĶ ¬ +âĹ Ģ +ðŁĴ ĵ +ê¹ İ +ìĤ IJ +ìĶ Į +Ñ Ľ +âĶ Ī +ë² ³ +ãİ ŀ +Õ ¡ +íĤ µ +ðŁ¤ Ķ +ëĢ Ķ +ìĬ IJ +íĻ ī +âľ ¦ +ëľ ¯ +ìł ¯ +ëĶ § +Î ¦ +Ë Ī +ìī ¼ +âĹ Ĭ +ëľ © +ëľ ° +ï¾ IJ +ë¿ Ķ +ìĹ ® +ì· Į +ïº § +Î Ĵ +ëµ Ļ +ï» Ĭ +ì° Ķ +íİ Ħ +ðŁĴ Ĺ +Ạ´ +ì° ¢ +íľ ¼ +ê½ Ĥ +ì± Ķ +ìī ´ +âĸ ¾ +íĪ ° +ëĭ Ľ +âĿ £ +ï½ ª +ðŁĴ ľ +Ë ĺ +ãħ ¤ +âĨ Ĺ +íĸ Ħ +âĻ ¬ +ìķ ° +ïº ľ +âī ¡ +ãĢ ĵ +ìij ¥ +íĮ į +íī ģ +ë» Ĺ +íľ ł +íľ © +âľ Ī +íĢ Ħ +ìĸ ĩ +ì¢ ĩ +íŀ Ļ +ëª ¹ +ãĤ Ľ +ðŁĺ ± +ëį Ł +๠ħ +êµ ¶ +Ù « +ìĶ ģ +âľ ª +ï¾ Ī +ðŁĻ Į +âļ ¡ +Î ļ +ì¼ Ī +ï¾ Ķ +ï¾ Ĥ +êµ ī +ïº » +ðŁĴ ĭ +á¹ £ +Ó Ļ +ìĨ ľ +ìĹ £ +âľ © +ìľ Ļ +ïº ° +Ạ² +ìŀ £ +âĿ Į +âĺ ģ +ìķ İ +Ä ½ +Û ģ +ãĦ ± +ëŁ ¿ +íĮ ¸ +ê½ ī +ìı ł +ðŁį Ģ +âĨ Ķ +ëŃ ¡ +ï» ģ +ï¼ Ħ +ðŁĴ ¥ +âĺ Ľ +íĹ · +ëij ¡ +Î ł +Î ¤ +âĦ ĵ +ïº · +Î Ļ +ëı Ķ +ì§ ¤ +âĶ ĥ +ãĦ · +Ç Ĵ +ðŁ¥ ° +ëĶ ķ +ìļ ¥ +ì¸ Ħ +íĽ Ķ +ïº ĩ +ïº ¬ +ðŁĺ ¢ +ë¹ ¡ +ìĶ ¹ +Å ³ +Ë Ŀ +íİ ij +ï¾ ĵ +ðŁĴ ļ +ëĬ ij +êº ¾ +íĨ ° +à ¿ +Ð Ħ +ëĮ IJ +ë½ Ģ +ì· Ħ +ðŁ ĵį +ðŁĻ Ī +âĹ Ī +ê¿ ĩ +ì¼ Ħ +íİ « +ðŁĩ · +âĶ ĭ +âļ ł +ë± ī +ì į° +ìĻ Ī +É ª +ïº ĭ +ðŁĺ ľ +Î Ł +ðŁ ĻĤ +âļ ½ +Å Ī +ë¹ Ķ +íĮ ľ +๠ı +ìĸ ¹ +íĪ Ń +ðŁ¥ ĩ +ãĦ ´ +ëĶ ¥ +ìŃ Ī +âĪ Ĩ +ëĸ ³ +ë± ĥ +ìŀ ¦ +ï» IJ +Î ľ +âľ § +Ï į +ìł ĵ +âĹ ķ +ëĴ Ģ +ï» Ģ +ðŁĶ ´ +ê½ ģ +ëĮ Ī +ëİ Į +ãĤ İ +⦠ģ +ì½ § +ï¯ ¾ +âĿ ¯ +ภħ +ðŁĻ Ħ +âĿ Ģ +ðŁĶ ¹ +âĩ IJ +êµ µ +âĩ Ķ +ë¶ IJ +ðŁĴ Ľ +Î ¾ +íĥ ¬ +âĿ Ħ +Ò £ +ãĢ ° +âĪ ij +âĺ ¼ +âī ł +Ò ¯ +ïº ¯ +ê¿ ¨ +âľ ĸ +Ê ĸ +íĢ Ģ +ê¾ Ģ +íĹ Ŀ +âĶ £ +ãİ ľ +ëĶ Ľ +ëľ ¸ +ï º« +ê¿ ° +ðŁĩ ¹ +Ç IJ +Û Ĵ +ë£ » +ïº ĸ +Ñ ļ +ëĬ ł +Û ķ +ê¹ ¡ +ë¿ ľ +ì² ¼ +ï¨ ij +ë¥ µ +ìį ¸ +íħ ħ +íij ¹ +Ö Ģ +ï³ Į +ãħ £ +ìij ¤ +ì½ ķ +ëķ ł +ðŁĮ ¿ +íĥ Ķ +ìĽ ģ +Î ¶ +âŀ ľ +ìĬ ĺ +íĽ Ĺ +ë© § +ìī ĺ +Õ ¶ +á¹ ĩ +ðŁİ ģ +ï½ ¿ +ï¼ Ĥ +á¼ IJ +âľ ķ +âŀ ¢ +ëĦ ¨ +ì» « +ì¯ Ķ +ì° ľ +ðŁĴ ° +íħ Ŀ +ãİ ı +ë³ ¶ +Ò ĵ +âĨ ³ +ìĥ ´ +íģ ĺ +âĸ Ģ +ë² Ļ +ภĥ +á½ ¶ +Ä ķ +⬠ĩ +ë¤ ĺ +ðŁİ µ +âľ ļ +ïº ı +Î ¡ +âĹ ī +ðŁĴ « +Ð Ī +ìĸ Ħ +ì§ Ļ +ï» ĥ +ðĿij Ĵ +ëŃ Ħ +âĿ ¥ +âĿ ĸ +âĺ Ŀ +Ê ¹ +Ḡ¥ +âĢ ¿ +ãħ ħ +ê¸ ģ +ëķ ¡ +ëį ¥ +âĪ © +ê» Ħ +ë® Į +Ò ± +âĪ Ĺ +ëł Ļ +ïº Į +Ë IJ +ðŁĺ ³ +ðŁij © +ðŁİ ¶ +ì¿ µ +ðŁ¤ © +ê· ¤ +ëĮ Ķ +ïº IJ +Ï İ +ì¶ ¥ +ï½ Ĭ +á¹ Ń +ë¤ ¼ +âĸ « +ì§ ł +á¼ Ģ +ê» ij +ëĮ ģ +íĢ ¸ +âĻ Ľ +ðŁĴ ŀ +âĸ ° +ðĿij ĸ +ëĿ ¤ +ठ¦ +ì´ ĺ +ðŁĺ ĩ +ëĶ ¤ +Î Ĺ +ðŁĻ ĩ +Ë Ľ +ì© ¡ +âĪ § +Õ ¥ +Ñ Ļ +ëIJ ¬ +ëĸ Ħ +ðŁĮ · +ìĹ Į +ðŁĺ ¥ +ëĪ ´ +ï» ļ +É Ľ +ïº Ħ +ï» ı +Å Į +ë² ļ +ìĭ £ +ïº Ģ +Î ĵ +ðŁĺ Į +Ë Ļ +ëŀ ı +ðŁĶ ¸ +ðŁĵ · +ëģ ½ +íģ ½ +ðŁĴ ¡ +ðŁĮ ± +ëº ı +ìģ ł +ìĥ IJ +ëı Ĺ +ì¸ ° +ëĪ ķ +Î Ŀ +âģ ī +ðŁĮ ¼ +íĮ ł +âĭ ¯ +áĥ ĺ +âľ ¤ +ê± Ķ +íĮ İ +ðŁĴ ¯ +ìı Ļ +íĹ ī +Ù Ń +ì½ ° +ïº ¿ +ï» ± +ì± Į +âĺ ķ +ðŁİ Ģ +Ä Ŀ +ë° § +ìĤ ¿ +áij ķ +ðŁį ĥ +âĩ ¨ +Î Ľ +ë§ ´ +ë³ ķ +á ijIJ +âĸ ĵ +ðĿ ijľ +âĻ » +íĤ ¥ +Õ ¸ +ãĪ ± +ëº Ģ +ì² ¸ +ïº Ľ +ðŁı Ĩ +ðŁĩ ª +âĿ ĵ +Ä Ģ +ì½ ¥ +ðŁĩ § +á½ · +âľ Ĥ +ìŀ ¼ +ï§ ¡ +ðŁĵ ¸ +âĻ ¯ +É Ķ +á½ ¸ +âĮ ª +ï» ĸ +ï¥ § +âļ « +âĶ Ĺ +ðŁĮ Ī +ï» © +ðŁĵ ² +Ï Ī +ðŁĺ ¡ +ðĿij İ +ìľ ½ +ì§ ¬ +ì§ Ĭ +á½ ³ +ìĮ ¤ +ëĤ į +âī Ĵ +ðŁij ¨ +âĺ ĺ +Ó © +âĤ ĵ +âĪ Ĥ +ï¹ ģ +ðŁĴ IJ +íħ ĥ +ðŁı ½ +ê· Ħ +ðŁĺ ı +ðŁĮ º +ðŁĺ Ķ +ï½ « +âľ İ +ëµ Ī +ðŁĩ ¸ +âĢ £ +âŀ Ķ +ëĺ ĺ +ìĥ ¬ +Ê ĥ +⬠ħ +ì© IJ +ðŁĻ Ĩ +ðŁİ Ħ +Ä ¾ +⣠¶ +áĥ IJ +âĺ » +ì± ķ +ìģ © +ë½ ķ +ìº £ +ðŁij Ī +ðŁĻ ĭ +ï¾ ĸ +Ò ļ +Õ « +ìĮ Ī +ë² § +ðŁĩ ® +ï½ Ŀ +ðŁį ģ +ìĹ ¥ +Ä ³ +ë½ IJ +íį ½ +íĽ ij +âĤ ¹ +ãħ ģ +ìĶ ½ +ðŁĶ ģ +ठ¯ +ê¾ ¹ +ëī ľ +âĹ ¡ +íķ Į +Î ĺ +ë£ ¹ +ìĻ ĵ +ðŁĩ ¦ +ðŁij Ģ +âĶ Į +á¿ ¦ +ëĦ Ľ +ìĦ £ +ìŃ Ļ +ï± ł +Î ŀ +Ê » +á¿ ¶ +âĿ Ŀ +ê± Ģ +ëĸ ´ +ãĦ ¹ +ðŁĴ İ +Ï ¹ +⼠ħ +ï» ķ +ãĥ ± +ï½ Ľ +ëĮ ķ +ë¹ ½ +ì¥ Ķ +ì¿ ¤ +ðŁĸ ¤ +Ñ Ĵ +ê¹ į +ëİ Ģ +ìĭ ¯ +ë» ¤ +ðŁĵ ŀ +ðŁĵ £ +ðŁĺ Ŀ +ìį ¹ +ìĹ ¡ +ì° IJ +á½ IJ +ï» Ī +âľ į +Ä ı +ðŁĮ ŀ +âĦ ¦ +ê½ Ŀ +ë» ĺ +ìĪ ± +âĶ ĺ +ðŁĮ » +âĤ ´ +âŀ ¨ +íIJ ģ +ê ¶Ī +âĺ ¢ +ðŁĺ Ī +ï½ © +âĦ Ĺ +ê° Ń +ê° ¸ +ë» ij +ì¥ ´ +ì» ¥ +ï¤ Ĭ +ï» Ĵ +ðŁĺ ķ +âĺ Ķ +ìĺ IJ +ðŁļ Ĺ +ëĹ Ħ +ë§ ı +Õ ½ +âĸ » +⣠µ +ìī ° +ï» ij +âĻ © +Î ¥ +ðŁĺ £ +âĬ Ĥ +ãħ Ĥ +ìħ ¸ +íı Ħ +âľ ½ +ì¦ Ļ +âĸ £ +ê± į +ê¿ ĭ +ì« Ħ +ìº ĩ +ðŁĩ µ +ðŁij ij +âľ ĺ +ðĿij Ľ +ìį ½ +ìº ī +ï¬ µ +ðŁĶ º +âĦ ® +íĥ ¤ +ðŁĩ º +ðŁĴ µ +íħ ¨ +ï½ ij +Î ¨ +ìĥ ¹ +ìĸ ķ +ì¹ µ +ðŁĵ ± +ठµ +ðŁij Ĭ +ðŁĴ Ħ +ðŁĴ Ŀ +ãĮ Ķ +ìĻ ģ +Ð ĩ +à® IJ +âĸ ¹ +á´ Ľ +âĹ ĺ +ëº ¨ +íĥ ī +ìĸ Į +ðŁIJ ¶ +ãĤ ij +Ë ĩ +Å ı +á½ ¹ +ìħ § +ï¹ ° +ðĿij ¡ +ðŁĶ Ŀ +ðŁĺ » +ðŁĴ ĥ +ðŁ¤ ¦ +ðŁį Ĵ +íĢ µ +âľ Ĩ +ë¹ ´ +ï§ ¤ +ï» Ļ +á´ Ĺ +ðŁĮ ´ +Í ¾ +ëĮ ij +ì¨ ĭ +ìµ ¸ +ðŁİ Ī +ðŁı ł +á½ ± +Û Ĩ +á¿ ĸ +âĢ Ľ +ì° ¼ +íķ ¥ +íĹ ´ +ðŁĩ ¬ +ì° Ŀ +âĪ ł +ï¼ ĩ +âĬ Ļ +âĿ ij +ëĦ ĭ +ëŀ Ĺ +ë° ī +ìĹ Ĭ +ì¢ Ĩ +íĮ ¥ +ï° ² +ðŁĵ ĸ +ðŁĺ ® +âļ ª +ðŁĺ ļ +âĿ ŀ +ðĿij Ł +ðŁİ Ĥ +Å ķ +áIJ Ī +êº ½ +ì± ł +ïº Ŀ +ê¿ ī +áĥ ł +ðŁı ĥ +ðŁĴ ¸ +âĿ ģ +âĹ ¾ +Ú ª +á¹ ĥ +íĬ ¬ +ðŁĩ ± +íİ Ń +ðŁĺ ŀ +ë¾ ° +á¹ Ľ +ëĽ ¸ +âĿ Ĥ +êĴ ³ +âĶ IJ +íĵ ° +âŀ ł +ê´ ĺ +ëħ ĺ +ë» ¥ +ì¾ ħ +ðŁĺ IJ +âĪ ª +ðŁij ģ +âĪ ´ +âĹ ģ +ëº IJ +ìŀ ¤ +ì± Ĺ +ðŁı ¾ +Î § +á½ » +âŀ ¥ +ìŁ Ī +ï» ī +âĸ Į +ãĥ ® +ðŁ¤ ¤ +âĩ ĵ +ì¼ ł +á´ ı +ë§ ¬ +ë» £ +ðŁĴ ¬ +ðŁį ĵ +Ä ¸ +Ù ¹ +Ê ¿ +á½ ° +ëķ ľ +ì° ¡ +ì° » +íİ į +ðŁİ ¯ +ðŁį Ĥ +ðŁij § +âĻ ¢ +áĨ ŀ +âĻ § +âļ ľ +âľ ī +ëĵ ¦ +ëŃ £ +ìĪ ı +ìĵ ± +Å Ń +Ê Ĭ +âĴ ¸ +âĩ © +ðŁĴ Ķ +Õ µ +Ð ī +Ò » +ë§ £ +ìĽ ľ +ì¿ ¡ +íĽ ħ +íĽ ¤ +ïº ¢ +âľ ĭ +âĪ Ī +ðŁĮ į +Ê ľ +ëĬ ª +ëĴ ¹ +ïº ² +âĸ Ħ +ãħ Ī +ëļ ¤ +íİ © +âĪ ¨ +ðŁ¤ ª +áĥ ļ +ê³ ¶ +íĬ ķ +ðŁĺ ¬ +âĪ « +ðŁij ĭ +Ò IJ +íĬ ¿ +ðŁĶ µ +ðŁĴ ¨ +ðŁĮ Ļ +ëĩ © +âľ ³ +ë¨ ģ +ëº Ħ +ìĻ ij +ìº ħ +íı Ī +ðĿij Ļ +ðŁĴ ĺ +ãİ ¥ +âĿ ı +âľ ° +ï¯ ¿ +ëµ IJ +ì¼ IJ +ïº ± +Õ ´ +ï¬ Ģ +âľ ´ +ðŁ¤ Ń +ðŁij Ĩ +âĽ Ķ +ê· ĵ +ìĮ Į +ðŁ¤ · +Û Ķ +ðŁ§ ¡ +ðŁĺ ĵ +Î ĸ +âı ° +ê² ľ +ëĭ ³ +ëİ ħ +ë° Ī +ï® IJ +ðŁı ¡ +âĨ ª +âĵ Ķ +âľ Ĭ +Ï ² +Ü IJ +ðŁĩ ³ +Ö Ĥ +âľ ı +ìĸ Ĺ +ì« Ļ +ðŁĺ ² +Ä Ń +âĻ Ń +âĶ ı +âĹ Į +ðŁĺ ¯ +áµ Ĵ +íĬ ł +Ä · +Ê ģ +à¤ Ł +á¹ ģ +á¼ ° +á¿ Ĩ +â « +â« ¸ +ëį « +ì³ ĩ +ì¼ ¤ +íĽ ¨ +ðŁĴ Ł +Ê Ģ +Ê ³ +ëĵ IJ +âķ ° +âĿ ĩ +Ç Ģ +Ç Ķ +É ´ +âĺ ļ +âĺ ľ +ê¶ Ĥ +ì« Ĵ +ì± Ī +ðŁĩ ¨ +ðŁİ ¥ +ðŁĵ Ŀ +Ä § +ðĿ ijIJ +Û Ī +ठ¬ +ì¬ IJ +íĹ ¥ +âĻ ¨ +ðŁį ´ +ï¹ ı +Ë ĭ +ðŁ¥ º +âĸ ¨ +íĻ ĭ +âĪ ħ +ëģ Ļ +ëŀ ł +ìĨ ¥ +âĢ ĸ +ðŁ¤ ĺ +ðŁIJ » +áµ ķ +Ç Ŀ +âĺ ı +ïº ļ +ï» Ĥ +ðŁļ © +ìĪ Ł +Ë Ĭ +⤠µ +ðŁĴ § +ã ħį +ë© © +Æ ¬ +Î ĩ +âĩ § +âĵ ļ +ìĤ ¯ +ìĪ ¯ +ëĨ ĭ +âľ ¯ +ðŁļ Ģ +Ú ĺ +Ú ¨ +âľ Ń +ê² ħ +íĮ ° +íľ Ļ +ðŁĮ Ĭ +ðŁİ ĵ +ðŁĺ Ļ +Ë ĥ +ðŁĴ ģ +ðŁij İ +âĺ ¹ +ðŁĺ « +ðŁĴ » +ëĤ µ +ìĿ Ĭ +íĮ » +Ò ³ +á½ ² +âŀ ŀ +ëĤ ij +ëĿ Ī +ì£ ¤ +ï» ¯ +ðŁĩ © +ðŁ¥ ³ +âĴ ¼ +ðŁ¦ ĭ +âĺ Ĥ +ðŁĺ ° +ðŁĻ ĥ +ðŁĺ Ĵ +Û İ +Ï ķ +Ḡ¤ +ë£ ½ +ìĬ ¥ +ðĿij ī +É IJ +ðŁį İ +âķ ¯ +âķ ¹ +ຠ² +ï¾ ł +ë¹ ķ +ïº Ĩ +Ê º +Ó § +âĨ ł +ëĥ ĩ +ìİ Ī +ìŁ ¤ +ï± ¢ +âķ ¬ +âĺ ł +ðŁİ Ĭ +ãį į +ãİ İ +âĺ ° +âľ ĥ +ãħ ī +ë¯ Ī +ë¹ ¤ +ìı Ń +ðĿij ¢ +ðŁIJ ¾ +Å ĭ +ðŁij ¶ +âĶ Ľ +ï¿ ¢ +áĥ ¡ +Ä ¼ +Å Ĩ +Ñ IJ +ìĥ Ľ +ìĺ Į +ì± ¤ +íħ ģ +íļ ĥ +ï³ Ĭ +ðĿij Ķ +ðŁĩ « +âĭ ° +ðŁĺ ¨ +âĤ © +Õ ¬ +Ḡį +á» ´ +âĨ ĺ +âĺ ¯ +ãħ ı +ìł ¬ +âĻ Ķ +ðŁĶ Ķ +ðŁĺ ł +ðŁĻ Ĭ +à® ľ +á¹ ħ +âĹ IJ +âĿ Ī +âŀ ½ +ìĥ ħ +ðĿij ł +Æ ¢ +âĭ Ļ +ê° Ľ +ëĿ µ +ë£ Ł +ìı ľ +ïº ģ +ðŁĴ Ń +âĬ ĥ +ðŁIJ ° +ãħ Į +Ü ĵ +âŀ ķ +á½ ģ +ìķ ³ +ðĿij Ŀ +ðŁİ ¬ +É ¡ +à¤ Ĺ +áIJ ī +ì© ľ +ì¶ § +ï³ ī +ï» ħ +ðĿIJ ŀ +ठ¶ +ðŁĵ ¢ +ðŁį ĭ +ðŁĴ ħ +ï¾ ķ +⬠Ĩ +âĪ µ +ðŁ¤ ij +áĥ £ +Æ Ħ +Ñ ¹ +á¼ Ķ +ê° ł +ê´ Į +ê· IJ +ëĽ ´ +ì± ĺ +ï® Ń +ïº ¹ +ïº ¾ +âľ Ĺ +âĿ ¦ +ðŁij ¦ +áĥ Ĺ +Ù ² +á½ ´ +âĪ ı +âľ ® +ê¹ ° +ë² µ +ìĦ Ģ +ì© Ŀ +ïº ŀ +ïº ½ +ðŁĩ Ń +Ë Ĥ +ðŁį ij +ðŁį Į +ðŁĶ » +ê¹ ¬ +ìĬ Ń +ìľ · +ðŁĽ ij +Ç § +ë¼ Ľ +ïº ¡ +ïº º +ðĿij ļ +ðŁĵ ¦ +ðŁĶ İ +ðŁĹ ĵ +áĥ Ķ +âľ Ĵ +âľ ¡ +ðŁĮ µ +âĶ ķ +ëĢ Ŀ +ðŁį Ĭ +âĺ ĥ +ìĺ ħ +ঠ¬ +ðŁ¦ ģ +âİ ¯ +ðŁIJ ķ +Ñ ¿ +ॠ¤ +༠ĭ +ê· Ī +ì« Į +ðŁĩ ° +âĿ ī +ì« Ģ +íĿ Ħ +ðĿIJ ¢ +ðŁļ ¨ +âĻ ¤ +ðŁĺ © +ðŁį į +ðŁĺ ij +ðŁļ ļ +Ö Ħ +ë « +ë« ¼ +ठı +á¿ · +âĮ © +âĺ IJ +âŀ £ +ê¸ ± +ê¼ ¿ +ëĦ Ŀ +ìı ´ +ìļ ¤ +ì¿ ± +íİ IJ +ðŁĴ ¢ +ì´ IJ +âĩ ij +âĶ ĵ +âģ ¾ +Ü Ŀ +ðŁ į° +â´ ° +Æ ı +Ï Ł +Ú º +Û ĥ +áĦ Ĵ +âĪ Ł +âĿ į +ãĦ ² +ìľ ħ +ì¤ ı +ðŁĩ ² +êº Ħ +ðŁİ ¤ +âľ £ +⸠Ŀ +ï¸ µ +ຠ§ +áĢ Ļ +âķ ł +Õ ¯ +âı © +ðĿij £ +ðŁĴ £ +Å ĺ +ॠIJ +âģ ĥ +âĮ ĺ +ê» Į +ìĮ Ķ +ðĿij ĺ +ðŁ¤ ĵ +Õ ¿ +à¤ Ń +âĮ ļ +âľ Ŀ +ðŁIJ ¼ +Ë Į +âķ ļ +ï¦ Ĺ +âĿ ķ +âķ £ +ðŁIJ ± +à® ¤ +Ñ ¾ +ठļ +ठľ +ìĪ Ħ +ìļ ľ +ðŁİ ® +É Ĵ +Ú · +ຠį +âĨ µ +â Īĺ +âĿ Ĭ +ë¿ į +ìIJ Ī +ìļ ĺ +ì¯ § +íĥ ¯ +ìĸ ı +ï¸ ° +ðŁĩ ¯ +ðŁ§ ļ +ðŁĺ µ +ðŁĺ · +ðŁĮ ³ +ຠ¥ +Ä ī +Ä ¥ +âľ ¶ +á¿ ¾ +âĬ ± +âĺ ¾ +ê° ī +ê¼ ° +ëº ij +ðŁĶ Ĭ +ðŁĸ IJ +Å ¤ +Ò « +à® ® +âĮ Ī +âĹ Ĺ +ëĦ µ +ëħ ľ +ëľ ¹ +ðĿij ¥ +ðŁĴ ¿ +ðŁĽ Ĵ +Ê Ĵ +áŀ ĵ +ðŁIJ Ŀ +ðŁ¦ Ħ +ðŁį · +âĺ Ł +ï¸ ¶ +ðŁ¤ Ł +Ô ± +âĨ ² +âĪ İ +âľ « +ëĩ ½ +ëı IJ +ëķ Ħ +ï¦ ³ +ï§ Ŀ +ïº Ļ +ðŁij » +ðŁĵ º +êµ ¼ +ìĮ © +ðŁĮ ² +È ± +íĶ ķ +ðŁĺ ¤ +ãĮ ¢ +Ê Ķ +ठ¡ +á¼ Ī +ëİ ĥ +ë© ± +ë® Ī +ðĿIJ « +âĬ ķ +ëĥ ł +ë» ¬ +íĭ Ķ +Õ ¤ +á¼ ± +âľ ¥ +âĺ Ħ +âĪ ¥ +âļ ķ +ðŁij Ħ +ðŁİ ħ +àº Ļ +âĶ ¬ +á½ µ +Õ ¾ +Ö ģ +âĹ Ķ +ê¿ į +ëĸ µ +ë© İ +ë® ´ +ìķ ´ +áĥ ľ +á¼ ¡ +âĶ Ĭ +âķ ® +âĹ ¼ +ðŁį ¾ +ðŁĽ į +ðŁij Ĺ +ðŁ¤ ŀ +âľ Ħ +Õ Ģ +ঠ² +Ë ī +⣠¨ +Ä ¯ +Ï Ĭ +á´ ľ +ë¹ ³ +ï³ ĭ +ï¿ ł +Ä ª +âĤ ¸ +âľ ± +ê» IJ +ëĭ » +ë§ ¸ +ìŀ ¿ +ì© ¨ +ì ŃIJ +ì° ¿ +íħ Ł +ðĿIJ § +ðĿij ij +ðŁĮ İ +ðŁĵ ® +ðŁķ Ķ +âĹ Ļ +âĹ » +âŀ § +ìŁ Ŀ +âľ ¬ +ãĥ ° +âģ Ī +â ĵĺ +ðŁ ĴĮ +ï¬ ĥ +àº Ķ +ìĶ ° +ðŁĺ ª +× Ģ +ìĥ ¨ +ïŃ ĭ +ðŁį ķ +ðŁĺ ´ +Ï ³ +á¼ Ħ +á½ ħ +âĩ ¢ +âķ Ń +ìĺ » +íĬ ¤ +Ü ĺ +⤠´ +âĹ į +áŀ Ł +ðŁį º +áŀ ļ +ðŁı Ĭ +ðŁIJ · +Ê Į +á½ º +âģ » +ê½ Į +ëĪ Ĺ +ë Ĺı +ì¿ ° +íĢ ¼ +íį ħ +ï· ² +ðŁĮ ı +ðŁį « +ðŁį ³ +ðŁİ ° +ðŁij ° +ðŁĴ ² +á¥ Ļ +ðŁIJ Ł +ï¿ ¡ +ðŁĹ £ +ðŁį ľ +âľ ² +ãİ ¢ +ðŁĶ ° +á¼ ¸ +á½ ij +Ä İ +áĦ Ģ +âĻ ķ +ëł Ŀ +ìĪ ´ +ïŃ Ń +Ó ľ +Ô Ģ +ëĢ ľ +ëĥ Ķ +ìĬ Ľ +ì« ij +ìº ¥ +ìº ¬ +ðĿij ¦ +ðŁĶ ¶ +ì¾ ¨ +ðĿIJ ļ +ðŁį » +ðŁĴ į +ðŁ¤ ¡ +ðŁķ Ĭ +â½ ĩ +âĵ IJ +ðŁį Ń +ðŁį ª +ðŁĶ Ĩ +Ò ¡ +á´ ĩ +É Ĺ +Ü Ķ +âĦ İ +âĿ ĥ +ëĹ Ģ +ï² Ķ +ïº Ī +ðĿIJ » +ðŁĴ Ĭ +ðŁļ « +Ñ ° +Ñ ³ +ठ· +âĹ ł +ðŁij ¤ +ï¾ ĩ +âĺ ĵ +ðŁį µ +ðŁ¤ ¨ +âĸ Ń +à® ´ +Ü ¢ +Ü ¬ +à´ ® +ðŁķ º +Ô ¹ +Õ £ +à´ ¯ +á ´Ģ +âĮ ī +âľ IJ +âŀ ¦ +ê¹ ½ +ëĮ ľ +ðŁı ¥ +ðŁĵ © +Ò ¹ +Ó ĺ +ठħ +âĿ § +Æ Ĺ +âĹ ½ +ðŁij « +ðŁİ § +ðŁij £ +âľ » +ðŁĻ ħ +ðŁĺ ĸ +ðŁĴ ® +ຠ° +ðŁĶ ľ +ðŁį Ħ +ðŁ¤ Ŀ +á ĥĿ +áŀ Ģ +âĩ ¦ +Ê ¾ +Ò ® +Õ ¼ +ठĨ +âĹ ħ +âļ ĵ +âļ ĸ +ê¿ © +ë¯ Ħ +ìIJ IJ +ìŀ ° +ì§ Ń +íĭ ĭ +íİ ¨ +íĻ § +ï² ij +ðŁİ Ĺ +Ù ³ +ðŁij ¸ +ঠ® +ðŁij ķ +Ú µ +âĢ ¾ +âŀ ° +ðŁij ¯ +ðŁİ ¼ +ðŁı ģ +Ä º +Ê ı +Ú ³ +âı ± +ê½ Ī +ëĿ Į +ìĮ ī +ìĹ · +ìŀ ´ +íĹ ¹ +íľ ¨ +ðĿĹ ² +ðŁĮ IJ +ðŁİ Ļ +ðŁı µ +íĽ Ļ +ðĿij ħ +ðŁĺ ¶ +âĵ ħ +âķ ¥ +ðŁį ı +ï¦ İ +Õ © +ðĿIJ Ħ +Ó £ +Ú ¿ +âĻ ļ +ðŁĶ Ĺ +Ḡ« +âĭ ® +âĸ ¦ +⼠½ +âľ µ +ãħ Ĩ +ãħ Ĭ +ëĦ Ļ +ëĿ ¨ +ë¥ Ħ +ìĦ ¦ +ì§ ° +ì§ ¹ +íī Ī +ï§ ij +ï» ĩ +ðŁĮ ¾ +ðŁı ĸ +ðŁIJ ij +ðŁĴ ³ +ðŁĵ Ĩ +Û ĩ +Ü ķ +á½ ½ +ëĦ ľ +à´ ² +à´ ³ +àº Ń +áĥ Ľ +âĿ Ķ +âij ħ +áĥ ¥ +ðŁĵ ħ +âŀ ³ +á´ µ +ï¹ ¡ +ï¹ ¶ +Î Ĩ +ठ¥ +áī µ +âĿ Ļ +âĿ ± +ëī ł +ëİ ł +ëı Ľ +ë¿ ħ +ìĶ ¸ +íij ¯ +íŀ ī +íŀ Ľ +ï§ Ħ +ïŃ ĺ +ïº ¦ +ï» ¸ +ðĿij Ĥ +ðĿij ı +Ï ij +Ú ł +áĢ Ķ +áŀ Ķ +á¹ ¢ +ëĦ ¸ +ðĿIJ ¨ +ðŁĩ ´ +Õ ° +ðŁij ł +ðŁį Ĩ +ðŁı Ģ +ðŁ ijIJ +ðŁį ĩ +ðŁIJ £ +áĪ Ń +Ü ª +ðŁ ĮĢ +áŀ ĺ +âĩ Ħ +ðĿIJ Ģ +Ê Ļ +âĶ ¼ +ðŁı ¿ +Æ · +È ł +Ñ ½ +âĤ ¨ +ê´ Ń +ê¹ » +ëĶ ¨ +ìĪ Ģ +ì¾ ° +íĨ Ī +ï® § +ï¯ ½ +ðŁĶ ħ +ðŁĶ ® +Å ¢ +Ê ° +Ñ ¸ +ठ£ +âĬ Ĺ +ëª Ħ +ï¹ · +ïº ħ +ðĿIJ µ +ðŁĮ ¶ +ðŁĵ ° +ðŁĶ · +ðŁĸ Ĵ +ðŁ¤ ² +ëī © +ðŁİ Ĩ +ðŁ§ IJ +ðŁį ® +âĨ º +âĿ ¢ +ðŁij ª +ðŁij ± +âĨ ¡ +áŀ ı +Ú ķ +ðŁį ¹ +ðŁĴ Ģ +Ë ® +Ó ¨ +Ö ħ +ठĩ +âĤ ¡ +âĪ ķ +âĺ ī +ê¹ ¼ +ê¼ IJ +ì½ ¸ +ðĿIJ ¬ +ðŁı ħ +ðŁij Ļ +ðŁĴ ī +ðŁ¤ Ļ +È ĺ +É ³ +É ¹ +Ù º +áĢ Ħ +á¿ ³ +âļ ĺ +âĿ Ĩ +ëĨ ī +ìĸ į +ìĺ ĩ +ì¥ ĺ +íĸ ħ +íĻ ij +ï® Ĭ +ï¿ Ń +ðĿĴ IJ +ðĿĹ ¢ +ðŁĶ ĸ +ðŁĶ ¨ +ðŁļ ij +ðŁļ ² +Æ ¸ +âĹ ¥ +ðĿIJ Ń +ðŁį ½ +âĹ ij +âĵ ĩ +ðŁĶ ± +âľ ¼ +ï¹ ĥ +âķ ± +ãĢ Ĺ +ðŁı ĭ +ðŁļ ´ +ðĿIJ ® +Ä ļ +Õ ı +Ä ¶ +áĥ ij +á¹ ¬ +Ä Ī +Ä Ĵ +Ò ° +Ó ķ +â IJ +âIJ £ +âĹ ¢ +âļ Ļ +ãħ Ĺ +ê° ¬ +ê³ ª +ê» Ģ +ëĦ ´ +ëİ ģ +ëĿ Ķ +ë¬ ½ +ëŃ į +ìĩ ³ +ì° ¹ +íĮ ¹ +íŀ Ŀ +ï® ĭ +ï ¶Ī +ðĿĴ Ĥ +ðŁ¥ Ģ +ðŁ¦ ħ +Ê ĺ +á¼ ij +âģ İ +ðŁį ŀ +âĨ ĸ +âĨ Ļ +ðŁİ ĥ +âĦ ¡ +âĭ ± +ðŁĶ į +ಠ¨ +áµ ĥ +âĶ « +⦠¿ +ðŁĩ » +Æ ¤ +Ò ı +Ò · +Û ī +à® ķ +Ḡ³ +ï¬ ± +ðŁĨ Ķ +Ú Ń +Û ¦ +áħ ¡ +âĦ ¹ +ê¿ İ +ëķ Ķ +ë¼ ī +ìļ § +ì² µ +ì´ ¨ +íĬ Ī +íĸ IJ +ðĿĹ ĺ +ðŁĩ ¿ +ðŁİ ĸ +ðŁij ħ +ðŁ ĵĺ +ðŁļ Ļ +ðŁĽ µ +à¶ ½ +⼠µ +ðĿIJ ³ +ðĿIJ ¸ +âļ Ķ +ðŁij Ń +Ó ij +âĶ ¯ +ðŁħ ¿ +ðŁĺ ¹ +ï¿ « +â¼ ¤ +ðŁĴ ĩ +ðŁĵ İ +ðŁĸ ĭ +ঠ¸ +ðĿIJ į +Ä ² +Ï ĭ +Ñ ¬ +Ú ¬ +Ü Ĵ +á´ ¬ +ï¨ Ħ +É £ +Ë ij +Ï µ +Ò Ŀ +Û ¥ +Ü ł +๠Ľ +áĥ ķ +áĬ ķ +á¾ ¶ +âĤ · +âĩ ¾ +âķ © +âĸ IJ +âĺ ª +âĺ ® +âĿ ļ +âĿ Ń +âŀ ± +âµ İ +ãı Ĭ +ë© ĵ +ìĹ ¾ +ìª Ħ +íĵ Į +íķ ¼ +ïŃ ¬ +ðĿij Ĩ +ðĿij ŀ +ðĿĸ Ĭ +ðŁİ ¸ +ðŁı Ħ +ðŁij µ +ðŁĴ ł +ðŁĶ ĺ +ðŁ¥ Ĥ +Å ª +à· ĥ +á´ ¼ +âĬ ° +ë³ ı +ë´ £ +ï¥ ľ +ðŁĵ Ī +ðŁķ ¯ +ðŁ§ Ģ +âĻ IJ +ðŁĨ Ĺ +ðŁĵ ķ +ðŁ§ ģ +Ü « +âĿ IJ +Õ ķ +འķ +âŀ Ŀ +ঠķ +ðĿIJ ¶ +É ¢ +Î Ħ +áĨ ¢ +âĤ ± +Õ į +à¡ ķ +á´ ° +Ḡ© +⼠· +âĿ ® +ê¡ ĵ +ëı ¤ +ëĹ IJ +ëµ Į +ìij Ī +íı ¿ +íĹ µ +ðĿIJ İ +ðŁĨ ĺ +ðŁı Ł +É ¥ +Õ » +à¡ Ķ +ठĸ +á´ ¸ +âİ Ļ +âİ ¥ +âı ³ +ëģ ķ +ëĬ ī +ì¡ į +ì¹ ¡ +ï¦ ¶ +ï¬ Ł +ï® « +ï® ¯ +ï± ĥ +ï ·» +ïº µ +ðĿĹ Ķ +ðĿĹ ¡ +ðŁİ ¨ +ðŁĶ Ĵ +Ú Ľ +ठ§ +âŀ ¹ +áĢ Ģ +ðŁį ħ +âĹ ¤ +ठł +ðŁIJ ¥ +áĥ Ĵ +ðŁı Ŀ +ðŁį ¼ +ãĮ § +âĿ Ľ +ðŁIJ Ī +ঠ¯ +áĢ ŀ +ãĢ ĸ +áŀ Ļ +ঠª +Õ Ĩ +âĬ Ĩ +âľ ¾ +ðŁIJ Ĺ +ï¹ ¿ +Ä ¦ +Ü Ł +ಠł +ಠ¥ +áŀ ī +á´ ¥ +á´ © +á½ Ģ +á½ ¡ +âĨ ķ +âŀ ¯ +ê¡ ij +ëij £ +ë± Į +ìĪ ij +ìľ Ķ +ìŀ ½ +ì¨ į +ðĿij Ģ +ðŁĮ Į +ðŁį ¦ +ðŁį © +ðŁIJ ļ +ðŁĵ Ĵ +ðŁĵ ¹ +ðŁ¥ ij +Ä ĭ +Ë Ĺ +Ñ « +Õ ¢ +Ú ° +â ĮĢ +âĹ Ĥ +âĹ £ +âľ Ľ +âĿ Ĵ +âĿ ĺ +âŀ Ļ +âŀ ² +ãİ į +ê¡ IJ +ëŀ ĸ +ìĬ Ŀ +ìĽ ¤ +ì¡ ĭ +ì¨ ° +íĹ Ļ +ï¥ ¸ +ï³ į +ï» İ +ðĿij ĵ +ðŁĵ Ĭ +ðŁļ ¼ +ï¦ ģ +ðĿķ Ĵ +ðŁ ijľ +ðŁij ¿ +ðŁĩ ½ +à· Ħ +âĸ ´ +ãį ī +âĬ ĩ +ðŁ§ ¸ +Ú ¡ +â¾ ĥ +ðŁĹ » +âĵ ij +ðŁ¤ ¸ +ðŁ¤ ¯ +êĴ ° +ðĿIJ ĵ +âĶ ´ +êĴ ± +áĢ ĺ +â ĽĦ +ï¹ ¹ +Ó Ķ +áĥ ± +Ü ¡ +ß ŀ +âĻ ı +âľ ¸ +ìij ¨ +ðĿIJ Ŀ +ðĿIJ ¥ +ðŁį ī +ðŁij ¼ +ðŁ¥ Ŀ +Æ Ķ +Ý ¬ +ठ« +ຠļ +á´ ´ +á½ ĸ +âĤ ¶ +âİ ¢ +âĿ ħ +⣠« +ãİ Ľ +ë® ¨ +ëº Į +ë¼ ĺ +ìĨ Ŀ +ìľ ³ +ìŀ Į +ì£ Ĺ +ìª ĺ +ì» ¹ +ï· ¼ +ïº Ĥ +ðĿIJ ´ +ðĿIJ ¼ +ðŁĮ ļ +ðŁı « +ðŁĴ ¤ +ðŁĴ ¶ +ðŁĴ ¼ +Ê ķ +Ê ½ +â² Ł +ãī ł +ê¡ Ĵ +ëľ Ģ +ìĥ ¾ +ì¸ ¤ +ï¥ ģ +ðĿļ Ĭ +ðŁļ ĥ +âŀ Ľ +ìħ ´ +áĦ ĭ +âĩ Ĺ +ï§ · +âĺ ĸ +ðŁIJ ¦ +⸠ľ +ðŁĴ ´ +ðŁ¤ ļ +ãĬ Ĺ +âĮ Ľ +áĪ Ľ +༠º +â½ ī +ðŁı ¢ +âĵ ŀ +âĺ ½ +ãĢ Ļ +ðŁ¤ ® +Å IJ +áĥ ¬ +ðĿĹ » +ðŁį ĸ +Æ Ĭ +Ê Ł +ß ĭ +ठĭ +áµ Ķ +á¿ ĥ +âĦ ī +âĮ ĭ +âı ² +âĵ Ī +âĵ ¢ +âķ Ķ +âļ ij +âĿ ĭ +âĿ İ +â µľ +âµ £ +ëĴ Ī +ëľ ģ +ë¶ ĩ +ìį » +ìĺ Ń +ì§ ¢ +íĹ Ģ +ï§ Ĭ +ï ¬¸ +ï± ¡ +ðĿIJ º +ðĿij § +ðĿĺ ¦ +ðŁĵ ¥ +ðŁĺ Ł +ðŁ¥ IJ +Ä ĸ +É ¨ +áĢ IJ +áĥ ĵ +Ạĵ +á¼ ¶ +á½ Ħ +âĤ ¤ +âĮ ľ +âĮ Ł +âİ ł +⼠¸ +âµ į +âµ ı +âµ ĵ +ãĢ ĺ +ë ·¸ +íħ ¼ +ï¦ Į +ïŃ Ħ +ïŃ İ +ðĿĻ ļ +ðĿļ ĺ +༠ĵ +ëŃ ħ +áIJ Ľ +ãİ ¾ +ï¨ Ģ +ðŁĹ ½ +âĻ ŀ +Ë ĸ +âĹ ŀ +ðŁ¤ « +ðŁĺ Ĺ +ï½ ¦ +ðŁ¤ ¢ +âģ ĩ +ãĢ µ +ðŁį Ķ +áĬ ł +ðŁĺ ¼ +ðĿĹ ® +ðŁIJ ³ +ðĿIJ ĭ +ðŁĨ ļ +ðŁĶ Ľ +Ñ » +Ü ¨ +à® ² +âľ ŀ +âµ Ļ +êµ £ +ì¸ ¨ +ðĿ IJľ +ðĿĺ ° +ðŁĶ ½ +Ç » +Ç ¿ +Ê ĩ +Î IJ +Ð Ģ +Ñ ¡ +Ñ ² +Ò Ĵ +Ù ¶ +ß ķ +à¶ ± +áIJ ģ +âģ ŀ +âĸ § +âĽ Ī +âľ ľ +âľ ¹ +⣠¹ +⤠ĩ +ê² Ĭ +ê¾ ľ +ë¯ IJ +ë³ IJ +ìħ © +ìIJ ¬ +ìij ¹ +ï¤ Ķ +ï¦ ļ +ï¬ ł +ïŃ Ķ +ïº ¶ +ðĿĴ ı +ðĿĸ Ĩ +ðĿĹ ¶ +ðŁı Ĥ +ðŁIJ ½ +ðŁĴ © +ðŁĵ ½ +ðŁĹ ¨ +ðŁĹ º +ðŁĺ ¸ +ðŁ¥ § +Å Ĺ +Ê İ +Ò Ļ +× ² +à¤ Ī +á¼ ´ +á¿ ij +âµ ī +ãħ ĵ +ì½ ´ +ðĿĸ ĵ +ðŁĵ Ĺ +ðŁĶ ª +ðŁĸ į +Ï Ĵ +ðŁij ¬ +áĥ Ļ +âĨ ¬ +âĶ ¤ +⼠¹ +âĻ Ł +ðŁļ ¶ +ðŁij ¾ +âĪ ĭ +ðŁIJ ¯ +à¼ İ +âľ · +ï¨ Ļ +âĶ » +ðŁij ¹ +áĦ ī +ຠª +â¾ ı +â½ ħ +ãİ ĸ +Ñ ´ +Õ ® +Ú ¼ +áĢ ķ +áĨ ¼ +ëŃ ı +ðŁIJ ¸ +ðŁļ £ +Æ Ŀ +Ô » +áĥ ¢ +ðŁį ¯ +É ¦ +Õ ¦ +âĻ ĭ +ï¬ « +ðĿĹ ¦ +Ç ļ +É ± +ठī +á´ Ħ +âĻ ĵ +⼠° +⣠ª +ëĥ ĺ +ë¢ ¸ +ìĤ ij +ï® Ķ +ðĿķ ĸ +ðĿĹ § +ðŁĩ ¼ +ðŁĵ ĭ +ðŁļ ľ +ðŁ¥ ¤ +Ä ® +Å · +ß Ĭ +ॠ¥ +à® ª +áŀ Ħ +áµ Ģ +Ḡħ +á¼ ¢ +âĪ Ŀ +âĬ ¹ +âĴ ¶ +âķ ´ +⼠± +⼠³ +⼠º +âŀ Ł +ãı Ħ +ê¸ Ķ +ê¹ Ł +ëĩ ° +ë¹ » +ìĤ ¥ +ìĽ » +ì° Ł +íĥ ° +íĨ º +íļ ½ +ï¤ ´ +ï¥ ¾ +ï³ Ŀ +ðĿIJ ¦ +ðĿĴ ľ +ðĿĴ Ł +ðĿļ Ĺ +ðŁİ Ń +ðŁı ĵ +ðŁı ³ +ðŁı º +ðŁIJ į +ðŁij ĥ +ðŁĴ ı +ðŁ¤ ĸ +ðŁ¤ µ +Õ ² +âµ Ķ +ëĺ ¬ +ï¦ £ +Ê Ĥ +áĨ « +áŀ ij +ðĿĸ İ +ðĿĹ ĸ +áĦ ĥ +âĩ ł +áĢ ¡ +འĦ +âŀ ¸ +ï¦ Ļ +âĩ ļ +ðŁIJ ¬ +ðŁIJ ¢ +â¾ Ĵ +ðŁIJ ¤ +ðŁĶ « +ãĢ ŀ +ï¸ º +ðŁĺ º +â½ ´ +ðŁĨ ķ +âģ ¿ +ðŁį ¨ +ಠķ +ðŁļ ĺ +áŀ ħ +ঠħ +áŀ ¢ +ਠľ +â ļĮ +ãĢ ½ +à· ´ +âĵ Ľ +áĢ ľ +ìĨ ¨ +Ë © +Ü Ĺ +âĭ ¼ +ðŁĻ ī +Å Ĭ +É ĵ +Ê ² +Î ° +Ñ ¼ +Ô ¿ +à¡ IJ +༠ľ +འ¦ +á¶ ľ +âĤ ² +âĨ ¨ +âĬ ¥ +âķ § +âĻ ľ +ãĭ ¡ +ë´ ¬ +ë¶ ij +ìī ¿ +ìİ ħ +ìł ± +ì° § +ï² ¡ +ðĿĴ Ľ +ðĿķ £ +ðĿĹ ľ +ðŁį ² +ðŁİ © +ðŁIJ IJ +ðŁIJ ł +ðŁij ½ +ðŁĴ ij +ðŁĵ ľ +ðŁķ µ +ðŁ ļĮ +ðŁĽ £ +Ê ĭ +Ó ¯ +Ù ¸ +ß Ķ +ß Ļ +à¡ ĵ +á´ į +Ḡ¿ +âı º +âĸ ¥ +ë¤ ½ +íľ ij +ðĿIJ ¹ +ðĿĸ Ķ +ðĿļ İ +ðŁĵ Ħ +ðŁ¦ · +Æ ĥ +à¦ Ł +âĮ Ĥ +âĺ Ń +â² ļ +ëĿ ķ +ðŁİ £ +à® ĩ +འĨ +áħ µ +áĹ ľ +âĢ ½ +âĮ £ +âģ ½ +ðŁĵ ¬ +ðŁ¤ § +âĩ ª +â½ £ +âĹ Ł +ï¨ Ĺ +êĴ ª +ðŁĽ Ģ +Ç Ĥ +ðŁ¥ ¶ +ðŁİ į +ï¿ © +ðŁij Ĵ +áµ Ī +ï¸ ¿ +áħ © +â¾ ¦ +à° ¤ +á´ ĸ +ਠ¬ +àº Ĺ +༠» +Ñ º +ਠª +á´ ³ +ðĿIJ Ī +à» Ģ +á´ ¿ +âĤ į +âĩ ¡ +⼠ª +ðĿIJ Ĥ +ðĿĴ ķ +ðŁ IJľ +Ê į +Ñ ± +འĥ +ë® IJ +ìĽ ¡ +ìľ ģ +ðĿIJ ¿ +ðĿķ ł +ðŁij Ľ +Æ ª +Ï º +Ó ¬ +Ù ¿ +Ý £ +ઠī +à® ¹ +འij +áĨ ¯ +áµ ĩ +âĩ ¥ +âı ª +âĻ ° +âļ Ń +âļ ¾ +ãħ Ħ +êĢ ° +ê° Ĺ +ê² ĭ +ê² » +ê¶ ľ +ê¼ ĩ +ê½ ¹ +ëĤ Ł +ëħ Ī +ëĭ ¢ +ë§ Ł +ëª Ĩ +ëµ Ģ +ì½ ± +íĩ ĺ +íľ ľ +ï§ ¾ +ï± µ +ï² ¢ +ï² ¤ +ðĿĴ Ĭ +ðĿĺ ¯ +ðŁį Ĺ +ðŁı į +ðŁIJ ĺ +ðŁĵ ¡ +ðŁĶ ŀ +ðŁ¤ ³ +ðŁ¥ ģ +ðŁ¥ Ĺ +ðŁ¦ Ĭ +Ä µ +Æ ¦ +Ç µ +É ¯ +Î ı +Õ Ħ +Ü ¥ +འģ +ᨠł +âķ « +ãİ ī +ë· ´ +ìĨ İ +ìİ Į +ì£ µ +íĽ ł +ï§ ª +ï³ ı +ï» º +ðĿij ģ +ðĿij ĩ +ðĿĴ Ĩ +ðŁİ ł +ðŁIJ Ķ +ðŁij Ł +Å ĸ +ठĮ +á¾ ½ +ê¦ Ĵ +à® Ł +á´ ± +ðŁı ° +ðŁIJ ŀ +à½ Ģ +áĢ ħ +âĬ ¿ +ðŁIJ § +ἠģ +â¼ Ī +âĶ ¿ +ðŁ¥ ´ +â¼ ¿ +ðŁ§ ľ +ãħ ¿ +âĦ « +ãĢ ³ +ãĬ Ļ +â¼ Ģ +ï ¦¬ +ðŁı ¬ +ðŁĵ » +áĬ Ľ +áĦ ħ +ຠĬ +ຠĽ +áħ ³ +ðŁij ® +à® ± +âĺ ĩ +ðĿIJ ı +à´ µ +à» ģ +འı +འ¢ +ᥠ± +âĤ £ +ï¥ ¦ +ïŃ Ļ +ï´ © +ï¹ Ĥ +ðŁį £ +ðŁķ ¹ +Ï ĸ +à¶ ¸ +ຠ¢ +áĭ Ń +âİ Ŀ +âĹ Ŀ +âĻ Ī +âĻ İ +ê½ ¥ +ì³ Ķ +ì¼ ij +ï± ° +ðĿij ĥ +ðŁĮ ª +ðŁį ¡ +Å İ +Ê ¦ +Ñ § +Ó İ +Ô ´ +Ú Ī +ß ĵ +ß § +à¤ Ķ +áĪ « +áĪ µ +áĹ © +á´ ł +á¼ ł +âĢ Ĺ +âģ ij +âĦ ı +âĸ ĩ +â² £ +ãĦ ³ +ãī ® +ê³ Ĺ +ëĦ Ĵ +ëĸ « +ë¡ Ħ +ë¹ ° +ë½ ģ +ìĦ ģ +ìĮ ĺ +ìŁ Į +ì³ ī +ì¼ ķ +ï¬ » +ï³ İ +ï¹ ¸ +ï¹ ¾ +ðĿIJ Ĩ +ðĿij · +ðĿĽ ¼ +ðŁİ ı +ðŁİ ŀ +ðŁIJ Ļ +ðŁij Ĥ +ðŁĵ ģ +ðŁĸ ± +ðŁļ į +ðŁļ § +ðŁĽ ¡ +ðŁ¤ Ĵ +ðŁ¥ ŀ +ðŁ¥ © +ðŁ¦ Ģ +ðŁ¦ ĸ +Ë ¢ +Ü ļ +à® µ +áĢ ģ +áī ° +âı Ń +âĻ ¿ +ê³ ĺ +ëı Ŀ +ëķ ĥ +ìħ Į +ìĴ ¸ +ìĽ Ł +íħ Ħ +íľ « +ï§ ĺ +ï¿ ¬ +ðŁı · +ðŁĶ § +ðŁ¥ Ī +Æ ĸ +áŀ ĩ +áŀ ĸ +âģ º +âĹ ľ +âŀ © +ê¦ Ń +ëĻ ¤ +ïŃ ¼ +ðĿĻ ĸ +ðĿĻ £ +ðĿĻ ¤ +ðŁĮ Ŀ +ðŁĶ ij +ðŁĽ ł +ຠĩ +âĺ £ +ãĦ ¨ +ðĿĸ Ĺ +Ó ĵ +âĨ £ +ðŁ¥ ī +ðŁĮ ł +ðŁĺ ½ +ãİ ł +Å § +ðŁIJ Ĵ +ï§ IJ +ðŁĺ ¿ +âĪ ¬ +ðŁIJ ® +⣠± +ಠ¡ +â¾ ¼ +à° ² +Ë ¶ +âĸ ¿ +Õ Ī +áŀ İ +áħ ¥ +áŀ Ĺ +Õ § +ðŁ¤ IJ +ðŁį ł +ঠ¤ +à¶ º +âĻ į +ìĺ Ļ +íĺ ĵ +ï¹ º +ðŁĽ ³ +Å ī +á´ İ +âı ľ +âĶ ³ +ê¸ · +ì¡ Ķ +ðĿĴ Ī +ðĿĴ į +ðĿĴ ¹ +ðĿĵ ĩ +ðĿķ Ł +ðĿĹ ¹ +ðŁĮ ħ +ðŁı ´ +Ä Ķ +Ä ¤ +Å µ +Ç ¾ +Ï ŀ +Ï ¶ +Ô ³ +Ü Ĩ +ß © +à¡ Ĵ +ठĺ +à¶ ļ +འĸ +áģ Ĭ +áĥ ŀ +áĦ Ĥ +áĭ « +á´ º +Ḡ£ +Ḡª +á¹ Ĥ +á¼ · +á¿ ĩ +âĩ Į +âı ¬ +âĻ Į +â® Ł +â´ » +âµ Ł +ê¦ ķ +ê¦ ª +ê¦ ® +ê² Ħ +ê¾ IJ +ëĥ ij +ëķ ĭ +ë¡ ¸ +ë¬ Ģ +ìĩ ¤ +ìĪ © +ìľ ķ +ìŃ ĺ +ì· ° +ì ·¸ +íľ Ģ +ï¤ £ +ï§ į +ï± Ħ +ï³ ij +ðĿIJ ¤ +ðĿĴ ĵ +ðĿĴ ¶ +ðĿĹ ¼ +ðĿĻ Ĭ +ðŁĩ ¾ +ðŁĮ Ľ +ðŁĮ ® +ðŁİ ĩ +ðŁİ ² +ðŁı Ľ +ðŁij ¥ +ðŁij ´ +ðŁĴ Ĩ +ðŁĵ Ĥ +ðŁĵ § +ðŁķ IJ +ðŁĸ ķ +ðŁĺ § +ðŁĻ Ģ +ðŁļ Ĵ +ðŁĽ « +ðŁ¤ ł +ðŁ¥ ļ +ðŁ¥ Ľ +ðŁ¥ £ +Ç ¯ +È § +Î Ĭ +Ò ² +× ° +Û ij +áĥ © +áĦ Į +áĪ į +áī ¥ +áı Ĥ +âģ ± +âĬ ¢ +âĹ ĵ +âĿ ° +ë¿ ¡ +ìĽ © +íģ Ń +íĨ ³ +íĬ Ħ +íĵ ¸ +ï¥ £ +ï¥ ´ +ï± IJ +ï± ¯ +ï³ ļ +ðĿĸ ĺ +ðĿĺ Ģ +ðŁIJ Ĭ +ðŁIJ Į +ðŁij ļ +ðŁĵ ĥ +ðŁļ Ľ +ðŁļ ª +ðŁ¤ ° +Ä ´ +áĥ ® +áĹ ¨ +âĻ ® +â² ŀ +ãĪ Ķ +ì ħį +ãħ ĥ +ï¥ ¡ +ຠ¡ +Õ İ +Õ º +⬠Ľ +â½ ¤ +ðĿIJ ² +âŀ µ +áĢ Ľ +âĶ ħ +âĨ Ł +â¼ Ĭ +ðŁĮ ½ +ðŁļ ¿ +ï¦ Ĭ +ãĦ £ +⼠© +ï© Ľ +ðŁį ± +â¾ ¨ +à´ ¤ +áŀ ģ +ຠŀ +Ê ļ +ðĿIJ Ĵ +à´ ± +áŀ ľ +à® © +à° Ĺ +à´ ļ +âĩ £ +ï¦ ķ +Õ ħ +Æ ĺ +âĤ ¦ +âĶ Ħ +ï¦ Ł +ï¦ « +ðĿIJ ģ +ðĿIJ ĥ +ðŁį ¸ +ðŁIJ ² +Å ¶ +É ĸ +ß ĺ +ภ¦ +à½ Ķ +áĨ · +âģ ķ +âĵ Ĥ +âĿ ľ +ï¥ ¥ +ï¬ ® +ðĿĹ Ŀ +ðĿĹ ¿ +ðŁİ ¾ +ðŁĹ Ŀ +ðŁ¦ Į +Æ ħ +Ç ª +Ò Ĺ +Ü Ľ +ß ł +à¡ ij +áī £ +áĬ Ń +á¹ ¡ +âŀ ¼ +âŀ ¾ +â´ ± +ãī ¡ +ê³ ¯ +ë½ Ī +ìĤ ĺ +ìī ij +ì «ĺ +íĮ ĥ +íĻ ° +ï¤ Ĺ +ðŁĮ ¬ +ðŁĮ ° +ðŁį ¤ +Ä » +Å ĩ +Æ ¨ +É ķ +Ò ¢ +Ò º +Ö į +× ± +Ú ± +Ú ½ +Û IJ +ठĽ +à· Ģ +๠ļ +ຠ« +á´ ¹ +á ½Ķ +á¾ ³ +âĤ Ĵ +âĨ ´ +âĩ Ŀ +âī ħ +â Į¨ +âĵ ĵ +âĸ ¢ +âļ ¬ +âŀ Ń +â² Ĵ +ãİ ¿ +ê¿ ´ +ëĪ ± +ëį ¬ +ëİ IJ +ëIJ « +ëĶ « +ë± ģ +ìĥ ¥ +íĮ ¼ +ïŃ ĵ +ï® ¥ +ï² ° +ðĿIJ ĩ +ðĿIJ ij +ðĿij Į +ðĿĵ ª +ðĿķ ļ +ðĿĺ ª +ðĿĺ ¼ +ðĿļ Ľ +ðŁĩ ¶ +ðŁĮ Ħ +ðŁĮ ķ +ðŁĮ ¤ +ðŁĮ § +ðŁį ¬ +ðŁİ ĭ +ðŁİ » +ðŁı ¨ +ðŁIJ ĩ +ðŁij ĵ +ðŁĵ IJ +ðŁĵ Ļ +ðŁĶ ¼ +ðŁķ Ĵ +ðŁĸ ı +ðŁĸ ¥ +ðŁ¤ ¬ +ðŁ¥ Ĭ +ðŁ¥ Ĵ +ß Į +ຠĦ +á¼ µ +âķ ¡ +â² ¤ +â´ ¼ +âµ ¢ +ãĪ ¯ +ëĵ ¸ +ëŁ ĩ +ëº į +ðĿĻ § +ðŁį Ī +ðŁĶ ¬ +ðŁĸ Ĭ +ðŁ¤ ¾ +Ë ¡ +Ü © +âĮ ¡ +âŃ ij +â² ¦ +ë© ī +ì¼ Ń +ï¿ ¤ +ðĿĴ İ +ðĿĹ ¥ +ðŁIJ µ +ðŁķ ¶ +ðŁķ ¸ +ðŁ¤ ľ +Õ ª +áĪ ĭ +ðŁ¥ µ +ï° ģ +áµ IJ +âķ ĵ +áĢ ĸ +âĭ Ī +É ŀ +âŀ ® +ॠ° +ãĨ ģ +ðŁĴ ± +ðŁı Ń +áĨ ¨ +ðŁį ļ +ðŁ¦ IJ +á´ » +âĺ Į +à´ ķ +Õ ± +áħ ® +ðĿIJ Į +Å ¦ +ຠķ +âľ Ļ +Ë ³ +Ô µ +âķ Ĵ +ðĿĹ Ĺ +ðĿĹ ł +Ú ļ +ঠ§ +âĨ Ŀ +âĻ ī +ãĮ » +ì¹ Ĭ +ðĿĹ º +ðŁ§ ĺ +ì³ £ +ï¬ Ŀ +ðŁij º +Ç Ł +Î Ī +Î « +Ñ ¥ +Ô ² +Õ ¨ +Ü ¦ +ঠĨ +ঠ¥ +áIJ ¢ +á¼ ģ +á¼ ĺ +á¼ ¦ +âĵ Ŀ +ãĪ ° +ãİ Ĺ +ê² ¡ +ë¨ Ģ +ì£ Ķ +ì´ ¤ +ìµ Ŀ +ï§ ´ +ïŃ Ĭ +ï² Ł +ðĿIJ · +ðĿij ĭ +ðĿĵ ī +ðĿĺ µ +ðŁĴ · +ðŁĽ © +ðŁ§ ¹ +Å Ķ +Ê ŀ +Ë ¥ +Î Į +Ñ © +Ó IJ +Ó ł +Ú ij +Ú Ĵ +ß ¨ +àª Ī +áIJ ĥ +á¹ ¯ +âĤ ĭ +âĤ µ +âĦ ħ +âĦ ł +âĪ £ +âī º +âī » +âĬ Ľ +âĮ IJ +âİ ĵ +âĺ ¸ +âĻ Ĵ +âļ Ĵ +âľ ĩ +âľ ł +â´ · +âµ ĸ +ãĦ ¸ +ãī ¢ +ãī ° +êĩ ´ +ê´ ¸ +êº ł +ëĤ ı +ëĤ ¢ +ëIJ Ģ +ëº ´ +ìĥ ľ +ìį ħ +ì¤ « +ì± ¦ +ìº ij +ì¼ ģ +ì¿ ³ +íĤ ģ +íħ ¡ +íĴ Ĥ +íĴ ī +íľ Ħ +ïŃ ª +ï® ¬ +ï¯ ¦ +ï± ª +ï² ı +ï ´Ģ +ï» Ĩ +ï¿ ¦ +ðĿij Ĺ +ðĿĸ Ļ +ðŁĮ ¡ +ðŁį Ŀ +ðŁį § +ðŁİ « +ðŁı ĺ +ðŁı ª +ðŁIJ ĭ +ðŁIJ Ľ +ðŁIJ º +ðŁij ĸ +ðŁij ŀ +ðŁij · +ðŁĵ Ģ +ðŁ ĶĦ +ðŁĶ Į +ðŁķ Ļ +ðŁĻ į +ðŁĻ İ +ðŁ¦ į +Ç ° +É Ł +Ê Ĩ +Ô ¼ +Ú ľ +ঠ¡ +ঠ¶ +áĴ ĥ +á¼ © +âĵ ķ +â² Ī +ê° ° +ê¹ ł +êº ħ +ëĦ ¹ +ë¯ ĵ +íIJ Ī +ï§ ¶ +ï® ij +ï² ¨ +ðĿĴ ī +ðĿĴ Ķ +ðĿĹ ¨ +ðĿĻ ŀ +ðĿļ Ĵ +ðĿļ ķ +ðŁIJ İ +ðŁ¤ ķ +ðŁ§ Ķ +Ï ° +Ô Ŀ +âĮ Ĭ +âĴ ¾ +ãī £ +ïŃ © +ðĿļ ŀ +Ê ij +ঠ¦ +áĦ ĩ +âī ĥ +â² Ģ +ìŁ İ +ðĿij ¶ +ðĿĵ ² +ðŁ İ· +ðŁļ ¹ +ຠģ +áł ł +ãĦ ļ +ðŁIJ ¿ +ἠļ +âķ ³ +ðŁIJ Ń +âĴ ¹ +ðĿĸ ļ +âĻ ĸ +ãĪ ² +âĨ ¾ +áĦ Ĩ +âķ Ľ +ðŁ¤ į +â½ ¥ +ðŁ Į¨ +âĪ ® +ãĮ ĺ +ãį ij +ï¹ Ģ +âĵ Ĺ +âĬ Ħ +ðŁı ¹ +Ë Ĵ +ðŁ¤ ± +ãı ľ +ðŁİ Į +ï¥ Ń +ঠ£ +ðŁİ ¹ +ãĬ Ł +à´ ° +ðĿIJ Ķ +à´ ¨ +འļ +âľ º +Õ · +ðŁij ³ +ঠľ +âĺ ĭ +âĻ Ĭ +ãĢ Ľ +È ĭ +à® ° +áĥ ¨ +âĦ ķ +íij Ģ +ðĿĵ ĥ +ðŁ¦ Ķ +Ä ¿ +Å Ģ +Æ ³ +É ļ +Ö ĥ +Ü £ +ß Ł +à¦ Ń +à§ ¡ +à¶ » +ຠ£ +འĩ +Ḡ¨ +á½ Ī +â½ ¬ +ê¡ Ķ +ì³ Ħ +ï¨ ī +ðĿIJ ¡ +ðĿĺ ¢ +ðŁį ¿ +ðŁİ Ł +ðŁı ī +ðŁĶ IJ +ðŁļ ħ +ðŁ¤ ½ +Æ į +Ç « +Ç ½ +È ļ +Î ī +Ó ¤ +Ó ª +Õ Ĭ +Ù ¼ +Ú ´ +ß Ŀ +à¶ ľ +á¼ ķ +á¿ ¥ +âİ ŀ +ãĢ ļ +ãī ¤ +ê³ ¸ +ê· ģ +ëĵ Ħ +ëĵ ķ +ì¨ Ķ +ì± ¨ +ðĿIJ ¾ +ðĿij » +ðĿĶ ¼ +ðĿķ Ŀ +ðĿĺ Ń +ðŁĨ Ļ +ðŁĵ ¤ +ðŁĶ Ł +ðŁĹ ¼ +Ä ľ +Æ ģ +Æ ¿ +Ç ³ +Ç · +É ĥ +É ł +Ê ī +Ê § +Ë ² +Ï ´ +Õ ģ +Õ ŀ +Ö ĩ +Û Ĥ +Û ĵ +ß Ĺ +ß ¦ +ঠ¹ +à® ³ +à´ ¸ +à» Ĥ +áĪ Ŀ +áĪ ª +áĭ µ +áIJ Ĭ +áĴ ª +áļ ĸ +áŀ Ľ +á´ ¢ +áµ ı +áµ Ń +á¶ « +Ḡı +ẠĴ +á¼ ¥ +á½ ķ +á½ ¼ +âĤ Ĭ +âĦ Ĥ +âĦ © +âĩ ī +âī £ +âĮ ł +âİ Ł +âı ® +âķ ĺ +âĹ ĸ +âĺ © +âĻ ij +âĻ ² +âļ Ľ +ãĦ Ł +ãī ± +ãİ ļ +ê¡ ķ +êª ĸ +ê° ¹ +ê² Ĩ +êµ Ħ +ëĩ ¬ +ëĭ ¯ +ëı ł +ëĴ ¬ +ëĸ Ī +ëĸ ½ +ëĺ Ķ +ëŀ ¸ +ë¸ ħ +ë» ł +ë¿ Ł +ìĤ µ +ìĬ ī +ìľ ° +ìł ĭ +ìł Ķ +ì¥ ¡ +ìŃ Ŀ +ì¼ ¬ +íĪ ĩ +íī ľ +íį Ħ +íĽ ¾ +íĿ £ +ï¤ © +ï¤ ¯ +ï¦ ľ +ï¦ § +ï§ ľ +ï¨ Ī +ï¬ ª +ï ¬´ +ïŃ ½ +ï® ī +ï¯ ŀ +ï° Ĵ +ï± ĩ +ï¿ Ħ +ðĿIJ ħ +ðĿij Ħ +ðĿij º +ðĿĴ Ĺ +ðĿĵ ® +ðĿķ Ľ +ðĿķ ŀ +ðĿĸ ij +ðĿĺ ģ +ðĿĺ Ĩ +ðĿĺ ¶ +ðĿĻ ¢ +ðĿļ ľ +ðŁĮ ĥ +ðŁĮ ¦ +ðŁį Ł +ðŁİ İ +ðŁı Ļ +ðŁIJ © +ðŁIJ « +ðŁIJ ´ +ðŁij Ķ +ðŁĵ ī +ðŁĵ Ľ +ðŁĶ ī +ðŁĸ ¼ +ðŁĹ ĥ +ðŁĹ ¯ +ðŁļ ĩ +ðŁļ IJ +ðŁļ µ +ðŁ¤ ¶ +ðŁ¥ ĭ +ðŁ¥ ĵ +ðŁ¥ ® +ðŁ¦ İ +ðŁ¦ ł +ðŁ§ Ĵ +ðŁ§ ¨ +Æ IJ +Ç į +Ó Ģ +Ô Ľ +ಠ° +à´ Ļ +áĢ Ĵ +ê² Ŀ +ê¹ ¹ +ë© ¥ +ìĸ Ķ +ï¤ ģ +ï¤ ı +ï¦ ī +ï¦ ĵ +ï§ ī +ï² Ŀ +ðĿĹ ŀ +ðĿĹ ± +ðŁĮ ĭ +ðŁį ¶ +ঠļ +ìķ ľ +ðĿIJ ¯ +ðĿļ Ŀ +à° ¨ +འĺ +འł +á¡ ¥ +á¾ ° +âģ į +âĶ ° +⬠ľ +ðĿIJ ł +ðĿij ¯ +ðĿĹ Ľ +ðĿĵ » +ðĿĸ Ī +âŀ » +áŀ ł +â¡ ± +â» ij +ðŁ§ µ +ï¦ ¢ +ðŁij ĺ +ãĤ Ķ +â¼ Ł +ãĬ ¤ +ï¦ Ŀ +ãĮ ¦ +âĢ ¸ +ðŁĶ Ļ +ã ¹ +ã¹ ¦ +ï¹ ħ +ï© Į +ãī ¨ +ï¸ ½ +âį ¥ +ðŁļ ī +ðŁ¥ ľ +âĵ ľ +â» Ŀ +ï¨ ľ +ðŁĴ Ĵ +áĦ ij +â¾ ŀ +ï¨ ģ +à´ ª +áĦ İ +âŀ ´ +ঠ· +áħ ¬ +áŀ § +âĨ ¢ +âķ ¦ +âľ ij +Ë ¬ +Õ IJ +à¼ Ķ +Ê ¤ +Ë ¨ +ठŀ +à» ĥ +༠ļ +âĵ ¥ +âķ ľ +ðŁIJ ĸ +á¼ Ļ +á¼ ¤ +ìĨ ° +È Ĥ +Ê ± +à® ļ +áĥ § +á´ ĭ +á´ ® +âĿ ¡ +âŀ · +ëĿ ¡ +ï§ ¢ +ï¯ ¡ +ðĿķ ķ +ðŁħ ° +ðŁ¦ ¸ +Ç ¸ +Ó ŀ +Ô ¶ +Ö Ĩ +Ú ģ +Û ĭ +áİ ¥ +á¾ ¿ +âĶ Ń +âĶ ® +êĢ Ģ +ê± ĺ +ëIJ Ń +ë½ Ħ +ìĶ IJ +ì¸ Į +íģ ł +íĻ ± +ï¥ ī +ï¨ ĸ +ðĿij ´ +ðĿĸ Ĵ +ðĿĺ ¨ +ðĿ ļĮ +ðŁIJ ¡ +ðŁij ¢ +ðŁĵ Ķ +Å ħ +Æ İ +È © +Ò ª +Ô ĥ +áĥ « +Ḡĩ +âĽ Ł +ê» Ń +ë¨ Ħ +ìŁ Ģ +ì¤ ´ +íļ IJ +ï¤ ³ +ðŁŁ ¢ +Æ § +È ¼ +Ê Ŀ +Ë Ħ +Ë ħ +Ë į +Ë § +Ò ¥ +Õ Ķ +Ø ı +Ø ¼ +ß IJ +ß ľ +ठĵ +à¦ Ļ +à® ĵ +à¶ ´ +༠į +༠Ĵ +འ£ +áĢ Ĥ +áĢ Ĭ +áĦ Ħ +á Īĺ +áĭ Ĭ +áĮ į +áij ĭ +áŀ Ĥ +áł ¢ +á¡ Ŀ +á´ ¦ +áµ į +áµ ¨ +Ḡ¡ +Ḡ¯ +á¼ £ +âģ Ĥ +âĦ ĺ +âĦ ľ +âĦ ³ +âĦ µ +âĨ ¦ +âĩ Ĩ +âĪ · +âĬ ļ +âĮ « +âĮ ¯ +âİ Ľ +âİ ľ +âİ ¤ +âİ ¦ +âİ ® +âij ī +âĶ ī +âķ Ļ +âĸ Ĥ +âĹ Ń +âĺ Ĭ +âĺ į +âĺ Ĵ +âļ Ĩ +⼠§ +⼠² +âŀ ĺ +⥠Ħ +â´ ³ +â´ ½ +âµ Ī +ãī ¯ +ãİ ij +ã§ ¬ +êĻ ¬ +ê§ ģ +ê³ ¬ +ê´ ŀ +ê» ľ +ëħ ĵ +ëĭ ¼ +ëį ĸ +ëĸ ± +ëĿ ° +ë¡ ¹ +ë¢ ´ +ë£ Ģ +ë¤ ł +ë¨ ķ +ëŃ ¥ +ìĦ ¶ +ìħ ¤ +ìĮ ķ +ìį ª +ìı © +ìĴ Ģ +ìĶ ¯ +ìĿ Ķ +ìĿ ľ +ìł Ń +ì§ ¦ +ì¨ © +ì² ¬ +ì³ ¥ +ì¼ ¯ +íĢ « +íĢ Ń +íĥ ¸ +íĵ ģ +íķ ¬ +íĹ ¸ +íĽ ķ +íľ Ń +íĿ Ĺ +ï¤ Į +ï¤ ª +ï§ ¿ +ï¬ Ħ +ï¬ ħ +ïŃ ij +ïŃ « +ïŃ º +ï® Ĥ +ï® ¢ +ï® ¨ +ï° İ +ï° ł +ï² £ +ï³ IJ +ï³ Ĵ +ï³ ĺ +ï³ ľ +ï¹ ¼ +ï¿ ¨ +ðĿIJ © +ðĿĴ ļ +ðĿķ Ķ +ðĿķ ¤ +ðĿĸ Į +ðĿĹ £ +ðĿĹ ° +ðĿĹ ´ +ðĿĺ Ĥ +ðĿĺ ¥ +ðĿĺ ® +ðĿĺ ¸ +ðĿĻ Ģ +ðĿĽ ¾ +ðĿľ ı +ðŁĮ ģ +ðŁĮ ľ +ðŁĮ ¥ +ðŁĮ ¯ +ðŁį IJ +ðŁİ Ĵ +ðŁı Ķ +ðŁı ķ +ðŁı ® +ðŁIJ Ĥ +ðŁIJ ī +ðŁIJ ¹ +ðŁĶ ķ +ðŁĶ ļ +ðŁķ ij +ðŁķ £ +ðŁĹ ŀ +ðŁĹ ¡ +ðŁĹ ¿ +ðŁļ Ĩ +ðŁļ Ĭ +ðŁļ ĵ +ðŁļ ķ +ðŁļ ¾ +ðŁĽ ģ +ðŁĽ İ +ðŁĽ ı +ðŁ¤ ´ +ðŁ¥ ķ +ðŁ¥ ĸ +ðŁ¥ ł +ðŁ¥ ¥ +ðŁ¦ Ĩ +ðŁ¦ ī +ðŁ¦ ļ +ðŁ§ ij +ðŁ§ ¥ +ðŁ§ ¿ +Å ° +Æ º +É § +ઠĩ +à® £ +áĪ Ī +áĬ ¤ +áĭ ® +áĮ Ī +áĮ µ +ᥠ² +âĵ Ł +êĻ ³ +ê° Ĭ +ëķ ģ +ëķ ¨ +ìĬ ģ +ï¦ µ +ï¬ ² +ðĿĸ į +ðĿĺ Į +ðĿĺ ³ +ðĿĻ © +ðŁį Ļ +ðŁĸ ĸ +áī ³ +áĭ ¨ +áĸ ĩ +áŀ Į +á¹ § +âķ ª +âŀ ļ +â² ĺ +ê ķ +êķ ¥ +ï¤ · +ï® £ +ï¯ ł +ðĿĴ ĸ +ðĿķ ĺ +ðĿĸ ĩ +ðĿĹ Ł +ðĿĹ ª +ðĿĹ ¯ +ðĿĻ ł +ðŁĵ ı +à¦ Ĺ +âĴ » +â² ł +ðĿĵ µ +Ê £ +à° ľ +áĬ ¢ +áŀ IJ +Ḡ· +âĦ Ľ +âĩ Ģ +âĩ Ĭ +êĴ ¦ +ê¦ ł +ï® ¤ +ðŁį Ľ +ðŁ¤ Ľ +ᨠ¾ +âŀ º +áķ ¯ +ἠı +âĩ Ĥ +âĶ ¹ +âĻ Ĺ +ðŁĸ ¨ +ê¦ ı +ઠ° +áļ ¨ +ðŁ¤ ¥ +ðŁ§ ¢ +ãIJ Ĥ +ãĦ ¥ +ðŁĸ Į +â¼ Ĵ +ãĬ § +âį © +ðŁ¦ ij +âĶ · +ï© IJ +ï© ¡ +ðĵ Ī +ðĵĪ Ĵ +â» Ħ +ï¨ Ĵ +âĦ ª +Ò § +Ú Į +âĢ ¶ +⺠ł +â» ģ +âĨ ¸ +áĦ IJ +ãħ IJ +à» Ħ +áĹ ª +âĨ ¼ +âĩ ĭ +âĩ ĺ +âĮ ij +âĸ © +ðĿIJ Ĺ +Ä Ĭ +ঠī +ìī ł +É ¤ +ß į +ß ı +áµ Ĺ +âĤ ¥ +âĵ ī +âĶ ł +âĶ ¨ +âķ Ħ +ä ¤ +ä¤ Ģ +ê» ¸ +ï® ģ +ðĵ Ĥ +ðĵĤ ĥ +ðŁ¦ ķ +Æ Ľ +ঠĩ +ãı ĺ +ï® ¼ +Ú ĵ +Ú Ŀ +ঠĵ +à¶ ¯ +á´ ħ +á½ Ļ +âģ ¼ +âĸ İ +â¼ © +ä Ķ +äĶ Ģ +ë» ¡ +ìĽ ½ +íģ Ħ +ï¥ ¼ +ï± ī +ï¹ » +ðĿĸ ĭ +ðĿĻ Ī +ðĿĻ ª +ðĿ ϶ +ðŁIJ Ħ +ðŁIJ Ĩ +áİ ¢ +ḠĮ +âĿ ´ +ðŁı ¸ +È Ŀ +É ¸ +Î ħ +Ï ľ +Ó ¢ +Õ ¹ +à´ ħ +àº Ī +áĭ ° +áij İ +áł µ +á¡ ł +á´ ī +Ḡµ +á¿ ´ +âĵ £ +âĶ ¶ +â½ ¯ +ê² ¥ +ê¿ ĺ +ëģ İ +ëİ Ī +ëĶ ¯ +ë² ° +ìĺ ¯ +ìĽ ¸ +ìŀ Ĺ +ì§ ĺ +ì¬ ¬ +ì· ¬ +íģ ħ +íĵ Ķ +íĽ Ŀ +ï¤ ® +ï¤ ¹ +ï¥ ² +ï¯ ĸ +ðĿĵ ħ +ðĿĻ Ħ +ðŁĵ ¶ +ðŁĹ Ĵ +ðŁ¥ Ķ +ðŁ¥ Ń +Å ® +Å ´ +Æ ī +Æ « +Ç ģ +Ç £ +Ç º +Ç ¼ +È į +È ¯ +É ľ +Ê ¬ +Ë ģ +Ë ¤ +Ë µ +Ï Ľ +Ò ¤ +Ò ¬ +Ó ı +Ó Ľ +Ó ¡ +Ó ³ +Ô Į +Ô ¬ +Õ ³ +Ù » +Ú ī +Ú § +Ü ľ +ß ª +ठĿ +ঠĽ +ਠĨ +ઠķ +ઠ¡ +à® İ +à° ¬ +ൠ» +ൠ¼ +à¶ ł +à¶ Ń +à¶ ¶ +à· Ĩ +༠½ +áĢ ļ +áħ ¢ +áĨ ¸ +áĪ Ģ +áĪ ķ +áĪ ° +áī ¡ +áī ¤ +áĬ ¦ +áĬ « +áĭ ĭ +áĭ į +áİ ¯ +áij Ń +áķ Ĺ +ᣠĽ +ᥠĴ +á© ī +áŃ º +á´ ¡ +áµ ĺ +áµ Ľ +á¶ ł +Ḡģ +Ḡĭ +á¹ Ļ +á¹ Ŀ +á¹ ¦ +Ạħ +á¼ Ĥ +á½ ĥ +á½ į +á½ § +á¾ · +âĢ µ +âĤ İ +âĦ Ŀ +âħ Ģ +âĨ ŀ +âĨ § +âĩ ħ +âĪ ĥ +âī ı +âī ½ +âĬ ŀ +âĬ ¡ +âĬ § +â Ĭ¶ +âĭ Ħ +âİ Ĵ +âİ ¡ +âİ £ +âİ ª +âı İ +âĵ ĥ +âĵ ĸ +âĵ ¨ +âķ ĭ +âķ ĸ +âķ ¢ +âķ ² +âĸ Ĩ +âĸ Ĭ +âĸ į +âĸ ® +âĺ ¡ +âĺ ¦ +âĺ ± +âĺ ¿ +âĻ ĺ +âĻ Ŀ +âļ ° +⼠ij +âŀ ª +⤠Ŀ +⤠¢ +⤠· +â§ « +â¨ Ń +⨠¯ +â± £ +â² İ +âµ Ľ +ãħ Ķ +ãĪ ı +ãī ² +ãī ³ +ãĬ ij +ãĭ Ľ +ãİ IJ +ê² ¤ +ê· ¿ +ê¹ ŀ +ê» ¨ +ê¼ į +ê¿ ¸ +ëĥ ¬ +ëĩ IJ +ëĭ ł +ëį ¯ +ëĹ Į +ëĹ ij +ë¥ Ģ +ëª ĥ +ëª ¯ +ë± ¡ +ë³ ĵ +ë³ ½ +ë µľ +ìĤ ³ +ìħ ¥ +ìĩ ½ +ìı ¨ +ìı ¸ +ìķ į +ìĸ ĸ +ìŁ ¨ +ì¢ ĥ +ì¢ į +ì¥ ij +ì§ ¼ +ì© ĥ +ì® ľ +ì® ¸ +ì³ ij +ì´ ¥ +ì¾ ĥ +íħ ¦ +íĪ ¿ +íĵ ½ +íķ ³ +íĸ ı +íĹ ł +íĿ « +ï¤ ĵ +ï¤ ĺ +ï¥ İ +ï¥ ¶ +ï¦ ħ +ï¦ ½ +ï§ ĩ +ï¬ Ĩ +ï¬ ³ +ï® ĩ +ï® Ī +ï® Ŀ +ï® © +ï® ± +ï¯ ĺ +ï¯ Ļ +ï¯ ¢ +ï¯ £ +ï¯ ¤ +ï¯ ¥ +ï± Ĥ +ï² Ĩ +ï² ª +ï´ ¼ +ïº ī +ïº Ĭ +ïº ¥ +ðĿij ¨ +ðĿij © +ðĿij ² +ðĿ ĴĮ +ðĿĴ ª +ðĿĴ ® +ðĿĵ Ĥ +ðĿĵ Ī +ðĿĵ ¯ +ðĿĶ ¨ +ðĿķ Ģ +ðĿķ Ĩ +ðĿķ ¦ +ðĿķ § +ðĿķ « +ðĿķ · +ðĿĹ µ +ðĿĹ ¸ +ðĿĺ Ħ +ðĿĺ Ļ +ðĿĺ ł +ðĿĺ ¬ +ðĿĻ į +ðĿĻ ij +ðĿĻ ¡ +ðĿ ύ +ðĿĻ · +ðĿļ į +ðĿĽ ¿ +ðŁ ĥ +ðŁĥ ı +ðŁħ ĺ +ðŁ ī +ðŁī ij +ðŁİ ¡ +ðŁİ ª +ðŁİ ± +ðŁİ ³ +ðŁİ º +ðŁı İ +ðŁı Ĺ +ðŁı ļ +ðŁı ŀ +ðŁı ¦ +ðŁı § +ðŁIJ ģ +ðŁIJ ħ +ðŁIJ ĵ +ðŁĴ Ĥ +ðŁĵ ij +ðŁĵ ĵ +ðŁĵ ¨ +ðŁĵ « +ðŁĶ ĭ +ðŁĶ Ń +ðŁĶ ¯ +ðŁķ Ĺ +ðŁļ Ĥ +ðŁļ ¢ +ðŁļ ¦ +ðŁļ ¬ +ðŁĽ ĭ +ðŁĽ Į +ðŁĽ ¬ +ðŁĽ ¶ +ðŁŁ ¡ +ðŁ¥ ĺ +ðŁ¥ Ł +ðŁ¥ ¦ +ðŁ¦ ĩ +ðŁ¦ Ī +ðŁ§ Ĭ +ðŁ§ Ĺ +ðŁ§ ¤ +Ê · +Ë ¹ +á¹ ļ +á½ ¥ +âĦ Ł +ê² ¯ +ê» « +ë° · +ìĥ Ĩ +ìĽ Ŀ +ì¨ ī +ì« ı +ï¯ ķ +ðĿľ ĭ +É ² +Ò Ń +Ó Ī +འĽ +áĭ ĵ +áĻ Ń +áł © +á¹ ® +âĦ Ĵ +âĨ » +âµ ĥ +ëĢ ¨ +ëł § +ìī ¥ +ìĮ ľ +ìĹ ¶ +ì¨ Ī +ìª ¾ +íı ½ +íļ Ķ +íĽ µ +ï¤ ¸ +ï¦ IJ +ï§ Ĺ +ï§ ļ +ï¬ ¯ +ðĿIJ Ĭ +ðĿķ Ĺ +ðĿĹ ļ +ðĿļ ĸ +ðŁħ ´ +È ĥ +É Ŀ +Ï ± +Ó Ĺ +ठ¢ +áħ ł +áī ¦ +áij Į +áĴ ¼ +áŀ ¡ +áł ¨ +áł Ń +ᨠħ +á¨ Ķ +á´ ĺ +á¶ ¦ +á¸ İ +á¼ ħ +á¼ ¹ +âĨ ¯ +âĵ İ +ãı Į +ê ī +êī Ĥ +ëĨ § +ëĿ ± +ì¢ ¡ +íĪ ½ +ï¤ ĩ +ï¤ Ľ +ðĿIJ ķ +ðĿĵ ¸ +ðĿĵ ¼ +ðĿĹ ķ +ðĿĺ Ī +ðŁı £ +ðŁı ¤ +ðŁĹ Ħ +Ñ · +Ò ł +áµ ĸ +á¼ ¨ +ë¬ Ħ +ï° ´ +âĪ ½ +Õ Ń +Ú ¹ +à¥ Ł +áĢ Ĩ +áŀ Ĵ +ãĢ ¶ +ê¦ « +ï¸ ĵ +ðĿIJ Ľ +ðĿĺ Ĺ +ðŁı ľ +ì« Ń +ðŁ§ ŀ +འĤ +âĨ ¿ +âĩ ı +âĵ ģ +âĶ § +âķ ģ +âķ ¤ +ê¦ Ĺ +ê¦ ¤ +ðŁı Ī +áŀ ķ +Ô ½ +àª Ĺ +ଠĨ +âķ ķ +ï½ ł +â¼ ¦ +â¼ ¯ +â¾ · +âĶ ĸ +ଠĵ +âĺ Ĺ +âį ĭ +ï¨ Ŀ +â¼ ¥ +ï¦ ª +âĦ Ĭ +ãĢ ´ +âį ¢ +ð¡ Ī +ð¡Ī ½ +ï© ¨ +ãĢ » +ãı ĥ +ï¦ ¡ +ï¨ ĺ +ðŁIJ ĥ +ðŁĨ ĸ +ðŁĹ ¾ +ãĦ ĩ +Þ ĭ +â¼ ¼ +ï¨ Ń +Þ Ģ +Þ Ħ +Þ Ī +Þ IJ +âĮ Ħ +â» ĺ +ãŁ ¢ +á ħ§ +ðIJĮ ¿ +Ë » +à² Ĺ +áĢ ĩ +áŀ Ĭ +âķ ĩ +ãĩ ¼ +ãİ ° +Õ Ĵ +Ü Ī +ß ¥ +à¿ IJ +áĢ Ł +âĨ ¥ +âķ Į +â½ Ģ +â½ ° +â¾ Ĭ +ä Ħ +äĦ Ģ +ðĵ IJ +ðĵIJ į +ðŁİ ¦ +âĤ ¯ +âĬ ĺ +âĦ į +Ê µ +Ñ ¶ +Ú ĥ +à¦ Ķ +à´ ¦ +áİ ¶ +áĵ ķ +á¹ ¨ +âĤ ł +âĩ ° +âĹ Ĵ +â¿ Ĭ +ê· ± +ì¹ ķ +íĪ © +ïŃ Ģ +ðĿĴ ¸ +ðĿĵ Ĭ +ðĿĺ © +Ç ¦ +É « +áĬ ¨ +È ¹ +Ê ¯ +Î ª +Ú Ģ +áĮ ¸ +áİ » +áı ķ +áı ´ +á² Ĥ +á½ ¨ +âı Ŀ +âĺ Ļ +ëĥ ¨ +ëĦ ¼ +ëĪ Ļ +ë£ ħ +ìĶ ¼ +ìķ Ŀ +ìļ ¬ +ìľ ± +ï¥ Ĥ +ï¦ ¹ +ï¬ ¹ +ïŃ ģ +ï³ Ī +ðĿĶ ħ +ðĿĺ ¤ +ðĿĻ ı +ðĿĻ Ļ +ðŁķ ī +ðŁ§ Ļ +Ḡij +ê´ ¼ +ëģ į +ëĹ ´ +ëĿ ³ +ë° ŀ +ë° ¢ +ëµ ĺ +ìĤ Ķ +ìĦ Ħ +ì¼ ļ +íĢ ł +íĬ ± +íĮ ĸ +ï¤ ij +ï¦ ´ +ï¦ ¸ +ï´ į +ðĿĺ · +Ä ¬ +Å ¬ +Æ Ģ +Æ ĭ +Æ ľ +Ç ij +Ç ĺ +Ç ŀ +Ç ¥ +Ç ® +É ° +É ¶ +É · +É ½ +Ê Ī +Ê IJ +Ë İ +Ë Ł +Ë ¦ +Ë ¯ +Ï IJ +Ï ĵ +Ï ¢ +Ï ¤ +Ï ª +Ï Ń +Ï ® +Ï » +Ñ ł +Ñ Ń +Ò ¨ +Ó Ŀ +Ô ¡ +Ô · +Õ ī +Õ ĵ +Õ ĸ +Õ ļ +Õ Ŀ +Ö İ +Ø ¿ +Ú ħ +Ú į +Ú Ķ +Û Ĭ +Û ¾ +Ü Ļ +Ý Ĵ +Ý ĺ +ß Ĵ +ß ĸ +ठĬ +ठIJ +ঠı +ঠĸ +à§ Ł +ઠ® +ઠ¹ +à® ħ +à® Ĩ +à° ¡ +à° ° +ಠļ +ಠ® +ಠ¯ +à´ Ł +à´ · +ൠ¾ +à¶ ij +à¶ ŀ +༠¼ +འĵ +áĢ ĵ +áĤ ¦ +áĥ ĸ +áĥ Ń +áĥ ¯ +áħ ¨ +áħ ª +áĨ ° +áĪ ģ +áĪ İ +áĪ ĵ +áĪ ¥ +áĪ ² +áĪ ´ +áĪ » +áī ł +áī ² +áī ¶ +áĬ £ +áĬ ¥ +áĬ ª +áĭ ĺ +áĭ ² +áĭ ¶ +áĮ £ +áį ¡ +áį £ +áİ ¬ +áİ ¾ +áIJ ¡ +áķ ķ +áĸ ± +áĹ IJ +áĹ Ń +áĺ ī +áļ ± +áĽ Ł +áŀ ¥ +áŁ Ķ +áł £ +áł ª +áł ° +áł ´ +ᤠĸ +ᥠ£ +á ® +á® ł +á ¯ +á¯ Ļ +á ° +á° į +á´ Ĭ +á´ ¾ +áµ ģ +áµ İ +áµ ŀ +áµ ¤ +á¶ ħ +á¶ ĺ +á¶ Ł +á¶ ¢ +á¶ ¤ +á¶ ± +á¶ » +Ḡī +Ḡŀ +Ḡº +á¹ ĵ +á¹ Ĺ +á¹ ª +ẠĬ +Ạı +ẠĽ +á¼ ĥ +á¼ Į +á¼ ¿ +á½ Ĥ +á½ ĵ +á½ Ĺ +á½ ¦ +á¾ ± +á¾ ´ +á¿ ĺ +á¿ Ł +á¿ ¸ +âģ ĺ +âĤ ij +âĤ Ľ +âĤ ¿ +âĦ ĩ +âĦ ŀ +âĦ ± +âĩ Ł +âĩ ² +âĪ ¤ +âĪ ¶ +âī Ĥ +âī ¾ +âĬ ¨ +âĬ ³ +âĬ · +âĭ Į +âĭ ĺ +âĮ ķ +âĮ ¥ +âĮ µ +âĮ º +âį £ +âį ² +âį µ +âİ ĩ +âı ĥ +âı IJ +âı ł +âı ¤ +âı ¶ +âı ¸ +âı ¹ +âij Ĥ +âĴ · +âĴ º +âĵ ¡ +âĵ ¤ +âĶ ¾ +âĸ ĺ +âĸ µ +âĹ ª +âĹ · +âĺ ¨ +âĺ « +âĺ ² +âĺ ³ +âĻ Ĩ +âļ ¤ +âļ ¥ +⼠ĵ +⼠´ +⼠¾ +âŀ « +âŀ ¿ +⣠· +⤠ij +⤠« +⤠¶ +⤠½ +â§ ª +â¨ Ģ +â ©½ +⬠¡ +⬠¢ +⬠¤ +â² ĸ +â² ª +âµ Ģ +⸠® +⸠½ +ãĢ ł +ãĢ · +ãĦ Į +ãĦ ĺ +ãħ ij +ãĪ İ +ãĪ IJ +ãĬ ľ +ãĮ ĵ +ãĮ ł +ãİ Ł +ãİ ¤ +ãİ § +㬠® +ä Ī +äĪ Ģ +ä ° +ä° Ģ +ê ħ +êħ ī +êĩ Ĺ +ê Ī +êĪ į +ê§ Ĥ +ê§ Ĭ +êª Ģ +ê² Ī +ê² į +ê³ Ģ +êµ ł +ê½ IJ +ê¾ Ī +ê¿ ± +ëĥ ı +ëĦ ij +ëħ ¤ +ëĩ ¸ +ëĪ ¼ +ëī ħ +ëĬ £ +ëĭ º +ëį ŀ +ëIJ Į +ëķ ¸ +ëĺ ł +ëĻ ĩ +ëĻ Ī +ëľ ½ +ëŀ Ķ +ëł ľ +ë£ IJ +ë§ Ģ +ë§ Ĭ +ëª Ģ +ë¬ Ń +ë¯ ¾ +ë³ ľ +ë´ Ĭ +ëµ ī +ë· ľ +ë¸ Ģ +ë¹ ĭ +ìģ Ħ +ìĤ £ +ìĤ » +ìĦ µ +ìħ Ĵ +ìī Ī +ìī Ķ +ìĬ Į +ìĬ Ļ +ìIJ ´ +ìĵ º +ìķ ļ +ìķ º +ìĸ ľ +ìĹ ª +ìĺ ľ +ìĻ ¤ +ìļ Ľ +ìļ º +ìĿ ħ +ìĿ ı +ìĿ Ń +ìĿ ¶ +ìł Ľ +ì¡ Ī +ì¢ ī +ì¢ Ķ +ì© ł +ìŃ Į +ì¯ © +ì´ £ +ì¸ ķ +ì¹ Ł +ì¾ ¡ +ì¿ Ļ +íģ ĩ +íģ ī +íĩ Ģ +íĪ ¶ +íĸ ij +íĸ ¤ +íĹ ħ +íľ ı +íĿ Ŀ +ï¤ Ĵ +ï¤ ķ +ï¤ ¬ +ï¥ ħ +ï¥ ĩ +ï¥ ı +ï¥ ļ +ï¥ Ł +ï¦ Ħ +ï¦ Ī +ï¦ ¨ +ï¦ © +ï¦ ² +ï§ ģ +ï§ ĥ +ï§ Ķ +ï§ ł +ï§ £ +ï§ ® +ï ŃIJ +ïŃ ĸ +ïŃ ¦ +ïŃ ´ +ïŃ µ +ïŃ ¶ +ïŃ ¸ +ï® Į +ï® İ +ï® ŀ +ï® Ł +ï® ¡ +ï® ª +ï¯ Ķ +ï¯ Ĺ +ï¯ ļ +ï¯ Ľ +ï¯ Ŀ +ï¯ Ł +ï¯ § +ï¯ ¨ +ï¯ « +ï¯ ¯ +ï¯ ° +ï¯ ± +ï¯ ² +ï¯ ³ +ï¯ ´ +ï¯ µ +ï¯ ¶ +ï° Ģ +ï± ħ +ï± Ķ +ï± ´ +ï² ģ +ï³ ķ +ï· ½ +ï¸ ķ +ï¸ ± +ï¹ £ +ï¹ ½ +ï» į +ï¾ ± +ðĿIJ Ļ +ðĿIJ ½ +ðĿij ¤ +ðĿij ® +ðĿij µ +ðĿĴ ĥ +ðĿĴ Ħ +ðĿĵ Ń +ðĿĵ · +ðĿĶ ĸ +ðĿĶ ŀ +ðĿĶ ¢ +ðĿĶ ¦ +ðĿĶ ¬ +ðĿķ Ħ +ðĿķ Ĭ +ðĿķ İ +ðĿķ Ļ +ðĿķ ľ +ðĿķ Ń +ðĿķ ³ +ðĿķ ¸ +ðĿķ ¾ +ðĿ ĸī +ðĿĸ ı +ðĿĺ ĩ +ðĿĺ ī +ðĿĺ ĸ +ðĿĺ Ľ +ðĿĺ ŀ +ðĿĺ « +ðĿĺ ¾ +ðĿĻ ĩ +ðĿĻ ī +ðĿĻ ĭ +ðĿĻ İ +ðĿĻ ĺ +ðĿĻ ¥ +ðĿļ ĥ +ðĿļ IJ +ðĿļ Ķ +ðĿľ ĥ +ðŁĦ · +ðŁħ Ŀ +ðŁħ ¾ +ðŁĨ Ĥ +ðŁĨ ĵ +ðŁĮ Ĥ +ðŁĮ Ĩ +ðŁĮ ī +ðŁĮ ij +ðŁĮ ĺ +ðŁĮ © +ðŁĮ « +ðŁį ¢ +ðŁį ¥ +ðŁİ Ľ +ðŁİ ¢ +ðŁİ ´ +ðŁij ¡ +ðŁĴ ¾ +ðŁĵ Ń +ðŁĶ Ī +ðŁĶ ¦ +ðŁĶ ² +ðŁĶ ³ +ðŁķ ĵ +ðŁķ ķ +ðŁķ ĺ +ðŁķ Ł +ðŁķ · +ðŁĹ ³ +ðŁļ Ħ +ðŁļ Ķ +ðŁļ ĸ +ðŁĽ IJ +ðŁĽ ¤ +ðŁĽ ¸ +ðŁ ł +ðŁł ³ +ðŁ¤ ¹ +ðŁ¥ ĥ +ðŁ¥ ¨ +ðŁ¥ ª +ðŁ¥ ¾ +ðŁ¦ ĥ +ðŁ¦ Ĵ +ðŁ¦ Ļ +ðŁ¦ ¶ +ðŁ§ ł +ðŁ§ ª +ðŁ§ Ń +ðŁ§ ² +𣠷 +ð£· Ń +ð¦ ĺ +ð¦ĺ Ĵ +Æ ij +Ç Ļ +È ® +Ø ł +Ú Ħ +Ü Ģ +ß ¢ +áī Ģ +áĬ IJ +áİ ł +Ạŀ +ëĪ ŀ +ëķ Ł +ë£ ģ +ë¤ Ĺ +ìĦ ¥ +ìħ ij +ìĸ IJ +ìĽ Ľ +ì£ ķ +íİ ı +íĽ ĵ +ï¥ º +ï³ Ľ +ï´ « +ðĸ § +ðĸ§ · +ðĿķ ģ +ðŁIJ ª +ðŁĴ Ī +ðŁĵ ł +ðŁķ Ľ +ðŁķ ´ +Ñ Ŀ +Ó Ĭ +ॠ² +ઠª +áĥ ¤ +áį IJ +á¶ ° +á¼ Ŀ +á½ © +âĭ ĭ +âĴ ½ +âĻ ¾ +â ½Ķ +â¾ ¯ +ãĦ Ĵ +ãħ ļ +ëIJ į +ë· ģ +ìĭ Ģ +ìļ Ŀ +ì¥ ° +ìº ´ +íĭ ī +íĿ ½ +ï¦ Ģ +ï¦ ¿ +ï§ ħ +ï§ ĵ +ïŃ ¯ +ï® Ĩ +ðIJ¤ ķ +ðĿIJ Ł +ðĿĴ ħ +ðĿĵ ľ +ðĿĶ ° +ðĿĶ » +ðĿĺ į +ðĿĻ ¯ +ðŁĦ ½ +ðŁħ Ĥ +ðŁħ Ķ +ðŁħ ½ +ðŁĵ ´ +ðŁ§ ĸ +Ó Ĵ +Ḡ² +ëī ¼ +Ç ı +È ĵ +Ê ¸ +Õ Ĥ +Û ħ +ß ¡ +ß £ +à® ¯ +à° Ī +ಠ¸ +ຠ® +༠ķ +áĢ İ +áĨ ¡ +áIJ ĭ +áIJ ķ +áij ¯ +áŀ Ĩ +ᨠķ +á© Ī +âģ ħ +âĨ ļ +âĶ İ +âł © +â² Ĥ +â² Ķ +â² ¨ +ãĬ ļ +íĵ ² +ðĿij Ī +ðĿij ¬ +ðĿij ¹ +ðĿĴ ¾ +ðĿĵ ± +ðĿĵ ½ +ðĿķ ¯ +ðĿķ » +ðĿĺ ½ +ðĿļ Ĩ +ðŁĦ ° +ðŁIJ ¨ +Ò ķ +ಠħ +ï¨ Ĩ +ðĿij ° +ðŁĦ ¸ +Ô İ +Ø į +Ù µ +ಠ¶ +áĢ Ī +áĺ Ĺ +áł ¸ +á¡ ¡ +ᨠ² +á© ģ +á´ · +áµ § +âķ ¨ +âļ ģ +â¾ Ŀ +ãĢ ¼ +ãĦ ı +êĴ « +ê¦ ¥ +ê¦ © +ê¦ ² +ìĺ ¼ +íĵ IJ +ðĵ ĩ +ðĵĩ ¼ +ðĿķ ¿ +ðŁĽ ´ +ë¨ ľ +ಠµ +à´ İ +à¼ Ģ +âĩ ĸ +ãĪ « +âĵ Ģ +áħ ´ +áļ ¾ +ἠŀ +ἠ« +ᥠ´ +âĨ Ľ +âĨ ¶ +âĩ ¤ +âķ Ł +âĺ · +âļ IJ +ðŁ§ ´ +á¹ ³ +âĶ į +âĶ Ĵ +âĶ © +âĶ ¦ +â¾ µ +ઠľ +ઠ¤ +âĩ Ļ +âĶ ± +âķ Ģ +â½ Ĭ +ï½ Ł +ଠ¡ +ðł ® +ðł® · +âķ ĥ +â° Ķ +ãĬ ¦ +ðŁİ IJ +ãĩ ° +â¼ Ŀ +â¾ Ķ +â½ Ĵ +âł Ĵ +ï¨ ¦ +ï© Ĵ +ï¨ ² +ï© ĸ +ðĵı ¸ +ãĮ ĥ +ðĸ ¤ +ðĸ¤ IJ +ï¦ Ń +âĬ ħ +â¾ ³ +ä´ ¥ +ï© ķ +ðŁĮ Ķ +áŀ ĭ +âļ į +â¼ ĭ +ãİ ĺ +ðIJĮ ² +É © +áİ ij +âĨ ® +âĩ ĥ +âļ İ +ãĩ ± +ãĭ © +ãĮ ¶ +êĻ ª +ëİ ¬ +ï¨ IJ +ï¨ Ľ +ï© Ĭ +ï© į +ðĵ ħ +ðĵħ º +Ï ¡ +È ij +É Ĥ +Ô ĵ +ß İ +à´ § +áĢ ī +áĢ ĭ +áĢ ij +áĢ ł +áļ Ļ +ᨠĦ +ᨠ© +ᨠ¹ +á© ĵ +ᬠľ +á´ Ļ +áµ ij +âĤ Ń +âĨ ° +âľ ģ +â½ IJ +ãĭ ¯ +ãĮ ½ +íĨ ¢ +ï¤ ¿ +ðŁ Ĥ +ðŁĤ » +È Ĵ +Í º +Ô ¥ +Õ ij +Ú ¶ +à§ İ +à¶ ® +ຠĸ +ຠľ +ຠ½ +áĥ » +áħ ¯ +áĭ ŀ +áĸ ķ +á ´Ī +á¶ Ĩ +Ḡľ +á¹ ¼ +á¿ ¨ +âĦ ĭ +âĦ Ń +âĪ ± +âĮ ĵ +âĶ ĩ +âĶ ¢ +â± ® +â² Ħ +ãĩ ¾ +ãĪ ¬ +ë¸ ¡ +ìIJ ī +íĻ Ľ +ðĿķ ª +Æ ¹ +Í ² +Ó ģ +Û ¼ +ঠ« +áħ Ł +áī Ĩ +áį Ī +Ạĸ +á½ ī +âĶ ¸ +â½ © +ê ľ +êľ ¥ +êµ ħ +ëĤ Ķ +ëĦ ł +ëĩ Ĺ +ëĻ Ŀ +ìļ ¯ +ìļ · +ìŁ Ľ +ì· IJ +íŁ ¬ +íŁ ® +íŁ ° +ï¦ Ĩ +ï¦ ± +ï² ŀ +ï³ ¤ +ï³ ¥ +ðIJĮ ¸ +ðĿĶ ı +ðĿķ ® +ðĿĺ £ +à¦ Ī +âı ı +ãĦ ĸ +ê² ĩ +ëĸ ĺ +ëľ · +ëŀ Ĵ +ë¡ ĵ +ë¢ ī +ë£ ĥ +ë§ ĭ +ë² ĭ +ìĤ · +ìĪ ķ +ì Į¨ +ìĵ » +ìĸ Ĭ +ìĻ ¬ +ìĿ » +ì¦ ģ +ìµ ¤ +ì· ĥ +íĢ ľ +íħ ī +íį ł +íı ħ +íij ± +íķ ķ +íĸ ł +íĿ ķ +Æ Ļ +Æ ļ +Æ ŀ +Ç ĥ +Ç Ĭ +Ç ľ +Ç ¤ +Ç Ń +Ç ¹ +È Ģ +È ģ +È ħ +È ī +È Ĺ +È Ł +È ¤ +È ¥ +È ¨ +È µ +È º +È » +É Į +É ® +Ê ħ +Ê ¥ +Ê ¨ +Ë ĵ +Ë Ķ +Ë ł +Ë £ +Ë ¸ +Í ´ +Ï Ĺ +Ï ĺ +Ï Ļ +Ï ļ +Ï Ŀ +Ï ¨ +Ï ¬ +Ï ¾ +Ï ¿ +Ñ ª +Ò Ģ +Ò ľ +Ò ¼ +Ò ½ +Ó Ĥ +Ó ħ +Ó ĩ +Ó į +Ó ĸ +Ó Ł +Ó « +Ó ± +Ô Ĩ +Ô ĩ +Ô º +Õ ĭ +Ö ī +Ø Ī +Ø Ĭ +Ø ½ +Ø ¾ +Ù · +Ú Ĥ +Ú Ĭ +Ú ĸ +Ú Ĺ +Ú £ +Ú « +Ú ¸ +Û Ģ +Û į +Û ½ +Ü ī +Ü ¤ +Ý § +Ý ´ +Þ ĥ +Þ ¤ +Þ ¥ +ß ļ +ß Ľ +ß ¤ +àł į +àł ĵ +àł ³ +à¡ ¢ +ॠł +à§ ł +à§ º +ਠĬ +ਠIJ +ਠ® +ਠ¯ +ਠ° +ਠ¸ +ઠĨ +ઠ³ +ઠµ +ઠ½ +ଠĮ +ଠĺ +ଠ½ +à® ĥ +à® ¸ +à° Ĩ +à° ķ +à° ¦ +ಠĨ +ಠĬ +ಠĮ +ಠIJ +ಠĽ +ಠ¤ +ಠ¦ +ಠª +ಠ² +ಠ¹ +à´ Ĩ +à´ ı +à´ Ĺ +à´ « +à´ ¹ +ൠº +ൠ½ +à¶ ħ +à¶ Ĭ +à¶ Ķ +à¶ § +à¶ « +à¶ ° +༠Ħ +༠ħ +༠Ĭ +à½ Ļ +འ¡ +འ§ +à¿ Ģ +à¿ Ļ +áĢ Ŀ +áĢ § +áĢ © +áĢ ¿ +áģ µ +áĤ ģ +áĤ ½ +áĥ Ĥ +áĥ ª +áĦ Ĭ +áĦ ¢ +áħ ¦ +áħ Ń +áĨ ® +áĨ ± +áĨ » +á ĩ +áĩ Ĥ +áĪ ħ +áĪ ī +áĪ Į +áĪ IJ +áĪ Ĵ +áĪ Ļ +áĪ ļ +áĪ ľ +áĪ ŀ +áĪ © +áĪ ³ +áĪ º +áĪ ½ +áī ħ +áī ¢ +áī ± +áī ´ +áĬ ĥ +áĬ į +áĬ ĸ +áĬ ® +áĬ ¸ +áĭ Ľ +áĭ Ŀ +áĭ ³ +áĮ ģ +áĮ ħ +áĮ ¥ +áĮ ¦ +á Į¨ +áį Ĭ +áį į +áį ķ +áį ĸ +áį ¢ +áį ¤ +áİ Ĵ +áİ ª +áı ģ +áı IJ +áı Ł +áIJ Ĥ +áIJ ĸ +áIJ Ŀ +áIJ ŀ +áIJ Ł +áIJ ł +áij ĸ +áĴ ĭ +áĴ į +áĴ ¡ +áĵ « +áĶ ķ +áķ ĭ +áķ ij +áķ Ļ +áķ ļ +áķ Ľ +áķ ¤ +áķ ¦ +áķ ® +áķ ¼ +áĸ ĵ +áĹ Ĺ +áĹ ¢ +áĹ ¯ +áĹ · +áĺ Ħ +áĺ ij +ἠĤ +áĽ Ļ +áŀ į +áł Ĩ +áł ¡ +áł ¦ +áł ® +áł ¯ +áł ² +áł · +á¡ į +á¡ ŀ +á¡ ¤ +á ¡´ +á¡ µ +ᤠĵ +ᥠĸ +ᥠ° +ᨠ¦ +ᨠ§ +ᨠ¨ +ᨠª +ᨠ¬ +ᨠ¯ +ᨠ³ +ᨠµ +á© ĥ +ᬠķ +áŃ £ +á ± +á± ļ +á² ł +á´ ĵ +á´ ¶ +áµ Ĥ +áµ Į +áµ ¥ +áµ ´ +á¶ ĩ +á¸ Ī +Ḡł +Ḡ§ +Ḡ´ +Ḡ¾ +á¹ Ģ +á¹ ĸ +á¹ Ł +á¹ ł +á¹ « +á¹ ± +á¹ · +á¹ ¿ +ẠĦ +Ạį +Ạij +áº Ĺ +á¼ ī +á¼ ĵ +á¼ Ń +á½ ĭ +á½ Ĵ +á½ ł +á½ £ +á¾ Ħ +á¾ ı +á¾ ij +á¾ Ĺ +á¾ ¦ +á¾ § +á¾ ¾ +á¿ Ħ +á¿ ĵ +á¿ ¡ +á¿ ¬ +âģ ļ +âĤ Į +âĦ ģ +âĦ Ķ +âĦ £ +âĦ § +âĦ ¯ +âĦ ° +âĦ ´ +âħ ħ +âĨ ľ +âĨ « +âĨ Ń +âĨ ± +âĨ ¹ +âĨ ½ +âĩ ĩ +âĩ ľ +âĩ µ +âĪ ī +âĪ Ĭ +âĪ ĸ +âĪ ľ +âĪ ¾ +âī Ģ +âī ĭ +âī Į +âī ĵ +âī ľ +âī ´ +âī ¿ +âĬ Ĭ +âĬ ĭ +âĬ Ķ +âĬ ĸ +âĬ £ +âĬ ¦ +âĭ İ +âĭ ª +âĭ ² +âĮ ¦ +âĮ § +âį º +âİ Ī +âİ ¨ +âİ ¬ +âİ ³ +âİ ¼ +âİ ¾ +âı Į +âı ļ +âı « +âı ¯ +âı µ +âĴ ľ +âĴ Ŀ +âĴ « +âĵ Ħ +âĵ Ĭ +âĵ Ļ +âĵ © +âĶ ij +âĶ Ļ +âĶ ļ +âĶ ¥ +âķ ħ +âķ ī +âķ į +âķ ı +âķ ŀ +âĸ ļ +âĸ ¯ +âĹ ĥ +âĹ ļ +âĹ ¬ +âĹ ´ +âĺ Ī +âĺ ¤ +âĺ ¥ +âĺ § +âĺ ¬ +âĻ ģ +âĻ ± +âļ ĥ +âļ Ħ +âļ ħ +âļ ı +âļ ļ +âļ ŀ +âļ Ł +âļ ± +âļ ² +âľ Ģ +âľ Ł +âľ ¢ +âĿ µ +⣠¡ +⣠¦ +⣠§ +⣠³ +⣠¾ +⣠¿ +âł ĩ +⤠Ħ +⤠º +⥠Ĥ +⥠¹ +â§ ī +â§ ¼ +â§ ½ +⨠į +⬠Ĭ +â¬ Ł +âŃ ŀ +â® ŀ +â® ³ +â¯ Ī +⯠ij +â± ł +â± ± +â² Ń +â´ ¹ +âµ ķ +⸠¾ +â º« +â¼ Ĩ +â¼ ł +â½ Ł +â½ ¼ +â¾ Ľ +â¾ § +â¿ ĥ +â¿ » +ãĤ ķ +ãĤ Ł +ãĦ Ľ +ãĦ ¡ +ãĦ ¶ +ãĦ º +ãħ Ĵ +ãħ Ł +ãĨ Ģ +ãĩ » +ãĪ ij +ãĪ Ń +ãĪ ® +ãĪ ³ +ãĪ ¹ +ãī ¥ +ãī ¦ +ãī ¹ +ãī ¿ +ãĬ ŀ +ãĬ ¨ +ãĭ ij +ãĭ ¥ +ãĭ ´ +ãĭ º +ãİ Ħ +ãİ ķ +ãİ ¯ +ãı Ĥ +ãı Ī +ãı ĵ +ãı ĸ +ãı ± +ãIJ ± +ãŁ ģ +ã ¢ +㢠¨ +ã ¨ +㨠³ +ã« ª +ã« ´ +ã¶ ³ +㺠¾ +ä Ģ +äĢ Ģ +ä ĭ +äĭ Į +ä ĮĢ +äIJ Ģ +ä łĢ +ä ł +äł ¼ +ä § +ä§ ŀ +ä¨ ° +ä¨ º +ä ´Ģ +ä · +ä· ħ +ä ·¸ +ê Ĥ +êĤ « +ê Į +êĮ ¼ +ê į +êį ² +êĴ µ +ê ĵ +êĵ ½ +êĻ Ń +êĿ Ľ +êĿ ¥ +ê ŀ +êŀ Ĭ +ê¦ Ĩ +ê¦ ĩ +ê¦ Ł +ê¦ ¨ +ê§ Ī +ê © +ê© Ł +êª ĭ +êª ij +êª ķ +êª Ĺ +êª ľ +êª ® +êª ± +êª » +êª ¼ +ê« Ģ +ê« Ŀ +ê° ĥ +ê° ĺ +ê± ľ +ê² ĵ +ê² ļ +ê³ Ļ +ê³ ¾ +ê´ Ĺ +ê´ Ļ +êµ Ľ +ê¶ ĥ +ê¶ ķ +ê¶ ¨ +ê¸ © +ê¸ ¿ +ê ¹Ħ +ê¹ Ĩ +ê¹ ī +ê¹ ĵ +ê¹ ¢ +ê¹ £ +ê¹ ¸ +êº ³ +ê¿ ı +ê¿ ķ +ê¿ § +ëĢ © +ëģ ħ +ëĥ µ +ëĦ ĸ +ëĦ Ĺ +ëĦ ¢ +ëħ Ĥ +ëĨ IJ +ëĩ ľ +ëĪ ĭ +ëĪ ļ +ëī į +ëī ¨ +ëĬ ļ +ëĬ ¡ +ëĭ ľ +ëĭ ª +ëĮ ĺ +ëĮ ¤ +ëĮ ¸ +ëİ Ł +ëı ¨ +ëIJ Ħ +ëIJ ı +ëIJ ´ +ëIJ ¸ +ëij ģ +ëij ¿ +ëĴ ¨ +ëĵ · +ëĶ ® +ëĶ ² +ëķ § +ëĸ Ķ +ëĸ ª +ëĺ Ń +ëļ Ģ +ëļ ł +ëĽ Ķ +ëĽ © +ëľ ħ +ëŀ ķ +ëŀ ° +ëŁ IJ +ëł ¡ +ë¡ ŀ +ë¡ £ +ë¡ µ +ë£ Ħ +ë£ į +ë¤ ³ +ë¦ į +ë¦ ı +ë¦ ³ +ë§ Ħ +ë§ Ĩ +ë§ į +ë§ ľ +ë§ « +ë§ » +ë¨ ® +ë© Ĥ +ë© Ń +ëª ´ +ë¬ ľ +ë¬ ł +ë¬ « +ë¬ ¾ +ëŃ ¬ +ë® ĺ +ë® ¹ +ë¯ ķ +ë¯ ľ +ë° ¨ +ë° ª +ë± Ķ +ë² ĺ +ë² Ľ +ë² ± +ë² ´ +ë´ ½ +ëµ ¤ +ëµ ¨ +ë· Ĺ +ë· ĺ +ë¸ ĵ +ë¸ ľ +ë¹ ª +ëº ĥ +ëº ĺ +ëº µ +ë» ´ +ë¼ IJ +ë¾ Ķ +ìģ Ń +ìĤ ł +ìĤ ® +ìĥ ı +ìĥ Ļ +ìĦ º +ìħ ¢ +ìĨ Ģ +ìĨ ħ +ìĨ ¤ +ìĨ ¦ +ìĨ ¬ +ìĩ ± +ìĪ µ +ìĭ ¨ +ìĭ ´ +ìĮ ° +ìį ľ +ìİ Ĺ +ìİ ĺ +ìİ ¼ +ìij ī +ìij Ŀ +ìij » +ìĴ Ķ +ìĴ ¯ +ìĵ © +ìķ IJ +ìķ ĸ +ìĸ ł +ìĸ ¾ +ìĹ ĥ +ìĹ Ĺ +ìĹ ľ +ìĹ ¨ +ìĺ Ĥ +ìĺ Ħ +ìĺ ı +ìĺ ¾ +ìĺ ¿ +ìľ § +ìĿ IJ +ìĿ ĸ +ìĿ · +ìŀ į +ìŀ ı +ìŀ ¨ +ìŀ ª +ìŀ ³ +ìł ¡ +ìł ´ +ìł ¹ +ì¡ Ģ +ì¡ ª +ì¡ µ +ì¢ IJ +ì¢ ¨ +ì£ Į +ì£ Ļ +ì£ ³ +ì¦ ij +ì§ ¥ +ì§ ´ +ì§ ¾ +ì¨ ĵ +ì¨ ķ +ì© ° +ì© » +ì© ¼ +ìª Ĺ +ì¬ Ķ +ì¬ ĺ +ì® ® +ì¯ ķ +ì¯ ĺ +ì° İ +ì° ¯ +ì± ĥ +ì± µ +ì² § +ì² ® +ì² ¯ +ì³ ¬ +ì´ ĭ +ì´ ¢ +ìµ ¥ +ì¶ £ +ì¸ Ī +ì¸ Ļ +ìº ¤ +ìº Ń +ì» ½ +ì¼ Ļ +ì½ ¬ +ì¾ Ģ +ì¿ ħ +ì¿ ½ +íĢ ħ +íģ ¦ +íĤ ħ +íĥ ¶ +íĥ ¹ +íĦ Ķ +íħ £ +íĨ Ħ +íĨ § +íĨ ¹ +íĩ ¼ +íī ¤ +íĬ ½ +íĭ Ĥ +íĭ ij +íį Ī +íį Ļ +íį ¿ +íİ ¶ +íIJ Ŀ +íĴ ľ +íĵ Ŀ +íĵ ª +íĵ ± +íĵ · +íĵ ¼ +íĶ Ļ +íĶ ł +íķ ļ +íķ Ľ +íķ ŀ +íķ Ł +íķ § +íķ ¶ +íĸ Ĭ +íĸ ĭ +íĸ į +íĸ Ķ +íĸ ĺ +íĸ ¡ +íĸ ¬ +íĹ £ +íĹ ¿ +íĺ ĸ +íĺ Ń +íļ ° +íĽ į +íĽ ½ +íĿ Ł +íĿ Ń +íĿ ´ +íŀ ľ +ï¤ ī +ï¤ Ń +ï¤ ² +ï¤ µ +ï¤ ¼ +ï¥ Ģ +ï¥ ij +ï¥ Ĵ +ï¥ ķ +ï¥ ĺ +ï¥ Ļ +ï¥ « +ï¥ ¬ +ï¥ ° +ï ¥¿ +ï¦ ĭ +ï¦ ı +ï¦ Ķ +ï¦ ĸ +ï¦ ĺ +ï¦ Ľ +ï¦ ł +ï¦ ® +ï¦ ¯ +ï¦ º +ï¦ » +ï¦ ¾ +ï§ Ĩ +ï§ ĸ +ï§ Ľ +ï§ ŀ +ï§ Ł +ï§ § +ï§ ³ +ï§ º +ï§ ½ +ï¨ ĥ +ï¨ ļ +ï¨ ¢ +ï© Ł +ï¬ ¤ +ï¬ ¬ +ï¬ ¼ +ïŃ Ĵ +ïŃ ķ +ïŃ Ľ +ïŃ Ŀ +ïŃ ŀ +ïŃ Ł +ïŃ ¤ +ïŃ § +ïŃ ¨ +ïŃ ® +ïŃ ° +ïŃ ± +ïŃ · +ïŃ ¹ +ïŃ » +ï® Ģ +ï® ĥ +ï® Ħ +ï® ħ +ï® į +ï® Ĵ +ï® ĵ +ï® ķ +ï® ¦ +ï® ® +ï® ° +ï¯ ĵ +ï¯ ľ +ï¯ © +ï¯ ª +ï¯ ¬ +ï¯ Ń +ï¯ ® +ï¯ · +ï¯ ¹ +ï¯ » +ï¯ ¼ +ï° ĥ +ï° Į +ï° IJ +ï° ĺ +ï° Ļ +ï° ľ +ï° ŀ +ï° ¢ +ï° ® +ï° ° +ï° ¼ +ï° ¿ +ï± Ģ +ï± ģ +ï± Ī +ï± ĭ +ï± ı +ï± Ń +ï² Ģ +ï² ĩ +ï² Ī +ï² ĭ +ï² İ +ï² Ĵ +ï² ľ +ï² ł +ï² ¬ +ï² » +ï³ ĩ +ï³ Ķ +ï³ £ +ï³ « +ï´ ĺ +ï´ ° +ï´ ½ +ï ¶ +ï¶ ° +ï¸ ĸ +ï¸ ´ +ï¸ ¹ +ï¹ į +ï¹ Ĺ +ï¹ ¢ +ï¹ ¤ +ï¹ © +ï¹ ± +ï¾ ° +ï¿ Ĥ +ï¿ ® +ðIJĮ ° +ðIJĮ ¹ +ðIJĮ º +ðIJĮ ½ +ðIJį Ĥ +ðIJį ĥ +ðIJį Ħ +ðIJ İ +ðIJİ ¹ +ðIJ¤ Ĥ +ðIJ¤ į +ðIJ¤ ı +ðIJ¤ ĵ +ðIJŃ ī +ðIJŃ į +ðIJ° ĩ +ðIJ° ° +ðij Ĥ +ðijĤ Ħ +ðij ĺ +ðijĺ ģ +ðĴ Ģ +ðĴĢ ¸ +ðĴ ģ +ðĴģ º +ðĴ Ħ +ðĴĦ · +ðĴ Ĭ +ðĴĬ ij +ðĴ ĭ +ðĴĭ Ĺ +ð ĴĮ +ðĴĮ ¨ +ðĵĥ ¢ +ðĵĥ ° +ðĸ ł +ðĸł ļ +ðĿĦ ĥ +ðĿĦ ħ +ðĿĦ ķ +ðĿĦ Ļ +ðĿĦ ± +ðĿĦ ´ +ðĿĦ ¹ +ðĿħ İ +ðĿħ ª +ðĿĨ £ +ðĿĨ ³ +ðĿĨ ¹ +ðĿĩ Ĭ +ðĿĩ Ĺ +ðĿĩ ļ +ðĿĩ ľ +ðĿĩ ł +ðĿIJ ī +ðĿIJ ĸ +ðĿIJ ĺ +ðĿIJ £ +ðĿIJ ± +ðĿij Ĭ +ðĿij Ń +ðĿij ¼ +ðĿij ½ +ðĿĴ ° +ðĿĴ · +ðĿĴ ¿ +ðĿĵ ģ +ðĿĵ ĭ +ðĿĵ İ +ðĿĵ Ĵ +ðĿ ĵĺ +ðĿĵ ¢ +ðĿĵ ¦ +ðĿĵ « +ðĿĵ ¿ +ðĿĶ İ +ðĿĶ ± +ðĿĶ ´ +ðĿĶ · +ðĿĶ ¸ +ðĿĶ ½ +ðĿķ Ĥ +ðĿķ ĥ +ðĿķ ĭ +ðĿķ ı +ðĿķ IJ +ðĿķ ¥ +ðĿķ ´ +ðĿķ º +ðĿĸ IJ +ðĿĸ Ľ +ðĿĸ Ŀ +ðĿĸ ŀ +ðĿĹ © +ðĿĹ ³ +ðĿĹ ½ +ðĿĺ Ĭ +ðĿĺ ĭ +ðĿĺ Ķ +ðĿĺ ± +ðĿĺ ´ +ðĿĺ ¿ +ðĿĻ Ĵ +ðĿĻ Ŀ +ðĿĻ Ł +ðĿĻ ¬ +ðĿĻ Ń +ðĿĻ » +ðĿĻ ¾ +ðĿļ Ī +ðĿļ ĭ +ðĿļ ij +ðĿļ Ł +ðĿļ ł +ðĿļ £ +ðĿĽ ½ +ðĿľ Ĥ +ðĿľ Ķ +ðĿľ Ļ +ðŁ Ģ +ðŁĢ Ħ +ðŁĦ ² +ðŁĦ ¶ +ðŁħ IJ +ðŁħ ĸ +ðŁħ ļ +ðŁħ Ľ +ðŁħ ¦ +ðŁħ ¶ +ðŁħ » +ðŁħ ¼ +ðŁĨ ĥ +ðŁĨ Ĩ +ðŁĨ İ +ðŁĪ ¯ +ðŁĪ ² +ðŁĪ ¹ +ðŁĮ ĩ +ðŁĮ ĵ +ðŁį ĺ +ðŁİ ij +ðŁİ ¿ +ðŁı ı +ðŁı Ĵ +ðŁı © +ðŁı ¯ +ðŁIJ Ģ +ðŁij Ŀ +ðŁĴ ¹ +ðŁĴ º +ðŁĵ Ł +ðŁĵ ª +ðŁĵ ¼ +ðŁĶ Ģ +ðŁĶ Ĥ +ðŁĶ ĥ +ðŁĶ ĩ +ðŁĶ ĵ +ðŁĶ ¢ +ðŁĶ ¤ +ðŁĶ © +ðŁķ ĸ +ðŁķ ļ +ðŁķ ľ +ðŁķ Ŀ +ðŁķ ŀ +ðŁķ ł +ðŁķ ¢ +ðŁķ ³ +ðŁĸ ĩ +ðŁĸ ij +ðŁĸ ¶ +ðŁĹ ģ +Ñ ¨ +Ú İ +á¡ Į +Ḡ° +áº Ģ +á¼ ® +á½ Ŀ +âĦ ¬ +âļ § +⼠¤ +ã³ ¬ +êĻ ĭ +ê¸ ij +ëĶ ī +ëĹ į +ë¡ ij +ë¯ ij +ë» ħ +ë¼ Ŀ +ìĦ IJ +ìī ¡ +ìĭ ² +ìı ± +ìĹ ¤ +ìĿ © +ìĿ ¿ +ìŁ Ļ +ìł ° +ì¥ ī +íĬ Ń +íķ ® +ï® ı +ðŁħ ± +ðŁĨ Ĵ +ðŁķ ĭ +É ĺ +Ê ĵ +Õ ĥ +à´ ´ +འħ +áĨ º +áĪ Ĭ +áĪ ¨ +áĪ ¾ +áī IJ +áĮ ĥ +áĮ ½ +áĶ Ń +áł Ĥ +áł ¬ +ᨠ¸ +á© ĭ +á¶ ı +á¾ Ķ +á¿ IJ +á¿ ļ +âĻ Ļ +âļ Ĥ +âļ Ĺ +â¡ ¢ +⤠¦ +ëĸ ° +ë¤ Ĥ +ë§ ł +ë± ĭ +ë± IJ +ìĽ ¢ +ìľ ¾ +ì³ ħ +ì» ģ +íģ » +íĥ Ļ +íĵ ĸ +íĵ Ń +íķ ± +íĽ ľ +ï¤ ħ +ï¤ Ĩ +ï¦ ĥ +ï§ © +ï¨ Ĥ +ðIJ¤ Ķ +ðIJŃ ĵ +ðIJ° ¼ +ðĿĵ ŀ +ðĿĵ ° +ðĿĻ ľ +ðĿļ ģ +ðŁħ ¢ +ðŁı ĩ +È ² +Ê ¶ +Ô Ī +Ô ij +Ý ĵ +Ý ¥ +ठij +ॠ± +ଠī +à° ³ +à° µ +à² Ł +áĢ ı +áģ ¼ +áī ¨ +áĬ Ĵ +áĭ © +áĮ Ħ +áĮ Ķ +áIJ § +á ĴĮ +áĶ ħ +áĶ Ĭ +áł Ħ +ᨠģ +Ḡĥ +Ḡ» +âĶ ŀ +âĺ µ +âļ £ +â² ¢ +ãĪ ª +ä¶ µ +ê² Ļ +ê² ´ +ê³ Ĥ +ë¡ ¼ +ìĨ Ĭ +ì¼ ĩ +íĭ į +íĵ ¬ +íĵ ® +íĵ ¶ +íĵ » +ï¤ ¦ +ï¥ ł +ï¥ ± +ïŃ ² +ðIJŃ Ĭ +ðIJ ±ħ +ðĸ ¥ +ðĸ¥ ¨ +ðĿij ³ +ðĿĵ ķ +ðĿĵ ¬ +ðĿĵ ¹ +ðĿĵ ¾ +ðĿĶ ĵ +ðĿķ į +ðĿķ ¡ +ðĿķ ± +ðĿĸ ĸ +ðĿĺ ı +ðĿĺ IJ +ðĿĺ ļ +ðĿĻ ® +ðĿĻ ° +ðĿĻ ¸ +ðĿĻ º +ðĿĻ ¼ +ðĿĻ ½ +ðĿĻ ¿ +ðĿļ Ħ +ðĿļ ı +ðŁħ ħ +ðŁħ ĵ +Æ Ī +àł Į +áĻ ³ +á ļĮ +ἠħ +ἠIJ +ᤠĬ +ḠĬ +âĶ ½ +âķ Ĭ +⼠ĩ +⼠ı +âĿ ª +âĿ « +⣠° +ãĦ į +ãĦ ĵ +ãĦ § +ãħ ĸ +ãī « +ê¦ Ķ +ï± Ĭ +ຠĤ +áħ £ +á¥ Ķ +ᥠ¤ +âĨ ¤ +âĨ · +âĩ ŀ +âĸ ¤ +âŀ ¶ +ãĪ ¼ +ï¨ · +ðĵı § +âĶ ² +âĢ ´ +âĴ Ł +âĴ ¡ +â° Ĥ +â° į +â° İ +â° IJ +â° ij +â° Ł +â° ł +â° ¡ +â¼ Ń +ãĬ ¥ +âĴ ł +â½ º +ãĩ º +ãĩ ½ +ï¨ Ĭ +áķ · +âį ¨ +âº Ł +â½ Ĺ diff --git a/8B-Final/model-00001-of-00004.safetensors b/8B-Final/model-00001-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a81ad7d55c3cc48caf0dc0838e2ebad6884147ca --- /dev/null +++ b/8B-Final/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b4d685dada41f95f4819c7379aa9a8b972fd289bbb7852f7b07404e825bb146 +size 4900119584 diff --git a/8B-Final/model-00002-of-00004.safetensors b/8B-Final/model-00002-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6702711313a01553a2cf72e5b1ec707f6a06a8e8 --- /dev/null +++ b/8B-Final/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a64dca24aa2621614d2e8bf5aad957e03a1ecd1bcf255d3d48284306f1a12e5 +size 4915960368 diff --git a/8B-Final/model-00003-of-00004.safetensors b/8B-Final/model-00003-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ed6285059fdc0d60cb49b9203a9661852b09188 --- /dev/null +++ b/8B-Final/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6faad15cdd550c7128bd5fcdb7924f4837ea0d5788e4578d0768290048d4a6e7 +size 4983068496 diff --git a/8B-Final/model-00004-of-00004.safetensors b/8B-Final/model-00004-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9335c910a59750f0fcb120da60fb0e792c00c260 --- /dev/null +++ b/8B-Final/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67f6f7e76048838f33aeff339bebf445e37251ae818825cb42e79d0bf244ff59 +size 1578092152 diff --git a/8B-Final/model.safetensors.index.json b/8B-Final/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..d44734c3a04564c8b51a12abf4d5f4eea8e399f2 --- /dev/null +++ b/8B-Final/model.safetensors.index.json @@ -0,0 +1,406 @@ +{ + "metadata": { + "total_size": 16377194496 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/8B-Final/special_tokens_map.json b/8B-Final/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..2f354a35553e358107a20dcd5fac3aa160626006 --- /dev/null +++ b/8B-Final/special_tokens_map.json @@ -0,0 +1,22 @@ +{ + "additional_special_tokens": [ + "", + "", + "", + "" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/8B-Final/tokenizer.json b/8B-Final/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..04a20d9647f12faad384184b1a8f3ae14692f8ac --- /dev/null +++ b/8B-Final/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e0a5c807e135eba1d85b949b0dce8afd1e8906430f41670602df72c30b2379d +size 11423791 diff --git a/8B-Final/tokenizer_config.json b/8B-Final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ff1acbf47f5ad19d432c3a26998ed1bc570bfa43 --- /dev/null +++ b/8B-Final/tokenizer_config.json @@ -0,0 +1,279 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "", + "", + "", + "" + ], + "bos_token": null, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n\\n\\n\\n\\n' }}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/8B-Final/vocab.json b/8B-Final/vocab.json new file mode 100644 index 0000000000000000000000000000000000000000..4783fe10ac3adce15ac8f358ef5462739852c569 --- /dev/null +++ b/8B-Final/vocab.json @@ -0,0 +1 @@ +{"!":0,"\"":1,"#":2,"$":3,"%":4,"&":5,"'":6,"(":7,")":8,"*":9,"+":10,",":11,"-":12,".":13,"/":14,"0":15,"1":16,"2":17,"3":18,"4":19,"5":20,"6":21,"7":22,"8":23,"9":24,":":25,";":26,"<":27,"=":28,">":29,"?":30,"@":31,"A":32,"B":33,"C":34,"D":35,"E":36,"F":37,"G":38,"H":39,"I":40,"J":41,"K":42,"L":43,"M":44,"N":45,"O":46,"P":47,"Q":48,"R":49,"S":50,"T":51,"U":52,"V":53,"W":54,"X":55,"Y":56,"Z":57,"[":58,"\\":59,"]":60,"^":61,"_":62,"`":63,"a":64,"b":65,"c":66,"d":67,"e":68,"f":69,"g":70,"h":71,"i":72,"j":73,"k":74,"l":75,"m":76,"n":77,"o":78,"p":79,"q":80,"r":81,"s":82,"t":83,"u":84,"v":85,"w":86,"x":87,"y":88,"z":89,"{":90,"|":91,"}":92,"~":93,"¡":94,"¢":95,"£":96,"¤":97,"¥":98,"¦":99,"§":100,"¨":101,"©":102,"ª":103,"«":104,"¬":105,"®":106,"¯":107,"°":108,"±":109,"²":110,"³":111,"´":112,"µ":113,"¶":114,"·":115,"¸":116,"¹":117,"º":118,"»":119,"¼":120,"½":121,"¾":122,"¿":123,"À":124,"Á":125,"Â":126,"Ã":127,"Ä":128,"Å":129,"Æ":130,"Ç":131,"È":132,"É":133,"Ê":134,"Ë":135,"Ì":136,"Í":137,"Î":138,"Ï":139,"Ð":140,"Ñ":141,"Ò":142,"Ó":143,"Ô":144,"Õ":145,"Ö":146,"×":147,"Ø":148,"Ù":149,"Ú":150,"Û":151,"Ü":152,"Ý":153,"Þ":154,"ß":155,"à":156,"á":157,"â":158,"ã":159,"ä":160,"å":161,"æ":162,"ç":163,"è":164,"é":165,"ê":166,"ë":167,"ì":168,"í":169,"î":170,"ï":171,"ð":172,"ñ":173,"ò":174,"ó":175,"ô":176,"õ":177,"ö":178,"÷":179,"ø":180,"ù":181,"ú":182,"û":183,"ü":184,"ý":185,"þ":186,"ÿ":187,"Ā":188,"ā":189,"Ă":190,"ă":191,"Ą":192,"ą":193,"Ć":194,"ć":195,"Ĉ":196,"ĉ":197,"Ċ":198,"ċ":199,"Č":200,"č":201,"Ď":202,"ď":203,"Đ":204,"đ":205,"Ē":206,"ē":207,"Ĕ":208,"ĕ":209,"Ė":210,"ė":211,"Ę":212,"ę":213,"Ě":214,"ě":215,"Ĝ":216,"ĝ":217,"Ğ":218,"ğ":219,"Ġ":220,"ġ":221,"Ģ":222,"ģ":223,"Ĥ":224,"ĥ":225,"Ħ":226,"ħ":227,"Ĩ":228,"ĩ":229,"Ī":230,"ī":231,"Ĭ":232,"ĭ":233,"Į":234,"į":235,"İ":236,"ı":237,"IJ":238,"ij":239,"Ĵ":240,"ĵ":241,"Ķ":242,"ķ":243,"ĸ":244,"Ĺ":245,"ĺ":246,"Ļ":247,"ļ":248,"Ľ":249,"ľ":250,"Ŀ":251,"ŀ":252,"Ł":253,"ł":254,"Ń":255,"ĠĠ":256,"ĠĠĠĠ":257,"in":258,"Ġt":259,"ĠĠĠĠĠĠĠĠ":260,"er":261,"ĠĠĠ":262,"on":263,"Ġa":264,"re":265,"at":266,"st":267,"en":268,"or":269,"Ġth":270,"ĊĊ":271,"Ġc":272,"le":273,"Ġs":274,"it":275,"an":276,"ar":277,"al":278,"Ġthe":279,";Ċ":280,"Ġp":281,"Ġf":282,"ou":283,"Ġ=":284,"is":285,"ĠĠĠĠĠĠĠ":286,"ing":287,"es":288,"Ġw":289,"ion":290,"ed":291,"ic":292,"Ġb":293,"Ġd":294,"et":295,"Ġm":296,"Ġo":297,"ĉĉ":298,"ro":299,"as":300,"el":301,"ct":302,"nd":303,"Ġin":304,"Ġh":305,"ent":306,"id":307,"Ġn":308,"am":309,"ĠĠĠĠĠĠĠĠĠĠĠ":310,"Ġto":311,"Ġre":312,"--":313,"Ġ{":314,"Ġof":315,"om":316,");Ċ":317,"im":318,"čĊ":319,"Ġ(":320,"il":321,"//":322,"Ġand":323,"ur":324,"se":325,"Ġl":326,"ex":327,"ĠS":328,"ad":329,"Ġ\"":330,"ch":331,"ut":332,"if":333,"**":334,"Ġ}":335,"em":336,"ol":337,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":338,"th":339,")Ċ":340,"Ġ{Ċ":341,"Ġg":342,"ig":343,"iv":344,",Ċ":345,"ce":346,"od":347,"Ġv":348,"ate":349,"ĠT":350,"ag":351,"ay":352,"Ġ*":353,"ot":354,"us":355,"ĠC":356,"Ġst":357,"ĠI":358,"un":359,"ul":360,"ue":361,"ĠA":362,"ow":363,"Ġ'":364,"ew":365,"Ġ<":366,"ation":367,"()":368,"Ġfor":369,"ab":370,"ort":371,"um":372,"ame":373,"Ġis":374,"pe":375,"tr":376,"ck":377,"âĢ":378,"Ġy":379,"ist":380,"----":381,".ĊĊ":382,"he":383,"Ġe":384,"lo":385,"ĠM":386,"Ġbe":387,"ers":388,"Ġon":389,"Ġcon":390,"ap":391,"ub":392,"ĠP":393,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":394,"ass":395,"int":396,">Ċ":397,"ly":398,"urn":399,"Ġ$":400,";ĊĊ":401,"av":402,"port":403,"ir":404,"->":405,"nt":406,"ction":407,"end":408,"Ġde":409,"ith":410,"out":411,"turn":412,"our":413,"ĠĠĠĠĠ":414,"lic":415,"res":416,"pt":417,"==":418,"Ġthis":419,"Ġwh":420,"Ġif":421,"ĠD":422,"ver":423,"age":424,"ĠB":425,"ht":426,"ext":427,"=\"":428,"Ġthat":429,"****":430,"ĠR":431,"Ġit":432,"ess":433,"ĠF":434,"Ġr":435,"os":436,"and":437,"Ġas":438,"ect":439,"ke":440,"rom":441,"Ġ//":442,"con":443,"ĠL":444,"(\"":445,"qu":446,"lass":447,"Ġwith":448,"iz":449,"de":450,"ĠN":451,"Ġal":452,"op":453,"up":454,"get":455,"Ġ}Ċ":456,"ile":457,"Ġan":458,"ata":459,"ore":460,"ri":461,"Ġpro":462,";čĊ":463,"ĉĉĉĉ":464,"ter":465,"ain":466,"ĠW":467,"ĠE":468,"Ġcom":469,"Ġreturn":470,"art":471,"ĠH":472,"ack":473,"import":474,"ublic":475,"Ġor":476,"est":477,"ment":478,"ĠG":479,"able":480,"Ġ-":481,"ine":482,"ill":483,"ind":484,"ere":485,"::":486,"ity":487,"Ġ+":488,"Ġtr":489,"elf":490,"ight":491,"('":492,"orm":493,"ult":494,"str":495,"..":496,"\",":497,"Ġyou":498,"ype":499,"pl":500,"Ġnew":501,"Ġj":502,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":503,"Ġfrom":504,"Ġex":505,"ĠO":506,"ld":507,"Ġ[":508,"oc":509,":Ċ":510,"Ġse":511,"Ġle":512,"--------":513,".s":514,"{Ċ":515,"',":516,"ant":517,"Ġat":518,"ase":519,".c":520,"Ġch":521,"":589,"ust":590,"que":591,"Ġres":592,"))":593,"'s":594,"Ġk":595,"ans":596,"yst":597,"unction":598,"********":599,"Ġi":600,"Ġus":601,"pp":602,"one":603,"ail":604,"====":605,"name":606,"Ġstr":607,"Ġ/":608,"Ġ&":609,"ach":610,"div":611,"ystem":612,"ell":613,"Ġhave":614,"err":615,"ould":616,"ull":617,"pon":618,"ĠJ":619,"_p":620,"Ġ==":621,"ign":622,"St":623,".Ċ":624,"Ġpl":625,");ĊĊ":626,"form":627,"put":628,"ount":629,"}ĊĊ":630,"dd":631,"ite":632,"Ġget":633,"rr":634,"ome":635,"ĠâĢ":636,"aram":637,"cc":638,"Ġ*/":639,"ER":640,"In":641,"les":642,"_s":643,"ong":644,"ie":645,"Ġcan":646,"ĠV":647,"erv":648,"pr":649,"Ġun":650,"row":651,"ber":652,"Ġdo":653,"ll":654,"Ġel":655,"Ġself":656,"ated":657,"ary":658,"Ġ.":659,"']":660,"ud":661,"Ġen":662,"ĠTh":663,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":664,"te":665,"_c":666,"uct":667,"Ġab":668,"ork":669,".get":670,"Ġ#":671,"aw":672,"ress":673,"ob":674,"Name":675,"app":676,"['":677,"Ġall":678,"ory":679,"ition":680,"ance":681,"ear":682,"Ġcont":683,"vent":684,"ia":685,"Ġwill":686,"IN":687,"ĠĠĠĠĠĠĠĠĠ":688,"return":689,"Ġ":755,"\",Ċ":756,"ec":757,"ĠIn":758,"ph":759,"Ġ|":760,"_f":761,"Ġvar":762,"ence":763,"Id":764,"ree":765,"ink":766,"lect":767,"ug":768,"eth":769,"Ġelse":770,"----------------":771,"cont":772,"Ġso":773,"atic":774,"Ġlo":775,"pro":776,"ton":777,"ss":778,"own":779,"abel":780,"oint":781,"ous":782,"eld":783,"ST":784,"The":785,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":786,"RE":787,"\":":788,"olor":789,"tp":790,"eg":791,"key":792,"ude":793,"ĠSt":794,"ound":795,"Ġar":796,"\");Ċ":797,"ener":798,"ser":799,"bject":800,"essage":801,"fer":802,"Ġmore":803,"ations":804,"ents":805,"Ġhis":806,"Ġthey":807,".S":808,"ĠY":809,"use":810,"ne":811,"ish":812,"old":813,"_d":814,"io":815,"ield":816,"Ġper":817,"Cont":818,"ings":819,"####":820,"Ġdata":821,"Ġsa":822,"ef":823,"fo":824,"Ġone":825,"eng":826,"Ġdis":827,"AT":828,"Ġname":829,"Ġtrue":830,"val":831,"led":832,".f":833,"Ġne":834,"Ġend":835,".T":836,"cre":837,"ark":838,"log":839,"Ex":840,"error":841,"_id":842,"urre":843,"ange":844,"Ġnull":845,"rray":846,"Ġmy":847,"pan":848,"ict":849,"ator":850,"View":851,"List":852,"ĉreturn":853,"âĢĿ":854,"Ġpre":855,"Ġx":856,"clude":857,"arg":858,"ov":859,".h":860,"Ġ>":861,"Ġtheir":862,"')":863,"irst":864,"ick":865,"gh":866,"LE":867,"OR":868,"Ġprivate":869,"tem":870,"čĊčĊ":871,"user":872,"Ġ)":873,"com":874,".A":875,"\";Ċ":876,"Ġid":877,"read":878,"Ġwho":879,"_b":880,"\">Ċ":881,"Ġtime":882,"Ġman":883,"ry":884,"========":885,"roup":886,"rop":887,"public":888,"vel":889,"umber":890,"ble":891,"Ġwhich":892,"****************":893,"Ġany":894,"Ġfalse":895,"we":896,"Ġvalue":897,"Ġli":898,"\")":899,"nder":900,"gr":901,"Ġno":902,"param":903,"fig":904,".com":905,"Ġapp":906,"_l":907,"ions":908,".D":909,"ĠCh":910,"Ġabout":911,"Ġadd":912,"Ġsu":913,"Ġstring":914,"ID":915,"Ġover":916,"string":917,".l":918,"ource":919,"_C":920,"]Ċ":921,"Ġqu":922,"ĠString":923,"ca":924,"SE":925,"Ġro":926,"sh":927,"ual":928,"Type":929,"son":930,"new":931,"ern":932,"Ġag":933,"AR":934,"];Ċ":935,"].":936,"Ġ?":937,"ical":938,"Ġdes":939,"uth":940,"ix":941,"ays":942,"Ġtype":943,"'t":944,"ault":945,"Ġinter":946,"var":947,".b":948,"Ġpart":949,".d":950,"urrent":951,"IT":952,"EN":953,"enc":954,"(f":955,"ra":956,"value":957,"cho":958,"utton":959,"ose":960,"Ġ!=":961,"ater":962,"é":963,"reate":964,"oll":965,"pos":966,"yle":967,"ng":968,"AL":969,"using":970,"ames":971,"Ġ{čĊ":972,"ates":973,"ely":974,"Ġwork":975,"Ġem":976,"inal":977,"Ġsp":978,"Ġwhen":979,".set":980,"ĠĠĠĠĠĠ":981,"):Ċ":982,"to":983,"quire":984,"indow":985,"lement":986,"pect":987,"ash":988,"[i":989,"Ġuse":990,".F":991,"pec":992,"Ġad":993,"ove":994,"ception":995,"ength":996,"include":997,"ader":998,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":999,"atus":1000,"Th":1001,"itle":1002,"rit":1003,"void":1004,"().":1005,"(Ċ":1006,"Ġoff":1007,"Ġother":1008,"Ġ&&":1009,"';Ċ":1010,"ms":1011,"Ġbeen":1012,"Ġte":1013,"ml":1014,"co":1015,"nc":1016,"ervice":1017,"Ġ%":1018,"**Ċ":1019,"ann":1020,"ade":1021,"ĊĊĊĊ":1022,"lock":1023,"const":1024,"ponse":1025,"Ġsup":1026,"++":1027,"date":1028,"Ġacc":1029,"Ġhad":1030,"Ġbu":1031,"ĠRe":1032,"Ġwere":1033,"Ġfile":1034,"Ġwould":1035,"ĠâĢľ":1036,"ven":1037,"iss":1038,"Ġour":1039,"class":1040,"raw":1041,"Ġyear":1042,"Data":1043,"Ġval":1044,"Ġsome":1045,"fter":1046,"ys":1047,"Ġ///":1048,"round":1049,"view":1050,"Ġpe":1051,"Ġthere":1052,"Ġsaid":1053,"du":1054,"of":1055,"line":1056,"/*":1057,"duct":1058,"Ġher":1059,"ĠĠĠĠĠĠĠĠĠĠĠĠĠ":1060,"Res":1061,"Ġco":1062,"Ġcomm":1063,"ise":1064,"min":1065,"ĠĠĠĠĊ":1066,"#include":1067,"ethod":1068,".P":1069,"ute":1070,"Ġass":1071,"Int":1072,"ask":1073,"loc":1074,"Ġlike":1075,"ody":1076,"Ġlet":1077,"load":1078,"Ġam":1079,"rol":1080,"Ġgr":1081,"yp":1082,"Ġalso":1083,"ĠIt":1084,"url":1085,"ific":1086,"ors":1087,"_P":1088,"_n":1089,"igh":1090,"Ġthan":1091,"Com":1092,"AN":1093,"UL":1094,"ating":1095,"ĠThis":1096,"ref":1097,"_S":1098,"Ġstatic":1099,"roll":1100,"Ġjust":1101,"Ġresult":1102,"ian":1103,"idth":1104,"Ġthem":1105,"));Ċ":1106,"der":1107,"reak":1108,"Con":1109,"://":1110,"ule":1111,"...":1112,"arch":1113,"ement":1114,"Ġ<<":1115,"ush":1116,"ense":1117,"arr":1118,"Ġinto":1119,"cess":1120,"amp":1121,"ied":1122,"ument":1123,"Ġ\\":1124,"],":1125,"wo":1126,"als":1127,"Ġwhat":1128,"anc":1129,"Value":1130,"='":1131,"olum":1132,"Ġpos":1133,"ages":1134,"ayer":1135,"Ġsc":1136,"ues":1137,"\")Ċ":1138,"_T":1139,"Ġlist":1140,"(s":1141,"Ġcase":1142,"Ch":1143,"ĉĉĉĉĉ":1144,"////////":1145,"ponent":1146,"Ġz":1147,"Ġkn":1148,"let":1149,"DE":1150,"red":1151,"Ġfe":1152,"Ġ},Ċ":1153,"Ġ,":1154,"(t":1155,"Ġfirst":1156,"');Ċ":1157,"word":1158,"Ġimport":1159,"Ġact":1160,"Ġchar":1161,"CT":1162,"ĠTr":1163,"ople":1164,"={":1165,"ĉf":1166,"ient":1167,"cent":1168,".j":1169,"lection":1170,"))Ċ":1171,"Ġonly":1172,"Ġprint":1173,"mer":1174,".W":1175,"ock":1176,"Ġ--":1177,"Text":1178,"Ġop":1179,"ank":1180,"Ġits":1181,"Ġback":1182,"[\"":1183,"Ġneed":1184,"Ġcl":1185,"Ġsub":1186,"Ġla":1187,"((":1188,".\"":1189,"Object":1190,"Ġstart":1191,"file":1192,"(self":1193,"ner":1194,"ey":1195,"Ġuser":1196,"Ġent":1197,"ĠCom":1198,"its":1199,"ĠCon":1200,"ouble":1201,"ower":1202,"item":1203,"very":1204,"ĠWe":1205,"lick":1206,"ĠQ":1207,"php":1208,"ttp":1209,"':":1210,"ics":1211,"Ġunder":1212,"Ġ*Ċ":1213,".L":1214,");":1215,"ices":1216,"Ġreg":1217,")čĊ":1218,"ĉpublic":1219,"SS":1220,"Ġthen":1221,"reat":1222,"ious":1223,".G":1224,"ek":1225,"irect":1226,"heck":1227,"cript":1228,"ning":1229,"ĠUn":1230,"Ġmay":1231,"ĠWh":1232,"Bo":1233,"Item":1234,"struct":1235,".st":1236,"ream":1237,"ible":1238,"loat":1239,"Ġorg":1240,"und":1241,"sum":1242,"_in":1243,"../":1244,"_M":1245,"Ġhow":1246,"rite":1247,"'Ċ":1248,"To":1249,"ww":1250,"Ġpeople":1251,"index":1252,".n":1253,"http":1254,"(m":1255,"ector":1256,"Ġind":1257,"Ġjav":1258,"],Ċ":1259,"ĠHe":1260,"_st":1261,"ful":1262,"ole":1263,"){Ċ":1264,"Ġshould":1265,"opy":1266,"elp":1267,"ier":1268,"_name":1269,"erson":1270,"ION":1271,"ote":1272,"Ġtest":1273,"Ġbet":1274,"rror":1275,"ular":1276,"ãĢ":1277,"ĠÐ":1278,"bs":1279,"ting":1280,"Ġmake":1281,"Tr":1282,"Ġafter":1283,"arget":1284,"RO":1285,"olumn":1286,"rc":1287,"_re":1288,"define":1289,"Ġright":1290,"right":1291,"day":1292,"Ġlong":1293,"[]":1294,"(p":1295,"td":1296,"cond":1297,"ĠPro":1298,"Ġrem":1299,"ptions":1300,"vid":1301,".g":1302,"Ġext":1303,"Ġ__":1304,"')Ċ":1305,"pace":1306,"mp":1307,"Ġmin":1308,"stance":1309,"air":1310,"action":1311,"wh":1312,"type":1313,"util":1314,"ait":1315,"ĊĊ":1339,"Ġshe":1340,"\"]":1341,"aph":1342,"Ġexp":1343,"erty":1344,"ĠSe":1345,"Ġpar":1346,"unc":1347,"ET":1348,"Ġread":1349,"print":1350,"Ġrel":1351,"Ġform":1352,"Ġdr":1353,"Exception":1354,"input":1355,"Ġtrans":1356,"########":1357,"order":1358,"By":1359,"Ġaw":1360,"ities":1361,"uff":1362,"play":1363,".add":1364,"ĠâĢĵ":1365,"Ġwant":1366,"Ġcomp":1367,"ments":1368,"Ġ||":1369,"az":1370,"be":1371,"Ġnumber":1372,"Ġrequire":1373,"ĠEx":1374,"Ġcol":1375,"Ġkey":1376,"ember":1377,"Ġtwo":1378,"Ġsize":1379,"Ġwhere":1380,"UT":1381,"result":1382,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":1383,"ough":1384,"orld":1385,"ood":1386,"uch":1387,"ative":1388,"ger":1389,"arent":1390,"Ġ/*":1391,"Ġarg":1392,"Ġwhile":1393,"(this":1394,"Ġrec":1395,"Ġdif":1396,"State":1397,"Ġspec":1398,"ride":1399,"_F":1400,"Ġlook":1401,"AM":1402,"ility":1403,"eter":1404,"âĢĻt":1405,"ĊĊĊ":1406,"ayout":1407,"--------------------------------":1408,"ager":1409,"Ġcould":1410,"Ġbr":1411,"ends":1412,"ures":1413,"Ġknow":1414,"ets":1415,"ĠIf":1416,"ĠSh":1417,".w":1418,"back":1419,"Ġser":1420,"Ġ+=":1421,"Ġfr":1422,"());Ċ":1423,"Ġhand":1424,"Ind":1425,"ULL":1426,"Im":1427,"();ĊĊ":1428,"Ġmost":1429,"Ġtry":1430,"Ġnow":1431,"rough":1432,">čĊ":1433,"ackage":1434,"Ġhim":1435,"._":1436,"ify":1437,"Ġbreak":1438,"Ġ);Ċ":1439,"ren":1440,"#define":1441,"itt":1442,"Ġap":1443,"ĉc":1444,"(n":1445,"ĠYou":1446,":ĊĊ":1447,"-m":1448,"Ġevery":1449,"ustom":1450,"lient":1451,"ocument":1452,"cription":1453,"Error":1454,"-b":1455,"о":1456,"][":1457,"trans":1458,"Ġpoint":1459,"Ġstd":1460,"Ġfil":1461,"Time":1462,"Ġmod":1463,"Ġ->":1464,"Ġerror":1465,"ah":1466,"Ġtext":1467,"roller":1468,"lose":1469,"ql":1470,"Ġpol":1471,"><":1784,".B":1785,"-c":1786,"Ġopen":1787,"Ġest":1788,"ĠĠĠĠĠĠĠĠĊ":1789,"Ġnext":1790,"IM":1791,"ÑĤ":1792,"OT":1793,"ó":1794,"Ġfollow":1795,"content":1796,"ĠĠĠĠĠĠĠĠĠĠĠĠ":1797,"Ġinclud":1798,"HE":1799,"ĠRes":1800,"Ġhref":1801,"и":1802,"Ġcar":1803,"ypes":1804,"image":1805,"Un":1806,"Ġbool":1807,"AD":1808,"Ġgame":1809,".Form":1810,"rows":1811,"*/":1812,"velop":1813,".Drawing":1814,"Ġpath":1815,"ision":1816,"Ġeach":1817,"ĠPl":1818,"_type":1819,"Path":1820,"nection":1821,"Ġav":1822,"').":1823,"Ġsupport":1824,"ENT":1825,"rem":1826,"\").":1827,"Ġown":1828,"Ġcor":1829,"count":1830,"miss":1831,"ually":1832,"Ġmem":1833,"std":1834,"ience":1835,"search":1836,"\"ĊĊ":1837,"Form":1838,"Ġsex":1839,"ename":1840,"Ġsign":1841,"Ġet":1842,"ĠĠĠĠĠĠĠĠĠĠ":1843,"','":1844,"ĠApp":1845,"Ġthose":1846,"off":1847,"Ġerr":1848,"Ġsystem":1849,"Ġbest":1850,"code":1851,"Ġsame":1852,"Ġdi":1853,"uss":1854,"Ġcreate":1855,"ather":1856,"Array":1857,".in":1858,"fe":1859,"Service":1860,"UN":1861,"ats":1862,"ĠZ":1863,"alth":1864,"Ġmade":1865,"true":1866,"AB":1867,"Ġmark":1868,"rid":1869,"ified":1870,",čĊ":1871,"yn":1872,"press":1873,"Ġgroup":1874,"Ġfin":1875,"ĠLicense":1876,"Field":1877,"eger":1878,"Ġworld":1879,"iness":1880,"ty":1881,"Ġprocess":1882,"(b":1883,"Ġcre":1884,"arn":1885,"ives":1886,"Ġmain":1887,"ideo":1888,"_g":1889,"AG":1890,"valid":1891,"img":1892,"PI":1893,"Ġcolor":1894,"Ġreport":1895,"Ġtake":1896,"rib":1897,"OM":1898,"Ġday":1899,"Request":1900,"Ġsk":1901,"bers":1902,"ĉs":1903,".Add":1904,"oot":1905,"Image":1906,"Ġcomple":1907,"ollection":1908,"Ġtop":1909,"Ġfree":1910,"AS":1911,"De":1912,"ĠOn":1913,"IG":1914,"eta":1915,"Date":1916,"Ġaction":1917,"Over":1918,"itor":1919,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":1920,"not":1921,"Ġindex":1922,"her":1923,"icon":1924,"On":1925,";čĊčĊ":1926,"ivity":1927,"mand":1928,".Windows":1929,"OL":1930,"Ġreal":1931,"Ġmax":1932,"land":1933,"....":1934,"raph":1935,"Ġbuild":1936,"leg":1937,"assword":1938,"?ĊĊ":1939,"â̦":1940,"ook":1941,"uck":1942,"Ġmessage":1943,"test":1944,"ivers":1945,"Ġinput":1946,"Ġart":1947,"Ġbetween":1948,"Get":1949,"enter":1950,"ground":1951,"ene":1952,"á":1953,".length":1954,"Node":1955,"(i":1956,"Class":1957,"for":1958,"ĠâĢĶ":1959,"ten":1960,"oin":1961,"Ġke":1962,"ui":1963,"ĠIN":1964,"Ġtable":1965,"sub":1966,"ĠLe":1967,"Ġhead":1968,"Ġmust":1969,"////////////////":1970,".util":1971,"Context":1972,"Ġorder":1973,"Ġmov":1974,"over":1975,"Ġcontin":1976,"Ġsay":1977,"static":1978,".Text":1979,"ĠclassName":1980,"pany":1981,"Ġter":1982,"head":1983,"rg":1984,"Ġproduct":1985,"This":1986,".âĢĿ":1987,"ĠBut":1988,"loy":1989,"Ġdouble":1990,"sg":1991,"Ġplace":1992,".x":1993,"message":1994,"Ġinformation":1995,"private":1996,"Ġoper":1997,"ced":1998,"db":1999,"\">":2179,"aterial":2180,"iled":2181,"Ġput":2182,"Qu":2183,"ÑĢ":2184,"ung":2185,"map":2186,"ĉĉĉĉĉĉĉĉ":2187,"Ġlevel":2188,"Component":2189,"book":2190,"creen":2191,"_RE":2192,"Ġconfig":2193,"ãģ":2194,"Or":2195,".data":2196,"Ġdocument":2197,"\",\"":2198,"tribute":2199,"ux":2200,"Log":2201,"ference":2202,"post":2203,"_e":2204,"Ġlocal":2205,"andom":2206,"assert":2207,"Val":2208,"lected":2209,"ina":2210,"atabase":2211,"Add":2212,"Ġcontent":2213,".print":2214,"signed":2215,"ric":2216,".\"ĊĊ":2217,"Ġfa":2218,"!ĊĊ":2219,"-f":2220,"ived":2221,"Ġquest":2222,".ex":2223,"Ġfloat":2224,"Ġdevelop":2225,"оÐ":2226,"Map":2227,"ading":2228,"Ġposs":2229,"UE":2230,"namespace":2231,"_O":2232,"ĉb":2233,".Get":2234,">(":2235,"json":2236,"etails":2237,"Ġtoo":2238,"Ġextends":2239,"ĠNone":2240,"Ġfore":2241,"(String":2242,"format":2243,"Ġgreat":2244,"inter":2245,"cale":2246,"Ñģ":2247,"ron":2248,"iving":2249,"Ent":2250,"ency":2251,"xt":2252,"oy":2253,"Ġmonth":2254,"Ġhapp":2255,"Ġsuper":2256,"bar":2257,"default":2258,"_de":2259,"ords":2260,"ln":2261,"({Ċ":2262,"ĠInd":2263,"ases":2264,"Ġtitle":2265,"Ġcontext":2266,"oh":2267,"-p":2268,"Em":2269,"Ġmet":2270,"Test":2271,"Ġlife":2272,"_v":2273,"ĠUS":2274,"UI":2275,"ocation":2276,"md":2277,"Ġ[Ċ":2278,"Ġ]":2279,"sw":2280,"Ġincre":2281,"script":2282,"ential":2283,"ways":2284,".de":2285,"Ġsrc":2286,"Ġcatch":2287,"ĠAmeric":2288,"//Ċ":2289,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":2290,"Ġpay":2291,"plit":2292,"âĢĶ":2293,"Ġcoun":2294,"obj":2295,".php":2296,"Ġchange":2297,"ething":2298,"'re":2299,"aster":2300,"los":2301,"lation":2302,"ĠĠĊ":2303,"Le":2304,"ä":2305,"({":2306,"ready":2307,"ĠNo":2308,"Ġposition":2309,"Ġold":2310,"Ġbook":2311,"abled":2312,"bug":2313,"Hand":2314,"};ĊĊ":2315,"isplay":2316,"aving":2317,"Ġgover":2318,"Ġversion":2319,"System":2320,"nect":2321,"response":2322,"Style":2323,"Up":2324,"angu":2325,"Ġthree":2326,"init":2327,"ero":2328,"Ġlaw":2329,"endif":2330,"Ġbase":2331,"email":2332,"(l":2333,"_V":2334,"Ġconf":2335,"ATE":2336,"Ġduring":2337,"tes":2338,"Ġconsole":2339,"ĠPr":2340,"Ġspe":2341,"ves":2342,"path":2343,"ialog":2344,"dition":2345,"_to":2346,"ards":2347,"Ġagainst":2348,"etwork":2349,"ĠPh":2350,"_L":2351,"cur":2352,"imit":2353,"With":2354,"Ġpower":2355,"ium":2356,"';ĊĊ":2357,"Ġwom":2358,"left":2359,"ources":2360,"atri":2361,"ĠIm":2362,"ĠMan":2363,"orth":2364,"${":2365,"quals":2366,"ese":2367,"_size":2368,"Ġiss":2369,"otal":2370,"-g":2371,"ique":2372,"rame":2373,"Ġwidth":2374,"erg":2375,")(":2376,"ittle":2377,"TR":2378,"ĠThey":2379,"ences":2380,"rl":2381,"ons":2382,"Ġlabel":2383,".y":2384,"-t":2385,"update":2386,"anel":2387,"sc":2388,".to":2389,"Ġproject":2390,"ü":2391,"Ġelement":2392,"Ġsuccess":2393,"ĉĉĊ":2394,".sh":2395,"ram":2396,"ched":2397,"())Ċ":2398,"Ġ(Ċ":2399,"Ġdate":2400,"Ġtot":2401,"_ST":2402,"All":2403,"ification":2404,"ĉvar":2405,"Ġtri":2406,"chem":2407,"my":2408,"Ġbig":2409,"ĠAd":2410,"ĠAt":2411,"ots":2412,"num":2413,"Act":2414,"Ġmap":2415,"era":2416,"cope":2417,".$":2418,",âĢĿ":2419,"Ġpop":2420,"Ġfew":2421,"Ġlen":2422,"uid":2423,"eters":2424,"ules":2425,"ÃŃ":2426,"source":2427,"https":2428,"Ġdem":2429,"Ġear":2430,"################":2431,"Ġmatch":2432,"ories":2433,"aces":2434,"ĠCl":2435,"Ġnode":2436,"irc":2437,"local":2438,"unity":2439,"};Ċ":2440,"Ġanother":2441,"<<":2442,"ogle":2443,"Ġsit":2444,"ework":2445,"TE":2446,".I":2447,"NS":2448,"ology":2449,"ought":2450,".Cont":2451,">>":2452,"Ġcare":2453,"state":2454,"ĉprivate":2455,"Ġeffect":2456,"++)":2457,"_file":2458,"ending":2459,"Line":2460,"For":2461,"ior":2462,"ĠSc":2463,"Ġfun":2464,".Size":2465,"ĉelse":2466,"])":2467,"start":2468,"vious":2469,"Ġ},":2470,"ours":2471,"Ġleg":2472,"Ġservice":2473,"Ġsince":2474,"iron":2475,"Label":2476,"Ġnon":2477,"Ġlos":2478,"iction":2479,"Ġfull":2480,"acter":2481,"board":2482,"gress":2483,"Ġturn":2484,"ither":2485,".size":2486,"Ġbody":2487,"resh":2488,"eturn":2489,"(_":2490,"yles":2491,"ormal":2492,"pi":2493,"Ġsomething":2494,"!--":2495,"uint":2496,"Ġprodu":2497,"Ġstand":2498,"Ġproble":2499,"Ġavailable":2500,"mt":2501,"ĠBl":2502,"Ġ...":2503,"Ġblock":2504,"Input":2505,"Ġkeep":2506,"Count":2507,"open":2508,"Ġ['":2509,"Ġthrow":2510,"uilder":2511,"Action":2512,"Ġthings":2513,"True":2514,"Ġurl":2515,"ĠBo":2516,"printf":2517,"Ġred":2518,"js":2519,".create":2520,"ĠOr":2521,"Status":2522,"Instance":2523,"Ġcontrol":2524,"Ġcome":2525,"Ġcustom":2526,"location":2527,"model":2528,"ĠčĊ":2529,"Ġsource":2530,"Ġeas":2531,".out":2532,"]ĊĊ":2533,"oney":2534,"Ġawait":2535,"Ġpartic":2536,"AP":2537,"ublish":2538,"odes":2539,"_pro":2540,"ply":2541,"riter":2542,"Ġprov":2543,"Ġmill":2544,"HT":2545,"])Ċ":2546,"Ġchang":2547,"Ġask":2548,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":2549,"Ġoutput":2550,"Ġemail":2551,".push":2552,"Ġ}čĊčĊ":2553,"ination":2554,"atrix":2555,"Table":2556,"uccess":2557,"]);Ċ":2558,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":2559,"Ġdisc":2560,"([":2561,"Ġbusiness":2562,"height":2563,".html":2564,"ta":2565,"field":2566,"Ġrequired":2567,"_R":2568,"Ġgovern":2569,"}čĊčĊ":2570,"lex":2571,".,":2572,"ĠSet":2573,"urch":2574,"///":2575,"ts":2576,"af":2577,"Ġmight":2578,"istory":2579,"Str":2580,"Ġnever":2581,"Response":2582,"arse":2583,"ada":2584,"ĠHow":2585,"Ġ*)":2586,"Ġ;":2587,"Ġhard":2588,"Ad":2589,"Ġintern":2590,"used":2591,"(data":2592,"mod":2593,"annel":2594,"Ġnp":2595,"ugg":2596,"Ġ/>Ċ":2597,"Ġcalled":2598,"body":2599,"Ġcho":2600,"(r":2601,"_set":2602,"ird":2603,"Ġ>=":2604,"Ġ};Ċ":2605,"Ġoptions":2606,"ĠGener":2607,"Ġheight":2608,"Point":2609,"You":2610,"ety":2611,"Click":2612,"Ġsmall":2613,"Ġide":2614,"Ġaccess":2615,"anguage":2616,"Ġprotected":2617,"Ġjob":2618,"ĠThere":2619,"Def":2620,"Ġaddress":2621,"Ġuint":2622,"Not":2623,"oo":2624,"aps":2625,"":2759,"ĉĠĠĠ":2760,"\"))":2761,"Content":2762,"_W":2763,"plement":2764,"Ġwon":2765,"Ġvideo":2766,"adi":2767,"point":2768,"%%":2769,"Ġgl":2770,"erved":2771,"viron":2772,"IF":2773,"uted":2774,"ãĥ":2775,"'m":2776,"Ġcert":2777,"Ġprof":2778,"Ġcell":2779,"ari":2780,"Ġplayer":2781,"ais":2782,"Ġcost":2783,"Ġhum":2784,"(R":2785,"Ġoffic":2786,"ks":2787,".text":2788,"atures":2789,"Ġtotal":2790,"Ġ*/ĊĊ":2791,"ope":2792,"Ġstat":2793,"UM":2794,"Ġload":2795,"ights":2796,"Ġclear":2797,"uro":2798,"Ġtechn":2799,"upport":2800,"IR":2801,"Ġrow":2802,"Ġseem":2803,"Ġq":2804,"Ġshort":2805,"ĠNot":2806,"ipp":2807,"Group":2808,"section":2809,"max":2810,"irl":2811,"Ġoverride":2812,"Ġcompany":2813,"Ġdone":2814,"\");čĊ":2815,"Ġgre":2816,".Re":2817,"Ġbelie":2818,"rist":2819,"Ġhealth":2820,"ANT":2821,"()ĊĊ":2822,"ĠBe":2823,".value":2824,"ĠGr":2825,"ottom":2826,"Ġargs":2827,"PT":2828,"status":2829,"func":2830,"uments":2831,"-h":2832,"Number":2833,":čĊ":2834,"ĠLog":2835,"erver":2836,"Ġ),Ċ":2837,"ament":2838,"Ġobj":2839,"inc":2840,"Ġchildren":2841,"icy":2842,"IZ":2843,"ands":2844,"ably":2845,"Ġdistrib":2846,"Ġcur":2847,"erial":2848,"Ġdays":2849,"reated":2850,"rect":2851,"-l":2852,"irm":2853,"idden":2854,"omb":2855,"Ġinitial":2856,".js":2857,"Ġâ":2858,"Query":2859,"Ġonline":2860,"imal":2861,".con":2862,"au":2863,"Url":2864,"control":2865,"irection":2866,"Ġinstance":2867,"ORT":2868,"ĠFr":2869,"where":2870,"Ġjavax":2871,"Ġorgan":2872,"apter":2873,"Ġreason":2874,"options":2875,"ĠMar":2876,"(a":2877,"Ġwithin":2878,".âĢĿĊĊ":2879,"ODE":2880,"_DE":2881,"admin":2882,"ended":2883,"Ġdesign":2884,"ĠData":2885,"une":2886,"ĠFile":2887,"root":2888,"Ġcent":2889,"Ġarr":2890,"_add":2891,"len":2892,"page":2893,",'":2894,"_str":2895,"Ġbro":2896,"ability":2897,"outh":2898,"/c":2899,"pose":2900,"irtual":2901,"earch":2902,"_url":2903,"argin":2904,"Http":2905,"Ġschool":2906,"ava":2907,"Ġconsider":2908,".label":2909,"ĠArray":2910,"web":2911,"opt":2912,".println":2913,"ulation":2914,"Ġfunc":2915,"PL":2916,"Ġ\"\\":2917,"ĠText":2918,"actory":2919,"(function":2920,"null":2921,"Ġeng":2922,"down":2923,"Ġinclude":2924,"ĠEn":2925,"ĠDr":2926,"Ġdb":2927,"!!":2928,"side":2929,"Ġinit":2930,"quired":2931,"ĠShe":2932,"Column":2933,"react":2934,"Ġann":2935,"Ġstop":2936,"Ġlater":2937,"ĠThat":2938,"ention":2939,"df":2940,"UG":2941,"ILE":2942,"Ġclient":2943,"raft":2944,"ffer":2945,"POST":2946,"elper":2947,"Ġlove":2948,"quote":2949,"oud":2950,"Ġjson":2951,"Ġable":2952,"Ġmen":2953,"AX":2954,"ĠCopyright":2955,"ö":2956,"avig":2957,"req":2958,"Client":2959,"});Ċ":2960,".Com":2961,"erc":2962,"ilt":2963,"pecial":2964,"_com":2965,"room":2966,".Name":2967,"Ġgive":2968,"amb":2969,"ike":2970,"Ġcondition":2971,"client":2972,"ators":2973,":\"":2974,"Ġcopy":2975,"uture":2976,"iversity":2977,"ernal":2978,"{{":2979,"ĠCan":2980,"ounc":2981,"do":2982,"Ġocc":2983,"Ġappro":2984,"thers":2985,"ze":2986,"Ġeither":2987,"ĠFl":2988,"Ġimportant":2989,"Ġlead":2990,"attr":2991,"ART":2992,"Equal":2993,"Ġda":2994,"etch":2995,"entity":2996,"Ġfamily":2997,"adding":2998,"Ġoption":2999,"Ġexist":3000,"ica":3001,"ĠObject":3002,"'ve":3003,"vers":3004,"itional":3005,"output":3006,"ĠTrue":3007,"ĠOF":3008,"_time":3009,"Ġoffer":3010,"Ġ});ĊĊ":3011,"HER":3012,"egin":3013,"\"\"":3014,"Ġwater":3015,"Ġche":3016,"ĠMy":3017,"ored":3018,"Ġstep":3019,"ances":3020,"CK":3021,"AY":3022,"à¸":3023,"struction":3024,"(C":3025,"ouch":3026,"Stream":3027,"active":3028,"ama":3029,"Entity":3030,"product":3031,"(){Ċ":3032,"Ġgovernment":3033,"ĠID":3034,"ajor":3035,"And":3036,"Ġdisplay":3037,"л":3038,"Ġtimes":3039,"Ġfour":3040,"Ġfar":3041,"Ġpresent":3042,"ĠNS":3043,"Ġ\\Ċ":3044,"uest":3045,"Ġbas":3046,"echo":3047,"child":3048,"ifier":3049,"Handler":3050,"Ġlib":3051,"Property":3052,"translation":3053,"Ġroom":3054,"Ġonce":3055,"Ġ[]":3056,"center":3057,"================================":3058,"Ġresults":3059,"Ġcontinue":3060,"Ġtalk":3061,"_get":3062,"Ġgrow":3063,".sw":3064,"eb":3065,"ĠPublic":3066,"OP":3067,"ecute":3068,"ols":3069,"Ġ**":3070,"\");ĊĊ":3071,"Ġmass":3072,"ured":3073,".class":3074,"omic":3075,"Ġmean":3076,"ips":3077,"Ġaut":3078,");čĊčĊ":3079,"Ġuntil":3080,"Ġmarket":3081,"Ġarea":3082,"uit":3083,"Ġlength":3084,"ĠWith":3085,"structor":3086,"event":3087,"\"><":3088,"ĠSp":3089,"IV":3090,"Ġmus":3091,"iff":3092,"Ġkind":3093,"author":3094,"ounds":3095,"mb":3096,"_key":3097,"width":3098,"pository":3099,"Ġlight":3100,"uk":3101,"Row":3102,"ohn":3103,"alf":3104,"vironment":3105,"apper":3106,"ollections":3107,"Ġside":3108,"_info":3109,"Ġexample":3110,"imary":3111,"Ġwr":3112,"Ġcamp":3113,"cribe":3114,"\"/":3115,"Ġmiss":3116,"way":3117,"Ġbased":3118,"Ġplan":3119,"Vis":3120,"omain":3121,"unk":3122,"Ġaway":3123,"UP":3124,"":3370,"Ġden":3371,"obile":3372,"change":3373,"ĠĠĠĠĠĠĠĠĠĠĠĠĊ":3374,"ici":3375,"na":3376,"ĠForm":3377,"Ġsort":3378,"Select":3379,"pare":3380,"Ġthought":3381,"_con":3382,"Ġtask":3383,"ocus":3384,"ĠDE":3385,"ĠMin":3386,"Ġopt":3387,"ĉbreak":3388,"umer":3389,"KE":3390,"then":3391,"Ġdet":3392,"ĠTest":3393,"ports":3394,"Ġreview":3395,"('/":3396,"move":3397,"Ġswitch":3398,"ERT":3399,"patch":3400,"annot":3401,"ãĤ":3402,"Ġabove":3403,"itive":3404,"Ġquestion":3405,"ĠQu":3406,"ãĢĤĊĊ":3407,"gle":3408,"Ġword":3409,"Ġprovide":3410,"ĠReturn":3411,"Ġresearch":3412,"ão":3413,"ustr":3414,"Ġpublish":3415,"chema":3416,"}}":3417,"ĠCON":3418,"-in":3419,"allback":3420,"Ġcover":3421,"\\\\":3422,"color":3423,"ĠIS":3424,"Ġwhether":3425,"imate":3426,"isc":3427,"Bar":3428,"Ġdiv":3429,"Be":3430,"ourn":3431,"Ġhaving":3432,"lem":3433,"player":3434,"abs":3435,"amera":3436,"ney":3437,"Ġexc":3438,"gether":3439,"plied":3440,"ao":3441,"[$":3442,"Ġ++":3443,"ipe":3444,"show":3445,"/d":3446,"[:":3447,"agement":3448,"lev":3449,"_ID":3450,"rary":3451,"ades":3452,"_se":3453,"ause":3454,"Ġemploy":3455,"Ġ*/čĊ":3456,"Ġfre":3457,"Ġ'@":3458,"Ġcomplet":3459,"Ġlarge":3460,"ral":3461,"\\x":3462,"Ġfac":3463,">":3578,"Ġface":3579,"CTION":3580,"Ġsave":3581,"Ġtyp":3582,"dev":3583,"(\"#":3584,"AGE":3585,"container":3586,"edit":3587,"QL":3588,"Ġitems":3589,"Ġsocial":3590,"ien":3591,"ĠReact":3592,").ĊĊ":3593,"Ġmar":3594,"Ġredu":3595,"ĠRE":3596,".put":3597,"Ġmajor":3598,"Cell":3599,"next":3600,"Ġexpected":3601,"Ġyet":3602,"Ġindiv":3603,"tributes":3604,"atis":3605,"amed":3606,"Ġfood":3607,"Source":3608,"(string":3609,"Ġ+Ċ":3610,"ites":3611,"dr":3612,"Ġmembers":3613,"Ġcomb":3614,"items":3615,"ĠPer":3616,"TH":3617,"=True":3618,"Ġbar":3619,"_SE":3620,"comm":3621,"(w":3622,")ĊĊĊ":3623,"Ġsend":3624,"Ġinc":3625,"unsigned":3626,"FA":3627,"Ġparams":3628,"apping":3629,"ros":3630,"ugin":3631,"fa":3632,"Ġconnection":3633,"Ġ};ĊĊ":3634,"Ġbecome":3635,"Mode":3636,"Ġev":3637,"Ġdiff":3638,"ĠUnited":3639,"Height":3640,"fully":3641,"images":3642,"Ġmakes":3643,"Ġglobal":3644,"Ġcontact":3645,"':Ċ":3646,"Ġabs":3647,"аÐ":3648,"float":3649,"Ġexcept":3650,"ĠPol":3651,"Child":3652,"typ":3653,"Ġcertain":3654,"ión":3655,"OUT":3656,"Ġimpro":3657,"iles":3658,"Ġ-->Ċ":3659,"ĠPart":3660,"values":3661,"oss":3662,"/**":3663,"ilit":3664,"ĠEvent":3665,"curity":3666,"ster":3667,"Ġcharacter":3668,"Ġnews":3669,"Ġ\",":3670,"Ġdevice":3671,"cel":3672,"login":3673,"heet":3674,"Default":3675,"@\"":3676,"ĉĠ":3677,"click":3678,"(value":3679,"ĠAb":3680,"Ġprevious":3681,"ERROR":3682,"ocal":3683,"Ġmaterial":3684,"Ġbelow":3685,"ĠChrist":3686,"Ġmedia":3687,"cover":3688,"ĠUI":3689,"Ġfail":3690,"Ġblack":3691,"Ġcomponent":3692,"ĠAmerican":3693,"Ġadded":3694,"Ġbuy":3695,"stit":3696,"Ġcame":3697,"Ġdelete":3698,"property":3699,"oding":3700,"Ġcard":3701,"rops":3702,"Ġhttps":3703,"Ġroot":3704,"Ġhandle":3705,"CC":3706,"Back":3707,"emplate":3708,"Ġgetting":3709,"_by":3710,"mail":3711,"_sh":3712,".assert":3713,"ĠDec":3714,"(true":3715,"Ġcomput":3716,"Ġclaim":3717,"'=>":3718,"ĠSub":3719,"Ġair":3720,"ops":3721,"nav":3722,"ements":3723,"(id":3724,"Ġenter":3725,"anged":3726,"End":3727,"Ġlocation":3728,"Ġnight":3729,"Ġdoing":3730,"ĠRed":3731,"lin":3732,"}ĊĊĊ":3733,"vider":3734,"Ġpick":3735,"Ġwatch":3736,"essages":3737,"Ġhuman":3738,"Ġdam":3739,"pend":3740,"dir":3741,"Ġtax":3742,"Ġgirl":3743,"reet":3744,"Ġbox":3745,"Ġstrong":3746,"(v":3747,"rel":3748,"Ġinterface":3749,"Ġmsg":3750,"fect":3751,"_at":3752,"Ġhouse":3753,"Ġtrack":3754,"');ĊĊ":3755,"je":3756,"ĠJohn":3757,"istr":3758,"(S":3759,"ube":3760,"Ġce":3761,"itted":3762,"VER":3763,"*)":3764,"parent":3765,"Ġapplication":3766,"any":3767,".swing":3768,"Ġpack":3769,"\\u":3770,"Ġpract":3771,"Ġsection":3772,"ctx":3773,"Ġunsigned":3774,".Point":3775,"ĠOne":3776,"ı":3777,"iple":3778,"aid":3779,"Ñĥ":3780,"Vector":3781,"byte":3782,"Ġwait":3783,"ĠÃł":3784,"Ã¥":3785,"Ġtogether":3786,"Ġthrows":3787,"FO":3788,"'))":3789,"host":3790,"ising":3791,".view":3792,"Ġterms":3793,"framework":3794,"-r":3795,"Ġapply":3796,"Ġsession":3797,"Options":3798,"uggest":3799,"Ġothers":3800,"witter":3801,"Ġfund":3802,"Init":3803,"__(":3804,"ensor":3805,"GET":3806,"Ġseveral":3807,"ii":3808,"[j":3809,"IO":3810,"Ġtemplate":3811,"Position":3812,"Ġecon":3813,"achine":3814,"Ġil":3815,".spring":3816,"main":3817,"elt":3818,"iment":3819,"Rec":3820,"mm":3821,"ĠUniversity":3822,"ursor":3823,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":3824,"GL":3825,"icture":3826,"ithub":3827,"cer":3828,"cast":3829,"From":3830,"ales":3831,"Ġsubject":3832,"password":3833,"ny":3834,"Ġesc":3835,".write":3836,"ï¼Į":3837,"What":3838,".H":3839,"Ġhistory":3840,"ĠFe":3841,"Ġindividual":3842,"unit":3843,"Ġ-->":3844,"Ġdu":3845,"IST":3846,"Ġusers":3847,"fs":3848,"false":3849,"unt":3850,"Title":3851,"Ġmot":3852,"Ġfuture":3853,"ached":3854,"Ġstarted":3855,"Ġmode":3856,"Ġ'<":3857,"_array":3858,"Ġax":3859,"'];Ċ":3860,"ires":3861,"There":3862,"ught":3863,"tml":3864,"posed":3865,"icult":3866,"Ġtook":3867,"Ġgames":3868,"Ġ}}":3869,"Ġ?>Ċ":3870,"Ġproducts":3871,"Is":3872,"Ġbad":3873,"ĠDes":3874,".path":3875,"'ĊĊ":3876,"ĠPost":3877,"avel":3878,"(:":3879,"Ġneeds":3880,"Ġknown":3881,"Fl":3882,"Ġexec":3883,"Ġseen":3884,"ume":3885,"Ġborder":3886,"Ġlive":3887,"temp":3888,"Per":3889,"Ġvariable":3890,"iet":3891,"ĠDef":3892,"Ġge":3893,"eme":3894,"_back":3895,"first":3896,"Ġprovided":3897,"////////////////////////////////":3898,"Ġfilename":3899,"Ġhope":3900,"uly":3901,"auto":3902,"find":3903,"_string":3904,"btn":3905,"itude":3906,"Attribute":3907,"Ġyoung":3908,".txt":3909,"Ġwebsite":3910,"ĠProp":3911,"Ġey":3912,">();Ċ":3913,"ional":3914,"ARR":3915,"ictionary":3916,"urther":3917,".":3997,"tx":3998,"Ġpur":3999,"uel":4000,"ymbol":4001,"uation":4002,"anger":4003,"Ġbackground":4004,"ecess":4005,"efined":4006,"........":4007,"Ġdescription":4008,"Ġrepresent":4009,"\"));Ċ":4010,"pression":4011,"rowser":4012,"Ġseries":4013,"wards":4014,"($_":4015,"aise":4016,"Ġhot":4017,"acity":4018,"ries":4019,"actions":4020,"Create":4021,"adio":4022,"amples":4023,"Ġoriginal":4024,"ensive":4025,"font":4026,"stream":4027,"using":4028,".springframework":4029,"server":4030,"Ġbill":4031,"ACK":4032,"ilename":4033,"Ġframe":4034,"Ġ=Ċ":4035,"Edit":4036,"adius":4037,"Ġdraw":4038,"anks":4039,"Ġdeter":4040,"Ġcomes":4041,"_int":4042,"Ġforeach":4043,"angle":4044,"Ġelect":4045,"pected":4046,"Header":4047,"istration":4048,"False":4049,"ĠGame":4050,"Ġfilter":4051,"Activity":4052,"Ġlarg":4053,"inition":4054,"Ġ\"<":4055,"ised":4056,"Ġremove":4057,"ĠTrans":4058,"met":4059,"see":4060,"Format":4061,"Command":4062,"ĠEX":4063,"None":4064,"Ġfront":4065,"ASE":4066,"ĠRec":4067,"oundation":4068,"Ġvo":4069,"=\\\"":4070,"(*":4071,"Change":4072,".Write":4073,"group":4074,"ients":4075,"uy":4076,"****************************************************************":4077,"Ġdig":4078,"hr":4079,"(-":4080,"Ġgen":4081,"number":4082,"vec":4083,"urope":4084,"entry":4085,"LL":4086,"Ġste":4087,"Valid":4088,"'],":4089,"_param":4090,"Ġselected":4091,"Ġaccording":4092,"ĠDis":4093,"Ġutil":4094,"Buffer":4095,"_error":4096,"Ġassoci":4097,"_SIZE":4098,"Ġwor":4099,"Ġprintf":4100,"rag":4101,"Âł":4102,"DD":4103,"ĠVal":4104,"Ġactiv":4105,"Eng":4106,"etime":4107,"Ġvirtual":4108,"aign":4109,"aur":4110,"ĠPres":4111,"ĠException":4112,"Ġanything":4113,"ĠOff":4114,"Ġhours":4115,"Ġwar":4116,"Args":4117,"aging":4118,"Ġmodels":4119,"ĠTime":4120,"Ob":4121,"ams":4122,"joy":4123,"Ġearly":4124,".read":4125,"Ġcenter":4126,"ĠInitial":4127,"Ġlanguage":4128,"length":4129,"xy":4130,"Ġsn":4131,"Ġinf":4132,"Post":4133,"Ġago":4134,"Ġeasy":4135,"_code":4136,"ĠANY":4137,"_ch":4138,"Ġdownload":4139,"(T":4140,"aved":4141,"âĢĵ":4142,"Ġstudents":4143,"Ġfig":4144,"light":4145,"xx":4146,"Ġbuffer":4147,"ĠDep":4148,"ĠMath":4149,"ITH":4150,"Ġvari":4151,"Ġdue":4152,"Factory":4153,"Ġpor":4154,"Ġep":4155,"otype":4156,"Ġcannot":4157,"Ġwhite":4158,"čĊ":4424,".annot":4425,"Ġcollection":4426,"'.":4427,"Ġsimilar":4428,"Ġtaken":4429,"(\"%":4430,"Order":4431,"']Ċ":4432,"-md":4433,"ĠTH":4434,"aced":4435,"Ġisn":4436,"/j":4437,"Ġson":4438,"graph":4439,"ĠInteger":4440,"Ġnecess":4441,"reen":4442,"Ġum":4443,"Ġ\\<":4444,"Ġmoment":4445,"Ġbring":4446,"Ġindic":4447,"ysis":4448,"Level":4449,"verse":4450,"urrenc":4451,"_test":4452,"Ġentire":4453,"Down":4454,"Ġ}ĊĊĊ":4455,"(result":4456,"ĠRead":4457,"è":4458,"Mod":4459,"Ġtrying":4460,"\"),Ċ":4461,"Ġmember":4462,"ĠCor":4463,"ODO":4464,"-control":4465,"untime":4466,"ĠSim":4467,"Dialog":4468,"plot":4469,"_on":4470,"Ġphys":4471,"}/":4472,"Ġnamespace":4473,"ĉčĊ":4474,"acc":4475,"Player":4476,"ARE":4477,"Ġfoot":4478,"Ġboard":4479,"part":4480,"Ġsus":4481,"wise":4482,"ĠMc":4483,"Ġpush":4484,"ATA":4485,"Ġplease":4486,"ried":4487,"weet":4488,"bit":4489,"ided":4490,"VE":4491,"ĠSw":4492,"UB":4493,"Ġtypes":4494,"edia":4495,"Ġclos":4496,"acebook":4497,"When":4498,"Ġedit":4499,"igger":4500,"Ġenerg":4501,"Container":4502,"Ġphot":4503,"ĠCount":4504,"ĠEurope":4505,".Is":4506,"ĠRuss":4507,"peed":4508,"ĠStr":4509,"Ġpy":4510,"Ġcult":4511,"Ġdefined":4512,"ccount":4513,"Ġobt":4514,".Location":4515,"Ġthread":4516,"ille":4517,"Ġinstead":4518,"strong":4519,"ĠSec":4520,"URE":4521,"Ġidea":4522,".se":4523,"emy":4524,"selected":4525,"Connection":4526,"acing":4527,"thread":4528,".next":4529,"Ġcoll":4530,"Ġfilm":4531,"istic":4532,"Ġcompet":4533,"Ġconn":4534,"though":4535,"Ġcompan":4536,"ocket":4537,"Ġteach":4538,"=(":4539,"Ġphone":4540,"Ġactive":4541,"delete":4542,"tries":4543,"Ġmo":4544,"Ġdeath":4545,"});ĊĊ":4546,"ocol":4547,"Widget":4548,"Ġarticle":4549,"rodu":4550,"andid":4551,"Ñĭ":4552,"ĠCr":4553,"ka":4554,"():":4555,"lood":4556,"ĉĉĉĊ":4557,"Ġalmost":4558,"Ġsell":4559,"ervlet":4560,"rip":4561,"Unit":4562,"Ġapplic":4563,"Ġconnect":4564,"Ġfeature":4565,"Ġvia":4566,"'),":4567,"Ġlim":4568,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":4569,"ĠGu":4570,"Engine":4571,"Ġens":4572,"Ġenvironment":4573,"block":4574,"HERE":4575,"NULL":4576,"gy":4577,"tag":4578,")).":4579,"exp":4580,"Ġcompl":4581,"Ġinstall":4582,"Ġcomplete":4583,"queue":4584,"atural":4585,"Ġgeneral":4586,"thon":4587,"Ġasked":4588,"ores":4589,"(res":4590,"Ġreserved":4591,"SP":4592,"Ġâ̦":4593,"ÅĤ":4594,"Ġsignific":4595,"Off":4596,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":4597,"ĠAg":4598,"ĠJust":4599,"ĠError":4600,"Ġinfl":4601,"adata":4602,"Ġicon":4603,"asks":4604,"''":4605,"_LO":4606,"?.":4607,"account":4608,"Ġ(*":4609,"')ĊĊ":4610,"rap":4611,"_var":4612,"ĠFOR":4613,"Ġparty":4614,"ĠYour":4615,"cat":4616,"stry":4617,".new":4618,"boot":4619,"ĠNov":4620,"Ġvector":4621,"Ġnormal":4622,"Ġfurther":4623,"Repository":4624,"Ġdatabase":4625,"attle":4626,"Ġmusic":4627,"Ġspeed":4628,"Ġdoc":4629,"process":4630,"IGHT":4631,".parse":4632,"Ġtaking":4633,"Ġviol":4634,"ceed":4635,"ĠAfter":4636,"Ġforward":4637,"Ġcrit":4638,"\"/>Ċ":4639,"rot":4640,"Ġfailed":4641,"efore":4642,"Ġconcern":4643,"oe":4644,"ba":4645,"Ġsender":4646,"Ġterm":4647,"has":4648,"=\"#":4649,"Ġpotential":4650,"Num":4651,"Ġpublished":4652,".close":4653,"ĠImage":4654,"straint":4655,"UD":4656,"ĠOb":4657,"Ġprobably":4658,"lim":4659,"\":Ċ":4660,"olume":4661,"Ġconsum":4662,"ague":4663,"ensions":4664,"Ġinvestig":4665,"-year":4666,"');":4667,"-sm":4668,"Ġenjoy":4669,"orig":4670,"ering":4671,"cp":4672,"leased":4673,"plements":4674,"Ġreturns":4675,"pat":4676,"BO":4677,"ĠHouse":4678,".Label":4679,"Ġweight":4680,"ighb":4681,"Ġconditions":4682,"Ġexception":4683,"description":4684,"Ġtrad":4685,"-to":4686,"Ġ{}":4687,"Ġmodule":4688,"END":4689,".ap":4690,".props":4691,"Ġconstructor":4692,"aves":4693,"Ġfavor":4694,"ĠNow":4695,";i":4696,"ĠMain":4697,"_k":4698,"eries":4699,"âĢĻll":4700,"transform":4701,"imestamp":4702,"Pre":4703,"Ġmer":4704,".res":4705,"stant":4706,"Location":4707,"_NAME":4708,"Ġloss":4709,"ĠĊĊ":4710,"net":4711,"Ġengine":4712,"Block":4713,"Ġissues":4714,"Ġparse":4715,"ĠBar":4716,"Ġstay":4717,"ĠJSON":4718,"Ġdom":4719,"airs":4720,"wner":4721,"Ġlower":4722,"\",čĊ":4723,"ĠDem":4724,"ufact":4725,"Ġps":4726,"Ġperfect":4727,"RL":4728,"Ġeduc":4729,"ls":4730,"emory":4731,"ARRANT":4732,"uge":4733,"Ġexact":4734,".key":4735,"alled":4736,"ech":4737,"ief":4738,"\\/":4739,"oke":4740,"Ġformer":4741,"alloc":4742,"Ġsix":4743,"ida":4744,"Ġmargin":4745,"Ġheart":4746,"ald":4747,"pack":4748,".getElementById":4749,"ĠWARRANT":4750,"Ġrather":4751,"Ġbuilding":4752,"erman":4753,"lice":4754,"Ġquestions":4755,"izes":4756,"lege":4757,"irectory":4758,"Ġje":4759,"Ġcas":4760,"props":4761,"utf":4762,"Ġsecurity":4763,"Ġhowever":4764,"weight":4765,"Ġinside":4766,"Ġpresident":4767,"Char":4768,"ĠWITH":4769,".map":4770,"Ġgraph":4771,"Ġtag":4772,"_status":4773,"Ġattempt":4774,"opp":4775,"uses":4776,"ĉconst":4777,"Ġround":4778,",$":4779,"Ġfriends":4780,"Email":4781,"?>":4782,"Resource":4783,"KEY":4784,"osp":4785,".query":4786,"ĠNorth":4787,"ables":4788,"istrib":4789,"_class":4790,"ello":4791,"That":4792,"к":4793,"pecially":4794,"ĠPresident":4795,"Ġcampaign":4796,"Ġalt":4797,"area":4798,"Ġchall":4799,"Ġopport":4800,".Con":4801,"Ġenergy":4802,"like":4803,".string":4804,"ington":4805,")*":4806,"yy":4807,"Ġprofession":4808,"irth":4809,"Ġseg":4810,"æľ":4811,"Ġhor":4812,"iers":4813,"can":4814,"Ġbehind":4815,"Product":4816,"fg":4817,"ĠSk":4818,".jpg":4819,"?:":4820,"];ĊĊ":4821,"Ġcallback":4822,"ĠHttp":4823,"ÑĮ":4824,"long":4825,"MS":4826,"ATH":4827,"Ġraise":4828,"Ġwanted":4829,"rown":4830,"utor":4831,"lt":4832,"]=":4833,"eline":4834,"MA":4835,"Ġsepar":4836,"cs":4837,"semb":4838,"Dis":4839,"bserv":4840,"ĠWill":4841,"Ġpolicy":4842,"Ġthird":4843,"phone":4844,"Ġbed":4845,"/g":4846,".__":4847,"ĠInc":4848,"izing":4849,".remove":4850,"instance":4851,".type":4852,"Ġserv":4853,"Each":4854,"Ġhar":4855,"ĠMessage":4856,"(key":4857,"SELECT":4858,"Pos":4859,"));čĊ":4860,"Ġrecomm":4861,"Ġtraining":4862,"ĠEnt":4863,"ĠChar":4864,"icht":4865,"(file":4866,"Ġprior":4867,"Game":4868,"Ġexit":4869,"Params":4870,".core":4871,"PC":4872,"nes":4873,"anced":4874,"(request":4875,"Password":4876,"}>Ċ":4877,"Ġmag":4878,"Ġrelease":4879,"Ġshall":4880,"udent":4881,"ĠSouth":4882,"ando":4883,":'":4884,".TabIndex":4885,"sk":4886,"anner":4887,"isset":4888,"Ġoutside":4889,"ledge":4890,"Ġå":4891,"ĠRob":4892,"Ġimm":4893,"!Ċ":4894,"ĠWeb":4895,"Des":4896,"BC":4897,"ancial":4898,"Route":4899,"Dec":4900,"ferences":4901,"Ġpurch":4902,"ĠModel":4903,"ctor":4904,"gn":4905,"_start":4906,"_un":4907,".*":4908,"ises":4909,"Ġground":4910,"Ġunique":4911,"Ġbeaut":4912,"{\"":4913,"Ġpour":4914,"ĠOct":4915,"Ġtree":4916,"sets":4917,"_res":4918,"')->":4919,"_reg":4920,"(\"\\":4921,"Ġbyte":4922,"Bl":4923,"Ġdating":4924,"Ġmatter":4925,"ĠRem":4926,"Ġ'../":4927,"ĠAug":4928,"ĠLa":4929,"Ġ$(":4930,"ournal":4931,"iam":4932,"Ġshows":4933,"write":4934,"Ġball":4935,"Ġsimply":4936,"Ġfast":4937,"Ġmemory":4938,"ASS":4939,"ĠOf":4940,"oved":4941,"ante":4942,"aul":4943,"istry":4944,")));Ċ":4945,"Ġfit":4946,"_":5129,"\")ĊĊ":5130,"ox":5131,"application":5132,"Ġ]Ċ":5133,"ĊĊĊĊĊĊ":5134,"Ġsoon":5135,"ctions":5136,"inger":5137,"Ġjoin":5138,"ĠPe":5139,"Ġë":5140,"Ġlas":5141,".E":5142,"css":5143,"/or":5144,"ĠStart":5145,"ĠTO":5146,"Ġsubs":5147,"conn":5148,"components":5149,"DEBUG":5150,"quare":5151,"Function":5152,"endar":5153,".index":5154,"Ġfill":5155,"ÄĻ":5156,"Ġchoose":5157,"how":5158,"ĠAmerica":5159,"assets":5160,"------------":5161,"ĠValue":5162,"Ġoffice":5163,"Ġveh":5164,"Ġtransform":5165,"ĠArt":5166,"Ġinde":5167,"Ġfn":5168,"Ġimplements":5169,"ango":5170,"plete":5171,"+\"":5172,"tmp":5173,"amily":5174,"Ġhash":5175,"missions":5176,"EST":5177,"gt":5178,"Provider":5179,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":5180,"Ġflag":5181,"Ġparticip":5182,"den":5183,"ĠReturns":5184,"Ġnote":5185,"ür":5186,"pm":5187,"ideos":5188,"Ġspecified":5189,"ĠEN":5190,"ester":5191,"olid":5192,"Ġupon":5193,"(std":5194,"ĉv":5195,"Ġ'\\":5196,"uz":5197,"Ġvert":5198,"Ġvict":5199,"ĉself":5200,"Ġ\"$":5201,".k":5202,"Ġgroups":5203,"github":5204,"lang":5205,"Ġmut":5206,"TO":5207,"Ġve":5208,"ĠPlease":5209,";ĊĊĊ":5210,"access":5211,"Ġ{\"":5212,"rea":5213,"Ġrisk":5214,"icker":5215,"oggle":5216,"ĉwhile":5217,"ANG":5218,".send":5219,"Ġwoman":5220,"Ġgets":5221,"Ġign":5222,"ĠId":5223,"_log":5224,"ONE":5225,"Ġevid":5226,"ĠHar":5227,"_sub":5228,"Ġendl":5229,"Ġincluded":5230,"());ĊĊ":5231,"ĠAp":5232,"igr":5233,"Ġsem":5234,"ĠBlack":5235,"doc":5236,"_table":5237,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":5238,"-up":5239,"Ġcause":5240,"Ġ..":5241,"Ġvan":5242,"_dict":5243,"Ġfocus":5244,"IND":5245,"CESS":5246,".Log":5247,"Ġmultiple":5248,"ido":5249,"Ġregard":5250,"-M":5251,"andler":5252,"ourse":5253,"Ġdeg":5254,".U":5255,"Ġaddition":5256,"Ġvarious":5257,"Ġreceive":5258,"ен":5259,"ĠHT":5260,"Obj":5261,"DF":5262,"Ġincrease":5263,"ĠOpen":5264,"];":5265,"Ġcommit":5266,"?Ċ":5267,"ategories":5268,"atory":5269,"ship":5270,"ĠMich":5271,"Ġhtml":5272,"romise":5273,"Ġleave":5274,"Ġstrateg":5275,"aven":5276,"ĠConsole":5277,"known":5278,"-n":5279,"_LE":5280,".component":5281,"Ġbre":5282,"Session":5283,"iance":5284,"Ġalign":5285,"typedef":5286,"_result":5287,"ĠWHERE":5288,".split":5289,"Ġreading":5290,"FAULT":5291,"Ġclo":5292,"Ġnotice":5293,"_pr":5294,"arter":5295,"Ġlock":5296,"Ġstandard":5297,"etic":5298,"ellow":5299,"Ġpadding":5300,"ĠHis":5301,"Ġstates":5302,"_cast":5303,"(P":5304,"aa":5305,"Ġinternal":5306,"ean":5307,"ĠPRO":5308,"ĠKey":5309,"Ġespecially":5310,"ming":5311,"Ġcross":5312,"Ġnational":5313,"_object":5314,"filter":5315,"Ġscript":5316,".update":5317,"_i":5318,"ĠAssert":5319,"/core":5320,"%%%%":5321,"Ġproblems":5322,"istor":5323,"Ġ.=":5324,"Ġarch":5325,"Ġwritten":5326,"Ġmilit":5327,"MENT":5328,".ch":5329,"cape":5330,"ĠMus":5331,"_config":5332,"ĠAPI":5333,"foot":5334,"Ġimages":5335,"endl":5336,".In":5337,"First":5338,"Ġplatform":5339,".prot":5340,"Option":5341,"ste":5342,"ĠTODO":5343,"Ġforce":5344,".cont":5345,"ĉecho":5346,"ĠDav":5347,"Ptr":5348,"(B":5349,"RT":5350,"ĠBase":5351,"]['":5352,"Ġannounc":5353,"console":5354,"ĠPy":5355,"ds":5356,".as":5357,"Ġprevent":5358,"apan":5359,"Ġ{'":5360,"}'":5592,"Ġdead":5593,"VAL":5594,"QUE":5595,"************************************************************************":5596,"Ġcharg":5597,"Return":5598,"Ġful":5599,"dom":5600,"Ġrules":5601,"Ġmodify":5602,"Ġeval":5603,"ham":5604,"atement":5605,"\\<":5606,"ula":5607,"=False":5608,"RA":5609,"Ġcontains":5610,"Ġstack":5611,"mar":5612,"Ġ{}Ċ":5613,"Ġundefined":5614,"Ass":5615,"ĠChina":5616,"vey":5617,"*Ċ":5618,"Ġplaying":5619,")/":5620,"actor":5621,"Ġbottom":5622,"lier":5623,"ĠNumber":5624,"Ġcouple":5625,"DC":5626,"ĠSO":5627,"gor":5628,".setText":5629,"success":5630,"command":5631,"Filter":5632,"ĠOur":5633,"_item":5634,"Ġctx":5635,"Ġroad":5636,"Version":5637,"case":5638,"urt":5639,"avior":5640,"ych":5641,"sembly":5642,"ĠProduct":5643,"Ġheld":5644,"afe":5645,"Ġincludes":5646,"&":5789,"CON":5790,"Ġrepl":5791,"Ġregular":5792,"Storage":5793,"ramework":5794,"Ġgoal":5795,"Ġtouch":5796,".widget":5797,"Ġbuilt":5798,"des":5799,"Part":5800,"(re":5801,"Ġworth":5802,"hib":5803,"game":5804,"Ġв":5805,"acion":5806,"ĠWhite":5807,"(type":5808,"(`":5809,"Ġnatural":5810,"Ġinj":5811,"Ġcalcul":5812,"ĠApril":5813,".List":5814,"Ġassociated":5815,"ĉSystem":5816,"~~":5817,"=[":5818,"Ġstorage":5819,"Ġbytes":5820,"Ġtravel":5821,"Ġsou":5822,"Ġpassed":5823,"!=":5824,"ascript":5825,".open":5826,"Ġgrid":5827,"Ġbus":5828,"Ġrecogn":5829,"Ab":5830,"Ġhon":5831,"ĠCenter":5832,"Ġprec":5833,"build":5834,"HTML":5835,"ĠSan":5836,"Ġcountries":5837,"aled":5838,"token":5839,"kt":5840,"Ġqual":5841,"Last":5842,"adow":5843,"Ġmanufact":5844,"idad":5845,"jango":5846,"Next":5847,"xf":5848,".a":5849,"Ġporno":5850,"ĠPM":5851,"erve":5852,"iting":5853,"_th":5854,"ci":5855,"=None":5856,"gs":5857,"Ġlogin":5858,"atives":5859,"']);Ċ":5860,"Äħ":5861,"Ġill":5862,"IA":5863,"children":5864,"DO":5865,"Ġlevels":5866,"Ġ{{":5867,"Ġlooks":5868,"Ġ\"#":5869,"ToString":5870,"Ġnecessary":5871,"ĠĠĠĊ":5872,"cell":5873,"Entry":5874,"Ġ'#":5875,"Ġextrem":5876,"Selector":5877,"Ġplaceholder":5878,"Load":5879,"Ġreleased":5880,"ORE":5881,"Enumer":5882,"ĠTV":5883,"SET":5884,"inq":5885,"Press":5886,"ĠDepartment":5887,"Ġproperties":5888,"Ġrespond":5889,"Search":5890,"ael":5891,"Ġrequ":5892,"ĠBook":5893,"/Ċ":5894,"(st":5895,"Ġfinancial":5896,"icket":5897,"_input":5898,"Ġthreat":5899,"(in":5900,"Strip":5901,"ìĿ":5902,"ção":5903,"Ġevidence":5904,"));":5905,"ĠBro":5906,"Ġ[];Ċ":5907,"Ġou":5908,"buf":5909,"Script":5910,"dat":5911,"Ġrule":5912,"#import":5913,"=\"/":5914,"Serial":5915,"Ġstarting":5916,"[index":5917,"ae":5918,"Ġcontrib":5919,"session":5920,"_new":5921,"utable":5922,"ober":5923,"Ġ\"./":5924,"Ġlogger":5925,"Ġrecently":5926,"Ġreturned":5927,"ččĊ":5928,")))Ċ":5929,"itions":5930,"Ġseek":5931,"Ġcommunic":5932,"Ġ\".":5933,"Ġusername":5934,"ECT":5935,"DS":5936,"Ġotherwise":5937,"ĠGerman":5938,".aw":5939,"Adapter":5940,"ixel":5941,"Ġsystems":5942,"Ġdrop":5943,"Ġstructure":5944,"Ġ$(\"#":5945,"encies":5946,"anning":5947,"ĠLink":5948,"ĠResponse":5949,"Ġstri":5950,"ż":5951,"ĠDB":5952,"æĹ":5953,"android":5954,"submit":5955,"otion":5956,"(@":5957,".test":5958,"ĊĊĊĊĊĊĊĊ":5959,"];čĊ":5960,"Ġdirectly":5961,"Ġ\"%":5962,"ris":5963,"elta":5964,"AIL":5965,"){čĊ":5966,"mine":5967,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":5968,"(k":5969,"bon":5970,"asic":5971,"pite":5972,"___":5973,"Max":5974,"Ġerrors":5975,"ĠWhile":5976,"Ġarguments":5977,"Ġensure":5978,"Right":5979,"-based":5980,"Web":5981,"Ġ-=":5982,"Ġintrodu":5983,"ĠInst":5984,"ĠWash":5985,"ordin":5986,"join":5987,"Database":5988,"Ġgrad":5989,"Ġusually":5990,"ITE":5991,"Props":5992,"?>Ċ":5993,"ĠGo":5994,"@Override":5995,"REF":5996,"Ġip":5997,"ĠAustral":5998,"Ġist":5999,"ViewById":6000,"Ġserious":6001,"Ġcustomer":6002,".prototype":6003,"odo":6004,"cor":6005,"Ġdoor":6006,"ĠWITHOUT":6007,"Ġplant":6008,"Ġbegan":6009,"Ġdistance":6010,"()).":6011,"Ġchance":6012,"Ġord":6013,"came":6014,"pragma":6015,"Ġprotect":6016,"ragment":6017,"ĠNode":6018,"ening":6019,"Ñĩ":6020,"Ġroute":6021,"ĠSchool":6022,"hi":6023,"Ġneighb":6024,"After":6025,"licit":6026,"Ġcontr":6027,"Ġprimary":6028,"AA":6029,".WriteLine":6030,"utils":6031,"Ġbi":6032,"Red":6033,".Linq":6034,".object":6035,"Ġleaders":6036,"unities":6037,"Ġgun":6038,"onth":6039,"ĠDev":6040,"FILE":6041,"Ġcomments":6042,"_len":6043,"arrow":6044,"amount":6045,"Range":6046,"sert":6047,"GridView":6048,"Ġupdated":6049,"ĠMo":6050,"Ġinform":6051,"ociety":6052,"ala":6053,"Access":6054,"Ġhab":6055,"Ġcreat":6056,"_arg":6057,"ĠJanuary":6058,"ĠDay":6059,"\")čĊ":6060,"uple":6061,"document":6062,"gorith":6063,"menu":6064,"ĠOver":6065,"bb":6066,".title":6067,"_out":6068,"Ġled":6069,"uri":6070,"Ġ?>Ċ":6107,"run":6108,"Ġscene":6109,"(array":6110,"device":6111,"_title":6112,"agon":6113,"]čĊ":6114,"aby":6115,"Ġbecame":6116,"boolean":6117,"Ġpark":6118,"ĠCode":6119,"upload":6120,"riday":6121,"ĠSeptember":6122,"Fe":6123,"Ġsen":6124,"cing":6125,"FL":6126,"Col":6127,"uts":6128,"_page":6129,"inn":6130,"Ġimplied":6131,"aling":6132,"Ġyourself":6133,".Count":6134,"conf":6135,"Ġaud":6136,"_init":6137,".)":6138,"Ġwrote":6139,"NG":6140,".Error":6141,"ä»":6142,".for":6143,"Ġequal":6144,"ĠRequest":6145,"Ġserial":6146,"Ġallows":6147,"XX":6148,"Ġmiddle":6149,"chor":6150,"ø":6151,"erval":6152,".Column":6153,"reading":6154,"Ġescort":6155,"ĠAugust":6156,"Ġquickly":6157,"Ġweap":6158,"ĠCG":6159,"ropri":6160,"ho":6161,"Ġcop":6162,"(struct":6163,"ĠBig":6164,"Ġvs":6165,"Ġfrequ":6166,".Value":6167,"Ġactions":6168,"Ġproper":6169,"Ġinn":6170,"Ġobjects":6171,"Ġmatrix":6172,"avascript":6173,"Ġones":6174,".group":6175,"Ġgreen":6176,"Ġpaint":6177,"ools":6178,"ycl":6179,"encode":6180,"olt":6181,"comment":6182,".api":6183,"Dir":6184,"Ġune":6185,"izont":6186,".position":6187,"Ġdesigned":6188,"_val":6189,"avi":6190,"iring":6191,"tab":6192,"Ġlayer":6193,"Ġviews":6194,"Ġreve":6195,"rael":6196,"ĠON":6197,"rics":6198,"np":6199,"Ġcore":6200,"());čĊ":6201,"Main":6202,"Ġexpert":6203,"ĉĉčĊ":6204,"_en":6205,"Ġ/>":6206,"utter":6207,"IAL":6208,"ails":6209,"ĠKing":6210,"*/ĊĊ":6211,"ĠMet":6212,"_end":6213,"addr":6214,"ora":6215,"Ġir":6216,"Min":6217,"Ġsurpr":6218,"Ġrepe":6219,"Ġdirectory":6220,"PUT":6221,"-S":6222,"Ġelection":6223,"haps":6224,".pre":6225,"cm":6226,"Values":6227,"Ġ\"Ċ":6228,"column":6229,"ivil":6230,"Login":6231,"inue":6232,"Ġbeautiful":6233,"Ġsecret":6234,"(event":6235,"Ġchat":6236,"ums":6237,"Ġorigin":6238,"Ġeffects":6239,"Ġmanagement":6240,"illa":6241,"tk":6242,"Ġsetting":6243,"ĠCour":6244,"Ġmassage":6245,"ĉend":6246,"Ġhappy":6247,"Ġfinish":6248,"Ġcamera":6249,"ĠVer":6250,"ĠDemocr":6251,"ĠHer":6252,"(Q":6253,"cons":6254,"ita":6255,"Ġ'.":6256,"{}":6257,"ĉC":6258,"Ġstuff":6259,"Ġ:Ċ":6260,"ĠAR":6261,"Task":6262,"hidden":6263,"eros":6264,"IGN":6265,"atio":6266,"ĠHealth":6267,"olute":6268,"Enter":6269,"'>":6270,"ĠTwitter":6271,"ĠCounty":6272,"scribe":6273,"Ġ=>Ċ":6274,"Ġhy":6275,"fit":6276,"Ġmilitary":6277,"Ġsale":6278,"required":6279,"non":6280,"bootstrap":6281,"hold":6282,"rim":6283,"-old":6284,"ĠDown":6285,"Ġmention":6286,"contact":6287,"_group":6288,"oday":6289,"Ġtown":6290,"Ġsolution":6291,"uate":6292,"elling":6293,"]->":6294,"otes":6295,"ental":6296,"omen":6297,"ospital":6298,"ĠSup":6299,"_EN":6300,"Ġslow":6301,"SESSION":6302,"Ġblue":6303,"ago":6304,"Ġlives":6305,"Ġ^":6306,".un":6307,"inst":6308,"enge":6309,"Ġcustomers":6310,"Ġcast":6311,"udget":6312,"ï¼ģ":6313,"icens":6314,"Ġdetermin":6315,"Selected":6316,"_pl":6317,"ueue":6318,"Ġdark":6319,"//ĊĊ":6320,"si":6321,"thern":6322,"ĠJapan":6323,"/w":6324,"PU":6325,"ĠEast":6326,"ovie":6327,"Ġpackage":6328,"Ġnor":6329,"Ġapi":6330,"bot":6331,"\"];Ċ":6332,"_post":6333,"ulate":6334,"Ġclub":6335,"'));Ċ":6336,"Ġloop":6337,"PIO":6338,"ione":6339,"shot":6340,"Initial":6341,"Ġplayed":6342,"register":6343,"rought":6344,"_max":6345,"acement":6346,"match":6347,"raphics":6348,"AST":6349,"Ġexisting":6350,"Ġcomplex":6351,"DA":6352,".Ch":6353,".common":6354,"mo":6355,"Ġ'../../":6356,"ito":6357,"Ġanalysis":6358,"Ġdeliver":6359,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":6360,"idx":6361,"Ãł":6362,"ongo":6363,"ĠEnglish":6364,"Ċ":9992,"_default":9993,"ĠDatabase":9994,"rep":9995,"ESS":9996,"nergy":9997,".Find":9998,"_mask":9999,"Ġrise":10000,"Ġkernel":10001,"::$":10002,".Q":10003,"Ġoffering":10004,"decl":10005,"ĠCS":10006,"Ġlisted":10007,"Ġmostly":10008,"enger":10009,"Ġblocks":10010,"olo":10011,"Ġgoverning":10012,"\\F":10013,"Ġconcent":10014,".getText":10015,"Ġmb":10016,"Ġoccurred":10017,"Ġchanging":10018,"Scene":10019,"_CODE":10020,"Beh":10021,"\"The":10022,"Ġtile":10023,"ĠAssociation":10024,"ĉP":10025,"alty":10026,"_ad":10027,"odies":10028,"iated":10029,"Ġprepared":10030,"possible":10031,"Ġmort":10032,"TEST":10033,"Ġignore":10034,"Ġcalc":10035,"Ġrs":10036,"ĠassertEquals":10037,"Ġsz":10038,"ĠTHIS":10039,".\"Ċ":10040,"Ġcanvas":10041,"java":10042,"Ġdut":10043,"VALID":10044,".sql":10045,".input":10046,"Ġaux":10047,"Sup":10048,"Ġartist":10049,"Vec":10050,"_TIME":10051,".stringify":10052,"etween":10053,"ĠCategory":10054,"Ġ[-":10055,"ĠDevExpress":10056,"ĠJul":10057,"Ġring":10058,".ed":10059,"YY":10060,"Let":10061,"TextField":10062,"Ġflat":10063,"_print":10064,"ĠOTHER":10065,"adian":10066,"Ġchecked":10067,"ele":10068,"Align":10069,"standing":10070,"Ġ[],":10071,"Ġlab":10072,"ucky":10073,"ĠChristmas":10074,"(image":10075,".module":10076,"Ġlots":10077,"Ġslightly":10078,"(final":10079,"erge":10080,"è¿":10081,"ĠPolice":10082,"ĠRight":10083,"Ġaward":10084,"ĠOS":10085,"Ġ{}ĊĊ":10086,"Ġptr":10087,"oves":10088,"icated":10089,"ем":10090,"Ġmanage":10091,"oliday":10092,"Amount":10093,"oolStrip":10094,"tbody":10095,"Nav":10096,"wrap":10097,"BB":10098,"Ġwatching":10099,"arios":10100,"Ġoptional":10101,"_K":10102,"ĠLicensed":10103,".Map":10104,"Timer":10105,"ĠAP":10106,"ĠRev":10107,"(o":10108,",c":10109,"umin":10110,"etailed":10111,"ĠHy":10112,"Ġblank":10113,"agger":10114,"ĠSelf":10115,"()[":10116,".make":10117,"earn":10118,"channel":10119,";Ċ":10133,"World":10134,"Ġpython":10135,"Ġlif":10136,"Ġtrav":10137,"Ġconven":10138,"company":10139,"ĠClub":10140,"Ver":10141,"Btn":10142,"Ġzone":10143,"products":10144,"ĠEduc":10145,"Ġverify":10146,"ĠMil":10147,"ono":10148,"]);ĊĊ":10149,"ENCE":10150,"Ġpacket":10151,"Ġcer":10152,"Ġenumer":10153,"Ġpars":10154,"formed":10155,"Ġoccup":10156,"tre":10157,"Ġexercise":10158,"Day":10159,"_sum":10160,"Ġasking":10161,"aption":10162,"Ġorders":10163,"Ġspending":10164,"ĠERR":10165,".Dis":10166,"ĠUtil":10167,"âĢľI":10168,"\\'":10169,"?)":10170,"/>Ċ":10171,"Ġemot":10172,"Ġinfluence":10173,"ĠAfrica":10174,"atters":10175,"Ùħ":10176,".session":10177,"Ġchief":10178,"ĉĉĉĉĉĉĉĉĉĉĉ":10179,"Ġtom":10180,"cluded":10181,"serial":10182,"_handler":10183,".Type":10184,"aped":10185,"Ġpolicies":10186,"-ex":10187,"-tr":10188,"blank":10189,"merce":10190,"Ġcoverage":10191,"Ġrc":10192,"_matrix":10193,"_box":10194,"Ġcharges":10195,"ĠBoston":10196,"Pe":10197,"Ġcircum":10198,"Ġfilled":10199,"Ġnorth":10200,"ictureBox":10201,"ĉres":10202,"è®":10203,"Ġtermin":10204,"Ġ[â̦":10205,"IRECT":10206,"Ġber":10207,"Ġ\"../../":10208,"retch":10209,".code":10210,"_col":10211,"ĠGovernment":10212,"Ġargv":10213,"ĠLord":10214,"asi":10215,"Exec":10216,"ĉlet":10217,"vertis":10218,"Ġdiscussion":10219,"enance":10220,"outube":10221,"typeof":10222,"Ġserved":10223,"ĠPut":10224,"ĉx":10225,"Ġsweet":10226,"Before":10227,"ategy":10228,".of":10229,"ĠMaterial":10230,"Sort":10231,"ONT":10232,"igital":10233,"Why":10234,"Ġsust":10235,"Ġç":10236,"abet":10237,"Ġsegment":10238,"Ġ[],Ċ":10239,"ĠMuslim":10240,"ĠfindViewById":10241,"cut":10242,"_TEXT":10243,"ĠMary":10244,"Ġloved":10245,"Ġlie":10246,"ĠJO":10247,"Ġisset":10248,"month":10249,"Ġprime":10250,"ti":10251,"ĠCarol":10252,"Use":10253,"ĠPop":10254,"ĠSave":10255,"Interval":10256,"execute":10257,"dy":10258,"ĠIran":10259,"_cont":10260,"ĉT":10261,"Ġphase":10262,"checkbox":10263,"week":10264,"Ġhide":10265,"Ġtil":10266,"Ġju":10267,"Custom":10268,"burg":10269,"/M":10270,"TON":10271,"Ġquant":10272,"Ġrub":10273,"ixels":10274,"Ġinstalled":10275,"Ġdump":10276,"Ġproperly":10277,"(List":10278,"Ġdecide":10279,"apply":10280,"Has":10281,"Ġkeeping":10282,"Ġcitizens":10283,"Ġjoint":10284,"pool":10285,"Socket":10286,"_op":10287,"Ġweapon":10288,"gnore":10289,"ĠExec":10290,"otten":10291,"ĠMS":10292,"Ġ(-":10293,"ĠReview":10294,"Ġexamples":10295,"Ġtight":10296,"!(":10297,"DP":10298,"ĠMessageBox":10299,"Ġphotograph":10300,"URI":10301,"ét":10302,"low":10303,"ĠGrand":10304,".persistence":10305,"Ġmaintain":10306,"Ġnums":10307,"Ġzip":10308,"ials":10309,"ĠGets":10310,"peg":10311,"ĠBuffer":10312,"~~~~":10313,"rastructure":10314,"ĠPL":10315,"uen":10316,"obby":10317,"sizeof":10318,"Ġpic":10319,"Ġseed":10320,"Ġexperienced":10321,"Ġodd":10322,"Ġkick":10323,"Ġprocedure":10324,"avigator":10325,"-on":10326,",j":10327,"ĠAlthough":10328,"ĠuserId":10329,"accept":10330,"Blue":10331,"IColor":10332,"layer":10333,"available":10334,"Ġends":10335,".table":10336,"Ġdataset":10337,"bus":10338,"Ġexplain":10339,"(pro":10340,"ĠCommittee":10341,"Ġnoted":10342,"]:Ċ":10343,"Dim":10344,"stdio":10345,".\",Ċ":10346,"_source":10347,"ĠWeek":10348,"ĠEdge":10349,"Ġoperating":10350,"Ġeste":10351,"ipl":10352,"agination":10353,"Ġproceed":10354,"Ġanimation":10355,".Models":10356,"ĠWatch":10357,"iat":10358,"Ġoppon":10359,"/A":10360,"Report":10361,"Ġsounds":10362,"_buf":10363,"IELD":10364,"Ġbund":10365,"ĉget":10366,".pr":10367,"(tmp":10368,"Ġkid":10369,">ĊĊĊ":10370,"Ġyang":10371,"NotFound":10372,"ÑĨ":10373,"math":10374,"@gmail":10375,"ĠLIMIT":10376,"redients":10377,"Ġvent":10378,"avigate":10379,"Look":10380,"Ġreligious":10381,"Ġrand":10382,"rio":10383,"(GL":10384,"_ip":10385,"uan":10386,"iciency":10387,"ĠChange":10388,">čĊčĊ":10389,"ĠEntity":10390,"Ġrencontre":10391,"ĠRet":10392,"plan":10393,"én":10394,"BOOL":10395,"uries":10396,"train":10397,"Definition":10398,"============":10399,"zz":10400,"Animation":10401,"ĠOK":10402,"_menu":10403,".bl":10404,"_score":10405,"Ġacad":10406,"(System":10407,"Ġrefresh":10408,"'=>$":10409,".Graphics":10410,"amento":10411,"pid":10412,"tc":10413,"Ġtips":10414,"Ġhomes":10415,"Ġfuel":10416,"âĸ":10417,"_helper":10418,"ĠĠčĊ":10419,"ĠRoom":10420,".Close":10421,"_attr":10422,"ĠMount":10423,"ĠEv":10424,"arser":10425,"_top":10426,"eah":10427,"ĠDelete":10428,"ãĢį":10429,"uke":10430,"Ġusage":10431,"aria":10432,"_dev":10433,"Ġtexture":10434,"Ġconversation":10435,"eper":10436,"Bean":10437,"done":10438,"nonatomic":10439,"ĠSecond":10440,"Ġshooting":10441,"_pre":10442,"Components":10443,"Ġ]ĊĊ":10444,"__,":10445,"stitution":10446,".Char":10447,">();ĊĊ":10448,"Ġpresented":10449,"Ġwa":10450,"oker":10451,"-ĊĊ":10452,"iner":10453,"Ġbecoming":10454,"Ġincident":10455,"Att":10456,"Ġrevealed":10457,"forc":10458,"Ġboot":10459,".page":10460,"Enumerator":10461,"_->":10462,"Photo":10463,"Ġspring":10464,".\",":10465,"ĠDictionary":10466,"BJECT":10467,"Ġlocations":10468,"Ġsamples":10469,"InputStream":10470,"ĠBrown":10471,"Ġstats":10472,"quality":10473,"Ñħ":10474,"-dis":10475,"Ġhelping":10476,"Ġped":10477,"(se":10478,"ĠWho":10479,"alian":10480,"internal":10481,"Ġft":10482,">().":10483,"->{":10484,"Ġmine":10485,"Ġsector":10486,"Ġgro":10487,"Ġopportunities":10488,"Ġü":10489,"Ġmp":10490,"Ġalleged":10491,"Ġdoubt":10492,"Mouse":10493,"About":10494,"_part":10495,"Ġchair":10496,"Ġstopped":10497,"loop":10498,"entities":10499,"Ġapps":10500,"ansion":10501,"Ġmental":10502,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":10503,"FR":10504,"Ġdefend":10505,"care":10506,"Ġideal":10507,"/api":10508,"urface":10509,"Ġele":10510,"ulator":10511,"ĠRights":10512,"anguages":10513,"Ġfunds":10514,"Ġadapt":10515,"Attributes":10516,"Ġdeploy":10517,"opts":10518,"Ġvalidation":10519,"Ġconcerns":10520,"uce":10521,".num":10522,"ulture":10523,"ila":10524,"Ġcup":10525,"Ġpure":10526,".Fore":10527,"ĠHashMap":10528,".valueOf":10529,"asm":10530,"MO":10531,"Ġcs":10532,"Ġstores":10533,"Ġ************************************************************************":10534,"Ġcommunication":10535,"mem":10536,".EventHandler":10537,".Status":10538,"_right":10539,".setOn":10540,"Sheet":10541,"Ġidentify":10542,"enerated":10543,"ordered":10544,"Ġ\"[":10545,"Ġswe":10546,"Condition":10547,"ĠAccording":10548,"Ġprepare":10549,"Ġrob":10550,"Pool":10551,"Ġsport":10552,"rv":10553,"ĠRouter":10554,"Ġalternative":10555,"([]":10556,"ĠChicago":10557,"ipher":10558,"ische":10559,"ĠDirector":10560,"kl":10561,"ĠWil":10562,"keys":10563,"Ġmysql":10564,"Ġwelcome":10565,"king":10566,"ĠManager":10567,"Ġcaught":10568,")}Ċ":10569,"Score":10570,"_PR":10571,"Ġsurvey":10572,"hab":10573,"Headers":10574,"ADER":10575,"Ġdecor":10576,"Ġturns":10577,"Ġradius":10578,"errupt":10579,"Cor":10580,"Ġmel":10581,"Ġintr":10582,"(q":10583,"ĠAC":10584,"amos":10585,"MAX":10586,"ĠGrid":10587,"ĠJesus":10588,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":10589,".DE":10590,"Ġts":10591,"Ġlinked":10592,"free":10593,"ĠQt":10594,"Ġ/**čĊ":10595,"Ġfaster":10596,"ctr":10597,"_J":10598,"DT":10599,".Check":10600,"Ġcombination":10601,"Ġintended":10602,"-the":10603,"-type":10604,"ectors":10605,"ami":10606,"uting":10607,"Ġuma":10608,"XML":10609,"UCT":10610,"Ap":10611,"ĠRandom":10612,"Ġran":10613,".sort":10614,"Ġsorted":10615,".Un":10616,"_PER":10617,"itory":10618,"Ġpriority":10619,"ĠGal":10620,"ĠOld":10621,"hot":10622,"ĠDisplay":10623,"(sub":10624,"_TH":10625,"_Y":10626,"ĠCare":10627,"loading":10628,"Kind":10629,"_handle":10630,",,":10631,"rase":10632,"_replace":10633,".addEventListener":10634,"ĠRT":10635,"Ġentered":10636,"gers":10637,"Ġich":10638,"(start":10639,"/app":10640,"Ġbrother":10641,"Memory":10642,"Outlet":10643,"Ġutf":10644,"prec":10645,"Ġnavigation":10646,"ORK":10647,"Ġdst":10648,"Detail":10649,"Ġaudience":10650,"Ġdur":10651,"Ġcluster":10652,"unched":10653,"Ġ],":10654,"Ġcomfortable":10655,".values":10656,"ĠTotal":10657,"Ġsnap":10658,"Ġstandards":10659,"Ġperformed":10660,"hand":10661,"(\"@":10662,"åŃ":10663,"Ġphil":10664,"ibr":10665,"trim":10666,"Ġforget":10667,"Ġdoctor":10668,".TextBox":10669,"icons":10670,",s":10671,"ĠOp":10672,"Sm":10673,"Stop":10674,"ĉList":10675,"ĉu":10676,"Comment":10677,"_VERSION":10678,".Xtra":10679,"Person":10680,"rb":10681,"LOB":10682,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":10683,"ĠCentral":10684,"ICK":10685,"raq":10686,"Ġputting":10687,"Ġmd":10688,"ĠLove":10689,"Program":10690,"Border":10691,"oor":10692,"Ġallowing":10693,"after":10694,"Ġentries":10695,"ĠMaybe":10696,"]).":10697,"ĠShort":10698,")\\":10699,".now":10700,"friend":10701,"Ġprefer":10702,"ĠGPIO":10703,"osis":10704,"ĠGameObject":10705,"Ġskip":10706,"Ġcompetition":10707,"_match":10708,"lications":10709,"_CONT":10710,".groupBox":10711,"Ġals":10712,"\"We":10713,"_eq":10714,"lan":10715,"_search":10716,"ĠMusic":10717,"asis":10718,"Ġbind":10719,"ĠIsland":10720,"rum":10721,"(E":10722,"Ġseat":10723,"Video":10724,"Ġack":10725,"reek":10726,"={()":10727,"Ġrating":10728,"Ġrestaurant":10729,"DEX":10730,"(buf":10731,"pping":10732,"uality":10733,"Ġleague":10734,"Ġfocused":10735,"apon":10736,"$data":10737,"CLUD":10738,"CLUDING":10739,"Ġabsolute":10740,"(query":10741,"Ġtells":10742,"Ang":10743,"Ġcommunities":10744,"Ġhonest":10745,"oking":10746,"Ġapart":10747,"arity":10748,"/$":10749,"_module":10750,"ĠEnc":10751,".an":10752,".Config":10753,"Cre":10754,"Ġshock":10755,"ĠArab":10756,"IENT":10757,"/re":10758,"Ġretrie":10759,"ycler":10760,"isa":10761,"ĠOrgan":10762,".graph":10763,"Ġí":10764,"ĠBAS":10765,"Enum":10766,"Ġpossibly":10767,"ÑĢаÐ":10768,"ĠJapanese":10769,"Ġcraft":10770,"ĠPlace":10771,"Ġtalent":10772,"Ġfunding":10773,"Ġconfirmed":10774,"Ġcycle":10775,"/x":10776,"GE":10777,"Ġhearing":10778,"Ġplants":10779,"Ġmouth":10780,"pages":10781,"oria":10782,"ĠRemove":10783,"_total":10784,"Ġod":10785,"ollapse":10786,"door":10787,"Ġbought":10788,"Ġaddr":10789,"ARCH":10790,"_dim":10791,"dden":10792,"Ġdecades":10793,"REQUEST":10794,"Ġversions":10795,"fire":10796,"Ġmoves":10797,"fb":10798,"Ġcoffee":10799,".connect":10800,"ĠRow":10801,"Ġschema":10802,"Scope":10803,"-Type":10804,"Ġfighting":10805,"Ġretail":10806,"Ġmodified":10807,"TF":10808,"Files":10809,"nie":10810,"_command":10811,"stone":10812,"ĠÑĤ":10813,"_thread":10814,"Ġbond":10815,"ĠDevelopment":10816,"Ġpt":10817,"FORM":10818,"plet":10819,"Ġidentified":10820,"cpp":10821,"Ġcoding":10822,"oked":10823,"ĠMaster":10824,"IDTH":10825,"Ġresidents":10826,"redit":10827,"ĠPhoto":10828,"=-":10829,"unte":10830,"ateur":10831,"_STATE":10832,"ĠSing":10833,"Ġsheet":10834,".val":10835,"orse":10836,"Ġhers":10837,"Ġdetermined":10838,"Common":10839,"Ġwed":10840,"_queue":10841,"PH":10842,"ĠAtl":10843,"cred":10844,"/LICENSE":10845,"Ġmes":10846,"Ġadvanced":10847,".java":10848,".Sh":10849,"Go":10850,"kill":10851,"fp":10852,"_settings":10853,"Ġpal":10854,"Ġtruck":10855,"Ġcombined":10856,"Ġ\"${":10857,"ĠCorpor":10858,"Ġjoined":10859,"ĠJose":10860,"ĠCup":10861,"uns":10862,"estival":10863,"levision":10864,"Ġbroken":10865,"Ġmarriage":10866,"ĠWestern":10867,"Ġrepresents":10868,"ĠTitle":10869,"Ġss":10870,".Ass":10871,"ongoose":10872,"iento":10873,"<>();Ċ":10874,"Ġabsolutely":10875,"Ġsmooth":10876,"TERN":10877,"ĠUnless":10878,"Word":10879,"Ġmerge":10880,"igan":10881,"ĠVol":10882,"Ġnn":10883,".getId":10884,"Ġз":10885,"Ġsexy":10886,"Ġseeking":10887,"Single":10888,".this":10889,"Ġkom":10890,"bound":10891,";\"":10892,"ĠfontSize":10893,"_df":10894,"Ġinjury":10895,"(H":10896,"Ġissued":10897,"_END":10898,":self":10899,"Ġpatch":10900,"Ġleaves":10901,"Ġadopt":10902,"FileName":10903,"ãĢIJ":10904,"Ġexecutive":10905,"ĠByte":10906,"]))Ċ":10907,"Ġnu":10908,"outing":10909,"cluding":10910,"-R":10911,".options":10912,"Ġsubstant":10913,"avax":10914,"ĠBUT":10915,"Ġtechnical":10916,"Ġtwice":10917,"Ġmás":10918,"Ġunivers":10919,"yr":10920,"Ġdrag":10921,"ĠDC":10922,"Ġsed":10923,"Ġbot":10924,"ĠPal":10925,"ĠHall":10926,"forcement":10927,"Ġauch":10928,".mod":10929,"notation":10930,"_files":10931,".line":10932,"_flag":10933,"[name":10934,"Ġresolution":10935,"Ġbott":10936,"(\"[":10937,"ende":10938,"(arr":10939,"Free":10940,"(@\"":10941,"ĠDistrict":10942,"PEC":10943,":-":10944,"Picker":10945,"ĠJo":10946,"ĠĠĠĠĠĊ":10947,"ĠRiver":10948,"_rows":10949,"Ġhelpful":10950,"Ġmassive":10951,"---Ċ":10952,"Ġmeasures":10953,"ĠRuntime":10954,"Ġworry":10955,"ĠSpec":10956,"ĉD":10957,"ãĢij":10958,"Ġ){Ċ":10959,"Ġworse":10960,"(filename":10961,"Ġlay":10962,"Ġmagic":10963,"ĠTheir":10964,"oul":10965,"stroy":10966,"ĠWhere":10967,"Ġsudden":10968,"Ġdefe":10969,"Ġbinding":10970,"Ġflight":10971,"ĠOnInit":10972,"ĠWomen":10973,"ĠPolicy":10974,"Ġdrugs":10975,"ishing":10976,"('../":10977,"ĠMel":10978,"peat":10979,"tor":10980,"Ġproposed":10981,"Ġstated":10982,"_RES":10983,"Ġeast":10984,"ĠCONDITION":10985,"_desc":10986,"Ġwinning":10987,"folio":10988,"Mapper":10989,"ĠPan":10990,"ĠAnge":10991,".servlet":10992,"Ġcopies":10993,"LM":10994,"Ġvm":10995,"åį":10996,"Ġdictionary":10997,"Seg":10998,"elines":10999,"ĠSend":11000,"Ġiron":11001,"ĠFort":11002,".domain":11003,"Ġdebate":11004,"NotNull":11005,"eq":11006,"acher":11007,"lf":11008,"ĉfmt":11009,"Ġlawy":11010,"ÄŁ":11011,"ĠMen":11012,"Ġtrim":11013,"(NULL":11014,"Ġ!!":11015,"Ġpad":11016,"Ġfollows":11017,"\"][\"":11018,"requ":11019,"ĠEp":11020,".github":11021,"(img":11022,"eto":11023,"('\\":11024,"Services":11025,"umbnail":11026,"_main":11027,"pleted":11028,"fortunately":11029,"Ġwindows":11030,"Ġplane":11031,"ĠConnection":11032,".local":11033,"uard":11034,"}\\":11035,"==\"":11036,"andon":11037,"ĠRoy":11038,"west":11039,"iginal":11040,"emies":11041,"itz":11042,"'):Ċ":11043,"ĠPeter":11044,"Ġtough":11045,"Ġreduced":11046,"Ġcalculate":11047,"Ġrapid":11048,"customer":11049,"Ġefficient":11050,"Ġmedium":11051,"Ġfell":11052,".ref":11053,"ĠCas":11054,"Ġfeedback":11055,"Speed":11056,"(output":11057,"aje":11058,"Ġcategories":11059,"Ġfee":11060,"};":11061,"Ġdeleted":11062,"reh":11063,"Ġproof":11064,"Desc":11065,"Build":11066,"Ġsides":11067,".ArrayList":11068,"-%":11069,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":11070,"ر":11071,".match":11072,"ли":11073,"Ġfeels":11074,"Ġachieve":11075,"Ġclim":11076,"_ON":11077,"ĠCD":11078,"Ġteacher":11079,"_current":11080,"bn":11081,"_PL":11082,"isting":11083,"Enable":11084,"GEN":11085,"Ġtv":11086,"Ġsock":11087,"Ġplays":11088,"Ġdiscount":11089,"ĠKE":11090,"ĠDebug":11091,"Fore":11092,"ĠIraq":11093,"Ġappearance":11094,"Mon":11095,"Ġstyled":11096,"ĠHuman":11097,"iot":11098,"ĠHistory":11099,"Ġsac":11100,"ĠCollection":11101,"Ġrecommended":11102,".Selected":11103,"Ġorganizations":11104,"Ġdiscovered":11105,"cohol":11106,"adas":11107,"ĠThomas":11108,"May":11109,"Ġconserv":11110,"Ġdomin":11111,"ĠFollow":11112,"ĠSection":11113,"ĠThanks":11114,"Username":11115,"Ġrecipe":11116,"Ġwonderful":11117,".sleep":11118,"_if":11119,"ĉĊĉĊ":11120,"orno":11121,"Ġru":11122,"_target":11123,".\"\"":11124,"à¦":11125,"EventArgs":11126,"Ġinputs":11127,"Ġfif":11128,"Ġvision":11129,"cy":11130,"ĠSeries":11131,")(((":11132,"Ġtrading":11133,"Ġmarker":11134,"Begin":11135,"Ġtypically":11136,"Ġcauses":11137,"dropdown":11138,"_DEBUG":11139,"Ġdetect":11140,"country":11141,"!\");Ċ":11142,"ĉR":11143,"appy":11144,"Ġcref":11145,"('<":11146,"\"=>":11147,"ĠLE":11148,"reader":11149,"Ġadministr":11150,"õ":11151,"ucket":11152,"Ġfashion":11153,".char":11154,"izar":11155,"Ġdisable":11156,"Ġsuc":11157,"ĠLive":11158,"issue":11159,"Ġmetadata":11160,"flags":11161,"ĠðŁ":11162,"Ġcommitted":11163,"Ġva":11164,"Ġrough":11165,"Ġ'''Ċ":11166,"Ġhighlight":11167,"_vars":11168,"VO":11169,"Ġencoding":11170,"-Z":11171,"_sign":11172,"$(\"#":11173,"Ġrain":11174,"reatest":11175,"ĠEND":11176,"Selection":11177,"Ġcandidates":11178,"Ġsav":11179,".Empty":11180,"Ġdecisions":11181,"Ġcollabor":11182,"ridge":11183,"feed":11184,"ression":11185,"Ġpersons":11186,"VM":11187,"ega":11188,"_BIT":11189,"According":11190,"acked":11191,"Ġdollars":11192,"_loss":11193,"ĠCost":11194,"}\"Ċ":11195,"Notification":11196,"Ġprostit":11197,"Ġauthority":11198,".rec":11199,"Ġspokes":11200,"ĠToday":11201,"istant":11202,"ĠHead":11203,"âĢĿ.":11204,"ertainment":11205,"cean":11206,"culate":11207,"Ġven":11208,"However":11209,"_arr":11210,"Ġtokens":11211,"Graph":11212,"ĠJud":11213,"ĠVirgin":11214,"ĠSerial":11215,"unning":11216,"Mutable":11217,"agers":11218,".csv":11219,"Ġdeveloping":11220,"Ġinstructions":11221,"Ġpromise":11222,"Ġrequested":11223,"_encode":11224,"/\"":11225,"ĠIcon":11226,"uilt":11227,"-day":11228,"Ġintelligence":11229,".IS":11230,"ĠObservable":11231,"ĠHard":11232,"Bool":11233,"idential":11234,".Anchor":11235,"Ġselling":11236,"CI":11237,"AGES":11238,"tle":11239,"bur":11240,"UFFER":11241,"RY":11242,"Ġbigger":11243,"Ġrat":11244,"Ġfamous":11245,"Ġtypename":11246,"Ġexplained":11247,"}}Ċ":11248,"Ġnuclear":11249,"-N":11250,"Ġcrisis":11251,"ĠEnter":11252,"Ġanswers":11253,"/${":11254,"/pl":11255,"Ġsequ":11256,"_next":11257,"mask":11258,"Ġstanding":11259,"Ġplenty":11260,"ĠCross":11261,"ĉret":11262,"dro":11263,"ĠCast":11264,"=true":11265,"ĠChris":11266,"icio":11267,"ĠMike":11268,"Decimal":11269,"addComponent":11270,"Len":11271,"Ġcock":11272,"Ġ#{":11273,"URN":11274,"":11403,"Ġ*=":11404,"ĠPS":11405,"Ġdangerous":11406,"[p":11407,"OME":11408,"Other":11409,"ĠStringBuilder":11410,"Points":11411,"heading":11412,"Ġcurrency":11413,"Ġpercentage":11414,"_API":11415,"Ġclassic":11416,"thead":11417,"ĠMO":11418,"FE":11419,"Idx":11420,"await":11421,"Ġè":11422,"Ġaccident":11423,"Ġvariant":11424,"Ġmyst":11425,"ĠLand":11426,"ĠBre":11427,"Ġharm":11428,"ĠAcc":11429,"Ġcharged":11430,"iones":11431,"Visibility":11432,"arry":11433,"ĠLanguage":11434,"Ġwalking":11435,"\".ĊĊ":11436,"ifer":11437,"Ġleadership":11438,".From":11439,"ynam":11440,"Ġtimestamp":11441,"ipt":11442,"ĠHas":11443,"REFER":11444,"ĠIts":11445,"Ġlistener":11446,"UTE":11447,"_description":11448,"Ġexperiences":11449,"Ġcreates":11450,"RS":11451,"cart":11452,"black":11453,"Ġchoices":11454,"war":11455,"Ġ'''":11456,"Ġordered":11457,"Ġevening":11458,"Ġpil":11459,"Ġtun":11460,"ĠBad":11461,"(app":11462,"random":11463,"Ġexplicit":11464,"Ġarrived":11465,"Ġfly":11466,"Ġeconom":11467,"-mail":11468,"Ġlists":11469,"Ġarchitect":11470,"ĠPay":11471,"Ġds":11472,"ĠSol":11473,"Ġvehicles":11474,"Hz":11475,"-com":11476,"Ġking":11477,"_equal":11478,"ĠHelp":11479,"Ġabuse":11480,"--;Ċ":11481,"Ġextr":11482,"Ġchemical":11483,"ä¿":11484,"Ġorient":11485,"Ġbreath":11486,"ĠSpace":11487,"(element":11488,"wait":11489,"DED":11490,"igma":11491,"Ġentr":11492,"Ġsob":11493,"-name":11494,"Ġaffected":11495,"ika":11496,"Ġcoal":11497,"_work":11498,"Ġhundreds":11499,"Ġpolitics":11500,"subject":11501,"Ġconsumer":11502,"ANGE":11503,"Ġrepeated":11504,"Send":11505,"Ġ#[":11506,"Ġprotocol":11507,"Ġleads":11508,"useum":11509,"Every":11510,"Import":11511,"(count":11512,"Ġchallenges":11513,"Ġnovel":11514,"Ġdepart":11515,"bits":11516,".Current":11517,"Ġ`${":11518,"oting":11519,"(\\":11520,"Ġcreative":11521,"Ġbuff":11522,"Ġintroduced":11523,"usic":11524,"modules":11525,"Are":11526,"-doc":11527,"language":11528,"_cache":11529,"Ġtod":11530,"?>{{":11764,"ĠResource":11765,"ĠStandard":11766,"ĠPrem":11767,"updated":11768,"ivalent":11769,"Ġassets":11770,"_temp":11771,"Ġinterests":11772,"Ġhardware":11773,"ĠRom":11774,"ĠShare":11775,"Ġ''Ċ":11776,"Ġ*,":11777,"ĠTake":11778,"ĠImages":11779,"_CHECK":11780,"(typeof":11781,"ĠJun":11782,"\\<^":11783,"Ġliqu":11784,"Ġworst":11785,"ymbols":11786,"ĉĉĉĠĠĠ":11787,"Ġdrivers":11788,"ĠDocument":11789,"eno":11790,"ĠTechnology":11791,"Ġapproved":11792,"umps":11793,"Ġsnow":11794,"formance":11795,"_ASSERT":11796,"uits":11797,"ÙĨ":11798,"Ġdifferences":11799,".Visible":11800,"ĉĉĉčĊ":11801,"ĠPs":11802,"_fetch":11803,"Ġtodo":11804,".',Ċ":11805,"Ġsel":11806,"urers":11807,"invalid":11808,"Ġtweet":11809,"VEL":11810,"Ġresearchers":11811,"Ġsprintf":11812,"ĠRO":11813,"Ġpel":11814,".Trans":11815,"Ġillegal":11816,"dialog":11817,"smarty":11818,"lg":11819,"_MIN":11820,"Ġhero":11821,"final":11822,"Ġpp":11823,".Le":11824,"Ġci":11825,"ĉRT":11826,"Ġsuggested":11827,"pdf":11828,"aching":11829,"ĠRo":11830,"ĠProperties":11831,"ĠSi":11832,"Ġbuying":11833,"Ġmu":11834,"Ġlands":11835,"ifiers":11836,"ĠFILE":11837,"ROUP":11838,"Ġholder":11839,"ĠSon":11840,"Ġsympt":11841,".route":11842,")?":11843,"Ġargc":11844,"Ġfort":11845,"Ġcasino":11846,"_category":11847,"Ġforum":11848,"prefix":11849,"apture":11850,"Tube":11851,"ems":11852,"imize":11853,"Ġnue":11854,"aus":11855,"course":11856,"ATOR":11857,"()),":11858,"Advertis":11859,"INGS":11860,"Ġacknow":11861,"ĠKorea":11862,"pling":11863,"Ġworker":11864,"PLIED":11865,"hal":11866,"ĠRichard":11867,"Elements":11868,"ĉĉĉĠ":11869,"star":11870,"Ġrelationships":11871,"Ġcheap":11872,"ACH":11873,"ĠXML":11874,",&":11875,"ĠLouis":11876,"Ġride":11877,"_FAIL":11878,"Ġchunk":11879,"[s":11880,"_OUT":11881,"Ġchosen":11882,"_[":11883,"/(":11884,"ĠJeff":11885,"_sl":11886,"priv":11887,"ĠCanadian":11888,"Ġunable":11889,"_FLAG":11890,"Ġnos":11891,"high":11892,"Ġlift":11893,"fun":11894,"(){":11895,"elly":11896,"yclerView":11897,"_as":11898,"_LIST":11899,"Ġradi":11900,".getValue":11901,"ĠAngeles":11902,"ĠSpan":11903,"_instance":11904,"itors":11905,"Ġmigration":11906,"AK":11907,"Oh":11908,"®":11909,".selected":11910,"ĠGT":11911,"Ġadvance":11912,"ĠStyle":11913,".DataGridView":11914,"ection":11915,"Ñİ":11916,"pio":11917,"rog":11918,"Ġshopping":11919,"ĠRect":11920,"Illuminate":11921,"OU":11922,"ĉarray":11923,"Ġsubstantial":11924,"Ġpregn":11925,"Ġpromote":11926,"IEW":11927,".Layout":11928,"Ġsigns":11929,"/.":11930,"Ġletters":11931,"Board":11932,"ctrl":11933,"\"\\":11934,"ĠJones":11935,"Ġvertex":11936,"Ġja":11937,"Ġaffili":11938,"Ġwealth":11939,"ĉdefault":11940,"Ġsignificantly":11941,"Ġec":11942,"Ġxs":11943,"actual":11944,".per":11945,"_step":11946,"anvas":11947,"mac":11948,"Ġtransl":11949,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":11950,"Iterator":11951,"Ġoch":11952,"agnostic":11953,"ĠDuring":11954,"ĠDEFAULT":11955,"Ġtill":11956,"Ġsignature":11957,"Ġbird":11958,"ĠOl":11959,"ĠIr":11960,"HS":11961,"avatar":11962,"ESSAGE":11963,"Ġelev":11964,"Ġmt":11965,"ĠNav":11966,"Ġrelax":11967,"Ġplate":11968,"ITEM":11969,"(date":11970,".not":11971,"Ġgrade":11972,"Ġ}),Ċ":11973,"?\"ĊĊ":11974,"iences":11975,"High":11976,"ĠDIS":11977,"disabled":11978,"QUI":11979,"Ġnoise":11980,"aux":11981,"ĠUP":11982,"osa":11983,"Ġvoc":11984,"Ġ))":11985,"ocom":11986,"_OFF":11987,"ĠDb":11988,"Lock":11989,".eclipse":11990,",d":11991,"ĠDraw":11992,"Ġ\"(":11993,"Ġvisited":11994,"ĠâĪ":11995,"Ġsucceed":11996,"Ġimpossible":11997,"aire":11998,"ĠTurn":11999,"Ġdish":12000,"FG":12001,"Ġsensor":12002,"ANN":12003,"aba":12004,"Ġsurg":12005,"]);čĊ":12006,"Ġfp":12007,"_an":12008,"-J":12009,"-G":12010,"ĠJob":12011,"Convert":12012,"ĠKEY":12013,"Ġauthors":12014,"_server":12015,"\\r":12016,"Ġ-*-":12017,"flex":12018,"Ġsoc":12019,"Ret":12020,"Ġsalt":12021,"Ġâ̦ĊĊ":12022,"ĠClear":12023,"(page":12024,"-danger":12025,"Ġrooms":12026,"conv":12027,"#{":12028,".op":12029,"ĠArea":12030,"_SC":12031,"hen":12032,"Ġbegins":12033,"-y":12034,"Ġexcited":12035,"Ġignored":12036,"Ġbonus":12037,"student":12038,"ĠMember":12039,"Ġrelatively":12040,"ĠLow":12041,"ĠProdu":12042,"ateway":12043,"posure":12044,"Ġthick":12045,"aniel":12046,"(view":12047,"ĠCrush":12048,"Extension":12049,"Il":12050,"eed":12051,"LOC":12052,".im":12053,".Items":12054,"Ġconflict":12055,".prevent":12056,"ĠonCreate":12057,"uv":12058,"iser":12059,"Ġwave":12060,"Mar":12061,"ĠCommunity":12062,"iche":12063,"ĠNothing":12064,"[m":12065,"ĠLee":12066,"riends":12067,"ère":12068,"!!!":12069,"anz":12070,".result":12071,"ĠSK":12072,"_PARAM":12073,"Ġdemocr":12074,"BackColor":12075,".exists":12076,"\"It":12077,"(options":12078,"razy":12079,"aser":12080,"\\Database":12081,"alendar":12082,"_ass":12083,";}Ċ":12084,"vertex":12085,"inecraft":12086,"Warning":12087,"argo":12088,"Ġactor":12089,"ĠInstead":12090,"ĠUsing":12091,"Self":12092,"@interface":12093,"Ġspeaking":12094,"ĠParis":12095,"ĠLICENSE":12096,".node":12097,"ĠFood":12098,"EIF":12099,"ĠBi":12100,".Start":12101,"ĠIB":12102,"Ġuniversity":12103,"ĠHeader":12104,".product":12105,"Copy":12106,"etc":12107,"rical":12108,"Ġ>>>":12109,"books":12110,"Ġalgorithm":12111,"Ġ'__":12112,"(javax":12113,"Ġnumerous":12114,"Share":12115,"Have":12116,"Ġrecru":12117,"Ġprove":12118,".substring":12119,"health":12120,"ел":12121,"Ġdecimal":12122,"Ġcommission":12123,"scription":12124,"xC":12125,"Ġsummary":12126,"atted":12127,"Ġcloser":12128,"finished":12129,"()){Ċ":12130,"ĠWood":12131,"_fields":12132,"ku":12133,"_items":12134,"Flag":12135,"Ġconfidence":12136,"ĠFederal":12137,"dux":12138,"Ġcompat":12139,"Ġvertical":12140,"й":12141,"ès":12142,";\">Ċ":12143,"_manager":12144,"()))Ċ":12145,"IDE":12146,":\",":12147,"__Ċ":12148,"ĠWay":12149,"ÑĪ":12150,"Temp":12151,"ĠSTR":12152,"ritten":12153,"Sync":12154,"ĠAV":12155,"ĠCEO":12156,"ĠGuid":12157,"Ġenvironmental":12158,"Ġcorresponding":12159,"ĉconsole":12160,"Ġjustice":12161,"ĠJS":12162,"Ġlived":12163,"gar":12164,"ĠGraph":12165,"ĠStat":12166,"ĠiPhone":12167,".al":12168,"ĠHD":12169,"Ġoccur":12170,"Ġthreshold":12171,"Ġonclick":12172,"REG":12173,".GraphicsUnit":12174,"Meta":12175,"ž":12176,"Ġcum":12177,".gnu":12178,"ë":12179,"Ġobtained":12180,"Ġcomplaint":12181,"Ġeating":12182,"Ġtar":12183,"_task":12184,"Ġopts":12185,"(to":12186,"Pass":12187,"Ġplastic":12188,"tility":12189,"ĠWin":12190,".preventDefault":12191,"pile":12192,"ĠGar":12193,"Ġquantity":12194,"_last":12195,"Ġgreatest":12196,"Dao":12197,"_DIS":12198,"ĠUsed":12199,"ĠHP":12200,"riting":12201,"SION":12202,"blue":12203,"domain":12204,"Ġscores":12205,"Normal":12206,"_admin":12207,"ĠASSERT":12208,"Then":12209,"***":12210,"dist":12211,"lon":12212,"Ġhate":12213,"shal":12214,"ImageView":12215,"database":12216,"Ġpand":12217,"Ġlogic":12218,"=false":12219,"bg":12220,"ĠConfiguration":12221,"Ġnur":12222,"OG":12223,"Ġmarried":12224,":+":12225,"Ġdropped":12226,"Ġregistration":12227,"ом":12228,"ultiple":12229,"izers":12230,"shape":12231,".copy":12232,"Ġwearing":12233,"ĠCath":12234,"Ġdedicated":12235,"Ġ...Ċ":12236,"Ġadvoc":12237,"ĠFamily":12238,"Ġstatements":12239,"ematic":12240,"ampionship":12241,"Ġmotiv":12242,"ĠHave":12243,"Ġblow":12244,"Job":12245,"cert":12246,"_vector":12247,"install":12248,"ĠCOPY":12249,"embed":12250,"DIR":12251,"ĠSpring":12252,"Ġexhib":12253,"cdn":12254,"ĠComment":12255,"ĠOptional":12256,".player":12257,"ĠDark":12258,"(pos":12259,"ĠShould":12260,"Ġcentre":12261,"ĠGuard":12262,"ów":12263,"Ġtrouble":12264,"ENER":12265,"(unsigned":12266,"_service":12267,"Ġns":12268,"uling":12269,"ĠMexico":12270,"ĠNY":12271,"mysql":12272,"Ġlic":12273,"åľ":12274,"Mr":12275,"-fl":12276,"ĠCustomer":12277,"idi":12278,"Ġ?>ĊĊ":12279,"rible":12280,"ĠпÑĢ":12281,"Ġsizes":12282,"_STRING":12283,"validation":12284,"ĠJon":12285,"(Http":12286,"addClass":12287,"Nodes":12288,"Ġfragment":12289,"Ġspoke":12290,"Ġwaste":12291,"Join":12292,"Ġillustr":12293,"eli":12294,"cient":12295,"Ġaid":12296,"Ġprosec":12297,"'){Ċ":12298,"Ġpassing":12299,"Ġfaces":12300,"Shape":12301,"_Z":12302,"iti":12303,"Ġalle":12304,"Ġrobot":12305,"ĠĠĠĠĠĠĠĊ":12306,"ĠSpe":12307,"Ġreceiving":12308,"ĠDetails":12309,"Ġ\")":12310,"mg":12311,"_REF":12312,"Ġcomparison":12313,"*,":12314,"ĠFound":12315,"_session":12316,"(U":12317,"/F":12318,"Ġxxx":12319,"Network":12320,"ders":12321,"Ġcapture":12322,"Ġcorre":12323,"ĠLtd":12324,"ĠAdv":12325,"[@":12326,"Ġclip":12327,"Mill":12328,"ĠProfile":12329,"Ġendif":12330,"Ġoblig":12331,"describe":12332,".element":12333,"riterion":12334,"LD":12335,"ered":12336,"Ġfavour":12337,"score":12338,"ĠFilter":12339,"attributes":12340,"Ġchecks":12341,"Inflater":12342,"ĠPlus":12343,"Ġscientific":12344,"Ġprivacy":12345,"Head":12346,"Ġfeat":12347,"Ġdegrees":12348,"ĠPale":12349,";\">":12350,"Ġfilms":12351,"ĠAudio":12352,"ĠTag":12353,"ĠEnergy":12354,"itar":12355,"parator":12356,"Ġfellow":12357,"Ġevt":12358,"ĠTri":12359,"ĠDAM":12360,"cloud":12361,"ĠPassword":12362,"ĠDemocrats":12363,"ĠAcad":12364,"$lang":12365,"Ġreb":12366,"())ĊĊ":12367,"нÑĭ":12368,"ĠBur":12369,"readcr":12370,"Ġhex":12371,"Console":12372,"ctl":12373,"ousel":12374,"ĠWilliam":12375,"Ġaz":12376,"_PORT":12377,"Ġpractices":12378,"Ġanywhere":12379,"ĠPosition":12380,"Ġ->Ċ":12381,"iams":12382,".username":12383,"placeholder":12384,"Ġoder":12385,"ĠSecretary":12386,"ĠiT":12387,"mond":12388,"events":12389,"?âĢĿ":12390,".Sub":12391,"Ġattached":12392,"Ġnão":12393,"Ġestate":12394,".action":12395,"Ġfigures":12396,"Ġ});čĊ":12397,"Ġsubscri":12398,".tag":12399,"nam":12400,".plot":12401,"noon":12402,"liament":12403,"Character":12404,".tab":12405,"Ġwinter":12406,"ĠVariable":12407,"Ġtrees":12408,"Ġproud":12409,"(V":12410,"_load":12411,"Ġhier":12412,"ĠEcon":12413,"Ġfd":12414,"Ġvictims":12415,"Rest":12416,"iana":12417,"Ġfake":12418,".Println":12419,"Ġstrlen":12420,"Ġsad":12421,"Ġble":12422,"Prot":12423,"Ġbuttons":12424,"Ġtelevision":12425,"Ġlogo":12426,"extension":12427,"ĉj":12428,"stein":12429,"aciones":12430,"Ġ\"\"\"ĊĊ":12431,"Ġsimp":12432,"Ġrecorded":12433,"Ġbrings":12434,"Ġprincipal":12435,"Ġfees":12436,"(source":12437,"kdir":12438,"Ġutils":12439,"Ġcorrectly":12440,"fil":12441,"Ġwel":12442,"Pair":12443,"-button":12444,"scale":12445,"verify":12446,"[c":12447,"Ġ---":12448,"Ġescape":12449,"ikes":12450,"LowerCase":12451,"ician":12452,"Ġchapter":12453,"ĠTYPE":12454,"Ġshadow":12455,"Ġawesome":12456,"WE":12457,"elif":12458,"Ġlambda":12459,"Ġdistinct":12460,"Ġbare":12461,"-off":12462,"Ġcolour":12463,".appendChild":12464,"olec":12465,"aga":12466,".fill":12467,"ĉsuper":12468,"Ġadj":12469,"(position":12470,".getItem":12471,"Short":12472,"Ġtotally":12473,"VD":12474,"ĠTre":12475,"_ep":12476,"vements":12477,"ĠSolution":12478,"Ġfundament":12479,"Follow":12480,"Ġfacility":12481,"Ġhappening":12482,"OF":12483,".textBox":12484,"Span":12485,"Ġ«":12486,"iden":12487,"Ġexceed":12488,"(parent":12489,"Ġcp":12490,"ç»":12491,"Ġhasn":12492,"Ġpri":12493,"Ġconsequ":12494,"nen":12495,"ĠINTO":12496,"Ignore":12497,"ĠFuture":12498,"Ġcarbon":12499,"ĠSteel":12500,"fmt":12501,"okie":12502,"Ġspl":12503,"(title":12504,"-info":12505,"Ġdeals":12506,"Ġfixture":12507,"ea":12508,"Div":12509,"Ġtested":12510,"_return":12511,")ĊĊĊĊ":12512,"upported":12513,"ĠCook":12514,"Ġpaying":12515,"ĠIll":12516,"Ġarrested":12517,"ĠPrime":12518,"_callback":12519,">,Ċ":12520,"driver":12521,"Once":12522,"abb":12523,"_bytes":12524,"ĠSets":12525,"(Object":12526,"Ġcc":12527,"Ġshell":12528,"alo":12529,");//":12530,"(log":12531,"ctors":12532,")":13004,"Ġ$(\".":13005,".pos":13006,"Ġboys":13007,"Ġwedding":13008,"Ġagents":13009,"=\"_":13010,"ĠArmy":13011,"Ġhint":13012,"vision":13013,"Ġtech":13014,"ĠConnect":13015,"Ġlegend":13016,"ĠBet":13017,".Base":13018,"Subject":13019,"Ġlit":13020,"Remove":13021,"Ġ\":":13022,"ĠFinal":13023,"pearance":13024,"ĠiTunes":13025,"Ġparticipants":13026,"ĠPython":13027,"Ġbusy":13028,"iel":13029,"vertices":13030,"ĠtemplateUrl":13031,"ĠClose":13032,"Img":13033,"ĠCorporation":13034,"timestamp":13035,"Ġextend":13036,"Ġwebsites":13037,"Ġpossibility":13038,"оÑĤ":13039,"Ġkö":13040,"Ġmeat":13041,"Ġrepresentation":13042,"Ġĉĉ":13043,"_START":13044,".apply":13045,"ĠValley":13046,"ĠSuccess":13047,"Hi":13048,"Ġnob":13049,"ĠIEnumerable":13050,"_select":13051,"geo":13052,".\")Ċ":13053,"Ġturning":13054,"Ġfabric":13055,"(\"\");Ċ":13056,"Ġperspective":13057,"éĹ":13058,"ĠSn":13059,"Thank":13060,";j":13061,".Parameters":13062,"ĉĠĠĠĠĠĠĠĠĠĠĠ":13063,"Ġfacts":13064,"Ġunt":13065,".instance":13066,"################################################################":13067,"-end":13068,"ĠJOIN":13069,"ĠHen":13070,"Ġuri":13071,"åIJį":13072,"Ġна":13073,"ĠInfo":13074,"Ġconducted":13075,"ĠÃ¥":13076,"OURCE":13077,"Ġwine":13078,"John":13079,".Errorf":13080,"ĠAge":13081,"ounded":13082,"Ġrealize":13083,"Ġ];":13084,"Ġsubsequ":13085,",m":13086,"(User":13087,"iano":13088,"Ġaccompl":13089,"isp":13090,".std":13091,"éĩ":13092,"ĠBed":13093,".setAttribute":13094,"BR":13095,"keep":13096,"ĠALL":13097,"Ġisol":13098,"amma":13099,"Package":13100,"Ġoccasion":13101,"-success":13102,"ед":13103,"ĠLIMITED":13104,"strip":13105,"()ĊĊĊ":13106,"istribution":13107,"Colors":13108,"Ġ+:+":13109,"DidLoad":13110,"aler":13111,"Ġtid":13112,"ĠLED":13113,"ĠLinked":13114,"ĠCart":13115,"())čĊ":13116,"_READ":13117,"Ġkilling":13118,"ĠPHP":13119,"fection":13120,"Ġinstances":13121,"cv":13122,"\"/>":13123,"Ġsf":13124,"Ġtaxes":13125,"_location":13126,"ĠBitcoin":13127,"uable":13128,"rank":13129,"ignore":13130,"track":13131,"ка":13132,"Ġshouldn":13133,"ĠOP":13134,"=>{Ċ":13135,"Ġkm":13136,"Ġhelper":13137,"_head":13138,"ĠWhether":13139,"oco":13140,"_bl":13141,"Ġstatistics":13142,"Ġbeauty":13143,"Ġtog":13144,"tip":13145,"ëĭ¤":13146,"Ġcsv":13147,"(sql":13148,"stdlib":13149,"weak":13150,"Ġlikes":13151,"Äį":13152,"Ġrepeat":13153,"Ġapartment":13154,"Ġemph":13155,"_edit":13156,"Ġvit":13157,"ĉtype":13158,"Even":13159,"uten":13160,"Ġcircumstances":13161,"bian":13162,"Ġsugar":13163,"Windows":13164,"ìŀ":13165,"Ġobserved":13166,"/data":13167,"Ġcalendar":13168,"Ġstrike":13169,"ĠRES":13170,"_sc":13171,"fony":13172,"orem":13173,"(z":13174,"power":13175,"etect":13176,"ĠSat":13177,".description":13178,"Ġgang":13179,"ĠSports":13180,"ongs":13181,"ĠBundle":13182,".sum":13183,"once":13184,"Ġaccused":13185,"Ġexplore":13186,"Ġapproximately":13187,"Ġlosing":13188,"thesis":13189,"ĠFund":13190,"Ġdiagn":13191,"Autowired":13192,"properties":13193,"Ġ_.":13194,"Ġcnt":13195,"cedure":13196,"Ġyy":13197,"Ġgrant":13198,"sock":13199,".innerHTML":13200,"Ġ]);Ċ":13201,"ĠCONFIG":13202,"='$":13203,"]];Ċ":13204,"UND":13205,"Ġglob":13206,"Ġdire":13207,"uffle":13208,"_MEM":13209,"Ġauthentic":13210,">(\"":13211,"Ġdecade":13212,"ĠImport":13213,"Ġoriginally":13214,"ĠjQuery":13215,"Ġindicate":13216,"Ġourselves":13217,"Sw":13218,".lbl":13219,"enerate":13220,"Ġbasically":13221,"ĠHom":13222,"Ġ+#+":13223,"ĠBritain":13224,"ĠKar":13225,"toEqual":13226,".stop":13227,"Ġmodal":13228,"isi":13229,"Ġsuggests":13230,"Ġdtype":13231,"Ġtur":13232,"bf":13233,"Ġconnections":13234,"ĠBefore":13235,"isted":13236,"mouse":13237,"Ġpulled":13238,".build":13239,"Ġlegislation":13240,"Ġforth":13241,"pad":13242,"ego":13243,".Now":13244,"Ġexciting":13245,"}ĊĊĊĊ":13246,"Ġcompr":13247,"Ġshares":13248,"Ġrig":13249,"green":13250,"_vec":13251,"Ġenumerate":13252,"Auto":13253,"icator":13254,"ĠRay":13255,"asse":13256,"Ġholiday":13257,"Ġnullable":13258,"gun":13259,"_details":13260,"Ġwrapper":13261,"seq":13262,"ĠYoung":13263,"juana":13264,"Ġ\"__":13265,"license":13266,"serve":13267,"^(":13268,"iders":13269,".Remove":13270,"ropdown":13271,"'S":13272,"pin":13273,"(token":13274,".Default":13275,"Ġreasonable":13276,"ampion":13277,"ĠSociety":13278,"Ġbei":13279,"erves":13280,"rad":13281,"ĠFox":13282,"_images":13283,"Ġwheel":13284,"')[":13285,"Ġcfg":13286,"(By":13287,"Constructor":13288,"Ġvary":13289,".swift":13290,"Ġproxy":13291,"ĉH":13292,"ĠAnother":13293,"ĠPen":13294,"Ġchecking":13295,"Ġjest":13296,"manager":13297,"Origin":13298,"ugs":13299,"oir":13300,">čĊ":15956,"Ġrelief":15957,"lap":15958,"quer":15959,"_parent":15960,"heap":15961,"LOSE":15962,"Ġcombine":15963,"ĠRose":15964,"owers":15965,"Ġprocedures":15966,"ĠSort":15967,"anim":15968,"variant":15969,"ehicle":15970,"Ġsigning":15971,"Primary":15972,"currency":15973,"Ġsexe":15974,"oen":15975,"theta":15976,"eman":15977,"Ġimpressive":15978,"('_":15979,"ĉU":15980,"ĠTextStyle":15981,"_cnt":15982,"Ġslice":15983,"(':":15984,"Ġunderstood":15985,"His":15986,"Ġinformed":15987,"Ġnick":15988,"(TAG":15989,"hd":15990,"Ġelections":15991,"esture":15992,"ĠSanta":15993,"ĠCoast":15994,".pdf":15995,"inciple":15996,".clone":15997,"born":15998,"uta":15999,"Ġlicensed":16000,"Cr":16001,"Ġbread":16002,"ĠHouston":16003,"Ġnod":16004,"Ġhopes":16005,"ĠCGRect":16006,"Ġguilty":16007,".gif":16008,"Ġrose":16009,".Common":16010,"Tip":16011,"ANK":16012,"ĠFC":16013,"During":16014,"ĠSymfony":16015,"Ġdefensive":16016,"km":16017,")>":16018,"archive":16019,"ĠURI":16020,"ycling":16021,"-o":16022,"ĠWebsite":16023,"AMP":16024,"ishment":16025,"Ġdoctors":16026,"Direct":16027,"ARI":16028,"ĠRedirect":16029,"ieren":16030,"_dist":16031,"yo":16032,"ĠProgress":16033,"Ġzum":16034,"Ġmemor":16035,"ĠED":16036,"Ġjur":16037,"æį®":16038,"_TABLE":16039,"Ġuuid":16040,"Expr":16041,".head":16042,"('%":16043,"pointer":16044,"Ġestimate":16045,"ĠGreg":16046,"Ġloader":16047,"ĠiOS":16048,"Ġmens":16049,"[y":16050,"Ġrefused":16051,"Ġprecision":16052,"isch":16053,"ĠACTION":16054,"Cloud":16055,"sWith":16056,"(ret":16057,"_ADDR":16058,"_conf":16059,"(df":16060,"Ġlocked":16061,"Ġrising":16062,"ãĥ»ãĥ»":16063,"ĠMs":16064,"Ġscenes":16065,"_EXT":16066,"_raw":16067,"_the":16068,"people":16069,"Ġrecon":16070,"ĠFun":16071,"Ġbless":16072,"ĠUpdated":16073,"ün":16074,"ĠĠĠĠĠĠĠĠĠĠĠĠčĊ":16075,"pection":16076,"Release":16077,".logger":16078,"ĠSY":16079,"Ġcounsel":16080,"urd":16081,"_true":16082,"Ġeverybody":16083,"ivot":16084,"Ġhence":16085,"ĠNAS":16086,"Ġopposed":16087,"unknown":16088,"ĠDESC":16089,"ĠChair":16090,"failed":16091,"ĠINCLUDING":16092,"Ġwriters":16093,"{}Ċ":16094,"ÃŃt":16095,"_copy":16096,"}:":16097,"ĠBat":16098,"Ġconverted":16099,"eding":16100,"placement":16101,"ĠHost":16102,"Sound":16103,"им":16104,"Ġsought":16105,"mid":16106,"Ġsalary":16107,"ogg":16108,"âĦ¢":16109,"bul":16110,"Ġwir":16111,"validator":16112,"_STAT":16113,".store":16114,"ĠBattle":16115,"ın":16116,"Ġ-->ĊĊ":16117,"Trump":16118,"dot":16119,"ĠCONT":16120,".fetch":16121,"Ġcontinu":16122,"was":16123,"Ġfraud":16124,"_tmp":16125,"mitter":16126,".pictureBox":16127,"GA":16128,"Ġtournament":16129,".Input":16130,"[r":16131,"exion":16132,"centage":16133,"ĠKorean":16134,"undef":16135,"ĠAvailable":16136,"reshape":16137,"Ġkit":16138,"ĠStruct":16139,"ĠSUB":16140,"Answer":16141,"_lib":16142,".twitter":16143,"Ġore":16144,"ĠDragon":16145,".Ext":16146,",k":16147,"Ġexplanation":16148,"refs":16149,"ĠDrive":16150,"ĠTraining":16151,".Has":16152,"intage":16153,"big":16154,"ologist":16155,"ennis":16156,"Ùĩ":16157,"Ġchicken":16158,"ĠĠĠĠĠĠĠĠĠĠĊ":16159,"çĽ":16160,"ãģ§":16161,"Ġpeak":16162,"Ġdrinking":16163,"Ġencode":16164,"ĠNEW":16165,"malloc":16166,"ĉfprintf":16167,"Ġ=================================================================":16168,"including":16169,"Ġprinciples":16170,"ĠMah":16171,"storage":16172,"-key":16173,"Ġkeyword":16174,"%;":16175,"Ġtrained":16176,".contrib":16177,"Ġkv":16178,"__':Ċ":16179,"ĠBoy":16180,"parameter":16181,"Ġsuite":16182,"Ġthousand":16183,"Ġcoordinate":16184,"-generated":16185,"íķĺ":16186,"generated":16187,"Ġadmitted":16188,"Ġpussy":16189,"#w":16190,"Ġswim":16191,"union":16192,"Na":16193,"ĠRoyal":16194,".channel":16195,"Updated":16196,"_ROOT":16197,"Ġvital":16198,"raction":16199,"ĠCrusher":16200,"Ġpreced":16201,"Ġhorizontal":16202,"Blueprint":16203,"Ġattrs":16204,"Ġsmoke":16205,"ÐĴ":16206,".Equals":16207,"FB":16208,"ĠResources":16209,"rolling":16210,"Ġpasses":16211,"ĠNum":16212,"rotate":16213,"etype":16214,"\\\",":16215,"Ġsensitive":16216,"Ġtall":16217,"?âĢĿĊĊ":16218,"Proxy":16219,"iy":16220,"_section":16221,"âĢĶâĢĶâĢĶâĢĶ":16222,"brid":16223,"Ġcircuit":16224,"atan":16225,"ENC":16226,"Ġdriven":16227,"Ġvoted":16228,"Ġeducational":16229,"Ġinteraction":16230,"abetes":16231,"Ġtone":16232,"ĠInitializeComponent":16233,"Ġmerely":16234,"Ġìŀ":16235,"cookie":16236,"_div":16237,"ĠUILabel":16238,"vely":16239,"});čĊ":16240,"_ENT":16241,"#+#+":16242,"articles":16243,"ĠSouthern":16244,"Ġstronger":16245,"ĠGiven":16246,"ĠEric":16247,"ĠIR":16248,"abstract":16249,"Under":16250,"nable":16251,"Ġincrement":16252,"oven":16253,"Ġcoin":16254,"_timer":16255,"Ġsuffered":16256,"ĠFREE":16257,"'].\"":16258,"ĠQueen":16259,"stats":16260,"Ġmeetings":16261,"Ġentering":16262,"Ġalongside":16263,"(session":16264,"itals":16265,"Ġfoundation":16266,"ĠCredit":16267,".div":16268,"_ALL":16269,"pcion":16270,"_stat":16271,"icking":16272,"Defaults":16273,"_src":16274,"Ġoutputs":16275,"/B":16276,"Ġenthus":16277,"-bl":16278,".ForeColor":16279,"ĉtemp":16280,"Face":16281,"Ġinteract":16282,"Ġweird":16283,"Mount":16284,"rell":16285,"udents":16286,"Ġrequirement":16287,"ĠSus":16288,"IER":16289,"Ġelected":16290,"reference":16291,"ĠME":16292,"Ġservers":16293,".wait":16294,"Ġsnapshot":16295,"ilton":16296,"Ġtries":16297,"Ġtipo":16298,".Time":16299,">w":16300,"Ġmountain":16301,"Ġpounds":16302,"Ġ[...":16303,"exists":16304,"ĠngOn":16305,"_MAP":16306,"Ġflying":16307,"xiety":16308,"ĉvalue":16309,"_DB":16310,"uno":16311,"Ġseats":16312,"TURN":16313,".author":16314,"!)":16315,"orce":16316,"Ġindicated":16317,".sin":16318,"Ġassignment":16319,"imiento":16320,"ĠFrame":16321,"_gen":16322,"inery":16323,"_)":16324,"messages":16325,".settings":16326,"ĠMean":16327,"ĠMuseum":16328,"irq":16329,"attach":16330,"ĠPalestin":16331,"_QU":16332,"_tags":16333,"Ġcasual":16334,"emen":16335,"ASSWORD":16336,"$s":16337,"ĠCirc":16338,"ой":16339,"etric":16340,"/P":16341,"Ġepoch":16342,"The":16357,"ĠAk":16358,"Ġgrass":16359,"/*čĊ":16360,"(dis":16361,"Ġguns":16362,"Ġtb":16363,"ĠKevin":16364,".args":16365,"ĠAh":16366,"oped":16367,"(J":16368,"columns":16369,"arguments":16370,"ĠWithEvents":16371,"_full":16372,"ĠDefense":16373,"Simple":16374,"Ġdeaths":16375,"Ġextensive":16376,"ĠStill":16377,"ĠExpression":16378,"ĠAgency":16379,"Ġperforming":16380,"FX":16381,"Ġusuario":16382,"UAL":16383,"Side":16384,"odos":16385,"aptop":16386,"Ġcredentials":16387,"_cap":16388,"atient":16389,"ĠDisney":16390,"Ġai":16391,"Ġchip":16392,"Ġvolt":16393,".makeText":16394,"%%%%%%%%%%%%%%%%":16395,"Ġbelief":16396,"_LOC":16397,"ĠCivil":16398,"Navigation":16399,"Ġreveal":16400,"Ġviolent":16401,"ĠFil":16402,"Ġcatalog":16403,"emed":16404,"scan":16405,".control":16406,"Ġconstitution":16407,"Country":16408,"Separator":16409,"_APP":16410,"topic":16411,"uetooth":16412,"MIN":16413,"Ġdescriptor":16414,"yt":16415,"ETHER":16416,"Ġdistribute":16417,"'}Ċ":16418,".trim":16419,".Line":16420,"Ġlbl":16421,"assertEquals":16422,"ĠDet":16423,"ombok":16424,"(width":16425,"Ġtort":16426,"ĠEXPRESS":16427,"aco":16428,"Using":16429,"ĠBrand":16430,"wall":16431,"EMENT":16432,"ĠCommunic":16433,"(Ċ":17055,"?>\"":17056,"Ġ///Ċ":17057,"Ġeiner":17058,"Ġweekly":17059,"ĉlogger":17060,"_pop":17061,"_man":17062,"Ġmigrations":17063,"Ġasks":17064,"Ġbs":17065,"Ġfalls":17066,".Where":17067,"-height":17068,"_feature":17069,".Min":17070,"Ġhyper":17071,"Ġvolatile":17072,"Ġtwenty":17073,"Typography":17074,"Unable":17075,"Det":17076,",f":17077,"-mod":17078,"Ġsettlement":17079,"Ġcontracts":17080,"nome":17081,"Bad":17082,"ĠBrian":17083,"(username":17084,"!!!!":17085,"Ġhack":17086,".Field":17087,"HR":17088,"ĠJordan":17089,"iza":17090,"ĠÂł":17091,"ĠSher":17092,".header":17093,"(other":17094,"ĠDub":17095,"(op":17096,"ĠRound":17097,"Ġvie":17098,"Ġappl":17099,"ĉJ":17100,"ĠInsert":17101,"ĠLP":17102,"regon":17103,"ĠMPI":17104,"Ġanchor":17105,"aca":17106,"ør":17107,"Ġade":17108,"anchor":17109,"quee":17110,"ĠTreeNode":17111,"Ġtargeted":17112,"Ġlaid":17113,"ABEL":17114,"vet":17115,"ĠOrigin":17116,"Ant":17117,".');Ċ":17118,"expect":17119,"edReader":17120,"ĠMajor":17121,"Ġinch":17122,"Compar":17123,"Ġpreview":17124,"Ġillness":17125,"ĠCONTRACT":17126,"ĠIndepend":17127,"uuid":17128,"Ġnome":17129,"Ġtc":17130,"ĠAvenue":17131,"isan":17132,"Ġphrase":17133,"_move":17134,"\")[":17135,"Ġprovision":17136,"Ġconcentr":17137,"_IR":17138,"ĠUt":17139,"()+":17140,"Ġnas":17141,"!,":17142,"ĠRobin":17143,"iations":17144,"atitude":17145,"Ġpx":17146,"ĠWithout":17147,"/bash":17148,"ekt":17149,"reement":17150,"Observer":17151,"ĠRegion":17152,"UBLIC":17153,"Ġ{//":17154,"KN":17155,"å·":17156,"GameObject":17157,"å¾":17158,"encoding":17159,"Ġ***":17160,"projects":17161,"Ġtk":17162,"Ġcheese":17163,"EMPL":17164,"aro":17165,"ĠاÙĦ":17166,"Ġconsists":17167,"refresh":17168,"ureau":17169,"ĠScanner":17170,"Ġsoil":17171,"Ġflavor":17172,"DataSource":17173,"Execute":17174,"ение":17175,"Ġshit":17176,"åĪĨ":17177,"Ċ":17419,"Ġsubsequent":17420,"posable":17421,"-fluid":17422,"Ġthorough":17423,"Ġpublicly":17424,"apters":17425,"ĠWilson":17426,"_PRE":17427,"yard":17428,"ä¼":17429,"ĉin":17430,"Ġrevers":17431,"Ġbullet":17432,"cribed":17433,"nesota":17434,"Ġ($_":17435,"annon":17436,"cursor":17437,"Ġclothing":17438,"ĠMulti":17439,":',":17440,"Ġvess":17441,"ordinator":17442,"Ġeinem":17443,"Cannot":17444,"Ġarmed":17445,"ĉV":17446,"ä¸Ĭ":17447,".Flat":17448,"ĠSep":17449,"ĠSubject":17450,"_font":17451,"Ġcharacteristics":17452,"Done":17453,"eln":17454,"############":17455,"POS":17456,"Ġdensity":17457,"ĠPlatform":17458,"-items":17459,"Ġovers":17460,"Ġpushing":17461,"ç¤":17462,".Connection":17463,"_term":17464,"Ġinitialization":17465,"________________________________":17466,"ç¬":17467,".document":17468,"lesh":17469,"ĉdocument":17470,"ĠPin":17471,"ça":17472,"Ġdefinitions":17473,".Path":17474,"_WRITE":17475,"ĠĉĊ":17476,"?>ĊĊ":17477,"Ġterrible":17478,"bean":17479,"ickets":17480,"ĠSV":17481,"Buy":17482,"(task":17483,"Ġregime":17484,"google":17485,"Ġcrack":17486,".visit":17487,"NUM":17488,"energy":17489,"Ġstruck":17490,"_sample":17491,".payload":17492,"Ġrevis":17493,"ĠScene":17494,"Ġpg":17495,"Ġbreakfast":17496,"URRENT":17497,".charAt":17498,"_exception":17499,"ĠAnton":17500,"Ġguidelines":17501,"Ġexhaust":17502,"ĠFinancial":17503,"Ġindent":17504,"Ġdesktop":17505,"Hidden":17506,"Failure":17507,"Ġprinciple":17508,"Ġiv":17509,"Ġseks":17510,"network":17511,"ĠnumberOf":17512,"ĠAlbert":17513,"ĉlong":17514,",.":17515,"Ġzeros":17516,"fade":17517,"ĠTyp":17518,"ĠTerm":17519,"ĠArts":17520,".Application":17521,"Ġbehalf":17522,"æĪ·":17523,"Ġmere":17524,"(`${":17525,"Ġawareness":17526,"elpers":17527,"flix":17528,"Ġweigh":17529,"Ġestimates":17530,".child":17531,"/O":17532,"ĠBitmap":17533,".bottom":17534,"Ġ**************************************************************************":17535,"Expect":17536,"ento":17537,"ĠForum":17538,"veral":17539,"Ġjail":17540,"Ġabilities":17541,"ĠHOLD":17542,"ĠCit":17543,"Ġdynam":17544,"Ġgray":17545,"ĉĉĉĉĉĉĉĉĉĉĉĉĉ":17546,".nextInt":17547,"antly":17548,"ĠARISING":17549,"(private":17550,"Ġrejected":17551,"ĠNic":17552,"Ġleather":17553,"={Ċ":17554,"alytics":17555,"thetic":17556,".Top":17557,".Page":17558,"={`":17559,"Ġ;čĊ":17560,"depth":17561,"mann":17562,"WD":17563,"ĠSom":17564,".Right":17565,"Ġ)}Ċ":17566,"Ġtrait":17567,"ÃĹ":17568,"iac":17569,"Ġrv":17570,"Sample":17571,".Xml":17572,"opped":17573,"ĠÑĦ":17574,"lists":17575,"Ġtear":17576,"iversary":17577,".collection":17578,"ĠConstitution":17579,"ĠHttpResponse":17580,"Ġbrill":17581,"ĠProm":17582,"hover":17583,"ĠMiami":17584,"Ġargue":17585,"_float":17586,"ĠãĤ":17587,"Ġnat":17588,"ĠTal":17589,"Ġintegration":17590,"(cur":17591,"Ġremoving":17592,"Ġcoeff":17593,"ĠThough":17594,"Ġforecast":17595,"ĠVegas":17596,"Site":17597,"Ġtrab":17598,"ĠHenry":17599,"-i":17600,"Ġinvolves":17601,"BT":17602,"Ġslo":17603,"Invoke":17604,"Ġlucky":17605,"rat":17606,"Ġ?Ċ":17607,"Ġhandled":17608,"(fd":17609,"contents":17610,"ĠOFF":17611,"RF":17612,"Ġsty":17613,"ĠMotor":17614,"tery":17615,"tax":17616,"MAP":17617,"ĠMrs":17618,"Ġphones":17619,"ĠUIView":17620,"\")));Ċ":17621,"(dev":17622,"ĠIrish":17623,"Ġws":17624,"DI":17625,"_OFFSET":17626,"ĠEvents":17627,"Ġstages":17628,"Ġ}//":17629,"Ġhaben":17630,"STANCE":17631,"ĠSin":17632,"ĠMoney":17633,"(top":17634,"Ġappointment":17635,"VERSION":17636,"metadata":17637,"_comment":17638,"Ġcolleagues":17639,"maps":17640,"âĺ":17641,"ĊĉĊ":17642,"(al":17643,"_req":17644,"Ġfut":17645,"Ġarchitecture":17646,"ĠWHETHER":17647,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":17648,"_screen":17649,"ĠstyleUrls":17650,"Ġmonster":17651,".up":17652,"phia":17653,"Ġprocessor":17654,"ĠTerr":17655,"=',":17656,"ĠManufact":17657,"ĠNT":17658,"kel":17659,"ibern":17660,"ĉfile":17661,"Ali":17662,"rientation":17663,"Ġ//!":17664,"apore":17665,"aneous":17666,"ĠCreat":17667,"folder":17668,"Ġhay":17669,"Suppress":17670,"(left":17671,"Ġeuro":17672,"Ġdisclaimer":17673,"ustry":17674,"ships":17675,"_fd":17676,"ĠFa":17677,"_insert":17678,"Ġrol":17679,"ifting":17680,"ĠComments":17681,"_br":17682,"Ġlosses":17683,"ĠAdded":17684,"charg":17685,"Ġпо":17686,"_system":17687,"ĠSometimes":17688,"ĠSpain":17689,"(group":17690,"ialis":17691,"Ġdollar":17692,"ĠArgs":17693,"quires":17694,"ĠTen":17695,".scss":17696,"Ġsurvive":17697,"usage":17698,"Ġjun":17699,"imiter":17700,"ï¼ģĊĊ":17701,"Ġfifth":17702,"toggle":17703,"Ġdecline":17704,"($\"":17705,"(Long":17706,"inge":17707,"Ġpilot":17708,"-light":17709,"-radius":17710,"Ġpodcast":17711,"Ġnaturally":17712,"Pages":17713,"为":17714,"ĠDespite":17715,"Ġlighting":17716,"Ġcrate":17717,"ĠBinary":17718,"Ġreducing":17719,"Ġeleg":17720,"ĠMouse":17721,"ĠTestBed":17722,"ĠbeforeEach":17723,"_ARRAY":17724,"Redirect":17725,"Ġflood":17726,"Ġships":17727,"Ġelectricity":17728,")*(":17729,"ê¸":17730,"ĠViet":17731,"hero":17732,"Ġdia":17733,"ĠKent":17734,"heart":17735,"Ġthreats":17736,"_acc":17737,"Ġsymbols":17738,"ischen":17739,"_inst":17740,"Criterion":17741,"ĠTIM":17742,".Height":17743,"ĠâĢĻ":17744,"();ĊĊĊ":17745,"Products":17746,"_SP":17747,"ĠCy":17748,"Ġdependent":17749,"este":17750,"Ġdatos":17751,"dit":17752,"ав":17753,"IGNAL":17754,"Ġlesson":17755,"\">'":17756,"ĠCover":17757,"ĠHope":17758,"ĠTimer":17759,"Ġdad":17760,"viders":17761,"ĠPhot":17762,"/?":17763,"ropy":17764,"oming":17765,"asion":17766,"Ġ\\(":17767,"ĠET":17768,"ĠReading":17769,"Ġepisodes":17770,"lm":17771,"echa":17772,"Ġneuro":17773,"Ġharmon":17774,"Ġliberal":17775,"-ind":17776,"DATA":17777,"Ġeveryday":17778,"Ġdivided":17779,"ĠActiveRecord":17780,"figure":17781,"UA":17782,"ä¹":17783,"riendly":17784,"tech":17785,".gameObject":17786,"иÑĤÑĮ":17787,"Ġmoon":17788,"ftime":17789,"Ġnoch":17790,"ĠTORT":17791,"ĠVM":17792,".initial":17793,"(child":17794,"Ġmusical":17795,"Ġoc":17796,"bas":17797,"ĠHay":17798,"_long":17799,"Ġmemset":17800,"iley":17801,"adelphia":17802,"SV":17803,"roat":17804,"_tx":17805,"Ġlon":17806,"ĠngOnInit":17807,"bp":17808,"ĠGolden":17809,"ACHE":17810,"Ġworried":17811,"azi":17812,"Ear":17813,"Take":17814,"(fp":17815,"burgh":17816,"_Data":17817,"gres":17818,"ĠOnt":17819,"pus":17820,"Ġtransparent":17821,"Ġpocket":17822,"Ġram":17823,"igrations":17824,".čĊčĊ":17825,"Ġ[(":17826,"Ġadopted":17827,"Ġreportedly":17828,"ĠDream":17829,"Ġ}));Ċ":17830,"losing":17831,"Ġteeth":17832,"ĠBooks":17833,"\",&":17834,"enny":17835,"LEMENT":17836,"Ġgel":17837,"ĠPlant":17838,"!âĢĿ":17839,".host":17840,"ĠReply":17841,"rength":17842,"Ġrecognition":17843,"Ġ}}>Ċ":17844,"LA":17845,"Ġmirror":17846,"Ġassistant":17847,"(device":17848,"Ġspiritual":17849,"builder":17850,"§":17851,"Ġoutr":17852,"Ġtt":17853,"ĠPER":17854,"Ġradical":17855,"Methods":17856,"Ġpace":17857,"udy":17858,"Ġgut":17859,"ĠGreek":17860,"Ġnonatomic":17861,"ĠPaper":17862,"_GPIO":17863,"Ġobst":17864,".Ad":17865,"vironments":17866,"ĠSov":17867,"(con":17868,"ĠTransaction":17869,".assign":17870,"ĉcatch":17871,"elter":17872,"Ġbitcoin":17873,"_GR":17874,"ĠčĊ":17989,"metic":17990,"Ġtransformation":17991,"åı·":17992,"Ġrgb":17993,"istributions":17994,"Ġimplicit":17995,"/in":17996,"destination":17997,"аÑĤÑĮ":17998,"Zero":17999,"Ġunset":18000,".where":18001,".go":18002,"Ġformation":18003,"Ġdeclaration":18004,"()čĊčĊ":18005,"ĠExpl":18006,"ĉĉĉĠĠ":18007,"/pro":18008,".JSON":18009,"Ġdesk":18010,".substr":18011,"//----------------------------------------------------------------------------":18012,"lyn":18013,"pson":18014,"disable":18015,"ĠFunc":18016,"ĉAssert":18017,"ĠMARK":18018,"Ġdefeat":18019,"Ġblind":18020,"Ġconstants":18021,".headers":18022,"UILD":18023,"Ġexpenses":18024,"Pixel":18025,"Ġhr":18026,"Ġfel":18027,"ĠEastern":18028,"_del":18029,"ĠCub":18030,"Ġsq":18031,"ĉcount":18032,"ĠDirectory":18033,"Ġexclus":18034,"Ġhistoric":18035,"Ġ------------------------------------------------":18036,"Ġcomposition":18037,"ĠdataGridView":18038,"ĠBurn":18039,"ĠBC":18040,"Master":18041,"Ġspawn":18042,"Ġbearing":18043,".SetActive":18044,"ilo":18045,"Ġgallery":18046,"Ġfounded":18047,"Ġavailability":18048,".sqrt":18049,"Ġpes":18050,"ĠDOM":18051,"mate":18052,"Oct":18053,"Ġmatched":18054,"itivity":18055,"Ġanxiety":18056,".price":18057,"ĠInstant":18058,"ìĬ":18059,"Ġtut":18060,"ICollection":18061,".shared":18062,"_sql":18063,"tbl":18064,"library":18065,"_destroy":18066,"ermal":18067,"ĠNotes":18068,"ĠEin":18069,"Ġsouthern":18070,"ĠOTHERWISE":18071,"Ġmacro":18072,".lower":18073,"cls":18074,"ContentView":18075,".link":18076,"constant":18077,"ĠBes":18078,"Ġsomebody":18079,"nb":18080,"\">{":18081,"(local":18082,".....":18083,"ĠNull":18084,"mx":18085,"Ġç":18086,"Ġpause":18087,"-----------":18088,"_MO":18089,"ĠCM":18090,"ĠforKey":18091,"ĠDVD":18092,"Ġclosest":18093,"_DEVICE":18094,"ĠStephen":18095,"ĠBBC":18096,"ĠTravel":18097,"Paint":18098,"ĠResults":18099,"ĠRule":18100,"Ġtp":18101,"Ġratings":18102,"cin":18103,"csv":18104,">/":18105,"ĠGOP":18106,"lad":18107,"ĠÑĢ":18108,"ĠindexPath":18109,"matrix":18110,"=f":18111,"arsed":18112,"Ġ});":18113,"ĠCos":18114,"ĠScore":18115,"Ġtak":18116,"ĠESP":18117,"ĠINC":18118,"_NULL":18119,"-flex":18120,"\"][":18121,"into":18122,"eland":18123,"Authorization":18124,"_FALSE":18125,"Ġgate":18126,"Ġvid":18127,"istent":18128,"TIME":18129,"Ġrewrite":18130,"Ġtie":18131,"Ġarchive":18132,".events":18133,".getParameter":18134,"ĠPermission":18135,"Ġprogramme":18136,"Ġé":18137,"jud":18138,"Ġcameras":18139,"(sys":18140,"ĠSyrian":18141,"Ġimprovements":18142,"Ġhip":18143,"Ġsuicide":18144,"Ġscholar":18145,"Ġcompatible":18146,"remote":18147,".down":18148,"FUNCTION":18149,"Ġmanaging":18150,"ĠUIKit":18151,".raw":18152,">>>>":18153,"Ġdemands":18154,"ellite":18155,"Ġdent":18156,"ĠMicro":18157,"åıĸ":18158,"'][$":18159,"ĠIE":18160,"imension":18161,"Ġtrem":18162,"Ġgained":18163,".with":18164,".ok":18165,"hou":18166,"Ġbom":18167,"ampaign":18168,"Ġjoining":18169,"fish":18170,"ĠaddSubview":18171,"Ġnorthern":18172,".cor":18173,"oret":18174,"Die":18175,"inish":18176,"_comp":18177,"Ġattended":18178,"Ġcollapse":18179,"ĠSS":18180,"acent":18181,"_EQUAL":18182,"ĠDeep":18183,"RGB":18184,"ĉtest":18185,"olves":18186,"uset":18187,"UnityEngine":18188,"writer":18189,"Resolver":18190,",%":18191,"ifference":18192,"_remove":18193,"onda":18194,"Ġfemme":18195,"decode":18196,"Branch":18197,"Ġflush":18198,"Ġinnovative":18199,"Tests":18200,"Ġ['./":18201,"Ġcovering":18202,".admin":18203,"ultipart":18204,"(lambda":18205,"namespace":18206,"ĠSport":18207,"Ġ!(":18208,"acles":18209,"Ġdepression":18210,"ĠKong":18211,"Ġpert":18212,"ĠConn":18213,"ĠOtherwise":18214,"/home":18215,"supported":18216,"Ġpink":18217,"Ġinvited":18218,"ños":18219,"_enabled":18220,"Ġ-Ċ":18221,"FW":18222,"eners":18223,"ĠMY":18224,"Ġsuggestions":18225,"Canvas":18226,"Ġfer":18227,"ĠMarketing":18228,"@Test":18229,"untu":18230,"ĠVen":18231,"ĠCou":18232,"ivals":18233,"Donald":18234,"limited":18235,"ĉĉĉĉĉĉĊ":18236,"Ġanalyst":18237,"(entry":18238,"Ġrepresentative":18239,"_attributes":18240,"Ġfur":18241,".hide":18242,"resp":18243,"adores":18244,"rides":18245,"ĠJosh":18246,"robot":18247,"ĠNAT":18248,"Ġsesso":18249,"Ġintegrated":18250,":true":18251,"parts":18252,"Ġstupid":18253,":event":18254,"@endsection":18255,"Ġpu":18256,".Table":18257,"ĠYii":18258,"`;ĊĊ":18259,"Ġclang":18260,"=\"\">":18261,"engan":18262,"_parameters":18263,".internal":18264,"ĠModern":18265,"Ġmetric":18266,"Ġsemi":18267,"={{Ċ":18268,".amazon":18269,"ĠBB":18270,"ainty":18271,"viewport":18272,"ĠstartActivity":18273,"dispatch":18274,"*****":18275,"Ġflav":18276,"ifferent":18277,"[this":18278,"Ġstake":18279,"Ġargued":18280,"viously":18281,".work":18282,"ĠOak":18283,"Old":18284,"(async":18285,"notes":18286,"Ġflip":18287,"Ġdisag":18288,"ĠTE":18289,"ĉerror":18290,"<'":18291,"Ġ»ĊĊ":18292,"Ġfiltered":18293,"ĠMach":18294,"Ġhung":18295,"_dump":18296,"_samples":18297,"-dismiss":18298,"Ġray":18299,"Implemented":18300,"DK":18301,"Ġjed":18302,"Ġbreaks":18303,"Ġfits":18304,".gr":18305,"ĠZero":18306,"oro":18307,"Ġequally":18308,"Ġ'[":18309,"Ġconcerning":18310,"<":18407,"Ġpromot":18408,"Ġincl":18409,"_only":18410,"를":18411,"ĠAttorney":18412,"-date":18413,"Ġlandscape":18414,"Ġfu":18415,"SY":18416,".prop":18417,"ĠArr":18418,"pag":18419,"ParallelGroup":18420,"':čĊ":18421,"Ġlogs":18422,"aunch":18423,"unci":18424,"nama":18425,"TableCell":18426,"issues":18427,".{":18428,"ecurity":18429,"_exec":18430,"olds":18431,"Ġhosts":18432,"Ġproto":18433,"_import":18434,"_sort":18435,"ĠBow":18436,"ĠNormal":18437,"ĠFarm":18438,".createParallelGroup":18439,"Rotation":18440,".err":18441,"Ġpleased":18442,"itage":18443,".Wh":18444,"ĉĉĠĠĠĠ":18445,"MR":18446,"ĠMORE":18447,"ĠNatural":18448,"_transform":18449,"BASE":18450,"eneral":18451,"utdown":18452,".commons":18453,"WT":18454,"Ġaan":18455,".Result":18456,"dog":18457,"Ġclicking":18458,"),ĊĊ":18459,"#line":18460,"Operator":18461,"Ġciv":18462,"Ġmerg":18463,"obuf":18464,"ngthen":18465,"Ġ[{":18466,"Ġcancell":18467,"trigger":18468,".:":18469,"WORK":18470,"declare":18471,"Ġdecrease":18472,"ÅĽci":18473,"loom":18474,".None":18475,"ĠMI":18476,"ĠJason":18477,"Ġhealthcare":18478,"iamond":18479,"sylvania":18480,"*x":18481,"ĠRa":18482,"[b":18483,"Ġprinting":18484,"phabet":18485,"ĠLabour":18486,"opper":18487,"Ġzijn":18488,"-target":18489,"_FUNCTION":18490,"Ġoct":18491,"ениÑı":18492,"åľ¨":18493,"Ġwestern":18494,"Ġcomputers":18495,"ĠRET":18496,"HashMap":18497,"[String":18498,"getValue":18499,"_DATE":18500,".Next":18501,"ĠFif":18502,"él":18503,"icked":18504,"æİ":18505,"-MM":18506,"Ġ{ĊĊĊ":18507,"Ġcontacts":18508,"Ġdigits":18509,"Produ":18510,"Ġunusual":18511,"Ġrapidly":18512,"tures":18513,"Ġangry":18514,"cancel":18515,"xxxx":18516,"_parser":18517,"idity":18518,"_PREFIX":18519,"Ġmehr":18520,"Ġrarely":18521,"ethe":18522,"opes":18523,"Ġ%.":18524,"works":18525,"Ġtheta":18526,"Ġcontribution":18527,"ĠTony":18528,"Ġsquad":18529,"ай":18530,"Ġîn":18531,"there":18532,"outed":18533,"ĉq":18534,"ĻĤ":18535,"good":18536,"LI":18537,"页":18538,"ĠLiving":18539,"izabeth":18540,"Ġkt":18541,"ĠDallas":18542,"]],Ċ":18543,"Ġ/>ĊĊ":18544,"Ġraising":18545,"/router":18546,"_game":18547,"ĠCUR":18548,"zens":18549,".es":18550,"ĠfontWeight":18551,"(func":18552,"notification":18553,"Ġ'../../../":18554,"Ġblame":18555,"ãĢĤĊĊĊĊ":18556,"anco":18557,"Identity":18558,"follow":18559,"Ġarts":18560,"xs":18561,"Ġofficially":18562,"ĠStudio":18563,"Ġrecommendations":18564,"Ġlocale":18565,"Ġamateur":18566,"ĠEnable":18567,"Ġcaps":18568,".End":18569,"-add":18570,"_gshared":18571,"ĠCT":18572,"Force":18573,"ĊĠĠĠĠĠĠĠĠĠĠĠĠĊ":18574,"Ġorange":18575,"Ġlp":18576,"Ġanswered":18577,".Grid":18578,"Ġdual":18579,"Ġstrategic":18580,"Ġnobody":18581,"Ġfatal":18582,"_est":18583,"(el":18584,"Ġìł":18585,"ĠBudd":18586,"AIT":18587,"_factor":18588,"-one":18589,"ĠHAVE":18590,"\"čĊčĊ":18591,"Prof":18592,"Ġär":18593,"strings":18594,"Ġdirty":18595,"ĠFace":18596,"ĠBegin":18597,"ĠBus":18598,"Ġwis":18599,"åŃĹ":18600,"Ġspeaker":18601,"Ġcarrier":18602,"ĠOm":18603,"Ġhadn":18604,"Allow":18605,"::__":18606,"Ġverb":18607,"ĠComplete":18608,"ĠEasy":18609,"Ġbills":18610,"ĠĠĊĊ":18611,"Vertical":18612,"Ġpron":18613,"ĠDefine":18614,"Ġlookup":18615,"variables":18616,"Ġpandas":18617,"umes":18618,"Ġinnoc":18619,"ĠsetUp":18620,"ĠChampionship":18621,"artist":18622,"ĠCType":18623,"Foundation":18624,"à¹Ī":18625,"ĠSetup":18626,"Ġrecipes":18627,"ĠUIColor":18628,"ĠFight":18629,"Ġauthorized":18630,"_click":18631,"_success":18632,"angan":18633,"ĠMountain":18634,"ĠDoctor":18635,"Ġegg":18636,"ĠMedicine":18637,"cles":18638,"`.Ċ":18639,"[int":18640,"dashboard":18641,"ĠAppro":18642,"-dr":18643,"Ġproduces":18644,"Ġrental":18645,"Ġreload":18646,"Ġarrival":18647,"spot":18648,"Ġundert":18649,"Ġequipped":18650,"Ġproved":18651,"Ġcenters":18652,"Ġdefines":18653,"also":18654,"Ġopacity":18655,"ĠUnfortunately":18656,"ĠIllinois":18657,"Ġне":18658,"ĠTemple":18659,"ĠTrail":18660,"ĠKelly":18661,"Ġmeasurement":18662,"Ġseparated":18663,"-circle":18664,"Hey":18665,"ĠREAD":18666,"igits":18667,"Ġib":18668,"ĠMOD":18669,"attery":18670,"аз":18671,"Ġvend":18672,"енÑĤ":18673,"ĠHttpClient":18674,"safe":18675,"_ASS":18676,"icit":18677,"ĠConstruct":18678,"ĠClo":18679,"ĠSix":18680,"_TOKEN":18681,"(block":18682,"Ġwarned":18683,"/*!":18684,"!Ċ":18769,"Ġinnovation":18770,"_\"":18771,"Ġ);čĊčĊ":18772,"Ġspots":18773,"Ġchoosing":18774,".cs":18775,"Ġflexible":18776,"UInt":18777,"Ġscratch":18778,"-al":18779,"Ġfestival":18780,"Ġoutstanding":18781,"================================================":18782,"Mean":18783,"ĠOregon":18784,"symbol":18785,".account":18786,"dney":18787,"'''":18788,"!\",":18789,"Ġparticle":18790,"Ãĥ":18791,"[MAX":18792,"IVER":18793,"ERENCE":18794,"NSMutable":18795,"ĠColumbia":18796,"_ĊĊ":18797,".fr":18798,"Ġcogn":18799,"VR":18800,"ĠMethods":18801,"ĠMade":18802,"ĠBR":18803,"ĠElse":18804,"Ġeggs":18805,"Ġswing":18806,"ĠInv":18807,"Ġdiseases":18808,"Ġfirms":18809,"Ġlemma":18810,"}`);Ċ":18811,"lings":18812,"Ġgym":18813,"uminum":18814,".Trim":18815,"Mem":18816,"Ġcriticism":18817,"ibernate":18818,"_TX":18819,"ioni":18820,"Ġguidance":18821,"Ġrepeatedly":18822,"Ġsupplier":18823,"Ġpainting":18824,".Fragment":18825,"edException":18826,"Ġwiring":18827,"Ġcourts":18828,"WEB":18829,"æľī":18830,"\\.":18831,"illance":18832,"Ġbrows":18833,"ĠPattern":18834,"PLICATION":18835,"ĠSummer":18836,"Chain":18837,"Ġcute":18838,"mercial":18839,"Ġdil":18840,"ĠFranklin":18841,"ĉglobal":18842,"INCLUDING":18843,"history":18844,"Ġlst":18845,"Qt":18846,"SDL":18847,"alia":18848,"iere":18849,"(...":18850,"ĉcin":18851,"iffs":18852,"velope":18853,"ĠRoot":18854,"cluster":18855,"UserName":18856,"igne":18857,"()Ċ":18949,"Ġapplying":18950,"Ġpromised":18951,"Ġox":18952,"ncia":18953,"ĠValidation":18954,"orts":18955,"_cur":18956,"elect":18957,"eye":18958,"(Data":18959,"Ġreporter":18960,"ĠBuff":18961,"Ġsr":18962,"Ġ\";":18963,"icky":18964,"Ġtempor":18965,"SN":18966,"Ġresident":18967,"pires":18968,"ysical":18969,"Ġendorse":18970,"ĠSong":18971,"isEmpty":18972,"leet":18973,"_util":18974,"Ġdistingu":18975,"ĠTalk":18976,"ĠMot":18977,"(default":18978,".Arg":18979,"gorithms":18980,"_words":18981,"immer":18982,"_reset":18983,"family":18984,"WW":18985,"Ġsavings":18986,"ĠâĢĿ":18987,"_enable":18988,"sidebar":18989,"Running":18990,"Ġali":18991,"Ġtestim":18992,"Ġwarnings":18993,"ĠChem":18994,"ĠExit":18995,"Ġfounder":18996,"pector":18997,"Ġrm":18998,"_dataset":18999,"ĠDas":19000,"Ġhan":19001,"Getty":19002,"ál":19003,"Ġny":19004,"Ġpoverty":19005,"Ġresulted":19006,".by":19007,"ĠVisit":19008,"Ġobtaining":19009,"/'.$":19010,"ĠĠĠĠĠĠĠĠĠĠĠĊ":19011,"shall":19012,"_LEFT":19013,"UIImage":19014,"_Name":19015,"have":19016,"ĠNob":19017,"lr":19018,"-footer":19019,"Ġnaked":19020,"ĠGarden":19021,"\\Facades":19022,"Ġgraduate":19023,"Ġfranchise":19024,"plane":19025,"Ġcontributions":19026,"ĠstringWith":19027,"Ġcrypto":19028,"Ġmovements":19029,"athers":19030,"Ġlifetime":19031,"Ġcommunicate":19032,"jar":19033,"ĠFragment":19034,"_IF":19035,"ĠNavy":19036,"ĠFigure":19037,"Ġsimulation":19038,"_stop":19039,"Ġreporters":19040,"Ġversus":19041,"aja":19042,"Ġα":19043,"Ġgovernor":19044,"ListItem":19045,"Ġsealed":19046,".Background":19047,"edi":19048,"ashing":19049,"Ġlip":19050,"ĠIh":19051,"merge":19052,"Ġnec":19053,"elocity":19054,"ATEG":19055,"Ġseeds":19056,"Ġfloating":19057,"_FA":19058,"walk":19059,"ĉuser":19060,"_depth":19061,"Ġwage":19062,"@app":19063,"Nil":19064,"([\"":19065,"(vector":19066,"Ġsecretary":19067,"ĠjPanel":19068,"vez":19069,"³³³³":19070,"direction":19071,"ĠEP":19072,"Ġhunt":19073,"JsonProperty":19074,"ĠPORT":19075,"]\",":19076,"ап":19077,"ĠForeign":19078,"panic":19079,"Ġtrials":19080,"ĠAle":19081,"Ġrural":19082,"-value":19083,"authorized":19084,"ĠScotland":19085,".drop":19086,"ĠMT":19087,"ç±":19088,"rowth":19089,"FilePath":19090,"Ġrecall":19091,"ifle":19092,"Ġcel":19093,"ĠSELECT":19094,"kn":19095,"_case":19096,"Ġcrop":19097,"sure":19098,"pot":19099,"ICS":19100,"Ġstem":19101,"Ġindustries":19102,"Put":19103,"Ġaber":19104,"roadcast":19105,"Icons":19106,")\")Ċ":19107,"æĪIJåĬŁ":19108,"gui":19109,"Ġassumed":19110,"Ġrx":19111,"EA":19112,"è§":19113,"ELL":19114,"Ġdose":19115,"Ġine":19116,"Ġdeeper":19117,"lider":19118,"Ġordinary":19119,"Ġgolf":19120,"_IMAGE":19121,"ĠNAME":19122,"(module":19123,"Ġatom":19124,"Ġbelt":19125,"Ġoffices":19126,"beta":19127,"Ġphilosophy":19128,"(JSON":19129,"-field":19130,"Ġintroduce":19131,"Ġconvenience":19132,"optim":19133,">\"Ċ":19134,"athy":19135,"Ġemployer":19136,"quate":19137,"Ġedited":19138,"Arguments":19139,"ĠNations":19140,"__)":19141,"Ġnose":19142,"ĠSample":19143,"')ĊĊĊ":19144,"Ġcake":19145,".getAttribute":19146,"HD":19147,"Modified":19148,"Ġpredicted":19149,"ÅĦ":19150,"anie":19151,"Sorry":19152,"(doc":19153,"wind":19154,"ieve":19155,"Ġprovisions":19156,"ATER":19157,"OTE":19158,"MY":19159,".Autowired":19160,"ĠBath":19161,".Boolean":19162,"Ġbackend":19163,".Mouse":19164,"ateral":19165,"paper":19166,"Const":19167,"ĠVR":19168,"_entity":19169,"_CTRL":19170,"ĠProtection":19171,"ĠGM":19172,"ĠStudy":19173,"Ġsoup":19174,"otime":19175,"'use":19176,"]\"":19177,"/users":19178,"aug":19179,"ĠHong":19180,"_norm":19181,"ãģ¨":19182,"Ġsecre":19183,"(Build":19184,"ĠContract":19185,"olas":19186,"Ġsauce":19187,"Ġaggressive":19188,"Ġracial":19189,"character":19190,"@@":19191,"Ġcompile":19192,"ĠVoid":19193,"_rem":19194,"_memory":19195,"kk":19196,"Ġmic":19197,"Same":19198,"Utility":19199,"ĠHtml":19200,"ĠXml":19201,"Ready":19202,"Ġgall":19203,"Ġallegedly":19204,"ĉĉĉĉĠĠĠ":19205,"ĠMetal":19206,"ĠPersonal":19207,"ĠborderRadius":19208,"rxjs":19209,"objects":19210,"Ġwanting":19211,"Ġbowl":19212,"vendor":19213,"offsetof":19214,"ĠRs":19215,"ĠRating":19216,"Ġrally":19217,"_NODE":19218,"ĠMix":19219,"Ġadvertis":19220,"Ġnarrative":19221,"sal":19222,"Ġmc":19223,"SError":19224,"Ġfingers":19225,"Ġaccompany":19226,"Ġtired":19227,"Ġstride":19228,"Ġgui":19229,"elist":19230,"Locale":19231,"Ġreleases":19232,"iking":19233,"Ġanger":19234,")))ĊĊ":19235,"allest":19236,"Summary":19237,"(O":19238,"(for":19239,"Ġbasketball":19240,"Ġroads":19241,"ĠInstall":19242,"ĠFab":19243,"itmap":19244,"Ġ))Ċ":19245,"Ġintersection":19246,"ighbor":19247,"ĠBry":19248,"ĠHERE":19249,"Software":19250,"elfare":19251,"acs":19252,"Ġtrailer":19253,".getClass":19254,"chars":19255,"Ġregulation":19256,"Ġrefers":19257,"Ġdestruction":19258,"Ġcontinuous":19259,"ĠAustin":19260,"é¢":19261,"akan":19262,".window":19263,"ĠTemplates":19264,"Ġabsence":19265,":n":19266,"Ġdisorder":19267,"flash":19268,"Ġdelet":19269,"boards":19270,"ĠĠĉ":19271,"ROP":19272,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":19273,"Ġacqu":19274,"Ġlawsuit":19275,"ĠReviews":19276,"Ġgarage":19277,"timer":19278,"Ġej":19279,"ĠRectangle":19280,"Ġflowers":19281,"ilst":19282,"ĠInstance":19283,"Super":19284,"det":19285,"disposing":19286,"ĠES":19287,"ĠIC":19288,"vere":19289,"Sk":19290,"_channels":19291,"puted":19292,"/null":19293,"nnen":19294,"ĠGallery":19295,"_global":19296,"Authentication":19297,"ĠRank":19298,"Ġblocked":19299,"Ġcalm":19300,"market":19301,"ĉval":19302,"Ġaug":19303,"period":19304,"ĠConstant":19305,"Ġ?>\">Ċ":19306,"Ġlobby":19307,"pal":19308,"Ġsink":19309,"iah":19310,"С":19311,"urname":19312,"Ġconver":19313,"Ġinvestigate":19314,"Christ":19315,"Hub":19316,"ĠIND":19317,"ĠPed":19318,"uras":19319,"ĉurl":19320,"ĠTro":19321,"Ġpreferences":19322,"Ġguaranteed":19323,"`ĊĊ":19324,"Ġportions":19325,"Ġevalu":19326,"'>;ĊĊ":19421,".AutoScaleMode":19422,"Ġcats":19423,"Ġregistry":19424,"ulus":19425,"FI":19426,"payload":19427,"-search":19428,"Ġstaying":19429,"acious":19430,"Decoration":19431,"Review":19432,"Inf":19433,"Keep":19434,"itis":19435,",String":19436,"Coord":19437,"Ġpero":19438,"Sex":19439,"ĠAtlanta":19440,"uesta":19441,"Argb":19442,">*":19443,"}_":19444,"Footer":19445,"Ġemployed":19446,"_bound":19447,"vide":19448,".func":19449,"$scope":19450,"Ġspo":19451,"ĠAnal":19452,"ounced":19453,"around":19454,"Ġrestriction":19455,"Ġshops":19456,"åĢ":19457,"ĠLatin":19458,"-col":19459,"Ġbarely":19460,"ĠEuro":19461,"Er":19462,"Ġfaire":19463,"_distance":19464,"_unlock":19465,"Quote":19466,"IVATE":19467,"ĠåĪ":19468,"Ġaimed":19469,"ĠRetrie":19470,".iter":19471,"Ġwrapped":19472,"Ġagreements":19473,"strument":19474,"(product":19475,"Ġstudied":19476,".setValue":19477,"Ġye":19478,"ĠCache":19479,"MBOL":19480,"Ġquarterback":19481,"Ġsyntax":19482,".getElementsBy":19483,".version":19484,"website":19485,"Runner":19486,"_single":19487,"ativ":19488,"ĠAltern":19489,"ĠBeautiful":19490,"rightarrow":19491,"Ġdiversity":19492,"plash":19493,"(co":19494,".Fill":19495,"Ġtyping":19496,"Ġclar":19497,"Hit":19498,"OO":19499,"acco":19500,"worth":19501,"Ġscripts":19502,"ĠMuslims":19503,"ĠLL":19504,"erving":19505,"(boolean":19506,"Ġbaseball":19507,"ĠCAN":19508,"MAIL":19509,"depend":19510,"Ġrespective":19511,"Ġconstexpr":19512,".*;ĊĊ":19513,"']))Ċ":19514,"Ġyard":19515,"Ġidentical":19516,"ifecycle":19517,"USH":19518,"upiter":19519,".validate":19520,"cli":19521,"ISTER":19522,"Indicator":19523,"Fail":19524,"Ġdemocracy":19525,".var":19526,"Ġsatisfied":19527,"-------------":19528,"encer":19529,"hor":19530,"Ġrounds":19531,"DAO":19532,"oa":19533,"Ġflask":19534,"=c":19535,"[]Ċ":19536,"/dist":19537,"Ġparte":19538,"Ġconfirmation":19539,"eron":19540,"aware":19541,"":19542,"Ġdependencies":19543,"ĠVideos":19544,"-row":19545,"Ġ**/Ċ":19546,"Ġnou":19547,"Ġhover":19548,"æŀ":19549,"Ġnin":19550,"ĠUSD":19551,"Mac":19552,"_Load":19553,"Ġoutcomes":19554,"_socket":19555,"Ġqueries":19556,"wm":19557,"Ġhitting":19558,"inux":19559,"Mich":19560,"udge":19561,"ATAB":19562,"Ġvulnerable":19563,"ä¾":19564,"Ġportfolio":19565,":YES":19566,"ĉmap":19567,"Bound":19568,"Ġiteration":19569,"incess":19570,"Ġactors":19571,"ĠQual":19572,"_clean":19573,"ãĢijãĢIJ":19574,"MSG":19575,"Green":19576,"ĠOfficer":19577,"Ġsmoking":19578,">',":19579,"ĠFlo":19580,"++;":19581,"olygon":19582,"Ġbulk":19583,"Ġdrama":19584,"Ġexceptions":19585,"osed":19586,"Ġ+čĊ":19587,"Ġlegacy":19588,"CV":19589,"Ġcontributed":19590,"ĠTerms":19591,"Ġbt":19592,"Ġuntuk":19593,"Ġalien":19594,"===Ċ":19595,"ĉVector":19596,"Ġls":19597,"Online":19598,".facebook":19599,"numeric":19600,"ockets":19601,"Aut":19602,"bury":19603,"-redux":19604,"ĠRedistributions":19605,"GLOBALS":19606,"urrencies":19607,"Ġtons":19608,"âĢĻ,":19609,"Ġê":19610,"(col":19611,"ĠSymbol":19612,"Ġstayed":19613,"ĠML":19614,"Ġmunicip":19615,"Ġsexo":19616,"Sen":19617,"nr":19618,"Ġgains":19619,"Ġshortly":19620,".Menu":19621,"ý":19622,"KNOWN":19623,"Ġoperators":19624,"-V":19625,"ĠPatrick":19626,"/add":19627,"_CO":19628,"iration":19629,"(post":19630,"Posts":19631,"/_":19632,"Ġplug":19633,"Ġintellectual":19634,"Ġmetab":19635,"Ġpregnancy":19636,"ĠPremier":19637,"nm":19638,"Ġprediction":19639,"ĠMinistry":19640,"Three":19641,"valuate":19642,"ĠMini":19643,"bu":19644,"оз":19645,"\";čĊ":20078,"ĠSav":20079,".Bold":20080,"Ġenables":20081,"ĉtmp":20082,"Ġmanually":20083,"ĠSqu":20084,"userid":20085,".function":20086,".cache":20087,"LOPT":20088,".Services":20089,"ddit":20090,"tim":20091,">>":20154,"station":20155,"lore":20156,"atype":20157,"ishop":20158,"/****************************************************************":20159,"ComboBox":20160,"Ġvacation":20161,"Ġinitiative":20162,"ĠdefaultValue":20163,"concat":20164,"ĠKh":20165,"ĠWelcome":20166,"izedName":20167,"Migration":20168,"Ġgradient":20169,"Hot":20170,"Ġhardly":20171,"elo":20172,"ĠStudents":20173,"Ġloose":20174,"atz":20175,".Send":20176,"'/":20177,"Ġuniversal":20178,"Ġenterprise":20179,"Ġregex":20180,"Ġvisitor":20181,"ĠFly":20182,"Seq":20183,"à¸Ļ":20184,"ĠVisual":20185,"Ġlibraries":20186,"atoes":20187,"Payment":20188,"Ġpent":20189,"Ġgathered":20190,"VRTX":20191,"ĠDM":20192,"Split":20193,"Ġletting":20194,"ÐĿ":20195,"_errors":20196,"epoch":20197,"PARAM":20198,"cu":20199,"ÑģÑĤв":20200,"olutions":20201,"Editing":20202,"fonts":20203,"Ġallocated":20204,"ĠBased":20205,"(Y":20206,"ĠJudge":20207,"Ġbrothers":20208,"FILES":20209,"ço":20210,"wb":20211,"_PI":20212,"'^":20213,"Ġsword":20214,".services":20215,"Ġnl":20216,"Tim":20217,"igg":20218,"ĠMoore":20219,"Ġcryptoc":20220,"åĩº":20221,"_posts":20222,"otate":20223,"?'":20224,"....ĊĊ":20225,"Ġkl":20226,"=\"$":20227,"Ġdecoration":20228,"ạ":20229,"ĠDIRECT":20230,"GUI":20231,")=>{Ċ":20232,"Ġnewsletter":20233,"Ġprecis":20234,"(point":20235,"ĠEquipment":20236,"uty":20237,"ĠDave":20238,"Ġparticipation":20239,"uarios":20240,"xit":20241,".As":20242,"ETER":20243,"orous":20244,"Ġshield":20245,"[]>":20246,"ilitary":20247,".origin":20248,"Ġpromotion":20249,"Unt":20250,"Ġct":20251,"TRA":20252,"ViewHolder":20253,"Ġsigma":20254,"delta":20255,"arehouse":20256,"contract":20257,"(Vector":20258,"Ġcompete":20259,"/form":20260,"/components":20261,"Ġnr":20262,"ĠIndones":20263,"ĠоÑĤ":20264,"ĠVolume":20265,".files":20266,"(resp":20267,"/models":20268,"Ġsurf":20269,"standard":20270,"/o":20271,"ĠXCTAssert":20272,"VICES":20273,".Code":20274,"SED":20275,"Ġactivate":20276,"Delta":20277,"Ġlimitation":20278,"rij":20279,"Ġpregnant":20280,":^(":20281,"Ġsour":20282,"pie":20283,"Ġexpense":20284,"ication":20285,"ĠLarge":20286,"Ġ±":20287,"ĠBowl":20288,"(models":20289,"/N":20290,"Pa":20291,".reload":20292,"Ġwondering":20293,"Execution":20294,"ĉĠĠĠĠĠĠ":20295,"ĠGraphics":20296,"ĠContin":20297,"_job":20298,"ĠgetName":20299,"ĠMagn":20300,"ĠDWORD":20301,"mad":20302,"Ġnh":20303,"features":20304,"}\");Ċ":20305,"heets":20306,"(train":20307,"zn":20308,"Ġrecruit":20309,".connection":20310,"Ġbarrel":20311,"Ġsteam":20312,"_setting":20313,"Ġangular":20314,"aneously":20315,"Ġbil":20316,"ĠNorm":20317,"(!$":20318,"ibt":20319,"%(":20320,"Ġposit":20321,"ĠFather":20322,"intendo":20323,"Live":20324,"Ġports":20325,"Ġmej":20326,"Ġlanding":20327,"ponder":20328,"Ġcod":20329,"_HEADER":20330,".Margin":20331,"Ġballs":20332,"Ġdiscussions":20333,"Ġblend":20334,"Hex":20335,"Ġfarmers":20336,"Ġmaintaining":20337,"ĠĠĠčĊ":20338,"syn":20339,"[T":20340,"rus":20341,"uffers":20342,"Ġcontributors":20343,"_sys":20344,".Debug":20345,"Ġconstructed":20346,"omes":20347,"?id":20348,"slider":20349,"Ġsuppliers":20350,"scriber":20351,"pes":20352,"Ðŀ":20353,"\":čĊ":20354,"\\Controller":20355,"))ĊĊĊ":20356,"Ġlua":20357,"Multi":20358,"ENS":20359,"Src":20360,"Ġpetition":20361,"Ġslave":20362,"looking":20363,"VERT":20364,"ĉvector":20365,"Special":20366,"hh":20367,"anne":20368,"ĠNiger":20369,"/views":20370,"zing":20371,"endant":20372,"(":20591,".Product":20592,"Forms":20593,"NEW":20594,"Pay":20595,"ĉboolean":20596,"_contact":20597,"ĠElectric":20598,"skip":20599,"Ġwur":20600,"Ġchronic":20601,"_driver":20602,"ĠSab":20603,"ĠUlt":20604,"ĠRad":20605,"STATUS":20606,"ĠLewis":20607,"OB":20608,"Ġgifts":20609,".Rec":20610,"TRUE":20611,"Ġintensity":20612,"Marker":20613,".compare":20614,"ffic":20615,"Cookie":20616,"ĠBaby":20617,"ĠBigDecimal":20618,"ilet":20619,"ĠHOLDERS":20620,"ĠLady":20621,"Ġlung":20622,"ĠAlabama":20623,"Ġdess":20624,"`);Ċ":20625,"ĠBuilder":20626,"_region":20627,"Ġneutral":20628,"Both":20629,"Ġhp":20630,"Ġhorn":20631,"Ġsegments":20632,"ĠEC":20633,"\"=>\"":20634,"(rec":20635,"ĠPi":20636,"GM":20637,"Ġlaptop":20638,"Scalar":20639,"isd":20640,"-dialog":20641,"ĠAnderson":20642,"Ġmistakes":20643,"ĠHan":20644,"jes":20645,"estination":20646,"Ġpromises":20647,"bid":20648,"ĠScient":20649,"GIN":20650,"ĠPerformance":20651,"bage":20652,".users":20653,"leading":20654,"Ġoral":20655,"Graphics":20656,"_PTR":20657,"hang":20658,"Ġinev":20659,"processing":20660,"Factor":20661,"ĠNA":20662,"$string":20663,"Ġgrounds":20664,".SaveChanges":20665,"clock":20666,"cripcion":20667,"ĠNewton":20668,"gc":20669,".includes":20670,"Ġblast":20671,"Ġ'-'":20672,"Ġpuede":20673,".Session":20674,"Ġgrep":20675,"_final":20676,"ĠGay":20677,"ĠGive":20678,"iri":20679,"-star":20680,"ĠUIImage":20681,"_epoch":20682,"ubb":20683,"enth":20684,"Ġelite":20685,"Ġcampaigns":20686,"ĠPorno":20687,"_assign":20688,"Protocol":20689,"ĠBeing":20690,"ĠAirport":20691,"Ġconventional":20692,"ĠWat":20693,"ĠCI":20694,"ETA":20695,"ĠAnthony":20696,"Ġtablet":20697,"(format":20698,"Ġconsistently":20699,"ĠIowa":20700,"Ġavatar":20701,".cursor":20702,"![":20703,"Ġhanging":20704,"Her":20705,"Such":20706,"';ĊĊĊ":20707,"orgeous":20708,"()==":20709,"ĠviewModel":20710,"Ġãĥ":20711,"Ġels":20712,"ĠAgent":20713,"Fetch":20714,"apor":20715,"Ġcx":20716,"pread":20717,"ĠPier":20718,"oeff":20719,"Sn":20720,"ĠVirtual":20721,"Apr":20722,".White":20723,"_MOD":20724,"ĠPoints":20725,"失":20726,"Ġgenes":20727,"Ġvendor":20728,"Ġmainstream":20729,"Ċ":20758,"Filename":20759,"Ġsne":20760,"ĠFootball":20761,"Ġrival":20762,"Ġdisaster":20763,"ionic":20764,"ĠDamage":20765,".Resource":20766,"-en":20767,"ĠTypes":20768,"getString":20769,"(board":20770,"Ġbol":20771,"plain":20772,"zym":20773,"า":20774,"Ġscanner":20775,"ilder":20776,"_msgs":20777,"æı":20778,"(intent":20779,"Ġdestruct":20780,"Ġbust":20781,"ĠEmploy":20782,"oni":20783,"ĠUIViewController":20784,"Ġodds":20785,"earer":20786,"Geometry":20787,"Ġyii":20788,"_EXPORT":20789,"ĠAttack":20790,"Ġniet":20791,"Ġimpression":20792,"ĠGil":20793,"_prob":20794,"ĠCF":20795,"ĠExperience":20796,"/plugins":20797,".Method":20798,"Ġbeliefs":20799,"Native":20800,"_build":20801,"Ġvig":20802,"Ġranks":20803,"covered":20804,"such":20805,"Guard":20806,".pack":20807,"adder":20808,"ivia":20809,"lng":20810,"ĠвÑĭ":20811,"Timestamp":20812,"_now":20813,"Ġpoker":20814,"Ġunc":20815,"Ġshapes":20816,"-types":20817,"_period":20818,"pk":20819,"Ġveteran":20820,"Ġsono":20821,"Ġappointed":20822,"overflow":20823,".driver":20824,"_cat":20825,"utt":20826,"plant":20827,"imb":20828,"ĠAccept":20829,"Ġconcert":20830,"ĉnode":20831,"ĉz":20832,"?>čĊ":20833,"Ġbanned":20834,"ĉĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":20835,"Ġtoxic":20836,"Ġdisappe":20837,"ÈĽ":20838,"Ġgrace":20839,"ateful":20840,"Reply":20841,"ĠCruz":20842,"Ġscrap":20843,"Ġkeywords":20844,"simp":20845,"Ġmortgage":20846,"Ġcyber":20847,"ĠExecute":20848,"Ġlatitude":20849,"ifu":20850,".COM":20851,"dbo":20852,"Ġsorts":20853,"ĠGas":20854,"omial":20855,".Local":20856,"Cells":20857,".Replace":20858,"Strings":20859,".fit":20860,"ĠThird":20861,"%\",Ċ":20862,"Ġ{}\".":20863,"ĠSony":20864,"Ġ[:":20865,"Ġfallen":20866,".')Ċ":20867,"inh":20868,"ĠMC":20869,"Ġredis":20870,"Codes":20871,"Ġprofiles":20872,"hook":20873,"Reducer":20874,"_FUNC":20875,"Ġnavigate":20876,"strlen":20877,"Ġhorm":20878,"áŀ":20879,"ĠSR":20880,".boot":20881,"Ġdigest":20882,"ĉheader":20883,".findOne":20884,"æģ":20885,"DbType":20886,"nia":20887,"_merge":20888,"Ġdonne":20889,"/Getty":20890,"_CHAR":20891,"Ġbands":20892,".URL":20893,"artial":20894,"Ġfreq":20895,"Ġsist":20896,"Ng":20897,"Ġrendering":20898,"\\Core":20899,"Widgets":20900,"ĠVA":20901,"Ġactivists":20902,"Ste":20903,"=_":20904,"alla":20905,"Stamp":20906,"Ġloads":20907,"Ġxx":20908,"ĠLearning":20909,".Mvc":20910,"uir":20911,"(\"$":20912,"Ġconnecting":20913,"ReadOnly":20914,"uru":20915,"ĠEag":20916,"BIT":20917,"_DEL":20918,"å§":20919,"arrass":20920,"external":20921,"ĠYOUR":20922,"ĠBrew":20923,"ĠFive":20924,"Ġresize":20925,"igid":20926,"eration":20927,"ĠÑį":20928,"åĬł":20929,"ĠCatch":20930,"Ùģ":20931,"ĠLeon":20932,"amil":20933,".Body":20934,"Clip":20935,"/list":20936,".br":20937,"EditText":20938,"ĉdb":20939,".Game":20940,"(BuildContext":20941,"backend":20942,".Red":20943,"facebook":20944,".urls":20945,"mr":20946,"rolled":20947,"-------":20948,"Ġintervention":20949,"Ġretirement":20950,"ĠKit":20951,"ĠPRE":20952,"UpperCase":20953,"ĠSocket":20954,"Ġ:-":20955,"Ġstudying":20956,"ĠMetro":20957,"arded":20958,"Ġconversations":20959,"Called":20960,"Ġexamine":20961,"ertificate":20962,".gz":20963,"-responsive":20964,"Ġrefund":20965,"_network":20966,"allowed":20967,"empt":20968,"Ġmeals":20969,"Categories":20970,"Ġtraveling":20971,"Ġkg":20972,"Ġshame":20973,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":20974,"Ġexplicitly":20975,"Ġmathematic":20976,"ĠSuite":20977,"ĠRGB":20978,"******/":20979,"Ġmixture":20980,"learning":20981,".template":20982,"atts":20983,"wx":20984,"ĉctx":20985,".properties":20986,"Ġdrinks":20987,"ĠEither":20988,"setText":20989,".getData":20990,".zip":20991,"Ġreveals":20992,".Ċ":21005,"Ġranked":21006,"_impl":21007,"ĠHandles":21008,"Ġhosted":21009,"Ġupdating":21010,"album":21011,"éĿ":21012,"Ġshader":21013,"Editors":21014,"-round":21015,"[]{":21016,"Ġsep":21017,"ĠHi":21018,"TEM":21019,"lookup":21020,".man":21021,"_INPUT":21022,"Ġthreatened":21023,"_IMPORT":21024,"Ġdrops":21025,"ruit":21026,"sid":21027,"both":21028,"ĠExcel":21029,"Ġjer":21030,"ordinary":21031,"ей":21032,"VIEW":21033,"reply":21034,"Ġ):Ċ":21035,"colors":21036,"verified":21037,"_Tr":21038,"_parse":21039,"Ġcongress":21040,"Promise":21041,"ints":21042,"ĠMother":21043,".Api":21044,"ĠDuration":21045,"ĠfirstName":21046,"inheritdoc":21047,"ĠMars":21048,"Ġapr":21049,"ODY":21050,"Ġvisits":21051,"Ġhealing":21052,"letters":21053,")));čĊ":21054,"future":21055,".Framework":21056,"Ġkiss":21057,"Ġinvolve":21058,"Ġsilent":21059,"adows":21060,"Ġanybody":21061,"sch":21062,"Ġsolely":21063,"-img":21064,"Ġpropri":21065,"Ġinstruct":21066,"Ġlicenses":21067,"Ġmeth":21068,"Ġcondem":21069,"ĠDomain":21070,"ĠHarris":21071,"ĠsÃ¥":21072,"CEPT":21073,"Batch":21074,"@extends":21075,"ĠCONTRIBUT":21076,".DataFrame":21077,"_packet":21078,"recision":21079,"Ġfocusing":21080,".ht":21081,"__\":Ċ":21082,":Get":21083,"ĠKC":21084,"Ġpassage":21085,"Segment":21086,"_center":21087,"-zA":21088,"_BL":21089,"Ġconvin":21090,"Ġclassified":21091,"ĠNSMutable":21092,"_ap":21093,"tile":21094,"Rectangle":21095,"(nums":21096,"vens":21097,"ĠUIButton":21098,"ĠFeder":21099,"amo":21100,"Ġoutline":21101,"ĠParser":21102,"Ġâī":21103,"ĠWorks":21104,".Schema":21105,"Ġengines":21106,"_common":21107,"_old":21108,"ĠsetContentView":21109,"Ġ///<":21110,"ĠBT":21111,"fm":21112,"Ġdivers":21113,"_weights":21114,"emark":21115,"ĠACT":21116,"Ġproportion":21117,"overlay":21118,".dirname":21119,"ĠGit":21120,"_REFERENCE":21121,"<>":21122,"lb":21123,"_rule":21124,"è´¥":21125,"ĠPutin":21126,"Ġsleeping":21127,"():čĊ":21128,"Ġpreserve":21129,"Ġparliament":21130,"ĠLooking":21131,"Ġpicking":21132,"ĠDispatch":21133,"Ġslip":21134,"ëĵ":21135,"ĠLyn":21136,"_signal":21137,"configuration":21138,"ĠPitt":21139,"aden":21140,"procedure":21141,"Ġenthusi":21142,"fight":21143,"ĠConsider":21144,"Ġtorn":21145,"Connected":21146,".cos":21147,"_groups":21148,"ĠThink":21149,"Ġdeliber":21150,"Ġresid":21151,"working":21152,".columns":21153,"ĠCalled":21154,"Ġeslint":21155,">\",":21156,"_DOWN":21157,"hist":21158,"ĠAdvanced":21159,"Ġrewards":21160,"actors":21161,"Ġsilence":21162,"Ġmyth":21163,"Ġneur":21164,"Ġauction":21165,".GetString":21166,"eks":21167,"(project":21168,"ĉmsg":21169,"ĉoutput":21170,"Ġcomplaints":21171,",S":21172,"Ġtbl":21173,"Ġ,ĊĊ":21174,"riors":21175,"ahren":21176,"Ġlawyers":21177,"redux":21178,"_symbol":21179,"offee":21180,"_RESULT":21181,"(Name":21182,"UTC":21183,".currentTime":21184,"Ġorganis":21185,".arg":21186,"Ġminim":21187,"wick":21188,"Ġreceives":21189,"Balance":21190,"Ġspeaks":21191,"ĠDays":21192,"ĠBelow":21193,"tipo":21194,"Present":21195,"Ġreserv":21196,"hp":21197,"Ġrit":21198,"_RIGHT":21199,"--)":21200,"Ġchairman":21201,"DIS":21202,"ĠBOOST":21203,"Ġexperiments":21204,"__);Ċ":21205,"Ġstamp":21206,"Ġfert":21207,"Ġfond":21208,"Ter":21209,"elve":21210,"uren":21211,"+i":21212,"endency":21213,"Ġvirtually":21214,"...\"":21215,"ï½ŀ":21216,"-cent":21217,"_unique":21218,"Ġpricing":21219,"mic":21220,"RESH":21221,"Ġ:::":21222,"Ġannotation":21223,"ĠCircle":21224,"ongodb":21225,"itas":21226,"Ġ%(":21227,"(component":21228,"Ġоб":21229,"(port":21230,"-hour":21231,".obj":21232,"LBL":21233,"Ġjury":21234,"GBT":21235,"Ġspy":21236,"ĠProfessional":21237,"Ġ\"\";ĊĊ":21238,"Ġstriking":21239,"Ġdiscrimination":21240,"Ġpays":21241,"lict":21242,"entes":21243,"Ġthrowing":21244,"ĠPlugin":21245,"(def":21246,"ĠRuntimeException":21247,"ĠMigration":21248,"Ġdic":21249,"bag":21250,"onia":21251,"Ġcorruption":21252,"(Map":21253,"Ġprz":21254,".dto":21255,"Ġacquire":21256,"StateToProps":21257,"Ġloving":21258,"ож":21259,"_pattern":21260,"Ġemotions":21261,"Ġpublisher":21262,"_be":21263,"Ġcouples":21264,"oj":21265,"ĠChart":21266,"Ġtrop":21267,".tool":21268,"Ġestablishment":21269,"Ġdol":21270,"Ġtower":21271,"Ġlane":21272,"ĠSydney":21273,"Ġfilling":21274,"claimed":21275,"Ġdialogue":21276,"Ġconvention":21277,"booking":21278,"parency":21279,"æ±":21280,"ĠGeneric":21281,"\\Schema":21282,"Ġranges":21283,"/ch":21284,"Ġpanels":21285,"Ġruled":21286,"çĶŁ":21287,".ts":21288,"_sets":21289,"Ġcleanup":21290,"Previous":21291,"ĠAnimal":21292,"($(":21293,"ĠAve":21294,"ollar":21295,"_eval":21296,"ĉName":21297,"(tree":21298,"Ġ\"]":21299,"Ġduties":21300,"='/":21301,"Clicked":21302,"Ġdifferently":21303,"ĠClark":21304,"Ġdit":21305,"ologists":21306,"Ġsynd":21307,"Ġsends":21308,"-known":21309,"kb":21310,"ĠModal":21311,"itative":21312,"Ġracing":21313,"Ġhighlights":21314,"ĠSimon":21315,"ĠCaptain":21316,"ä¿¡":21317,"ĠCB":21318,"contin":21319,"aran":21320,"Ġphysics":21321,"retty":21322,"etal":21323,".md":21324,"axios":21325,"Ġspeakers":21326,"Ġprep":21327,"Ġawarded":21328,"ì§Ģ":21329,"ĠCorn":21330,"ĠNature":21331,"UDIO":21332,"Ġproj":21333,"-pre":21334,"[u":21335,"Features":21336,"ĠisEqual":21337,"Binary":21338,"sig":21339,"Ġconfusion":21340,"ĠHat":21341,"Ġktó":21342,".configure":21343,"MON":21344,"/edit":21345,"_Add":21346,",true":21347,"Ġcli":21348,"ErrorMessage":21349,"-loader":21350,"Dimensions":21351,"ultiply":21352,"Ġ{!!":21353,"ĠSqlCommand":21354,"Ġspoken":21355,"Ġpics":21356,"Ġtoy":21357,"(Key":21358,"ĠLoop":21359,"ب":21360,"EATURE":21361,"inction":21362,"_setup":21363,"wrapper":21364,"Ġtong":21365,"cular":21366,"Opt":21367,".Pl":21368,"=\",":21369,"(length":21370,"umn":21371,"Ġchrom":21372,"Ġsevent":21373,"ĠIllegalArgumentException":21374,"ĉstart":21375,"Ġbegun":21376,"CEPTION":21377,"dataset":21378,"ĠFailed":21379,"cols":21380,"Ġknee":21381,"imore":21382,".splice":21383,"shell":21384,"iggers":21385,"Ġthemes":21386,"ĠDJ":21387,"ĠAssistant":21388,"-$":21389,"Maybe":21390,"Ġordering":21391,"ĠIntelligence":21392,"ĠMassachusetts":21393,"Ġfailing":21394,"elson":21395,"Great":21396,"=i":21397,".rest":21398,"Ġinvite":21399,"-disable":21400,".GroupBox":21401,"âĢĻest":21402,"Ġtackle":21403,"gv":21404,"etter":21405,"Ġ),čĊ":21406,"_rules":21407,".warn":21408,"functions":21409,"ĠChristians":21410,"Ġbacked":21411,"Ġslider":21412,"Ġenjoying":21413,"nest":21414,"Ġhij":21415,"_ms":21416,"//*":21417,"Annotations":21418,"ĠVariables":21419,"":21620,"cycle":21621,"ĠBull":21622,"paths":21623,"Ġunp":21624,"ĠviewDidLoad":21625,"_Model":21626,"ĠassertTrue":21627,"Ġrated":21628,"Decl":21629,"verted":21630,"ĠDat":21631,"brew":21632,"Ġpointing":21633,"Ms":21634,"ĠPointer":21635,")'":21636,"_non":21637,"ĠSEC":21638,"Ġyeah":21639,"gency":21640,"initialize":21641,"fly":21642,"[pos":21643,",g":21644,"Tele":21645,"Ġjoke":21646,"Ġclause":21647,".findById":21648,"enes":21649,"(instance":21650,"£":21651,"Ġslic":21652,"_home":21653,"Ġ*/}Ċ":21654,"_pages":21655,"(service":21656,"RP":21657,"ĠAmong":21658,".getCurrent":21659,"ãĤ¹":21660,"Ġslee":21661,"=[Ċ":22071,"oler":22072,"Ġlibert":22073,"Ġ`Ċ":22074,"Ġwenn":22075,"lated":22076,"Ġimmune":22077,"(Node":22078,"ĠProblem":22079,"ĠAbs":22080,"logs":22081,"Ġ../":22082,"ĠADC":22083,"Ġ}}\">Ċ":22084,">');Ċ":22085,"=b":22086,"ĠWind":22087,"lahoma":22088,"Ġallocate":22089,"orian":22090,"Ġprescription":22091,"-quality":22092,"ĠMayor":22093,"inely":22094,"endforeach":22095,"ĠComplex":22096,"kom":22097,"TY":22098,"]].":22099,".Style":22100,"_many":22101,"','$":22102,"Ġbarrier":22103,"ĠFetch":22104,"ĠMarvel":22105,"Ġresist":22106,"ого":22107,"bidden":22108,"ĠRunnable":22109,":false":22110,"Ġbuilds":22111,"ĠStage":22112,"Ġdub":22113,"empo":22114,".site":22115,";ĊĊĊĊ":22116,"ĠDenver":22117,"Ġrevel":22118,"Ġtriggered":22119,"Ġdice":22120,"_fail":22121,"Ġgc":22122,"ĉX":22123,"ĠThrowable":22124,".router":22125,"ĠRevolution":22126,"ÑĢа":22127,"_NON":22128,"Ł¥":22129,"Ġelder":22130,"Ġabroad":22131,"Ġе":22132,"ĠAdult":22133,"blr":22134,"glyphicon":22135,"Ġpromoting":22136,"Ġiz":22137,"ĠSolid":22138,"_loader":22139,"early":22140,".enabled":22141,"-edit":22142,"ĠUL":22143,"_play":22144,"ĠInterrupt":22145,"Ġadvantages":22146,"ucle":22147,"Ġmechanical":22148,".tableLayoutPanel":22149,"ĠWorking":22150,"Ġanonymous":22151,"Rating":22152,"igious":22153,"_phone":22154,".addActionListener":22155,"Ġfran":22156,"unden":22157,"Ġ*)&":22158,"_bool":22159,"ulative":22160,"Ġcone":22161,"ĠMult":22162,"Ġmö":22163,"ĠForward":22164,"]):Ċ":22165,"Ġconvinced":22166,"acted":22167,"ãģĵ":22168,"ĠConfigure":22169,"Ġceiling":22170,"Der":22171,"Ġpassengers":22172,"Groups":22173,"Ġsoccer":22174,"/W":22175,"aviors":22176,"swith":22177,"ĠZone":22178,".Options":22179,"ĠMom":22180,"ieder":22181,"Arrays":22182,"Ġtreatments":22183,"Ġprotecting":22184,"fac":22185,"Ġpickle":22186,"ButtonItem":22187,"Ġblocking":22188,"strar":22189,"ò":22190,"ĠExport":22191,"Ġthrew":22192,"otta":22193,"ĠBASE":22194,".ws":22195,".LEADING":22196,"orderBy":22197,"_delay":22198,"ĠPu":22199,".dll":22200,"ĠChoose":22201,"Police":22202,"ĠBEGIN":22203,"boxes":22204,"Ġdiamond":22205,",l":22206,"Ġĉĉĉ":22207,"Ġcurious":22208,"tv":22209,"Ġerotische":22210,"ackages":22211,"ĉSet":22212,"Tick":22213,".border":22214,"staticmethod":22215,"Ġcher":22216,"invoice":22217,"Ġcru":22218,"Ġdefect":22219,"_metadata":22220,"relation":22221,"ikan":22222,"[N":22223,"(Qt":22224,"(Base":22225,"æģ¯":22226,"beat":22227,"ĠEmpty":22228,"ĉo":22229,"_shift":22230,"Ġregret":22231,"Those":22232,"Cent":22233,"ĠPortug":22234,"ĠIslands":22235,"ĠTIME":22236,"Management":22237,"-sp":22238,"ême":22239,"Ġnotion":22240,"unifu":22241,"PK":22242,"è¡Į":22243,"ĠCURLOPT":22244,"\\\"\\":22245,"UV":22246,"çº":22247,"dra":22248,"cou":22249,"=`":22250,"ĠDestroy":22251,"rp":22252,".cancel":22253,"GG":22254,"runtime":22255,"ĠVue":22256,"Ġprogressive":22257,"/services":22258,"Ġrunner":22259,"_FRAME":22260,".ToolStripMenuItem":22261,"Ġ','":22262,"delay":22263,"=utf":22264,"Ġscreening":22265,"Ġpulling":22266,"omas":22267,"Ġanth":22268,"-new":22269,"/local":22270,"ĠiPad":22271,"Ġtwitter":22272,"Ġdying":22273,"Ġheaven":22274,"ĠUInt":22275,"ĠSenator":22276,"Ġpresum":22277,"ĠWalker":22278,"Ġovercome":22279,"etection":22280,"Ġembarrass":22281,"China":22282,"Include":22283,"ROLL":22284,"ĠdataType":22285,"David":22286,"ร":22287,"lop":22288,"-month":22289,"Ġscar":22290,"ĠSafe":22291,"Ġ****************************************************************":22292,"Ġaccessories":22293,"Ġramp":22294,"_USE":22295,"Ġcontrad":22296,"))]Ċ":22297,"Ġprest":22298,"ĠHR":22299,"ĠRap":22300,"Ġusize":22301,"Ġcapability":22302,"Ġcort":22303,"-next":22304,"Ġburden":22305,"_reader":22306,"Ġ@@":22307,"regular":22308,"ĠKa":22309,"MAN":22310,"Ġastr":22311,"Ġ'')Ċ":22312,"Ġfed":22313,"Ġparsing":22314,"ĠYears":22315,"Ġbroker":22316,"\":{\"":22317,"Ġakt":22318,"Inventory":22319,"abeled":22320,"Ġargparse":22321,"*******Ċ":22322,"versation":22323,"Ġcord":22324,"ĠTi":22325,"Ġhopefully":22326,"Ġah":22327,"verb":22328,"Ġstolen":22329,".Entry":22330,"Ġexpecting":22331,"Orientation":22332,"Ġpowered":22333,"Ġpersist":22334,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":22335,"']);":22336,"')),Ċ":22337,"ĠCash":22338,"ĉitem":22339,"grades":22340,"ropol":22341,"basic":22342,"Ġ\");čĊ":22343,"Ġawards":22344,"(range":22345,"-all":22346,"ĠIBOutlet":22347,"ĠIndeed":22348,"----------------------------------------------------------------------------":22349,"Ġstomach":22350,"Ġflower":22351,"Ġsew":22352,"_times":22353,"avis":22354,"QString":22355,"ĠRoutes":22356,"_prot":22357,"Ġcomedy":22358,"Ġlogout":22359,"Ġwooden":22360,"Ġposter":22361,"piece":22362,".Join":22363,"ĠPok":22364,"celona":22365,"mutex":22366,";čĊčĊčĊ":22367,"Ġstrikes":22368,"Loaded":22369,")arg":22370,"esa":22371,"United":22372,"Ep":22373,"PELL":22374,"ĠAtlantic":22375,"ullet":22376,"apple":22377,"Ġsettled":22378,"acon":22379,"Ġprinter":22380,"ĠGC":22381,"å®ļ":22382,"Ġrendered":22383,",âĢĻ":22384,"heit":22385,"social":22386,".ge":22387,"ĠRick":22388,"ĠUtah":22389,"got":22390,"onical":22391,"ĠScroll":22392,"ĠSciences":22393,"Ġjug":22394,"Ġampl":22395,"enti":22396,"LEFT":22397,"Ġtabs":22398,"Ġenormous":22399,".getKey":22400,"locate":22401,".EX":22402,".storage":22403,".We":22404,"Ġtoast":22405,"ĠAdditionally":22406,"ĠNOW":22407,"_UPDATE":22408,"Ġtransferred":22409,"tha":22410,".Display":22411,"_ui":22412,"IDEO":22413,"Ġmeaningful":22414,"ĠMoscow":22415,",this":22416,"ĠVictoria":22417,"æĶ¹":22418,"ĠÐŁ":22419,".stack":22420,"ĠBarn":22421,"paredStatement":22422,":string":22423,"Ġbij":22424,"ĠSTATE":22425,"Ġemployers":22426,"ĉinput":22427,"(|":22428,"Ġlex":22429,"invoke":22430,"ĉnum":22431,"++,":22432,"atial":22433,"orses":22434,"Ġfork":22435,"_txt":22436,"ĠAntonio":22437,"Ġ(<":22438,"averse":22439,"Ġdevast":22440,"ãĢĢ":22441,".Dec":22442,"ĠGard":22443,"/ui":22444,".%":22445,"tri":22446,"Ġrolled":22447,"ValuePair":22448,"itten":22449,"ĠTher":22450,"Ġvrou":22451,"ĠFlow":22452,"ĠFinance":22453,"ĠComb":22454,"HC":22455,".setVisible":22456,"isl":22457,"Ġpk":22458,"Ġupset":22459,"(raw":22460,"ĠVice":22461,"eatures":22462,"ĠLang":22463,"Looking":22464,"ĠAST":22465,"Ġtrips":22466,"ĠJustin":22467,"browser":22468,"=\"'.$":22469,".vertices":22470,"-co":22471,"}/{":22472,"Ġ?,":22473,"ĠDomin":22474,"ĠBelg":22475,"\"<":22476,"Ġsuppose":22477,"addy":22478,"Ġwalks":22479,"ERRU":22480,"_filters":22481,"Preferred":22482,"scene":22483,"еÑģ":22484,"ĠAffairs":22485,"Ġ\"#{":22486,"ĠonSubmit":22487,"Ġstocks":22488,"/view":22489,"gree":22490,"-get":22491,"hit":22492,"Jo":22493,".getC":22494,"Initialized":22495,"ÑĤи":22496,"cuts":22497,"(Type":22498,"ĠAgreement":22499,"ĠVietnam":22500,"Ġ/*!":22501,"Ġpizza":22502,"-view":22503,"_em":22504,"Ġlhs":22505,"Ġmuy":22506,"ĠIdent":22507,"ĠFriends":22508,"Ġabund":22509,"_AD":22510,".timestamp":22511,"-'":22512,"Ġduplicate":22513,"Ġhunting":22514,"Ġregulatory":22515,"iao":22516,"amous":22517,"ĠEntertainment":22518,"[A":22519,"iatric":22520,"_CLIENT":22521,"ĠKids":22522,"/pkg":22523,"Break":22524,")));ĊĊ":22525,"ĠShape":22526,"Ġrelating":22527,"Interrupt":22528,"ableOpacity":22529,"embre":22530,"Ġmystery":22531,"Ġjournalists":22532,"ritable":22533,".Link":22534,"Ġstopping":22535,"CRET":22536,".DB":22537,"Ġpopularity":22538,"Ġgew":22539,"Ġimpr":22540,"setValue":22541,"FLAG":22542,"ĉmax":22543,"Ġbake":22544,"wy":22545,"ĠEconomic":22546,"Ġencontr":22547,"Ġfname":22548,"/de":22549,"Rank":22550,"Ġbugs":22551,".sm":22552,"Ġmedian":22553,"DOWN":22554,"ĠSure":22555,"AtIndex":22556,"ĠDick":22557,"Ġ(__":22558,".delta":22559,"Fr":22560,"Ġsuggesting":22561,"ĠRecyclerView":22562,",e":22563,"START":22564,"/****************************************************************************":22565,"xford":22566,"Ġreceipt":22567,"CLAIM":22568,"readonly":22569,"Ġengaging":22570,"Ca":22571,"asma":22572,"Ġensuring":22573,"English":22574,"ĠVancouver":22575,"hyth":22576,"Ġpurchasing":22577,"ĠPI":22578,".word":22579,"(sp":22580,".home":22581,":def":22582,"Ġgig":22583,"ĠVe":22584,"forum":22585,"ĠMitch":22586,"Bay":22587,"_FL":22588,"Ġsoll":22589,"_columns":22590,"Ġminority":22591,"bird":22592,"Ġhanded":22593,"SSL":22594,"STAT":22595,"Ġnervous":22596,"ĥ½":22597,"ĠfilePath":22598,"CREATE":22599,"Aw":22600,"Ġpens":22601,"seed":22602,"ĠCompute":22603,"olk":22604,"ĠAsset":22605,"reach":22606,"'),čĊ":22607,"navigation":22608,"LF":22609,"/util":22610,"ĠPub":22611,"ĠâĶ":22612,"cion":22613,"##Ċ":22614,"III":22615,"TagName":22616,"Ġamid":22617,"permission":22618,"ifiable":22619,"xFFFFFFFF":22620,"ни":22621,".Buffer":22622,"_irq":22623,"dark":22624,"Ġretval":22625,".fire":22626,"production":22627,".listen":22628,"ĠWeather":22629,"Ġbuyers":22630,".ne":22631,"erp":22632,"ĠPent":22633,"Ġwelfare":22634,"ĠpageSize":22635,"ĠStadium":22636,"erta":22637,"Ġlev":22638,"ampa":22639,"Pager":22640,"Ġcharging":22641,"ĠNetflix":22642,"|null":22643,"_random":22644,".xpath":22645,"Ġstere":22646,"ĠISIS":22647,"ponses":22648,"(loc":22649,"eyond":22650,"ĠOfficial":22651,"ĠMaryland":22652,"DataType":22653,"_par":22654,"{},":22655,"ĠEnjoy":22656,"_SHIFT":22657,"ĠAwards":22658,"_ENTRY":22659,"Ġseemingly":22660,"enticate":22661,"Ġhearts":22662,"_;ĊĊ":22663,"ĠHIV":22664,"Ġindivid":22665,"ĠFlag":22666,"_ctrl":22667,"ĠCallback":22668,",z":22669,"ĠGPU":22670,"ĉobj":22671,"ĠPhoenix":22672,"ĠBUS":22673,"Ġrubber":22674,"_AUTH":22675,"ĠSolutions":22676,"(location":22677,"Variables":22678,".setEnabled":22679,"_high":22680,"WO":22681,"Gesture":22682,"Ġretry":22683,"ĠobjectForKey":22684,"alloween":22685,"Ġmos":22686,"ĠCele":22687,"Ġikke":22688,"(cell":22689,"ĠMODE":22690,"rena":22691,"Ġdescribing":22692,"Ġphi":22693,"Ġrd":22694,"Ġdeserve":22695,"Ġwheels":22696,"å¸Ĥ":22697,"Ġcritics":22698,"Namespace":22699,"ĠFra":22700,"ĠĊĊĊĊ":22701,"Ġalla":22702,"Ġrequiring":22703,"æľŁ":22704,"utation":22705,"Ġdelayed":22706,"Ġadministrative":22707,"Ġbay":22708,".hidden":22709,"Tex":22710,"Ġboundaries":22711,"Ġ]);ĊĊ":22712,"ĠFollowing":22713,"~/":22714,"Fi":22715,"_conv":22716,"_TITLE":22717,"Ġdesde":22718,"ICollectionView":22719,"Alias":22720,"Ġbite":22721,"patient":22722,"_COMMAND":22723,"Completed":22724,"ĉelif":22725,"(<":22726,"Business":22727,"ĠPool":22728,"Ġpursue":22729,"ĠBan":22730,"_steps":22731,"_DECL":22732,"umble":22733,"Ġcombo":22734,"ĠLayer":22735,".xr":22736,"Ġdup":22737,"---------":22738,"Ġmodifier":22739,"rob":22740,"rez":22741,"Ġathletes":22742,"Used":22743,"wear":22744,"Ġlegitimate":22745,"Ġ\"ĊĊ":22746,"Ġhv":22747,"Std":22748,"ĠHold":22749,"Ġsurviv":22750,"ĠAlliance":22751,"ĠEarly":22752,"Behavior":22753,"(font":22754,"/libs":22755,"Ġrectangle":22756,"Ġsinger":22757,"Ġamp":22758,"EqualTo":22759,"Ġ\".\"":22760,"Ġgirlfriend":22761,"å±":22762,"linear":22763,"observ":22764,"Ġpiù":22765,"Ġcomplement":22766,"WithValue":22767,"(password":22768,"take":22769,"Blank":22770,"ĠCompar":22771,"'\",":22772,"_policy":22773,"mongoose":22774,"_FAILED":22775,".report":22776,"Ratio":22777,".PerformLayout":22778,"usable":22779,"mers":22780,"_render":22781,"PEED":22782,"Ġlesb":22783,"ĉE":22784,"_tool":22785,"Ġladies":22786,"оÑģ":22787,"))))Ċ":22788,";;;;":22789,".dot":22790,"Ġnest":22791,"peak":22792,"ukkit":22793,"eca":22794,"_SW":22795,"Ġ&(":22796,"ĠOklahoma":22797,"Ġbanking":22798,"ĠNintendo":22799,"Ġreproduce":22800,"_elements":22801,"_mac":22802,"proxy":22803,"Ġremarkable":22804,"}/${":22805,"Ġouts":22806,".hasNext":22807,"MODE":22808,"Ġanime":22809,".conn":22810,"Unique":22811,"Dom":22812,"Ġimportantly":22813,"itty":22814,"Ġjuice":22815,"Tw":22816,"ĠPartners":22817,"Ġattacking":22818,"Ġportable":22819,"amiento":22820,".PictureBox":22821,".gen":22822,"Ġoptimal":22823,"Ġrecre":22824,"Ġjournalist":22825,"ĠExtract":22826,"ĠMoreover":22827,"ĠmarginTop":22828,".Ap":22829,"Ġfiring":22830,"NaN":22831,"ĉtemplate":22832,"ад":22833,".En":22834,"Ġdefence":22835,"ĠTel":22836,"ilen":22837,"jan":22838,"=data":22839,"ĠUrl":22840,"ĠReuters":22841,"(total":22842,"ĠFifth":22843,"Ġessays":22844,"Ġinterpretation":22845,"Ġcharity":22846,"ĠRules":22847,"Ġsubsection":22848,"styled":22849,"azer":22850,"lags":22851,"LIST":22852,"Ġuploaded":22853,"Ġtrash":22854,"Ġregistr":22855,"Ġseller":22856,">';čĊ":22857,"ĠstartTime":22858,"çĻ":22859,"sy":22860,"(HttpServletRequest":22861,"Ġtrap":22862,"GC":22863,"Ġembedded":22864,"Ġsurrounded":22865,"imits":22866,"TX":22867,"ylinder":22868,"ĠFal":22869,"Ġsentences":22870,"ĠJa":22871,"IFICATION":22872,"weapon":22873,"ovation":22874,"Ġcoat":22875,"Ġinterpol":22876,"Ġlips":22877,"ĠKy":22878,"Ġvectors":22879,"_am":22880,"Ġintake":22881,".world":22882,"Ġinbox":22883,"ĠMAC":22884,"_ab":22885,"(nameof":22886,"Ġentert":22887,"Ġgathering":22888,"ĠSIM":22889,"++.":22890,"nya":22891,"'}}":22892,"ĠUPDATE":22893,"Ġpac":22894,"(html":22895,"ĠSant":22896,"iating":22897,"ĠIdeas":22898,"Ġspray":22899,"ĠHart":22900,"Ġverification":22901,"adesh":22902,"/modules":22903,"ĠMind":22904,"ĠSizedBox":22905,"Ġshelter":22906,"Ġheroes":22907,"atty":22908,"Ġcertified":22909,"sj":22910,"Ġêtre":22911,"ÅĤo":22912,"Ġpublishing":22913,"ĠMalays":22914,".getUser":22915,"ĠProvider":22916,"ĠLinkedList":22917,"ĠBor":22918,"ROUND":22919,"did":22920,"tain":22921,"pire":22922,"ĠJenn":22923,"tel":22924,"ande":22925,"_front":22926,"ĠMcG":22927,"TestMethod":22928,"à¸Ń":22929,"Ġoccasionally":22930,"ĠWales":22931,"Ġexercises":22932,"ĠÐĴ":22933,"-plus":22934,"Ġvalidator":22935,"Ġprayer":22936,"LATED":22937,"_author":22938,"Ġlabour":22939,"++Ċ":22940,"-equiv":22941,"ĠGPL":22942,"Ġfacebook":22943,"simple":22944,"gly":22945,"Processor":22946,"ipy":22947,"Ġ*>":22948,"Ġcleared":22949,"ĠPush":22950,"Ġpenis":22951,"Structure":22952,"lij":22953,"ĠMorgan":22954,"Ġhandful":22955,"\".Ċ":22956,"|\\":22957,"Ġ********************************":22958,"ĠAqu":22959,"_IC":22960,".loads":22961,"Ġmeter":22962,"ĠMarine":22963,"::{":22964,"ĠTS":22965,"ĠArrays":22966,".Title":22967,"GRAM":22968,"termin":22969,"Ġcoinc":22970,"Else":22971,"_states":22972,"-run":22973,"members":22974,"astro":22975,"ĠonPress":22976,"Ġbeings":22977,"Ġabandoned":22978,"Ġtaxp":22979,"owners":22980,".mode":22981,"Ġdiagnosis":22982,"Ġ_Ċ":22983,"ĠKnight":22984,"ĉA":22985,"Ġobserve":22986,"),'":22987,"!\")Ċ":22988,"ĠPara":22989,"Ġvariation":22990,"(False":22991,"ĠAnti":22992,"Ġgri":22993,"Ġhomeless":22994,"?v":22995,"Ġbez":22996,".Server":22997,"release":22998,"ĠPatri":22999,"Ġchars":23000,"Ġranking":23001,"activation":23002,"Ġwides":23003,"qr":23004,".Sql":23005,"acular":23006,"ĠBot":23007,"_sync":23008,"Ġhappiness":23009,"Ġvolunteers":23010,"Ġsits":23011,"/<":23012,"[e":23013,"(fileName":23014,"Ġcapac":23015,"ĠMaria":23016,"father":23017,"Ġgram":23018,"*i":23019,"Ġcaso":23020,"_draw":23021,"ĠRaw":23022,"ĠIterator":23023,"ĠPadding":23024,"PD":23025,"BOX":23026,"ĠSPECIAL":23027,"Ġfecha":23028,"Ġvide":23029,"ĠLeader":23030,"以":23031,"$(\".":23032,"Ġdiameter":23033,"Ġmild":23034,"Ġrocks":23035,"appings":23036,"directory":23037,".flush":23038,"ĠJess":23039,"UNIT":23040,"ĠPear":23041,"Ġmandatory":23042,"Sur":23043,"qt":23044,"Ġstreams":23045,"Ġcooperation":23046,"ĠSac":23047,"Ġcheaper":23048,"ĉch":23049,"animation":23050,"fare":23051,"(height":23052,"(True":23053,"NY":23054,"Ġwrest":23055,"Ġpolls":23056,"Ġencountered":23057,"ĠMarketable":23058,"_PASSWORD":23059,"_SELECT":23060,"ĠArabia":23061,"_clock":23062,"Ġvoy":23063,"Ġиз":23064,"Ġstir":23065,"isible":23066,"-effect":23067,".created":23068,"Ġtoys":23069,"ĠTradable":23070,"Ġrust":23071,"Ġstrcpy":23072,"_timestamp":23073,"Ġtalented":23074,",null":23075,"ĠJobs":23076,"ĠPortland":23077,"Ġweakness":23078,"Throw":23079,"ĠAngel":23080,"ä¿®":23081,"Ġuncert":23082,"ï¼īĊ":23083,"ĠìĿ´":23084,"Which":23085,"Ġ[-]:":23086,"Something":23087,"Ġconvicted":23088,"kle":23089,"edium":23090,"Ġbranches":23091,"Ġbases":23092,"ç®":23093,"Ġcomplexity":23094,"ĠFig":23095,".reshape":23096,"$db":23097,"_CONST":23098,"ĠTes":23099,".runtime":23100,"Ġdeny":23101,"ĠBSD":23102,"Ġkr":23103,"hatt":23104,"ĠStatic":23105,"Ġuniversities":23106,"Replace":23107,"Ġdrove":23108,"Ġadoles":23109,"_plugin":23110,"ĠLGBT":23111,"Ġtex":23112,"duction":23113,"EDI":23114,"ĠTed":23115,"_URI":23116,"Ġreception":23117,"arten":23118,".Single":23119,"rice":23120,"scious":23121,"_bg":23122,"Ġwages":23123,"ĠServlet":23124,"UILayout":23125,"Ġformatted":23126,".Mod":23127,"',Ċ":23174,"Ġexpanding":23175,"ĠHamilton":23176,"ĠContrib":23177,".Tables":23178,"Activ":23179,"HH":23180,"ocommerce":23181,"_;":23182,"Ġamongst":23183,"owing":23184,"ĠCold":23185,"APH":23186,"Ġpsychological":23187,"_tensor":23188,"Ġpackaging":23189,"ĠSweden":23190,"Ġpare":23191,"Ġaggregate":23192,"Ġmoderate":23193,"_hand":23194,"Ġdesignated":23195,"Ġdrum":23196,"ĠgetUser":23197,"ĠCreek":23198,"_scope":23199,"ĠTransfer":23200,"ĠMarg":23201,"Ġfighters":23202,"Wnd":23203,"ĠSel":23204,"ĠLaunch":23205,"Ġemerging":23206,"iframe":23207,"ĠAdditional":23208,"Ġfears":23209,"Ġsatellite":23210,"_:":23211,"Ġdisposing":23212,"GetValue":23213,"HttpPost":23214,"ATIVE":23215,"ulary":23216,"Views":23217,"Ġattending":23218,"ĠTennessee":23219,"ĠMission":23220,"Ġmedication":23221,"ĠWy":23222,"ĠAnna":23223,"ع":23224,"ĠVertex":23225,".types":23226,"Organ":23227,".DataGridViewTextBoxColumn":23228,"ĠRS":23229,"Ġtempo":23230,"(App":23231,"VersionUID":23232,".point":23233,"ĠDutch":23234,"Hours":23235,"LU":23236,"Ġquoted":23237,".builder":23238,"ĠPerfect":23239,"ĠAlways":23240,"_two":23241,"Ġexclusively":23242,"ĠCra":23243,"ificar":23244,"ĠAWS":23245,"ingham":23246,"complex":23247,"kernel":23248,"Ġgravity":23249,"Ġwi":23250,"Ġoverview":23251,"ĠWant":23252,"ĠWP":23253,"(sh":23254,".rotation":23255,"States":23256,"ĠTeen":23257,"_components":23258,"ìĪĺ":23259,"Received":23260,"Ġlyrics":23261,"rites":23262,"ĉĉĉĉĉĠ":23263,"-American":23264,"[num":23265,"/python":23266,"ĠUART":23267,"Ġapple":23268,"ĠJonathan":23269,"Ġmomentum":23270,"ั":23271,"Ĥ¹":23272,"Ġmich":23273,"andra":23274,"Ġbiological":23275,"ĠMens":23276,"Ġ%%":23277,"elsea":23278,"ĠMexican":23279,".randint":23280,"Ġtale":23281,"ĠValidate":23282,"Ġdefeated":23283,".htm":23284,"Ġcopper":23285,"=/":23286,"cosystem":23287,"Ġrip":23288,"decimal":23289,".VISIBLE":23290,"ĠTa":23291,"ĉĉĉĉĉĉĉĉĉĉĉĉĉĉ":23292,"Ġdownloaded":23293,"environment":23294,"Ġnomine":23295,"building":23296,"ĠSpot":23297,"ipheral":23298,"Ġalto":23299,"quet":23300,"ĠFT":23301,"/get":23302,"/master":23303,"WIN":23304,"åħĥ":23305,"West":23306,"argc":23307,"Ġproducers":23308,"ĠMuch":23309,"_storage":23310,"credit":23311,"CONT":23312,"Ġvet":23313,"Ġvoices":23314,"('',":23315,"Ġinstruments":23316,"ĠMSG":23317,"esse":23318,"repository":23319,"omics":23320,"Ġdealer":23321,"Still":23322,"Ġbanner":23323,"ascii":23324,"Ġremarks":23325,"[js":23326,"Ġshorter":23327,"gulp":23328,"Ġmyster":23329,"Ġkun":23330,"ĠBird":23331,"Ġtiene":23332,"nut":23333,"ĠUm":23334,"Ġwise":23335,"Yeah":23336,"INESS":23337,"_begin":23338,"-heading":23339,"Course":23340,"ĠčĊčĊ":23341,"ombie":23342,"graded":23343,"ĠGPS":23344,"Ġże":23345,"Fit":23346,"caption":23347,"ön":23348,"/image":23349,"lia":23350,"(mod":23351,"Ġleak":23352,"enza":23353,"/H":23354,"ĠHappy":23355,"Dist":23356,"nx":23357,"ĠGovernor":23358,"(last":23359,"teacher":23360,"ĠSent":23361,"support":23362,"jectory":23363,"ĠÙħ":23364,"Registration":23365,"ĠGray":23366,",false":23367,"Ġadjusted":23368,"(settings":23369,"'Ċ":23431,"-fold":23432,"æĬ":23433,"ĠBetter":23434,"Ġ\"\\<":23435,"spacing":23436,"Ġfurnished":23437,"oser":23438,"]}Ċ":23439,"Ġ$\"":23440,"pull":23441,".Post":23442,"(ip":23443,"Ĺı":23444,".front":23445,"nte":23446,"ĠFM":23447,"guid":23448,"Ġnegotiations":23449,"agonal":23450,"Ġtremend":23451,"ungeon":23452,"Adv":23453,"carousel":23454,"ÃŁe":23455,"_DESC":23456,"Ġhammer":23457,"áºŃ":23458,"ĠĠĠĠĠĠĠĠĊĊ":23459,"-core":23460,"-service":23461,"Ġcorners":23462,"ĠSF":23463,"pred":23464,">A":23465,"ĠJLabel":23466,"Ġromantic":23467,"Ġtestimony":23468,"osc":23469,"ĠGeneration":23470,"asures":23471,"_internal":23472,"Ġprints":23473,"Ġ])Ċ":23474,"ĠCleveland":23475,"repo":23476,"Disc":23477,"Ġ\">Ċ":23478,"����":23479,"Ġnearest":23480,"_tb":23481,"(require":23482,"EOF":23483,"-child":23484,"Ġbudd":23485,".XtraEditors":23486,"alties":23487,"\\\":\\\"":23488,"Words":23489,"Ġlocally":23490,"Ġpurchases":23491,"Drawer":23492,"extract":23493,"Ġexecut":23494,"}'.":23495,"userdata":23496,"Ġfocuses":23497,"-minute":23498,"ĠPublish":23499,"ogo":23500,"Ġmountains":23501,"Bot":23502,"}>{":23503,"Ġtension":23504,"rod":23505,"mesh":23506,"Ġtransformed":23507,",R":23508,"()}Ċ":23509,".long":23510,"Ġgorgeous":23511,"ĠSchedule":23512,"Ġoldest":23513,"Ġsubprocess":23514,"(IN":23515,"yect":23516,"ĠCooper":23517,"arness":23518,"ĠMonitor":23519,".part":23520,"ĠNBC":23521,"Ġcotton":23522,"Ġhol":23523,"Ġrgba":23524,"ĠBio":23525,"Continue":23526,"Pod":23527,"Ġparticipating":23528,"clusions":23529,"(ByVal":23530,"ì":23531,"ĠHOW":23532,"_setopt":23533,"Ġaccompanying":23534,"aton":23535,"Ġ/\\":23536,"ĠAuthentication":23537,"ién":23538,"ĠBarack":23539,"/*.":23540,"Ġeager":23541,"ĠCancel":23542,"$":23586,"OLEAN":23587,"OKIE":23588,"IBILITY":23589,"UAGE":23590,"ĠSurvey":23591,"Ġresign":23592,"wing":23593,"Ġsecrets":23594,"Ġchips":23595,"JSONObject":23596,"Desktop":23597,"_SYMBOL":23598,"(resource":23599,"ĠĊ":23600,"Ġnewest":23601,"uli":23602,"Ġdesert":23603,"Ġdip":23604,"ĠPow":23605,"Ġequation":23606,"Ġpossibilities":23607,"ĠFed":23608,"osph":23609,"Ġ[%":23610,"Ġbubble":23611,"etherlands":23612,"Ġcement":23613,".auto":23614,"_AN":23615,"âĢĻ.":23616,"selection":23617,"ĠBond":23618,"Den":23619,"-O":23620,".getType":23621,".Window":23622,"pres":23623,"Ġswinger":23624,"\"})Ċ":23625,"Ġpip":23626,"Ġmice":23627,"Ġcompound":23628,"-plugin":23629,"iko":23630,"Ġcenturies":23631,"icular":23632,"-inline":23633,"ĉkey":23634,">\\<":23635,"ENSION":23636,"Ġ[čĊ":23637,"Ġprecisely":23638,"Ġété":23639,"ĠPast":23640,"ĠCambridge":23641,"-full":23642,"Ġanalyze":23643,"ĠSteven":23644,"Ġnem":23645,"due":23646,"oren":23647,"Ġmuscles":23648,"ijing":23649,"/-":23650,"ĠKennedy":23651,"RM":23652,"ossible":23653,"Ġactress":23654,"Ġdolor":23655,"å½ķ":23656,"Need":23657,".toggle":23658,"ĠRace":23659,"wers":23660,".material":23661,"ĠDue":23662,"ĠPel":23663,"#print":23664,"Ġindependence":23665,"exus":23666,"Shadow":23667,"Ġencoder":23668,"(level":23669,"ĠSwift":23670,".doc":23671,"_selection":23672,"ĠserialVersionUID":23673,"Labels":23674,"Ġperformances":23675,".Tag":23676,"ĠNHL":23677,"izen":23678,"/UIKit":23679,"_CONTROL":23680,"Ġearnings":23681,"ĠAlt":23682,"_HANDLE":23683,"Ctx":23684,"Ġpersu":23685,"Ġtran":23686,"ç¨":23687,"_CHANNEL":23688,"Ġsatisfaction":23689,"ĠGP":23690,"iox":23691,"mitt":23692,"lando":23693,"Ġpig":23694,"inals":23695,"ência":23696,"Surface":23697,"ĠUUID":23698,"Ġbeneficial":23699,"Ġsequences":23700,"ĉmemset":23701,"Ġmagical":23702,"«":23703,"Ġworn":23704,"ASC":23705,"popup":23706,"COMP":23707,"_before":23708,"eness":23709,"Ui":23710,"Les":23711,".require":23712,".Serializable":23713,"addGap":23714,"Ġauthorization":23715,".pyplot":23716,"urray":23717,"latitude":23718,"frames":23719,"ajs":23720,"Ġcompass":23721,"Ġobservations":23722,"_sup":23723,".environ":23724,"Ġtriple":23725,"ĠRuby":23726,"Ġdrain":23727,"_FILTER":23728,"San":23729,"UMP":23730,"NullException":23731,"ĠGab":23732,"owe":23733,"ĠTurkish":23734,"_sequence":23735,"ĠGrant":23736,"uela":23737,"Ġwo":23738,"Ġcube":23739,"iq":23740,"Ġdisorders":23741,"Ġextraordinary":23742,"Ġctrl":23743,"ĠSeq":23744,"entr":23745,"Ġsanctions":23746,"utsch":23747,"Reports":23748,"Ġinherit":23749,"Period":23750,"Ġphotography":23751,"ĠFramework":23752,"Ġspecialist":23753,"Ġ?ĊĊ":23754,"_selected":23755,".Player":23756,"Ġallocation":23757,"(account":23758,"Ġstructural":23759,"vable":23760,"-offset":23761,".AppCompatActivity":23762,"ам":23763,".AddWithValue":23764,"Ġicons":23765,"Ġshutdown":23766,"_low":23767,"ĠCompare":23768,"ĠCe":23769,"=head":23770,"lam":23771,".predict":23772,"_DEC":23773,"ĠSleep":23774,"ĠGratis":23775,"Ġsuggestion":23776,"ĠDEL":23777,"caff":23778,"avirus":23779,"Nothing":23780,"ŀĭ":23781,"Ġwidespread":23782,"Ġmechanisms":23783,"ĠtextAlign":23784,"occup":23785,"ĠRail":23786,":NS":23787,"Ġfiber":23788,"Ġmk":23789,"Ġvintage":23790,"-long":23791,".reduce":23792,".Entities":23793,"(record":23794,"Ġpleasant":23795,"FRING":23796,".Cells":23797,"OTT":23798,"ĉelseif":23799,"_confirm":23800,"ĠViewGroup":23801,"sym":23802,"Ġpray":23803,"Ġsuspected":23804,"Contains":23805,"Ġborders":23806,"ĠcomponentDid":23807,"ASSERT":23808,"Ġinfinite":23809,"-order":23810,"Ġhello":23811,"ĠGrade":23812,".currentTimeMillis":23813,"apolis":23814,"zh":23815,"ĉObject":23816,":\\\\":23817,"HO":23818,"valuation":23819,"Ġvocab":23820,"Ġcoupon":23821,"atabases":23822,".GetType":23823,"Learn":23824,"]=\"":23825,"ĠGary":23826,"otive":23827,"Ġash":23828,"Ġbib":23829,"XXXX":23830,"Ġbalanced":23831,"VALUE":23832,"ĠNat":23833,"_Ad":23834,"<":23976,"Ġfool":23977,"Ġesk":23978,".Null":23979,"ĠDies":23980,"_OUTPUT":23981,"_TYPED":23982,"Ġpainted":23983,"Ġsophistic":23984,"ĠBear":23985,"*n":23986,"_PACK":23987,"Ġdelivering":23988,"ĠCOUNT":23989,"åįķ":23990,"Ġjeg":23991,"-car":23992,"fname":23993,"Ġranging":23994,"ĠNeg":23995,"/******/":23996,"ĠCHAR":23997,"Ġultra":23998,"Grad":23999,"=t":24000,"Ġjudges":24001,"ĠDise":24002,"anners":24003,"Ġscal":24004,"_cal":24005,"ĠCONNECTION":24006,"_embed":24007,"(fn":24008,"ĠCraft":24009,"ĠPas":24010,"\")->":24011,".convert":24012,".resource":24013,"ĠSTATUS":24014,"ông":24015,"ĠTit":24016,"Ġclassroom":24017,"ĠArchitect":24018,"ĠKings":24019,"Ġsteady":24020,"/*!Ċ":24021,"ĠGene":24022,")\";Ċ":24023,"icia":24024,"stan":24025,"ĠConstruction":24026,"umper":24027,"wc":24028,"ĠCBS":24029,"inging":24030,"-party":24031,"(driver":24032,"MARK":24033,"Ġnested":24034,"eward":24035,"Ġdependency":24036,"Ġmales":24037,"ĠONE":24038,"ĠProduction":24039,"][$":24040,"ãĥ¼ãĥ":24041,"_LOAD":24042,"ĠBol":24043,"elry":24044,"łéϤ":24045,"ĠRequire":24046,"Ġplacing":24047,"xxx":24048,"CALE":24049,"Ġthumb":24050,"Choose":24051,"Ġprototype":24052,"VOID":24053,"Ġlesbian":24054,"Ġtraits":24055,"Sharp":24056,"Ġconsume":24057,"Truth":24058,"ĠactionPerformed":24059,"ĠEnvironmental":24060,"ĠDean":24061,"Ġestado":24062,"same":24063,"Ġnumeric":24064,"Ġtransit":24065,".Email":24066,"-side":24067,"_RUN":24068,"ĠVillage":24069,"_OPEN":24070,"è¦":24071,".rem":24072,"-warning":24073,"anya":24074,"PropertyChanged":24075,"Ġ(!_":24076,"(check":24077,"ilia":24078,"ĠSoft":24079,"steps":24080,"ĠMadrid":24081,"MemoryWarning":24082,"Ġhandlers":24083,"Ġexperiencing":24084,"Ġinspect":24085,"buttons":24086,"ReceiveMemoryWarning":24087,"chemy":24088,"Links":24089,"Ġurllib":24090,".SystemColors":24091,"ĠEigen":24092,"Ġpunishment":24093,":UIControl":24094,"bara":24095,"-set":24096,"Ġ}čĊčĊčĊ":24097,"Ġtolerance":24098,"Ġinterfaces":24099,".redirect":24100,"ighbors":24101,"csrf":24102,"_background":24103,".Utils":24104,"_HT":24105,"ĠInterest":24106,"imos":24107,"Ġgrants":24108,"Ġexamined":24109,"ÐĶ":24110,"Ġcf":24111,"forge":24112,"backs":24113,"ĠObjects":24114,"_sent":24115,".entry":24116,"ĠTHEN":24117,"ellido":24118,"cia":24119,",res":24120,"/stdc":24121,".nd":24122,"(Int":24123,"ĠAuthors":24124,"ĠAppCompatActivity":24125,"'{":24126,"Ġmedi":24127,"Music":24128,"igm":24129,"ceipt":24130,"Ġauss":24131,"Ġtargeting":24132,"ĠKeys":24133,"hn":24134,":]Ċ":24135,"Ġmineral":24136,"î":24137,".ca":24138,"omed":24139,"Ġsheets":24140,"Ġcamb":24141,"Ġdeadly":24142,".inject":24143,"(unit":24144,"ĠSelection":24145,".gms":24146,"(connection":24147,"Ġ$(\"":24148,"émon":24149,"ĠCurrently":24150,"pte":24151,"_paths":24152,"leaf":24153,"Ġimplications":24154,"posal":24155,"ä½į":24156,"[/":24157,"ancia":24158,"éĽ":24159,"mul":24160,"cie":24161,"Ġgeile":24162,"imals":24163,"UIView":24164,"Ġsurre":24165,"serialize":24166,"ISO":24167,"Ġarbitrary":24168,"Ġsockaddr":24169,".fn":24170,"ĠMerc":24171,"Ġcasting":24172,"KeyDown":24173,"ĠnewValue":24174,"opens":24175,"Todo":24176,"Ġflexibility":24177,"ĉĉĉĉĠĠ":24178,"Velocity":24179,"ún":24180,"rowing":24181,"Ġcomputed":24182,"`)Ċ":24183,"statement":24184,"Ġri":24185,"_cart":24186,"Low":24187,"transfer":24188,".nav":24189,"Ġgrave":24190,"ĠDoor":24191,"ĉalert":24192,".subscribe":24193,"-profile":24194,"ĉbase":24195,"ĠâĪĴ":24196,"__ĊĊ":24197,"Ġengineers":24198,"Ġexplosion":24199,"Ġdari":24200,"ĉLog":24201,"onal":24202,"Ġisolated":24203,"{i":24204,"ĠMsg":24205,"Future":24206,"Ġracist":24207,"-wrap":24208,"ĠVers":24209,"borg":24210,"ISION":24211,"ĠÑĢаÐ":24212,"ĠYan":24213,"initWith":24214,"Ġnomin":24215,"(empty":24216,"ÃŃn":24217,"ãĤ¤":24218,"ĉwidth":24219,"Ġchamber":24220,"/ajax":24221,"EMP":24222,"Ġneces":24223,"ivos":24224,"logic":24225,"*)&":24226,"cripts":24227,"RowAt":24228,"iblings":24229,"Ġears":24230,"Ġcomputing":24231,"Ġmaker":24232,"ĠNeither":24233,"breadcrumb":24234,"Ġserialize":24235,"ĠWithin":24236,"Ġdell":24237,"_TRACE":24238,"=a":24239,"Ġwishes":24240,"-inch":24241,"ĠDor":24242,"Ġinnocent":24243,"ĠDol":24244,"Ġintens":24245,"forced":24246,"ĠBIT":24247,"Ġphotographs":24248,"Ġcasa":24249,"ĠLen":24250,"\\Framework":24251,".Simple":24252,"Ġdear":24253,")/(":24254,"ippi":24255,"Ġowns":24256,"Players":24257,"Ġproposals":24258,".pi":24259,"usalem":24260,"Damage":24261,"Ġcalories":24262,"ĠCreative":24263,"Ġ[$":24264,"Ġ//čĊ":24265,"AndView":24266,"ème":24267,".custom":24268,"_factory":24269,"commands":24270,"_look":24271,"Ġstrcmp":24272,"YN":24273,"aired":24274,"Ġaudit":24275,"оÑģÑĤ":24276,"ĠReverse":24277,"ropriate":24278,"etics":24279,"';Ċ":24352,"Ġpepper":24353,"Ġshed":24354,"ĠMedium":24355,"ĠCookie":24356,"Ġoverseas":24357,"edor":24358,"asurement":24359,"åŃĺ":24360,"Ġ'.'":24361,"Ġphp":24362,"ĠPROC":24363,"Ġexceptional":24364,"(th":24365,"ĠJet":24366,"Ġoccupied":24367,".setImage":24368,"ĠRelated":24369,"ucker":24370,"Members":24371,"PRINT":24372,"ĠGlo":24373,"_VIEW":24374,"}\",Ċ":24375,"Ġadoption":24376,"[])Ċ":24377,"ĠMissouri":24378,"ĠLincoln":24379,"erald":24380,"Popup":24381,"Ġfate":24382,"-bootstrap":24383,"fections":24384,"ĠPoll":24385,"_ARGS":24386,"inance":24387,"-home":24388,".),":24389,"_done":24390,":ĊĊĊ":24391,"Ġdiscussing":24392,"ĠSQLException":24393,"Ġelectro":24394,"ĉreq":24395,"Ġzw":24396,"Ġlui":24397,"Ġovernight":24398,"$user":24399,"ĠWAY":24400,"Ġallerg":24401,"Ġdisappointed":24402,"Ġradiation":24403,"Ġimpressed":24404,"ificates":24405,"Ġtob":24406,"CLASS":24407,"Ġcuda":24408,"_det":24409,"-post":24410,"ulu":24411,"Translation":24412,"-hand":24413,".year":24414,"ĠMongo":24415,"Ġunclear":24416,".engine":24417,"WEBPACK":24418,"rices":24419,"_ACCESS":24420,"Ġholidays":24421,"percent":24422,".Identity":24423,"ĠGov":24424,"Ġpassionate":24425,"!!.":24426,"ĠGreece":24427,"plusplus":24428,"'));":24429,"GP":24430,"Ġexcit":24431,".tabPage":24432,"_cond":24433,"Ġsponsor":24434,"MODULE":24435,"_proc":24436,"Ġ$Ċ":24437,"Ġrational":24438,".Tool":24439,"Ġihr":24440,"cca":24441,"åĵģ":24442,"ĠEstate":24443,"IBUTE":24444,"ActionPerformed":24445,"ĠSolar":24446,"¦Ĥ":24447,"Ġequity":24448,"tid":24449,"Ġrecip":24450,".simple":24451,"mk":24452,"ĠLuke":24453,"ĠGuardian":24454,"Ġencrypted":24455,"Ġdominant":24456,".place":24457,"ĠNV":24458,"Ġtongue":24459,"(Get":24460,"Ġstainless":24461,".Play":24462,"Ġeb":24463,"aci":24464,".buffer":24465,"readcrumbs":24466,"Ġvaccine":24467,"prom":24468,"ĠuserInfo":24469,"Ġslug":24470,"SerializedName":24471,"-wide":24472,"Ġreactions":24473,"ĠYang":24474,"ĠAdds":24475,"(userId":24476,"Ġplates":24477,"ĠMEM":24478,"Ġbail":24479,"Inside":24480,"eted":24481,"Ġelsif":24482,"Ġsake":24483,"Ġcycles":24484,"ĠìĹ":24485,"ĉI":24486,"-collapse":24487,"ĠGMT":24488,"Declaration":24489,"Ġgros":24490,"Ġreaches":24491,"Ġcustody":24492,"Until":24493,"tu":24494,"ĠChen":24495,"Ġnx":24496,"(addr":24497,"ĠOffer":24498,"Ġcolleg":24499,"assador":24500,"Ġmapper":24501,"ĠSIGNAL":24502,"ĠBloom":24503,"ĠHoll":24504,"ĠImper":24505,"-des":24506,"_site":24507,"Proc":24508,"Equ":24509,"Ġatomic":24510,"ĠWoman":24511,"sent":24512,"scar":24513,"Ġintelligent":24514,"ĠGetting":24515,"ĠRegistration":24516,"ĠPhill":24517,"Ġkiller":24518,"unicode":24519,"ĊĉĉĊ":24520,"ĠJacob":24521,"ĠConst":24522,"Ġlocate":24523,"Ġcaus":24524,"ĠScholar":24525,"Ġconstitutional":24526,"Ġinflation":24527,"ĠGot":24528,"=array":24529,"endum":24530,"Ġtranslated":24531,"Ġdivorce":24532,"Entries":24533,"Ġsor":24534,"ĠQuote":24535,"irlines":24536,"UK":24537,"Ġexcel":24538,"(opt":24539,"ĠADV":24540,",:,":24541,"Ġcontacted":24542,"ĠDA":24543,"Ġrings":24544,"ĠIndustrial":24545,".getContext":24546,"Ġforgotten":24547,"ĠTan":24548,"Ġpants":24549,"Ġov":24550,"Ġdecoder":24551,"ĠPartial":24552,"Ġvc":24553,"Ġbattles":24554,"Arial":24555,"FRINGEMENT":24556,"irates":24557,",w":24558,"aintenance":24559,"ĠOd":24560,"ĠTechnologies":24561,"åīį":24562,"ĠCarter":24563,".findAll":24564,"Nome":24565,"Ben":24566,"ĠUsage":24567,"ĠPicture":24568,"Ġbadly":24569,"_panel":24570,"Ġpatent":24571,"ĠProtocol":24572,"lotte":24573,"ĉplayer":24574,"jections":24575,"Ġdou":24576,"_release":24577,"urniture":24578,"_tax":24579,"ĠFields":24580,".dataset":24581,"_master":24582,"CLUDE":24583,"ĠPharm":24584,"bst":24585,"Ġoperational":24586,".cell":24587,"Ġidentifying":24588,"Ġjwt":24589,"tuple":24590,"ĠTC":24591,"ĠCro":24592,"ixmap":24593,"-components":24594,"general":24595,"Ġoz":24596,"_De":24597,"_double":24598,"ĠToo":24599,".ViewGroup":24600,"gate":24601,"dings":24602,"photos":24603,"Ġgrande":24604,"ollect":24605,"_lin":24606,"Ġawful":24607,"filters":24608,"Ġalternate":24609,"esp":24610,"Ġcompress":24611,"eo":24612,"ĠScale":24613,"Ġindirect":24614,"Ġinvoice":24615,"ĊĊĊĊĊĊĊĊĊĊĊĊĊĊĊĊ":24616,"Starting":24617,"ĠPlayers":24618,"iele":24619,".then":24620,"Ord":24621,"ĠTuple":24622,"Ġbout":24623,"ĠStatistics":24624,"Preview":24625,"Ġpuzzle":24626,"ĠWidth":24627,"STATE":24628,"Ġoverlay":24629,"ĉon":24630,"Ġinfr":24631,"Ġsmallest":24632,"locked":24633,"ÑĤо":24634,"ssl":24635,"Ġdeemed":24636,"Ġsco":24637,"reck":24638,"ĠjButton":24639,"Ġmissions":24640,"ç§°":24641,".SelectedIndex":24642,"TABLE":24643,"Sept":24644,"Ġacknowledge":24645,"Ġstrtotime":24646,"ĠTell":24647,"ĠDak":24648,"Ġaluminum":24649,"Ġfence":24650,"ĠStars":24651,"CONFIG":24652,"Ġretrofit":24653,"Ġemphasis":24654,"/header":24655,"ĠSomething":24656,"inished":24657,"='\".$":24658,"ĠValidators":24659,"Ġpolar":24660,"sections":24661,".aspx":24662,"Ġaspir":24663,".Mock":24664,"CodeGen":24665,"Ġpeut":24666,"Ġaccepting":24667,"Ġbacking":24668,"Picture":24669,"/ap":24670,"ег":24671,"_SEC":24672,"-use":24673,"annotation":24674,"Ġcognitive":24675,"Ġgrip":24676,"hour":24677,"ĠLegal":24678,"Ġepic":24679,".toolStrip":24680,".notify":24681,".Last":24682,"ORIZ":24683,"Middleware":24684,"criptions":24685,"lash":24686,"_FOUND":24687,"ĠLiverpool":24688,"Ġ{}\",":24689,"Install":24690,"Ġnit":24691,"Ġfigured":24692,"[len":24693,".Win":24694,".platform":24695,"Ġgambling":24696,"(dt":24697,"avery":24698,"ĉinclude":24699,"Whether":24700,"Routing":24701,"Ġtherap":24702,"Remote":24703,"ĠLoss":24704,"yll":24705,"Ġapproached":24706,"ĠVehicle":24707,"ĠAlpha":24708,"Ġvocê":24709,"answers":24710,"NSDictionary":24711,"consider":24712,"unused":24713,"ĠFan":24714,"orable":24715,"fre":24716,"ĠDISCLAIM":24717,"ĠActor":24718,".]":24719,"toHave":24720,".userId":24721,"Ġspeeds":24722,"eway":24723,"Ġrecurs":24724,"Ġг":24725,"_priv":24726,"!âĢĿĊĊ":24727,"Choice":24728,"Ġsettle":24729,"Ġplanes":24730,"'},":24731,"Tom":24732,"ITER":24733,"!\"Ċ":24734,"å»":24735,"achelor":24736,"Ġseparation":24737,"Ġdal":24738,"adj":24739,"Ġregisters":24740,"riz":24741,"ĠNotice":24742,"Ġlu":24743,"Ġcourage":24744,"Ġaxes":24745,"cellent":24746,".async":24747,"Ġcompatibility":24748,"ç«":24749,"Ġ!ĊĊ":24750,"ĉtitle":24751,"YLE":24752,"ĉmessage":24753,"UUID":24754,"OLDER":24755,"ĠHH":24756,"ĠStyleSheet":24757,"Ġaccessed":24758,".validation":24759,"tasks":24760,"Ġpollution":24761,".canvas":24762,"Ġingredient":24763,"ĠCabin":24764,"Ah":24765,"oldown":24766,"ĠNOI":24767,"ĠÃĹ":24768,"[f":24769,"educ":24770,"yalty":24771,"(not":24772,"_State":24773,"amen":24774,"Ġdao":24775,"udad":24776,"ellers":24777,"}&":24778,"licity":24779,"_WINDOW":24780,"Ġtatto":24781,"valor":24782,".Range":24783,"Ġreferenced":24784,"ĠReserve":24785,"Money":24786,"SCRIPT":24787,"/product":24788,"choices":24789,"Ġtin":24790,"ãĤĵ":24791,"Ġseparator":24792,"Ġpkg":24793,"ammed":24794,"ĠMAT":24795,"!!ĊĊ":24796,"Ġraid":24797,"Ġmotivation":24798,"ĠXP":24799,"ĠBackground":24800,"ĠQuaternion":24801,".defineProperty":24802,"iker":24803,"ĉparent":24804,"ĠOriginally":24805,"antage":24806,"ĠHans":24807,"Ġtimeline":24808,".cur":24809,"opic":24810,"ĠSequ":24811,"must":24812,"ĠCoal":24813,"Ġformatter":24814,"_RGB":24815,"Ġ_(\"":24816,"'}),Ċ":24817,"Ġ=================":24818,"ĠFUNCTION":24819,"Ġlng":24820,"icates":24821,"live":24822,"_engine":24823,"Ġtowns":24824,"'))ĊĊ":24825,"ĠPK":24826,"(api":24827,"ĉscanf":24828,"packet":24829,".phone":24830,"áĢ":24831,"ĠAndy":24832,"_NAMES":24833,"PLY":24834,"Ġmins":24835,"imi":24836,"Ġbrick":24837,"Ġblade":24838,".stdout":24839,"}`;Ċ":24840,"Shift":24841,"ĉsb":24842,"ĠChecks":24843,"Ġphenomenon":24844,"Avatar":24845,"Ġministry":24846,"rose":24847,"ĉFile":24848,"Ġtitled":24849,"(LOG":24850,"Ġgan":24851,"design":24852,"(),čĊ":24853,"Ġbones":24854,"stm":24855,"ÅĽÄĩ":24856,"ĠInputStream":24857,"Ġvolunt":24858,"ĠSerializable":24859,"Ġfighter":24860,"ĠDrag":24861,"Twitter":24862,"Ġsubsid":24863,"ç¼":24864,"Ġforums":24865,".loading":24866,"logged":24867,"_this":24868,"Ġterrain":24869,"Ġirre":24870,"ĠIng":24871,"ĠCN":24872,"_objects":24873,".uid":24874,"Ġconsciousness":24875,"TINGS":24876,"ĠGall":24877,"Ġportray":24878,"ĠDeveloper":24879,"Ġparticipant":24880,"Ġ\";čĊ":24881,"/model":24882,"ĠOperations":24883,"^\\":24884,"ĠLater":24885,"Ġraises":24886,"-none":24887,".meta":24888,"='.$":24889,"Finished":24890,"Ġreplacing":24891,"Ġsampling":24892,"ĠJen":24893,"\"There":24894,"REAL":24895,"ALE":24896,"ìĬ¤":24897,"Orders":24898,"_parameter":24899,"ĠOlympic":24900,"Ġtrès":24901,"Ġarena":24902,"iol":24903,";?>":24904,"Ġimpacts":24905,"ĠWS":24906,":get":24907,"Ġflights":24908,"ĠRussell":24909,"camera":24910,"Fn":24911,"sigma":24912,"Ġforcing":24913,"Ġlocals":24914,"Ġdeparture":24915,"Ġcelebration":24916,"ĠSay":24917,"ï¼Ĵ":24918,"ĠHills":24919,".hasOwnProperty":24920,"Ġtypings":24921,".API":24922,"Ġdonation":24923,"OperationException":24924,".Activity":24925,"cplusplus":24926,"ĠCharlie":24927,"Ġimported":24928,"Ġdann":24929,"Ġoccasions":24930,"Ġimplementing":24931,"Ġpurple":24932,".dialog":24933,"SQLException":24934,"erno":24935,"Ġwars":24936,"Ġpaste":24937,"Ġdecreased":24938,"Ġharsh":24939,"Ġelabor":24940,"inputs":24941,"ĠViews":24942,"ĠerrorMessage":24943,"_mul":24944,"ĉwrite":24945,"ĠCop":24946,"ĠAnnual":24947,"(button":24948,"Ġvida":24949,"bars":24950,"ĠHarvard":24951,"ĉexpect":24952,"Ġindexes":24953,"Ġdocumentary":24954,"Ġflesh":24955,"ORLD":24956,"ĠDelta":24957,"MAND":24958,"Brush":24959,"-column":24960,"Ġdevelopments":24961,"methodVisitor":24962,"slice":24963,"ĠPDO":24964,"Ġinvesting":24965,"irable":24966,"Ġxmlns":24967,"ï¼Ľ":24968,"arta":24969,"Ġtheories":24970,"_city":24971,"Ġ$__":24972,"Creating":24973,"(pr":24974,"Dropdown":24975,"ismatch":24976,"ĠNET":24977,"'])){Ċ":24978,"ĠValues":24979,"ĠSEO":24980,"ĠSTAT":24981,"Ġecosystem":24982,"Ġtempt":24983,"Ġ\\\\":24984,"Ġ//{Ċ":24985,"ĠChristopher":24986,"ĠKentucky":24987,"ĠHttpServletResponse":24988,"Ġhybrid":24989,"yon":24990,"Ġfeeding":24991,"ĠExtra":24992,"Norm":24993,"ITCH":24994,"ĠSean":24995,"ĠUpload":24996,"mun":24997,"pur":24998,"Ġpersistent":24999,"ĠIDC":25000,"ĠPerform":25001,".merge":25002,"_room":25003,"Meanwhile":25004,"!='":25005,"ĠWel":25006,"ArgsConstructor":25007,".Database":25008,"Ġcounting":25009,"()*":25010,"ĶåĽŀ":25011,"ĠTOP":25012,"mill":25013,"ĠDT":25014,"IGNED":25015,"ĠKB":25016,"Ġcomply":25017,"South":25018,"_collection":25019,"Chapter":25020,"Ġexplaining":25021,"_AM":25022,"_ts":25023,"cards":25024,"Ġquel":25025,"Ġpole":25026,"Ġtouchdown":25027,"ĠOthers":25028,"Ġpeers":25029,"ĠTypeError":25030,"Ġsixth":25031,"Ġcheer":25032,"Ġdispute":25033,"usc":25034,")],":25035,"thumb":25036,"Ġhiding":25037,"ĠSIG":25038,"likes":25039,"ĠPAGE":25040,".Reflection":25041,"Ġheadquarters":25042,"TING":25043,"ĠGhost":25044,"MLE":25045,"$Ċ":25046,"Ġcontrary":25047,"extend":25048,"']).":25049,"FFECT":25050,"ĠPinterest":25051,"úmero":25052,"ricane":25053,"ĉsession":25054,"Ġcrystal":25055,"-Control":25056,"overnment":25057,"ograf":25058,"-action":25059,"volume":25060,"ften":25061,"Ġuncon":25062,"Ġanimate":25063,"Ġlease":25064,"scr":25065,"Ġrefuse":25066,"ãĢĭ":25067,"ftp":25068,"information":25069,"Ġevaluated":25070,"Ġinjection":25071,"Ġjack":25072,"Ġworkshop":25073,"注":25074,"PTH":25075,"ĠTs":25076,"offer":25077,"ĉos":25078,"Ġkingdom":25079,"Missing":25080,"Ġlawmakers":25081,"extField":25082,"Ġsinging":25083,"abi":25084,"/client":25085,".media":25086,"ATEGORY":25087,"Signature":25088,"%',Ċ":25089,"ĠFuck":25090,"][:":25091,"Ġsensors":25092,"/com":25093,"ĠPrimary":25094,".SQL":25095,"_program":25096,"Ġpills":25097,"Ġintegral":25098,"Ġfleet":25099,"Ġdropping":25100,".sl":25101,"Been":25102,"Ġpets":25103,"Ġadvised":25104,"Ġdragon":25105,"_EDIT":25106,"(im":25107,"FER":25108,"ĠDrug":25109,"(random":25110,"Ġcompression":25111,"oust":25112,"[%":25113,"Ġbuyer":25114,"hop":25115,"Roles":25116,"manage":25117,"Ġpainful":25118,"ĠBranch":25119,"-modal":25120,"enant":25121,"ĠMesh":25122,"/font":25123,"ĠGraham":25124,"Ġâĺ":25125,"Ġnc":25126,"ĠFrancis":25127,"Ġspecification":25128,"Ġdamages":25129,"-config":25130,"Ġtheoret":25131,"secure":25132,"_multi":25133,"aceutical":25134,"Ġdemanding":25135,"enne":25136,"ISTS":25137,"()));ĊĊ":25138,"Reason":25139,"Recent":25140,"phase":25141,"Ġpsy":25142,"_MAN":25143,"Ġvolunteer":25144,"å¿":25145,"istributed":25146,"lio":25147,"Ġproductivity":25148,"_comm":25149,"Spring":25150,"nis":25151,".weight":25152,"ĠCancer":25153,"Alloc":25154,"ĠTweet":25155,"Ġseparately":25156,"ĉcheck":25157,"_properties":25158,".Unit":25159,"_CLK":25160,"Ġgt":25161,"Ġ();ĊĊ":25162,"Ġhandy":25163,"ĠThompson":25164,"Ġunnecessary":25165,"ĠReader":25166,"GN":25167,"=request":25168,"ĠUtility":25169,".Repository":25170,"ĠAx":25171,"hydr":25172,"ieu":25173,"Ġthy":25174,"Ġlt":25175,"_mail":25176,"ä¿®æĶ¹":25177,"ailand":25178,"ĠPhilip":25179,"Ġbitter":25180,"Ġbetting":25181,"Ġtimed":25182,"ocks":25183,"'a":25184,"Ġalgorithms":25185,"Ġreinterpret":25186,"Ġtoss":25187,"rogen":25188,"Ġhoped":25189,"(selected":25190,"Ġventure":25191,"TEX":25192,"ĠLeave":25193,".Substring":25194,"Ġgrateful":25195,"uka":25196,"ĠConsumer":25197,"Ġaggreg":25198,"Circle":25199,"à¸ģ":25200,"_blocks":25201,"Ġlegally":25202,"Ġ\"|":25203,"ãĥĥ":25204,".board":25205,".Ab":25206,"Functions":25207,"recipe":25208,"èĩ":25209,"ĠOxford":25210,"Ġwholes":25211,".Build":25212,"_changed":25213,"hai":25214,"Ġdepartments":25215,"Imp":25216,"Ġcoalition":25217,"INFRINGEMENT":25218,"Ġempower":25219,"itches":25220,"North":25221,"Ġinflamm":25222,"ONSE":25223,"Ġmissile":25224,"ĠRaj":25225,"ĠIssue":25226,"Ġatoi":25227,"caled":25228,".Controllers":25229,"ĠWolf":25230,"Ġcrushers":25231,"á»ĩ":25232,".Auth":25233,".addAttribute":25234,"his":25235,"Ġboots":25236,".clean":25237,"camp":25238,"Ġtenant":25239,"Ġtune":25240,"Ġ{}'.":25241,"Ġworkout":25242,"Repo":25243,"Ġpartially":25244,"MISSION":25245,"jamin":25246,"ĠSB":25247,"Ġdetermination":25248,"Ġ'');Ċ":25249,"ĠBeng":25250,"Ġvos":25251,"Ġinhab":25252,"/lang":25253,"sburgh":25254,"Executor":25255,"hone":25256,"ĠChallenge":25257,"_links":25258,".Level":25259,"Ġunderground":25260,"-code":25261,"Ġoptimization":25262,"logging":25263,"_dest":25264,"Ġsnake":25265,"Ġchemicals":25266,"_IMPORTED":25267,"adoop":25268,"ĠTHAT":25269,"managed":25270,"Ġreduces":25271,"ĠREAL":25272,"ĠGuy":25273,"_GENERIC":25274,"/********************************":25275,".amount":25276,"Ġdere":25277,"getTime":25278,"Ġpant":25279,"anonymous":25280,"Ġharmony":25281,"ĠAlan":25282,"Ġscenarios":25283,"Ġdirt":25284,"htags":25285,"Mc":25286,"Shell":25287,"rin":25288,"{čĊčĊ":25289,".pow":25290,"ĉclient":25291,"Ġconspiracy":25292,"Ġadmission":25293,"ĠRegional":25294,"ĠViewController":25295,"ĠPhilippines":25296,"Ġdepos":25297,"Ġpap":25298,"ĠPad":25299,"Paul":25300,".ComboBox":25301,"Ġtutor":25302,"ĠRecipe":25303,"writing":25304,"Ġcontributor":25305,"OTH":25306,"Small":25307,"VI":25308,"Ġhacer":25309,"equ":25310,"ĠExamples":25311,"human":25312,".messages":25313,"ĉtyp":25314,"Ġ(čĊ":25315,"ĠSSL":25316,"LEN":25317,"ĠRomney":25318,"(grid":25319,"ĉmin":25320,"Ġ>ĊĊ":25321,"Ġfruits":25322,"Ġvoter":25323,"Inline":25324,"pane":25325,"ĠCollections":25326,"charset":25327,"Ġspam":25328,"zb":25329,"itemap":25330,"Ġsucceeded":25331,"_COL":25332,"Ġelapsed":25333,"imeter":25334,"Ġrecovered":25335,"Tensor":25336,"hattan":25337,".setup":25338,"isto":25339,"(head":25340,"ĠSIZE":25341,"Ġtactics":25342,"Ġdistur":25343,"Ġpreval":25344,"icios":25345,"(Value":25346,"_cols":25347,"ĠFat":25348,"Ġseal":25349,"Ġsons":25350,"Ġensures":25351,"Ġpressing":25352,"=&":25353,"igenous":25354,"Ġharassment":25355,"_JSON":25356,"Ġignor":25357,"ynomial":25358,"omer":25359,"_static":25360,"Ġsignificance":25361,"Ġcircles":25362,"_System":25363,"Ġdiscipline":25364,"Ġdressed":25365,"Ġsphere":25366,"Ġclimb":25367,"_actions":25368,"ĠBab":25369,"Ġ'=',":25370,"_schema":25371,"\"use":25372,"Ġunders":25373,"Ġcups":25374,".screen":25375,"/new":25376,"Ġappearing":25377,"TOP":25378,"vised":25379,"clang":25380,"Ġinvestigators":25381,"Ġmysterious":25382,"Ġpromising":25383,"Ġqualify":25384,"Ġcave":25385,"Ġequip":25386,"=x":25387,"GT":25388,"(link":25389,".velocity":25390,".erase":25391,"oter":25392,"++++++++":25393,"profit":25394,"Ġzones":25395,"_uid":25396,"-ser":25397,"Ġobjectives":25398,"Ġmilf":25399,"webkit":25400,"(match":25401,"neh":25402,"ĠAssociated":25403,"ĠTodo":25404,"=d":25405,"Cam":25406,"Ġvocal":25407,"Ġsudo":25408,"(EX":25409,"Ġtrou":25410,"ABC":25411,".bean":25412,"ĠGround":25413,"ĠREST":25414,"weets":25415,"Ing":25416,"imon":25417,"_bus":25418,"ĠCOLOR":25419,"unto":25420,"Ġfoss":25421,"ĠLinks":25422,"äng":25423,"/forms":25424,"prises":25425,"Ġachievement":25426,"CALL":25427,"елÑĮ":25428,"ĠVerify":25429,"_SOURCE":25430,"aptcha":25431,"IDD":25432,"_reference":25433,"Gold":25434,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":25435,"Receiver":25436,"Ġaj":25437,"_direction":25438,"}]":25439,"ĠCompet":25440,"Ġbang":25441,"ĠCass":25442,"-url":25443,"techn":25444,"ĠJerusalem":25445,"longitude":25446,"');čĊčĊ":25447,"Ġwinners":25448,"Tasks":25449,"ĠDMA":25450,"Ġtooltip":25451,"İ·":25452,"ĠBra":25453,"_duration":25454,"cury":25455,"parents":25456,"---->(":25526,"ĠKir":25527,"Ġintros":25528,"Ġsketch":25529,"Ġskilled":25530,"Ġimmer":25531,"Ġadequate":25532,"_rep":25533,"(header":25534,"_like":25535,"Ġperceived":25536,"ssh":25537,"Ġassuming":25538,"Ġff":25539,"_uuid":25540,"ulas":25541,"Ġdemocratic":25542,".entities":25543,"Series":25544,"aphore":25545,"Ġnewer":25546,"}(":25547,"SEC":25548,"airo":25549,"Ġcommod":25550,"Ġprivilege":25551,"Ġdeux":25552,"ĠHop":25553,".'/":25554,"ctic":25555,".';Ċ":25556,"C":25630,"ĠWarren":25631,"Ġoptimizer":25632,"ĠSERVICES":25633,"_oper":25634,"getAttribute":25635,"ĠMcK":25636,"_self":25637,".rs":25638,"\")ĊĊĊ":25639,"GetComponent":25640,"erce":25641,"Ġtous":25642,"units":25643,"']);čĊ":25644,"Zoom":25645,"/E":25646,"Ġobsc":25647,"Ġfastest":25648,"online":25649,"Ġpeaceful":25650,"ffen":25651,"Ġcargo":25652,"ĉpr":25653,"Ġseeks":25654,"zu":25655,"Trim":25656,"Ġward":25657,"Ġverd":25658,"Ġblogs":25659,".exceptions":25660,"ĠPremium":25661,"ĠNetherlands":25662,"Safe":25663,"Finish":25664,"ĠAlbum":25665,"_ACC":25666,"=this":25667,"virtual":25668,"]>":25669,"_LABEL":25670,"ĠNich":25671,"_win":25672,"ĠAaron":25673,"WP":25674,";$":25675,"aims":25676,"ĠImageView":25677,"Ġendless":25678,"ERA":25679,"_DISABLE":25680,"Ġcancelled":25681,"-us":25682,"Ġinspection":25683,"emin":25684,"ĠGrey":25685,"-open":25686,"Ġiterations":25687,".owner":25688,"Ġkeras":25689,".Password":25690,"ĠRy":25691,"ĠINS":25692,"Air":25693,"ĠSeveral":25694,".TabStop":25695,"INGLE":25696,"ĠHair":25697,"ĠCanvas":25698,"AAAA":25699,"Ġflaw":25700,"cedes":25701,".Report":25702,"íĬ":25703,"ĠTips":25704,"criptors":25705,".transaction":25706,".Spring":25707,"Ġviewer":25708,"Ġinsights":25709,"è¾ĵ":25710,"ordion":25711,"UINT":25712,"seek":25713,"ĠAuf":25714,"ìŀIJ":25715,"Ġstrain":25716,"Tooltip":25717,"Ġdz":25718,"ignal":25719,"adt":25720,"Ġuc":25721,"finite":25722,"Ġnm":25723,".cmd":25724,"ĠMySql":25725,"[data":25726,".jackson":25727,".tree":25728,"RequestParam":25729,"_agent":25730,"\")]čĊ":25731,"Ġassass":25732,"(Constants":25733,":ss":25734,"ĠMAN":25735,"+-+-":25736,"ĠBottom":25737,"prints":25738,"ĠSame":25739,"@Autowired":25740,"swap":25741,"ición":25742,"Ġprotesters":25743,"Ġhoney":25744,"ĠVeter":25745,"(Calendar":25746,"-ad":25747,"ĠBrooklyn":25748,"Life":25749,"_VAR":25750,"zech":25751,"ĠCALL":25752,"_CAST":25753,"ĠElection":25754,"Ġthickness":25755,"Very":25756,"_INTEGER":25757,"-dev":25758,"))))":25759,"apat":25760,"oooo":25761,"demo":25762,"ĠparseFloat":25763,"ĠRather":25764,"STIT":25765,"maker":25766,"[current":25767,"chrono":25768,"Ġchrist":25769,"ãģª":25770,"ĠDetail":25771,"ưá»":25772,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":25773,"Ġsul":25774,"idency":25775,"Que":25776,"Ġelegant":25777,"apons":25778,"Ġdishes":25779,"Ġintegers":25780,"(read":25781,"findViewById":25782,"ĠAmount":25783,"ĠSkip":25784,"Ġhabits":25785,"*)(":25786,"Ġmonsters":25787,"MAC":25788,":end":25789,"Ġfrank":25790,"Assembly":25791,"Ġdfs":25792,"Ġneut":25793,"_TYPES":25794,"equal":25795,"loyd":25796,"(uri":25797,"Ġchi":25798,"Ġdefendant":25799,"Ġconflicts":25800,"Ġvil":25801,"-js":25802,"ĠPeace":25803,"Ġmutable":25804,")sender":25805,"ĠFocus":25806,"建":25807,"Ġappreciated":25808,"sleep":25809,"ĠRED":25810,"Culture":25811,"Ġdesigners":25812,"_generator":25813,"codes":25814,"/ex":25815,".GetValue":25816,"umbled":25817,".scalajs":25818,"peror":25819,"Ġveterans":25820,"Ġ})čĊ":25821,"Ġunfortunately":25822,"_CREATE":25823,"Mass":25824,"ĠCLAIM":25825,"ĠMeet":25826,"_support":25827,"Bank":25828,"().Ċ":25829,"Dark":25830,"_LOW":25831,"ĠMining":25832,"ĠOwner":25833,"iera":25834,"Cliente":25835,"Ġencouraging":25836,">S":25837,"Ġboyfriend":25838,"ĠHalf":25839,"ĠACC":25840,"Aff":25841,"_ar":25842,"-life":25843,"cx":25844,".JButton":25845,"izado":25846,".zero":25847,".openqa":25848,"oton":25849,".textContent":25850,"Ġtoll":25851,"atie":25852,"Ġballot":25853,"-number":25854,".Exception":25855,"ĉparams":25856,"circle":25857,"-map":25858,"Ġnap":25859,"ĠRobot":25860,"ĠIch":25861,"registration":25862,"Amazon":25863,"rollment":25864,"(exp":25865,"Ġtanks":25866,"ĠGordon":25867,"Ġmachinery":25868,"Ġbaseline":25869,"æĭ":25870,"Ø©":25871,"ĠConvention":25872,"ĉconfig":25873,"ookies":25874,"mult":25875,"Records":25876,"ĠEST":25877,"Ġgarbage":25878,"Ġconform":25879,"idal":25880,"Ġbarg":25881,"Ġsurvived":25882,"Ġinvestigations":25883,".containsKey":25884,"--------------------------------------------------------------------------Ċ":25885,"ortion":25886,"Ġhorr":25887,"_http":25888,"Ġmant":25889,"];čĊčĊ":25890,"binary":25891,"empl":25892,"Ġinquiry":25893,"ĠMeanwhile":25894,"Ġcollecting":25895,".EntityFramework":25896,"\",ĊĊ":25897,"ĠPic":25898,"@Inject":25899,"ickness":25900,"ĠBinding":25901,"Ġcontrolling":25902,"reverse":25903,"Ġchairs":25904,"sembled":25905,"(add":25906,"Disabled":25907,"anas":25908,".translate":25909,"-----------Ċ":25910,"Ġreflected":25911,"\"]ĊĊ":25912,"External":25913,"Arrow":25914,"Singleton":25915,"%x":25916,"ĠÅ":25917,"Ġancest":25918,"ĠOrleans":25919,"ĉcmd":25920,"Ġprohibited":25921,"ithmetic":25922,"(channel":25923,"_css":25924,"Forward":25925,".socket":25926,"Ġluc":25927,"âĨ":25928,"ĠFirefox":25929,"ĠMovies":25930,")_":25931,".ends":25932,"(shape":25933,"Ġdealt":25934,"Ġsaves":25935,"Ġglory":25936,"Ġmejor":25937,"Ġbreathing":25938,"Ġeller":25939,"getData":25940,"Ġangles":25941,"Ġtoolbar":25942,"Ġspacing":25943,"IPS":25944,"Ġfloors":25945,"_ACTIVE":25946,"Ġshuffle":25947,"/shared":25948,"ĠEle":25949,"edish":25950,"Ġwebcam":25951,".expect":25952,"iloc":25953,"ĠIncludes":25954,"Ġtweeted":25955,"Ġ:)":25956,"ĠEssay":25957,"Fix":25958,"-between":25959,"_web":25960,".conv":25961,"Ġracism":25962,"Ġreflects":25963,"umm":25964,"иÑĤе":25965,"_footer":25966,"/docs":25967,"ĠPour":25968,"NgModule":25969,".initialize":25970,"patterns":25971,"_In":25972,"ĠAbb":25973,"*čĊ":25974,"Ġsentiment":25975,"buff":25976,"_counts":25977,"Ġreuse":25978,"chunk":25979,"Ġimposed":25980,"PrimaryKey":25981,"Foreground":25982,"Ġconsumed":25983,"?!":25984,"Ġdick":25985,"Ġchron":25986,"ĠFern":25987,"Ġresponsive":25988,"Ġinsect":25989,"iculty":25990,"Ġrw":25991,"Ġalike":25992,"Ġsubset":25993,"ĠCookies":25994,"ĠPair":25995,"Ġtier":25996,"IFO":25997,"avour":25998,"ĠQU":25999,",sizeof":26000,"Ġmerged":26001,"mv":26002,"itol":26003,"ylon":26004,"Ġjumped":26005,".role":26006,"ensaje":26007,"Rules":26008,"Ġbrowse":26009,"Animator":26010,"Ġyoga":26011,"Ġvariants":26012,"Ġcourtesy":26013,"uran":26014,"pbs":26015,"elseif":26016,"Alt":26017,"ĠLane":26018,"CLK":26019,"IMARY":26020,"_PROPERTY":26021,"ï¼IJ":26022,"Ġchan":26023,"Ġgradually":26024,"Ġshake":26025,"Ġblonde":26026,"...\");Ċ":26027,"-sex":26028,"Ġgameplay":26029,"acies":26030,".refresh":26031,"USB":26032,"ĠPlot":26033,"Was":26034,"issippi":26035,"ĠTensor":26036,"Ġcryptocurrency":26037,"Ġdifficulties":26038,"Deleted":26039,"Without":26040,"_append":26041,"_ver":26042,"\"))čĊ":26043,"Ġhonestly":26044,"Ġpivot":26045,"Ġtemps":26046,"_ps":26047,"ĠUnlike":26048,"[:-":26049,"VS":26050,"_inf":26051,"Ġjunior":26052,"Ġanimations":26053,"Ġfilepath":26054,"?{{$":26076,"Ġunicode":26077,"places":26078,"ĠCoffee":26079,".SE":26080,"ĠPAR":26081,"(txt":26082,"gebra":26083,"Ġfires":26084,"MainWindow":26085,"medium":26086,"Ġ(âĢľ":26087,"Ġlg":26088,"Ġcmp":26089,"/base":26090,"_layers":26091,"_entries":26092,"Ġadminister":26093,"ĠSUCH":26094,"BP":26095,"ĠScottish":26096,"ĉčĊĉčĊ":26097,"guard":26098,"ĠStrong":26099,"Insn":26100,"ĠCAP":26101,"asury":26102,"ĠSEE":26103,"Clock":26104,"erie":26105,"\\models":26106,"Ġ$$":26107,"ĠCab":26108,"Ġwurde":26109,"Ġsoldier":26110,"Ġclips":26111,"Ġarrangement":26112,"ĠWonder":26113,"ĠHorn":26114,"Ġscared":26115,"Ġcure":26116,"mkdir":26117,"Ġaligned":26118,"ĠPink":26119,"Ġlanded":26120,"Dimension":26121,"ScrollPane":26122,".chat":26123,".With":26124,"ĠTrain":26125,"].Ċ":26126,"Ġthirty":26127,"Ġdurable":26128,"Ġld":26129,"Ġlateinit":26130,"Ġcharts":26131,"Ġinsult":26132,".Fatal":26133,"_ct":26134,"Ġmasks":26135,"CLUDED":26136,"President":26137,"Ġcolours":26138,"gments":26139,".attributes":26140,"ĠFlex":26141,"ĠClock":26142,"ÃŃcul":26143,"imen":26144,"JO":26145,"ĠRegex":26146,"_LINK":26147,"Ġcouch":26148,"ĠINPUT":26149,"Ġbeating":26150,"business":26151,"preced":26152,".unit":26153,"ĠFel":26154,"Never":26155,"ospel":26156,".startswith":26157,"ĠEPA":26158,".only":26159,"Ġpreventing":26160,"yer":26161,"ColumnName":26162,"Ġelevation":26163,"flu":26164,"icycle":26165,"Ġoffline":26166,"Toolbar":26167,"Ġcompeting":26168,")].":26169,"Ġmog":26170,"ĠisValid":26171,"Ask":26172,"_av":26173,"_lat":26174,"ANC":26175,"ĠJoh":26176,"kers":26177,"Ġguards":26178,"Ġchains":26179,"ĠSimpleDateFormat":26180,".static":26181,"Ġvessel":26182,"Ġmud":26183,"Ġstabil":26184,"Ġstret":26185,"gm":26186,"amation":26187,"çľ":26188,"-with":26189,"Ġros":26190,"_PA":26191,"Ġresultado":26192,"Ġconfidential":26193,"ĠTokyo":26194,"ĉusing":26195,"ĠMathf":26196,"ombine":26197,"ĠESPN":26198,"Ġdealers":26199,"Ġdismissed":26200,"TRY":26201,"Ġteens":26202,"records":26203,"Ġwings":26204,"gallery":26205,"accounts":26206,"_LIB":26207,"Ġjacket":26208,"ĠNSObject":26209,"Ġstones":26210,"ĠDelivery":26211,"ĠDiet":26212,"/watch":26213,"Ġtoilet":26214,"ĠGuest":26215,".day":26216,"Ġintval":26217,"Visit":26218,"Ġinvestigated":26219,"Ġpentru":26220,"ĠTheatre":26221,"andidates":26222,"Lang":26223,"ĠServ":26224,"Ġcontrollers":26225,"ĠsetTitle":26226,"NP":26227,"amy":26228,"flat":26229,"(ui":26230,"_document":26231,"èĥ½":26232,"ĠCoin":26233,"ĠAdams":26234,"ptic":26235,"Ġproductive":26236,"Ġaccomplished":26237,"čĊčĊčĊčĊ":26238,"Ġdeferred":26239,"ientes":26240,"Ġsinc":26241,"olars":26242,"Rightarrow":26243,"Ġvariations":26244,"(offset":26245,".LayoutInflater":26246,"Ġsuspend":26247,"Ġprevention":26248,"_private":26249,"_js":26250,"âĺħ":26251,"Ġwieder":26252,"atum":26253,"ĴĮ":26254,"Ġappearances":26255,".Document":26256,"Ġvalidates":26257,"calendar":26258,"}\";Ċ":26259,".demo":26260,"conut":26261,"Ġcorrection":26262,"ĠDeal":26263,"Ġbatteries":26264,".duration":26265,",\\":26266,"_marker":26267,"multi":26268,"Ġhalt":26269,"Ġcms":26270,"Ġshaped":26271,"Bro":26272,"reduce":26273,"Ġ####":26274,"CTOR":26275,"ĠBenef":26276,"Ġiconic":26277,"Ġpiano":26278,"Ġeffectiveness":26279,"|.Ċ":26280,"Ġajax":26281,"Ġvolumes":26282,"ม":26283,"Ġcljs":26284,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":26285,"aths":26286,"raits":26287,"大":26288,"Ñĸ":26289,"_mult":26290,"Ġfascinating":26291,"Average":26292,"Ġpré":26293,"ĠChairman":26294,".findElement":26295,"_pin":26296,"Ġcomparing":26297,"Ġdarkness":26298,"-Fi":26299,"-server":26300,"Ġselecting":26301,"sterdam":26302,"ĠParts":26303,"FORMATION":26304,"Ġnoting":26305,"Ġpile":26306,"ogs":26307,"Ġpalette":26308,"_do":26309,"itize":26310,"()(":26311,"Ġdefining":26312,"Ġremainder":26313,"Units":26314,"_TASK":26315,"HttpClient":26316,"Social":26317,"Ġfundra":26318,"NR":26319,"chest":26320,"Currency":26321,".adapter":26322,"Ġdop":26323,"unting":26324,"ANGUAGE":26325,"\"He":26326,"ĉindex":26327,"_package":26328,".Icon":26329,"Ġrepet":26330,"mass":26331,"=\".$":26332,"ĠSud":26333,"Ġlid":26334,"province":26335,"ìľ":26336,"GPIO":26337,"Ðļ":26338,"ĠMySQL":26339,"Ġdocs":26340,"ĠGA":26341,"Ġipsum":26342,"Kernel":26343,"Ġaccepts":26344,"Ġfitting":26345,"Ġcuando":26346,"Ġduplic":26347,"ĠBrother":26348,"ĠKle":26349,"nums":26350,"Ġmorph":26351,"Ġ########":26352,"ĠCGPoint":26353,"manual":26667,"ĠTechnical":26668,"Ġcorporation":26669,"ĠHW":26670,"anka":26671,"TAIL":26672,"istas":26673,"Ġperforms":26674,"ĠBehavior":26675,".For":26676,"_ORDER":26677,"ĠKick":26678,"Ġcallbacks":26679,"_dr":26680,"uego":26681,"hub":26682,"ufficient":26683,"sky":26684,"Ġbp":26685,"htable":26686,"ĠONLY":26687,"ĠAUTHORS":26688,".Argument":26689,"\"};Ċ":26690,"ĠThunder":26691,"ĠKom":26692,".Should":26693,"AUTH":26694,"ahu":26695,"_payment":26696,"Ġstarter":26697,"ìĦľ":26698,"ìļ©":26699,"Blog":26700,".patch":26701,"Ġgoverned":26702,"assy":26703,"-found":26704,"Ġtheater":26705,"ĠFontWeight":26706,"ĠBatman":26707,"\"If":26708,".Random":26709,"_delta":26710,"ĠCE":26711,"Authenticated":26712,"Ġdrone":26713,"Ġcous":26714,"radius":26715,"Mer":26716,"(None":26717,"ĠNJ":26718,"_headers":26719,"Ġamer":26720,"pytest":26721,"ĠActions":26722,"ĉĉĉĠĠĠĠ":26723,"Ġett":26724,"Ġholy":26725,"Ġuncomfort":26726,"ĠNin":26727,"ĠDecimal":26728,"ĠMessages":26729,".sender":26730,"]])Ċ":26731,"Ġembrace":26732,"Though":26733,"/sp":26734,"Ġcultures":26735,"Ġhighway":26736,"tar":26737,".fail":26738,"_hidden":26739,"ĠcomponentDidMount":26740,"ĠWright":26741,"Ġjag":26742,"_il":26743,"../../../":26744,"igu":26745,"Food":26746,"Ġace":26747,"Ġaños":26748,"USD":26749,"Ġmutual":26750,"Logic":26751,"Ġtemple":26752,"Ġbriefly":26753,"ĠTrip":26754,"classmethod":26755,"defaults":26756,"Ġchunks":26757,",,,,":26758,"ĠReason":26759,"$id":26760,"-ups":26761,"Ġdamn":26762,"Ġtrucks":26763,"Ġunlimited":26764,"Ġsculpt":26765,"ĠCards":26766,"Ġautor":26767,"ĠTesting":26768,"Ġdiese":26769,"shops":26770,"ç´":26771,"(payload":26772,"ĠPATH":26773,"ĠMemorial":26774,"Ġridiculous":26775,"egree":26776,"-winning":26777,"Ġrehab":26778,"Ġsophisticated":26779,"wpdb":26780,"ĉpath":26781,"!\";Ċ":26782,"_SYS":26783,".speed":26784,"Ġsoap":26785,"suffix":26786,"Wrap":26787,"Ġenhancement":26788,"Ãī":26789,"úb":26790,"Ġplaylist":26791,"Ġmixing":26792,"antidad":26793,"=\"\";Ċ":26794,"ĠRevision":26795,"ĠBeat":26796,".inc":26797,"-way":26798,"encias":26799,"ulers":26800,"Cat":26801,"idel":26802,"ĠShip":26803,".setColor":26804,"Ġthreatening":26805,".modules":26806,"Ġafterwards":26807,"ĠDashboard":26808,"ĊĠĊ":26809,"Signal":26810,"Ġprimer":26811,"orneys":26812,"iciary":26813,"Ġligne":26814,"_predict":26815,"Ġaest":26816,"_https":26817,">:":26818,"ĠLex":26819,"Ġrencontres":26820,"egral":26821,"scala":26822,"_family":26823,"ÃŁen":26824,"_sym":26825,"Ġuncertainty":26826,"ĠVALUE":26827,"Ġ};čĊčĊ":26828,"Ġbroader":26829,"Ġhorses":26830,"ãģĿ":26831,"ĠKal":26832,"oba":26833,"_INET":26834,"ĠKill":26835,"jquery":26836,"amination":26837,"[@\"":26838,"Ġmuj":26839,"###Ċ":26840,"FirstOrDefault":26841,"thenReturn":26842,"Che":26843,"/footer":26844,"Ġparks":26845,"asje":26846,"ĠGulf":26847,"Ġmodest":26848,".Init":26849,"ï¼ŁĊĊ":26850,"Ġprospects":26851,"Ġsvg":26852,"Ġåı":26853,".Dialog":26854,"_NET":26855,"Ġ(($":26856,"Ġek":26857,"ĠWarning":26858,"ĠMK":26859,"":27166,"ĠRepair":27167,"_BE":27168,"Brand":27169,"uart":27170,"preview":27171,"Ġinitiatives":27172,"running":27173,"bang":27174,"ĉupdate":27175,"ĠCoach":27176,"Rich":27177,"Ġyoutube":27178,"Ġritual":27179,"appa":27180,"ĠRobinson":27181,"precision":27182,"////////////////////////////////////////////////////////////////////////////":27183,"=[]Ċ":27184,"Ġcelebrated":27185,"OTO":27186,"Ġinclusion":27187,"JP":27188,"';čĊčĊ":27189,"Ġnotable":27190,"(_.":27191,"Managed":27192,"Ġguides":27193," ":27194,"atedRoute":27195,"ĠAdjust":27196,"Ġcolored":27197,"_scores":27198,"ĠTesla":27199,"_progress":27200,".inst":27201,"['_":27202,".flags":27203,"Ġfclose":27204,"_OPER":27205,"ży":27206,"_note":27207,"Ġtransgender":27208,"åķ":27209,"RIPT":27210,"Ġabsent":27211,"Ġamet":27212,"Ġoperand":27213,"ë©":27214,"Ġhood":27215,"toLowerCase":27216,"avo":27217,"ĠCircuit":27218,"ĠLind":27219,"--}}Ċ":27220,"=m":27221,"Ġsuppress":27222,"ĠMAP":27223,"iang":27224,"-admin":27225,"Ġsidebar":27226,"ĠBu":27227,"ĠHex":27228,",F":27229,"ĠSignal":27230,"Ġtransparency":27231,"ĠFederation":27232,"/V":27233,"Req":27234,"Ġpulse":27235,"Ġtends":27236,"Numbers":27237,"%'":27238,"Ġdeport":27239,"datas":27240,"_UINT":27241,"_tra":27242,"oko":27243,"Ġ\"?":27244,"compet":27245,"solete":27246,"undry":27247,"Ġoverlap":27248,"}`,Ċ":27249,".ly":27250,"_summary":27251,"ĠLost":27252,".Center":27253,"Ġdisability":27254,".Serialization":27255,"Ġgeom":27256,"Ġ?:":27257,"ĠWo":27258,"Ġshipped":27259,"Ĥæķ°":27260,"Ġugly":27261,"Ġexcitement":27262,"Ġexterior":27263,"Ġcheckout":27264,"Ġkur":27265,",D":27266,"ĠAlaska":27267,"Ġsynthetic":27268,"ĠBudget":27269,"ĠSubscribe":27270,"Ġ&Ċ":27271,"ÈĻi":27272,"ĠYu":27273,"ĉquery":27274,"}.Ċ":27275,"Ġtraged":27276,"assen":27277,"Ġaccommodation":27278,"Ġphysician":27279,"Ġrenamed":27280,"Ġtidak":27281,"zÄħ":27282,"Ġminus":27283,"nych":27284,"_EXCEPTION":27285,"threads":27286,"Ġtire":27287,"_created":27288,"ensure":27289,"Ġworthy":27290,"Ġexcuse":27291,"Ġcloth":27292,".parentNode":27293,"/platform":27294,"ĠUFC":27295,"ĠGtk":27296,"unny":27297,"Ġgibt":27298,"keley":27299,"hum":27300,"(tx":27301,"ĉdev":27302,"Ġoutfit":27303,"doors":27304,"Ġfon":27305,"icut":27306,"volatile":27307,"Ġhomosex":27308,"Maximum":27309,"Ġexpend":27310,"Ġ});ĊĊĊ":27311,"Eq":27312,"onders":27313,"department":27314,"ĠPhysics":27315,"\"});Ċ":27316,"Ġparad":27317,".Str":27318,"Ġsele":27319,"IFIED":27320,"Ġdelivers":27321,"ivan":27322,"Ġresponsibilities":27323,"Ġadvocates":27324,"èµ":27325,"ĠRID":27326,".parameters":27327,"Metrics":27328,"ronics":27329,"ĠUITableViewCell":27330,"Absolute":27331,"ipse":27332,"ylum":27333,"MLElement":27334,"_VALID":27335,"\\<^":27530,"Ġios":27531,"sound":27532,"\"];":27533,"Ġfreed":27534,"rottle":27535,"ĠLower":27536,"[count":27537,"åĿ":27538,"Ġpale":27539,"ĠWayne":27540,"earth":27541,"_categories":27542,"UCK":27543,".metadata":27544,"Ġsummon":27545,"HOME":27546,"олÑĮз":27547,"Ġmanufactured":27548,"Ġdock":27549,"Ġcompetitors":27550,"_MODEL":27551,"okia":27552,"ĠHey":27553,"ο":27554,"Ġbackward":27555,"ĠPOSS":27556,"ropa":27557,"Ġcri":27558,"_OBJ":27559,"Transport":27560,"-high":27561,"Ġerotik":27562,"_slot":27563,"Ġartic":27564,"_framework":27565,"-serif":27566,"ĠSqlDbType":27567,"')(":27568,"+\"/":27569,"Ġwore":27570,"Sil":27571,"Ġstoring":27572,"ĠPhase":27573,"uant":27574,"Ġbump":27575,"inho":27576,"Ġdign":27577,"Ġbacks":27578,"qq":27579,"(hash":27580,"Ġgeo":27581,"Ġtender":27582,"Logo":27583,"!)Ċ":27584,"ĠMX":27585,"ĠArthur":27586,"essoa":27587,"_Ch":27588,"Ġbedrooms":27589,"=\"#\"><":27590,"Ġthroat":27591,"insic":27592,".integer":27593,"Ġprimitive":27594,"Truthy":27595,"Ġfacilitate":27596,"Ġcreativity":27597,"ĠDNS":27598,"Ġgra":27599,"uez":27600,"Ġcountless":27601,"ĠPoland":27602,"'M":27603,"ĠDist":27604,"Ġvest":27605,"Ġcertification":27606,"á»ij":27607,"held":27608,"extensions":27609,"(static":27610,"Ġgrades":27611,"ĠUber":27612,"ãģŁ":27613,"Ġ[])Ċ":27614,"datos":27615,"ĠgetData":27616,"ĠCharg":27617,"ĠBS":27618,".microsoft":27619,".video":27620,".direction":27621,"->{'":27622,"lua":27623,"apest":27624,"Ġboiler":27625,"erek":27626,"Ġdecides":27627,".jar":27628,"ISC":27629,"ĠWords":27630,"(CON":27631,"EMPLATE":27632,"reeze":27633,"shots":27634,"apps":27635,"unted":27636,".setName":27637,"::<":27638,"-bold":27639,"ê²":27640,"å¯Ĩ":27641,"Longrightarrow":27642,"Ġunfair":27643,"Ġearning":27644,"Ġshelf":27645,"UREMENT":27646,"Ġidle":27647,"_MENU":27648,".Custom":27649,"AGER":27650,"-\"":27651,"_switch":27652,"because":27653,")view":27654,"mare":27655,"_condition":27656,"ĠStarting":27657,"Mvc":27658,"(pre":27659,"dump":27660,"_LOCK":27661,"atetime":27662,".callback":27663,"ĠCer":27664,"opol":27665,"ibrary":27666,"Ġreservation":27667,"ĉĉĉĉĉĉĉĊ":27668,"lector":27669,"graduate":27670,"Ġgenerous":27671,"Ġion":27672,"ricao":27673,"mq":27674,"_complete":27675,"(cursor":27676,"ĠFormControl":27677,":center":27678,"Ġsubstitute":27679,"ĠPlanning":27680,"Ġpension":27681,"Ġrecommendation":27682,"ĠTags":27683,"Ġgef":27684,"Ġalbums":27685,"Ġwashing":27686,"roc":27687,"Ġtrains":27688,"atings":27689,"Ġexponent":27690,"ackbar":27691,"-ln":27692,"ág":27693,".DataAnnotations":27694,"ĠEIF":27695,"ĠMalaysia":27696,"ĉPORT":27697,"onus":27698,"Ġclever":27699,"Ġpeu":27700,">ĊĊĊĊ":27701,"ĠArguments":27702,"Ġdebugging":27703,"(right":27704,"'D":27705,"compute":27706,"Ġfinest":27707,"ORAGE":27708,"Ġspectacular":27709,"phrase":27710,"Ġindia":27711,"Ġlegendary":27712,"birth":27713,"Ġcomposite":27714,"Ġgrows":27715,"ĠTD":27716,"Ġepid":27717,"Ġlaunching":27718,"]][":27719,"Minutes":27720,"ĠCha":27721,"Ġcleaned":27722,"Ġwitnesses":27723,"ukan":27724,"ĉType":27725,"Ġhabe":27726,"paragraph":27727,"ĠJPanel":27728,"ĠHann":27729,"Ġvaried":27730,"ĠPokemon":27731,"ĠMUST":27732,"åĬ¨":27733,".visibility":27734,"opup":27735,"^[":27736,".expand":27737,"Ġ\"',":27738,".fasterxml":27739,"_auto":27740,"ĠSheet":27741,"marker":27742,"Parcel":27743,"ews":27744,"ĠStrategy":27745,"-making":27746,"Ġunve":27747,"Ġtrailing":27748,"Ġclicks":27749,"ĠGetComponent":27750,"ĉcontent":27751,"IGENCE":27752,"ERNEL":27753,"NSMutableArray":27754,"Ġbreat":27755,"Ġharmful":27756,"¶Ī":27757,"Ġbesides":27758,"Ġboring":27759,"Ġbrutal":27760,"vang":27761,"(parse":27762,"quick":27763,"Ġpytest":27764,"Ġswitching":27765,"()]Ċ":27766,"ĠìĦ":27767,"LER":27768,"ĉfont":27769,"Ġnett":27770,")]ĊĊ":27771,"(/\\":27772,"æŀľ":27773,"toArray":27774,"Ġbreed":27775,"ĠCAR":27776,"ĠWeapon":27777,"Abs":27778,"tot":27779,"ĠsetName":27780,"aptive":27781,"Ġ:,":27782,"Ġescaped":27783,"orden":27784,"ĠPri":27785,"thumbnail":27786,"Ġdescriptions":27787,"/styles":27788,"ĠPCI":27789,"Ġalphabet":27790,"asticsearch":27791,"NOTE":27792,"Ġcialis":27793,"ĠGriff":27794,"Ġporque":27795,"Ġproteins":27796,"plays":27797,"Ġstating":27798,"Ġimagination":27799,"Ġfacial":27800,"ĠMechan":27801,"Ġarranged":27802,"_used":27803,"Ġarrangements":27804,"ĠPipe":27805,"hostname":27806,"Ġprovinc":27807,"Tit":27808,".FlatStyle":27809,"ĠSplit":27810,"ĠLoader":27811,".cc":27812,"Ġclinic":27813,"----------------------------":27814,"Ġbaking":27815,"ĠENT":27816,"neath":27817,"ãĢģĊĊ":27818,"ANE":27819,".EntityFrameworkCore":27820,"appers":27821,".ic":27822,"ĠNgModule":27823,"ĠFORM":27824,"Ġ';":27825,"-profit":27826,"hw":27827,"enemy":27828,"ĠEye":27829,"Ġcaution":27830,"town":27831,"Ġurged":27832,"ĠJimmy":27833,"ynchronous":27834,"-sized":27835,"making":27836,",{":27837,"]',":27838,"_Object":27839,"ahoma":27840,"Ġactivist":27841,"INVAL":27842,"ĠCommercial":27843,"ĠOrlando":27844,"(tab":27845,"Ġب":27846,"Algorithm":27847,"Ġheritage":27848,"GetMapping":27849,"Ġfailures":27850,"rios":27851,"ativa":27852,"Ġtet":27853,"Ġcarpet":27854,"(Z":27855,"three":27856,"Ġdisclosure":27857,".ERROR":27858,"_called":27859,"Ġdial":27860,"Ġoccasional":27861,".Err":27862,"Ġfuncion":27863,"caffold":27864,"Ġreleasing":27865,"ï¼īĊĊ":27866,"_Value":27867,"ĠVari":27868,"yellow":27869,"Ġstruggles":27870,".cal":27871,"ĠDakota":27872,"ĉclose":27873,"Ġsandwich":27874,"Ġanalytics":27875,"Ġ**)":27876,"&#":27877,"ĠJos":27878,"Ġpassive":27879,"ATTR":27880,"Throwable":27881,"ĠMun":27882,"ĠUint":27883,"(disposing":27884,"arak":27885,"ĠLeaders":27886,"Ġaffecting":27887,"ĠitemView":27888,"Ġeconomics":27889,"fv":27890,"à¹Ģ":27891,".rb":27892,"ĠOverall":27893,"Ġwealthy":27894,"Ġevolved":27895,"nda":27896,"ĠHus":27897,"restrict":27898,"umen":27899,"ĠAgricult":27900,"!ĊĊĊ":27901,"Ġexpires":27902,"Ġspokesperson":27903,"interval":27904,"Ġâ":27905,"Ġqueen":27906,"(nil":27907,"ingo":27908,"Heap":27909,"Ùİ":27910,"Ġcomplain":27911,"Sym":27912,"ĠClone":27913,"ĠRu":27914,"ĠWILL":27915,"ĠCrystal":27916,"/content":27917,"ingen":27918,"ointment":27919,"LastName":27920,"avicon":27921,"ĠIBM":27922,"ĠDimension":27923,"anh":27924,"icipants":27925,"ĠAnne":27926,".progress":27927,"Ġalgo":27928,"obil":27929,"ĠVoice":27930,"ĠFE":27931,"Ġgli":27932,"Ġved":27933,"Ġprevents":27934,"\\Column":27935,"Ġfolk":27936,"etti":27937,"Ġmn":27938,"ĠCLASS":27939,"Ġdisplaying":27940,"ĠKl":27941,"ĠFerr":27942,"duto":27943,".ib":27944,"Ġdados":27945,"'name":27946,"-space":27947,"Ġitalian":27948,"Ġinverse":27949,"Ġdense":27950,"uter":27951,"ĠIEnumerator":27952,"-sign":27953,"Ġnationwide":27954,"Ġpersona":27955,"Ġsolved":27956,"Ġdramatically":27957,"Logout":27958,"Ġgrav":27959,"Ġanalyses":27960,"ollo":27961,"Ġlamp":27962,".team":27963,"ĠErot":27964,"=[\"":27965,"Ġdancing":27966,"Ġ?>/":27967,"Ġcater":27968,"ffe":27969,"ĠSha":27970,"ĠBos":27971,"ĠREQUIRE":27972,"ĠMonster":27973,"ĠRB":27974,"ĠIDE":27975,"Ġsuits":27976,"ĠformData":27977,"(theta":27978,"Ġspatial":27979,"=NULL":27980,"ĠSqlConnection":27981,"Ġà":27982,"ĠVenez":27983,"ĠMorning":27984,"Ġpublications":27985,"ĠNONINFRINGEMENT":27986,"firstName":27987,"uds":27988,"Would":27989,"_HEAD":27990,"Ġinvested":27991,"stable":27992,"fred":27993,"Ġcommander":27994,"SES":27995,"âĢĶa":27996,"anche":27997,"ĠMovement":27998,"ë³":27999,"Suite":28000,"Ġjurisdiction":28001,"리":28002,"ĠBeth":28003,"jQuery":28004,"ĠIsa":28005,"Ġdental":28006,",*":28007,"ĠLimit":28008,"iliation":28009,"=\"{":28010,"bast":28011,"Ġturb":28012,"isy":28013,"OOK":28014,"Ġadvocate":28015,"imag":28016,"LECTION":28017,"лÑĮ":28018,"(category":28019,".dec":28020,"Ġuniqu":28021,"_sn":28022,"Ġattracted":28023,"ĠÃī":28024,"ĠRunning":28025,"_edges":28026,"ĠDisable":28027,"_AS":28028,"åĽ¾":28029,"Ġnetworking":28030,"_branch":28031,"Having":28032,"toBeTruthy":28033,"GI":28034,"Ġcamps":28035,"sep":28036,"-part":28037,"Ġ)ĊĊĊĊĊĊĊĊ":28038,"ustralia":28039,"ĠReports":28040,"rito":28041,"Ġwaist":28042,"_plus":28043,"ĠWW":28044,"-person":28045,"April":28046,"Ġsar":28047,".tar":28048,"Ġagricultural":28049,"tic":28050,"Ġtcp":28051,"ĠsetValue":28052,"agento":28053,"ĠAppe":28054,"piler":28055,"CADE":28056,"Ġanche":28057,"atcher":28058,"Ġcomics":28059,"Ġlbs":28060,"_segment":28061,"']=$":28062,"itters":28063,"icher":28064,"GINE":28065,"Ġutilize":28066,"ĠCursor":28067,"_expression":28068,"Ġdag":28069,"x":28257,".Task":28258,"money":28259,"ibaba":28260,"'});Ċ":28261,"ĠSpecific":28262,"ĠLinear":28263,"_OPT":28264,"HashCode":28265,"(Player":28266,".ContainsKey":28267,"Ġcollapsed":28268,"transparent":28269,"_RANGE":28270,"Viewer":28271,"(cfg":28272,"Ġsorting":28273,"Ġinfected":28274,"ĠNach":28275,"Ġaccommodate":28276,".elements":28277,"_PART":28278,"ĠSexy":28279,"=get":28280,"(year":28281,"Ġxhr":28282,":]":28283,"owski":28284,"Ġsummar":28285,"Ġ¿":28286,"Ġinte":28287,"Ġworkflow":28288,"ĠTaiwan":28289,"versions":28290,"åıij":28291,"Ġsurprisingly":28292,"Ġoptical":28293,"Ġproces":28294,"Ġdisagree":28295,"Ġnuevo":28296,"ĠCAM":28297,"sorted":28298,"leases":28299,"istle":28300,"Ident":28301,"ĉevent":28302,"jected":28303,"Chunk":28304,"Vars":28305,".provider":28306,"Ġproceedings":28307,"Ġinclusive":28308,"Ġartwork":28309,"endants":28310,"ï¼ļĊ":28311,"seen":28312,"Ġlig":28313,"Ġmakers":28314,"_fun":28315,"Ġlengths":28316,"PathVariable":28317,"[item":28318,"ี":28319,"Dead":28320,"FFFFFF":28321,"ĠUrban":28322,"uples":28323,"ichen":28324,"(nullptr":28325,".spec":28326,",System":28327,"URATION":28328,"(job":28329,"å¼ı":28330,"Ġtracker":28331,"ÅĻ":28332,"ĠMR":28333,"ĠSQLite":28334,"Ġdto":28335,"Ġ;;Ċ":28336,"Ġmint":28337,"ĠIntroduction":28338,"cao":28339,"Ġquestioned":28340,"Ġfitted":28341,"revision":28342,"sq":28343,"Ġmig":28344,"_units":28345,"_async":28346,"Ġflick":28347,"});ĊĊĊ":28348,"Ġnotre":28349,"}`,":28350,"Filters":28351,"Ġmundo":28352,"_days":28353,"Ġfrm":28354,"utc":28355,"Ġvals":28356,"ewidth":28357,"ĠGenerator":28358,"ĠArtist":28359,"ĠIDs":28360,"ĠArticles":28361,"reater":28362,"ĠComponentFixture":28363,".=":28364,"Ġrou":28365,"-no":28366,".bukkit":28367,"egg":28368,"ĠDiff":28369,"atics":28370,"ÑĥÑĩ":28371,"âĢĶĊĊ":28372,"ĠCharlotte":28373,"bye":28374,"Ġ});čĊčĊ":28375,"ĠVik":28376,"ĠBrow":28377,"Ġlv":28378,"ĠGib":28379,"-wing":28380,"GLIGENCE":28381,"(Il":28382,"ĠEngineer":28383,".Wait":28384,"ĠPictures":28385,"Ġrhet":28386,"Ġthermal":28387,"Ġpraise":28388,"<>();ĊĊ":28389,"ĠSpider":28390,"Pause":28391,"ĠBaker":28392,"Ġslower":28393,"Ġ}]Ċ":28394,"_enqueue":28395,"Ġdisappeared":28396,"ĠTicket":28397,"INUX":28398,"_LOCAL":28399,"аÑģÑģ":28400,"@Injectable":28401,"community":28402,"GestureRecognizer":28403,"åĽ½":28404,"Ġscales":28405,"Ġ-(":28406,"/'+":28407,"ĠSit":28408,"Ġexecutives":28409,"arding":28410,"Ġadvers":28411,"Ġbackwards":28412,"ĉcontext":28413,"ĠHamp":28414,"ĠPF":28415,"ĠDeck":28416,"ĠCraig":28417,"American":28418,"Ġbell":28419,"Ġprol":28420,"ufen":28421,"Ġrng":28422,"arshal":28423,"ĠSimply":28424,"firstname":28425,"shore":28426,"July":28427,"Ġmortality":28428,"ĠâĨĴĊĊ":28429,"Helpers":28430,"Ġbenchmark":28431,"emade":28432,"Ġorganisations":28433,".gson":28434,"ĠTextField":28435,"Ġcivilians":28436,".Arrays":28437,"ĠMississippi":28438,"Ġintermediate":28439,"getUser":28440,"_cluster":28441,"Relative":28442,"foreign":28443,".querySelectorAll":28444,"ForeignKey":28445,"Ġreasonably":28446,"---------Ċ":28447,"Cards":28448,"ĠKam":28449,"ĠThor":28450,"Ġroller":28451,"-element":28452,"ĠCurrency":28453,"ddie":28454,"ALLY":28455,"ĠRA":28456,"Ġpermet":28457,"aaaa":28458,"Ġhomework":28459,"ĠVit":28460,"Ġmold":28461,"ĠFer":28462,"[start":28463,"Ġstatistical":28464,"Ġscary":28465,"_HOME":28466,".Begin":28467,"Construct":28468,"ogenic":28469,"ĠDEALINGS":28470,"Ġtambién":28471,"ixon":28472,".ind":28473,"acre":28474,"Ġtransforms":28475,"ĠNap":28476,".Block":28477,"ussia":28478,"piration":28479,"ulent":28480,"Ġceil":28481,"Clause":28482,"naire":28483,"TES":28484,"Ġneat":28485,"STD":28486,"ĠRegExp":28487,"perform":28488,":)":28489,"Ġunions":28490,"Ġsublic":28491,"Ġwinds":28492,"loating":28493,"glich":28494,"Ġpagination":28495,"Skill":28496,"Apply":28497,"ĠOperator":28498,"istogram":28499,"Ġqualities":28500,"Cross":28501,"Ġdecom":28502,"],\"":28503,"ĠJuan":28504,".modal":28505,".Child":28506,"ĠRoger":28507,"STITUTE":28508,":CGRectMake":28509,"alette":28510,"Ġsta":28511,"aside":28512,"Ġblur":28513,"ĠWa":28514,"ifetime":28515,"reed":28516,"controls":28517,"Ġbins":28518,"Ġпол":28519,"*/,Ċ":28520,"UIS":28521,"ĠRou":28522,"ĠDemo":28523,"-awesome":28524,"ĠChain":28525,"Ġhasta":28526,"ĠBart":28527,".KEY":28528,"Ġvendors":28529,"nofollow":28530,"ĠDest":28531,"_builder":28532,"Ġargues":28533,"_answer":28534,"goto":28535,"ĠRESULT":28536,"ĠMON":28537,"Ġpoder":28538,"oons":28539,"_CASE":28540,"Ġreplic":28541,"Ġfinancing":28542,"ĠDATE":28543,"cern":28544,"_track":28545,"ties":28546,"/logo":28547,"ĠNEGLIGENCE":28548,"getType":28549,">T":28550,"bet":28551,"girl":28552,"ĠINCIDENTAL":28553,"-site":28554,".trigger":28555,"ĠLisa":28556,"_inputs":28557,"Ġrelatives":28558,"LoggedIn":28559,"Configure":28560,"IK":28561,".accept":28562,"Resume":28563,"ĠDraft":28564,"Ġ*>(":28565,"ĠWA":28566,"edian":28567,"erness":28568,"ĠLayoutInflater":28569,"*/čĊčĊ":28570,"othy":28571,"Ġobligation":28572,"Subscribe":28573,"Ġthumbnail":28574,"exist":28575,"Ġinsisted":28576,"ĠUICollectionView":28577,"ĠAngular":28578,"Ġtablets":28579,"ĠImpact":28580,"ãĢįĊĊ":28581,"aho":28582,"Ġcharacteristic":28583,"gd":28584,"Ġ=================================================":28585,"ourt":28586,"`.":28587,"Appro":28588,"Coordinate":28589,"Remember":28590,"Ġmarine":28591,"]=='":28592,"ĠAdministrator":28593,".getDefault":28594,"Ġforgot":28595,"ĠStructure":28596,"Vue":28597,"arsing":28598,"moment":28599,"kw":28600,"_cursor":28601,"Attack":28602,"Ġathletic":28603,"Ġdiagnosed":28604,"Ġende":28605,"åĪłéϤ":28606,"House":28607,"ĠPARAM":28608,"Ġwiki":28609,"ĠOpp":28610,"Ġconservation":28611,"Ġsnd":28612,"_tem":28613,"substr":28614,"ĠCape":28615,".sim":28616,"UTION":28617,"anan":28618,"âĢĻun":28619,"Ġgy":28620,"-work":28621,"Ġcompelling":28622,"='#":28623,"ĉsub":28624,"Ġdirectories":28625,"íĬ¸":28626,"Ġtouches":28627,"outines":28628,".Collection":28629,"schedule":28630,".lat":28631,"ĠDoctrine":28632,"CAA":28633,"ĠRefer":28634,"Ġshifts":28635,"Ġlikelihood":28636,"preter":28637,"ĠFemale":28638,"Ġintercept":28639,"Ġlou":28640,"çĻ»":28641,"Ġrug":28642,"ĠCrown":28643,"Ġ****************************************************************************":28644,"-product":28645,"Ġprompted":28646,"ungle":28647,"docker":28648,"ĠTu":28649,"ĠUnique":28650,"_Error":28651,"ulos":28652,"ĠâĦ":28653,"Ġ(`":28654,"Getting":28655,"_scal":28656,"ĠEnh":28657,"üt":28658,"Ġsustained":28659,"Ġpatches":28660,"Ġprosper":28661,"ĠGaza":28662,"_light":28663,"Ġincons":28664,"--------Ċ":28665,"ĉĉĠĠĠĠĠĠ":28666,"SF":28667,"CN":28668,":\";Ċ":28669,"ĠCollins":28670,"(*)":28671,"Ġcompilation":28672,"']čĊ":28673,"Ġconsequence":28674,",...":28675,"Ġdm":28676,"ĠBLOCK":28677,"Cluster":28678,"Ġski":28679,"(argc":28680,"Tuple":28681,"Ġjoins":28682,"ĠSheriff":28683,"War":28684,"indi":28685,"Ġcommented":28686,"HOST":28687,"Ġinvitation":28688,"apanese":28689,"Ġpermits":28690,"precedented":28691,"_zone":28692,"ĠAmy":28693,"_RD":28694,"Minimum":28695,"Ġinvocation":28696,".enable":28697,"ichten":28698,"-owned":28699,"\"id":28700,"_POINTER":28701,"Fac":28702,"Ġspecifications":28703,"Ġnomination":28704,"Ġgp":28705,"<(":28706,"Ġrobots":28707,"ĠJerry":28708,"Ġholders":28709,"Ġwand":28710,"cms":28711,"Ġ}))Ċ":28712,".Toast":28713,"ĠIList":28714,"Based":28715,"zoom":28716,"/style":28717,"ĠBeck":28718,"Men":28719,"Ġcontributing":28720,"Ġundo":28721,"ĠOH":28722,"ĠaddObject":28723,"Ġeigen":28724,"signup":28725,"éĶĻ":28726,"Ġdistant":28727,"PARATOR":28728,"ĠMari":28729,"Ġmá":28730,"Emp":28731,"ós":28732,"ĠìĪĺ":28733,"evt":28734,"+j":28735,"park":28736,"ĠStay":28737,"ĠDun":28738,"Ġsoy":28739,">%":28740,"azines":28741,"Ġtiempo":28742,"(me":28743,"present":28744,".This":28745,"Ġeditors":28746,"FIELD":28747,".Work":28748,"ĠUniverse":28749,"Ġdrunk":28750,".timer":28751,"Ġaltered":28752,"ĠNar":28753,"ëł¥":28754,".Active":28755,"idor":28756,"çŃ":28757,".deltaTime":28758,"Ġawkward":28759,""":28760,"ĠSafari":28761,"Ġtricks":28762,"MENTS":28763,"division":28764,"Ġvarying":28765,"ĠHighway":28766,"Ġphotographer":28767,"ĠStewart":28768,"Ġlasting":28769,".Pre":28770,".amazonaws":28771,"ĠLuck":28772,".Description":28773,"ĠNaz":28774,"neg":28775,"Ġcó":28776,"<<\"\\":28777,"ĠSurv":28778,"ĠUnc":28779,"Recipe":28780,".BorderStyle":28781,"Ġmodifications":28782,"-at":28783,"ATFORM":28784,"hdr":28785,"ako":28786,"Ġsublicense":28787,"ĠJump":28788,"Ġbeim":28789,"ĠManhattan":28790,".bool":28791,"_hw":28792,"ÑĤÑĮ":28793,"Bin":28794,"Ġgateway":28795,"\"\":":28796,"ĠUIS":28797,":\"+":28798,"-def":28799,"ĠRegular":28800,"/testing":28801,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":28802,"stringstream":28803,"Ġdispar":28804,"Ġmobil":28805,"-read":28806,"ĠAdapter":28807,"ĠChampions":28808,"Ġscheduler":28809,"Ġkills":28810,"ĠMultiple":28811,"irror":28812,"Ġgods":28813,"ADO":28814,"akte":28815,"ĠUsuario":28816,".circular":28817,"Ġrecept":28818,"ĠExpr":28819,"Ġelderly":28820,"Ġnicely":28821,"Ġbeste":28822,"Want":28823,"Ġclassical":28824,".sprite":28825,"objc":28826,"ĠMason":28827,"Ġsistema":28828,".Black":28829,"eso":28830,"ĠZeit":28831,"Ġdivid":28832,"Ġenters":28833,"_subject":28834,"ĠPlanet":28835,".warning":28836,"ĠGram":28837,"_tokens":28838,"Ġhouseholds":28839,"_customer":28840,"userName":28841,"cross":28842,"Ġpione":28843,"Ġassists":28844,"_SM":28845,"ibo":28846,"Ġloyal":28847,"Ġuseless":28848,"#elif":28849,"ĠUltimate":28850,"Come":28851,"gel":28852,"Ġdich":28853,"xyz":28854,"ikel":28855,"obra":28856,"_scan":28857,"ĠInterior":28858,"ĠNice":28859,"Ġplac":28860,"ĉtarget":28861,"Ġviral":28862,"asso":28863,"()/":28864,"unde":28865,"ĠAdobe":28866,"Os":28867,"visited":28868,"ĠOW":28869,"ĠFeed":28870,"ĠSequence":28871,"Ġmanages":28872,"inson":28873,"ĠLouisiana":28874,"{})":28875,"ĠHab":28876,"ĠLD":28877,"Ġbip":28878,"prites":28879,"(elem":28880,".hibernate":28881,"élé":28882,"Ġohne":28883,"_transaction":28884,"Ġannunci":28885,"Published":28886,"ĠHonda":28887,"ĠTam":28888,"ĠPacket":28889,"_selector":28890,"Ġchallenged":28891,"Processing":28892,"-hover":28893,"Ġtrainer":28894,"_cancel":28895,"ĠNSDictionary":28896,"abric":28897,"ĠMLS":28898,"_sensor":28899,"Ġshrink":28900,"ĠFX":28901,"threshold":28902,"ĉHX":28903,"-mark":28904,"`.`":28905,"Scheme":28906,"(full":28907,"_writer":28908,"ĠSys":28909,"Ġfled":28910,"ĠCin":28911,"-widget":28912,"ĠPrevious":28913,"Gender":28914,"_question":28915,"Feed":28916,"Ġscrut":28917,"(prefix":28918,"ãĢĤãĢĤ":28919,"Ġinfections":28920,"Parts":28921,"Ġhierarchy":28922,"_DELETE":28923,"ĠPatient":28924,"_pay":28925,"Ġpromoted":28926,"Ġìĭ":28927,"Ġcivilian":28928,"Ġagriculture":28929,"ĠPiece":28930,"Ġstance":28931,"utsche":28932,"Assign":28933,".ACTION":28934,"Fig":28935,"_radius":28936,"ĠSync":28937,"ducer":28938,"failure":28939,"ensed":28940,"ptime":28941,"BM":28942,"_datetime":28943,"quivo":28944,"QUEUE":28945,"èĢħ":28946,"Appear":28947,"Ġsummit":28948,":void":28949,"Ġvine":28950,"认":28951,"onne":28952,"_TRANS":28953,".green":28954,"_cc":28955,"Ġhungry":28956,"Ġ\">":28957,"());čĊčĊ":28958,"Extract":28959,"izens":28960,"Ġsolver":28961,"Notify":28962,"Ġenglish":28963,"ĠShopping":28964,"interfaces":28965,"REQ":28966,"Ġilleg":28967,"ĠUIImageView":28968,"Ġdisconnect":28969,"ĠUntil":28970,"ĠConservative":28971,"@Column":28972,"Ġshifted":28973,"Ġ:čĊ":28974,"Ġfich":28975,"Ġdla":28976,"Ġshoe":28977,"\"),čĊ":28978,"ularity":28979,"_RESP":28980,"Weather":28981,"UIApplication":28982,".iterator":28983,"Ġaging":28984,".Parent":28985,"owie":28986,"(equal":28987,"ĠConv":28988,"/default":28989,"Ġmeasuring":28990,".prev":28991,".IsValid":28992,".Fat":28993,"ĠsÄĥ":28994,"keywords":28995,"without":28996,"Ġsovere":28997,"Ġexchanges":28998,"Ġmelt":28999,"Ġislands":29000,"ĠIntegr":29001,"Ġjumping":29002,"Ġgle":29003,"Ġjournalism":29004,"Ġdated":29005,"Localized":29006,"ĠRefresh":29007,"Particle":29008,"Ġaa":29009,"ĠSTRICT":29010,"Ġbod":29011,".Process":29012,"_AUTO":29013,"ĠPublished":29014,"every":29015,"Ġtechnological":29016,"lsx":29017,"Ġirrit":29018,"Additional":29019,"Ġdelimiter":29020,"_language":29021,"-area":29022,"boys":29023,"ĠTube":29024,"Ġwat":29025,"Ġmechanics":29026,"_owner":29027,"Spell":29028,"ĠStories":29029,".AppendLine":29030,"TableView":29031,"hem":29032,"stick":29033,"ollower":29034,"IFF":29035,"ĠUV":29036,"ollision":29037,"SUB":29038,"Ġcomparable":29039,"Ġdonde":29040,"sales":29041,"llvm":29042,"Ġ}],Ċ":29043,"OTTOM":29044,"ĠPurpose":29045,"Lab":29046,"Ġinterviewed":29047,"ois":29048,"asil":29049,".setId":29050,"ĠInstruction":29051,"-->":29052,"ĠModified":29053,"ationally":29054,"ĠMeeting":29055,"误":29056,"#region":29057,"Ġrouting":29058,".focus":29059,"ĠYouth":29060,"<":29348,"Ġunto":29349,"ologically":29350,"ĠMul":29351,"VIDIA":29352,"Ġslim":29353,"ĠCommissioner":29354,"(on":29355,"Ġunderneath":29356,"/db":29357,"vote":29358,"(Message":29359,"ĠPope":29360,"Defined":29361,"Ġswift":29362,"urf":29363,"Ġadapted":29364,"SEL":29365,"Ġrevenues":29366,"Ġdivine":29367,"=y":29368,"Gradient":29369,"_act":29370,"Ġ/*!<":29371,"Ġpolygon":29372,"ĠFDA":29373,"ĠCarr":29374,"atables":29375,"(stdout":29376,"Ġrefriger":29377,"Ġcoordin":29378,"avorites":29379,"ÑĪи":29380,"Ġcompassion":29381,"ĠPOSSIBILITY":29382,"-secondary":29383,"uracy":29384,"Ġcompromise":29385,"_AV":29386,"_os":29387,"Ġbeside":29388,"ĥĿ":29389,"Ġln":29390,".plugins":29391,"Capacity":29392,"alah":29393,".bin":29394,"ĠCRC":29395,"_balance":29396,"ĠflexDirection":29397,"Ġambit":29398,"Ġnickname":29399,"ĠForces":29400,"CLE":29401,"ĠShell":29402,"Ġsail":29403,"ĠWriter":29404,"ĠAlice":29405,"dw":29406,"ĠIndians":29407,"ĠMarshall":29408,"_SRC":29409,"Ġnormalized":29410,"ĠJag":29411,"ãĤĴ":29412,"zeit":29413,"rpc":29414,"ÃŃc":29415,".inline":29416,"Ġtravers":29417,"_numeric":29418,"Ġutilities":29419,"Ġevac":29420,"INPUT":29421,"ĉregister":29422,"MX":29423,"ĠCampbell":29424,"Ġdatasets":29425,"Ġdemanded":29426,"ĠinitialState":29427,"gan":29428,"Ġei":29429,"Unexpected":29430,"-web":29431,"trait":29432,",Y":29433,"ĠTodd":29434,"Ġskeleton":29435,"Ġoptimize":29436,"第":29437,"ĠUpon":29438,"ĠStObject":29439,"Ġaplic":29440,".'P":29478,"vron":29479,".UN":29480,"Ġpainter":29481,"izarre":29482,"Ġlav":29483,"Ġpom":29484,"preg":29485,"=function":29486,"(serial":29487,"ifica":29488,"uming":29489,"åľ°":29490,"ãģĤ":29491,"-op":29492,"UCH":29493,"ĠHend":29494,".propTypes":29495,"Ġyo":29496,"Ġroutines":29497,"Ġcaring":29498,"Sem":29499,"Ġreserves":29500,"Ġpriorities":29501,"redits":29502,"ISTR":29503,"ContentType":29504,"ĠSchw":29505,"/media":29506,"Ġestr":29507,"Ġclimbing":29508,"-week":29509,"cherche":29510,"sensor":29511,"ToArray":29512,"ĠMontreal":29513,"Ġclouds":29514,"ĠInjectable":29515,"ĠRice":29516,"Ġpropaganda":29517,"_provider":29518,"Ġindoor":29519,"Ġinaug":29520,"Ġdiplom":29521,"Ġmessaging":29522,"_mut":29523,"å¦Ĥ":29524,"Ġkw":29525,"ONS":29526,"arians":29527,"RPC":29528,")]čĊ":29529,"-ray":29530,"ĠSor":29531,"mall":29532,"Ġmarketplace":29533,"Ġvtk":29534,"Ma":29535,"ogan":29536,"igi":29537,"Ġsponsored":29538,"ĠDani":29539,".SEVER":29540,">'.$":29541,"multipart":29542,"ĠWol":29543,"ĠtableName":29544,"ĠUsername":29545,"BackgroundColor":29546,"Ġfright":29547,"_EMAIL":29548,"September":29549,"_vals":29550,"opia":29551,"Ġspotted":29552,"-Ch":29553,"ĠdataSource":29554,"/\"Ċ":29555,"екÑĤ":29556,"ĠRequestMethod":29557,"ĠReplace":29558,"-do":29559,"ahn":29560,"ĠPhD":29561,"].ĊĊ":29562,"NON":29563,"gement":29564,"ĠThr":29565,"Ġquietly":29566,"Ġtorture":29567,"Ġteas":29568,"ĠCY":29569,"Ġatr":29570,"development":29571,"-detail":29572,"Ġlighter":29573,"Ġarguing":29574,"Ġdeserves":29575,"Ġcurriculum":29576,"_CONTEXT":29577,"ÅĤy":29578,"HITE":29579,"ĉID":29580,"/uploads":29581,"Ġtits":29582,"reo":29583,"_drop":29584,".UTF":29585,"Ġpickup":29586,"Ġgrocery":29587,"ĠPure":29588,"Ġeasiest":29589,"Phil":29590,".feature":29591,"(\"*":29592,"Ġinvestor":29593,"tok":29594,"Ġjar":29595,"Los":29596,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":29597,".queue":29598,"-speed":29599,"Mal":29600,"umblr":29601,"ĠCONST":29602,"ĠHRESULT":29603,"ĠDance":29604,"(filePath":29605,"Ġattributed":29606,"à¥į":29607,"ĠBund":29608,"coins":29609,"Ġsão":29610,"Ġpir":29611,"personal":29612,"Ġprelim":29613,"Ġpropose":29614,"ĠTL":29615,"]])":29616,"ĠSubscription":29617,"ĠKre":29618,",len":29619,".FirstOrDefault":29620,")--":29621,"_products":29622,".GetBytes":29623,"Ship":29624,"Ġencrypt":29625,"ĠSG":29626,"ĠMyst":29627,"hir":29628,"Ġiterate":29629,"Ġintend":29630,".mockito":29631,"Ġchapters":29632,"(angle":29633,"ĠVlad":29634,"设":29635,"'.ĊĊ":29636,"ResponseBody":29637,"ĠAbd":29638,"deal":29639,"Ġbarriers":29640,"-outline":29641,"bill":29642,"ĠFalls":29643,"_second":29644,".include":29645,".ceil":29646,"Ġoccupation":29647,"phony":29648,".moveTo":29649,"ĠJennifer":29650,"ASTER":29651,";\"><":29652,"ĠEnabled":29653,"Ġterminate":29654,"ĠIo":29655,"lations":29656,"ĠTHEORY":29657,"Ġearliest":29658,"Ġrack":29659,"ĠScar":29660,"shake":29661,"chip":29662,"Ġuv":29663,"Ġalliance":29664,"пиÑģ":29665,"ĠGOODS":29666,"zione":29667,"ĠVI":29668,"Ġ{-":29669,"Ġfiltering":29670,"Ġmiscon":29671,".DockStyle":29672,"Ġbush":29673,"Ġjunk":29674,"æĮ":29675,"ĠQUE":29676,"Ġhooks":29677,"Ġfirmware":29678,"Ġmiddleware":29679,"dic":29680,"ĠOakland":29681,"Ġarrives":29682,"Payload":29683,"pixel":29684,"]|":29685,"ĠstartDate":29686,".PRO":29687,"_audio":29688,"Ġmidfield":29689,"igidbody":29690,"ĠSwiss":29691,"ĠClip":29692,"ĠDump":29693,"ĠTextBox":29694,"Ġgeh":29695,"yield":29696,"ods":29697,"Ġreferendum":29698,"Backend":29699,"ĠCream":29700,"Ġdominated":29701,"ĠArchive":29702,"Ġriders":29703,".prepareStatement":29704,"Ġquando":29705,"Ġchef":29706,"wiki":29707,"inel":29708,"ampling":29709,"(\"\\\\":29710,"Ġsag":29711,"_proxy":29712,"ãģķ":29713,"pdo":29714,".getElementsByTagName":29715,"Ġdemonstration":29716,"ĠNPC":29717,"Ġarchivo":29718,"endance":29719,"Ġefficiently":29720,"(actual":29721,".tableView":29722,"Ġmush":29723,"Ġbears":29724,"_threads":29725,"jas":29726,"ahun":29727,"Ġneural":29728,"Ġdesigning":29729,"ĠGDP":29730,"Ġlifted":29731,"缮":29732,"ĠJoint":29733,"ĠInclude":29734,"ĠGiants":29735,"Ġwithdrawal":29736,"ĠRent":29737,"native":29738,"ĠSeek":29739,"gression":29740,"_CPU":29741,"\\S":29742,"ĠShield":29743,"Ġsolic":29744,"Ġboom":29745,"yecto":29746,"Ġmanufacture":29747,"ĠâĢĭ":29748,"Ġbbox":29749,"Ġearthqu":29750,"ollectors":29751,":@\"%":29752,"Ġloops":29753,"Je":29754,"alking":29755,"ĠWhats":29756,"ĠBoys":29757,".book":29758,"ARGE":29759,"_pixel":29760,"Ġsuspects":29761,"ι":29762,"usp":29763,"ĠBMW":29764,"ieces":29765,"(person":29766,"å¼Ģ":29767,"é»":29768,"ĠPodcast":29769,"Ġbou":29770,"(Item":29771,"û":29772,"(Input":29773,"HttpGet":29774,"Ġburg":29775,")^":29776,"BOARD":29777,"*/,":29778,"Ġgulp":29779,"ĠBenn":29780,"Ġdecks":29781,".statusCode":29782,"Ġacute":29783,"Ġhug":29784,"ugu":29785,"Ġpled":29786,",\"%":29787,"hape":29788,"Ġзап":29789,"ĠMaine":29790,".real":29791,"Ġdalam":29792,"ĠMinor":29793,".Float":29794,"disp":29795,"Ġtl":29796,"Ġencount":29797,"=>$":29798,"Ġfg":29799,"tees":29800,"ĠRecomm":29801,"äl":29802,"Ġchemistry":29803,"Blocks":29804,"OID":29805,"Ġforex":29806,"ĠAppend":29807,"Ġ{*":29808,"ĠSupply":29809,"CGFloat":29810,"(bl":29811,"Ġate":29812,"adora":29813,"Ġgust":29814,"Associ":29815,">.Ċ":29816,"FETCH":29817,".serial":29818,"widgets":29819,"ardless":29820,"iefs":29821,"_FULL":29822,"ernetes":29823,"ĠPred":29824,"ØŃ":29825,"äºĭ":29826,"ubernetes":29827,"ĠLaura":29828,"Ġlabeled":29829,"Highlight":29830,"Ġannoying":29831,"/update":29832,"(description":29833,"Ġintimid":29834,"$c":29835,"\")))Ċ":29836,".AP":29837,"Ġ[]*":29838,"ĠEXIT":29839,".Host":29840,"ĠOPEN":29841,".sendMessage":29842,"_camera":29843,"_tile":29844,"Ġtherm":29845,"onomous":29846,"Ġdisadv":29847,"Ġnaar":29848,"indexOf":29849,"ĠPP":29850,".protocol":29851,"AFE":29852,"Ġtextures":29853,"################################################":29854,"umbai":29855,".stats":29856,"ĠGE":29857,"Ġie":29858,"ĠSTD":29859,"ĠMann":29860,".reflect":29861,"KB":29862,"Ġdive":29863,".wav":29864,"/*----------------------------------------------------------------":29865,"/settings":29866,".lifecycle":29867,"Ġdaughters":29868,"orus":29869,"uber":29870,"NING":29871,"stri":29872,"ĠTip":29873,"Ġzn":29874,"Ġswitched":29875,"inet":29876,"uffy":29877,"ĠTransportation":29878,"(conf":29879,"frica":29880,"ĠXL":29881,"ĠLead":29882,"_percent":29883,"__":29899,"permissions":29900,"ĠDetermine":29901,".Man":29902,"Ġadvances":29903,".InputStream":29904,"Ġstrongest":29905,"ĠeBay":29906,"Ġ#-":29907,"Ġdirname":29908,"ĠSMS":29909,"Ġmedications":29910,"Ġamended":29911,"Ġchurches":29912,"ĠImperial":29913,"$row":29914,"ĠMadison":29915,"ĠInsp":29916,"Ġaffair":29917,"Ġpsychology":29918,"vh":29919,"Ġseverity":29920,"âĢIJ":29921,"Ġstrips":29922,"AH":29923,"vertising":29924,"Ġconse":29925,"IMAGE":29926,"ĠStats":29927,"ĉsc":29928,".Cursor":29929,"Ġfreeze":29930,"sson":29931,"(xml":29932,"ĠSusan":29933,".tile":29934,"eded":29935,"ĠĠĠĠĉĉĉ":29936,"uelle":29937,"ĠMitchell":29938,"based":29939,"Operand":29940,"½æķ°":29941,"ĠFF":29942,"ĉstrcpy":29943,"ounces":29944,"ildo":29945,".executeQuery":29946,"Ġapproaching":29947,"ĠSeven":29948,"Ġnuts":29949,"Ġric":29950,"assignment":29951,"Ġcalculator":29952,"ĠMurphy":29953,"ĠBou":29954,"íĦ":29955,"Ġbutt":29956,"Ġticks":29957,"Projects":29958,"ilib":29959,".textColor":29960,"mov":29961,"_logo":29962,"(template":29963,"ĠINIT":29964,"ĠimageView":29965,"scriptions":29966,"ORITY":29967,"Consumer":29968,"Ġunprecedented":29969,"Ġtourist":29970,"Ġbron":29971,"Ġcontractor":29972,"Ġlicence":29973,"ĠNam":29974,"æ¯":29975,"(transform":29976,"_ATT":29977,"Pref":29978,"ĠGam":29979,"Ġvessels":29980,"Ġhav":29981,"Later":29982,".ToLower":29983,"Ġurls":29984,"Ġbreakdown":29985,"Ġpenalties":29986,"Ġfoster":29987,"ĠUE":29988,"Ġclue":29989,"comed":29990,"åIJįç§°":29991,"-main":29992,"Ġpts":29993,"Ġcounted":29994,"icts":29995,"/post":29996,"Ġgetattr":29997,"Ġping":29998,"ANCEL":29999,"Ġpec":30000,"Ñħод":30001,"antom":30002,"ĠBlueprint":30003,"ĠEventEmitter":30004,"Ġlä":30005,"æ²":30006,"Ġstraw":30007,"(comp":30008,"'une":30009,">N":30010,"-client":30011,"esModule":30012,"-base":30013,"Ġretreat":30014,"_simple":30015,"ĉĉĉĉĉĉĠ":30016,"fee":30017,"')čĊčĊ":30018,"ControlItem":30019,"Ġsubscribers":30020,"please":30021,"ĠEff":30022,"Ġpound":30023,"ĠBytes":30024,"ĠTea":30025,"_activity":30026,"Ġmaxim":30027,"Ġopcode":30028,"BSD":30029,".constant":30030,";}":30031,"ombres":30032,"Ġcareers":30033,").ĊĊĊĊ":30034,"Ġspreading":30035,"-expanded":30036,"ĠOrd":30037,"amarin":30038,"Ġmobility":30039,"Unfortunately":30040,"akk":30041,"NL":30042,"_redirect":30043,"ĠPG":30044,"ĠSensor":30045,"bol":30046,"tap":30047,"_MEMORY":30048,"ĠUIAlert":30049,"plitude":30050,"Website":30051,"ĠLogo":30052,"love":30053,"[ind":30054,"Ġaltogether":30055,"Ġwondered":30056,"Ġesper":30057,"ĠLiberal":30058,"Ġoss":30059,"Ġelit":30060,"Ġstiff":30061,"odox":30062,"_mentions":30063,"ĠDouglas":30064,"_pid":30065,"ĠCK":30066,"ĠinitWithFrame":30067,".blog":30068,"pkg":30069,"anghai":30070,"QUIRED":30071,"uu":30072,"Ġmkdir":30073,"ATAL":30074,"Ġunh":30075,"inces":30076,"sth":30077,"Ġhypothesis":30078,"Ġcata":30079,"ĠTB":30080,"ĠClar":30081,"Ġpredecess":30082,"Ġsituated":30083,"-world":30084,"))/":30085,"Ġheadlines":30086,".stat":30087,"Ġoutbreak":30088,"spath":30089,"_FLAGS":30090,"ĠServletException":30091,"Sun":30092,"FROM":30093,"ĠDir":30094,"ãĥ»ãĥ»ãĥ»":30095,"_coord":30096,"ĠOptim":30097,"Monitor":30098,".bit":30099,"XXX":30100,"Ġtodas":30101,"feld":30102,"ÑĢи":30103,"imir":30104,"Ġpolitically":30105,"Ġmolecular":30106,"Ġtraded":30107,"Ġ{{$":30108,"ĠSwedish":30109,"Ġ'@/":30110,"_REAL":30111,"Ġwarehouse":30112,"today":30113,",L":30114,"orp":30115,"false":30392,"Ġspa":30393,"ĠNear":30394,"ìķ":30395,"Ġintrig":30396,"_members":30397,"wave":30398,"Ġanalysts":30399,"_OS":30400,"edin":30401,"ĠFri":30402,"Ġretrieved":30403,"Regular":30404,"_obs":30405,"EXPORT":30406,"')}}\"":30407,"\"class":30408,"__((":30409,"bucket":30410,"Ġstro":30411,"ĠPatch":30412,"ystick":30413,"fulness":30414,"apos":30415,"Da":30416,"ĉĉĉĉĉĠĠĠ":30417,"Ġenrich":30418,"unordered":30419,"hole":30420,"Cong":30421,"';ĊĊ":30463,"STRUCT":30464,"QR":30465,"IDs":30466,"(arguments":30467,"_aux":30468,"(Event":30469,"_PRIVATE":30470,"ĠTrek":30471,"Ġdownloads":30472,"mutable":30473,"_STRUCT":30474,"(wx":30475,"Ġdomains":30476,"jspx":30477,"ĠViagra":30478,"Commands":30479,"Js":30480,".cfg":30481,"ContentPane":30482,"ĠEditText":30483,"à¥įà¤":30484,"Attach":30485,"ĠARM":30486,"positive":30487,"ĠGenerated":30488,"Ġseized":30489,"=:":30490,"Ġelectronics":30491,"ĠAppComponent":30492,"/',Ċ":30493,".equalsIgnoreCase":30494,"Doctrine":30495,"disk":30496,"ĠPolitical":30497,"CHO":30498,"":30584,"ĠBeauty":30585,"Ġ`<":30586,"Ġtouching":30587,"Ġ|--":30588,"ĉflag":30589,"normalize":30590,"Ġtrapped":30591,"Ġestablishing":30592,"/build":30593,"AJ":30594,"fy":30595,"-react":30596,"avn":30597,"RIPTION":30598,"Ġkut":30599,"ĠFashion":30600,"ĠInform":30601,"curities":30602,"{Ċ":30634,"Ġgarlic":30635,"Ġrepr":30636,"Ġreplies":30637,"(prop":30638,"Ġspirits":30639,"Ġinspire":30640,"Ġbasement":30641,".reject":30642,"Ġhints":30643,"Ġpolling":30644,"ĉĠĊ":30645,"_rating":30646,"Ġcath":30647,"avier":30648,"Ġcompressed":30649,"ĠVS":30650,"]'":30651,"Ġjudicial":30652,"ĠTrend":30653,"training":30654,"ESTAMP":30655,"ognition":30656,"Äģ":30657,"SENT":30658,"ventions":30659,"Ġconsultant":30660,"umph":30661,"ĠuserService":30662,",NULL":30663,"kh":30664,"Dear":30665,"_BAD":30666,"itations":30667,"Ġmetaph":30668,"'é":30669,"andise":30670,"-font":30671,".chart":30672,"Ġsg":30673,"_Controller":30674,".jpeg":30675,"ĠULONG":30676,"ĉgame":30677,"(ss":30678,"ĠMaj":30679,"ĉgo":30680,"ĠSad":30681,"ĠBerg":30682,"ĠMine":30683,"Pack":30684,"Ġresistant":30685,"ĠROM":30686,"Ġpeg":30687,"ĠStanford":30688,"ĠYahoo":30689,"Ġscaled":30690,"Ġlan":30691,"=[]":30692,"\"/>ččĊ":30736,"Ġsud":30737,"ĉbackground":30738,"Ġscholars":30739,"-muted":30740,"ará":30741,"Ġ=====":30742,"Ġ____":30743,"Creat":30744,"enever":30745,"/wp":30746,"ĠVPN":30747,"ErrorCode":30748,")],Ċ":30749,"(builder":30750,"ĠEnemy":30751,"Sensor":30752,"usa":30753,"Ġtriggers":30754,"Ġplayoffs":30755,"_REQ":30756,"Ġ(~":30757,"ĠBarry":30758,"Ġpermanently":30759,"ĠRUN":30760,"Ġbure":30761,".Fatalf":30762,"Ġchick":30763,"ĉpanic":30764,"psi":30765,"oka":30766,"éĢī":30767,">[":30768,"Ġunderstands":30769,"ĠJunior":30770,"ĠINFO":30771,"=mysqli":30772,"ustain":30773,"-source":30774,"serv":30775,"ĠCREATE":30776,".au":30777,"Ġsells":30778,"ĠĠĊĠĠĊ":30779,"Europe":30780,"zw":30781,"preh":30782,"ĠNSA":30783,"Ġxy":30784,"ิ":30785,"ĠBeyond":30786,"Instead":30787,"NonQuery":30788,"Ġarise":30789,"Ġavoided":30790,".emplace":30791,"_models":30792,"}),Ċ":30793,"Ġhid":30794,"Ġ&_":30795,".points":30796,".getWidth":30797,".Exec":30798,"Ġ////":30799,"ĠSessions":30800,"...\\":30801,"ĠColomb":30802,"Ġacceleration":30803,"restore":30804,"Ġile":30805,"obic":30806,"}Ċ":31296,"plaint":31297,"getText":31298,"Ġindividually":31299,"Ġcheckbox":31300,"UY":31301,"ĠLamb":31302,"Ġdysfunction":31303,"ĠLar":31304,"à°":31305,"ĠCreating":31306,"');ĊĊĊ":31307,"\"They":31308,"locations":31309,"_CORE":31310,"Interaction":31311,"umbnails":31312,"ĠPartner":31313,"brit":31314,"Ġlesser":31315,"ĠSlot":31316,"setAttribute":31317,"ĠWave":31318,".po":31319,"/store":31320,"Ġbrowsing":31321,"_pd":31322,"sume":31323,"sed":31324,"Curve":31325,"Ġplasma":31326,"Ġsuspicious":31327,"ìĿ¸":31328,"ĠBah":31329,"ĠExplicit":31330,"_CC":31331,".ClientSize":31332,"\\View":31333,"Ġsubstit":31334,"loon":31335,"ĠGAME":31336,"ĠBrid":31337,"Ľå»º":31338,"_User":31339,"Ġsquares":31340,"fone":31341,"Ġsacred":31342,"ughs":31343,"]interface":31344,"ĠThrow":31345,"ĠKirk":31346,"Ġempire":31347,"Ġassessed":31348,"Tax":31349,"ĠHeaven":31350,"-buffer":31351,"_STATIC":31352,"éné":31353,"-bordered":31354,"Ġpunct":31355,"(mode":31356,"Ġkeine":31357,"Sent":31358,"ĠCalcul":31359,"ĠEve":31360,"Ġstylish":31361,"Ġoils":31362,".TestCase":31363,"Ġtrademark":31364,"Ġliterary":31365,"Ġconcentrations":31366,"ĠRelations":31367,"(Class":31368,"Ġstdin":31369,"Ġvæ":31370,"backup":31371,".VERSION":31372,".AutoScaleDimensions":31373,"starter":31374,"Transactional":31375,"-panel":31376,"Studio":31377,"kc":31378,"ĠChamber":31379,"ĠSpiel":31380,"Ġrho":31381,"اÙĦ":31382,"!'":31383,".Attributes":31384,"Ġmurdered":31385,"apeutic":31386,"Ġintimate":31387,"ĠtextField":31388,"ĠBuffalo":31389,"dummy":31390,"\"%":31391,"ĠLiberty":31392,"obar":31393,"ĠTank":31394,"ĠPopular":31395,"ervisor":31396,"ĠIniti":31397,"ĠMall":31398,"ĠPrior":31399,"CAP":31400,"ĠClay":31401,"ĠCertificate":31402,".Lock":31403,"-strip":31404,"-driven":31405,"/all":31406,"ĠMessageBoxButtons":31407,"_SECRET":31408,"_pb":31409,"Ġrats":31410,"ाà¤":31411,"Ġnt":31412,".Router":31413,"_topic":31414,"Ġtennis":31415,"ĠPUBLIC":31416,"ĠActivatedRoute":31417,"Ġ',Ċ":31418,"Ġcostume":31419,"Ġjokes":31420,".Handle":31421,"ĉbyte":31422,"Ġflavors":31423,"(cc":31424,"Ġpersonas":31425,"ĉimage":31426,"ĠNazi":31427,"Ġgrammar":31428,"Ġúlt":31429,"Ġvalve":31430,"Ġvic":31431,"ĠRachel":31432,"_invalid":31433,"Prefs":31434,"stdint":31435,"(route":31436,"Ġhtmlspecialchars":31437,"Ġpeoples":31438,"pline":31439,"Ġnv":31440,"ĠQuant":31441,"oppers":31442,"ĠcurrentUser":31443,"ĠCatal":31444,"Ġreconc":31445,"Ġconjunction":31446,"lx":31447,"amburg":31448,"Ġinfluential":31449,"danger":31450,"inders":31451,"Ġ%@\",":31452,".configuration":31453,"osome":31454,".identity":31455,"Ġpicker":31456,"nost":31457,"ĠDIY":31458,"August":31459,"ablo":31460,"Leaf":31461,"ĠReco":31462,"cko":31463,"DOC":31464,"ĠHerm":31465,":any":31466,"ĠInterview":31467,"ĠTex":31468,"xfe":31469,"(work":31470,"Ġleap":31471,"Heading":31472,"Ġquarters":31473,"\\Bundle":31474,"reb":31475,"Perhaps":31476,"ĠGmbH":31477,"Birth":31478,"ĉsum":31479,"ĠWatson":31480,".nil":31481,"ç¡":31482,"{}ĊĊ":31483,"icaid":31484,"Getter":31485,"\"name":31486,"Ġ\"čĊ":31487,"_none":31488,"zm":31489,"acute":31490,"uesto":31491,"Ġsous":31492,"Ġrebuild":31493,"Ġnewspapers":31494,"ĠHaz":31495,"Ġkits":31496,"ifo":31497,"Blur":31498,"Ġsuited":31499,"-In":31500,"à¯":31501,"ĠKeith":31502,"ĠNorway":31503,"INIT":31504,"ireccion":31505,"ieties":31506,"_usage":31507,"ĠDoug":31508,"rise":31509,"Ġtrillion":31510,"imited":31511,"ĠREL":31512,"alic":31513,"Ġcriticized":31514,"theorem":31515,"Ġcease":31516,"Ġsidew":31517,"ĠTerry":31518,"Ġsubsidi":31519,"Ġfirmly":31520,"Ġaws":31521,"Ġhott":31522,"Ġdressing":31523,"badge":31524,"ĠApplications":31525,"è¿ĶåĽŀ":31526,"Ġlaughed":31527,"Ġhobby":31528,"Ġmusicians":31529,"Ġ*.":31530,".placeholder":31531,"Ġcounters":31532,"ĠCapitol":31533,"SDK":31534,"Ġhelmet":31535,"andbox":31536,"quit":31537,"Ġcriminals":31538,"Ġteenager":31539,"(update":31540,"Gl":31541,".selection":31542,"Ġdischarge":31543,"Ġpresenting":31544,"ufacturer":31545,"_UNKNOWN":31546,"Ġstressed":31547,"åύ":31548,"Proto":31549,"_correct":31550,"haus":31551,"Ġrenov":31552,"Ġfirearms":31553,"Ġtechnically":31554,"-browser":31555,"Ġcandy":31556,"Stroke":31557,"Ġexecutor":31558,"Ġoccurrence":31559,"ĠIPv":31560,"_INTERFACE":31561,"ĠRetrieve":31562,".bad":31563,"Exchange":31564,"Navbar":31565,"ĠKid":31566,"(getApplicationContext":31567,"_STOP":31568,"ĠBoss":31569,"Listeners":31570,"Ġshooter":31571,"ĠAlb":31572,"äch":31573,"Ġpix":31574,".keyCode":31575,"alone":31576,"Ġabsurd":31577,"ĠCum":31578,"ĠNewtonsoft":31579,"ikt":31580,"Ġlaughing":31581,"Ġcapitalism":31582,"reeNode":31583,"Tx":31584,"_QUERY":31585,".Sleep":31586,"(login":31587,"WebElement":31588,"Ġcelebrating":31589,"Ġdeprecated":31590,"Ġmaar":31591,"Ġartistic":31592,"_ASSOC":31593,"ĠBorderRadius":31594,"ĉwp":31595,"Ġsurvivors":31596,"Inner":31597,"-red":31598,"Ġprosecution":31599,"_pp":31600,"(\"$":31682,"Ġcomma":31683,"unchecked":31684,"graphics":31685,"rors":31686,"GROUND":31687,"(public":31688,"Ġcustomized":31689,"ĠArkansas":31690,"ĠRew":31691,"Ġexpiration":31692,"×ķ":31693,"ĠCul":31694,"Ġnons":31695,".Filter":31696,"Ġsenator":31697,"_definition":31698,"ashington":31699,"ymph":31700,"/J":31701,"Ġfuse":31702,"ramid":31703,"ĠSupplier":31704,"Ġautocomplete":31705,"Ġ}),":31706,".\"ĊĊĊ":31707,"_functions":31708,"ĉto":31709,".eval":31710,"ĠTObject":31711,"References":31712,"Ġheated":31713,"HAL":31714,"Ġ))}Ċ":31715,"}$":31716,"ĠBarr":31717,"_UNIT":31718,"+$":31719,"ĠgetValue":31720,"iped":31721,"chied":31722,"(vm":31723,"cue":31724,"_integer":31725,"_course":31726,"third":31727,"Ġrevised":31728,"**/Ċ":31729,"_DIRECT":31730,"OutOf":31731,"(\"(":31732,"ĠFeel":31733,"Ġreass":31734,"Ġsubtitle":31735,"peri":31736,"nf":31737,"Ġenjoys":31738,"Ġtreats":31739,")this":31740,"-tabs":31741,"ancers":31742,"Ġcontinent":31743,"Ġcardio":31744,"Ser":31745,".question":31746,"Ġphrases":31747,"Validators":31748,"Ġpopul":31749,"ĠlÃŃ":31750,"song":31751,"_INTERNAL":31752,"Ġadviser":31753,"Ġpuzz":31754,"Ġambitious":31755,"ĠTob":31756,"ĠDP":31757,"Ġpresidency":31758,"Ġsurrender":31759,"Ġwatches":31760,"_binary":31761,"ĠSoon":31762,"Ġcanada":31763,"(\"\")Ċ":31764,"]='":31765,"ĠBrandon":31766,"epsilon":31767,"rw":31768,".addChild":31769,".Copy":31770,"Principal":31771,"Photos":31772,"Ġmarginal":31773,"Ġbasics":31774,"eing":31775,"Must":31776,"_String":31777,"Ġole":31778,"Magento":31779,".customer":31780,"(prev":31781,"ล":31782,"Ġloyalty":31783,"Cog":31784,"Ġprotocols":31785,"ĠCompanies":31786,"Ġtheoretical":31787,"Ġaccessing":31788,"ĠZen":31789,".ones":31790,"attice":31791,"_world":31792,"zes":31793,"Ġtattoo":31794,"Ġmenos":31795,"Ġintersect":31796,"\"];ĊĊ":31797,"belie":31798,"Ġinactive":31799,".readline":31800,"-labelled":31801,".done":31802,"lickr":31803,"ĠWORK":31804,"Ġderivative":31805,"Ġdatabases":31806,"âĤĤ":31807,"Ġsx":31808,".isArray":31809,"Ġys":31810,"Ġpada":31811,"ĠBullet":31812,"(`/":31813,"isActive":31814,"ĠCGSize":31815,"(equalTo":31816,"ĠColumbus":31817,"Ġmarry":31818,"DEV":31819,"_limits":31820,"rones":31821,"IAS":31822,"Ġtau":31823,"mino":31824,"_Write":31825,"ĠWine":31826,"Ġ[['":31827,"ĠPull":31828,"riters":31829,"rients":31830,"Ġshifting":31831,"upp":31832,"_TIMER":31833,"ĠConditions":31834,"ấ":31835,"ĠOrders":31836,"ĠStrength":31837,"æīĢ":31838,"Ġvalidity":31839,"Ġfot":31840,"etur":31841,"Ġbolt":31842,"åĨħ":31843,"ĠAlong":31844,"oshi":31845,"Ġassumptions":31846,"Ġmagazines":31847,"_SPI":31848,"Ġpunt":31849,"_PRODUCT":31850,"Ġrelay":31851,"ĠJavascript":31852,".te":31853,"-es":31854,"Ġwidgets":31855,"(fs":31856,"\";":31923,"atching":31924,"ĠKnowledge":31925,"ĉThe":31926,";margin":31927,"lessness":31928,"opard":31929,"umatic":31930,"()));čĊ":31931,"Ġfals":31932,"(cache":31933,"TypeId":31934,"éĢļ":31935,"_choice":31936,"ĠGoth":31937,"ĠSites":31938,"MG":31939,"_border":31940,"Indices":31941,"Comparer":31942,"ĠRedistribution":31943,"Ġcloset":31944,"Ġversatile":31945,"Inputs":31946,"********************":31947,"Ġobesity":31948,"quiz":31949,"gra":31950,"(global":31951,"åĬ¡":31952,"Ġcollector":31953,"Ġkor":31954,"ovable":31955,"ADC":31956,"ĠEventHandler":31957,".nc":31958,"Ġplayback":31959,"ientos":31960,"_perm":31961,"_WARNING":31962,"ĠOlympics":31963,".norm":31964,"ĠBroadcast":31965,"_small":31966,"drive":31967,".iloc":31968,"Ġtyped":31969,"MEM":31970,"_cons":31971,"DMETHOD":31972,"Ġlun":31973,".distance":31974,"(par":31975,"poon":31976,"Ġbast":31977,"activities":31978,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":31979,":čĊčĊ":31980,"SER":31981,")&&":31982,"_lst":31983,"ĠPolish":31984,"Ġknocked":31985,"Ġfrustration":31986,"aukee":31987,"Ġphosph":31988,"iquid":31989,"_coeff":31990,"æŃ¤":31991,"Latest":31992,"ĠDust":31993,"Tipo":31994,"Ġmaintains":31995,"Ġmarsh":31996,"incinn":31997,"lbl":31998,"Care":31999,"Ġneighborhoods":32000,"_gpio":32001,"ĠArsenal":32002,"Dem":32003,"ĠWhe":32004,"_hook":32005,"Ġldc":32006,"ĠHarper":32007,"ĠBerkeley":32008,"Ġgraduated":32009,"Percent":32010,"Ġarriving":32011,"ĠAdventure":32012,"(scope":32013,"('*":32014,"quarter":32015,"ĠMarie":32016,"Speaking":32017,"_codegen":32018,"Ġimmun":32019,"caster":32020,"ãĤĮ":32021,"åķĨ":32022,"ĠDimensions":32023,".record":32024,"Ġtexto":32025,"ĠMichelle":32026,"Pending":32027,"(by":32028,"_PAR":32029,"ucht":32030,"bee":32031,".Thread":32032,"ampire":32033,"know":32034,"ĠClinical":32035,"ĠmarginBottom":32036,"Ġdistinguish":32037,".Full":32038,".undefined":32039,"ĠSequelize":32040,"############################################################################":32041,"Ġeducated":32042,"_OVER":32043,"åºı":32044,"ĠÂłĠÂł":32045,"_each":32046,"Ġurge":32047,"depart":32048,"Ġdonors":32049,"ĠAu":32050,"Ġbillions":32051,"Ġbelonging":32052,"_age":32053,"_Int":32054,"Ġsubstances":32055,"machine":32056,"!!!ĊĊ":32057,"Ġjsonify":32058,"ibbean":32059,"ĠCad":32060,"ĠendTime":32061,"Ġcycling":32062,"ĠUITextField":32063,"Ġleverage":32064,"Ġvanilla":32065,"eat":32066,"Launch":32067,"(pt":32068,"states":32069,"ĠControls":32070,"ĠRespons":32071,"ĠJake":32072,"Ġasleep":32073,"fortunate":32074,".nextLine":32075,"SizeMode":32076,"ìĿ¼":32077,"TestingModule":32078,"German":32079,"ĠInvestig":32080,".reverse":32081,"ĠBACK":32082,"(DateTime":32083,"Ġnonprofit":32084,"ĠExpect":32085,"Ġtanto":32086,"']),":32087,"ĉthe":32088,"Multiple":32089,"(getActivity":32090,"_WAIT":32091,"Ġjá":32092,"decor":32093,"levance":32094,"ĠGitHub":32095,"mination":32096,"_quantity":32097,".Scanner":32098,"ĠLion":32099,"éĶĻ误":32100,"Ġdre":32101,"Ġtantra":32102,"ĠcontentType":32103,"Ġfid":32104,"_alt":32105,"NSIndexPath":32106,"-pl":32107,"åĮĸ":32108,"Ġantibiot":32109,"tables":32110,"acial":32111,"ĠRegistry":32112,"Ġolive":32113,"igers":32114,"Ġsubscriber":32115,"_pres":32116,"ĠSyntax":32117,"Ġlovers":32118,".Byte":32119,"olders":32120,"_forward":32121,"always":32122,"Caption":32123,"Priv":32124,"ĠTampa":32125,"isateur":32126,"-labelledby":32127,"ĠToString":32128,"ĠìĤ¬":32129,"Ġinitiated":32130,"WF":32131,"Ġinstitutional":32132,"inject":32133,"ĠScr":32134,"Ġdoctrine":32135,"Ġspacious":32136,"isure":32137,"ĠAna":32138,"\"time":32139,"essaging":32140,"Ġcid":32141,"ĠNan":32142,"Ġincomplete":32143,"TAG":32144,"-build":32145,"December":32146,"Ġresidual":32147,"(PDO":32148,"ĠListen":32149,"Ġglyph":32150,"Ġgaps":32151,"nea":32152,".Rect":32153,"Ġsau":32154,"ĠPhotograph":32155,"Ġexecutable":32156,"ĠExpert":32157,"Coroutine":32158,"_sizes":32159,"ĠNL":32160,".isValid":32161,");}Ċ":32162,"-reg":32163,"Ġciting":32164,"cwd":32165,"ĠOttawa":32166,"ĠBatt":32167,"Ġrenewable":32168,"Ġpreliminary":32169,"Ġasylum":32170,"Ġwrist":32171,"Ġutiliz":32172,"Ġdetention":32173,"Fast":32174,"Ġange":32175,"incinnati":32176,"Ġsteering":32177,"ĠNaN":32178,"iosity":32179,"/page":32180,"Ġè¿":32181,"sterol":32182,"Ġdisg":32183,"(DB":32184,"ĠDESCRIPTION":32185,"Ġ_$":32186,"Ġobstacle":32187,"Ġbizarre":32188,"Ġextraction":32189,"_expected":32190,"Ġloses":32191,"ĠCelebr":32192,"ĠhtmlFor":32193,"Ġexploit":32194,"олÑĮзов":32195,"XYZ":32196,"Ġmagnet":32197,"amped":32198,"Ġatoms":32199,"Sources":32200,"pectives":32201,"Ñģли":32202,"Ġ=čĊ":32203,"Ġdare":32204,"ĠWalter":32205,"Ġbrightness":32206,"Ġannotations":32207,"ëı":32208,"iske":32209,"Schedule":32210,".images":32211,"rosso":32212,"Ġ\"..":32213,"gamma":32214,"Ġinstructor":32215,"Ġoverwrite":32216,"-am":32217,"Ġdevastating":32218,"ĠSaints":32219,"Ġhs":32220,"Ġbonuses":32221,"$output":32222,"ijd":32223,"(ActionEvent":32224,"monitor":32225,"Ġmattress":32226,"January":32227,".jp":32228,"Ġcaracter":32229,"Ġimpose":32230,"_rest":32231,"ĠSignature":32232,"Ġcoronavirus":32233,"ãģĬ":32234,"_compare":32235,"Measure":32236,"itated":32237,"elijk":32238,"igos":32239,"esar":32240,"Ġrushed":32241,"metry":32242,"_SEPARATOR":32243,"_WE":32244,"_ATTRIBUTE":32245,"Ġyaml":32246,"Ġspecs":32247,"ĠRah":32248,"pheric":32249,"ĠInvestment":32250,"äll":32251,"Ġappealing":32252,"Ġviewport":32253,"ç©":32254,"ĠmarginLeft":32255,"Ġsubtract":32256,"ĠEDIT":32257,"ĉArrayList":32258,"grading":32259,"ĠFailure":32260,"asper":32261,"EEK":32262,"(now":32263,")Ċ":32279,"Collision":32280,"ĠGreater":32281,"ĠRacing":32282,"alan":32283,"Ġmonetary":32284,",new":32285,"ĠSorry":32286,".Enable":32287,"ĠInstantiate":32288,"ollen":32289,"ë©´":32290,"ĠCalling":32291,"_hour":32292,"ADA":32293,"Ġshy":32294,")**":32295,"Ġ==>":32296,"Ġespecial":32297,"Ġinterpreted":32298,"!=\"":32299,"Ġpharmacy":32300,".single":32301,"ĠCialis":32302,"Ġparas":32303,".toUpperCase":32304,"ĠDemon":32305,"Prime":32306,"Ġrankings":32307,"Adding":32308,"_HASH":32309,"ĠExam":32310,"Ú©":32311,"ĠVictor":32312,"Okay":32313,"\"];čĊ":32314,"Ġfortune":32315,"ĠFETCH":32316,"expand":32317,".Interop":32318,"Ġbarn":32319,"æ¶Ī":32320,"uevo":32321,"Ġspeculation":32322,"âĶĢâĶĢâĶĢâĶĢ":32323,"ĠNu":32324,"ĠBlues":32325,"(fname":32326,"Ġinhabit":32327,"Ġ\\\"%":32328,"CES":32329,"ulario":32330,"_cr":32331,"Ġvalidated":32332,"Ġmidnight":32333,"anking":32334,"Ġincorporate":32335,"Ġpursuit":32336,"EXP":32337,"prime":32338,"Pid":32339,"-US":32340,"ĠNurs":32341,"ĠWheel":32342,"éĺ":32343,"Ġinp":32344,"Ġsupportive":32345,".member":32346,"ĠShot":32347,".CheckBox":32348,"Ġaffirm":32349,"Tor":32350,"FullYear":32351,"Ġconsiderably":32352,"credentials":32353,"_opts":32354,"Roll":32355,"(round":32356,"Ġcoment":32357,"_UART":32358,"Ġextending":32359,"RG":32360,"resultado":32361,"itu":32362,".getSession":32363,"Ġattraction":32364,"&D":32365,"$html":32366,"ĠJessica":32367,"ĠAssociate":32368,"añ":32369,"_ed":32370,"ĠLag":32371,"Ġorigins":32372,"())->":32373,"addEventListener":32374,"IALOG":32375,"åIJ¦":32376,".Compare":32377,"Album":32378,"ĠKu":32379,"\";ĊĊ":32423,"quisite":32424,"channels":32425,"/res":32426,"ĠAnalytics":32427,".appcompat":32428,"/to":32429,"ĠonError":32430,"(attr":32431,"IRM":32432,"Ġragaz":32433,"-as":32434,".Second":32435,"oriented":32436,"Ġdonn":32437,"Ġlightning":32438,"fid":32439,"ĠPle":32440,"ãģ¾ãģĻ":32441,"tro":32442,".True":32443,"Observable":32444,"×Ļ":32445,"umbing":32446,"Ġprospective":32447,"-filter":32448,"Ġpursuant":32449,"(points":32450,".Bind":32451,"Ġpalm":32452,"clearfix":32453,"ös":32454,"ĠGonz":32455,"Ġweaken":32456,"Drive":32457,"enido":32458,"lld":32459,"obox":32460,"anean":32461,"Got":32462,"ä¿Ŀ":32463,"Regex":32464,"æĥ":32465,"Ġsalad":32466,"assis":32467,"\"net":32468,"inheritDoc":32469,"ĠRV":32470,"quier":32471,"Ġclazz":32472,"Ä±ÅŁ":32473,"osterone":32474,"Ġairline":32475,".listdir":32476,"Ġdownloading":32477,"ĠPalm":32478,"waukee":32479,"<":32480,".BL":32481,"_INLINE":32482,"offs":32483,"<<(":32484,"_news":32485,"Ġchase":32486,"/><":32487,"Ġeuros":32488,"ĠEgyptian":32489,"ĠStainless":32490,"_BOOL":32491,"ĠGuild":32492,"ĠDynam":32493,"[indexPath":32494,"Ġï":32495,"Ġmemorable":32496,"ĠChampion":32497,"ResourceManager":32498,".Login":32499,"ĠFormer":32500,"yped":32501,"Ġlleg":32502,";\",":32503,"DWORD":32504,"Ġtaxi":32505,"Ġbombs":32506,"rah":32507,".tags":32508,"_tests":32509,"stones":32510,"âĢĿ)":32511,"[g":32512,"rtype":32513,"Ġvu":32514,"Ġhostile":32515,"Chars":32516,"ĠPatriots":32517,"/status":32518,"());Ċ":32872,"ajÄħ":32873,"_OCC":32874,"Ġplanets":32875,"æŁ¥":32876,"ĠDublin":32877,"Ġserie":32878,".printf":32879,"deep":32880,"`)":32881,"Ġ\\$":32882,"Ġμ":32883,"_VIDEO":32884,"endors":32885,"ĠCrypto":32886,"Far":32887,".Transparent":32888,".TR":32889,"iasm":32890,"_training":32891,"Ġteaches":32892,"ĠBelt":32893,"Ġlimiting":32894,"ĠKath":32895,"ĠIndexPath":32896,"Ġachievements":32897,"Ġserá":32898,"interopRequire":32899,"Ġdisse":32900,".If":32901,"arming":32902,"ulsion":32903,"Po":32904,"_DETAIL":32905,"Prototype":32906,"ĠCAL":32907,"Ġagrees":32908,".vo":32909,".ExecuteNonQuery":32910,"ĠTopic":32911,"Ġ'{}":32912,"Arm":32913,"Ġecc":32914,"Mag":32915,"Ġserialized":32916,"ĉconn":32917,"cached":32918,"=tf":32919,"ĠByteArray":32920,"protobuf":32921,"varchar":32922,"ĉASSERT":32923,"Ġliste":32924,"_trigger":32925,"·¸":32926,"Feel":32927,"Tahoma":32928,"ĠLik":32929,"Ġstructured":32930,"ergus":32931,".Initial":32932,"_ge":32933,"cljs":32934,".contact":32935,"Ġandere":32936,"$stmt":32937,"_CURRENT":32938,"ĠDiscover":32939,"$res":32940,"formatter":32941,"Ha":32942,"vangst":32943,"Ġemerge":32944,"ãĢĤâĢĿ":32945,"ĠCabinet":32946,"-square":32947,"éĥ¨":32948,"Ġrage":32949,"ĠAJ":32950,"ĠVT":32951,"shadow":32952,"ĠFaith":32953,"enames":32954,"pretty":32955,"hasil":32956,"party":32957,"Ġvarchar":32958,"Ġfotos":32959,"Ġalum":32960,"ĠBelgium":32961,".ylabel":32962,"Ġdej":32963,"_numbers":32964,"Ġhu":32965,".setAdapter":32966,"ĠUsually":32967,"(sample":32968,".Shared":32969,"Ġbooked":32970,"Ġ>>=":32971,"Ġminerals":32972,"\">":32991,"prog":32992,"boo":32993,"_md":32994,"_pack":32995,"(express":32996,"utz":32997,"\\Auth":32998,",id":32999,"ĠChile":33000,"actice":33001,"Ġrecruitment":33002,"Ġposes":33003,"Ġvulnerability":33004,"instanc":33005,"orum":33006,"dess":33007,"Ġxl":33008,"%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%":33009,"(fig":33010,"Ġdeleting":33011,".del":33012,")')Ċ":33013,"ĠWeekly":33014,"???":33015,"(strcmp":33016,"smith":33017,"Ġpursuing":33018,"-so":33019,"ĠApps":33020,"/'Ċ":33021,"Ġdecis":33022,"FORE":33023,"Everyone":33024,"Ġlanes":33025,"Virtual":33026,".attach":33027,"(Log":33028,"ĠMedicaid":33029,"(Path":33030,"ĠTurner":33031,"/application":33032,"Ġportrait":33033,"Ġoppose":33034,"checkout":33035,"Ġfinishes":33036,"_ME":33037,"Barrier":33038,"Song":33039,"VAR":33040,"Earlier":33041,"rella":33042,"Ġhast":33043,"azar":33044,"Ġpulls":33045,"ngx":33046,"Ġinspiring":33047,"ÑĥÑİ":33048,"-direction":33049,"Ġexplosive":33050,"ĠcreatedAt":33051,"sto":33052,"Ġwheat":33053,"ĠBuilt":33054,"'ai":33055,"Ġtracked":33056,"hammad":33057,"RowAtIndexPath":33058,"_heap":33059,"Due":33060,"Ġconnects":33061,".publish":33062,"emu":33063,"Ġbullets":33064,"BAR":33065,"olate":33066,"Ġinternally":33067,"Ġcatching":33068,"-password":33069,"ouched":33070,"æĢ§":33071,"eous":33072,"Ġxrange":33073,"Quality":33074,"vv":33075,"Manage":33076,"(($":33077,"acements":33078,"ĠBrothers":33079,"ĠHEAD":33080,"ĠUnsupported":33081,"san":33082,"esi":33083,"***Ċ":33084,"Ġadaptation":33085,"ĠWorker":33086,"']/":33087,".savefig":33088,"(trans":33089,"ج":33090,"nee":33091,"Correct":33092,"...\")Ċ":33093,"Ġsubmitting":33094,"-path":33095,"ĉlast":33096,"issan":33097,".xlabel":33098,"ĠSepar":33099,"/no":33100,"_best":33101,"ĠMills":33102,"_sock":33103,"(flag":33104,"Ġdestinations":33105,"emption":33106,"ĠFAIL":33107,"åĴĮ":33108,"Ġrp":33109,"fact":33110,"ĉlen":33111,"DAY":33112,"Ġseiz":33113,"_dst":33114,"lip":33115,".Linear":33116,"ĠBasket":33117,"$t":33118,"$i":33119,"-brand":33120,"ĠNeil":33121,"ĠEq":33122,"Ġthou":33123,"ogene":33124,"Ġscholarship":33125,"æĽ´":33126,"Ġswo":33127,"aginator":33128,"eni":33129,"(book":33130,"Ġblink":33131,"thus":33132,"ĠcancellationToken":33133,"ĠPalestinians":33134,"Ġprofitable":33135,"Ġbackpack":33136,"enson":33137,"true":33284,"ĠNYC":33285,"Ġbored":33286,"ĠDetect":33287,"Ġappar":33288,"Ġjeans":33289,"ĠTak":33290,"IOD":33291,"ĠHorse":33292,"(FILE":33293,"(?":33294,"rique":33295,"optimizer":33296,"nat":33297,"loys":33298,"ĉToken":33299,"oubted":33300,"uess":33301,"ocoa":33302,"DataMember":33303,"_POWER":33304,"classList":33305,"PushButton":33306,"ĠWiFi":33307,".Stream":33308,".guild":33309,"Ġnog":33310,"ĠPortugal":33311,"ĠUnter":33312,"Primitive":33313,"boss":33314,"ĠDeutsch":33315,"Ġerotic":33316,"Ġstrconv":33317,".TryParse":33318,"Ġgrams":33319,".Success":33320,"_pk":33321,"ĠHarvey":33322,"-minded":33323,".country":33324,"[]\"":33325,"Ġangel":33326,"Ġbeats":33327,"ĠVor":33328,"ilio":33329,".master":33330,"something":33331,"ĠPACK":33332,"(if":33333,"RequestBody":33334,"Ġantes":33335,"/widget":33336,"Ġmodo":33337,"ĠAW":33338,"finder":33339,"Ġoptimized":33340,"Ġmissiles":33341,"NB":33342,"ĉinternal":33343,"tex":33344,"ĠSri":33345,"Ġdamaging":33346,"ĠMais":33347,"-Allow":33348,"ĠZh":33349,"-alt":33350,"Ġ));ĊĊ":33351,"èī":33352,"Ġinfluences":33353,"Ġcatal":33354,"_REGISTER":33355,"ĠAPIs":33356,"-century":33357,"Ġbiology":33358,"ĠActual":33359,"Ġheels":33360,"TRACE":33361,"_DIG":33362,"Dataset":33363,"ĠMatter":33364,"Ġclassifier":33365,".wikipedia":33366,"ĠRogers":33367,"Ġdonated":33368,"rawler":33369,"enen":33370,"Ġcasinos":33371,"ortal":33372,"Ġprive":33373,"spe":33374,"ducers":33375,".ep":33376,"Ġgrasp":33377,"acji":33378,"Ġdairy":33379,"Ġbuses":33380,".comm":33381,".ins":33382,"ĠIRS":33383,"ĠBeer":33384,"adc":33385,"oard":33386,"_MET":33387,"Ġ'+'":33388,"rans":33389,"Ġkinda":33390,"ĠâĶĤ":33391,"ĠMaur":33392,"аг":33393,"Ġbandwidth":33394,"ibus":33395,"ĠDifferent":33396,"(mat":33397,"ĠResume":33398,"_UNS":33399,"establish":33400,"Ġfonction":33401,"Subscription":33402,"_company":33403,"Ġlightly":33404,".confirm":33405,".yaml":33406,"ĠBoost":33407,"Commerce":33408,"-template":33409,"_DELAY":33410,"ĠHI":33411,"Ġnavig":33412,"(Sender":33413,"ĠHS":33414,"_\"+":33415,"ĠREQUEST":33416,"Ġwifi":33417,"=\"\"Ċ":33418,"])->":33419,"Ġrope":33420,"Ġviolated":33421,"Ġglance":33422,"ĠKurd":33423,"Ġè®":33424,"deck":33425,"ĠISBN":33426,"Ġinfect":33427,"ĠFoo":33428,"Ġgetter":33429,"Ġtener":33430,"appe":33431,".hh":33432,"_hot":33433,"\".$":33643,"Ġrelies":33644,"(Console":33645,"International":33646,"->{$":33647,"Mid":33648,"Ġdissert":33649,"dds":33650,"Ġdeposits":33651,"ĉdriver":33652,"#ga":33653,"prising":33654,"println":33655,"Ġpresenter":33656,"Ġmines":33657,"CSS":33658,"ĠDual":33659,"(!(":33660,"Ġkam":33661,"ĠisLoading":33662,"ĠProtect":33663,".upper":33664,"arium":33665,"]:ĊĊĊ":33666,"Yii":33667,"-shirt":33668,"ĠIMAGE":33669,"_colors":33670,"Ġurgent":33671,".Container":33672,"!(Ċ":33673,"Saturday":33674,"Ġsocieties":33675,"ĠThan":33676,"ĠCod":33677,"=@":33678,"Ġattachments":33679,".mobile":33680,"Ġspite":33681,"Ġbounce":33682,"rawl":33683,"instancetype":33684,"ĠTruck":33685,"Ġmanipulation":33686,"(Config":33687,"-inst":33688,"Ġstor":33689,"itution":33690,"PreferredGap":33691,"ĠmainAxisAlignment":33692,"Ġlistened":33693,"'''ĊĊ":33694,"ottage":33695,"-project":33696,".APPLICATION":33697,"ĉroot":33698,"Ġwhit":33699,"Ġbilder":33700,"Ġker":33701,"Ġappliances":33702,"rowave":33703,"ìĿĢ":33704,"ematics":33705,"ĠOrg":33706,"oping":33707,"_SEARCH":33708,"Ġcham":33709,"addContainerGap":33710,"Ġ().":33711,"ĠArrow":33712,"Illegal":33713,"Currently":33714,"Ġusa":33715,"Ġpasswords":33716,"Ġrenown":33717,"avern":33718,"ĠEvil":33719,"Ġconcat":33720,"Ġduo":33721,"Ġvale":33722,"ĠBean":33723,"Ġindicators":33724,"cmath":33725,"ĠPump":33726,"November":33727,"ificant":33728,"_DOMAIN":33729,"regar":33730,"ĠPortal":33731,"\"$":33732,"Ġformerly":33733,"\"]:Ċ":33734,"ĠVisibility":33735,".getElementsByClassName":33736,"_RED":33737,"Ġchampions":33738,"à´":33739,"Valor":33740,"_es":33741,"*a":33742,"-repeat":33743,"Band":33744,".stage":33745,"Ġbureauc":33746,"Cnt":33747,"eten":33748,"-function":33749,"Ġmuito":33750,"PID":33751,"_editor":33752,"Ġcrashed":33753,"dead":33754,"kat":33755,"agh":33756,"ĠEXT":33757,"asser":33758,"-small":33759,"Ġrealiz":33760,"(Entity":33761,"ús":33762,"ĠActually":33763,"ĠElite":33764,"Ġhelm":33765,"(nonatomic":33766,"asher":33767,"Community":33768,"alleng":33769,"iry":33770,"ĠGrowth":33771,"Ġsue":33772,"Ġfrequencies":33773,"_descriptor":33774,".Attribute":33775,"Ġrecipients":33776,"_NS":33777,"/\"+":33778,"iban":33779,"Ġathlete":33780,"ĠIgn":33781,"_DMA":33782,"(ds":33783,"ĠRequirements":33784,"ADI":33785,"erez":33786,"\\Admin":33787,"braska":33788,"ĠRust":33789,"Relation":33790,"COD":33791,"ĠVERSION":33792,"emma":33793,")){":33794,".Duration":33795,"ĠCamb":33796,"-logo":33797,"Ġreadable":33798,"Ġcreators":33799,"()];Ċ":33800,"UpDown":33801,"-half":33802,".getMonth":33803,"(sf":33804,"Pic":33805,"Ġhunger":33806,".tx":33807,"Ġexceeded":33808,"_seed":33809,"(^":33810,"_sk":33811,".perform":33812,"Ġ>::":33813,"Ġmongo":33814,"=float":33815,"bindParam":33816,"Smart":33817,"ifa":33818,"Ġsecurities":33819,"Ġprejud":33820,"Ġ,\"":33821,"Ġcorps":33822,"Ġvra":33823,"amacare":33824,"iterr":33825,"(Media":33826,"uche":33827,"Ġcob":33828,"Ġliber":33829,".geometry":33830,"Locator":33831,"Ġsliding":33832,"Ġsurgical":33833,"_CUR":33834,"Ġconsect":33835,"[*":33836,"ĠResort":33837,"Stub":33838,"_DOUBLE":33839,"ĠSoph":33840,"Ġelectoral":33841,"_disable":33842,"ĠÑģо":33843,"ĠLightning":33844,"Ġmentions":33845,"ocy":33846,"Ġleaked":33847,"Ġrelaxing":33848,"Presenter":33849,"vsp":33850,"Ġguilt":33851,"=-=-":33852,".reply":33853,"ĠMirror":33854,"Camp":33855,"Ġ+#+#+#+":33856,"Ġ+#+#+#+#+#+":33857,".Author":33858,"Ġdirective":33859,"-hook":33860,"íĦ°":33861,"}ĊĊĊĊĊ":33862,"@pytest":33863,"_rand":33864,"mis":33865,"Ġcolorful":33866,"uje":33867,"lasses":33868,"ĠClasses":33869,".have":33870,"%),":33871,"é¢ĺ":33872,"Ġdisturbing":33873,"substring":33874,"ĠKoh":33875,"Invest":33876,"purchase":33877,"Ġrecycling":33878,"ĠART":33879,"ierarchy":33880,"Ġfps":33881,".checkBox":33882,"íķ´":33883,"_material":33884,"ducation":33885,"Ġfw":33886,"udit":33887,"Ġreviewing":33888,"ĠSid":33889,"Syntax":33890,"ĠWritten":33891,"argar":33892,"UME":33893,"/q":33894,"Classifier":33895,"Official":33896,"Ġjazz":33897,"Ġomega":33898,"Physics":33899,"Ġlugar":33900,"_accessor":33901,".commands":33902,"Ability":33903,"ĠBatch":33904,"RAM":33905,"Ġencounters":33906,".Qu":33907,"BYTE":33908,"ĠDistribution":33909,"Ġuso":33910,"ĠRecovery":33911,"approved":33912,"Ġdenial":33913,"/share":33914,"LinkedList":33915,")čĊčĊčĊ":33916,"uddy":33917,"Ġfines":33918,"Ġry":33919,"Unicode":33920,"ĉrender":33921,"Ġpremises":33922,"Ġpon":33923,"aliases":33924,"/Foundation":33925,"cuda":33926,"ĠCock":33927,",:)":33928,"(folder":33929,"Ġméd":33930,"drag":33931,"Ġtalents":33932,"ĠĠĠĊĊ":33933,"еÑģÑĤв":33934,"mob":33935,".yml":33936,"Ġaster":33937,"Ġdiscre":33938,"goal":33939,"ĠGTX":33940,"ĠSUCCESS":33941,"ĠLONG":33942,"(find":33943,"Ġsingular":33944,"_sz":33945,"ĠEthereum":33946,"..Ċ":33947,"Ġirres":33948,"')){Ċ":33949,"Ġministers":33950,"Steps":33951,"iversal":33952,"ĠNevertheless":33953,"-led":33954,"Ġ(%)":33955,"ç¡®":33956,"Ġtimezone":33957,"Ġstranger":33958,"(render":33959,"Ġshutil":33960,"Ġmph":33961,"Ġtrio":33962,"ppy":33963,"Ġpredomin":33964,"Ġendors":33965,"ĠRussians":33966,"ĉrow":33967,"Ġwizard":33968,".serialize":33969,"Ġcomplained":33970,"Ġsido":33971,"Ġdelighted":33972,"-me":33973,"ĠRav":33974,"Human":33975,"adays":33976,"recv":33977,"Working":33978,"Jump":33979,"ĠÃ¥r":33980,"ĠAutomatic":33981,"_Base":33982,"æł¼":33983,"aurants":33984,"¯":33985,"æ¸":33986,"(CType":33987,"IFI":33988,"(amount":33989,"Ġbelieving":33990,"=mysql":33991,"Ġfir":33992,"Ġrestoration":33993,"ereco":33994,"Т":33995,"_'+":33996,"Ġebook":33997,"Ġdebris":33998,"(inputs":33999,"AYOUT":34000,"Ġscreaming":34001,"avia":34002,"lander":34003,"Ġdistress":34004,"Ġassembled":34005,"ĠAvoid":34006,"(thread":34007,"ĠRPC":34008,"_EXIT":34009,"(queue":34010,"иÑģÑĤ":34011,"Dll":34012,"Ġskull":34013,"_pub":34014,"chez":34015,"minate":34016,"ensen":34017,"Ġinsane":34018,"bounds":34019,"ĠRosen":34020,"Ġconditioning":34021,"processed":34022,"videos":34023,"four":34024,".Conv":34025,"|;Ċ":34026,"Personal":34027,"cerpt":34028,":UIControlStateNormal":34029,"Ġdoses":34030,"ĠKarl":34031,"ĠFrequ":34032,".BASE":34033,"ĠVote":34034,"Ġconcurrent":34035,"ĠMessageBoxIcon":34036,"ĠÃĸ":34037,"ĠDubai":34038,"ĠRetail":34039,":number":34040,"ĠObserver":34041,"ĠBigInteger":34042,"_origin":34043,"_WORK":34044,"Frames":34045,"Ġnotably":34046,".âĢľ":34047,"Ġtropical":34048,"Ġniche":34049,"amina":34050,".sys":34051,"(tokens":34052,"modify":34053,"osit":34054,"strom":34055,"ĠComics":34056,"OPTION":34057,"Ticket":34058,"Ġfactories":34059,"Ġdisput":34060,"_File":34061,"ĠFinn":34062,"eee":34063,"ĠDiscord":34064,"_money":34065,".tpl":34066,"_safe":34067,"LB":34068,"Ġglut":34069,"JK":34070,".flow":34071,"-cont":34072,"gos":34073,"Ġhorizon":34074,"ĠRush":34075,"::*":34076,"Pipe":34077,"ulla":34078,"borough":34079,"heimer":34080,"(move":34081,"(Text":34082,"});čĊčĊ":34083,"welcome":34084,"ĠComponents":34085,"Ġgovernance":34086,"closed":34087,"ĉmargin":34088,"Ġlaundry":34089,"ĠTerminal":34090,"izards":34091,".âĢĶ":34092,".remote":34093,".radius":34094,"ĠQuebec":34095,"Ġdh":34096,"Tech":34097,"ĠMist":34098,"seller":34099,"_literal":34100,"Ġgenius":34101,"Ġbrains":34102,"gem":34103,"ĠMeasure":34104,"Ġcatast":34105,"rance":34106,".TextField":34107,"Ġconsuming":34108,"Ġ'\\''":34109,"oubtedly":34110,"ĠCertain":34111,"Ev":34112,"erti":34113,"being":34114,"Experience":34115,"Ġ//[":34116,"ĠArabic":34117,"ĠCrist":34118,"ĠAzure":34119,"Ġhora":34120,"ladesh":34121,"\\Blueprint":34122,"dar":34123,".rel":34124,"Ġsuprem":34125,"ĠReagan":34126,"ĠAttributes":34127,"-sidebar":34128,"ĠuseStyles":34129,"ĠAirlines":34130,"Ġhills":34131,"/xhtml":34132,"vinc":34133,"_mock":34134,"ĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":34135,"ĠPill":34136,".LayoutStyle":34137,"ĠCommander":34138,"]<":34139,"signature":34140,"Ġ{}čĊ":34141,"Ġhatred":34142,"Ġëĭ":34143,"olesterol":34144,"Ġ********":34145,"ancellor":34146,"crop":34147,"TIM":34148,"ĉĉĊĊ":34149,"ysqli":34150,"uitive":34151,"ĉunset":34152,"_sel":34153,"Ġmenus":34154,"tick":34155,"Ġconstitute":34156,"ĠElements":34157,"ĠRedis":34158,"aggio":34159,"_fp":34160,"_depend":34161,"emas":34162,"CAST":34163,"orange":34164,"jon":34165,"ĠEmily":34166,"Ġpotatoes":34167,"Ġreceptor":34168,"ĠElectronic":34169,"ĠLights":34170,"Ġcombining":34171,"ĠSomeone":34172,"Ġ########.":34173,"ĠTOD":34174,"/show":34175,"Xd":34176,".\"'":34177,"afx":34178,"Ġtragic":34179,"Styled":34180,"ĠMarco":34181,"Gallery":34182,"dale":34183,".âĢĿĊĊĊĊ":34184,"érie":34185,"/service":34186,"äºĨ":34187,"Ġambient":34188,"_SETTINGS":34189,".Adapter":34190,"lene":34191,"Ġtravels":34192,"Notice":34193,"Ġcleans":34194,"ĠFem":34195,"chair":34196,"Ñĥн":34197,"/my":34198,"_bad":34199,"ĠEconomics":34200,"ISA":34201,"_CNT":34202,"(Menu":34203,"äºİ":34204,"ĠRidge":34205,"Ġlengthy":34206,"Dot":34207,"Ġjumps":34208,"Ġhey":34209,"$pdf":34210,"Ġworm":34211,"Ġsut":34212,"Ġsher":34213,"iamo":34214,"ĠCalc":34215,"trieve":34216,"Ġcops":34217,"ĠChrom":34218,"Ġregulated":34219,"reatment":34220,"ĠHigher":34221,"oks":34222,"Ġdeze":34223,"LOCATION":34224,"ongsTo":34225,"Ġfinite":34226,"Ġvaries":34227,"Ġpositioned":34228,"'il":34229,"éĩij":34230,"Ġhike":34231,"(done":34232,"playlist":34233,"Ġada":34234,"Ġcoastal":34235,"ĠNancy":34236,".DateTimeField":34237,"CppCodeGen":34238,"ĠSimilarly":34239,"reur":34240,"ĠContr":34241,"ĠHidden":34242,"ĠBeta":34243,"atched":34244,"_install":34245,".Output":34246,"Lookup":34247,"ĠRichmond":34248,"quared":34249,"Ġmanga":34250,"-controls":34251,"ĠBernard":34252,"Large":34253,"Ġslices":34254,"Ġoffence":34255,"ĠMega":34256,"Ġestar":34257,"Ġjoints":34258,"Ġsumm":34259,"_platform":34260,"Buff":34261,".addSubview":34262,"Ġretained":34263,"Letter":34264,".dim":34265,"Ġessere":34266,"ĠScaffold":34267,"EXPECT":34268,"ĉRE":34269,".longitude":34270,"ünd":34271,"Ġstatue":34272,".addWidget":34273,"ĠCaribbean":34274,"addPreferredGap":34275,"ilde":34276,"UILabel":34277,"ĠOpport":34278,"Ġimperial":34279,"ursion":34280,"Ġmandate":34281,"Ġpromotional":34282,"Ġvk":34283,"iaÅĤ":34284,"Ġpyl":34285,"ĠCreation":34286,"озд":34287,"Ġsimpler":34288,".what":34289,"ĠRecent":34290,"Storm":34291,".quantity":34292,"ĠLov":34293,"\"-":34294,"ubbles":34295,"_notification":34296,"(world":34297,"urger":34298,"*(-":34299,":\"Ċ":34300,"hm":34301,"anship":34302,"ĠAlmost":34303,"Ġmotorcycle":34304,"_fee":34305,"Ġabsorb":34306,"ĠVincent":34307,"Ġsounded":34308,"ÃŃst":34309,"Ġpharmaceutical":34310,"htag":34311,"ĠKindle":34312,"italize":34313,"ĠEmperor":34314,"oustic":34315,"Ġspecialists":34316,"åħ¬":34317,"BorderStyle":34318,"/\\":34319,"RELATED":34320,"(',',":34321,"(expr":34322,"Ġht":34323,"åįĪ":34324,"_Create":34325,"Ġspecially":34326,"Ġ[];čĊ":34327,"Ġheel":34328,"Ġsept":34329,"_arch":34330,"(initial":34331,"%.ĊĊ":34332,"\\\",\\\"":34333,"Ġdiscusses":34334,"Ġupt":34335,"Ġ[&":34336,"Ġmanus":34337,".hand":34338,"ĠMAIN":34339,"ĠDenmark":34340,"Ġ],čĊ":34341,"Ġcryst":34342,"Ġnack":34343,"Coords":34344,"_inner":34345,"Ġmidst":34346,"Ġawake":34347,"ĠÐŀ":34348,"-break":34349,"ÃŃvel":34350,"_PASS":34351,"ĠParams":34352,"Ġdetr":34353,"Ġspider":34354,"ĠConcept":34355,"Ġprend":34356,"CHED":34357,".Exit":34358,"Ġpopulated":34359,"Ġvirtue":34360,"_SESSION":34361,"Ġnouvel":34362,"oauth":34363,"ĠданнÑĭ":34364,"rink":34365,".HeaderText":34366,"aturated":34367,"Ġerst":34368,"Ġåħ":34369,"à¥ĩ":34370,"_visible":34371,"eyer":34372,"Ġliable":34373,"Ġdebe":34374,"Ġbw":34375,"{-#":34376,"_WIN":34377,"dfs":34378,"Hover":34379,"ĠPUT":34380,"-angle":34381,"Ġnoble":34382,"Ġtraces":34383,"encv":34384,"ĠuserData":34385,"_ins":34386,"ĠSuz":34387,"Ġnewsletters":34388,"ĠModi":34389,"Ġentrepreneurs":34390,"Ġtribute":34391,"Ġrumors":34392,"Ġrr":34393,"ĠQuarter":34394,"ê³ł":34395,"Ġfeeds":34396,"óg":34397,"Ġenvelope":34398,"Ġlear":34399,"Ġkø":34400,"developer":34401,"Similar":34402,":\")Ċ":34403,"subscription":34404,"Modifier":34405,"italic":34406,"Ġnasty":34407,"Ġtermination":34408,"Ġcharming":34409,"ĠâŁ":34410,"tons":34411,".trace":34412,"hots":34413,"ĠUR":34414,"Mont":34415,"Ġjustified":34416,"ĠGang":34417,"inea":34418,"Ġbog":34419,"(ap":34420,"_$":34421,"Ġcontamin":34422,".Dot":34423,"ĉDebug":34424,"(exports":34425,"Ġpaired":34426,"ĠAssignment":34427,"Ġautomobile":34428,"ĵį":34429,"Ġphases":34430,"vw":34431,"@SuppressWarnings":34432,"=\\":34433,"rant":34434,"-ed":34435,"ĉawait":34436,"Ġcertificates":34437,"'>\"":34438,"Ġintact":34439,"CTRL":34440,"Mike":34441,"gregation":34442,"ATTERN":34443,"Ġrepublic":34444,"_upper":34445,"iliary":34446,"Ġcomputation":34447,"hire":34448,"ĠShin":34449,"_ANY":34450,"ĠManufacturer":34451,"ĠCarm":34452,"Ġbearings":34453,"_comb":34454,"cad":34455,"uristic":34456,"Ġwholesale":34457,"Ġdonor":34458,".interfaces":34459,"presso":34460,"ĠBrun":34461,"-close":34462,"prove":34463,"_SK":34464,"ĉframe":34465,"etros":34466,"ĠPain":34467,"_EXP":34468,"ĠLT":34469,"_fs":34470,".datas":34471,"ĉss":34472,"voir":34473,"ĠAxis":34474,"Major":34475,"=\"<":34476,"[h":34477,"Ġprofess":34478,"igrate":34479,"(score":34480,"Keyword":34481,"\"os":34482,"ĠĠĠĠĉĊ":34483,"analysis":34484,"Ġreplay":34485,".pass":34486,"\\d":34487,"tls":34488,"Ġsanct":34489,".light":34490,"_mobile":34491,"ÑģÑĤÑĮ":34492,"ĉtotal":34493,"uity":34494,"Ġpaused":34495,"NAS":34496,"Ġencore":34497,"loe":34498,"Ġ-*-ĊĊ":34499,".high":34500,"ampler":34501,"ĠSecure":34502,"Ġfragments":34503,"_vel":34504,"illary":34505,"ĠStein":34506,"ĠDawn":34507,"Ġmaximize":34508,"ย":34509,"Ġ/^":34510,"Ġcontinually":34511,"Ġshadows":34512,"ĉĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":34513,"ĠIActionResult":34514,"Ġinformación":34515,"CHECK":34516,".SelectedItem":34517,"bundle":34518,"olley":34519,"<":34681,"Ġtrajectory":34682,"_ring":34683,"Ġhydrogen":34684,"tron":34685,"Ġstatute":34686,"Ġconditional":34687,"Ġtray":34688,"-school":34689,"(widget":34690,"$config":34691,"Ġrequesting":34692,".uint":34693,"eton":34694,"brities":34695,"OfType":34696,"ADMIN":34697,"predict":34698,"Ġgegen":34699,"ĠHapp":34700,"OCUMENT":34701,"ĠApart":34702,"Ġ-----":34703,"roe":34704,"uide":34705,"justify":34706,"ĠSquad":34707,"Ġprofes":34708,".bot":34709,"_currency":34710,"innen":34711,"ĠMumbai":34712,"ĠNumbers":34713,"avanaugh":34714,"agnitude":34715,"âĢľThere":34716,"=http":34717,"çīĩ":34718,"Ġvb":34719,"+'{{$":34802,"Ġinode":34803,"sil":34804,"Ġhace":34805,"Ġseverely":34806,"ĠOverview":34807,"Ġspraw":34808,"Ġbeaches":34809,":left":34810,"·»":34811,"(${":34812,"ĠFIRST":34813,"ĠSpa":34814,"-ass":34815,"Ġbaise":34816,"ĠNODE":34817,"ĠPizza":34818,"Pet":34819,"(seq":34820,"\\\">Ċ":34821,"CppMethodPointer":34822,"Ġvp":34823,"Ġia":34824,"_seconds":34825,"emet":34826,"/blob":34827,"_THRESH":34828,"...čĊ":34829,"Dest":34830,"ĠNH":34831,".dataSource":34832,"ités":34833,"ĠJak":34834,"sell":34835,"Ġworkshops":34836,"\",Ċ":35452,"_Pin":35453,"uese":35454,"Ġoverrides":35455,"_ready":35456,"Advanced":35457,"Ġopi":35458,"-cart":35459,"(\"/\",":35460,"ĠDeb":35461,"CRY":35462,"ĠVertical":35463,"ĠOVER":35464,"ĠCorporate":35465,"Ġ\"\";":35466,"Ġstepping":35467,"ej":35468,"Ġaccusations":35469,"Ġoraz":35470,"_tail":35471,"Ġinduced":35472,"Ġelastic":35473,"Ġblown":35474,",//":35475,"Ġbackgrounds":35476,"âĢĻune":35477,"-sdk":35478,"ĠsetInterval":35479,"Ġincentives":35480,"Ġvegetable":35481,"_On":35482,"expanded":35483,"pix":35484,"_shader":35485,"ĠSPDX":35486,"@example":35487,"ĠWrapper":35488,".Zero":35489,"Positive":35490,"Ġspinner":35491,"Ġinvented":35492,"ĠGates":35493,"оÑĤоÑĢ":35494,"Ġcomparisons":35495,"è·":35496,".primary":35497,"dataProvider":35498,"additional":35499,"ĉoptions":35500,"snapshot":35501,".setHorizontal":35502,"Ġ\"{}":35503,"ĠFisher":35504,"halten":35505,"":35538,"ĠRegistered":35539,"INED":35540,"kal":35541,"parison":35542,"Ġobjeto":35543,"Vi":35544,"manda":35545,"Ġrenewed":35546,"ĠSof":35547,"essel":35548,".ndarray":35549,"Ġcrap":35550,"管":35551,".abspath":35552,"(up":35553,"Ġclearance":35554,"ĠTW":35555,"_COPY":35556,"ĠĠĠĠĠĠĠĠĠĠĠĠĉ":35557,"Ġforests":35558,"Ġarguably":35559,"ĠASS":35560,"hey":35561,"amel":35562,"_fore":35563,"ĠSoutheast":35564,"Ġabused":35565,"Ġpracticing":35566,"akedirs":35567,"主":35568,"_resources":35569,"Ġpond":35570,".Fixed":35571,"LastError":35572,"ĠPsychology":35573,"Ġ\"//":35574,"!:":35575,"Reusable":35576,"Ġmensaje":35577,"Ġrospy":35578,"Ġbour":35579,"Ġvarieties":35580,"Ġempath":35581,"(({":35582,"_org":35583,"ĠMes":35584,"ĠMagento":35585,"ISTORY":35586,"Unless":35587,"Ġhj":35588,"ĠDuty":35589,"Jun":35590,",size":35591,"Ġpaintings":35592,"Ġdispens":35593,"dart":35594,"Ġbehavioral":35595,"Ġrpc":35596,"calculate":35597,"fruit":35598,"_mm":35599,"ĉpthread":35600,"MaxLength":35601,"Ġcurrencies":35602,"_capacity":35603,"ĠOz":35604,"Ġfirearm":35605,"Ġcoefficient":35606,"Ġbankruptcy":35607,"wart":35608,"Ġfatigue":35609,"AVA":35610,"Ġespa":35611,"_pc":35612,"ĠQuotes":35613,"_LIGHT":35614,"ĠTickets":35615,"Ġrelates":35616,"Ġpublishers":35617,"Ġunlocked":35618,"Ġ//----------------------------------------------------------------":35619,"ĠInterruptedException":35620,"Ġoutlook":35621,"rn":35622,"Ġrebels":35623,"Written":35624,"Ġasian":35625,"otto":35626,"Ġĉĉĉĉ":35627,"_gpu":35628,"Txt":35629,".ImageView":35630,"Ġsuis":35631,"_tables":35632,".RecyclerView":35633,"Ġwhatsoever":35634,"èģ":35635,"]++;Ċ":35636,"assertTrue":35637,"_verify":35638,"ĠRivers":35639,"Ġ][":35640,"Jet":35641,"idian":35642,"Sibling":35643,"Ġgenres":35644,".Access":35645,"OPS":35646,"Ġtrivial":35647,"ส":35648,"alen":35649,"вед":35650,"ĠSword":35651,"Ġscrutiny":35652,"(cb":35653,"Ġcommerce":35654,"Ġguarantees":35655,"_adv":35656,"ĠLET":35657,"recio":35658,"Ġhilar":35659,"Ġbackyard":35660,"ãĢı":35661,"Ġillustrated":35662,"/vendor":35663,".Util":35664,"Ġwow":35665,"LOY":35666,"ĠMarshal":35667,"\">'.$":35668,"ĠBak":35669,"Ġmodifiers":35670,"dictionary":35671,"ĠStre":35672,"multiple":35673,"\")),":35674,"ĠCort":35675,"']\").":35676,"(admin":35677,"ĠCreator":35678,"Internet":35679,"(ms":35680,"logy":35681,"DECLARE":35682,"ĠMarcus":35683,"<<<<":35684,"ãģł":35685,"_my":35686,"(inst":35687,"Ġsciences":35688,"NDER":35689,".enter":35690,"Ġitu":35691,"Ġbehave":35692,"Pan":35693,"ombies":35694,"='<":35695,"'));čĊ":35696,"ĠMENU":35697,"ĠWorkers":35698,".NoError":35699,"Ġbindings":35700,"Ġdisabilities":35701,"{\\":35702,"ĠMunicip":35703,"Ġcores":35704,"urple":35705,"ĠNokia":35706,"usions":35707,"ĠFitness":35708,".handleChange":35709,"Ġjavascript":35710,"ìļĶ":35711,"(dec":35712,"Ġpacking":35713,"-depend":35714,"Ġtranscript":35715,"zeros":35716,"_alert":35717,"?\",Ċ":35718,"libs":35719,"±Ð¾ÑĤ":35720,"Ġ|ĊĊ":35721,"trained":35722,"ĠGent":35723,"ĠRab":35724,"xp":35725,"_configuration":35726,"天":35727,"_accept":35728,".recyclerview":35729,":url":35730,"ĠMuhammad":35731,"Ġprivileges":35732,"_bank":35733,"uku":35734,"wallet":35735,"ĠROOT":35736,"Ġencuent":35737,"?family":35738,"ĉposition":35739,"Ġcg":35740,"Ġprecip":35741,"methods":35742,"_fast":35743,"increment":35744,"ĠTiger":35745,"_OCCURRED":35746,"quip":35747,"ĠHAS":35748,"_dom":35749,"Ġwreck":35750,"bj":35751,"Ġdern":35752,"Ġorgans":35753,".entries":35754,"Ġ_('":35755,"ramento":35756,"ĠJamie":35757,"Ġpunk":35758,"IPP":35759,"Ġprograma":35760,"Ġattain":35761,"Ġproves":35762,"/sign":35763,"Ġanswering":35764,"Ġladder":35765,"****************************":35766,"ĠWalmart":35767,"ĠCONTENT":35768,"ductor":35769,"Ġverbal":35770,"ĠPID":35771,"crypto":35772,"_CALLBACK":35773,"Ġ=================================":35774,"Ġpotent":35775,"Ġshorts":35776,".Uri":35777,".uniform":35778,";border":35779,"ĠWer":35780,"Ġherein":35781,"lla":35782,"ĠIhr":35783,"Pixmap":35784,"literal":35785,"!)ĊĊ":35786,"generic":35787,"rust":35788,"_scripts":35789,"osto":35790,"itus":35791,"ĠCoalition":35792,"Ġremot":35793,"deploy":35794,"ĠEagle":35795,"ãĢģãĢĮ":35796,"Ġimportante":35797,"ĉobject":35798,"Ġseasonal":35799,"nej":35800,"aidu":35801,"BindView":35802,"ĠSierra":35803,"-bg":35804,"ĠmakeStyles":35805,"[offset":35806,"Games":35807,"Ġhormone":35808,"ARIO":35809,"heads":35810,"(select":35811,"ĠStarted":35812,"@param":35813,"_decl":35814,"_blog":35815,"Ġaño":35816,"\\Api":35817,"ĠMilwaukee":35818,"Provid":35819,"Animated":35820,"Ġcooler":35821,"ĠSeed":35822,".Edit":35823,"ÏĦ":35824,"ĠTaking":35825,"ĠborderColor":35826,"-founder":35827,".LoggerFactory":35828,"Ġ\"\"ĊĊ":35829,"ALT":35830,"ĠLate":35831,"EDIATE":35832,"Ġ);ĊĊĊ":35833,"afa":35834,"Ġcancellation":35835,"Atom":35836,"ĠBirmingham":35837,"empresa":35838,"HEMA":35839,"ascal":35840,"Ġupside":35841,".Version":35842,"ĠFolder":35843,"ĠEight":35844,"ĠVintage":35845,"ĠAppDelegate":35846,"ĠPrevention":35847,".separator":35848,"STM":35849,"(room":35850,"generator":35851,"Ġcattle":35852,"ĉZ":35853,"ĠParticle":35854,"'};Ċ":35855,"Ġneighbours":35856,"ĠStateless":35857,"Ġaltitude":35858,"Ġsaint":35859,"обав":35860,"Ġconvinc":35861,"ĠContents":35862,"Ġjeune":35863,"(ts":35864,"Serialization":35865,"(collection":35866,"ĠJazz":35867,"ĠDod":35868,"ĠRoch":35869,"acio":35870,"commended":35871,"DEFINE":35872,".onload":35873,"Ġspecialty":35874,"PLACE":35875,"_MOVE":35876,"Ġaccountable":35877,"Reuters":35878,"Ġficken":35879,"Ġdepr":35880,"Wow":35881,"Void":35882,".space":35883,"à¸Ĺ":35884,"Ġtq":35885,"ĠPets":35886,"<$":35887,"(Current":35888,"berries":35889,"planation":35890,"ĠlistOf":35891,"ĠThu":35892,"ĠPRINT":35893,"Ġmismo":35894,"Ġdoi":35895,"chk":35896,"ĠUnicode":35897,"(role":35898,"Ġvirgin":35899,"-->Ċ":36360,"Vol":36361,"ĠSSD":36362,"))),":36363,".Optional":36364,"Ġnurses":36365,"Ġorb":36366,"_pe":36367,");čĊčĊčĊ":36368,"placed":36369,"esser":36370,"Ġtherapeutic":36371,"Ġwhitespace":36372,"Ġaston":36373,"Successful":36374,"Ġpraised":36375,"ĠWes":36376,"Ġeighth":36377,"iral":36378,"Ġvrouw":36379,"Ġfaction":36380,"_bias":36381,"Ġwitch":36382,"Ġnpc":36383,"(sb":36384,"ĠRodrig":36385,"_big":36386,"Dependency":36387,"ĠAbraham":36388,"ardi":36389,"CAR":36390,"nos":36391,"Ġabundance":36392,"Ġnutrients":36393,"instein":36394,".Vert":36395,"ĠISS":36396,"D":36495,"Ġservlet":36496,"bastian":36497,"Ġ>&":36498,"SID":36499,"_clk":36500,"Ġdivisions":36501,"}',Ċ":36502,"Ġdildo":36503,"Ġparade":36504,"major":36505,"Ġaboard":36506,";++":36507,"Ġfusion":36508,"\"},{\"":36509,"ĠDialogResult":36510,"ĉarr":36511,"-em":36512,"_nr":36513,"(handler":36514,".NET":36515,".XtraReports":36516,"ĠShah":36517,"ĠBrief":36518,"-,":36519,"Ġprecio":36520,"ĉĉĉĠĠĠĠĠĠ":36521,"Ġtant":36522,"ĠGrande":36523,"/xml":36524,"_ICON":36525,"ĠRetro":36526,"unque":36527,"Ġnag":36528,"toFixed":36529,"XL":36530,"Ġdeclaring":36531,"ĠConcrete":36532,"ĠAmazing":36533,"ĉprintk":36534,"Ġdebates":36535,"DATED":36536,"Ġaesthetic":36537,"emetery":36538,"RoutingModule":36539,"ĠNashville":36540,"WAYS":36541,"Ġwolf":36542,"Ġobservers":36543,"OTA":36544,"anson":36545,"Ġea":36546,"Ġgreenhouse":36547,"ĵįä½ľ":36548,"Ġstair":36549,"Ġimmigrant":36550,"_apply":36551,"peare":36552,"ĠBloomberg":36553,"_PLAYER":36554,"Resp":36555,"æŃ£":36556,"Chooser":36557,"ĠICollection":36558,"Peter":36559,"Erro":36560,".detectChanges":36561,"Maps":36562,"Ġsqueeze":36563,"ĠHomes":36564,"wegian":36565,"Ġformatting":36566,"Ġnegotiate":36567,"uld":36568,"ĠNep":36569,"ĠQB":36570,"Ġeconomies":36571,"Ġ*/,":36572,"Ġredund":36573,"ĠAber":36574,".IsNullOrWhiteSpace":36575,"ycled":36576,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":36577,"_Sh":36578,"Ġskept":36579,"Ġrecreated":36580,"ĠgetType":36581,"Ġmargins":36582,"Ġcolonial":36583,"charts":36584,"//@":36585,"Ġprocessors":36586,"说":36587,"batis":36588,"æĦı":36589,"atorio":36590,"mentioned":36591,"Patient":36592,"Ġprey":36593,"Checkbox":36594,"_xpath":36595,".skip":36596,"ĠMormon":36597,"ĠMemoryStream":36598,"CREMENT":36599,"Ġku":36600,"meld":36601,"\\Data":36602,"ĠKernel":36603,"iltr":36604,"éĢģ":36605,"(profile":36606,"Carbon":36607,"ROLE":36608,"(pl":36609,"]*(":36610,".memory":36611,"Ġmedal":36612,"Ġadvisor":36613,"ität":36614,"Ġhdr":36615,"ierung":36616,"ĠProvides":36617,"(alpha":36618,"Ġteenagers":36619,"-parser":36620,".LatLng":36621,"]()Ċ":36622,"Ġfelony":36623,"ĉĉĉĊĉĉĉĊ":36624,"BOOK":36625,"Ġslash":36626,"Ġclearfix":36627,"ĠProphet":36628,"容":36629,"rightness":36630,"-fi":36631,".kind":36632,"erton":36633,"Jim":36634,"Ġmanipulate":36635,"Ġworksheet":36636,"olin":36637,"stars":36638,"Ġartifact":36639,"_EMPTY":36640,"ĉmain":36641,"-------------';":36709,"Ġexpressing":36710,"ĠIQ":36711,"ĠFact":36712,"/*******************************************************************************Ċ":36713,"_mass":36714,")):":36715,"Ġcondom":36716,"ĠcreateState":36717,"ometown":36718,"Ġirr":36719,"Ġ>(":36720,">B":36721,"iteration":36722,"ãĥª":36723,"Ġshirts":36724,"ounty":36725,"->$":36726,"_SIGN":36727,"ĠDale":36728,"Ġjj":36729,"Easy":36730,"Fre":36731,"ĠNy":36732,"Ġchlor":36733,"matched":36734,"ĠGerm":36735,"-UA":36736,"ĠNathan":36737,"education":36738,"-yard":36739,"-che":36740,"houses":36741,"ritional":36742,"Ġproximity":36743,"Ġdiesem":36744,"áºŃp":36745,"Ġdrought":36746,".audio":36747,"ĠLeo":36748,"Ġfavorable":36749,"inch":36750,"ĠDaw":36751,"ribly":36752,"_student":36753,"idable":36754,"OVE":36755,"Ġlacks":36756,"ouncing":36757,".business":36758,"Ġreopen":36759,"maybe":36760,"_GLOBAL":36761,"Ġdresses":36762,"ĠEdwards":36763,"ensible":36764,"ĠHardware":36765,"ĠExcellent":36766,"ĠTimeUnit":36767,"CTIONS":36768,"Ġschedules":36769,"Ġsegue":36770,"Opens":36771,"ammen":36772,"-Identifier":36773,"Ġstaring":36774,"Ġhappily":36775,"ĠHob":36776,"'_":36777,"Ġ\");":36778,"amentos":36779,"etched":36780,"Ġ/>}Ċ":36781,".Users":36782,"Ġinterrupted":36783,"Contacts":36784,"Ġregistro":36785,"inburgh":36786,"CHA":36787,"_imp":36788,"phis":36789,"say":36790,"Ġretailer":36791,".NODE":36792,"/maps":36793,"_LAST":36794,"ĠCharge":36795,"_guard":36796,"Collider":36797,"ĠStatelessWidget":36798,"\":[\"":36799,"(\"../../":36800,"ioxide":36801,"ĠSund":36802,"Ġ'';":36803,"unset":36804,"addWidget":36805,"лÑİ":36806,"elles":36807,"alker":36808,"Arc":36809,"Ġdeduct":36810,"GUILayout":36811,"ĠVilla":36812,"Ġforbidden":36813,"_where":36814,"Ġ\\/":36815,"ĠTib":36816,"_AX":36817,"]čĊčĊ":36818,"ĠBir":36819,"Ġbend":36820,"ĠMAKE":36821,"ĠMET":36822,"Ġfutures":36823,"Ġweighted":36824,"\"\"\"čĊ":36825,"Ġauthorize":36826,"(program":36827,"},{\"":36828,"Ġcoefficients":36829,"ês":36830,"PerPage":36831,"ĠBathroom":36832,"ĠPublishing":36833,"GPL":36834,"Ġsubmissions":36835,"ĠNUMBER":36836,"jÄħ":36837,"Ġadditionally":36838,"empre":36839,"ĠShel":36840,"otyp":36841,"Solution":36842,"Ġthunder":36843,"_ec":36844,"ĠĊĠĠĠĠĊ":36845,"ĠFellow":36846,"Ġkay":36847,"ĠnewState":36848,"ONTAL":36849,"Implementation":36850,".Look":36851,"Ġents":36852,"Ġlors":36853,"ĠBIG":36854,"fab":36855,"Ġaveraged":36856,"ĠFeedback":36857,"ĠWells":36858,"Ġmartial":36859,"Ġindul":36860,"ĠCommunist":36861,"ĠForex":36862,"ĠAgriculture":36863,"\"[":36864,"Ġquar":36865,"ĠKont":36866,"ĉview":36867,".Bytes":36868,"desktop":36869,"ĠMakes":36870,"akespeare":36871,".Nullable":36872,"Ġspotlight":36873,"VB":36874,"owy":36875,"(torch":36876,"tridge":36877,"_bounds":36878,"Ġapologize":36879,".addItem":36880,"antd":36881,"*);Ċ":36882,",u":36883,"(gen":36884,"ç»ĵ":36885,"reator":36886,"ĠCord":36887,"oupper":36888,".metro":36889,"Ġew":36890,"ĠWORD":36891,".After":36892,"Ġdetained":36893,"ĠHammer":36894,"existing":36895,"Ġost":36896,"Ġmonument":36897,"-custom":36898,"UserID":36899,"ĠNom":36900,"Ġrejection":36901,"(dim":36902,"Ġsingleton":36903,"ĉdie":36904,"ariance":36905,"reports":36906,"]!=":36907,"elda":36908,"Ġprevalence":36909,"_regs":36910,".\".":36911,"Ġfeminist":36912,"Codec":36913,"Ġ**Ċ":36914,"(labels":36915,"_MARK":36916,"FAILED":36917,"Ġadministered":36918,"WN":36919,"ĠĠĠĠĠĠĠĠĉĉ":36920,"Ġnoun":36921,"wig":36922,"Ġgotta":36923,"Ġrif":36924,"-im":36925,"ĠPaulo":36926,"ĠCommandType":36927,"]))ĊĊ":36928,"-zero":36929,"Training":36930,"Ġlord":36931,"_art":36932,"reddit":36933,"Cert":36934,"Ġpeso":36935,"Rot":36936,"Ġendanger":36937,".dr":36938,"userInfo":36939,"unts":36940,"nv":36941,"ĠTrailer":36942,"-first":36943,"(make":36944,"Ġbenefici":36945,"-black":36946,"iÃŁ":36947,"Ġundoubtedly":36948,"Ġmex":36949,"ĠAncient":36950,"(as":36951,"Ġdescent":36952,"Pick":36953,"Ġreplica":36954,"$obj":36955,"ähr":36956,"Ġarrows":36957,"fty":36958,"ĠLibya":36959,"uga":36960,"charged":36961,"Tur":36962,"Ġhomic":36963,"issen":36964,"ĠFake":36965,"Ġbeers":36966,"Ġscattered":36967,"(Time":36968,"UTIL":36969,"Ġbureaucr":36970,"/plain":36971,"Ġsticking":36972,"FAIL":36973,"ĠCovid":36974,"Third":36975,"_present":36976,"ĠPierre":36977,"Ġëª":36978,"Ġ[...]ĊĊ":36979,"Prob":36980,"ĠTraffic":36981,"icao":36982,"doctor":36983,"Ġ),ĊĊ":36984,"Tabs":36985,"alu":36986,"ï¼ļâĢľ":36987,"Ġinherent":36988,"_No":36989,"ritis":36990,"ĠProof":36991,".basename":36992,"ä¼ļ":36993,"Ġchim":36994,"ĠProtected":36995,"crit":36996,"Ġprone":36997,"Ġкон":36998,"ĠHeroes":36999,"Ġanxious":37000,"Ġanos":37001,"Ġweekends":37002,"Ġsext":37003,"Ġreducer":37004,"=UTF":37005,"half":37006,"ĠSaw":37007,".mm":37008,"Ġnueva":37009,".currentTarget":37010,".lua":37011,"_EXTENSION":37012,"ĉreg":37013,"ĠCtrl":37014,"_align":37015,"acceptable":37016,"Ġrushing":37017,"frac":37018,"Ġboasts":37019,"Five":37020,"±":37021,"ĠTemperature":37022,">):":37023,"Ġcharter":37024,"REATED":37025,"Ġsubjected":37026,"Ġopc":37027,"healthy":37028,"使ç͍":37029,"ĠScientific":37030,"Ġfrau":37031,"riages":37032,"à¸Ķ":37033,".inventory":37034,"ationale":37035,"Mad":37036,"minutes":37037,">>();Ċ":37038,"ĠEnv":37039,"Ġrecordings":37040,"Ġsuspicion":37041,"sqlite":37042,"ĉread":37043,"ãģ¦":37044,"Ġworries":37045,".putString":37046,"ĠShanghai":37047,"(uid":37048,"rer":37049,"ĠvÃŃde":37050,"\"):":37051,"Ġmethodology":37052,"ĠкоÑĤоÑĢ":37053,"ccc":37054,"avad":37055,"Ġinduction":37056,"ĉThread":37057,",string":37058,"ại":37059,"nehmen":37060,"uition":37061,"Ġ*__":37062,".emf":37063,"Ġìľ":37064,"/themes":37065,"ĠNine":37066,".One":37067,"ĠEmbed":37068,"Ġfaz":37069,"uations":37070,"Ġprivately":37071,"Ġling":37072,"[F":37073,"ushi":37074,"Ġlaunches":37075,"(KEY":37076,"GMT":37077,"Ġaiming":37078,"patible":37079,"ĠBiden":37080,"iw":37081,"ĠDegree":37082,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":37083,"Ġ$('<":37084,"ários":37085,"toUpperCase":37086,"ìłľ":37087,"ĠEUR":37088,"Ġoversight":37089,"Ġtablesp":37090,"Updates":37091,".makedirs":37092,"Ġhumidity":37093,"/template":37094,"Always":37095,"(IS":37096,"_cert":37097,"Dig":37098,"Ġunderway":37099,"orton":37100,"ĠHurricane":37101,"Ġspends":37102,"ĠSegment":37103,"Ġflies":37104,"ĠToggle":37105,"ĠLynch":37106,"Ġsenses":37107,"ĠKos":37108,"setEnabled":37109,"istically":37110,"Ġtester":37111,"Ġadministrators":37112,"Ġtagged":37113,"Ðĵ":37114,"Ġshortcut":37115,"ĠResolution":37116,"Ġsupervision":37117,"ĠAshley":37118,"Tracking":37119,"ulatory":37120,"andel":37121,"isten":37122,"Ġunre":37123,"(diff":37124,"ANTS":37125,"Ġrider":37126,"ĠsÄħ":37127,".Series":37128,"_orders":37129,"ORIZONTAL":37130,"Ġretention":37131,"ãĢĤčĊčĊ":37235,"Ġdiagonal":37236,"ĠCancellationToken":37237,"_Internal":37238,"Ġruin":37239,".Qt":37240,"ocratic":37241,"Tel":37242,"ĠAnswers":37243,"matic":37244,"Ġxp":37245,"atem":37246,"_jobs":37247,"_any":37248,"Ġseniors":37249,"Ġlandmark":37250,"ĠQList":37251,"Ġmaneu":37252,"otify":37253,"/\";Ċ":37254,"/server":37255,"ĠPhilosoph":37256,"utenant":37257,"(io":37258,"hz":37259,"Ġauthenticated":37260,"dv":37261,"-Compatible":37262,"Originally":37263,",function":37264,"ãĢĤčĊ":37265,"ĠRepresentative":37266,"asily":37267,"ircuit":37268,".dt":37269,"(math":37270,".Marshal":37271,"[,":37272,"ĠCities":37273,"_turn":37274,"|)Ċ":37275,"Ġcantidad":37276,"alter":37277,"ĉui":37278,"ĠNebraska":37279,"Ġskirt":37280,".bg":37281,"SharedPreferences":37282,"(style":37283,"Ġgrief":37284,"gew":37285,"Ġsafeg":37286,"olang":37287,"_lists":37288,"ìĽ":37289,"Ġgranite":37290,"Ġhottest":37291,".jdbc":37292,".Customer":37293,"Ġâī¤":37294,"Ġwaar":37295,"_scene":37296,"+'/":37297,"ĠJTextField":37298,"Ġseating":37299,"Ġwears":37300,"Ġ`/":37301,"Cases":37302,"ĠYoutube":37303,"ım":37304,"Ġbalcon":37305,",G":37306,"MetaData":37307,"-price":37308,"SCR":37309,"Unity":37310,"Ġtrunk":37311,"={`${":37312,"Ġearthquake":37313,"Partial":37314,"Ġsubst":37315,"Ġelimin":37316,"=\"'.":37317,"//*[@":37318,"Ġsupervisor":37319,"vrolet":37320,"_article":37321,"Ġpane":37322,"bio":37323,"Ġmotors":37324,"NM":37325,"Frank":37326,"Ġonion":37327,"-word":37328,"ItemClickListener":37329,"Ġbrit":37330,"endencies":37331,"Computer":37332,"_running":37333,"(day":37334,"-he":37335,"(named":37336,"ĠSach":37337,"оÑĩ":37338,"campaign":37339,".Abstract":37340,"(wrapper":37341,".pay":37342,"Ġuw":37343,"Geo":37344,"rails":37345,"/select":37346,"ichte":37347,"sons":37348,"EVENT":37349,"Ġaliment":37350,"Providers":37351,"Await":37352,"_INTERVAL":37353,".off":37354,"Ġgluten":37355,"_cloud":37356,"Ġwen":37357,".extract":37358,"ĉbutton":37359,"/MM":37360,"Party":37361,"Ġdemographic":37362,"_errno":37363,"Ġhiking":37364,"('')Ċ":37365,"\",@\"":37366,"Ġwit":37367,"rá":37368,"ologie":37369,"ĠStyles":37370,"ĠBrowserModule":37371,".RequestMapping":37372,"icans":37373,"PAGE":37374,"creation":37375,"ĠFerguson":37376,"uded":37377,"numbers":37378,"ĠGTK":37379,"Ġpresentations":37380,"ĠBobby":37381,"_span":37382,"estyle":37383,"Ġillegally":37384,"abela":37385,"Ġbattlefield":37386,"capacity":37387,"terror":37388,"]\");Ċ":37389,"Ġwarrior":37390,"leader":37391,"ĠDBG":37392,"ĠRevenue":37393,"Ġvigil":37394,"Ġcounterparts":37395,"(Error":37396,"ACTER":37397,"Ġheeft":37398,"Ġselections":37399,"zeug":37400,"tom":37401,"-two":37402,".;Ċ":37403,"_statement":37404,"ĠAid":37405,"ĠVul":37406,"_rgb":37407,"Ġprizes":37408,"Ġeditable":37409,"ĉform":37410,"ını":37411,".decor":37412,"Demo":37413,"lices":37414,"Ġenctype":37415,"ratulations":37416,"ĠROS":37417,"_chars":37418,"ĠJahr":37419,"partial":37420,"ÑĥÑĤ":37421,"ĠReceive":37422,"ĠLands":37423,"APTER":37424,"Ġchopped":37425,"..\"":37426,"ĠAnaly":37427,"ĠUID":37428,"ĠRadeon":37429,"ĠBee":37430,"Ġunm":37431,">M":37432,".findall":37433,"Tokenizer":37434,"ĠWHAT":37435,"Ġsj":37436,"Drawing":37437,"Ess":37438,"OND":37439,"Ĭ¶":37440,"(packet":37441,"âĢĶbut":37442,"Invocation":37443,"ĠNuclear":37444,"?;Ċ":37445,"Ġgrandes":37446,"ĠCrypt":37447,"remark":37448,"Ġ'../../../../":37449,"Ġinability":37450,"magic":37451,"cats":37452,"Ġsimulate":37453,":${":37454,"inflate":37455,"Ġener":37456,":NO":37457,"iples":37458,"Ġmerit":37459,"ĠRated":37460,"Ġglue":37461,"/blog":37462,"Ġgren":37463,"Ġthrilled":37464,".CH":37465,"uncan":37466,"ĠPRIMARY":37467,"Ġpersec":37468,"Ġfeared":37469,".MIN":37470,"ĠTheater":37471,"éĴ":37472,"ategorie":37473,"段":37474,"Ġappetite":37475,"square":37476,"ĠAlexand":37477,".UserId":37478,"_gt":37479,"_enter":37480,"Ġgraduates":37481,"FragmentManager":37482,"Authorize":37483,"-NLS":37484,"(My":37485,"Ġtriumph":37486,"usting":37487,"_PARAMS":37488,"Characters":37489,"(:,:,":37490,"_BUILD":37491,"MHz":37492,"Ġwashed":37493,"Ġuncle":37494,"Steve":37495,"ardown":37496,"${":37680,"_confirmation":37681,"Ġtrophy":37682,"Works":37683,"ĠElectronics":37684,"ĠMediterranean":37685,"_metrics":37686,"Ġannouncing":37687,"ĠDAY":37688,"_proto":37689,"Ġpear":37690,"baseUrl":37691,"ĉĉĉĉĉĉĉĉĊ":37692,"Ġcoordination":37693,":N":37694,".animate":37695,"ĠCotton":37696,"_hit":37697,"âľ":37698,"Ġjetzt":37699,"ifter":37700,"(fields":37701,"ownload":37702,"ificacion":37703,".cuda":37704,"ĠLiu":37705,">equals":37706,"ĠAce":37707,"ÑĢам":37708,"ĠSuperman":37709,"ĠGarcia":37710,"Ġarrests":37711,"agar":37712,"Ġ{})":37713,"Ġmacros":37714,"roupe":37715,"être":37716,"Ġtwisted":37717,"struments":37718,"_(\"":37719,"_vertices":37720,"ĠTransition":37721,"ик":37722,"[max":37723,"mind":37724,"ĠaccessToken":37725,"Ġunle":37726,"mus":37727,"cop":37728,"ĠFactor":37729,"Ġconced":37730,"Ġretr":37731,".linalg":37732,"-slider":37733,"obl":37734,"_StaticFields":37735,"Ġzombie":37736,"selling":37737,"Ġchap":37738,"Ġshaking":37739,"ĠTranslate":37740,"ĠAmsterdam":37741,"ĠETH":37742,"_EXTERN":37743,"kd":37744,"_disc":37745,"Ġpreceding":37746,"Ġprix":37747,"ObjectName":37748,"_modified":37749,"ardware":37750,"Ġ?>\">":37751,"ĠDW":37752,"`${":37753,"Ġ?>\">ĊĊ":37859,"Ġspinning":37860,"_pending":37861,"Matchers":37862,".Keys":37863,"ĠPV":37864,"enus":37865,"antis":37866,"Ġdiscard":37867,"Ġhaul":37868,"Ġempir":37869,"Ġpathway":37870,"Ġoak":37871,"мен":37872,"-induced":37873,"Ġimpair":37874,"ĠCalgary":37875,".isHidden":37876,"dz":37877,"_include":37878,"Ġgm":37879,"Ġ'('":37880,"PY":37881,"uggestions":37882,"Ġcommodity":37883,"cro":37884,"/sub":37885,"ĠgetInstance":37886,"ĠLegacy":37887,"ĠKil":37888,"Bal":37889,"(short":37890,"Inform":37891,"+x":37892,"*r":37893,"ĠHopefully":37894,"orate":37895,"Ġmachen":37896,"Ġtreaty":37897,"ĠOri":37898,".public":37899,"-horizontal":37900,"Ġtactic":37901,"Ġbord":37902,"wares":37903,"Ġammo":37904,"ĠLists":37905,"Ġequations":37906,"/her":37907,"ĠNSW":37908,"Bounding":37909,"_Collections":37910,"Ġavail":37911,".DropDown":37912,"è°":37913,"Ġhh":37914,"ĠlÃł":37915,".pb":37916,"Ġmemorial":37917,"ĠATTR":37918,"Ġexhausted":37919,"Ġtsp":37920,"ĉredirect":37921,"Ġlikewise":37922,"STER":37923,"Ljava":37924,"Ġcondemned":37925,"ocaust":37926,"(strict":37927,"Ġexempt":37928,"Ġsms":37929,"Ġexagger":37930,"SYS":37931,"Ġlounge":37932,":^":37933,"Ġtodd":37934,"deb":37935,"atorial":37936,"ĠPorter":37937,"Ġtuition":37938,"Ġexempl":37939,"Ġparen":37940,".lineTo":37941,"Ġkidney":37942,"Ġça":37943,"Ġcui":37944,"ï¼Į请":37945,"XC":37946,"Ġmoż":37947,"Ġnominated":37948,"lung":37949,"ImGui":37950,"ĠBuzz":37951,"Ġstereo":37952,"portal":37953,"resas":37954,"Ġklass":37955,"Ġdrafted":37956,"Ġprojectile":37957,"/gpl":37958,"(parameters":37959,"*)Ċ":37960,"Ġassisted":37961,"ĠNSInteger":37962,"sitemap":37963,":nth":37964,".Views":37965,".ArgumentParser":37966,"Ġmeer":37967,"zier":37968,"ĠDig":37969,"Ċ":38036,"Ġplag":38037,"pine":38038,"Ġblanket":38039,"Ġ:-":38643,"Ġlcd":38644,"---------------":38645,"(\"\"":38646,"Ġtactical":38647,"ĠRonald":38648,"extr":38649,"ĠFest":38650,"Ġfuer":38651,"-navigation":38652,"Ġkb":38653,"ghost":38654,"ĠhandleChange":38655,"_cls":38656,"()!=":38657,"Comparator":38658,".vm":38659,"ĠCox":38660,"_review":38661,"/@":38662,"_cookie":38663,"Ġrecognised":38664,"ldap":38665,"Threads":38666,"ĠSexual":38667,"ĠBearing":38668,"(SQL":38669,"Ġxr":38670,"Ġthigh":38671,"URLConnection":38672,"ĠSUV":38673,"ĠmContext":38674,"Ġincidence":38675,"ĠEste":38676,".sup":38677,"_te":38678,"(EXIT":38679,"CMD":38680,"/\">":38681,"Almost":38682,"ĠUne":38683,"Ġanderen":38684,"ĠSingleton":38685,"Ġbore":38686,"Think":38687,"Ġnarc":38688,"]initWith":38689,"_shop":38690,"(strategy":38691,"!',":38692,"herits":38693,"ĠDesk":38694,"_machine":38695,".netty":38696,"ında":38697,"=<":38698,"ĠQR":38699,"ĠSidebar":38700,".splitContainer":38701,"ĠonSuccess":38702,"Ġmonkey":38703,"Enjoy":38704,"(nodes":38705,"pectrum":38706,"Ġ(*(":38707,"ĉUINT":38708,",height":38709,"ĠNetworks":38710,".tail":38711,".linspace":38712,"Ġ\"...":38713,"Listen":38714,"Æ¡":38715,".Channel":38716,"-defined":38717,"Repeat":38718,"adjust":38719,"ERM":38720,"_application":38721,".assertNotNull":38722,"-stream":38723,"Ġrabbit":38724,"Ġpositioning":38725,"Ġwoke":38726,"Ġfing":38727,"Ġmultiplayer":38728,"Ġregistering":38729,"until":38730,"Ã¥n":38731,"(::":38732,"ussions":38733,"Ġpotato":38734,"ĠEquals":38735,".Sup":38736,"/apache":38737,"Ġ(=":38738,".\")":38739,".ptr":38740,"ĠSpeech":38741,".clip":38742,"ĠGabriel":38743,"Ġmusician":38744,"/issues":38745,".shop":38746,"ĠHier":38747,"_RET":38748,"_bucket":38749,"ãĥ¡":38750,"avs":38751,"Ġroz":38752,"flower":38753,"WriteBarrier":38754,"ĠMilan":38755,"Ġlegislature":38756,"ĠDoll":38757,"Ġproving":38758,".concatenate":38759,"âķIJ":38760,"Ġgchar":38761,"cdnjs":38762,"bles":38763,"ĠListing":38764,"ло":38765,".xrLabel":38766,"ĠSak":38767,"justice":38768,"ĠValentine":38769,"unless":38770,"Ġpiger":38771,"(run":38772,"Ġtestified":38773,"ANA":38774,"ĠRemoves":38775,"))));Ċ":38776,"recated":38777,"ĠRuntimeMethod":38778,"Ġconqu":38779,"ãĤ¢":38780,"Ġtissues":38781,"ailer":38782,"été":38783,"-Star":38784,"Ġflames":38785,".setIcon":38786,"Ġsupern":38787,"Ġvagina":38788,"-variable":38789,"Ġwellness":38790,"CUR":38791,"Ġbelle":38792,".getRequest":38793,"Ġpoco":38794,"benh":38795,"agens":38796,"Ġspill":38797,"ĠJur":38798,"Ġdispatcher":38799,"ного":38800,"emonic":38801,"(dirname":38802,"ĠÐĶ":38803,"Ġpasse":38804,"Ġganz":38805,"ricing":38806,"EU":38807,"Ġmujeres":38808,"essen":38809,".attribute":38810,"jj":38811,"ĉĉĠĊ":38812,"[^":38813,"Ġstrtolower":38814,"lexer":38815,"ectar":38816,"hotel":38817,".square":38818,"Ġrall":38819,"Ġlowered":38820,"handled":38821,"Market":38822,"ĠUses":38823,"ivas":38824,".Business":38825,"ãģĹãģ¦":38826,"DIV":38827,"Ġwasted":38828,"Ġavoir":38829,"êm":38830,"_ACCOUNT":38831,".et":38832,"ĉSDL":38833,"kap":38834,"Ġfox":38835,"uppet":38836,"{},Ċ":38837,"\",'":38838,"Favorite":38839,"PEND":38840,"ĠAES":38841,"}),":38842,"Ġdeduction":38843,"ĠpolÃŃt":38844,"ĠcomponentWill":38845,"ĠTelerik":38846,"_SELF":38847,"Ġmuse":38848,"Craft":38849,"Ġdens":38850,"ि":38851,"(tp":38852,"Ġtasty":38853,"Ġbalances":38854,"Ġdedication":38855,"ĠWallace":38856,"Ġunlaw":38857,"\\\">\\":38858,"Ġmum":38859,"-update":38860,"emente":38861,"Ġsoda":38862,"Republic":38863,"asmine":38864,"éric":38865,"(Status":38866,"ĠJsonConvert":38867,"ĠDisk":38868,".Redirect":38869,"Ġfilming":38870,"/mol":38871,"Ro":38872,"Ġville":38873,"Ġtrabaj":38874,"Ġsynthesis":38875,"rega":38876,"Ġrl":38877,"Scheduler":38878,"ISHED":38879,"currentUser":38880,"(errors":38881,"'h":38882,"_bot":38883,"ximo":38884,"ĠUSART":38885,"_super":38886,"_DECREF":38887,"ной":38888,"_ROW":38889,"Ġpromotes":38890,"ĠTA":38891,"Ġhoras":38892,"ĠRepresents":38893,"Ġnameof":38894,"ĠExc":38895,"ĠGarage":38896,"Ġseine":38897,",#":38898,"Ġherb":38899,"/resources":38900,"Ġpleaded":38901,".radioButton":38902,"Ġæĺ":38903,"Ops":38904,"ĠNest":38905,"cstring":38906,"ĠDefence":38907,"Ġrefere":38908,"_leaf":38909,"Ġrevelation":38910,"ë§":38911,".executeUpdate":38912,"_WORLD":38913,"Ġexpans":38914,"(\"\\\"":38915,"jab":38916,"Ġdoubts":38917,"ĠGeometry":38918,"Ġintroduces":38919,"Ġsenators":38920,"Ġcanal":38921,".helper":38922,"ĠBiology":38923,"_SENS":38924,".previous":38925,"-touch":38926,"abit":38927,"Ġimpacted":38928,"Ġbrackets":38929,".direct":38930,"accum":38931,"Ġtestosterone":38932,"ĉaction":38933,"ĠChance":38934,"Ġpeaks":38935,"CppCodeGenWriteBarrier":38936,"Ġunbelie":38937,"_press":38938,".Rel":38939,"angled":38940,"/templates":38941,"-->čĊ":38942,"lime":38943,"Ġsufficiently":38944,"_nt":38945,"Expand":38946,".isfile":38947,"ĠisEmpty":38948,"Ġqt":38949,"Ġmulher":38950,"acob":38951,"George":38952,"常":38953,"Ġassim":38954,"aso":38955,"Ġcomprised":38956,"OV":38957,"(CONFIG":38958,"ĉwriter":38959,"Ġdesp":38960,"Ġtenure":38961,"(cr":38962,".pool":38963,"ĠBrend":38964,"Ġcensor":38965,"(timeout":38966,"Ġplea":38967,".Wrap":38968,"Ġtightly":38969,"ĠWere":38970,"ĠIgnore":38971,"abei":38972,"Ġbridges":38973,"Ġcondemn":38974,"Ġsimplicity":38975,"Ġroutinely":38976,"Ġblacks":38977,"jb":38978,"ĠPit":38979,"Utf":38980,"Ġ/Ċ":38981,"reload":38982,"ĠsetObject":38983,"/global":38984,"Ġfatty":38985,"Ġsocks":38986,"Couldn":38987,"Ġerotisk":38988,"æĿ¡":38989,"ĠPressure":38990,"ĠMaz":38991,"npos":38992,"tolower":38993,"ĠEQ":38994,"uteur":38995,"ĠMoment":38996,"Ġeta":38997,"{{--":38998,"Ġgraphs":38999,"ĠGuar":39000,"rine":39001,"(--":39002,"ĠHttpStatus":39003,"(student":39004,"*np":39005,"Ġrailway":39006,"Ġasynchronous":39007,"_vm":39008,"'],'":39009,",text":39010,"merchant":39011,"(Guid":39012,"ĠGra":39013,"ixer":39014,"fetchAll":39015,".addListener":39016,"flip":39017,"*$":39018,">(),":39019,"Ġsunlight":39020,"assigned":39021,"Ġabc":39022,"ĠCOLUMN":39023,"ĠðŁĻĤĊĊ":39024,")...":39025,"Ġensemble":39026,"Ġnewline":39027,"_SINGLE":39028,"iedad":39029,"Ġdarker":39030,"ormap":39031,"Ġlion":39032,"plits":39033,"Ġillustration":39034,"ĠIEEE":39035,"Ġvista":39036,"ousands":39037,"*******":39038,"ĠTommy":39039,"Ġhue":39040,"Sel":39041,"Ġaura":39042,"ĠTherapy":39043,"Ġanimator":39044,".constraints":39045,"Ġvague":39046,"(\"\")":39047,"Ġvillain":39048,"Ġblessing":39049,"ĠstringBuilder":39050,"ĠMisc":39051,"ĠDIR":39052,"fax":39053,"-node":39054,"ĠWalking":39055,"ĠAU":39056,"sess":39057,"Ġgrill":39058,"VERTISE":39059,"ĠFoods":39060,"Ġtournaments":39061,"Ãĵ":39062,"ĠMarsh":39063,"Ġwonders":39064,"Longitude":39065,".CommandText":39066,"=input":39067,"_encoder":39068,"pageSize":39069,"ĠgetState":39070,">>Ċ":39071,".grey":39072,"pod":39073,"Ġreadings":39074,"Ġreconsider":39075,"Startup":39076,"Ġexcer":39077,".balance":39078,"_cycle":39079,"_Time":39080,"LOCAL":39081,"ĠEFI":39082,"ĠReyn":39083,".setForeground":39084,"byn":39085,"Ġdisconnected":39086,"ACTIVE":39087,"Ġembedding":39088,"ickers":39089,"Ġsurroundings":39090,"*c":39091,"Ġgarant":39092,"Ġbf":39093,"Ġwipe":39094,"Ġä¸ĭ":39095,"_TRA":39096,"adox":39097,"çķ":39098,"Ġsucks":39099,"ĠSongs":39100,"ĠAssociates":39101,"ĠBald":39102,"ĠBrett":39103,"venile":39104,"Ġvt":39105,"Ġinade":39106,"Ġresigned":39107,"ĠGlenn":39108,".pattern":39109,".DataBind":39110,"Ñĥм":39111,"LayoutInflater":39112,"chet":39113,"ĠTestament":39114,".ms":39115,"Ġpav":39116,"ĠReactDOM":39117,"urdy":39118,"ADATA":39119,"Mu":39120,"/actions":39121,"ĠJs":39122,"_extract":39123,"ĠBring":39124,":id":39125,"strt":39126,"ivation":39127,"Ġoutright":39128,"azu":39129,"loyment":39130,"иÑı":39131,"aldo":39132,"ĠPublisher":39133,"Education":39134,"Palette":39135,"_drv":39136,"Ġ($(":39137,"ĠAnda":39138,"Ġremedy":39139,"Ġinconsistent":39140,"tection":39141,"Ġregulators":39142,"Ġshortest":39143,"(pair":39144,"ĠInstallation":39145,"Ġdefendants":39146,"Ġ();":39147,"-large":39148,"Mel":39149,"Ġthreaten":39150,"нÑı":39151,"Ġfetish":39152,"otine":39153,"_dic":39154,"Ġ<$":39155,"Ġstagger":39156,"spi":39157,"$response":39158,"Serv":39159,"-born":39160,"jos":39161,"ĉimg":39162,"ĉWHERE":39163,"_lt":39164,"å½ĵ":39165,".cost":39166,"ĠTue":39167,".labels":39168,"ĠLV":39169,"wcsstore":39170,"ĠJesse":39171,"ห":39172,"Trade":39173,"Ġpredecessor":39174,"ëĤ":39175,"finally":39176,"_general":39177,"oggler":39178,"_REGION":39179,"nement":39180,"Ġblogger":39181,"ĠHarbor":39182,"ĠDataset":39183,"[w":39184,"Ġattendees":39185,".ico":39186,"maximum":39187,".Unlock":39188,"_SYNC":39189,"ágina":39190,"Ġdowns":39191,"ĠWii":39192,"])/":39193,"Ġkicking":39194,"unication":39195,"ĠDAC":39196,"ĠIDS":39197,"ĠRental":39198,"ĠcurrentTime":39199,"Ġvaccines":39200,"ĠDevil":39201,"Ġnors":39202,"_mouse":39203,"urrection":39204,"(no":39205,"Ġ>čĊ":39206,"Ġaggression":39207,"Ġbreeding":39208,".symbol":39209,"iman":39210,"AbsolutePath":39211,"ĠWHO":39212,"_flush":39213,"-root":39214,"arna":39215,"&M":39216,"Ġfathers":39217,"ĠRocket":39218,"iveau":39219,"Ġwander":39220,"Ġcompos":39221,"ĠWarrior":39222,"ĠSeat":39223,"ĠClinic":39224,"_invoice":39225,"(dispatch":39226,"Producto":39227,"aturing":39228,"ossier":39229,"ĠMAY":39230,"Ġdagger":39231,"Ġsanitized":39232,"ĠRFC":39233,"Ġproph":39234,"Ġurine":39235,"Ġgrind":39236,"ĠExpanded":39237,"descripcion":39238,"-fw":39239,"ĠKerry":39240,"=name":39241,"Ġchk":39242,"Ġnationally":39243,"Ġthee":39244,"Inc":39245,"Ġ?>>":39246,".RadioButton":39247,".HttpServletResponse":39248,"/Y":39249,"ĉfield":39250,"Ġhomme":39251,"yper":39252,"Physical":39253,"=v":39254,"Ġdriv":39255,"ĠErrors":39256,"ĠcÄĥ":39257,"Death":39258,"ĠWINDOW":39259,"Ġpoet":39260,"ĠSharp":39261,"ĠImmutable":39262,"ĉcreate":39263,"Ġgeht":39264,"ĠReform":39265,"aiser":39266,"ĠInitialization":39267,"Ġimmunity":39268,".compose":39269,"Ġlatency":39270,"ĠLebanon":39271,"ĠParad":39272,"Ġfuels":39273,"ĠExhib":39274,"coh":39275,"%\">Ċ":39276,"ĠCLI":39277,")initWith":39278,"-Za":39279,"_CLEAR":39280,"regn":39281,"Ġfinances":39282,".standard":39283,"_CATEGORY":39284,".library":39285,"Ġtravelers":39286,"_wp":39287,"ĠEvaluation":39288,"starting":39289,"Ġ)),Ċ":39290,"episode":39291,"ĠVariant":39292,"Ġdaemon":39293,"ĠJulia":39294,"ĠNR":39295,"Ġdoubles":39296,"'":39526,"Ġqueryset":39527,";}čĊ":39528,"ĠPopulation":39529,"utedString":39530,"resident":39531,"_FONT":39532,"ĠRespond":39533,"Ġobscure":39534,"Ġobservable":39535,"ĠContributors":39536,"kon":39537,"ĠMusk":39538,"exao":39539,"ĠTub":39540,"BootApplication":39541,"SOR":39542,".Horizontal":39543,".findBy":39544,".power":39545,"Ġpositively":39546,"venience":39547,"ĠJong":39548,"Ġwhistle":39549,"ĠзнаÑĩ":39550,"Ġlending":39551,"Ġdestructive":39552,"ĠonDelete":39553,"authorization":39554,"();?>":39555,"_original":39556,"science":39557,"atra":39558,"?,?,":39559,"ĠAsc":39560,"Ġconvincing":39561,"$a":39562,"orgen":39563,"_Date":39564,"ĠProvide":39565,"Ġlonely":39566,")'Ċ":39567,"exchange":39568,";?>Ċ":39569,".fast":39570,"Samples":39571,"London":39572,"'])čĊ":39573,"ĠIonic":39574,"Ġpesso":39575,"ĠKnights":39576,"ĠRaf":39577,"_attrs":39578,"Ġrepeal":39579,">Main":39580,"ĠOrdered":39581,"_New":39582,"=\"\">\";Ċ":39663,"ĠSERVER":39664,"ĠHEADER":39665,"_velocity":39666,"ĠInvoke":39667,".timestamps":39668,"Ġsulf":39669,"IQUE":39670,"Ġinhabitants":39671,"phins":39672,"azzo":39673,"Ġmono":39674,"Legend":39675,"Ġnonce":39676,"IFE":39677,";\";Ċ":39678,"-create":39679,"\"\",Ċ":39680,"permit":39681,"ĠImmigration":39682,"Ġpathname":39683,"ffective":39684,"âĻĢâĻĢ":39685,"Ġexams":39686,"-event":39687,"ĠTill":39688,"[mid":39689,"FIX":39690,";color":39691,"(Order":39692,"_traits":39693,"ĠorderBy":39694,"Ġsunt":39695,"ĠNicholas":39696,"ز":39697,"Ġsunny":39698,"iners":39699,"Ġaccessibility":39700,"ĠHB":39701,".comp":39702,"ĉop":39703,"Ġminorities":39704,"etheus":39705,"Ġcollaborative":39706,"prit":39707,"HIR":39708,"Ġwraps":39709,"ĉdraw":39710,"god":39711,"ĠIX":39712,".apps":39713,"ĠNM":39714,"Ġirrelevant":39715,"ĠTigers":39716,"Ġdiag":39717,"GV":39718,"ĠAccessories":39719,"kont":39720,"Ġsimplify":39721,"ĠFavorite":39722,"_tools":39723,"([]);Ċ":39724,"Ġtowers":39725,"Bes":39726,"Ġhunter":39727,"Ġsalon":39728,"(buff":39729,"ĉdebug":39730,"Ġmalware":39731,"Moving":39732,"-options":39733,")+'":39734,"ĠLOVE":39735,"_SOCKET":39736,"_fin":39737,"ĠDelaware":39738,"Ġsheriff":39739,"-invalid":39740,"ĠFULL":39741,"Ġпод":39742,"elas":39743,"\"strings":39744,"ĠRepresentatives":39745,"surface":39746,"resolved":39747,"htdocs":39748,")):čĊ":39749,"Ġpressures":39750,"Ġnorms":39751,"Ġpla":39752,"Ġsurname":39753,"Ġpostal":39754,"ĠDepart":39755,"Ġslaughter":39756,"orida":39757,"Ġhebben":39758,"Ġdesar":39759,"compact":39760,"_LANG":39761,"åIJĪ":39762,"opoly":39763,"_rad":39764,"ĠSTDMETHOD":39765,"Lazy":39766,"ĠĠĠĉ":39767,"...,":39768,"(web":39769,"ĠPont":39770,"Ġetwas":39771,"Ġupward":39772,"_hat":39773,"Ġ],ĊĊ":39774,"ĠbaseUrl":39775,"Ġworrying":39776,"-addon":39777,"(getClass":39778,"SPI":39779,"Ġcapturing":39780,")},Ċ":39781,"Effects":39782,"Ġcompetent":39783,"Ġfoul":39784,"Ġsubscribing":39785,"ĠOBJECT":39786,"IXEL":39787,"bucks":39788,"(edge":39789,"(pass":39790,"ĠPeterson":39791,"Ġboobs":39792,"ĠDelay":39793,"_square":39794,"elim":39795,"oters":39796,"_PC":39797,"%E":39798,"onclick":39799,"ĠSVG":39800,"Ġtopped":39801,"Ġfist":39802,"smart":39803,"ĠRalph":39804,"(owner":39805,"jours":39806,"Ġbronze":39807,"ĠArgumentException":39808,"(original":39809,"_SCALE":39810,"_cp":39811,"Ġrecommends":39812,".setStyle":39813,"Sure":39814,"LAND":39815,"Ġrepeating":39816,"Matt":39817,".Visibility":39818,"Ġenterprises":39819,".Setup":39820,"(scene":39821,"ĠReactive":39822,"urge":39823,"bw":39824,".Put":39825,"persist":39826,".cookie":39827,"ĠAudi":39828,"`s":39829,"supplier":39830,"(Form":39831,"¡":39832,"_so":39833,"ĮĢ":39834,"ĠLegion":39835,"tte":39836,"Nd":39837,"Loss":39838,"(attrs":39839,".scatter":39840,"Ġgroom":39841,"Ġglimpse":39842,"Ġnails":39843,"Ġcumulative":39844,"Ġfazer":39845,"_services":39846,".Num":39847,"ibilit":39848,"_resolution":39849,"ĠTx":39850,"uminium":39851,"opa":39852,".schedule":39853,"smtp":39854,"à¸ķ":39855,"urry":39856,"ük":39857,"goog":39858,"_signature":39859,".into":39860,"ĠSteps":39861,"Ġhomeowners":39862,"ĠNSURL":39863,"ĠPAC":39864,"ĠĠĠĠĠĠĠĠĠĠĠĠĊĊ":39865,">')Ċ":39866,"enh":39867,"Ġincap":39868,"$MESS":39869,"Ġmoins":39870,"ĠFi":39871,"Ġoffseason":39872,"pressions":39873,">.Ċ":39945,"ĠGrass":39946,"ĠGoal":39947,"_pdf":39948,"Handlers":39949,"Ġstacks":39950,".getFullYear":39951,"=[];Ċ":39952,"车":39953,",V":39954,"(split":39955,"Ñĥнк":39956,"Ġbakeca":39957,"Ġ~/.":39958,"pez":39959,"tails":39960,"ĠGlen":39961,"ĠsetImage":39962,"ĠComic":39963,"BLOCK":39964,"ĉThis":39965,"oader":39966,"Ġcapitalist":39967,"_STEP":39968,"(Boolean":39969,"ĠCorrect":39970,"rina":39971,"Ġconcaten":39972,"å®ŀ":39973,"():ĊĊ":39974,"Ġunanim":39975,"lli":39976,"alars":39977,"-ne":39978,"Ġdivor":39979,"ĠKickstarter":39980,"]._":39981,"*'+":40622,"åĿĢ":40623,"acency":40624,"(URL":40625,"_half":40626,"=l":40627,"ĠlistView":40628,"(section":40629,".toArray":40630,"+/":40631,"ĠRodriguez":40632,"istream":40633,"Ġeligibility":40634,"::-":40635,".newInstance":40636,"PB":40637,"ĠAssets":40638,"ĠComposite":40639,"ĠLabs":40640,"ĠHamas":40641,"++);Ċ":40642,"Ġblk":40643,"ĠNeo":40644,"Luc":40645,"@login":40646,"Ġunaware":40647,".met":40648,"_RELEASE":40649,"(ST":40650,"AMIL":40651,"rike":40652,"Ġ(){Ċ":40653,"(sprintf":40654,"ĠAccounts":40655,"ĠVIEW":40656,"ĠAj":40657,"ãĤ°":40658,"Ġwhisk":40659,"Ġidi":40660,"Ġrode":40661,"Ġihn":40662,"ĠElementary":40663,"Qty":40664,"Ġintriguing":40665,"Ġå¤":40666,"Jobs":40667,"ĉoffset":40668,"ĠAhmed":40669,"ĠTaliban":40670,"Ġèİ·åıĸ":40671,"Ġinjected":40672,".Authentication":40673,"_linear":40674,".Decimal":40675,"Ġapples":40676,"Ġshareholders":40677,"Ġbaked":40678,".diff":40679,"ĠEddie":40680,"okers":40681,"Ġconfronted":40682,"voices":40683,"Ġtus":40684,"ĠSpin":40685,"NODE":40686,"_Un":40687,"CTX":40688,"/google":40689,"Temperature":40690,"Ġ'').":40691,"Ġmagnificent":40692,"ĠstartIndex":40693,"sembles":40694,"Anyone":40695,"zk":40696,"ehen":40697,"ĠDame":40698,".strict":40699,"Ġreplaces":40700,"Ġlineback":40701,"Ġpushes":40702,"Ġcheek":40703,"ĠShi":40704,"_BYTES":40705,"REA":40706,"ản":40707,"_CONNECTION":40708,"Gateway":40709,"ĠTravis":40710,"ĠAX":40711,"ĠBasically":40712,"ĠUpgrade":40713,"àª":40714,"themes":40715,"ermo":40716,"kor":40717,"Female":40718,"_attach":40719,"ĠìĤ¬ìļ©":40720,"Ġpoz":40721,"==============Ċ":40722,"(symbol":40723,"ĠSector":40724,"__)ĊĊ":40725,"_padding":40726,"ï¼ļ\"":40727,"Ġfabs":40728,"Ġranged":40729,"setName":40730,"Ġperror":40731,"âĹ":40732,"ĠFileReader":40733,"Ġfulfilled":40734,"_Current":40735,"Ġdominate":40736,"Ġsmugg":40737,"PostMapping":40738,"_force":40739,"Ġbloc":40740,"ĠGiant":40741,"(video":40742,"ĠCU":40743,"SystemService":40744,"Ġelf":40745,"Ġkontakt":40746,"ëª":40747,"kees":40748,"gtk":40749,"ĠparamInt":40750,"Ġmarkup":40751,"uales":40752,"Ġaccounted":40753,"Ġgangbang":40754,"RYPT":40755,"ĠWrong":40756,"Ġcredited":40757,"ĠMESSAGE":40758,"Ġflaws":40759,"Ġbbw":40760,"Ġmetabolic":40761,"ĠOEM":40762,"/event":40763,"(Collectors":40764,"monton":40765,"appear":40766,"Ġopted":40767,"Ġcheat":40768,"Ġdav":40769,"ĠProceed":40770,"Ġê¸":40771,"anked":40772,"из":40773,"ansk":40774,"ĠHang":40775,"ĠCler":40776,"Ġdisgu":40777,"Ġcmap":40778,".cljs":40779,"Ġaument":40780,"lez":40781,"ĠJoined":40782,"_received":40783,"Ġaerial":40784,"otel":40785,"Ġgreet":40786,"\"s":40787,"ĠGenesis":40788,"ĠCalif":40789,"panion":40790,"Ġtailored":40791,"mapping":40792,"andExpect":40793,".track":40794,"atomy":40795,"ĠOw":40796,"ullah":40797,".Yes":40798,"ĠSimpleName":40799,"dbh":40800,"'en":40801,"Ġnonsense":40802,"Ġphilosophical":40803,"(getContext":40804,"Ġisso":40805,"ĠACE":40806,"startDate":40807,"ĠbÄĻd":40808,"ĠAUTHOR":40809,"ĠGlobe":40810,"Ġinsects":40811,"_Al":40812,"ushing":40813,"è®°":40814,"/Home":40815,"ĠLocalDate":40816,"needed":40817,"hesive":40818,"Ġillusion":40819,"äºĮ":40820,"Ġtrat":40821,"xo":40822,"/detail":40823,"_MATCH":40824,"Ġbroadband":40825,"Ġwal":40826,"ĠIllegalStateException":40827,"IRECTION":40828,"Ġnortheast":40829,"esium":40830,"ĠCliente":40831,"ulance":40832,"nty":40833,"Ġtecn":40834,"Devices":40835,"Ġgrains":40836,"ĠOg":40837,"ĠSEL":40838,"udiant":40839,"Ġ++;Ċ":40840,"Ġexplanations":40841,"occo":40842,"Ġdiets":40843,"Ġcohort":40844,"(controller":40845,".Iterator":40846,"-rich":40847,"rocess":40848,"GD":40849,"Ġcarbohydr":40850,"Ġfried":40851,"ĠEmployment":40852,"ìŀ¥":40853,"ĠLeonard":40854,"_${":40855,"quares":40856,"Ġcompanions":40857,"Ġparis":40858,"Ġstimulation":40859,"ĠZoo":40860,"Ġrelevance":40861,"ĠColour":40862,"Ġspear":40863,"otional":40864,"ĠLite":40865,"ĠKosten":40866,"Ġó":40867,"_attachment":40868,"orphic":40869,"Ġdamit":40870,"Ġdlg":40871,"Ġthrive":40872,"CHANGE":40873,"ĠApparently":40874,"Ġatual":40875,"Ġrooted":40876,"(images":40877,"awi":40878,"ariat":40879,"Ġcherry":40880,"STATIC":40881,"mnt":40882,"ĠUserId":40883,"illet":40884,"ĠHispanic":40885,"Ġnak":40886,"Ġcentro":40887,"Ġdims":40888,"_initialize":40889,"ık":40890,"ĠCenters":40891,"REN":40892,"Ġevolutionary":40893,"ĠTopics":40894,"_damage":40895,"emer":40896,"Ġrund":40897,"Ġpunished":40898,"Ġcubic":40899,"fair":40900,"[];ĊĊ":40901,"Ġinstantiate":40902,"Ġoversee":40903,"-delete":40904,"unteer":40905,"startTime":40906,"ĠPipeline":40907,"_GAME":40908,"ĠCir":40909,"ĉNull":40910,".Formatting":40911,"ucumber":40912,"ĠRide":40913,"Ġzoo":40914,"Ġchecker":40915,"åIJĮ":40916,"=C":40917,"Ġgrit":40918,"\");//":40919,"_xy":40920,"ĠDeclaration":40921,"Ġcallable":40922,"Foo":40923,"ĠListItem":40924,"Ġinaccur":40925,"mlin":40926,"ĉData":40927,"Ġevolving":40928,"awan":40929,"Ġcafe":40930,"folk":40931,"_IDX":40932,"ĠAnything":40933,"ĠPalestine":40934,"ĠGridView":40935,"Ġcolony":40936,"ĠGermans":40937,"(+":40938,".pid":40939,".jsx":40940,"ĠSuperior":40941,"Christian":40942,"ĠLect":40943,"ĉGame":40944,"Ġinstrumental":40945,"Animations":40946,"дал":40947,"ĠMoses":40948,"ĉĉčĊĉĉčĊ":40949,"zs":40950,"kte":40951,"ä¸ļ":40952,"_DIST":40953,"bitmap":40954,"dB":40955,"Ġpersistence":40956,"ÑĢоÑģ":40957,"$l":40958,"Bron":40959,"Ġ{|":40960,"_chart":40961,"ĠConsum":40962,"Ġhemp":40963,"Ġ\"))Ċ":40964,"Ġattackers":40965,"Ġknowledgeable":40966,"Ġcet":40967,"Ġviruses":40968,"'I":40969,"Ġpitcher":40970,"Ġsweeping":40971,"=list":40972,"aptops":40973,".depth":40974,"Ġinstructed":40975,"ĠRus":40976,"benhavn":40977,"Ġин":40978,"Sports":40979,"Ġonset":40980,"æĿĥ":40981,".RED":40982,"_si":40983,"ĠPST":40984,".onChange":40985,">tag":40986,"ĠRoh":40987,"_character":40988,"ĠLaws":40989,"ĠBachelor":40990,"_swap":40991,".reactivex":40992,"Ġrewarding":40993,"Medium":40994,"-[":40995,"ĠRecently":40996,"Joint":40997,"partition":40998,"ĠMinutes":40999,"Ġindo":41000,"Ġabsorbed":41001,"ĠGN":41002,"_IND":41003,"Ġsaber":41004,"Spawn":41005,"outputs":41006,"ĠJeffrey":41007,"Ġmedieval":41008,"hed":41009,"Guide":41010,"Ġpsycho":41011,"Ġglam":41012,"Elim":41013,"ädchen":41014,"_plain":41015,"ĠSau":41016,"-four":41017,"Ġanalyzing":41018,"QUERY":41019,"Ġtomato":41020,"_buttons":41021,"VEN":41022,".setStatus":41023,".Url":41024,"+ĊĊ":41025,"Ġcomplaining":41026,"degree":41027,"confirmed":41028,"Ġsubt":41029,"parsed":41030,"Ġtorque":41031,"Ġtroubled":41032,"ĠTARGET":41033,"Ġtrademarks":41034,"ĠCoordinate":41035,"ĠViv":41036,"Ġ//}ĊĊ":41037,"Ġaprès":41038,".getPosition":41039,"(KeyCode":41040,"ĠSilva":41041,"Ġmeteor":41042,"Ġendorsement":41043,"Overview":41044,"ĠPoss":41045,".Inject":41046,"Ġevenly":41047,"Ġvisualization":41048,"Ġwchar":41049,"ĠHDMI":41050,"Ġfunct":41051,"ickname":41052,"','','":41053,"Ġforwards":41054,"ManagedObject":41055,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":41056,"ĉserver":41057,"ĠOutlook":41058,"ĠChronicle":41059,"Ġdubbed":41060,"Ġdok":41061,"ĠWear":41062,".AL":41063,"paren":41064,".Interface":41065,"Interfaces":41066,".cod":41067,"Ġdib":41068,".Globalization":41069,"ĠAcademic":41070,"Ġassms":41071,"Autom":41072,"Ġlw":41073,"ĠNW":41074,"Ġ&&čĊ":41075,"Ġproblema":41076,"ĠManufacturing":41077,"limits":41078,"-mobile":41079,"Ġfilme":41080,"/map":41081,"Ġdoit":41082,"ĠInk":41083,"Ġsued":41084,".arr":41085,"Ġundermin":41086,"ĠProc":41087,"crollView":41088,"__$":41089,"Ġsidewalk":41090,"(that":41091,"ื":41092,"[q":41093,"grammar":41094,"Ġtë":41095,"quito":41096,"Ġspiral":41097,"extended":41098,"Ġfocal":41099,"Ġdigging":41100,"pas":41101,"ĠTall":41102,".proxy":41103,"itures":41104,"TRACT":41105,"ĠRealm":41106,"Ġfeder":41107,"Ġoriented":41108,"ĠAlternative":41109,"Ġowe":41110,"Ġsourced":41111,"inker":41112,".det":41113,"Sep":41114,"ĠQui":41115,"ĠPalmer":41116,"(_,":41117,"samples":41118,"oyer":41119,"ullan":41120,"quez":41121,"Edges":41122,"Ġshout":41123,"ĠAchie":41124,"Ġhaar":41125,"_Construct":41126,"Ġpremature":41127,"Ġrevert":41128,"').Ċ":41129,"Ġschn":41130,"filtered":41131,"nullptr":41132,"Saved":41133,"itecture":41134,"CLA":41135,"Ġvl":41136,"stell":41137,"ĉMe":41138,"ĠLip":41139,"national":41140,"Ġwholly":41141,"Ġsprings":41142,".Timer":41143,"ĉsrc":41144,"elsen":41145,"åħ¶":41146,"Ġcommunicating":41147,"ĠQuiz":41148,"Ġteng":41149,"Ġgez":41150,"ĠOutside":41151,".Sign":41152,"(cs":41153,"Ġdisputes":41154,"ĠWeiss":41155,"annes":41156,">No":41157,"ĠBach":41158,".removeAll":41159,"refer":41160,"/dashboard":41161,"ĠAjax":41162,"IndexChanged":41163,"ĠWeak":41164,"'\"Ċ":41165,"Ġsights":41166,"accessToken":41167,"ĠJoi":41168,"(domain":41169,"ĉcv":41170,"Ġcontinuation":41171,"Ġplum":41172,"adir":41173,".setMessage":41174,"Ġï¼Į":41175,"Ġswallow":41176,"ĠLamp":41177,"Ġqw":41178,"Ġuu":41179,"Coin":41180,"ubic":41181,"ĠDeals":41182,"race":41183,"Ġdictator":41184,"Ġmeme":41185,"turned":41186,"ĠJulie":41187,".gridColumn":41188,"Ġpuppy":41189,"Ġpam":41190,"Ġ){čĊ":41191,"Ġinviting":41192,"Ġfrench":41193,"vim":41194,"Ġwrapping":41195,"Ġ#-}Ċ":41196,"([-":41197,"Early":41198,"Ġshiny":41199,".faces":41200,"Ġrebell":41201,"abcdef":41202,"ält":41203,"Ġestimation":41204,"phys":41205,"losures":41206,"_REL":41207,"Ġexclusion":41208,"ĠSkype":41209,"weise":41210,"-stop":41211,"nothing":41212,"ĠEgg":41213,"isors":41214,"Richard":41215,"Ġcounseling":41216,"Ġcommem":41217,"ĠQMessageBox":41218,"ĠSynd":41219,"ĠFrost":41220,"ĠCompetition":41221,"ĠAwake":41222,"Ġted":41223,"iciones":41224,"ĠDevComponents":41225,"VERTISEMENT":41226,"otti":41227,".runner":41228,"Ġuniquely":41229,".flag":41230,"ĉrs":41231,"_generic":41232,"Ġ```Ċ":41233,"ACHINE":41234,"Ġmein":41235,"(Application":41236,"(br":41237,"Ġratios":41238,":,":41239,"ĠXCTest":41240,"ustainable":41241,"-www":41242,"itles":41243,"_TEMP":41244,"Ġsyst":41245,"umericUpDown":41246,"ĉassertTrue":41247,"Ġwf":41248,".peek":41249,"ĠBulg":41250,"Ġterrifying":41251,".MODE":41252,"ĠGW":41253,"ár":41254,"Ġfic":41255,"Ġcommitments":41256,"-tech":41257,"ĠLiquid":41258,"opez":41259,"zheimer":41260,"aña":41261,"-media":41262,"(animated":41263,"_goal":41264,"Ġgum":41265,"ystone":41266,".SET":41267,"ĠWend":41268,"setCellValue":41269,"Ġmsgs":41270,"cash":41271,"ALLOC":41272,"/aws":41273,"Ġmicrowave":41274,".Pointer":41275,"ĉConsole":41276,"_sorted":41277,"ĠFilip":41278,"Prod":41279,"Ġ//!<":41280,"ingroup":41281,"Ġks":41282,"_TRI":41283,"Ġteaspoon":41284,"ĠATT":41285,"Ġrecovering":41286,"ĠGLOBAL":41287,".Par":41288,"Ġ/>;Ċ":41289,"Ġmarble":41290,"ulators":41291,"ĠCycle":41292,"Ġherbs":41293,"_metric":41294,")!":41295,"_CLOCK":41296,"_Button":41297,"Harry":41298,"è¿Ľ":41299,"Ġstrains":41300,"ĠAppBar":41301,"ĠChan":41302,"/video":41303,"Ġbam":41304,".Progress":41305,"$f":41306,"lemen":41307,"Ġirregular":41308,"ĠDuncan":41309,"ĠMint":41310,"-video":41311,"া":41312,"ówn":41313,"ĠEMPTY":41314,"Ġstacked":41315,"ĠHA":41316,"_cut":41317,"Ġwherein":41318,"ĠWays":41319,"(counter":41320,"è¯ķ":41321,"FormGroup":41322,"Ġblew":41323,"courses":41324,"Ġproductos":41325,"rys":41326,"ĠRestr":41327,"Ġstyling":41328,">s":41329,"Ġpiv":41330,"Ġitertools":41331,"getRepository":41332,"ĠIk":41333,"_devices":41334,"layui":41335,"Ġhalfway":41336,"Ġfranç":41337,"Ġtuning":41338,"OA":41339,"_Node":41340,"arde":41341,"Ġfierce":41342,"licted":41343,"#čĊ":41344,"Ġbreakthrough":41345,"ĠErik":41346,"Ġbride":41347,"Ġ.\"":41348,"culus":41349,"inside":41350,"ĠIndianapolis":41351,"ĠEE":41352,"Ġyog":41353,"urret":41354,".fs":41355,".grad":41356,"_cards":41357,"_accuracy":41358,"_epi":41359,"queda":41360,"/org":41361,"éªĮ":41362,"Ġcompte":41363,"))[":41364,"Outside":41365,"Greater":41366,"ĠRenderer":41367,".actor":41368,"Accounts":41369,"Idle":41370,"_hours":41371,"erner":41372,"Joined":41373,"Ġmenj":41374,"requires":41375,"ĠOPER":41376,".removeChild":41377,"ĉsp":41378,"Ġesse":41379,"rift":41380,"xFE":41381,"ĠShakespeare":41382,"____________":41383,"Ġbudgets":41384,"ModelState":41385,"fillable":41386,"-component":41387,"ocos":41388,"ĠBUTTON":41389,"/io":41390,",out":41391,"sms":41392,"Thomas":41393,"ĠArmed":41394,"resume":41395,"Ġrotating":41396,"ĠVault":41397,"Ġseus":41398,".(*":41399,"Ġamino":41400,"Ġ[]);ĊĊ":41401,"Ġprovoc":41402,"nox":41403,".GetEnumerator":41404,"=======Ċ":41405,"æĸĻ":41406,"_scroll":41407,"Ġfilmed":41408,"ĠSoci":41409,"gap":41410,"gro":41411,"Vote":41412,"\"But":41413,"_RC":41414,"Animal":41415,"ÂĢ":41416,"ibile":41417,"Ġawaken":41418,"orest":41419,"inja":41420,"ĠIvan":41421,"(Command":41422,"Ġ*****":41423,"η":41424,"Ġkvinder":41425,"/helpers":41426,"_cases":41427,"tg":41428,"ìĦ¸":41429,"Registered":41430,"ĉpass":41431,"_digits":41432,"Ġcontour":41433,"Ġinfants":41434,"Ġjustification":41435,"ĠFortunately":41436,"Contr":41437,"ĠonCreateView":41438,"_SAMPLE":41439,"ĠallowNull":41440,"Ġnud":41441,"Ġfetched":41442,"_equ":41443,"ĠUnable":41444,"=\\\"\"":41445,">{Ċ":41446,"Ġcommittees":41447,"istema":41448,"+\".":41449,"ÃŃan":41450,"mant":41451,"Ġsoutheast":41452,"ï¼ĮĊ":41453,"dialogs":41454,"PROJECT":41455,"charger":41456,"-port":41457,"(uuid":41458,".export":41459,"Six":41460,"ĠRP":41461,"Prem":41462,"Ġconscience":41463,"ĠmarginRight":41464,"_distribution":41465,"yaml":41466,"resizing":41467,"Dock":41468,"ĠLocations":41469,"GY":41470,"Seed":41471,"BUFFER":41472,"ossip":41473,"ullen":41474,"Things":41475,"-self":41476,".poll":41477,"PLAYER":41478,"Ġå®":41479,"GROUP":41480,"ĠAway":41481,"Ġgospel":41482,"xfd":41483,"Mary":41484,"ĠPortable":41485,"TURE":41486,"Ġutilis":41487,"Ġseit":41488,"Ġstrand":41489,"Ġtransc":41490,"Ġ(^":41491,"ĠAlfred":41492,".mem":41493,".circle":41494,"Ġ~/":41495,"forcing":41496,"Ġriot":41497,"prox":41498,"THON":41499,"ización":41500,"ĠNI":41501,"rost":41502,"Ġdispro":41503,"_instances":41504,"ï¼ĮâĢľ":41505,"ographer":41506,"endas":41507,"ĠIsaac":41508,"ĠPine":41509,"/dis":41510,"ĠcolorWith":41511,"iterate":41512,"_stride":41513,"Ġpunto":41514,".EventArgs":41515,"(center":41516,"Ġneighboring":41517,"ĠPrison":41518,"ĠMessenger":41519,"Ġepidemic":41520,"dao":41521,"_complex":41522,"Ġgravel":41523,"_DIP":41524,"ément":41525,"ĠAri":41526,"_bitmap":41527,".quit":41528,"(valid":41529,"Ġpend":41530,"Ġrespiratory":41531,"Ġrebound":41532,"DefaultValue":41533,"ãĥŃ":41534,"Ġcommits":41535,".tests":41536,"_fr":41537,"itet":41538,".sf":41539,"Ġspacecraft":41540,"critical":41541,"Ġdepressed":41542,"ĠAnyObject":41543,"Ġunb":41544,"Ġdiscern":41545,"(mysql":41546,"Latin":41547,"ĠBog":41548,"ĠWildlife":41549,"ToFile":41550,"ioxid":41551,"@RestController":41552,"Ġ\"$(":41553,"Ġ<<\"":41554,"Ġdefects":41555,"Ġdatum":41556,"hin":41557,"Ġrealizar":41558,"anyahu":41559,"ĠSig":41560,"@Data":41561,"adaptive":41562,"ĠCatherine":41563,".cr":41564,"ĠCOOKIE":41565,"Ġpictured":41566,"ĠFighter":41567,"Queryable":41568,"ĠAnyway":41569,"ĠGLFW":41570,"_namespace":41571,"_ft":41572,"Ġ])":41573,"Organization":41574,"Ġconstitutes":41575,"Ġquand":41576,"(chunk":41577,"\"/>čĊ":41578,"ĠLakes":41579,"mainwindow":41580,"Carthy":41581,"spin":41582,"(csv":41583,":red":41584,"-commerce":41585,"ู":41586,"Ġdiscovering":41587,"Ġeco":41588,"_fac":41589,"inceton":41590,"ĠGreens":41591,"jwt":41592,"ص":41593,"ĠBroncos":41594,"ĠGoods":41595,"(GTK":41596,"ĠreturnValue":41597,"Ġsiempre":41598,"Ġneutr":41599,"went":41600,"ĠNatal":41601,"Ġenthusiastic":41602,"á»į":41603,"FN":41604,"/database":41605,"Catalog":41606,"Ġbrun":41607,"ĠKash":41608,"_Pl":41609,"iscrim":41610,",width":41611,"Ġinmates":41612,"Assignment":41613,"ĠHaven":41614,"Ġplayground":41615,"exam":41616,"@Controller":41617,"uliar":41618,".getParent":41619,"Ġ\";ĊĊ":41620,":size":41621,"issors":41622,"Ġfis":41623,"Ġalc":41624,"ensation":41625,"ĠNixon":41626,"Ġmighty":41627,"-str":41628,"_special":41629,"_ADC":41630,"ĠTwig":41631,"umbling":41632,"-address":41633,"Ġheroin":41634,"YTE":41635,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":41636,"Friend":41637,"Ġave":41638,"ĠPNG":41639,"ĠKurdish":41640,"DataSetChanged":41641,"Ġblades":41642,"bral":41643,"Steam":41644,"Ġsigu":41645,"IRTUAL":41646,"acos":41647,"UDP":41648,"(database":41649,"hec":41650,"ĠStrings":41651,"_scalar":41652,"ĉdesc":41653,"ĠTLS":41654,";\"Ċ":41655,"ĠCorbyn":41656,"SimpleName":41657,"uell":41658,"ĠEntre":41659,"ellites":41660,"-place":41661,"Ġfrankly":41662,"ĠErf":41663,"CEL":41664,"ĠpaÃŃs":41665,"Ġhedge":41666,"Ġlatent":41667,"ĠIRQ":41668,"ĠHerald":41669,"ĠPrec":41670,"ë³´":41671,".TEXT":41672,"Salary":41673,"Ġautumn":41674,"Ġtravail":41675,".Sum":41676,"Ġcared":41677,"Mor":41678,"Ġintuitive":41679,"Ġjournals":41680,"_IT":41681,"ĠTrou":41682,"ä¼ł":41683,"HasColumnName":41684,"Composite":41685,"Ġspice":41686,"_disk":41687,"_CODES":41688,"ĠIntroduced":41689,"iona":41690,"Ġnuestra":41691,"oct":41692,"ĠĠĠĠĊĠĠĠĠĊĠĠĠĠĊ":41693,"(parameter":41694,"Ġstudios":41695,"ĠprojectId":41696,"Ġbdsm":41697,".SqlClient":41698,"imizer":41699,"ĠCARD":41700,"+t":41701,"aan":41702,".sol":41703,"_Adjust":41704,"Ġrighteous":41705,"ĠLogging":41706,".filters":41707,"_TAB":41708,"ĉsys":41709,"rophic":41710,"otherapy":41711,"ĠBrowse":41712,"keyboard":41713,"RON":41714,"+\\":41715,"ropped":41716,"Ġextensively":41717,"fk":41718,"Ġlime":41719,"years":41720,"Exc":41721,"Ġsph":41722,"Ġcheating":41723,"andro":41724,"ÃŃo":41725,"Ġprince":41726,"oire":41727,"ĠDestination":41728,"ĠConverts":41729,"Ġupstream":41730,"oled":41731,"Ġservants":41732,"Ġsemantic":41733,"Ġcrunch":41734,"Ġeventual":41735,"runner":41736,"/error":41737,"Spin":41738,"Ġsecretly":41739,"Ġassemble":41740,".Person":41741,"enderror":41742,"_<":41743,"Ġpendant":41744,"Sleep":41745,"ĠChemistry":41746,"Ġbosses":41747,"lk":41748,"))),Ċ":41749,"Blockly":41750,"DEVICE":41751,"Ġreflecting":41752,"Ġample":41753,"Milliseconds":41754,"ĠPresidential":41755,"Ġusuarios":41756,"ĠNZ":41757,"ĠSalary":41758,"ĠAmanda":41759,"_np":41760,"jury":41761,"Ġkön":41762,"Ġtherapist":41763,"Ġhomosexual":41764,"ĠDrake":41765,"-window":41766,"ĠLocated":41767,".Driver":41768,"ĠVIDEO":41769,"Ġmerchants":41770,"ĠChest":41771,"-lock":41772,"/php":41773,"Ġmilano":41774,"_STYLE":41775,"arger":41776,"idea":41777,"GUID":41778,"advanced":41779,"meal":41780,"OptionsItemSelected":41781,"='%":41782,"ĠCham":41783,":data":41784,"(stat":41785,"WillAppear":41786,"Ġinformal":41787,"aji":41788,"Ġreproductive":41789,"ĠCAS":41790,"ãģ£":41791,"FUNC":41792,"ĠRuth":41793,")+(":41794,"CONST":41795,"ĠFans":41796,"ĠgroupId":41797,"xffffffff":41798,"Ġsampler":41799,"Ġ}}\">":41800,".the":41801,"Ġhollow":41802,"WAY":41803,"ĠFaculty":41804,"AttributedString":41805,"ĠLooks":41806,"ĠRex":41807,"jk":41808,"ĠMIL":41809,"Ġbard":41810,".Long":41811,"Ġlivest":41812,"Ġskal":41813,"icism":41814,"MAIN":41815,"Ġmucho":41816,"BODY":41817,"Ġese":41818,"ĉuse":41819,"Foot":41820,".SQLException":41821,"Ġinheritance":41822,"received":41823,"Ġputas":41824,"edis":41825,"alsa":41826,"ĠErrorMessage":41827,"Booking":41828,"Ġtract":41829,"acz":41830,"ĠCant":41831,"_regex":41832,"Ġideological":41833,"Ġjihad":41834,"hos":41835,"/sys":41836,"colm":41837,"(pool":41838,"Ġestán":41839,"ĠPending":41840,"emás":41841,"Ġktóry":41842,"));ĊĊĊ":41843,"transactions":41844,"Ġwield":41845,"itere":41846,"erture":41847,"_ss":41848,"Ġstretching":41849,"Ġprisoner":41850,".ReadAll":41851,"Ġbesch":41852,"--;čĊ":41853,"Ġcrisp":41854,"_SCAN":41855,"Ġae":41856,"Strict":41857,"ĠMinneapolis":41858,"ĠBoeing":41859,"aris":41860,"rek":41861,"_pipe":41862,"Ġpriests":41863,"(EIF":41864,"ehicles":41865,"ĠInteractive":41866,"between":41867,"ĉNullCheck":41868,"ĠBlair":41869,"ĠLt":41870,"_inline":41871,"ethyl":41872,"¼":41873,"_packages":41874,"Ġbarrels":41875,"_he":41876,"Ġregexp":41877,"_pts":41878,"_Handler":41879,"ingular":41880,"ĠNissan":41881,"ĠRanch":41882,"Ġperch":41883,"Unsupported":41884,"Smith":41885,"ĠLegends":41886,"Mi":41887,"Ġgf":41888,"steder":41889,"Ġacquiring":41890,"Ġsimulator":41891,"(),\"":41892,"receive":41893,"Ġinplace":41894,"ACTION":41895,"ĠWebDriver":41896,"filesystem":41897,"'+Ċ":41909,"Ġcredible":41910,"amat":41911,"playing":41912,".setImageResource":41913,"quel":41914,"Ġpodr":41915,"geom":41916,"Ek":41917,"ĠQatar":41918,"Ġgeld":41919,"?',Ċ":41920,"Ġcyl":41921,"(ax":41922,"ĠWI":41923,"urally":41924,"ĠBrasil":41925,"Ġsenza":41926,"aley":41927,"onen":41928,"Ġbah":41929,"Ġmolecule":41930,"Rad":41931,"è¿°":41932,"ANCH":41933,"-background":41934,"-agent":41935,"Ġprolifer":41936,":boolean":41937,"Ġtide":41938,"erializer":41939,"_;čĊ":41940,"Fee":41941,"**)":41942,"ergy":41943,"ĠHonor":41944,".Logging":41945,"iris":41946,"Ġundermine":41947,"ĠDy":41948,"Ġtyr":41949,"Ġdeque":41950,"Ġdamer":41951,"([])Ċ":41952,".layoutControlItem":41953,"peated":41954,"CAN":41955,"ragments":41956,"Land":41957,")]);Ċ":41958,"ĠSah":41959,"ĠDECL":41960,"Within":41961,"ĠNamespace":41962,"another":41963,"sembling":41964,".describe":41965,"Consum":41966,"ĠFear":41967,"given":41968,"Orange":41969,"This":41993,"ĠdataIndex":41994,"Ġprintable":41995,"ĠEyes":41996,"_targets":41997,"(Py":41998,".over":41999,"Ġbru":42000,"ampton":42001,"Ġplaintiff":42002,");Ċ":42013,"invest":42014,".*ĊĊ":42015,"Ġtélé":42016,"Ġsuperf":42017,"Ġcascade":42018,"DTD":42019,"Ġvivid":42020,"Ġsubsidies":42021,"ĠHass":42022,"Ġcollaps":42023,"Ġceramic":42024,"{}\".":42025,"ĠLeakage":42026,"-trash":42027,"collapsed":42028,"-social":42029,"ĠChad":42030,"Ġinclined":42031,"Ġsto":42032,"Ġstoryboard":42033,".payment":42034,"stackoverflow":42035,"ĠRaiders":42036,"Ġ#'":42037,"olicies":42038,"ìľ¼ë¡ľ":42039,"emap":42040,"Ġkj":42041,"Ġquota":42042,"ĠGardens":42043,"ë²Ī":42044,"ĠAngels":42045,"Ġoft":42046,"Ġlowercase":42047,"ĠiParam":42048,"Ġcheapest":42049,"unta":42050,"_pkt":42051,"icators":42052,"Ġleurs":42053,"Ġdecreases":42054,"ĉdefine":42055,"PREC":42056,"ammers":42057,"ĠPreparedStatement":42058,"(direction":42059,"Ġcrews":42060,"arked":42061,"ĠMemphis":42062,"ĠSell":42063,"GTK":42064,"Ġmaid":42065,":disable":42066,"éĽĨ":42067,"ĠPf":42068,"Ġalbeit":42069,"openh":42070,"?>\">Ċ":42071,".getSource":42072,"(scale":42073,"Du":42074,"ĠPIL":42075,"_refresh":42076,"Ġbets":42077,"(car":42078,"ĠVon":42079,"|--------------------------------------------------------------------------Ċ":42080,"ĠGrat":42081,"Much":42082,"(Dialog":42083,".stopPropagation":42084,"Ġtek":42085,"Ġexits":42086,"'],$":42087,"ĠphoneNumber":42088,"ucs":42089,"ecimal":42090,"--------------":42091,"inp":42092,".pojo":42093,"Ġcorpus":42094,"Ġpractitioners":42095,".pic":42096,"\"testing":42097,"ĠstringBy":42098,".NotNull":42099,"Ġrang":42100,".Dynamic":42101,"_Render":42102,"аÑĤа":42103,"Waiting":42104,"ĠWik":42105,"Ġoverwhelmed":42106,"%\">":42107,"ĠAE":42108,"}}>Ċ":42109,"uw":42110,"_typ":42111,"Ġbuckets":42112,"Ġgreeting":42113,"Ġlaughter":42114,"Ġantagon":42115,"uggestion":42116,"-email":42117,"ĉtop":42118,"Ġeros":42119,"_tri":42120,"Ġissuing":42121,"Ġhá":42122,"Ġisolate":42123,"Overflow":42124,",E":42125,"Ġnutritional":42126,"ĠAbbott":42127,"Ġnf":42128,".touch":42129,".fetchall":42130,"_zip":42131,"\")}Ċ":42132,"Ġamat":42133,"ĠCisco":42134,"ĠnÃ¥":42135,"PLEX":42136,"Ġsei":42137,"foto":42138,".toJson":42139,"å¤ļ":42140,"ĠKlein":42141,"Ġlibc":42142,"Ġminers":42143,"å¢":42144,"-print":42145,"ĠPride":42146,"Todos":42147,"Ġmasked":42148,"ĠsetData":42149,"Ġtelefon":42150,"Ġunhappy":42151,"ĠTables":42152,"geb":42153,"(debug":42154,"_allowed":42155,"-access":42156,"Ġlogistics":42157,"Ġgems":42158,"ĠMature":42159,"Ġrsp":42160,"ĠAlle":42161,".getBytes":42162,"\\web":42163,"ynchronized":42164,"Paragraph":42165,"Ġthrottle":42166,".sqlite":42167,"consulta":42168,"ĠSeah":42169,"Ce":42170,"Ġsubmar":42171,"ERE":42172,"Vous":42173,"Ġreddit":42174,"Ġsqlalchemy":42175,"-mile":42176,"ocide":42177,"Pour":42178,"}}\">Ċ":42179,"stead":42180,"Ġ@(":42181,"Ġ[])":42182,"ĠAds":42183,"Ġoverload":42184,"ridden":42185,"ĠDesert":42186,"ĠWrap":42187,"ĠPortuguese":42188,"etz":42189,"ĉfirst":42190,"Ġmilestone":42191,"æĹł":42192,"ÑĥÑī":42193,"(success":42194,"\")Ċ":42363,"ĠDollar":42364,"Ġemoji":42365,"Carousel":42366,"-player":42367,"Ġadjusting":42368,"Ġjuga":42369,"allenges":42370,"gene":42371,"(bodyParser":42372,"lopedia":42373,"ĠBehind":42374,"Ġsleeves":42375,"Ġdragging":42376,"ĠChevrolet":42377,"Ġbiz":42378,"ivities":42379,"ĠFrequency":42380,",char":42381,".WHITE":42382,"_preview":42383,")';Ċ":42384,"_ax":42385,"IONS":42386,".cpu":42387,".inputs":42388,"UBE":42389,"_feed":42390,"ĠSupplement":42391,"!).":42392,"esus":42393,"ĠUDP":42394,"Ġmicrophone":42395,"Ġconfirms":42396,".isNotEmpty":42397,"\":\"\",Ċ":42398,"_SCREEN":42399,"ĉexpected":42400,"+-+-+-+-":42401,"ĠHait":42402,"fastcall":42403,"Ġdepict":42404,"vb":42405,"_picture":42406,"ĉdescription":42407,"ĠWife":42408,"uci":42409,"Ġvicious":42410,"ä»ĸ":42411,"ueba":42412,"ĠsetUser":42413,"ãģ¡":42414,"Ġdiving":42415,"Ġopera":42416,"usercontent":42417,"arah":42418,")},":42419,"yun":42420,"velt":42421,"Ġuncovered":42422,"Ġhips":42423,"Ġoscill":42424,"Ġasserting":42425,"ĠXi":42426,".restore":42427,"kea":42428,"Ġspelling":42429,"Ġderive":42430,"abwe":42431,"ĠDow":42432,".setType":42433,"_vs":42434,"Ġcozy":42435,".categories":42436,"Org":42437,"_mgr":42438,"Ġdungeon":42439,"collectionView":42440,"ĠBlank":42441,"acias":42442,"ää":42443,"_cleanup":42444,"_ACTIVITY":42445,"Ġtriangles":42446,".MenuItem":42447,"Ġiphone":42448,"ĠWon":42449,"]]ĊĊ":42450,"ĠComparison":42451,".Doc":42452,"Ġcanonical":42453,"ĠSudan":42454,"'){":42455,"UpInside":42456,"builtin":42457,"ENCY":42458,"xbe":42459,"Ġchuck":42460,"Ġcontradict":42461,"Ġnuestro":42462,"Ġarchitectural":42463,"ĠFib":42464,"Ġcompares":42465,"*k":42466,"Cfg":42467,"çĦ¡":42468,"nten":42469,"Matches":42470,"ĠDOWNLOAD":42471,"_HANDLER":42472,"management":42473,"[S":42474,"ENG":42475,"ÂĢÂ":42476,"fang":42477,"Ġslipped":42478,"ĠLanka":42479,"escaping":42480,"Ġtackles":42481,"ĠPedro":42482,".Prop":42483,".''":42484,".Generated":42485,".NewGuid":42486,"atrigesimal":42487,"illon":42488,"Ġstatistic":42489,"species":42490,"holding":42491,"Drupal":42492,"Ġfundamentally":42493,"Ġbondage":42494,"Ġresolutions":42495,"InlineData":42496,"\\Type":42497,"estion":42498,".wrap":42499,"Ġwarriors":42500,"ĠLOCAL":42501,"Archive":42502,"Ġembraced":42503,"á»§":42504,".Ver":42505,"ĠAffordable":42506,"olesale":42507,"ĠApplied":42508,"ĠConversion":42509,"mega":42510,"_cam":42511,"Ġceremon":42512,"aurus":42513,"ĠVolk":42514,".opens":42515,"/about":42516,"ĠStd":42517,"journal":42518,"()){čĊ":42519,",\"\\":42520,"(Arrays":42521,"ĠDense":42522,"aseña":42523,"änner":42524,"/stat":42525,"userData":42526,"Ġgerman":42527,"Ġtz":42528,"worthy":42529,"FormatException":42530,"pherd":42531,"Ġsmiles":42532,"ĠWhenever":42533,"(adapter":42534,".badlogic":42535,"Ġbriefing":42536,".GridColumn":42537,"-char":42538,"dimension":42539,"ĠCopper":42540,"Ġninth":42541,"Ġ'{{":42542,"Ġrav":42543,"_Table":42544,"Ġderivatives":42545,"ĠRaise":42546,"ĠFut":42547,"armor":42548,"-padding":42549,"Ġremin":42550,"ĉstyle":42551,"ĠMembership":42552,"Ġspreads":42553,"Ġgalleries":42554,"ĠClarke":42555,"Ġconception":42556,"minute":42557,"Ġabusive":42558,"_adj":42559,"Ġterrific":42560,"Ġovert":42561,"ourcing":42562,"Ġentrada":42563,"levels":42564,"Ġcritique":42565,"Ġrespects":42566,"ĠMMA":42567,"iene":42568,"Ġencaps":42569,"ĠRaymond":42570,"Divider":42571,"ivable":42572,"baz":42573,"Ġ@_;Ċ":42574,"ĠClaire":42575,"Ġurging":42576,"CEE":42577,"Ġtransformer":42578,"discord":42579,"ĠJourney":42580,"tos":42581,"Ġcompetitions":42582,"ĠOBJ":42583,"ĠBis":42584,"Ġrelaxation":42585,"idy":42586,"_INSTANCE":42587,"ĠPref":42588,"dados":42589,"iciencies":42590,"ĠMediaQuery":42591,"ĠCube":42592,"ĠStrange":42593,"gpu":42594,"(days":42595,"_InitStruct":42596,"Ġfingerprint":42597,"emat":42598,"ĠGecko":42599,"Ġrails":42600,"ĠLum":42601,"straction":42602,"igung":42603,"(movie":42604,"_dictionary":42605,"_interrupt":42606,"ĠQC":42607,"iked":42608,"appendChild":42609,"recipient":42610,"ré":42611,"Ve":42612,"Ġtowel":42613,".lastIndexOf":42614,"Ġplacebo":42615,"ĠWie":42616,".esp":42617,"(Debug":42618,"operative":42619,"Ġdeceased":42620,"&id":42621,"ĉmutex":42622,"elic":42623,"Ġbapt":42624,"ĉčĊčĊ":42625,"Ġfarther":42626,"Half":42627,".disable":42628,".menuStrip":42629,"leccion":42630,"ĠresultCode":42631,"Ġcans":42632,"-election":42633,"female":42634,"_FIX":42635,"ausible":42636,"ĠPOWER":42637,"Ġreconstruction":42638,"Ġscans":42639,".XtraBars":42640,"âĢĺs":42641,"Removed":42642,"Ġparagraphs":42643,"_margin":42644,"Ġlymph":42645,"Ġbos":42646,"lington":42647,"ĠBaptist":42648,"Ġadvertisements":42649,"ĠManage":42650,"/yyyy":42651,"IOUS":42652,"ENCES":42653,"ĠFiction":42654,"ĉmenu":42655,"ĠFileOutputStream":42656,"ovan":42657,"ĠFeng":42658,"Ġskipping":42659,"getClass":42660,"anni":42661,"Ġrebounds":42662,"Ġpublicity":42663,"Ġingres":42664,"usement":42665,"Ġthoughtful":42666,".Chart":42667,"Ġhatte":42668,"passport":42669,"Ġhooked":42670,"ĠLens":42671,"Ġflagship":42672,"Ġstip":42673,"ĠGEN":42674,"Ġclues":42675,"ipv":42676,"ĠRise":42677,"ĠGew":42678,"tablename":42679,"Ġforemost":42680,"_validate":42681,"_analysis":42682,"olla":42683,"Ġqualifications":42684,"Ġdistributions":42685,"ĠFlower":42686,"Ġtense":42687,"Ġthankful":42688,"Ġclutch":42689,"Ġunified":42690,"roads":42691,"Ġsiti":42692,"Ġstall":42693,"_PRIORITY":42694,"cstdlib":42695,"_USERNAME":42696,".bytes":42697,"?page":42698,"ermalink":42699,"ĠVeget":42700,"/vnd":42701,"-author":42702,".NONE":42703,"ĠConcurrent":42704,"ĠCry":42705,"Ġstarters":42706,"ĠInteraction":42707,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":42708,"ĠLEVEL":42709,"Ell":42710,"ĠcomboBox":42711,"ĠTheresa":42712,"tek":42713,"_Handle":42714,"Ġaby":42715,".gdx":42716,",end":42717,"(Local":42718,"Ol":42719,"knife":42720,"arial":42721,"ĠHoff":42722,"Ġprostituerade":42723,"Doctor":42724,"Instances":42725,".SetValue":42726,"ĉfrom":42727,"Ġluxurious":42728,"Indent":42729,"Allocator":42730,"_DRAW":42731,"(\",\",":42732,"ĠFrances":42733,"ĠgroupBox":42734,"(schema":42735,"Printf":42736,"ORIES":42737,"-gradient":42738,"Ġreput":42739,"arin":42740,"_DONE":42741,"incre":42742,"ignty":42743,"Ġexert":42744,"Ġ-.":42745,"/App":42746,"-through":42747,"Ġdeclining":42748,"Ġdessert":42749,"Ġincumb":42750,"Ġdesignation":42751,".PORT":42752,",strong":42753,"Ġsandbox":42754,"Ġwines":42755,"ĠPav":42756,"$str":42757,"askell":42758,"Ġhö":42759,"ĠPY":42760,"GetInstance":42761,"TextInput":42762,"gameObject":42763,"/events":42764,"createdAt":42765,"ĠlocalVar":42766,"ĠWHITE":42767,"pered":42768,"ilege":42769,"efficient":42770,",color":42771,"cate":42772,"ĠCafe":42773,"Ġsimilarities":42774,"Ġpumps":42775,"ĠHungary":42776,".Username":42777,"Ġskate":42778,"Ġtouchdowns":42779,"Ġaccelerate":42780,"ĠHelen":42781,"OMEM":42782,"ĠKun":42783,"_vol":42784,"ĠfindAll":42785,"ĠMenschen":42786,"ahead":42787,");\"":42788,"kommen":42789,"Ġpossessed":42790,".argmax":42791,".transition":42792,"ARP":42793,"OLUME":42794,"(script":42795,"ĠÐĺ":42796,"ĠFinding":42797,"onces":42798,"Io":42799,"Bold":42800,"Ġrenewal":42801,"_DIALOG":42802,"Ġdisreg":42803,"INTERN":42804,"Ġtoute":42805,"Ġelectr":42806,"ĠGross":42807,"ĉtrue":42808,".Fields":42809,"ĠWIDTH":42810,"ĠDent":42811,"ĠÃģ":42812,"NSNotification":42813,"Ġaos":42814,"Ġmelee":42815,".Validation":42816,"ĠDEC":42817,"-dependent":42818,"Ġsuic":42819,"Traits":42820,"$message":42821,"ĠDear":42822,"ĉFILE":42823,"languages":42824,".Prot":42825,".addr":42826,"-generation":42827,"ICON":42828,"Ġtransplant":42829,"-description":42830,"Ġchasing":42831,"Ġchees":42832,"Ġ}*/Ċ":42833,"Trad":42834,"queries":42835,"/widgets":42836,"subpackage":42837,"Ġespec":42838,"Ġcracked":42839,"Ġcompetitor":42840,"Purchase":42841,"-team":42842,"olecular":42843,"orThunk":42844,"&P":42845,"Ġrelent":42846,"/#{":42847,"ĠproductId":42848,"Ġè¾":42849,"ĠLav":42850,"ĠAlter":42851,".Mode":42852,"ADIO":42853,"grp":42854,"æ·»åĬł":42855,"Quit":42856,"Ġdepths":42857,"-category":42858,"ĠDATABASE":42859,"SPELL":42860,"ĠFalcon":42861,"ĠQStringList":42862,"Ġ''.":42863,"ĠInstitution":42864,"damage":42865,"azor":42866,"belongsTo":42867,"verages":42868,"ĠNONE":42869,"ippets":42870,",\\Ċ":42871,"Ġfootprint":42872,"_archive":42873,"nak":42874,".getField":42875,"ĠReflection":42876,"Ġ']":42877,"ĠHBO":42878,"_discount":42879,"Ġincest":42880,"ĠDodge":42881,"ĠWade":42882,".NO":42883,"\"encoding":42884,"ĠBlockchain":42885,"Ġlawsuits":42886,"ĠMaint":42887,"chten":42888,"Ġétait":42889,"Ġktóre":42890,"_ctl":42891,"(timer":42892,"Battle":42893,"izo":42894,"ayed":42895,"IOR":42896,"ĠGlasgow":42897,"Ġsynth":42898,"_logs":42899,".pose":42900,"_AdjustorThunk":42901,"((&":42902,"Ġunsure":42903,"ystate":42904,"íķĺëĬĶ":42905,"OULD":42906,".ng":42907,"Ġdefaultdict":42908,"workspace":42909,"Ġselective":42910,"PickerController":42911,"YNAMIC":42912,".methods":42913,"Ġpathways":42914,"ĠFew":42915,"KG":42916,"CRYPT":42917,"following":42918,"ĠDLC":42919,"ĠSara":42920,"Ġpreset":42921,"estructor":42922,"ĠKurt":42923,"Ġairplane":42924,"Ġomp":42925,"ĠParents":42926,"ĠMartinez":42927,".complete":42928,"Ġbroadly":42929,"Ġscare":42930,"ĠMé":42931,"Ġelimination":42932,"Ġpoured":42933,"/sw":42934,"Ġcomun":42935,"Ġmasc":42936,"ĠOrganic":42937,"ĠStringUtils":42938,"ilateral":42939,"Ġreluctant":42940,"-age":42941,"Ġnz":42942,".\"\\":42943,"Ġpastor":42944,"alez":42945,"Ġefect":42946,"prov":42947,"/init":42948,"Ġpenn":42949,"unds":42950,"Ġssize":42951,"ĠProj":42952,"basename":42953,"Ġshells":42954,"ĠNeck":42955,"ĠEnforcement":42956,"vided":42957,"stown":42958,"Sphere":42959,"$r":42960,"ussen":42961,"afil":42962,"ĠTelegram":42963,"Ġanalytical":42964,"нÑĭе":42965,"usually":42966,"xn":42967,"Ġhistorian":42968,"ĠGregory":42969,"olph":42970,"ĠUna":42971,"Ġcontributes":42972,"%-":42973,"antiago":42974,"ÑĢед":42975,".region":42976,"Ġabrupt":42977,"ĠUnsupportedOperationException":42978,"ĠTASK":42979,"_finish":42980,"Ġnotorious":42981,"ĠVs":42982,"ĠMQ":42983,"Ġsunset":42984,"Ġunacceptable":42985,"arcer":42986,"Ġillumin":42987,"ĠOrb":42988,"Ġbh":42989,"Este":42990,"_dispatch":42991,"Ġripped":42992,"Ġtoujours":42993,"ĠParcel":42994,"_ll":42995,".userName":42996,".classes":42997,"SOURCE":42998,"(Number":42999,"елÑı":43000,"Ġheadphones":43001,"(side":43002,"constitution":43003,"annah":43004,"čĊĠĠĠĠĠĠĠĠčĊ":43005,"Ġcliff":43006,"-ref":43007,"Ġmostrar":43008,"ĠPowell":43009,"+y":43010,"ĠBG":43011,"_fragment":43012,".Port":43013,"Ġrealizing":43014,"paramref":43015,"Ġhometown":43016,"@Table":43017,"+\"--}}Ċ":43196,"French":43197,"EntityManager":43198,"ĠPlain":43199,"////////////////////////////////////////////////////////////////////":43200,"³":43201,"(RE":43202,"capt":43203,"Ġorganisms":43204,"Ġjets":43205,"olocation":43206,"ĠAppRoutingModule":43207,"Ġglorious":43208,"æľį":43209,"Ġdiscarded":43210,"ĉĉĉĉĠĠĠĠĠ":43211,"ĠArnold":43212,"lug":43213,"Ġparl":43214,"Ġhormones":43215,"Ġmah":43216,"ĠSonic":43217,"Ġorganizers":43218,"_PLATFORM":43219,".inv":43220,"Ġchord":43221,"ventional":43222,"ĉof":43223,"Episode":43224,".Enum":43225,"unkt":43226,"ĠDh":43227,"ĠJared":43228,"ĠNak":43229,"Ġintends":43230,"Endian":43231,"Ġaustralia":43232,"_cv":43233,"(resolve":43234,"Ġclinics":43235,"liked":43236,"ASHINGTON":43237,"inha":43238,"'*":43239,"ĠNP":43240,"_beh":43241,"Ġhf":43242,"Ġwür":43243,"categoria":43244,"$form":43245,"Ġsubway":43246,"ĠisActive":43247,"popular":43248,"Cour":43249,"Ġcooldown":43250,"Ġainsi":43251,"ĠGLuint":43252,"ereal":43253,"ĠarrayOf":43254,"Ġhatch":43255,"==========":43256,"resses":43257,"_PP":43258,".^":43259,"_decay":43260,"ĠBless":43261,"metrics":43262,"ĠCOPYING":43263,"ĠDumpster":43264,"ĠJosé":43265,"ĠDesigns":43266,"<":43269,"Ġ\"}Ċ":43270,"timezone":43271,"Ġeer":43272,"maxcdn":43273,"ĠESC":43274,"igaret":43275,"_connected":43276,"_reverse":43277,"Ġquestionable":43278,"ĠUSC":43279,"Ġtutti":43280,"Ġdropout":43281,"ĠActivities":43282,"ĠWinds":43283,"')));Ċ":43284,"Ġcongest":43285,"ģı":43286,"Ġprolonged":43287,"è¿Ļ":43288,"ĠCrossAxisAlignment":43289,"LEEP":43290,"ĠVALID":43291,"ĠGaz":43292,"Ġdependence":43293,"ĠPrix":43294,".CompilerServices":43295,"jump":43296,"Ġstrat":43297,"circ":43298,"ĠCUSTOM":43299,"xaa":43300,"Ġbmp":43301,"Ġbureau":43302,"Ġwaren":43303,"NX":43304,"(Window":43305,"ĠChristie":43306,"_FE":43307,"Ġtn":43308,"ĠOmega":43309,"communications":43310,"HomePage":43311,"completion":43312,"Ġsupplying":43313,"YPES":43314,"ável":43315,"åζ":43316,"(click":43317,"\\Contracts":43318,"/questions":43319,"Ġez":43320,"AMS":43321,".mesh":43322,"Ġ'\\Ċ":43373,"Robot":43374,"JsonObject":43375,"ĠDF":43376,"ĠProcessor":43377,"_should":43378,".protobuf":43379,"-users":43380,"Ġembry":43381,"FONT":43382,"Ġstartups":43383,"ĠDataSource":43384,")#":43385,"uros":43386,"_Color":43387,"Ġstandalone":43388,"}[":43389,"jd":43390,"Ġforgive":43391,"Ġngx":43392,"ĠGenerally":43393,"Ġconfigurable":43394,"/order":43395,"Ġvas":43396,"')\";Ċ":43397,"ĠRR":43398,"ĠTroy":43399,"Ġcompromised":43400,"ĠSwan":43401,"intendent":43402,"Central":43403,"_keeper":43404,"Ġarquivo":43405,"ĠReadOnly":43406,"_curve":43407,"kv":43408,"entin":43409,"è±":43410,"ĠEy":43411,".imread":43412,"ĠPam":43413,"iffe":43414,"ativity":43415,"xbc":43416,"Ġgrim":43417,"-filled":43418,"namese":43419,"']:":43420,"Ġaur":43421,"ĠGibson":43422,".MouseEvent":43423,"Ġlado":43424,"avadoc":43425,"Ġfamil":43426,"ĠModer":43427,"fps":43428,"ãĢĢãĢĢ":43429,"-example":43430,"ĠAlzheimer":43431,"ĠUtf":43432,"_arguments":43433,"Conclusion":43434,"textContent":43435,"remaining":43436,"Ġinterrupts":43437,"ĠBackup":43438,"ĠMong":43439,"Ġreceptors":43440,"histor":43441,".coroutines":43442,"Ġshouted":43443,"Alarm":43444,"Ġcombust":43445,"Ġgrote":43446,"ultural":43447,"(ids":43448,"--------------------------------------------------------------------------------":43449,"iplinary":43450,"Opts":43451,"ĠYale":43452,"localStorage":43453,"Ġequival":43454,"ĠFleet":43455,"\\b":43456,"*pi":43457,"ĠQLabel":43458,"æ¡":43459,"Ġvx":43460,"ĠACL":43461,"Ġsucesso":43462,"Ġperc":43463,"ĠNotre":43464,"Ġanarch":43465,"Ring":43466,"spb":43467,"Ġstrpos":43468,"stores":43469,"ĠMaple":43470,"(MainActivity":43471,"(\"\"))":43472,"ĠviewHolder":43473,"Quad":43474,"Ġigual":43475,"orsche":43476,".margin":43477,"Ġindie":43478,"Ġfranc":43479,"ĠFormBuilder":43480,"ĠParticip":43481,".flash":43482,"Ġstorms":43483,"Ult":43484,"Ġfen":43485,"[new":43486,"Ever":43487,"=\"Ċ":43488,"Ġlocalized":43489,"_follow":43490,"Ġnave":43491,"Ġdominance":43492,"(tile":43493,"Journal":43494,"ĠVC":43495,"Ġpenetration":43496,"ï¼ķ":43497,"Ġcompartment":43498,"Ġbids":43499,"Formatted":43500,"******/ĊĊ":43501,"(city":43502,"âĢĶit":43503,"[C":43504,"ĠuseCallback":43505,"aub":43506,")?.":43507,"ĠVAR":43508,"ĠSebastian":43509,"ĠMoss":43510,"Ġabundant":43511,"Greg":43512,"ÑĤа":43513,"_ci":43514,"Ġbibli":43515,"CRM":43516,"ĠAttempt":43517,"isme":43518,"dash":43519,"ãĢİ":43520,"_mu":43521,".FormattingEnabled":43522,"Indeed":43523,"-direct":43524,"Ġsucking":43525,"Ġpne":43526,"ocabulary":43527,"ĠPackers":43528,".Navigation":43529,"Ġpied":43530,"cribing":43531,"ĠStuart":43532,".ToDouble":43533,"ĠSecondary":43534,"Saving":43535,"ĠDut":43536,"ĠMadd":43537,"Magic":43538,",H":43539,".documentElement":43540,"ĠBST":43541,"Ġdiffers":43542,"Ġmoreover":43543,"_nd":43544,"SEARCH":43545,"пÑĢав":43546,"æ´":43547,"toMatch":43548,"Ġdecreasing":43549,"-member":43550,"ampus":43551,"(boost":43552,"Daily":43553,"DataGridView":43554,"ĠHttpContext":43555,"Ġhipp":43556,"_workers":43557,"-language":43558,"éĵ":43559,"Ġconsisted":43560,"athing":43561,"ĠMercury":43562,"$content":43563,"Ġpracticed":43564,"ĠModules":43565,"_DAY":43566,"Ġweaknesses":43567,"ĠLodge":43568,"Ġnar":43569,"ĠMate":43570,"Ġjp":43571,"ĠHttpHeaders":43572,"Ġsmo":43573,"ĠTOKEN":43574,"])(":43575,"Ġaqui":43576,"swagen":43577,"Ġsrv":43578,"ĉans":43579,"Around":43580,"ĠManuel":43581,"Ġfictional":43582,"ĠIMG":43583,"Ġ.'":43584,"ĠBerry":43585,"Ġwallpaper":43586,"sexual":43587,"iero":43588,"ĠçļĦ":43589,"ìĨĮ":43590,"BackingField":43591,"ĠAdrian":43592,"BASEPATH":43593,"Ġrepeats":43594,"Ġblues":43595,"Ġunpredict":43596,"_coll":43597,"stacle":43598,"ĠTumblr":43599,"ĠElf":43600,"Ġassurance":43601,"Ġcensus":43602,"ĠIMPORT":43603,"ENDER":43604,"anos":43605,"Ġ=(":43606,"ĠEllis":43607,"\"ĊĊĊĊ":43608,".win":43609,"ĠAbove":43610,"alon":43611,"_tick":43612,"Ġrepresentations":43613,"Ġæķ":43614,"wid":43615,"ĠArms":43616,"Lista":43617,"_failure":43618,"_cm":43619,".FlatAppearance":43620,"Ġthrone":43621,"Patch":43622,"ĠVoy":43623,"engl":43624,"Ġnegotiating":43625,">`":43626,"Ġshoots":43627,"ĠFPS":43628,".Year":43629,"ĠKiss":43630,"ención":43631,"reeting":43632,"FromFile":43633,"Ġresignation":43634,"Ø·":43635,"Ġtwins":43636,"ượ":43637,"Ġgebru":43638,".getContent":43639,".Tree":43640,"ĠEmployees":43641,"ĠFIFA":43642,"Ġcertainty":43643,"(Cl":43644,"Ġtotals":43645,"editable":43646,"à¥Ģ":43647,".Reporting":43648,"Mas":43649,"quiet":43650,".rules":43651,"ĠVO":43652,"conexion":43653,",K":43654,"Ġallocator":43655,"ĠPowder":43656,"\\Repository":43657,"Beat":43658,"_tipo":43659,"Ġ['',":43660,"_INTR":43661,"Ġ<<<":43662,"\");čĊ":43691,"dropIfExists":43692,"ĠBeg":43693,"_HAL":43694,"ĠcrossAxisAlignment":43695,"ĠEvidence":43696,"Ġpeculiar":43697,"Ġinstitute":43698,"veis":43699,"Ġfft":43700,"Ãģ":43701,"Ġzoekt":43702,"analy":43703,"ĠHomeland":43704,"Ġpenetr":43705,"uddenly":43706,"ĉelement":43707,"ĠBren":43708,"ĠTrudeau":43709,"ĠCuban":43710,"jam":43711,"uslim":43712,"_ev":43713,"Ġstems":43714,"}%":43715,"Ŀå§ĭ":43716,"Ġbranding":43717,"Ġcorrespondence":43718,".jquery":43719,"¢åįķ":43720,"ĠReads":43721,"(HttpStatusCode":43722,"assin":43723,"(slot":43724,"ĠGraduate":43725,"///<":43726,"Ġinformations":43727,"ENABLE":43728,"Ġpuis":43729,"Ġfinder":43730,"ĠBris":43731,"Ġnettsteder":43732,"_mid":43733,"Ġogs":43734,"ĠSterling":43735,"Ġarrog":43736,"strftime":43737,"|ĊĊ":43738,"Ġvox":43739,"ĠRegardless":43740,"Ġeso":43741,"ĠComfort":43742,".BooleanField":43743,"Ġuh":43744,"ACY":43745,"Ġsqueez":43746,"ĠVic":43747,"contro":43748,".lo":43749,"Ġire":43750,"ĠComedy":43751,"ë¶":43752,"Ġoriginated":43753,"Ġshipment":43754,"|max":43755,"_guid":43756,"levation":43757,"наÑı":43758,"(undefined":43759,"ĠDDR":43760,"Ġshootings":43761,"ĠLatino":43762,"ENDOR":43763,"Ġaveraging":43764,"Ġgreeted":43765,"Ġtheaters":43766,"ое":43767,"ĠdB":43768,"Ġgst":43769,"Ġdefinite":43770,".Storage":43771,".her":43772,"Ġafore":43773,"ĠReality":43774,"ĠGods":43775,"versed":43776,"Ġhandsome":43777,"Ġexcluding":43778,"(ad":43779,"Quotes":43780,"ĠScheme":43781,"?q":43782,"ĠTamil":43783,"Ticks":43784,"Ġpest":43785,"'n":43786,"Ġpornography":43787,"_modal":43788,"Ġ----------":43789,"Ġdisposable":43790,"FREE":43791,"Ġshark":43792,"CHE":43793,"Ġdepicted":43794,"Ġdemonstrations":43795,"ĠKilled":43796,"ĠRULE":43797,"Ġobsessed":43798,"Ġsimplified":43799,"Postal":43800,"Ġconceptual":43801,"Ġpst":43802,"Las":43803,"_PROJECT":43804,"ucceeded":43805,"olu":43806,"ÄŁi":43807,"Ġpersonalities":43808,"Ġreshape":43809,"Ġenclosed":43810,"ĉptr":43811,"Ġtutorials":43812,"Ġexploded":43813,"_DIRECTORY":43814,"åĨħ容":43815,"Ġcanon":43816,"Ġrecognise":43817,"PAD":43818,"ĠApprox":43819,"ĠRestore":43820,"ĠImportant":43821,"Ġheavier":43822,".Sequential":43823,"Earth":43824,"ĠMilk":43825,".setRequest":43826,".tem":43827,"Ġreconstruct":43828,"Ġskeptical":43829,"_Private":43830,"BUF":43831,"qua":43832,":a":43833,"Ġsek":43834,"Ġdwell":43835,"ossa":43836,"Ġrewarded":43837,"ий":43838,"(topic":43839,"_partition":43840,"Ġ__________________":43841,"Keywords":43842,"ĠFranco":43843,"Lite":43844,"Ġnaken":43845,"Ġза":43846,"OBJECT":43847,"Ġcrafts":43848,"ĠSwap":43849,".Xna":43850,".Connect":43851,"Ġbalcony":43852,"(real":43853,"ĠBarnes":43854,"bir":43855,"ĠTwenty":43856,"ayan":43857,"atars":43858,"ĠPropel":43859,"ĠIhnen":43860,"Upgrade":43861,"Ġcurb":43862,"-second":43863,"Ġneph":43864,".pres":43865,"ìŀħ":43866,".seq":43867,"Ġpadded":43868,"\"?":43869,"jl":43870,"ãĥ¬":43871,"')a":43875,"Coordinates":43876,"Ġenacted":43877,"ENTS":43878,"Ġlac":43879,".final":43880,"ĠPhpStorm":43881,"called":43882,"Ġinquiries":43883,".middleware":43884,"ĠDowntown":43885,"/';Ċ":43886,"Ġkilomet":43887,"accel":43888,"Ġquien":43889,"wstring":43890,"setData":43891,"Ġmanera":43892,"Ġmodular":43893,"rimp":43894,"Ġtariffs":43895,"âĢĻil":43896,"_THROW":43897,"/color":43898,"ĠHTMLElement":43899,"Ġcarro":43900,"Ġprere":43901,"Ġplotting":43902,"ĠPositive":43903,"ĠMachines":43904,"OTES":43905,"Ỽ":43906,"pleasant":43907,"Ġalte":43908,"Ġainda":43909,"these":43910,"Ġcors":43911,"ipay":43912,"ĠAdvisory":43913,"ĠRubio":43914,"jq":43915,"Ġlimestone":43916,"Ġdetached":43917,"设置":43918,"tenant":43919,"ĠDepth":43920,"alore":43921,"ĠÑģÑĤÑĢок":43922,"ĠFORE":43923,"ĠLay":43924,"presentation":43925,")');Ċ":43926,".subplots":43927,"Ïĥ":43928,"NOW":43929,"Gar":43930,"handles":43931,"abra":43932,"puties":43933,"ĠElectrical":43934,"Middle":43935,"ropic":43936,"ĠJD":43937,"ĠDyn":43938,"ĠBristol":43939,"ĠMcCarthy":43940,"Ġstriker":43941,"Ġenumerable":43942,"ĠEvan":43943,".defaults":43944,"quences":43945,")||":43946,"ĉtoken":43947,"âĹı":43948,"-dropdown":43949,"STORE":43950,"ĠGraphic":43951,"(pp":43952,"Expl":43953,"Ġupwards":43954,"ĠDistributed":43955,"ĠWEB":43956,"Jer":43957,"isNaN":43958,"çĶŁæĪIJ":43959,">R":43960,"üssen":43961,"efs":43962,"Ġuncover":43963,"Ġlud":43964,".calculate":43965,"Ġintptr":43966,"Ġmidfielder":43967,".Headers":43968,"Ġmf":43969,"eref":43970,".Metro":43971,"ĠSpeaking":43972,":b":43973,"Ġcryptocurrencies":43974,"Ġdemons":43975,"ĉEXPECT":43976,"Ġwicked":43977,"youtube":43978,":Int":43979,"ĠHindi":43980,"ĠCAT":43981,"Ġع":43982,"rar":43983,"omore":43984,"/per":43985,"/license":43986,"Ġreim":43987,"Ġawaiting":43988,"Ġlethal":43989,"ĠEF":43990,"rounded":43991,"ĠPlatinum":43992,"ĠвÑģе":43993,".coords":43994,".Device":43995,"/item":43996,"ĠWenn":43997,"compileComponents":43998,"ĠKinder":43999,".removeItem":44000,"Ġanda":44001,"bnb":44002,"Ġpra":44003,"(transaction":44004,"Ġembarrassing":44005,"ĉBOOL":44006,".contentView":44007,"Ġeventdata":44008,"atore":44009,"ĠprovidedIn":44010,"irma":44011,"Ġzona":44012,"_HW":44013,"æĻ":44014,"Ġstove":44015,"Ġcounterpart":44016,"_Product":44017,"_MANAGER":44018,"Ġinfring":44019,"ĠERA":44020,"_party":44021,"Ñij":44022,"Ġinici":44023,"_Request":44024,"Ġmiracle":44025,"ĠcancelButton":44026,"Spy":44027,"ató":44028,"Ġpolish":44029,"ĠNicole":44030,".displayName":44031,"\\Requests":44032,"ĠuseHistory":44033,"RouterModule":44034,"Ġstared":44035,"IDER":44036,"ÑĥнкÑĨи":44037,"Ġnota":44038,"$arr":44039,"pecified":44040,"Ġtopp":44041,"_DRIVER":44042,"/ng":44043,"åł":44044,"_tm":44045,"%timeout":44046,"\"":44488,"tlement":44489,"$(\"":44490,"FromString":44491,"ĠBild":44492,"Ġconventions":44493,"_native":44494,"ĠInspector":44495,"ĠPist":44496,"ubar":44497,"Ġregs":44498,"ĠPilot":44499,"Thus":44500,">'+":44501,"Ġcela":44502,".news":44503,"(Product":44504,"Living":44505,"Russia":44506,"Ġfacet":44507,"etical":44508,"Ġ['$":44509,"/[":44510,"ĠDire":44511,"Ġgases":44512,"ĠINFORMATION":44513,"ĠEat":44514,"ĠForums":44515,"ĠCharacters":44516,"_met":44517,"Ġìĭľ":44518,"Ġkings":44519,"achie":44520,"ĠLambda":44521,"Ġtimers":44522,"ĠLighting":44523,"ĠCasey":44524,"addir":44525,"andex":44526,".answer":44527,"ĠHip":44528,"ĠPrincip":44529,"StartDate":44530,"ĠãĢĮ":44531,"tres":44532,"Ġ&#":44533,".MaxValue":44534,"ĠProblems":44535,"Ġlatex":44536,"OfClass":44537,"ĠLynn":44538,"//'":44539,"Ġvoyage":44540,"Ġshuttle":44541,"ĠRoller":44542,"ĠRuntimeError":44543,"uya":44544,"Dic":44545,"ĉbuilder":44546,"Ġbullying":44547,"Ġsimplest":44548,".called":44549,"ĠLR":44550,"Ġmorality":44551,"Ġsturdy":44552,"tracking":44553,".swagger":44554,"_BIND":44555,"ITOR":44556,"-urlencoded":44557,"ĠÑħ":44558,"ĠTrinity":44559,"Ġtraps":44560,"Ġ|-":44561,"ĠsetText":44562,"Ġbargain":44563,"Ġbrakes":44564,".getCode":44565,"Ġmigrate":44566,"Ġribbon":44567,")return":44568,"Ġcharger":44569,"acom":44570,"ADIUS":44571,"ĠAmbassador":44572,"-after":44573,"Ġanni":44574,"ĉspin":44575,"Concept":44576,"ĠHenderson":44577,"ĠHOST":44578,".rank":44579,"ĠNortheast":44580,"Ġberlin":44581,"Ġrequis":44582,".feed":44583,"ĠsourceMapping":44584,"ĠRencontre":44585,".ajax":44586,"nestjs":44587,"Ġtrek":44588,"ĠNacional":44589,"Ġ&[":44590,"Ġpayable":44591,"ortex":44592,"Ġdept":44593,"fieldName":44594,"Ġcompletes":44595,"ĠRVA":44596,"Ġonions":44597,"alignment":44598,"Formats":44599,"Ġ'{$":44600,"HashSet":44601,"ĠBod":44602,".InvariantCulture":44603,"Ġsettlements":44604,"Ġhydr":44605,".updated":44606,"venth":44607,"(seconds":44608,"=\"/\"":44609,"Ġwebpage":44610,"(ĊĊ":44611,"Ġtir":44612,"Ġtoes":44613,"ĠBrick":44614,"Ġambition":44615,"Pot":44616,"=max":44617,"ETIME":44618,"Ġdepot":44619,"calls":44620,"ĠNorwegian":44621,"`:":44622,"Ġburger":44623,"Ġprofessors":44624,"ĠAllocate":44625,"-thirds":44626,"-chart":44627,"Ġford":44628,"*N":44629,".kotlin":44630,"Ġpaperwork":44631,"ĠDEVICE":44632,"%@\",":44633,"respect":44634,"(mp":44635,"é«ĺ":44636,"-if":44637,"Ġcushion":44638,"obot":44639,"Ġparc":44640,"SPACE":44641,"ĠNetanyahu":44642,"Ġselfish":44643,"feat":44644,"Ġclientes":44645,"-tools":44646,"Ġporch":44647,"Ġjq":44648,".verbose":44649,"Ġliberals":44650,"])ĊĊĊ":44651,"pies":44652,"NotBlank":44653,"(term":44654,"ÈĽi":44655,"_Params":44656,".normalize":44657,"Bullet":44658,"ASIC":44659,"(hex":44660,"_cliente":44661,"+,":44662,"_DI":44663,"Ġforthcoming":44664,"}\")]Ċ":44665,"seo":44666,"Um":44667,">Name":44668,"Ġcomfortably":44669,"irectional":44670,"WITH":44671,"/pr":44672,"ĠPoor":44673,"ĠVitamin":44674,"vic":44675,"GH":44676,"Ġpriorit":44677,"ĠNN":44678,"ĠClosed":44679,"¤í":44680,"ĠisOpen":44681,"\\Console":44682,"AndFeel":44683,".SUCCESS":44684,"_OPERATION":44685,"polation":44686,"ĠTas":44687,"psz":44688,">'.":44689,"CURRENT":44690,"Vendor":44691,"hosts":44692,"ĠErd":44693,">tagger":44694,"ĠsourceMappingURL":44695,"Ġmarathon":44696,"_closed":44697,"Ġexemption":44698,"Ġrecognizes":44699,"ideshow":44700,"'$":44701,"('/');Ċ":44702,"mits":44703,"warz":44704,"ĠCherry":44705,"µ¬":44706,"nor":44707,"porte":44708,"Ġwl":44709,"_backup":44710,".getBoolean":44711,".getResource":44712,"Ġdefinitive":44713,".EditText":44714,"ĠsÃŃ":44715,".CONT":44716,"ĠPLAYER":44717,".cards":44718,"ĠShore":44719,"('/')Ċ":44720,"cluir":44721,"WebDriver":44722,"(month":44723,"-release":44724,"Ġinspector":44725,"å£":44726,"ĠNF":44727,"_clip":44728,"åŃIJ":44729,"Ġinteracting":44730,".tmp":44731,"Ġ'''ĊĊ":44732,"Ġdee":44733,"Ġfrost":44734,"\"]))Ċ":44735,"ĠPlaces":44736,"Throws":44737,"fork":44738,"/day":44739,"iPhone":44740,"ĠMIC":44741,"Ġfolding":44742,"Ġcrore":44743,"ĠChiefs":44744,"pherical":44745,"(price":44746,".WriteString":44747,"Ġexiting":44748,"]',Ċ":44749,"ighting":44750,"Ingredient":44751,"(vertex":44752,"ĠscrollView":44753,"hf":44754,":new":44755,"SEN":44756,"sector":44757,"Ġspins":44758,"ĠScheduler":44759,"otechn":44760,"semicolon":44761,"FontOfSize":44762,"ĠSpecifically":44763,"flamm":44764,".ObjectId":44765,"Ġconta":44766,"_permissions":44767,"ĉFROM":44768,"ICODE":44769,"/kg":44770,"ĠHotels":44771,"-med":44772,"ĠDin":44773,"Ġnavy":44774,"getParam":44775,"Ġmend":44776,"Ġportrayed":44777,"ĠMetropolitan":44778,"Painter":44779,"Ġreferral":44780,"_good":44781,"Ġmarvel":44782,"osaic":44783,">(&":44784,".ur":44785,"Ġestos":44786,"William":44787,"Ġtimber":44788,"Ġquelques":44789,"ĠDocuments":44790,".Xaml":44791,"Ġbatches":44792,"éģĵ":44793,"ĠReleased":44794,"Tail":44795,"COOKIE":44796,"heid":44797,"_station":44798,"ĠVia":44799,"Sale":44800,"ĠRepeat":44801,"Ġpromin":44802,"ĠZo":44803,"-forward":44804,"ĠIon":44805,"itary":44806,"Ġjus":44807,"-request":44808,"Ġproudly":44809,"ĠStreaming":44810,"(MouseEvent":44811,"ĠSprint":44812,"_rotation":44813,"Repositories":44814,"Ġtart":44815,"ĠÑģв":44816,"Ġmappings":44817,"èª":44818,"Cu":44819,"Cycle":44820,"Ġbun":44821,"ĉlua":44822,"ãĥī":44823,"Ġ((!":44824,"Ġcollectively":44825,"ĠCond":44826,"Ġwszyst":44827,"(lib":44828,"openhagen":44829,"_skip":44830,".ColumnHeader":44831,"éĤ":44832,"perienced":44833,"ıè¿°":44834,"_props":44835,"Ġcontrace":44836,"Ġmatchup":44837,"abetic":44838,".members":44839,"RECT":44840,"(dat":44841,"Ġsog":44842,"renom":44843,"_Method":44844,"Customers":44845,"fullname":44846,"ZN":44847,"retry":44848,"Ġkap":44849,"ĠNeu":44850,"èĬ":44851,"addChild":44852,"willReturn":44853,"_permalink":44854,"Ġenergetic":44855,"ĠWet":44856,"ĠMorr":44857,"Ġgcd":44858,"counts":44859,",type":44860,"dig":44861,"(Login":44862,"Ġcracks":44863,"Ġbacterial":44864,"ĠMeat":44865,"ĠArmstrong":44866,"ĠBronze":44867,"Ġapproximate":44868,"_dirs":44869,"liga":44870,"ÅĤad":44871,"Ġkindness":44872,"Ġcontre":44873,"ĠEVERY":44874,"MET":44875,"Ġannouncements":44876,"gpio":44877,"ĠWaitForSeconds":44878,"ĠPhotoshop":44879,"Ġdiscontin":44880,"/dd":44881,"Ġtopology":44882,"anical":44883,".interface":44884,"aucoup":44885,".HashSet":44886,"ARIANT":44887,"(routes":44888,"ĠTeh":44889,"Ġhype":44890,"]\").":44891,"Ġslam":44892,"Ġbroth":44893,"-inter":44894,"ĠRid":44895,"-manager":44896,"Cancelar":44897,"ĠPagination":44898,"Ġsoundtrack":44899,"Ġposterior":44900,"Ġscrub":44901,"creating":44902,"-*":44903,"irteen":44904,".dy":44905,".symmetric":44906,"Ġ\"\".":44907,"===============":44908,"Ġchassis":44909,"ĠnumberOfRows":44910,"Developer":44911,"_bins":44912,"ĠOUR":44913,"rieb":44914,"Pros":44915,"ĠwiÄĻ":44916,"\"d":44917,"Ġasyncio":44918,"zeigen":44919,"_spi":44920,".ALL":44921,"Ġscrews":44922,"Chinese":44923,"ĠapiKey":44924,"Ġunsuccessful":44925,"ĠSeahawks":44926,"ORG":44927,"竳":44928,"Ġprofessionally":44929,"ĠCoupon":44930,"åŃĹæ®µ":44931,"Convention":44932,"Ġpolym":44933,"æīĭ":44934,"Ġsalvation":44935,"Ġengineered":44936,"ĠWrest":44937,"ĠGCC":44938,"Ġwarmer":44939,"LayoutConstraint":44940,"Ġaggrav":44941,"Scripts":44942,"venture":44943,"Ġrefrigerator":44944,"Ġinnovations":44945,"ĠRunner":44946,"NIC":44947,"ĠRolling":44948,"ControlEvents":44949,"Ġloos":44950,"pac":44951,"ĉpanel":44952,"efe":44953,"ĠBuddha":44954,"--------------Ċ":44955,"åºĵ":44956,"(forKey":44957,"Ġlumin":44958,"Ġ(?":44959,"ĠAIDS":44960,",user":44961,"imientos":44962,"contentType":44963,"antlr":44964,"é¦":44965,"ĠWelt":44966,"Production":44967,"might":44968,"ĠVII":44969,"\",(":44970,"Ġobserving":44971,"Ġdeliberate":44972,"(control":44973,"Ġwithd":44974,"Ġsemana":44975,"STACK":44976,"uchen":44977,"Nice":44978,"ĠDeutschland":44979,"ĠSpecifies":44980,"dma":44981,"izio":44982,"ĠFacts":44983,"_popup":44984,"ĠDirectors":44985,"{:":44986,"[R":44987,"ĠÑįлеменÑĤ":44988,"Ġplat":44989,"Ġdirecting":44990,"ä¸ī":44991,"ĠGilbert":44992,"â̦.ĊĊ":44993,".qml":44994,"Ġthereafter":44995,"Ġdisposition":44996,"draft":44997,"Ġsurgeon":44998,"ĠInsider":44999,"Blend":45000,"ĠTrev":45001,"trinsic":45002,"Topics":45003,"rieve":45004,"_FILENAME":45005,"Ġautres":45006,"Jose":45007,"Producer":45008,"erus":45009,"Ġpetit":45010,"ĠNEXT":45011,"ĠFilters":45012,"Ġreplicate":45013,"\"]).":45014,"Ġlenders":45015,"]\",Ċ":45016,";charset":45017,"CppObject":45018,"Ġfloral":45019,"ĠTipo":45020,"Ġcircuits":45021,"easy":45022,"(&$":45023,"itta":45024,"eryl":45025,"_COMMON":45026,"'}}>Ċ":45027,"-backed":45028,"(variable":45029,"(Index":45030,"Ġvoir":45031,"_locations":45032,"++){":45033,"ĠLouisville":45034,"Ġgratitude":45035,".Mockito":45036,"ĠPowers":45037,"ieurs":45038,"Ġgeographic":45039,"rale":45040,"Ġcra":45041,"ĠSpurs":45042,"iphertext":45043,"ACION":45044,"-common":45045,"Ġvictories":45046,"ĠFinals":45047,".shuffle":45048,"-million":45049,"_PROC":45050,"assume":45051,"Ġils":45052,"DBC":45053,"BootTest":45054,"Ġlavor":45055,".testing":45056,".ast":45057,"\"]/":45058,"moid":45059,"Ġqualification":45060,"gesch":45061,"ĉput":45062,"Ġairports":45063,"JI":45064,"Teacher":45065,"_uniform":45066,"Ġnama":45067,"ĠBast":45068,"ertype":45069,"capture":45070,"getAll":45071,"ĠReynolds":45072,"ooled":45073,".comments":45074,"Ġchin":45075,").*":45076,"Ġили":45077,"tgl":45078,"udos":45079,"ĠdÃŃas":45080,"chai":45081,".program":45082,"Ġpsz":45083,"ĉicon":45084,"phil":45085,"entral":45086,"_WRAP":45087,"ovi":45088,"Ġnostalg":45089,"Infinity":45090,"ĉyield":45091,"Ġvitamins":45092,"Quaternion":45093,"Sink":45094,"_goods":45095,"Ġ........":45096,"ĠWings":45097,"uridad":45098,"-story":45099,"\"])ĊĊ":45100,"idelity":45101,"TypeDef":45102,"Gtk":45103,"ĠíĮ":45104,"_Main":45105,"Ġchez":45106,"ĠRaven":45107,"Ġpayroll":45108,"Ġfreelance":45109,"LLU":45110,"ĠMend":45111,"eday":45112,"ApiModelProperty":45113,".FormBorderStyle":45114,"Ġeconomist":45115,"stanbul":45116,"Ġfreight":45117,"-Agent":45118,"(meta":45119,"Ġsymmetry":45120,"Ġ'..":45121,".Calendar":45122,"-aut":45123,"gf":45124,"pent":45125,"yclopedia":45126,"Ġwishing":45127,"ĊĊĊĊĊĊĊĊĊĊĊĊ":45128,"Ġgentleman":45129,"Ġê³":45130,"=#":45131,"Ġlectures":45132,"âĢľIn":45133,"Ġ!_":45134,"Ġhb":45135,"ĠVendor":45136,"Recently":45137,"_notes":45138,"æıIJ示":45139,"\"My":45140,"HeadersHeight":45141,"_SO":45142,"Ġunwilling":45143,"Ġsuperhero":45144,"gio":45145,"psy":45146,"ĠPeer":45147,"javax":45148,"&apos":45149,"ĠCrisis":45150,"ordinal":45151,"Memcpy":45152,"++++++++++++++++":45153,"-val":45154,"Ġworkbook":45155,"-ap":45156,"=k":45157,"Ġmetallic":45158,"_peer":45159,"ByPrimaryKey":45160,"_SD":45161,"uator":45162,"_SHADER":45163,")Math":45164,".Transform":45165,"Ġcows":45166,"Phi":45167,"ĠClem":45168,"(_(\"":45169,"ĠLud":45170,"-delay":45171,"ĠSecurities":45172,"ĠOrthodox":45173,"Symfony":45174,"(report":45175,"Ġentertain":45176,"EPS":45177,"izoph":45178,"exual":45179,"IRD":45180,"ä»İ":45181,"Ġlith":45182,"Ġsanitize":45183,"Ġfeminine":45184,"ISBN":45185,".authentication":45186,"_pipeline":45187,"/constants":45188,"ĠCONF":45189,"Ġlucr":45190,"ricia":45191,".ttf":45192,".setContent":45193,"Ġstan":45194,"orean":45195,"ĠLloyd":45196,".rawValue":45197,"Ġgor":45198,"ĠBrowns":45199,"Regression":45200,"Ġlowering":45201,"naissance":45202,"Ġblows":45203,"Ġamazed":45204,"Ġunrelated":45205,"Reviews":45206,"Ġruby":45207,"ĠModifier":45208,"Ġgiants":45209,".thread":45210,"Ġcontainment":45211,"ĠStartCoroutine":45212,"umat":45213,"orelease":45214,"ĠRandy":45215,"@endif":45216,"Digest":45217,"Ġsuburban":45218,"=\");Ċ":45219,"Ġannonce":45220,".variable":45221,"\\Foundation":45222,"Ġacre":45223,"Van":45224,"Ġtuples":45225,"dns":45226,"ĠStanding":45227,"_large":45228,"Ġboxing":45229,"SupportActionBar":45230,"ĠFortune":45231,"ĠRum":45232,"_multiple":45233,"archical":45234,"Ġfwrite":45235,"_quote":45236,"Ġfoolish":45237,"Ġcomprising":45238,"Ġоп":45239,"-selected":45240,"vf":45241,"maid":45242,"Nama":45243,"(datetime":45244,"Ġindirectly":45245,"gart":45246,"fixtures":45247,"chos":45248,"ĠHalo":45249,"Ġrecurring":45250,"-news":45251,"vil":45252,"ĠNursing":45253,"-produ":45254,"ĠHQ":45255,"\\HttpFoundation":45256,"enci":45257,"auen":45258,"Ġvy":45259,"ocracy":45260,"Ġdelegation":45261,"Ġasphalt":45262,"ĠsetSelected":45263,"kok":45264,"/rest":45265,"metics":45266,"ĠNSDate":45267,"Ġtravelled":45268,"Ġrecib":45269,"Ġmime":45270,"CLIENT":45271,"ĠGU":45272,"ĠHANDLE":45273,"/Q":45274,"[z":45275,"Ġbothered":45276,"ĠBBQ":45277,"ças":45278,"_examples":45279,"_FIN":45280,"ĠwhiteColor":45281,"Ġastronom":45282,"-dir":45283,"Ġsovereign":45284,"Ġbreeze":45285,"Ġinning":45286,"ĠEdmonton":45287,"gli":45288,".blogspot":45289,"jsx":45290,"Ġversa":45291,"ĠMohammed":45292,".Job":45293,"-toggler":45294,"ĠполÑĮзоваÑĤ":45295,"ardon":45296,"Ġnewborn":45297,"Ġnaval":45298,"noteq":45299,"Ġtumblr":45300,"Ġhentai":45301,"ĠTypically":45302,"Ġloot":45303,".Sprite":45304,"Flight":45305,"Ġwavelength":45306,"-sk":45307,"ĠElle":45308,"_exports":45309,"ĠÑı":45310,"ĠIH":45311,"izophren":45312,"Ġíģ":45313,"_primary":45314,"Ġmois":45315,"ĠBN":45316,"Ġsystemic":45317,"Ġdiferentes":45318,"INCT":45319,"Ġ''ĊĊ":45320,"$q":45321,"WidgetItem":45322,"clide":45323,"$file":45324,"Lemma":45325,"/table":45326,"agrid":45327,"ĠMongoDB":45328,"inte":45329,"Ġapprent":45330,"ÂŃing":45331,".Db":45332,"ĠÃĤ":45333,"hammer":45334,"='';Ċ":45335,"Ġbrokers":45336,"itlement":45337,"semblies":45338,"Ele":45339,"{x":45340,"Ġlastname":45341,"<-":45342,"Ġflatten":45343,"_band":45344,".Root":45345,".readFileSync":45346,"======":45347,".rx":45348,"?čĊ":45349,"Ġmetaphor":45350,"Ti":45351,"conte":45352,"Ġdebit":45353,"Ġcontempt":45354,"CppType":45355,"æĶ¯":45356,"FormField":45357,"ratio":45358,"osopher":45359,"Ġimplant":45360,"PURE":45361,"Ġalta":45362,"_management":45363,"Ġrefine":45364,"ĠCheckBox":45365,"ĠCharl":45366,"-version":45367,"conditional":45368,"venues":45369,"Ġrifles":45370,"Ġoffspring":45371,"Ġmilling":45372,"Ġsharply":45373,"Ġunderwater":45374,"(origin":45375,"_Control":45376,"Ġ.$":45377,"Plugins":45378,"Ġdrying":45379,"Ġillustrates":45380,"-u":45381,"Ġvegetarian":45382,"npc":45383,"Heart":45384,";',Ċ":45385,"comma":45386,"teenth":45387,"asan":45388,"/spec":45389,"_moves":45390,"-margin":45391,"Ġingen":45392,"³³³":45393,"Ġprojet":45394,"Ġotra":45395,"Ġbras":45396,".utc":45397,"Ġslept":45398,"=sub":45399,"abilit":45400,"poster":45401,"Ġsdk":45402,"ouncill":45403,"Ġwd":45404,"PreparedStatement":45405,"ĠDrum":45406,"(attribute":45407,"ĠEthernet":45408,"ĉDB":45409,"California":45410,"cube":45411,"[I":45412,".Created":45413,"ĠHM":45414,"Ġtracing":45415,"FormsModule":45416,"-you":45417,".currency":45418,"feeding":45419,"Ġtbody":45420,"Li":45421,"accion":45422,"nas":45423,"Ġtrouver":45424,"NONE":45425,"\"},čĊ":45426,"Ġftp":45427,"WithIdentifier":45428,"polate":45429,"FileInfo":45430,"Ġpursued":45431,"ĠĠĠĠčĊĠĠĠĠčĊ":45432,"DESCRIPTION":45433,"}*/Ċ":45434,"FromNib":45435,"Ġdecorative":45436,"_SSL":45437,"(chat":45438,"TLS":45439,"Ġsurprises":45440,"alculate":45441,"ĠSplash":45442,"(Configuration":45443,"ĠSEM":45444,"imson":45445,"/library":45446,"":45521,"GED":45522,"faq":45523,"Ġoptionally":45524,"_Dis":45525,"ĠSuccessful":45526,"ĠCensus":45527,"Ġincarcer":45528,"_CARD":45529,"Ġaviation":45530,"ĠGym":45531,"Authority":45532,".Bean":45533,"shader":45534,"NotExist":45535,"_TextChanged":45536,"ĠSTOP":45537,"(team":45538,"\"H":45539,"wg":45540,"Ġgrinder":45541,"Ġstripe":45542,"Ġpreservation":45543,"Claim":45544,"aversal":45545,"warehouse":45546,"targets":45547,"Trust":45548,"Ġallev":45549,",www":45550,"ousse":45551,"_chan":45552,"_Size":45553,"systems":45554,"Ġobjection":45555,"ĠKane":45556,"Ġcorros":45557,"ĠDSL":45558,"Ġua":45559,"ĠMH":45560,"ĠStrategic":45561,"_tcp":45562,"Ġê°Ĵ":45563,"Ġborrowed":45564,"ĠAch":45565,"ĉcommand":45566,"Ġgps":45567,"leston":45568,"ichever":45569,"ĠUA":45570,"Ġassaulted":45571,"Ġspecializes":45572,"ĉsearch":45573,"Hotel":45574,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠčĊ":45575,"ĠPitch":45576,"ĠÙģ":45577,"READY":45578,"Ġparental":45579,"Ġgéné":45580,"Ġdonnées":45581,"Ġdetain":45582,"TARGET":45583,"Ġprotagonist":45584,"ĠclearInterval":45585,"ĠIconButton":45586,"ĠGetAll":45587,"TypeInfo":45588,"EH":45589,"âĢľThey":45590,"Ġ{[":45591,"Ġgag":45592,"ĠÚ©":45593,"ĠDropdown":45594,".free":45595,"gone":45596,"imens":45597,"Ġinstal":45598,"ĉcurl":45599,"_CAN":45600,"ĠBone":45601,"ï¼Ķ":45602,"onyms":45603,"-government":45604,".bindingNavigator":45605,"ĠDans":45606,"ĠMcL":45607,"(en":45608,">(_":45609,"ÐĴÑĭ":45610,".*;čĊ":45611,"=j":45612,"-cor":45613,"Son":45614,".ToolStripItem":45615,"-around":45616,"_XML":45617,"endDate":45618,"Ġslack":45619,"Ġrotated":45620,"Ġnoqa":45621,"Ġcottage":45622,"Ġencontrar":45623,"_skill":45624,"houette":45625,"!čĊ":45626,".weather":45627,"Ġemphasized":45628,"å®¶":45629,"ĠÑģпиÑģ":45630,"ĠCompiler":45631,"(android":45632,"ĠâĢº":45633,".turn":45634,"Ġsuppression":45635,"_calls":45636,"Ġ*@":45637,"(strlen":45638,".hex":45639,"ĠBills":45640,"ĠRSA":45641,"ÏĤ":45642,"ĠEscape":45643,"ementia":45644,"Ġfrontend":45645,"Ġpint":45646,"_exc":45647,"zzo":45648,"[],Ċ":45649,"Ġ\"','\"":45650,".Environment":45651,"Ġaforementioned":45652,"Ġendure":45653,"prototype":45654,"therapy":45655,"ssi":45656,"Deg":45657,"_plugins":45658,".userInfo":45659,"Printer":45660,"ĠPROGRAM":45661,"Ġruins":45662,"Ġempirical":45663,"Ġcrawl":45664,"ĠBoiler":45665,"-comment":45666,".subplot":45667,"_et":45668,"Ġ'.',":45669,"minor":45670,"ĠCustoms":45671,"Ġyaw":45672,"underline":45673,"ĠComo":45674,"(('":45675,"(mean":45676,"Ġchaque":45677,"ĠBlocks":45678,".rad":45679,"ilibrium":45680,"Ġwebdriver":45681,"Ġmelhor":45682,"dana":45683,"ĠAbuse":45684,"ĠSouthwest":45685,"ĠParen":45686,"PERTIES":45687,"ĉIL":45688,"Ġscream":45689,"vu":45690,"Ġincomes":45691,"Ġnim":45692,"Ġlace":45693,"Ġcompensate":45694,"Reverse":45695,"Dat":45696,"_attack":45697,"Ġnour":45698,"achen":45699,"cek":45700,"\"+":45957,"Ġtokenizer":45958,"Ġsovereignty":45959,"ĠPence":45960,"()\");Ċ":45961,"Ġpessoas":45962,".Ge":45963,"ĠIncluded":45964,"Ġpagina":45965,"Ġexposing":45966,"еÑĪ":45967,"_SCRIPT":45968,"/$',":45969,"Thumbnail":45970,"×Ķ":45971,"webElementX":45972,"webElementXpaths":45973,"pressure":45974,"ĠCurry":45975,"_CP":45976,"OLUTION":45977,"ILES":45978,"protect":45979,"oola":45980,"Workspace":45981,"{};Ċ":45982,"ĠUNS":45983,"Ġsympathy":45984,"roker":45985,"Ġremodel":45986,"ĉcell":45987,"Ġatop":45988,".FullName":45989,"Ġfaut":45990,"ĠEasily":45991,"_dynamic":45992,"Ġframed":45993,"Ġmotive":45994,"è·¯":45995,"sam":45996,"Ġmarca":45997,"ĠTextEditingController":45998,"Ġdestructor":45999,"cream":46000,"Ġrude":46001,"ĠBold":46002,"ĠIndigenous":46003,"Ġgens":46004,"Ġrelacion":46005,"(system":46006,"ĠUIFont":46007,"_charge":46008,"USTER":46009,"EV":46010,".Namespace":46011,"Ġmerger":46012,"Ġcalloc":46013,"gang":46014,"BadRequest":46015,"Ġsper":46016,"-design":46017,"Ġâĩ":46018,"Chan":46019,"Ġorganism":46020,",)":46021,"=id":46022,"_plane":46023,"ĠCases":46024,"elfast":46025,"ĠLegislature":46026,"ĠFaker":46027,"Ġinvoking":46028,"-utils":46029,"().'":46030,".face":46031,"Ġguardian":46032,"myModal":46033,"Ġclipboard":46034,"ĠATM":46035,"Ġpeas":46036,"ĠSylv":46037,".calc":46038,"ĠContacts":46039,"intValue":46040,"Ġmodifying":46041,"ĠBarb":46042,".loss":46043,"_percentage":46044,"Asked":46045,"(lst":46046,"ategorical":46047,"-files":46048,"ĠRomania":46049,".Ac":46050,"Ġhai":46051,"ĠFlying":46052,"Ġż":46053,"jp":46054,"ĠTrainer":46055,".arc":46056,"_deg":46057,"Ġtraceback":46058,"OrFail":46059,"FLOW":46060,".old":46061,"oya":46062,"gmt":46063,"isempty":46064,"Ġvaccination":46065,"Ġobsolete":46066,"recognized":46067,"Ġruined":46068,"ĠRein":46069,"ĠTracking":46070,"xfb":46071,"اÛĮ":46072,"Ġvære":46073,"Ġbryster":46074,"ĠITS":46075,"Ġdestiny":46076,"Ġswear":46077,"Ġredes":46078,"Ġclf":46079,"Ġflipped":46080,"ĉhead":46081,"Bluetooth":46082,"ĠOverrides":46083,":Boolean":46084,"_=":46085,"_lr":46086,"spawn":46087,":index":46088,"VALUES":46089,"iskey":46090,"?\");Ċ":46091,".synthetic":46092,"ĠChecking":46093,"structures":46094,"iping":46095,"Ġvocals":46096,"-Up":46097,"ĠManufacturers":46098,"ĠMarriage":46099,"代çłģ":46100,"Ġgarner":46101,"_Client":46102,"parallel":46103,"RIEND":46104,"Ġvinegar":46105,"segue":46106,"JB":46107,"Ġcontacting":46108,"ĠCarroll":46109,"Ġoutreach":46110,"tensor":46111,"_variant":46112,"Ġtheat":46113,"licable":46114,"{|":46115,"tiny":46116,"_letter":46117,"Ġpencil":46118,"HeadersHeightSizeMode":46119,"iltro":46120,".autoconfigure":46121,".drag":46122,".useState":46123,"ĠBMI":46124,"hint":46125,"Compile":46126,"*\\":46127,"enary":46128,"Ġlvl":46129,".Cache":46130,"+=\"":46131,"_tv":46132,"ruitment":46133,"Ġfread":46134,"Articles":46135,"fila":46136,"Ġpackaged":46137,"âĺĨ":46138,"ATHER":46139,"ĠPlanned":46140,"scheme":46141,"Ġdiary":46142,"Ġoffenses":46143,"/F":46460,"ĠStick":46461,"Ġcerc":46462,"ĠSlee":46463,"ĉĉĠĠĠĠĠĠĠĠ":46464,"":46639,"ĉcol":46640,"VG":46641,"_boolean":46642,"recent":46643,"Ġ*)ĊĊ":46644,"ĠRainbow":46645,"ommen":46646,"Ġlur":46647,"Ġoppression":46648,"(\",\");Ċ":46649,"ĠFacility":46650,"DEFINED":46651,"Ġneon":46652,"Ġoffender":46653,"AFP":46654,"ĠCleaning":46655,"[]):":46656,"Ġundocumented":46657,".Repositories":46658,"ĠGuitar":46659,"аÑģÑģив":46660,"Skills":46661,"Ġtestimon":46662,"ryptography":46663,"ĠAmber":46664,"ĠStalin":46665,"Ġlone":46666,"Ġapenas":46667,"Ġdieses":46668,"ĠArduino":46669,"转":46670,"==-":46671,"_Act":46672,"Ġcoded":46673,"âĸł":46674,"amburger":46675,"-links":46676,"Ġarmour":46677,".High":46678,"getContent":46679,"stag":46680,"Ġheck":46681,"ĠìĹĨ":46682,"ĠMcConnell":46683,"ĠConcert":46684,"ĠAlloc":46685,"äre":46686,".replaceAll":46687,"Ġpartitions":46688,"rott":46689,"ĠFle":46690,"_TREE":46691,"reasonable":46692,"ĠReporting":46693,"Ġbillionaire":46694,"scores":46695,"mins":46696,"-eye":46697,"MORE":46698,"abort":46699,"ĠSWT":46700,"Ġinverted":46701,"ĠTeachers":46702,";n":46703,"Ġastro":46704,"нов":46705,"аниÑĨ":46706,"producto":46707,"countries":46708,"ĠOwen":46709,"Ġcontamination":46710,"Ġvibe":46711,"ĠElli":46712,".script":46713,"ĠOlive":46714,"DMA":46715,"vier":46716,":semicolon":46717,"-module":46718,"gressive":46719,"agu":46720,"_players":46721,"Ġresultados":46722,"started":46723,"scrollTop":46724,"=====":46725,"Ġweighing":46726,"Ġ[[[":46727,"zahl":46728,"(NS":46729,"ĠAssertion":46730,"league":46731,".setTextColor":46732,"ĉMessage":46733,"Ġmoms":46734,"_AF":46735,".wh":46736,"ALS":46737,"Ġautre":46738,"]ĊĊĊĊ":46739,".opacity":46740,"ĠBuddhist":46741,"Ġdeaf":46742,"ĠOrganisation":46743,"(Global":46744,"ensch":46745,"Ġheadache":46746,"ĠAlien":46747,"_inode":46748,"ĠStark":46749,"Ġæī":46750,"-lnd":46751,"oref":46752,"_feat":46753,"Ġpedestrian":46754,"Ġnominal":46755,"Ġballoon":46756,"Ġsprites":46757,"PrototypeOf":46758,"ĠApost":46759,"ĠFEATURE":46760,"OH":46761,"Ġrecess":46762,"ĠDonna":46763,"consumer":46764,"$GLOBALS":46765,"ĠGIF":46766,"-frame":46767,"Inicio":46768,"Ġpassages":46769,"DateString":46770,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":46771,".byte":46772,"Bug":46773,"initializer":46774,"pkt":46775,"odium":46776,"ĠDER":46777,".ops":46778,"leri":46779,"Ġgifted":46780,"Ġdetach":46781,"terrain":46782,"elters":46783,"ãģı":46784,".loader":46785,"ĠNGO":46786,"strncmp":46787,"Kh":46788,"(fontSize":46789,"rocket":46790,"Ġprecedent":46791,"ĠAurora":46792,"ĠExperiment":46793,"isphere":46794,"Encoded":46795,"ĠâĢĵĊĊ":46796,"Ġpyramid":46797,"ĠAnniversary":46798,"ofil":46799,"ëŁ":46800,"(plugin":46801,"Coeff":46802,"Ġcooperate":46803,"Ġpredominantly":46804,"ISM":46805,"Phrase":46806,"_DEFINE":46807,"Flip":46808,"AMILY":46809,"ĠMarkets":46810,"ĠStreamReader":46811,"ĠCombine":46812,"Ġmanuscript":46813,"zza":46814,",tp":46815,"Whatever":46816,"ITICAL":46817,"ighbour":46818,"DataProvider":46819,".Texture":46820,"privacy":46821,".SDK":46822,"Ġrecharge":46823,"Ġcpp":46824,"ĠCFG":46825,"(holder":46826,"(py":46827,"mot":46828,"Ġsavoir":46829,"ĠRosa":46830,"ĠPCs":46831,"ĠíĻ":46832,".heroku":46833,"Ġfren":46834,"ĠRiley":46835,"agate":46836,"Ġsond":46837,".xlsx":46838,"Ġhacked":46839,"stad":46840,"Gi":46841,"Ġsanity":46842,"ĠSqlDataAdapter":46843,"...\",":46844,"ĠPussy":46845,"Ġ****************":46846,"Ġhassle":46847,"_PARENT":46848,"ĠUAE":46849,"Ġbeginners":46850,"(Client":46851,"Ġstatistically":46852,".hour":46853,"edelta":46854,"Ġtraction":46855,"uelve":46856,"arat":46857,"Ġsauna":46858,"INVALID":46859,"Ġindictment":46860,"ALLE":46861,"Ġdissent":46862,"ĠTypography":46863,"Ġintentional":46864,"sit":46865,"ĠAnimals":46866,"Ġcountryside":46867,"Ġuart":46868,"}\\\"":46869,"Ġseamless":46870,"¾ç¤º":46871,"Ġautos":46872,"Ġ\"'\";Ċ":46873,"Flush":46874,"ANNOT":46875,"Ġalgebra":46876,"assoc":46877,"ĠWaters":46878,"Ġpreparations":46879,"ronym":46880,"[,]":46881,"Sans":46882,"Ġarmies":46883,"ipeg":46884,"Ġcreamy":46885,".art":46886,"etre":46887,"ĠAnimated":46888,"Ġunpleasant":46889,"emean":46890,"great":46891,"iÄħ":46892,"ĠEarlier":46893,"Ġchic":46894,"Ġpreserving":46895,"(exec":46896,"ĠInvestigation":46897,"ĉGPIO":46898,"Ġrigorous":46899,"ijo":46900,"=num":46901,"ĠtoolStrip":46902,")set":46903,"+\"&":46904,"ĠAcceler":46905,"Ġdevelopmental":46906,"isposable":46907,"Ġflawed":46908,"rene":46909,"Updating":46910,"Ġwatchdog":46911,"Ġdenominator":46912,"Ġsuburbs":46913,"Ġ...)":46914,"Ġconvictions":46915,"closure":46916,".IP":46917,"Ġtranslates":46918,".swt":46919,".Trace":46920,"Ġmettre":46921,".isEnabled":46922,"ĠEffective":46923,".toInt":46924,"Ġenchant":46925,"Ġstunned":46926,"Ġpoi":46927,"/code":46928,"adm":46929,".databinding":46930,"ĠLorem":46931,"________________________________________________________________":46932,"Ġledger":46933,"Ġcara":46934,"ĠGir":46935,"Ġwaits":46936,"Uno":46937,"Ġcwd":46938,"è¾ij":46939,"ĠTResult":46940,"Ġrejo":46941,"Ġemitted":46942,"ĠWestminster":46943,"ä¸Ģ个":46944,"nek":46945,"_Tis":46946,"Ġenact":46947,"ĉwith":46948,"orgia":46949,"Ġjue":46950,"Perform":46951,"SPATH":46952,".topic":46953,"ĠDaten":46954,"ầ":46955,"Ġsitio":46956,"_MM":46957,"\"So":46958,"bial":46959,"Ġscoped":46960,"Requires":46961,"ĠTOTAL":46962,"ĠChancellor":46963,"(contents":46964,"Ġstealth":46965,"devices":46966,"-pass":46967,"ilih":46968,"ĠMalcolm":46969,"ĠDepot":46970,"Ġconfigur":46971,"aussian":46972,"_constraint":46973,"веÑĤ":46974,"GRA":46975,"ĠRates":46976,".dataGridViewTextBoxColumn":46977,"ĠNobel":46978,"itics":46979,"Ġignorant":46980,"ĠReporter":46981,"ĠEbola":46982,"ĠShock":46983,"_relation":46984,"ĠNinja":46985,")c":46986,"Ġticker":46987,".isChecked":46988,"ĠSuppliers":46989,"ĠRapid":46990,"Levels":46991,"âĤ¬âĦ¢":46992,"ĉqueue":46993,"Ġchop":46994,"ĠUnix":46995,"reject":46996,"-calendar":46997,"(sort":46998,"ène":46999,"ercicio":47000,"Ġhect":47001,"CALLTYPE":47002,"roupon":47003,"Ġrentals":47004,"authors":47005,"{name":47006,"ĠFIFO":47007,"Ġlassen":47008,"ĠNous":47009,"Ġsnapped":47010,"Ġfertility":47011,"\"log":47012,"clicked":47013,"Ġplanting":47014,"Ġgb":47015,"/output":47016,"PEAT":47017,"Ġcategoria":47018,"Ġbach":47019,"Professor":47020,"inth":47021,"\"]čĊ":47022,"Recorder":47023,"serde":47024,"ĠTransmission":47025,"trad":47026,"Ġturbo":47027,"_VERTEX":47028,"\\Event":47029,"ilver":47030,"Ġbodily":47031,"ĠSources":47032,"Ġkillings":47033,".xrTableCell":47034,"Ġfolded":47035,"/legal":47036,"uner":47037,"ĠRifle":47038,"ĠMIDI":47039,"_SelectedIndexChanged":47040,".SizeType":47041,"ĠWebSocket":47042,"Ġseleccion":47043,"Sand":47044,"otros":47045,"Ġenvision":47046,"/etc":47047,"ĠMelissa":47048,"Spot":47049,"ное":47050,"_ARM":47051,"Attempt":47052,"ĠBI":47053,"ãģĶ":47054,"ĠDU":47055,"Ġbacklash":47056,"stride":47057,"/classes":47058,"ĠtextColor":47059,"_staff":47060,"oblin":47061,"agenta":47062,".collections":47063,"illage":47064,"'čĊčĊ":47065,"flatten":47066,"_sales":47067,"_MASTER":47068,"TW":47069,"_da":47070,"Pitch":47071,"phies":47072,"Ġzombies":47073,"ĠVERY":47074,"ĠPharmacy":47075,"ĠprogressBar":47076,"Ġhashtag":47077,"Sidebar":47078,"@stop":47079,"(pc":47080,"олж":47081,"MAKE":47082,"ĠCoron":47083,"Ġkvinner":47084,"ĠMaid":47085,"bob":47086,".titleLabel":47087,"Ġsuccesses":47088,"ĠDemocracy":47089,"ĠSurgery":47090,"Ġcougar":47091,"Ġcurso":47092,"Ġloro":47093,"istency":47094,"Senior":47095,"æk":47096,"ĠAAA":47097,"ĠBOOK":47098,"ко":47099,"WSTR":47100,"Ġ*/,Ċ":47101,"oyal":47102,".vector":47103,"ĠSPEC":47104,"SSF":47105,"Ġcompuls":47106,"ĠAppeals":47107,"ĠWinston":47108,"ĠMockito":47109,"contrib":47110,".available":47111,"entityManager":47112,"arias":47113,"_sale":47114,"_rs":47115,"Ġdecoding":47116,"Ġlocator":47117,"olith":47118,"Ġkol":47119,"Ġascii":47120,"ĠRut":47121,"/interface":47122,"ĉĉĉĉĉĉĠĠĠ":47123,"ĠNumer":47124,".flip":47125,"-del":47126,"Ġbolster":47127,"onomic":47128,"Ġzm":47129,"LG":47130,"FindBy":47131,"Ġadaptive":47132,"loo":47133,"Ġvue":47134,"(reverse":47135,"_canvas":47136,".roles":47137,"ificado":47138,"venient":47139,"\"As":47140,"ĠEntr":47141,"aligned":47142,"Ġbereits":47143,"///ĊĊ":47144,".gwt":47145,".employee":47146,"_cli":47147,"Ġanticipate":47148,"éĻIJ":47149,"Ġpik":47150,"Ġmushrooms":47151,"(tt":47152,"Ġoma":47153,"ĠSanchez":47154,"_google":47155,".Valid":47156,"ĠFileName":47157,"ivative":47158,"ked":47159,"-war":47160,"Ġmaturity":47161,"ид":47162,"Ġminer":47163,"Reducers":47164,"ĠLatLng":47165,"_STD":47166,"Digits":47167,"Calc":47168,"-upload":47169,"Ġhandic":47170,"ีà¹Ī":47171,"egrated":47172,"ĠSTM":47173,"Clients":47174,"ĠTurbo":47175,"SYNC":47176,"Ġphotographers":47177,".Out":47178,".character":47179,"BUILD":47180,".unlock":47181,"Ġarises":47182,"ĠCommands":47183,"(\"\");čĊ":47184,"_FORE":47185,";',":47186,"+\"'":47187,".Images":47188,"\"){":47189,"ĠMeyer":47190,"Ġnegatively":47191,"ĠDLL":47192,"Ġexe":47193,"Ġdeficiency":47194,"Ġwildly":47195,"-switch":47196,"construction":47197,"Ġexceptionally":47198,"ĠLiz":47199,"/java":47200,"Ġtheirs":47201,"ĠContemporary":47202,"lis":47203,".fillRect":47204,"ĠNFC":47205,"Ġrehe":47206,"(numbers":47207,"Ġraster":47208,"Ġfiguring":47209,"Ġshowc":47210,"ĠJill":47211,"Ġarcade":47212,"ĠConstructs":47213,"mdl":47214,"('|":47215,"Ġidentifiers":47216,"Ġstellar":47217,"(Connection":47218,"Ġ\"{{":47219,"yor":47220,"(mysqli":47221,"Ġdove":47222,"OfBirth":47223,".disconnect":47224,"_hi":47225,"Ġzwischen":47226,"ĠGrund":47227,"iros":47228,"_Array":47229,".onclick":47230,"ansom":47231,"Answers":47232,"ĉremove":47233,"Fa":47234,"Ġhurry":47235,"-inf":47236,"ĠgetClass":47237,"ĠRegulation":47238,"ĠFLAGS":47239,"misc":47240,"Ken":47241,"_heading":47242,"GHz":47243,"-entry":47244,"Ġbiography":47245,"Sig":47246,"-mf":47247,"Watcher":47248,"âĢľA":47249,"}px":47250,"Ġspicy":47251,"_sq":47252,"Lost":47253,"(track":47254,"али":47255,"Descending":47256,"((":47453,"survey":47454,"Ġíĺ":47455,"...')Ċ":47456,"ĠDivider":47457,"osl":47458,"_CANCEL":47459,"_prepare":47460,"stin":47461,"ĠHeath":47462,".PrimaryKey":47463,"ĠâĨIJ":47464,"ĠLocalDateTime":47465,"Ġcooperative":47466,"Learning":47467,".enqueue":47468,"Ġgoog":47469,"ĠRegression":47470,"imates":47471,"Ġvoyeur":47472,"ĠDrink":47473,"plug":47474,"Ġlender":47475,"mana":47476,"Ġpersonnes":47477,"ypse":47478,"Ġunlink":47479,"ĠRavens":47480,"Ġhurd":47481,"Ġperiodically":47482,"ARGS":47483,"ĠGH":47484,"characters":47485,"...\"ĊĊ":47486,"-establish":47487,"Ġdn":47488,"(condition":47489,"ĠGravity":47490,"Ġestas":47491,"_focus":47492,"Creature":47493,"(site":47494,"Ġcarr":47495,"ĠRL":47496,"ĠRI":47497,"ĠMoto":47498,"ASF":47499,"ĠLuckily":47500,"ĉRoute":47501,"Ġentropy":47502,"(\",\"":47503,"Collect":47504,"(contact":47505,"ĠFlorence":47506,"Ġpremiums":47507,"Ġlifecycle":47508,"Ġbans":47509,"xef":47510,"WebKit":47511,"ĠFloating":47512,"Ġcosa":47513,"Specific":47514,"ĠLoans":47515,"bread":47516,"Ġdescriptors":47517,"Ġ{:.":47518,"THREAD":47519,"ĠTrent":47520,"Ġscop":47521,"QA":47522,"ĠAntar":47523,"pel":47524,"_difference":47525,"_changes":47526,"(...)":47527,"ĠRotation":47528,"ĠLGPL":47529,"ĠJUST":47530,"(Task":47531,"_subset":47532,"ĠTRANS":47533,"åĬĽ":47534,"ĠScout":47535,"-popup":47536,"Ġsmoked":47537,"_Class":47538,"Ġturnover":47539,"brakk":47540,"ĠRocky":47541,"tas":47542,".RegularExpressions":47543,"ĠElliott":47544,"ĠSpinner":47545,"DUCTION":47546,"Ġlibre":47547,"Ġmolto":47548,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":47549,"ĠFTP":47550,"mpeg":47551,"(features":47552,"Ġbald":47553,"ĠVid":47554,"Ġshouting":47555,"Lint":47556,"Ġsockets":47557,"Ġprow":47558,"Ġnouvelle":47559,"iscard":47560,"ĠSponsor":47561,"Ġconsulta":47562,")));":47563,"Indian":47564,"ĠRaspberry":47565,"Ġteammate":47566,"ĠJWT":47567,"ĠGhana":47568,"Ġcakes":47569,"primer":47570,"forma":47571,"ergarten":47572,"_Manager":47573,"Ġpreseason":47574,"GAME":47575,"|\"":47576,"ĠBrock":47577,"Ġoccupy":47578,"Ġdecorations":47579,"ánd":47580,"Ġcot":47581,"Ġparan":47582,"Disk":47583,"remain":47584,">?":47585,"Strong":47586,"Ġfrance":47587,"ĠEra":47588,"-cr":47589,".BufferedReader":47590,"ĠParadise":47591,"ĠVAT":47592,"ĠAnders":47593,"Ġlimb":47594,"ampoo":47595,"Ġimperative":47596,"UTILITY":47597,"ĠRecognition":47598,"Ġragazze":47599,"Ġpops":47600,"ypress":47601,"Ġembargo":47602,"//{Ċ":47603,"Ġsyll":47604,"PTR":47605,"åŃĺåľ¨":47606,"Ġdidnt":47607,"Mailer":47608,"Ġacademics":47609,"ĠFrauen":47610,"neider":47611,"-rel":47612,"Ġrainbow":47613,"(In":47614,"Ġsliced":47615,"=============Ċ":47616,"(send":47617,"NSMutableDictionary":47618,"vos":47619,"(package":47620,"Ġordinance":47621,"viewer":47622,"ĠSantos":47623,"-selling":47624,"Ġgov":47625,"ettle":47626,"Ġfounders":47627,"Ġwaking":47628,"slashes":47629,"-pound":47630,"recht":47631,"ات":47632,".onClick":47633,"Ġnord":47634,"ständ":47635,"_when":47636,"UTERS":47637,"icc":47638,"Ġcapsule":47639,"ĠWid":47640,"Marc":47641,"ุ":47642,"rored":47643,"UGE":47644,"LOUD":47645,"ĠAudit":47646,"ipients":47647,"opian":47648,"ĠSue":47649,"Ġwurden":47650,".Helpers":47651,"Ġfactions":47652,"[np":47653,"-than":47654,"Ġreco":47655,"Ġkas":47656,"Ġcmds":47657,"/network":47658,"xbf":47659,"getColor":47660,"Ġbiased":47661,"ĠLak":47662,"Datas":47663,"vents":47664,"Ġë²":47665,"_PS":47666,".Validate":47667,"Invoker":47668,"Ġneuen":47669,"Ġjuvenile":47670,"VISION":47671,"Ġdevote":47672,"Ġlinha":47673,"Ġdiscounted":47674,"\\Config":47675,"Ġworthwhile":47676,"Ġskinny":47677,"ĠCourses":47678,"leys":47679,"ĠMortgage":47680,"Kevin":47681,"Ġannounces":47682,"])*":47683,"reservation":47684,"Ġæķ°":47685,"Ġprejudice":47686,"ĠStringComparison":47687,"Ġbeard":47688,"-win":47689,"ĠSão":47690,"ĉms":47691,"jal":47692,"ĠEarn":47693,"_ports":47694,"ĠNombre":47695,"_COR":47696,"ĠBUILD":47697,".sound":47698,"Yellow":47699,"Ġlinebacker":47700,"Ġcharitable":47701,"jug":47702,"_NONNULL":47703,"ĠDental":47704,"\">${":47705,"ĉmatch":47706,"Russian":47707,"Ġversch":47708,"Ġpinned":47709,"Ġadopting":47710,"OptionsMenu":47711,"Pag":47712,"Ġpairing":47713,"Ġtread":47714,"ercises":47715,"ĠSpread":47716,")i":47717,"ĠBAD":47718,"_tf":47719,"UIImageView":47720,"populate":47721,"bab":47722,"ĠÏĥ":47723,"[++":47724,"Ġopioid":47725,"Ġ##Ċ":47726,"dtype":47727,"ĠStarts":47728,"('/')":47729,"Ġpersonals":47730,"-market":47731,"Ġredundant":47732,"ĠEssential":47733,"Ġscrapy":47734,"Ġим":47735,"acl":47736,"Ġcrear":47737,"ĠBend":47738,"Ġrelieve":47739,"-room":47740,"wife":47741,"ĠvÃł":47742,"ĠQPoint":47743,"Ġquasi":47744,"ĠmethodName":47745,"\\xc":47746,"ĠPeru":47747,"/The":47748,".orm":47749,"Ġviz":47750,"/pdf":47751,"Located":47752,"Ġconfrontation":47753,"ĠChampionships":47754,"Ġhypert":47755,"Ġdj":47756,"ĠUserInfo":47757,"ĠåĪĽå»º":47758,"\\xb":47759,"(sim":47760,"Ġ==Ċ":47761,"Ġstaging":47762,"Ġdrastically":47763,"åѦ":47764,"lords":47765,".less":47766,"ведиÑĤе":47767,"ĠBucket":47768,"ĠMam":47769,".term":47770,"_pi":47771,"czy":47772,".pub":47773,"precio":47774,"ĠVirt":47775,"Ġroman":47776,"itat":47777,"Lex":47778,"_infos":47779,"İ":47780,".other":47781,"VELO":47782,"Ġponder":47783,"Ġhanno":47784,"(Page":47785,"doi":47786,"Ġpolite":47787,"Ġprogrammer":47788,"Dies":47789,"$d":47790,"Ġreplication":47791,"addColumn":47792,"frican":47793,"Ġleng":47794,"beer":47795,"oit":47796,"Ġwasting":47797,"ylim":47798,"measure":47799,"Neg":47800,"Ġpartie":47801,".console":47802,"ĠGuinea":47803,"TEL":47804,"_fact":47805,".chunk":47806,"Ġlent":47807,"Ġaller":47808,"Ġà¤ķ":47809,"_idle":47810,"Ġadmissions":47811,"JSONArray":47812,"Ġvibration":47813,".helpers":47814,"å¤ĸ":47815,"Ġhen":47816,"john":47817,"ĠìĥĿ":47818,"Ġjudgement":47819,"Ġgeen":47820,"terra":47821,"^{":47822,"ĠIz":47823,"Ġcâ":47824,"instances":47825,"Ġthreatens":47826,"Ġmüssen":47827,"KindOfClass":47828,"Ġstorytelling":47829,"_demo":47830,"rias":47831,"Privacy":47832,"hift":47833,"ĠYi":47834,"esor":47835,"íķł":47836,"ensitivity":47837,".Writer":47838,"à¸Ĥ":47839,"District":47840,".getJSONObject":47841,"Impro":47842,"(getResources":47843,"ĠSPELL":47844,"roduce":47845,"Ġslowed":47846,"Ġlinewidth":47847,"Ġhonesty":47848,"ĠCoord":47849,"ĠFork":47850,"ĠDispatchQueue":47851,"ĠCliff":47852,"ĠWiring":47853,"_TIMESTAMP":47854,"ollah":47855,"avoid":47856,"++];Ċ":47857,"semantic":47858,"-css":47859,"Ġveto":47860,"ĠMerr":47861,"Ġlegislators":47862,"CEEDED":47863,"Ġquestionnaire":47864,"ĠPills":47865,"Calculate":47866,"(core":47867,"'e":47868,"Ġdislike":47869,"ĠPreferences":47870,"_EXTERNAL":47871,"è°ĥ":47872,"Ġdodge":47873,"æľįåĬ¡":47874,".names":47875,".drawImage":47876,"_prom":47877,"uckland":47878,"Ġ<$>":47879,"ız":47880,"/site":47881,"项":47882,"rophe":47883,"Ġcompelled":47884,"Ġlaptops":47885,"Ġuni":47886,"CLOSE":47887,"Ġcasualties":47888,"ĠUniform":47889,"Terminal":47890,".\",\"":47891,"DAT":47892,"(TreeNode":47893,"ĠGandhi":47894,"(stmt":47895,"AXB":47896,"*M":47897,"Ġumbrella":47898,"animal":47899,"Ġgrpc":47900,"Ġwhereby":47901,"Ġfloats":47902,"ĉarg":47903,"Ġdbg":47904,"Ġexceeding":47905,"EventType":47906,".SaveChangesAsync":47907,"Ġ{{{":47908,"Ġowed":47909,"ahrenheit":47910,"Ġì§":47911,"Ġequipo":47912,"urai":47913,"Ġidol":47914,"]\")Ċ":47915,"_major":47916,"Ġentirety":47917,"ingerprint":47918,"ços":47919,"/account":47920,"ĉright":47921,"ursos":47922,"ĠEDT":47923,"_INSERT":47924,"Ġshining":47925,"Ġ<:":47926,"EdgeInsets":47927,"Ġcolonies":47928,".IM":47929,"ĉĠĉ":47930,"ROAD":47931,"CCCC":47932,"placing":47933,"ĠgetActivity":47934,"emacs":47935,"'%(":47936,".clicked":47937,"ĠThem":47938,"isia":47939,"Buscar":47940,".rename":47941,"Ġoath":47942,"Ġafterward":47943,"ĠUFO":47944,"APS":47945,"ĠJacksonville":47946,".some":47947,"Confirmed":47948,".scan":47949,"igInteger":47950,"Decorator":47951,"shield":47952,"ressive":47953,".did":47954,"请è¾ĵåħ¥":47955,"Ġshutter":47956,"Dam":47957,"Ġparenting":47958,"eyed":47959,"$item":47960,"-develop":47961,"Ġextracts":47962,"Ġdecentralized":47963,"ĠElsa":47964,"_spin":47965,"])+":47966,"-initial":47967,"Ġmultitude":47968,"Ġsensory":47969,"ĠMODEL":47970,"Ġsafeguard":47971,"ì¹":47972,"Ġhunters":47973,"ĠTiny":47974,"INO":47975,"decorate":47976,"ĠNoSuch":47977,"Ho":47978,"(Response":47979,"Ġruler":47980,"ĉshort":47981,"Ġcaster":47982,"ĠclientId":47983,"Ġpdb":47984,"ëıĦ":47985,"itic":47986,"ĠGameState":47987,"ĠnewItem":47988,")ĊĊĊĊĊĊ":47989,"ouis":47990,"noc":47991,".BLACK":47992,"_VECTOR":47993,"----------();":48281,".getP":48282,"anye":48283,"Ġneuron":48284,"ifold":48285,"ĠKnown":48286,"Bitcoin":48287,"Anyway":48288,"ayette":48289,"Ġ'['":48290,"Ãłnh":48291,"mgr":48292,"Ġcorrelated":48293,"Ġnause":48294,"Ġmentality":48295,"hasMany":48296,"ĠFG":48297,"ampie":48298,"ITU":48299,"Fs":48300,".Sp":48301,"_between":48302,"Dependencies":48303,"oug":48304,"Placeholder":48305,"=text":48306,"ĠManaging":48307,"ocalypse":48308,"åĮĹ":48309,"_mag":48310,"fld":48311,"âij":48312,"CAM":48313,"ĠHelpers":48314,"Ġdost":48315,"/out":48316,"Ġassassination":48317,".getImage":48318,"ĠKenny":48319,".')ĊĊ":48320,"){//":48321,"ĠRanger":48322,"Ġgek":48323,"Ġsincere":48324,"čĊ":48527,".getResources":48528,"Ġlump":48529,"_consts":48530,"(ext":48531,"ĉdir":48532,"âĿ":48533,"ĠpaddingTop":48534,"Ġobsession":48535,"Ġbanning":48536,"ĠAppModule":48537,"Ġpartisan":48538,"Ġcatalogue":48539,"Ġminors":48540,"Ġpitches":48541,"weep":48542,"Ġundertake":48543,"Ġthemed":48544,"audit":48545,".scrollTop":48546,"Ġrer":48547,"Ġsymptom":48548,"Ġopenings":48549,".blocks":48550,"openid":48551,"Ġassh":48552,"-save":48553,"ĠPig":48554,"Ġregain":48555,"Ġinicial":48556,"/favicon":48557,"ĉexp":48558,"Ġspices":48559,"iska":48560,"claims":48561,"mak":48562,"definitions":48563,"Ġcorrespondent":48564,"ĠCannabis":48565,"__,Ċ":48566,"ĠLucky":48567,"ĠGaussian":48568,"ĠNearly":48569,"CAD":48570,"']]Ċ":48571,"Ġadequately":48572,"ĠTITLE":48573,"constitutional":48574,"-mm":48575,"_override":48576,"Ġblas":48577,".readyState":48578,"Ġreminis":48579,"Ġreinforced":48580,"ĠCollabor":48581,"Ġdecorating":48582,"Ġbachelor":48583,"ERRUPT":48584,"Ġupright":48585,"ipation":48586,"ĠNoble":48587,"ĠvalueForKey":48588,"ĠsetLoading":48589,".Ignore":48590,"åģ":48591,"Globals":48592,"ĠMent":48593,"ASSES":48594,"Ġlimbs":48595,"ĠHUD":48596,"inci":48597,".iv":48598,"ĠQModelIndex":48599,"Fuse":48600,"Ġpedal":48601,"_FREQ":48602,"(verbose":48603,"Ġlongitud":48604,"ĠCharter":48605,"ê·¸":48606,"Ġbundles":48607,".ignore":48608,"umbo":48609,"EMA":48610,".......":48611,"sx":48612,".Card":48613,"Ġheute":48614,"Ġsteer":48615,"jumlah":48616,"Ġ{_":48617,"_Checked":48618,"Ġfax":48619,"ĠGust":48620,"itchens":48621,"Ġ))ĊĊ":48622,"Ġremarkably":48623,"/XML":48624,"-remove":48625,"_bt":48626,"Ġincub":48627,".package":48628,".currentThread":48629,"ĠHighlander":48630,".side":48631,"splash":48632,"Ġici":48633,"=D":48634,"Ġpuck":48635,"Ġballots":48636,"Ġhugely":48637,"coeff":48638,"ĠpData":48639,".COLUMN":48640,"ĠHealing":48641,"Ġordin":48642,"!),":48643,"Ġ'',čĊ":48644,"(md":48645,"ĠSask":48646,"čĊ":48668,"Ġrá":48669,"Ġblunt":48670,"ĠImageIcon":48671,"ifik":48672,"RTC":48673,"Ġfibers":48674,"Ġtoile":48675,".sent":48676,"ĠPyQt":48677,"$app":48678,"Ġmedio":48679,"Ġgranting":48680,"Ġtslint":48681,"ĠMö":48682,"(figsize":48683,"Ġhurricane":48684,"Ġlifes":48685,"ĠÃĦ":48686,"rocessing":48687,"_standard":48688,"-option":48689,"')))":48690,"Ġvacant":48691,"å·¥":48692,"ĠHollow":48693,"handleChange":48694,"Ġdivider":48695,"ĠEngineers":48696,"Ġsvens":48697,"Ġcompliant":48698,"tanggal":48699,"ĠCredits":48700,"ĠEmirates":48701,"RuleContext":48702,"Ġrealization":48703,"Ġdistracted":48704,"]+=":48705,"Ġaugment":48706,"ĠDw":48707,"otp":48708,"orrent":48709,"Editar":48710,".stock":48711,"Study":48712,"pections":48713,"ĠGameManager":48714,"=cut":48715,"Ġflock":48716,"ĠRomans":48717,"them":48718,"-hop":48719,"Ġscreenshots":48720,"Ġ/*!Ċ":48721,"Ġconversions":48722,"Ġnormalization":48723,"(configuration":48724,"Ġaeros":48725,"_security":48726,"!'Ċ":48727,"Bonus":48728,"ĠDRIVER":48729,"ĉDate":48730,"tie":48731,"ĠWyoming":48732,"Stand":48733,"itre":48734,"Ġshoppers":48735,"Ġdisadvantage":48736,"Ġliking":48737,"ç¬ij":48738,"Ġunderstandable":48739,"SEE":48740,"Ġhoy":48741,"Ġninete":48742,"Ġconfer":48743,"Ġnowrap":48744,"ĠVern":48745,",čĊčĊ":48746,"imestep":48747,"LayoutManager":48748,"à·":48749,"ĉwait":48750,"PLETED":48751,"Japan":48752,"Ġinduce":48753,"Ġå¯":48754,"озв":48755,"_ENDPOINT":48756,".horizontal":48757,"Ġaccelerated":48758,"rimon":48759,"IVES":48760,"Transactions":48761,"Lean":48762,"ĠSOUR":48763,"whether":48764,"yg":48765,"Ġoid":48766,"ĠEntityManager":48767,"OUNTRY":48768,"Ġfila":48769,"OLUMNS":48770,"INUE":48771,"ĠAnchor":48772,"TRAN":48773,"woo":48774,"blockquote":48775,"ĠNurse":48776,"ĠCarp":48777,"Ġredeem":48778,".try":48779,"ĠJP":48780,"Ġtimestamps":48781,"Ġ?>\"><":48782,"ĠREMOVE":48783,"ĠStarbucks":48784,"Really":48785,"Ġflooded":48786,".Callback":48787,"DropDown":48788,"ipro":48789,"Ġtended":48790,"lte":48791,"Ġproportions":48792,"-te":48793,"ĠRena":48794,"licate":48795,"forces":48796,".extra":48797,".authenticate":48798,"вод":48799,"¡°":48800,"ĠforControlEvents":48801,"Ġsenha":48802,"Ġkein":48803,"Ġminist":48804,"ĠPreference":48805,"ĠTelegraph":48806,"Ñĥп":48807,"strpos":48808,"Ġillnesses":48809,"Ġpigs":48810,"ĠgetIntent":48811,"Sol":48812,"Ġ¡":48813,"(cpu":48814,"[prop":48815,"screens":48816,"');?>":48817,"ĠActs":48818,"Ġstrdup":48819,"Ġaverages":48820,"anal":48821,"ĠCasual":48822,"GroupBox":48823,"ĠHandbook":48824,"/comments":48825,"Ġnumbered":48826,"Ġbroadcasting":48827,"çĽij":48828,".nativeElement":48829,".mu":48830,"ĠupdatedAt":48831,"ĠDoesn":48832,".AC":48833,".coll":48834,"Ġrecorder":48835,"_sha":48836,"Bg":48837,"bil":48838,"Ġbolts":48839,"Ġç¬":48840,"Ġimposing":48841,"ĠInformationen":48842,"_flashdata":48843,"economic":48844,"Remark":48845,"ucas":48846,"ĠOfficers":48847,"ĠTER":48848,"Walk":48849,"Ġmercado":48850,"_generate":48851,"HY":48852,"Calling":48853,"snap":48854,"scriptId":48855,".operation":48856,"ĠFlame":48857,"liness":48858,"Ġrented":48859,"_toggle":48860,"-changing":48861,"ĠTY":48862,"'util":48863,"EEP":48864,"Ġgraphql":48865,"ĠUni":48866,"Ġimpulse":48867,".Basic":48868,"Ġenergies":48869,"MARY":48870,"ĠMarcel":48871,"Ġmortal":48872,"Ġfres":48873,"mens":48874,"motion":48875,"Ġsampled":48876,"âĢľThat":48877,"iday":48878,"quipment":48879,"getInt":48880,"ĠAbsolute":48881,",'\"":48882,"uned":48883,".share":48884,"Ġ})(":48885,"mmm":48886,"ĠRising":48887,"ä»»":48888,"Ġunemployed":48889,"xfa":48890,".follow":48891,"ĉĉĉĉĠĠĠĠĠĠ":48892,"slt":48893,".Phone":48894,"Ġknives":48895,"Ġeve":48896,"onClick":48897,"]))čĊ":48898,"ĠWitness":48899,"ĉNS":48900,"ĠEOS":48901,"ĠStefan":48902,"ĠPriest":48903,"âĢĶwhich":48904,"GetString":48905,".By":48906,"Ġupstairs":48907,"Ġdetriment":48908,"broken":48909,"embro":48910,"Ġnicotine":48911,"ilion":48912,"Ġastonishing":48913,"_aff":48914,"ĠLesson":48915,"Ġaccidental":48916,"odor":48917,"Ġdecir":48918,"ĠnewName":48919,"+.":48920,"缸":48921,"igslist":48922,"ĠGithub":48923,"Ġsuccessive":48924,"racial":48925,"Ġenviron":48926,"éªĮè¯ģ":48927,"Ġredirected":48928,"TOTAL":48929,"Ġgrabbing":48930,"ĠLance":48931,"Ġforfe":48932,"_CB":48933,"å¾®":48934,"Elapsed":48935,"_way":48936,"(DialogInterface":48937,"_measure":48938,"xbb":48939,"Dog":48940,"Depart":48941,"-src":48942,"resolver":48943,"withstanding":48944,"_shell":48945,"ĠLastName":48946,"ĠAviation":48947,"Ġbeginner":48948,"(\"%.":48949,"(tool":48950,"Ġнов":48951,":init":48952,"(API":48953,"ĠMorrison":48954,"vtColor":48955,"Ġstaple":48956,"/INFO":48957,"Ġsupernatural":48958,"Ġsteak":48959,"timeline":48960,"zzle":48961,"\"`ĊĊ":48962,"Secondary":48963,"ĠNepal":48964,".StringUtils":48965,"Ġadam":48966,"Ġ(...":48967,"Ġsubstitution":48968,"Ġboarding":48969,"ĠKeyword":48970,"ĠAssault":48971,"dbcTemplate":48972,"ĠorderId":48973,"(engine":48974,".assertThat":48975,"ĠVenus":48976,"Ġhomicide":48977,"ĠAval":48978,"Ġgutter":48979,"ĠSupported":48980,"/part":48981,"Ġacclaimed":48982,"Histor":48983,"Ġmeses":48984,"über":48985,"ĠRenew":48986,"Ġgras":48987,"ĠEk":48988,"Ġinfile":48989,"indy":48990,".music":48991,".Scroll":48992,"ĠAges":48993,"ĠNaruto":48994,"ĠGather":48995,"Ġconfirming":48996,"=(\"":48997,"Ġpitched":48998,"oley":48999,"France":49000,"+'\"":49001,"$total":49002,"Ġonde":49003,"Ġditch":49004,"_sigma":49005,"Ġcontinuity":49006,"reward":49007,"-load":49008,"Ġproceso":49009,"Locked":49010,"staw":49011,"Ġspinal":49012,"lazy":49013,"!==":49014,"jest":49015,"Ġdun":49016,"ĠRodgers":49017,"ĉgrid":49018,"Ġlogos":49019,"ĠBengal":49020,".super":49021,"Provides":49022,"Ġnutrient":49023,".Timestamp":49024,"IZATION":49025,"åĨĮ":49026,"Ġfats":49027,"ĠXxx":49028,"ctica":49029,"Targets":49030,"Ġcontours":49031,"Ġreordered":49032,":Array":49033,"Ġtolerate":49034,"Vir":49035,"Ġterribly":49036,"Ġbricks":49037,"(&_":49038,"hb":49039,"Portal":49040,"ĠBread":49041,".which":49042,"ÂŃt":49043,"asInstanceOf":49044,"Ġjobject":49045,"ĉlength":49046,"_MT":49047,";\">čĊ":49048,"_EXIST":49049,"Ġmaternal":49050,"REL":49051,"Ġê²½ìļ°":49052,"hee":49053,"Ġlayouts":49054,"ĠLap":49055,"aisy":49056,"Ġstumbled":49057,"ĠUIG":49058,"ĠSco":49059,"Ġimpaired":49060,"RESSED":49061,"Ġabuses":49062,"VF":49063,"ARB":49064,".NAME":49065,"rch":49066,"primir":49067,"_completed":49068,"Ġpenny":49069,"Chrome":49070,"(begin":49071,"ernen":49072,"-checkbox":49073,"PlainOldData":49074,"ĠLPC":49075,"rade":49076,"spir":49077,"Ġconceived":49078,"Tips":49079,"ĠIoT":49080,"ĠGan":49081,"èģĶ":49082,"Ġbiases":49083,"Ġconsultants":49084,"pled":49085,"_ht":49086,"associated":49087,"],ĊĊ":49088,"Ġdelightful":49089,"ĠÑĤек":49090,"Helvetica":49091,"(load":49092,"-expand":49093,"_WIDGET":49094,"toa":49095,"ĠAkt":49096,"Ġomn":49097,"Ġclauses":49098,"Intel":49099,"*/}Ċ":49100,"_registration":49101,"ĠoldValue":49102,"Ġrestoring":49103,"Ġunreal":49104,"OVER":49105,"ĉĊĉĊĉĊ":49106,"ATS":49107,"_probe":49108,"Ġdivisor":49109,".updateDynamic":49110,"å¹³":49111,"Produces":49112,"stamp":49113,".jboss":49114,"ĉtask":49115,"!(:":49116,"Ġpsychic":49117,"@class":49118,"Martin":49119,"ĠPassed":49120,"clarations":49121,"hel":49122,"аÑĩ":49123,"ĉcopy":49124,"-bin":49125,"zan":49126,"igram":49127,"াà¦":49128,"(sig":49129,"ĠCaval":49130,"_##":49131,"Ġ%=":49132,"outlined":49133,"ĠAcid":49134,"Ġunpredictable":49135,"-dashboard":49136,"HexString":49137,"+c":49138,".Public":49139,"ẩ":49140,"Ġconveyor":49141,"ĠEB":49142,"Ġselects":49143,"Ġknocking":49144,"ĠCec":49145,"IBUTES":49146,"owaÄĩ":49147,"gatsby":49148,"*v":49149,"entropy":49150,"Ġdispatched":49151,"Ġcamel":49152,"ĠSaturn":49153,"Ġoverweight":49154,"(phone":49155,"parable":49156,"%B":49157,"_vectors":49158,"Ġbrewing":49159,"ĠTk":49160,"ĠDownloads":49161,"ĠSaved":49162,".Price":49163,"Ġcurved":49164,"ĠParenthood":49165,"è¶":49166,".pnl":49167,"pletely":49168,".Day":49169,"Ġadvertisers":49170,"Ġejec":49171,"Ġprzed":49172,"ë¯":49173,"!';Ċ":49174,"ĠKush":49175,"ĠTAB":49176,"Ġquests":49177,"Ġcoincidence":49178,"ummies":49179,"ĠKashmir":49180,"ĠEthics":49181,"_growth":49182,"Ġaktiv":49183,"Ġgrouping":49184,"å¢ŀ":49185,"_truth":49186,"åIJ¬":49187,"todos":49188,"iset":49189,"TexCoord":49190,"ätt":49191,"ĠZur":49192,"roys":49193,"_MAGIC":49194,"Ġbrewery":49195,"(State":49196,"ĠSMALL":49197,"ĠPlants":49198,"itbart":49199,"eacher":49200,"ĠAdelaide":49201,"Lu":49202,"Ġfick":49203,"undles":49204,"_loaded":49205,"ие":49206,"Poll":49207,"ritic":49208,"ELY":49209,"Ġ+'":49210,"ĠProfession":49211,"Ġstamps":49212,"ĠSew":49213,"scrollView":49214,"Ġcommunist":49215,"/problems":49216,"}čĊčĊčĊčĊ":49217,",o":49218,"Ġudp":49219,"Ġobese":49220,"approve":49221,"ancellation":49222,"_Game":49223,"ĠHashtable":49224,"adaptiveStyles":49225,"Ġpossesses":49226,".matcher":49227,"functional":49228,"Mrs":49229,"ĉsave":49230,"ĠDbType":49231,"Ġken":49232,"getContext":49233,"Ġmans":49234,"(rel":49235,"ĠBrotherhood":49236,")`Ċ":49237,"è§£":49238,".Information":49239,"OutOfRangeException":49240,"ĠSek":49241,"Cas":49242,"Ġbloggers":49243,"Either":49244,"(\"\"\"":49245,"Ġpinch":49246,"Ġcoarse":49247,")p":49248,"ĠPulse":49249,"Ġlearnt":49250,"Ġdentist":49251,"Ġonchange":49252,"Ġdirectives":49253,"(actions":49254,"nyder":49255,"ĠShir":49256,"Trait":49257,"_dep":49258,"ĠPET":49259,"ĠREP":49260,".AppSettings":49261,"cuador":49262,"idenav":49263,"Ġenvi":49264,"Ġslammed":49265,"ĠShoot":49266,"ĠdateFormat":49267,".joda":49268,"veys":49269,"Ġ).ĊĊ":49270,"Ġcareg":49271,"ĠParallel":49272,"_translation":49273,".functions":49274,".obs":49275,"RuntimeException":49276,"[]=":49277,"overview":49278,"ĠSchl":49279,"Ġnoisy":49280,"ĠOnPropertyChanged":49281,"Sending":49282,"Ġunfamiliar":49283,"Upon":49284,"ĠPrints":49285,".typ":49286,"Ġfleeing":49287,"ĉmove":49288,"(Un":49289,"Ġqr":49290,"׾":49291,"_beta":49292,"Ġskies":49293,"ĉme":49294,"WND":49295,"Ġstickers":49296,"blas":49297,"Ġinserts":49298,"Ġverses":49299,"ĠDew":49300,"Ġtangible":49301,"Ġhecho":49302,"POL":49303,"Ġteardown":49304,"omnia":49305,"IBE":49306,".cover":49307,"_strategy":49308,"^-":49309,"setPosition":49310,"uale":49311,"Signed":49312,"Ġiface":49313,"aseline":49314,".setTime":49315,"ĠMineral":49316,"ĠFighting":49317,"skins":49318,"Ġdiscrimin":49319,"Ġdansk":49320,"ĠPrinceton":49321,"acist":49322,"Ġ());Ċ":49323,"tracks":49324,"imonial":49325,"adecimal":49326,"EPROM":49327,"uggle":49328,".Notification":49329,"$mail":49330,"cantidad":49331,"ĠJung":49332,"Ġseekers":49333,"Ġplausible":49334,"tier":49335,"еж":49336,"Ġrapper":49337,"ĠMana":49338,"ĠHttpStatusCode":49339,"Ġburnt":49340,"loses":49341,"ĠFoto":49342,"ĠJsonObject":49343,"Instagram":49344,"Ġsyscall":49345,"Ġrealities":49346,"ĠMATLAB":49347,":^{Ċ":49348,"TERM":49349,"ĠCbd":49350,"ĠParagraph":49351,"Ġtravés":49352,"Ġconstructing":49353,"Ġswal":49354,"Ġpige":49355,"LLLL":49356,"-existing":49357,"Gets":49358,"Ġmelted":49359,"Ġmitigate":49360,"Hen":49361,"Ġhm":49362,"imas":49363,"ĠAo":49364,"ĠPerez":49365,"ĠDAL":49366,"Ġëĭ¤":49367,"Ġdivis":49368,"StoryboardSegue":49369,"ĠModify":49370,"ĠÃľber":49371,"_OVERRIDE":49372,".pem":49373,"untos":49374,"Ġespañ":49375,"Ġ{?":49376,"ĠPAY":49377,"_ipv":49378,"ĠFury":49379,"__.__":49380,"elow":49381,"-centered":49382,"checks":49383,"_Reg":49384,"-Javadoc":49385,"ĉload":49386,"ĠLikewise":49387,"اÙħ":49388,"UNE":49389,".sem":49390,"xcb":49391,"ĠCave":49392,"_sleep":49393,"Ġsilently":49394,"ĠExtreme":49395,".ToUpper":49396,"ĉCHECK":49397,"Ġcue":49398,"ĠQByteArray":49399,"Ġcorrupted":49400,"ĠDé":49401,"Ġimped":49402,"GetName":49403,"Ġinaccurate":49404,"Ġsober":49405,"ее":49406,"Ġbarcode":49407,"--){Ċ":49408,"inki":49409,"Ġép":49410,"Ġdri":49411,"ĠALT":49412,">>>>>>>>":49413,"onta":49414,"[L":49415,"Ġinteres":49416,"verting":49417,"Ġdiagnostics":49418,"pdev":49419,"è©":49420,"ĠIntegrated":49421,").'":49422,"_gc":49423,"$text":49424,".games":49425,"ĠTerra":49426,"'Re":49427,".transfer":49428,"_FIFO":49429,"getModel":49430,"Ġbland":49431,"ĠColeman":49432,"Ġprimes":49433,"ĠæĪ":49434,"Ġcrosses":49435,"nk":49436,"GING":49437,"Ġ'^":49438,"ĠBlob":49439,"Ġintercourse":49440,"ĠBlvd":49441,"Ġweighs":49442,"_regular":49443,"ĠPerth":49444,"Ġseparating":49445,"Ġbilled":49446,".tabControl":49447,"Ġpuppet":49448,"Ġutilization":49449,"Ġâĸł":49450,"Ġsucces":49451,"Ġlamps":49452,"_proj":49453,"Eric":49454,"Ġrenovation":49455,"ĠFamilies":49456,"ĠBits":49457,"partials":49458,"-Men":49459,"solution":49460,"Ġdwarf":49461,".INTEGER":49462,"ĠLOCK":49463,".ct":49464,"Ġexcerpt":49465,"ĠPix":49466,"ĠFirstName":49467,"ANTED":49468,"ĠAdmir":49469,"-help":49470,"Prior":49471,"ĠAlign":49472,".INSTANCE":49473,"LineEdit":49474,"('/:":49475,"Ġinet":49476,"odus":49477,".pkl":49478,"ĠKY":49479,"upert":49480,"Ġnerves":49481,"_gradient":49482,"}','":49483,"_unref":49484,"Ġsaturated":49485,"ĠConnected":49486,"ĠFN":49487,"EXIT":49488,"Ġteleport":49489,"Ġavait":49490,"PageRoute":49491,"Ġdivorced":49492,"(lang":49493,"fst":49494,"ĠTyr":49495,"Ġmessenger":49496,"ifstream":49497,"XS":49498,"ĠBanking":49499,"Ġinfectious":49500,"ĠMons":49501,"_LOOP":49502,"Ġzurück":49503,"Ġobtener":49504,"/repos":49505,"Vel":49506,"acro":49507,"ĠuserRepository":49508,"styleType":49509,"ĠSRC":49510,"VMLINUX":49511,"recursive":49512,"/bar":49513,"_chip":49514,"ominated":49515,"ĠNit":49516,"âĢĶto":49517,"ĠBuddh":49518,"омеÑĢ":49519,"ĠMAG":49520,"ĠCHE":49521,"_den":49522,".raises":49523,"_degree":49524,"Ġpumpkin":49525,"_templates":49526,"_MEDIA":49527,"ĠTimeline":49528,"Ġbots":49529,"ObjectType":49530,"Ġbuys":49531,".posts":49532,"CAL":49533,"waiting":49534,"ĠDaniels":49535,"Ġdabei":49536,"ĠSigma":49537,"ilor":49538,"igel":49539,",W":49540,"ADS":49541,"(panel":49542,"ì²´":49543,"itating":49544,".palette":49545,"Ġmosquito":49546,"Ġtego":49547,"(parseInt":49548,"Ġdespués":49549,"promise":49550,"Ġwij":49551,"typescript":49552,"ĠTv":49553,"_IDENTIFIER":49554,").ĊĊĊ":49555,"_flat":49556,"itsu":49557,"USR":49558,"experience":49559,"-fit":49560,"phinx":49561,"_thresh":49562,"Ġideally":49563,"ĠFreeman":49564,",DB":49565,"_rw":49566,"çŃī":49567,"Ub":49568,"_statistics":49569,"=\"\"><":49570,"Ġchore":49571,"Ġyork":49572,"installed":49573,"Additionally":49574,"Ġpstmt":49575,"ylko":49576,"::Ċ":49577,"Forest":49578,"Ġheadset":49579,"Ġgallon":49580,"ÑĢем":49581,"Ġwithdrawn":49582,"ĠCandidate":49583,"Ġmelting":49584,"Ġfreezer":49585,"Ġhl":49586,"_HELP":49587,"mime":49588,"(/*":49589,"Ġthirst":49590,"$return":49591,"memberof":49592,"еб":49593,"ĠHttpServletRequest":49594,"(ob":49595,"_Result":49596,"Ġasserted":49597,"Ġfulfilling":49598,"Ġstretches":49599,"parated":49600,"-funded":49601,"ĠåĽ":49602,"ingles":49603,"_ca":49604,".condition":49605,"ĠDisplays":49606,"Ġorang":49607,"ĠCRE":49608,"ĠglBind":49609,"ĠSelector":49610,"/type":49611,"ĠAlexa":49612,"chedules":49613,"ĠPeninsula":49614,"Ġparity":49615,"ĉdest":49616,"ĠDoors":49617,"čĊĉčĊ":49618,"_dimension":49619,"Ġaload":49620,".StoredProcedure":49621,"(paren":49622,"ĠBurke":49623,"')]Ċ":49624,"-engine":49625,"Ġquir":49626,"ĠHybrid":49627,"ĠDoe":49628,"Ġoutlines":49629,"ĠTrends":49630,"_NV":49631,"periments":49632,"ĠHin":49633,"?',":49634,"ĉText":49635,"FUL":49636,"Ġsmells":49637,"Ġslick":49638,"Ġmiserable":49639,"ĠArrayAdapter":49640,"ĠparamString":49641,"Hom":49642,"_literals":49643,"usuarios":49644,"Ġprompting":49645,"_lazy":49646,"ĠActivation":49647,"_oc":49648,"Weak":49649,"Ġanecd":49650,"ĠUCLA":49651,"=re":49652,"issement":49653,"ĠEscorts":49654,"Excellent":49655,"ĠPause":49656,"Ġrepositories":49657,"TOR":49658,"ariate":49659,"_iso":49660,"updates":49661,"halb":49662,"udiante":49663,"ë¡Ŀ":49664,"Ġnaive":49665,"ĠPeg":49666,"ĠLounge":49667,"ARGIN":49668,"(bin":49669,"OnClickListener":49670,"ĠFAILED":49671,"Ġlite":49672,"Ġdzie":49673,"ĠLiteral":49674,"ivor":49675,"fcntl":49676,"Ġeats":49677,"Ġqed":49678,"Unlock":49679,"riding":49680,"undai":49681,"=M":49682,"ATTER":49683,"ConfigureAwait":49684,"icias":49685,"ustomed":49686,"Ġsuccession":49687,"endTime":49688,"ĠJupiter":49689,"Ġjudging":49690,"dration":49691,"_docs":49692,".mo":49693,"Ġeducators":49694,"ĠVine":49695,"Cond":49696,"[out":49697,"qb":49698,"\\Validator":49699,"Ġmeanings":49700,"Ġpresently":49701,"Ġdividing":49702,"ottenham":49703,"ascular":49704,"Ġtrailers":49705,"ĠCLOSE":49706,"ами":49707,"âĢĻai":49708,"ĠGain":49709,"wor":49710,"Ġplanner":49711,"Ġdistributing":49712,"vat":49713,"months":49714,"xlabel":49715,"HF":49716,"Viol":49717,".BASELINE":49718,"еÑĤÑģÑı":49719,"ĠRotate":49720,"Ġtxn":49721,":bold":49722,"Ġbloss":49723,"Forgery":49724,"(embed":49725,"Ġjako":49726,"sprintf":49727,"their":49728,"Ġexhibits":49729,"-static":49730,"hecy":49731,"getActiveSheet":49732,".clients":49733,"ãģį":49734,"_hide":49735,"[word":49736,"Cb":49737,"addItem":49738,"axe":49739,"_radio":49740,"alion":49741,"modifier":49742,"Ġsaturation":49743,"Ġdenom":49744,"_pixels":49745,"mess":49746,"(fl":49747,"atif":49748,"Ġsecs":49749,"Ġprostitution":49750,"Ġgrandchildren":49751,"Ġparadise":49752,"ĠFeld":49753,"_BINARY":49754,"itous":49755,"à¹Ħ":49756,"Ġflashing":49757,"-sided":49758,"Ġcontradiction":49759,"/*ĊĊ":49760,"ylabel":49761,"ĠTet":49762,"Ġadmire":49763,"reso":49764,"Ġletz":49765,"ĠSEARCH":49766,"slots":49767,"ĠRewards":49768,"ĠHog":49769,"ĠNSData":49770,"stash":49771,"Fall":49772,"ĠAmer":49773,"LinearLayout":49774,"/photos":49775,"Ġfeather":49776,"Ġ|čĊ":49777,"Downloads":49778,".StartsWith":49779,"Ġ//#":49780,"ineTransform":49781,"Ġaffid":49782,"Vtbl":49783,"ĠRogue":49784,"scribed":49785,"Ġfauc":49786,"ĠMonroe":49787,"Ġdeclares":49788,"modern":49789,"reon":49790,"aybe":49791,"PASS":49792,"fers":49793,"_MULTI":49794,"ĠMathematics":49795,"Ġsudah":49796,"_ATTACH":49797,"ĠnumberWith":49798,"ĠSolomon":49799,"jin":49800,"ografia":49801,"öl":49802,"_design":49803,"culated":49804,"ĠLuna":49805,"iesz":49806,"Ġ=>'":49807,"Ġrevelations":49808,"Along":49809,"(ed":49810,"ĠFilename":49811,"Ġylabel":49812,"Secure":49813,"Ġbusca":49814,"agnosis":49815,"_RECE":49816,"Ġoverlapping":49817,"Extent":49818,"Ġanticipation":49819,"Checks":49820,"ĠALSO":49821,"orc":49822,"ilingual":49823,"itational":49824,"Ġadvancement":49825,"ouro":49826,"ĠPredicate":49827,"å¾Ĺ":49828,"eria":49829,"ĠPierce":49830,"orio":49831,"Ġmerits":49832,"Ġpeanut":49833,".Package":49834,"ĠConduct":49835,"_SENSOR":49836,"Ġboiling":49837,"Ġintra":49838,"ĠIGN":49839,"ĠFur":49840,".Refresh":49841,"ĠReach":49842,"_decoder":49843,".Exp":49844,"ĠÑĤак":49845,"pill":49846,",Q":49847,"ĠGrill":49848,"Ġpopping":49849,".Ag":49850,"Ġproyecto":49851,"Ġmileage":49852,"Ġecological":49853,"]]);Ċ":49854,"ĠÂŃ":49855,"subplot":49856,"acad":49857,"ĠTrying":49858,"recipes":49859,"$criteria":49860,"ĠPersian":49861,"-bound":49862,"MASK":49863,"ĠGesture":49864,"Ġkk":49865,"ĠPVC":49866,"Ġprohibition":49867,"Ġcomando":49868,"ĠLOOK":49869,"Shopping":49870,"Ġdistortion":49871,"čĊ":49917,".Dependency":49918,".QueryString":49919,".Owner":49920,"Ġexpiry":49921,"Thu":49922,"(Vec":49923,"Ġhazardous":49924,"Ġrpm":49925,"APON":49926,"ĠaddTarget":49927,"sville":49928,"pNet":49929,"ĠImg":49930,"ĠTIMER":49931,".Animation":49932,"Ġbek":49933,"Ġassort":49934,"Ġlebih":49935,"ĠbodyParser":49936,"Ġvibrating":49937,"IDL":49938,"Ġbutterknife":49939,"inters":49940,"Ġpersuade":49941,"ĠLGBTQ":49942,"èĭ":49943,".soft":49944,"Ġbeams":49945,"_sur":49946,".Def":49947,"Ġlabs":49948,"ĉplt":49949,"Ġskins":49950,"Ġtransferring":49951,"Ġimaginary":49952,"_End":49953,";background":49954,"Ġlaps":49955,"_COMMENT":49956,"(SDL":49957,"onds":49958,".Record":49959,"ĠImplements":49960,"_ticks":49961,"()))ĊĊ":49962,"Ġarose":49963,"]?":49964,"ĠMp":49965,"ĠICommand":49966,"Ġsculpture":49967,"Ġcontracted":49968,"\">'":50446,"kinson":50447,"Ġкол":50448,"ognitive":50449,"_li":50450,"Ġimminent":50451,"Ġaffinity":50452,".signal":50453,"Ġnotch":50454,"ĠSteelers":50455,"maxlength":50456,"KK":50457,"ĠEugene":50458,"_PWM":50459,"roi":50460,"ĠâĹı":50461,"ĠHamburg":50462,".Must":50463,"Ġaxe":50464,"enef":50465,"Ġambitions":50466,"ĠSpecies":50467,"ĠStress":50468,"Ġawhile":50469,"ĠбÑĥд":50470,"Ġwithstand":50471,"ĠDecoder":50472,"_inventory":50473,"Ġ{ččĊ":50474,"Ġtgt":50475,"Ġrailroad":50476,"WASHINGTON":50477,"Ġnegotiated":50478,"NST":50479,"-phone":50480,",U":50481,"Ġexercising":50482,"ụ":50483,"_PIXEL":50484,"avors":50485,"iterated":50486,"Ġvampire":50487,"adal":50488,"Ingrese":50489,"Ġung":50490,"jective":50491,".cells":50492,"Ġnano":50493,"Ġmarkdown":50494,"_RULE":50495,"(events":50496,"Ġluggage":50497,"MESSAGE":50498,"igkeit":50499,"$count":50500,"AttributeName":50501,"IGINAL":50502,"_Ent":50503,"ĠBF":50504,"ĠCOMMENT":50505,"_ini":50506,"ĠEuropeans":50507,"ĠBelle":50508,"åij½":50509,")['":50510,"åºĶ":50511,"ĠUseful":50512,".reference":50513,"()\",":50514,"_grade":50515,"ĠKaw":50516,"Ġsentencing":50517,"Ġsocialism":50518,"monster":50519,"_LAYER":50520,"Ġdeepest":50521,"wk":50522,"ĠNoise":50523,"###ĊĊ":50524,"Ġpréc":50525,"otle":50526,"ÑĤе":50527,"auf":50528,"ibal":50529,"Ġconquer":50530,">Email":50531,"Ġambulance":50532,"OAD":50533,"Ġ(\"%":50534,"ĠFI":50535,".fixture":50536,"Ġterse":50537,"ĠĠĠĠĉĉĉĉ":50538,"Ġsanctuary":50539,"ugi":50540,"ĠComparator":50541,"Definitions":50542,"Ġasthma":50543,"Ġlact":50544,"Ġhardwood":50545,".clock":50546,"Ġattracting":50547,"ĠMour":50548,"(distance":50549,"icits":50550,"Ġbonne":50551,"ĠACCESS":50552,".DeserializeObject":50553,"ĠTyped":50554,"Ġjeu":50555,"ĠappId":50556,"ĠClara":50557,"ĠHF":50558,"ĠReich":50559,"ipples":50560,"//--------------------------------------------------------------------------------":50561,"_delivery":50562,"erialization":50563,"Ġplaintiffs":50564,"Scient":50565,"shopping":50566,"ĠDummy":50567,"ĠWald":50568,"GroupName":50569,"Ġinscription":50570,"elog":50571,"::::::::":50572,"_ld":50573,"BackPressed":50574,".Raw":50575,"ĠOnTrigger":50576,"Ġmuseums":50577,"ĠBeen":50578,"ĠAdventures":50579,"Ġslate":50580,"Ġlett":50581,"Ġsund":50582,"ĠGin":50583,"ĠMechanical":50584,".ship":50585,"AppComponent":50586,"Ġdestined":50587,"Ġdwelling":50588,"Profiler":50589,"Prepare":50590,"zeich":50591,"Ġsilicon":50592,"(has":50593,"Ġ#%":50594,"VIDEO":50595,"Ġcollaborate":50596,"Lin":50597,"Ġscopes":50598,"(className":50599,"(sd":50600,"andin":50601,".ham":50602,"ServiceImpl":50603,"-described":50604,"Ġirony":50605,"stial":50606,"ĠHuawei":50607,"(repo":50608,"Ġunexpectedly":50609,"ĠKai":50610,".install":50611,"\\xf":50612,"Ġexhibited":50613,"_TCP":50614,"ĠOx":50615,"_CHO":50616,"Ġprostituerte":50617,"Ġvä":50618,"Ġsito":50619,"Ġconstituents":50620,"ĠContinued":50621,"ĠSAVE":50622,"rss":50623,"/message":50624,"ubes":50625,"Ġmisdemean":50626,"Ġtaxation":50627,"Ġstoryline":50628,"hair":50629,"ĠFinds":50630,"SIG":50631,"verification":50632,"~=":50633,".hp":50634,"Iterable":50635,"Ñĭе":50636,"atori":50637,"Ġctr":50638,"Rx":50639,"_);ĊĊ":50640,"dag":50641,".pin":50642,"Ġpseud":50643,"Ġinvo":50644,"ÑģÑĤÑĢ":50645,"_pix":50646,"为空":50647,"Ġsworn":50648,"âĢĶor":50649,"_registry":50650,"Ġdisasters":50651,"ĠROI":50652,"ĠâĢķ":50653,"aktu":50654,"forest":50655,"beiten":50656,"âĢĶI":50657,"ueva":50658,"egt":50659,"Ġspikes":50660,"URES":50661,"ĠRecommended":50662,"Ġexploited":50663,"ĠFrederick":50664,"_COMPLETE":50665,"ĠDrugs":50666,"!!!!!!!!":50667,"ĠRiv":50668,"STOP":50669,"ROOM":50670,"ĠPASSWORD":50671,"Cookies":50672,".El":50673,"á»Ń":50674,"ĠBert":50675,"Ġhashed":50676,"icester":50677,"Ġdecorator":50678,"ĠqueryString":50679,":;Ċ":50680,"Ġ\"[\"":50681,"otope":50682,"-Americ":50683,"ĠMatthews":50684,"URAL":50685,"âĢľ,":50686,"Summer":50687,"fos":50688,"_CONTAINER":50689,"_ACK":50690,"Ġfiltr":50691,"_disp":50692,"_Re":50693,"Ġfacile":50694,"аÑĪ":50695,"ĠìķĬ":50696,"Ġeben":50697,"Ġsprink":50698,"ĠQuint":50699,">V":50700,"Ġhistorians":50701,"ourmet":50702,"ĠMonitoring":50703,"ledger":50704,"cott":50705,"Ġware":50706,"GGLE":50707,"cars":50708,"ĠMEDIATEK":50709,"Ġvolupt":50710,"_View":50711,"HEL":50712,"(copy":50713,"(stats":50714,"Ġchromosome":50715,"ĠCurtis":50716,"-conf":50717,"(asset":50718,"Ġhvor":50719,"FileSystem":50720,"<>();čĊ":50721,"ocoder":50722,"ĠCannon":50723,")x":50724,"ĠSmooth":50725,"ĠSAS":50726,"_ce":50727,"ĉprev":50728,"_movie":50729,"Ec":50730,"_wall":50731,".ĊĊ":51278,"ogenesis":51279,"ĠOPTIONS":51280,"uptools":51281,"Ġmilitant":51282,"Ġexited":51283,"igar":51284,"ĠCOMM":51285,"ĠDisposable":51286,"aycast":51287,"Ġrowspan":51288,"Ġsynthes":51289,"Ġsondern":51290,"ĠĊ":54769,"ĠJacket":54770,"RATION":54771,".getSelectedItem":54772,"-init":54773,"ĠRegisters":54774,"_sep":54775,"ĠToolkit":54776,".dict":54777,"Ġxlabel":54778,"\\Table":54779,"toc":54780,"_combo":54781,"ĠCompact":54782,"Ġrugged":54783,"à¥ĩà¤":54784,"-management":54785,"')}}\">Ċ":54786,"ĠStamp":54787,"ıl":54788,"rox":54789,"Ġlandscapes":54790,"_NOTE":54791,"monary":54792,"cab":54793,"Ġmoet":54794,"xaf":54795,"rcode":54796,"-cli":54797,"_gate":54798,"[event":54799,"SPORT":54800,"gia":54801,"ĠSUPER":54802,"/Login":54803,"_shutdown":54804,"interrupt":54805,"Ġpretending":54806,"Ġfringe":54807,"ĠReds":54808,"ĠCUDA":54809,"ĠUNIX":54810,"vit":54811,"Ġbrig":54812,"drv":54813,"ĠConnector":54814,"Therefore":54815,"Ġlia":54816,"Detection":54817,"_actor":54818,"Ġtempfile":54819,"Ġeccentric":54820,"-role":54821,"Ġpadx":54822,"dent":54823,"Western":54824,"Ġê·¸":54825,"ĠApplicationRecord":54826,"Ġcampaigning":54827,"_runner":54828,"ĠCivic":54829,"aleigh":54830,"Ġdirekt":54831,".sul":54832,"ĠĠĉĉĉ":54833,"anten":54834,"Ġissuer":54835,"Ġassertions":54836,"(orig":54837,"ATIO":54838,"Ġleaned":54839,"äs":54840,".DTO":54841,"explode":54842,".Observable":54843,"Ġstaggering":54844,"Ġkidnapped":54845,"Ġprogrammers":54846,"ĠInnov":54847,".parameter":54848,"Ġdomination":54849,"Ġskeptic":54850,"Ġæĺ¯":54851,"Ġavoids":54852,".Verify":54853,"ubby":54854,"ĠASN":54855,"Ġformato":54856,"ĠBeatles":54857,"_brand":54858,"Ġinset":54859,"youtu":54860,"Ġtoc":54861,"-final":54862,"Showing":54863,"ĠDoub":54864,"ĠMesa":54865,"Adj":54866,"_medium":54867,"Creates":54868,"(endpoint":54869,"ĉUP":54870,"bbie":54871,"Ġstalk":54872,".databind":54873,".Scan":54874,"agents":54875,"$,":54876,"individual":54877,"+)/":54878,"ĉvm":54879,"(notification":54880,"Ġinex":54881,"ĠClassification":54882,"reno":54883,"Ġolig":54884,"-rated":54885,"Ġformulation":54886,"',{":54887,"Ġacept":54888,"_unpack":54889,"_CA":54890,".Pow":54891,"ĉim":54892,"Ġaluminium":54893,"ANO":54894,"Ġxn":54895,"Ġcómo":54896,"ĠIngredient":54897,"Ġseizures":54898,"åħ±":54899,"ificador":54900,"Ġsiguiente":54901,"ĠInfragistics":54902,"Ġduplicated":54903,"ĠDee":54904,"Ġnø":54905,"ĠACCEPT":54906,"(crate":54907,"иÑĤелÑĮ":54908,"-less":54909,"Ġinfinity":54910,"Analyzer":54911,"-Day":54912,"ritt":54913,"(cin":54914,"ĠGy":54915,"Ġmultiplied":54916,"uchi":54917,"ĠBaldwin":54918,"/ip":54919,"Ġshortcuts":54920,".ADD":54921,"Ġvigor":54922,"_instruction":54923,"(;":54924,"_eta":54925,"è¿ŀ":54926,"utorials":54927,"Ġboosting":54928,"bv":54929,"Ġacknowledges":54930,"Listening":54931,"FAQ":54932,";b":54933,"((-":54934,"Ġarchitects":54935,"Ġzwe":54936,"Ġpuls":54937,"ĠgetCount":54938,"verbs":54939,"ãĢľ":54940,"(Collection":54941,"kre":54942,"Ġjurisdictions":54943,"_bridge":54944,"ĠCrack":54945,"ĠDifficulty":54946,"KO":54947,"Reservation":54948,"_requires":54949,"Tour":54950,"ãģĹãģŁ":54951,".setCurrent":54952,"Ġky":54953,"ĠAlbany":54954,"Ġè§":54955,"ller":54956,"agna":54957,"workers":54958,".blank":54959,"ĠPrayer":54960,"MIC":54961,"Ġresilience":54962,"TeX":54963,"ĠLanguages":54964,"study":54965,"ĉcurr":54966,"Ġenzymes":54967,"Slug":54968,"ĠíĮĮ":54969,"stral":54970,"Ġtumors":54971,"Ġsegunda":54972,"='{":54973,"instruction":54974,"ĠLisp":54975,"/info":54976,"Ġ\"{$":54977,",:),":54978,"Ġgv":54979,"(ErrorMessage":54980,"Ġ'=":54981,"}-${":54982,".Documents":54983,"\"Well":54984,"Ġreminiscent":54985,"Ġgaz":54986,"iropr":54987,"ehr":54988,"Ġsuppressed":54989,"ersh":54990,".scrollTo":54991,"Ġcadena":54992,"ĠgameState":54993,"ÃŃm":54994,"(conv":54995,"ĠTomorrow":54996,"ĠCCT":54997,"Mongo":54998,"ulg":54999,".Camera":55000,".handlers":55001,"mph":55002,"Ġstk":55003,"Ġgenetics":55004,"ACING":55005,"Trivia":55006,"ĠBam":55007,"(marker":55008,".Stretch":55009,"ĠSunni":55010,"ĠBetty":55011,".tolist":55012,"unlikely":55013,".Rectangle":55014,"obsolete":55015,"ILON":55016,"innerText":55017,"embourg":55018,"aN":55019,"ĠVehicles":55020,"unlock":55021,":utf":55022,"nob":55023,"ĠSeeing":55024,"ĠNEVER":55025,"Ġtls":55026,"Ġfilles":55027,"Ġbenefited":55028,"ĠClint":55029,"*/),":55030,".fold":55031,"Ġposible":55032,"ADED":55033,"thouse":55034,".DAL":55035,"ĠOdd":55036,"rokes":55037,"ĠSunny":55038,"ĠPartialEq":55039,"_Buffer":55040,"ĠLevi":55041,"longrightarrow":55042,"eldon":55043,"gages":55044,"_warn":55045,".CreateTable":55046,"ĠDip":55047,"_questions":55048,".logic":55049,"Ġ#\"":55050,"={()=>":55051,"Ġtep":55052,"Ġjuicy":55053,"ìĤ¬":55054,"enko":55055,"ialect":55056,"Ùī":55057,"Ġonboard":55058,"Ġæı":55059,"ĉrt":55060,"_UTF":55061,"ĠQAction":55062,"âĢŀ":55063,"(Component":55064,"(audio":55065,".hit":55066,"gte":55067,"Ġprogrammed":55068,"stateParams":55069,"Ġpolyester":55070,"fires":55071,"byss":55072,"]=(":55073,"_quality":55074,"OfDay":55075,"ĠFairy":55076,"Ġyelled":55077,"opl":55078,"(userName":55079,"ĠDifference":55080,"Ġevaluations":55081,"iffany":55082,"Ġcyclists":55083,"Ġcidade":55084,"Ġtextbook":55085,"Ġprofiling":55086,"__),":55087,"dea":55088,".activate":55089,"Ġindications":55090,"Ðķ":55091,"TouchUpInside":55092,"Ġinvaluable":55093,"ĠMASK":55094,"Ġcontend":55095,"Freq":55096,"Ġrecruits":55097,"(interval":55098,"ĠUserProfile":55099,"Ġ'./../":55100,"edu":55101,"_Callback":55102,"Ġanalogy":55103,"ĠTrophy":55104,"apphire":55105,"Videos":55106,"ĠCher":55107,"ĠHav":55108,"â̦\"":55109,".validator":55110,"gfx":55111,"ĠUObject":55112,"classnames":55113,"triangle":55114,"ĠEncoder":55115,".spy":55116,"Ġpredators":55117,"=status":55118,"-safe":55119,":\",Ċ":55120,"ĠIncluding":55121,"Ġ{};čĊ":55122,"*cos":55123,"Ġendured":55124,".sulake":55125,"Ġnursery":55126,"Ġfragrance":55127,"Ġrebuilding":55128,"Ġnth":55129,"ĠFraser":55130,".setDate":55131,"ĠVince":55132,"_REST":55133,"Ġventilation":55134,"æµ·":55135,"cribes":55136,".asm":55137,"lpVtbl":55138,"ĠAbe":55139,"uisine":55140,",array":55141,"ĉclassName":55142,"errals":55143,"Ġ'ĊĊ":55144,"Checkout":55145,"Ġsolicit":55146,"Aux":55147,"_capture":55148,"Ġribs":55149,"ragon":55150,"viol":55151,"topics":55152,"FunctionFlags":55153,"ĠMarty":55154,"bike":55155,"ĠTucker":55156,"(kernel":55157,"ĠOps":55158,"CloseOperation":55159,"/demo":55160,"ilda":55161,"ĠlÃŃnea":55162,"APPING":55163,"Ġsuites":55164,".visitVarInsn":55165,"urus":55166,"ĠMinute":55167,"(manager":55168,"Ġbutterfly":55169,"Ġapare":55170,"Ġwolves":55171,"JWT":55172,"ĠSalon":55173,"ĉdelay":55174,"-eslint":55175,"isations":55176,".rpc":55177,")|(":55178,"ĠSnapchat":55179,"/mm":55180,"MN":55181,"ceries":55182,".textAlignment":55183,"ĠFrankfurt":55184,"Ġado":55185,"(newValue":55186,"(access":55187,"(Expression":55188,"ĠSignIn":55189,"ĠHaiti":55190,"_tp":55191,".setParameter":55192,"Minute":55193,"Ġmanuals":55194,"ricanes":55195,"ĠPTR":55196,"ĠOuter":55197,"Ġgetline":55198,"ocations":55199,"_CD":55200,"ĠLyon":55201,"/gui":55202,"_live":55203,"idan":55204,".geom":55205,"ĠborderBottom":55206,"imuth":55207,"_checkpoint":55208,"Ġmeu":55209,"ĠIrving":55210,"Ġpeuvent":55211,"(MAX":55212,"ĠARCH":55213,"Ġpov":55214,".sourceforge":55215,"Ġjamais":55216,"Ġark":55217,"ĠBaghdad":55218,"ĠCLEAR":55219,"MenuBar":55220,"Ġtrois":55221,"CHEDULE":55222,"Ġ#čĊ":55223,"(Call":55224,"$order":55225,"(Material":55226,"Ġencontrado":55227,"$list":55228,"ĠMETHODS":55229,".beginTransaction":55230,"_MAG":55231,"StyleSheet":55232,"Ġmajors":55233,"Ġindefinitely":55234,"cleanup":55235,"Ġhomeland":55236,"(dto":55237,"Dates":55238,"Presentation":55239,"ĠDK":55240,"={`/":55241,"ĉKey":55242,"(Block":55243,"_checkbox":55244,"needs":55245,"ĠonComplete":55246,"rico":55247,"Ġgleich":55248,"Ġxm":55249,"OOD":55250,"Better":55251,"ĠSQLITE":55252,".Book":55253,"xad":55254,"ĠGone":55255,"ĉdp":55256,"Ġdevotion":55257,"Ġstm":55258,"Ġobsess":55259,"ĠBackend":55260,"Queries":55261,"Ik":55262,"//****************************************************************":55263,"Ġdividends":55264,".parentElement":55265,"}\")ĊĊ":55266,"ĠMaterialPageRoute":55267,":num":55268,"Ġexplic":55269,"ĠOL":55270,"least":55271,"Oops":55272,"imentos":55273,"Ġinsurers":55274,"Ġheroic":55275,"ĉfields":55276,".imgur":55277,".btnCancel":55278,"ĠDetective":55279,"(sm":55280,"ĠMutableLiveData":55281,".lab":55282,"(([":55283,"Ġhairst":55284,"ĠTransactions":55285,"å¼Ģå§ĭ":55286,"ĠstdClass":55287,"uento":55288,"GIS":55289,"_cod":55290,"Instructions":55291,"Calls":55292,"PointerType":55293,"ĠRw":55294,"Ġassortment":55295,"ĠDIG":55296,"+r":55297,"_CERT":55298,"Ġinstability":55299,"Ġvib":55300,"onas":55301,"Ġroku":55302,"apellido":55303,"Ġangl":55304,"preneur":55305,"Ġfluids":55306,"isease":55307,"Ġdeed":55308,"quist":55309,"_CONSTANT":55310,"Ġequilibrium":55311,"_delegate":55312,"ĠQuantum":55313,"rei":55314,"Capabilities":55315,"rectangle":55316,"?><":55317,"alien":55318,"ĠJug":55319,"DNA":55320,"Tickets":55321,"Occurs":55322,"ĠHawk":55323,".setHorizontalGroup":55324,"\\Collection":55325,"ffiti":55326,"Ġrearr":55327,".setVerticalGroup":55328,"Ġcavity":55329,"Ġadulte":55330,"Facade":55331,"-wh":55332,"ĠLOL":55333,"ذ":55334,"Ġgrandparents":55335,"Swift":55336,"ĉwx":55337,"æīĢæľī":55338,"ifen":55339,"ffset":55340,"Beyond":55341,"//}ĊĊ":55342,"Ġwager":55343,"Ġbury":55344,"Ġcommence":55345,"registro":55346,"scient":55347,"ĠPercent":55348,"Ġдолж":55349,"(identifier":55350,".setModel":55351,"Ġseldom":55352,"nton":55353,"Ġappliance":55354,"amus":55355,"rysler":55356,"Ġpanties":55357,"enguins":55358,"Ġmimic":55359,"ĠonChanged":55360,"Ġalcoholic":55361,".reloadData":55362,"Charge":55363,"ĠFax":55364,"ĠjScrollPane":55365,"Empresa":55366,"Ġshattered":55367,"xba":55368,"Fonts":55369,"?s":55370,"Ġpostseason":55371,"retain":55372,"_rates":55373,"ĠrequestCode":55374,".todo":55375,"´s":55376,"CHK":55377,"ĠKeeping":55378,"engeance":55379,"Ġvscode":55380,"IPPING":55381,"DefaultCloseOperation":55382,"_raise":55383,"ĠOculus":55384,"ograms":55385,"raj":55386,"pci":55387,"Ġcorrosion":55388,".handleSubmit":55389,"Accessible":55390,"ĠPiano":55391,"little":55392,"ACL":55393,"Äĩe":55394,".unwrap":55395,"ĠConvers":55396,"ĠLeben":55397,"ioneer":55398,"ĠMerchant":55399,"ĠJorge":55400,"Ġembracing":55401,"Ġventa":55402,"ást":55403,"Ġviene":55404,"Ċ":55556,"-growing":55557,"Ġdeepcopy":55558,"Ack":55559,"eggies":55560,"Ġ__(\"":55561,"Ġnoir":55562,"terrorism":55563,"Ġanthem":55564,"agency":55565,"_PACKAGE":55566,"ĠClosure":55567,".registry":55568,"Ġmammals":55569,"L":55600,"Ġbluetooth":55601,".Deep":55602,"-standing":55603,"ácil":55604,"Ġrooft":55605,"ĠPaths":55606,"_iterations":55607,"InvalidArgumentException":55608,".spi":55609,"ĠUIAlertAction":55610,"uye":55611,"signin":55612,".priority":55613,"ĠEssays":55614,"='{$":55615,"Ġè¿ĶåĽŀ":55616,"_signed":55617,".persist":55618,"Ġredesign":55619,"ToLower":55620,"ĠNewman":55621,"=start":55622,"ĠIsraelis":55623,"asiswa":55624,"Speech":55625,"Ġnumeros":55626,"handlers":55627,"ĠWong":55628,"ĠмеÑĤод":55629,"Weights":55630,"ĠGujar":55631,"teil":55632,"ĠNonetheless":55633,"_EFFECT":55634,"Ġvect":55635,"ĠOsc":55636,"Ġcoats":55637,"ĠWheat":55638,"Ġgeek":55639,"ĠPROPERTY":55640,"worm":55641,"_constants":55642,"ĠBoulder":55643,"ĠParm":55644,"cole":55645,"ĠdefaultCenter":55646,"ĠRouge":55647,":A":55648,"xcf":55649,"ĠVenice":55650,"median":55651,"Ġredemption":55652,"Fresh":55653,"Ġcosm":55654,"Ġfigur":55655,"Ġrefurb":55656,"COPE":55657,".cd":55658,"Ġchords":55659,"ĠSgt":55660,"Åį":55661,"VPN":55662,"ĠSEND":55663,"ainen":55664,"_accounts":55665,"Ġtenth":55666,"Ġdissolved":55667,"":55907,"Ġlegitimacy":55908,"Ġoo":55909,"Slinky":55910,"Ġnationals":55911,".words":55912,";p":55913,"trap":55914,"omanip":55915,"Ġcues":55916,"Ġgraduating":55917,"Ġsemaphore":55918,"\"]);ĊĊ":55919,"acey":55920,"REET":55921,"Grab":55922,"ĠFelix":55923,"(Id":55924,"_neighbors":55925,"Ġmeaningless":55926,"(del":55927,"Ġjeder":55928,"ĠContentValues":55929,".absolute":55930,"/cl":55931,"Ġxb":55932,"datum":55933,"Ġtortured":55934,"Ġrubbing":55935,"Scores":55936,"ĠðŁĺī":55937,"Ġavons":55938,"Ġamsterdam":55939,"EOS":55940,"Hal":55941,"Ġtrustworthy":55942,"#=":55943,".EXTRA":55944,"Ġmano":55945,"isicing":55946,"-support":55947,"ĉcursor":55948,"ĠSpo":55949,"aimassage":55950,"Mission":55951,"[]{\"":55952,"Ġprinters":55953,"GREEN":55954,"Ġteg":55955,"Ġabdominal":55956,"!ĊĊĊĊĊĊ":55957,".Short":55958,"азв":55959,"ĠGifts":55960,"}\")":55961,"(binding":55962,"xce":55963,"âĢij":55964,"infos":55965,"FormData":55966,"Ġdart":55967,"Ġelems":55968,"(inv":55969,"YL":55970,"tin":55971,"GENER":55972,"ữ":55973,"ĠTaken":55974,"uckle":55975,":e":55976,"Ġspectral":55977,".baidu":55978,"/');Ċ":55979,"Ġgreedy":55980,"esion":55981,",,,,,,,,":55982,"Ġ/>,Ċ":55983,"InternalServerError":55984,"NSNotificationCenter":55985,"ĠAi":55986,"Ġspit":55987,"Ġaugmented":55988,"ĠstandardUserDefaults":55989,"FINITY":55990,"Race":55991,":C":55992,"ĠRECORD":55993,"ĠHighlight":55994,"Ġ'`":55995,"Ġdeficits":55996,"Ġnei":55997,"Ġresearched":55998,"Ta":55999,"Ġcopp":56000,".GetHashCode":56001,"):čĊčĊ":56002,"OnClick":56003,"ĠWellington":56004,"Ġrevival":56005,"æ¯Ķ":56006,"éĹ®":56007,"ĠNSS":56008,"Ġforn":56009,"Ġinté":56010,"ĠKuwait":56011,"_flip":56012,"_bo":56013,"_\\":56014,"Ġoccurrences":56015,"ĠScientists":56016,"SRC":56017,"ogens":56018,"igrant":56019,"REMOTE":56020,"ĠSID":56021,".opts":56022,"uve":56023,"()])Ċ":56024,"Ġlibertarian":56025,"ĠGlide":56026,"lesen":56027,"Ġforme":56028,"owania":56029,"Ġannoyed":56030,"Defs":56031,"ĠExecutor":56032,"Ġcasts":56033,".setChecked":56034,"ĠSharing":56035,".SerializeObject":56036,"Ġselectors":56037,"_OTHER":56038,"미":56039,"(super":56040,"(OS":56041,"_VERIFY":56042,"idunt":56043,"';Ċ":56045,"Ġvidéo":56046,"ĠNegro":56047,"ĠLords":56048,"ĠTours":56049,"Ġsoftly":56050,".receive":56051,"ĠERC":56052,"ĠdataSet":56053,"Badge":56054,"ĉEvent":56055,"Ġperl":56056,"Ġ{}\\":56057,"(sentence":56058,"OrUpdate":56059,"Ġdiminish":56060,"PIN":56061,"(draw":56062,".ToDateTime":56063,".EqualTo":56064,"(pin":56065,"-pencil":56066,"luent":56067,"ĠCaller":56068,"Ġplayful":56069,"-'+":56070,"xca":56071,"swick":56072,"){}Ċ":56073,"}:${":56074,"ĠMeth":56075,".getCell":56076,".break":56077,"Ġymax":56078,"='Ċ":56291,"ĠHiro":56292,"(TRUE":56293,"asurer":56294,"Ġcuer":56295,"Uber":56296,".Operation":56297,"Ġolan":56298,"Ġthrilling":56299,"'.":56321,"ĉvalid":56322,"\"\",":56323,"Instrument":56324,">J":56325,"Ġnostr":56326,"ĠRift":56327,"_Port":56328,"Ġveces":56329,"[['":56330,"Ġrallies":56331,"-series":56332,"Ġvv":56333,".uc":56334,"Ġrtn":56335,"StateChanged":56336,"(ins":56337,"ĠCla":56338,"------------Ċ":56339,"cus":56340,"ĠReload":56341,"//------------------------------------------------------------------------------------------------":56342,".seconds":56343,"_destination":56344,"Ġscrewed":56345,">c":56346,"Thickness":56347,"Designer":56348,"Ġgrids":56349,"nÄħ":56350,"(cookie":56351,"Trip":56352,"-Mobile":56353,"Ġvoll":56354,"Ġgenital":56355,"Ġconfisc":56356,"ĠConfederate":56357,"ĠwebView":56358,"Ġmise":56359,"Ġcler":56360,"(selection":56361,"$date":56362,"Ġsharpen":56363,"ragen":56364,"AndUpdate":56365,"Ġremix":56366,"Ġhtons":56367,"RW":56368,"MPI":56369,"Ġretrieval":56370,"Ġrichest":56371,".Decode":56372,":initComponents":56373,"ĠTValue":56374,"Saint":56375,"@include":56376,"ĠPERSON":56377,".sep":56378,"ĠLDAP":56379,"gba":56380,"ĠgroÃŁe":56381,"Ġreliably":56382,"ĠDFS":56383,".getItemId":56384,"Ġprésent":56385,".getToken":56386,"Ġchinese":56387,"ĠMeal":56388,"YOU":56389,"\">>ĊĊ":56948,"bower":56949,"Ġswapped":56950,"/install":56951,"Ġsinks":56952,"etrize":56953,"Ġdeclines":56954,"ĉmysql":56955,"ĠCString":56956,"ĠMotionEvent":56957,".Language":56958,"Road":56959,"ÑĤеÑĢ":56960,"ascimento":56961,"'))->":56962,".about":56963,"(editor":56964,"ĠRatings":56965,"income":56966,"Å¡e":56967,".dequeueReusableCell":56968,"ĠAustrian":56969,"Ġsulla":56970,"ĠTribunal":56971,"ĠDidn":56972,"оваÑĢ":56973,"Ġinspections":56974,"Boss":56975,"Ġcocktails":56976,"Ġapologized":56977,"_subplot":56978,"opal":56979,"+=(":56980,"Ġresonance":56981,"ibu":56982,"Ġ리":56983,"roma":56984,"reserve":56985,"pls":56986,"ĠTah":56987,"axies":56988,"OPLE":56989,"ĠDarren":56990,"ĠZombie":56991,"_Map":56992,"Ġ])ĊĊ":56993,"ĠQi":56994,"ĠSail":56995,"Ġrestrictive":56996,"Ġerosion":56997,"-par":56998,"WHITE":56999,"Ġoldu":57000,"Ġaperture":57001,"Ġbitcoins":57002,"texto":57003,"ĠComcast":57004,"Ġtimeless":57005,"enkins":57006,"Ġfeeder":57007,"/tmp":57008,"resden":57009,"+'_":57010,".Destroy":57011,"Ġçok":57012,"ĠDOCUMENT":57013,".lng":57014,".tagName":57015,"Ġkullan":57016,"egrate":57017,"Ġ(*.":57018,"ç¼ĸè¾ij":57019,"Ġhandshake":57020,"soc":57021,"_geometry":57022,"ĠDamascus":57023,"Minor":57024,"ĠKafka":57025,"ìŬ":57026,"Florida":57027,"_compute":57028,".expr":57029,"Ġparalle":57030,"ĠDiaz":57031,"cir":57032,"[target":57033,"Ġjoking":57034,"Ġglor":57035,"(setq":57036,"_handlers":57037,"Hang":57038,"Ġferr":57039,"riminal":57040,"ĉĠĠĠĠĉĉ":57041,"enties":57042,"defines":57043,"-tax":57044,"jsonp":57045,"ĠUPS":57046,"metro":57047,"__;Ċ":57048,"ĠUganda":57049,"])):Ċ":57050,"_td":57051,"xae":57052,"lw":57053,".OS":57054,"ĠLogged":57055,"acid":57056,"ĠMayo":57057,"aspect":57058,"Ġvaginal":57059,"Ġinitializing":57060,"Ġsteroids":57061,"fiction":57062,"GRE":57063,"gend":57064,"Ġliabilities":57065,"ĠLets":57066,"Mech":57067,"(nc":57068,"(change":57069,"Ġconnectors":57070,":k":57071,"Ġtast":57072,"!\");ĊĊ":57073,"things":57074,"rophy":57075,"luetooth":57076,"ĠSignUp":57077,".ctrl":57078,"Ġtherein":57079,"orda":57080,".escape":57081,"igator":57082,"Ġpetrol":57083,"Ġspecimen":57084,"Ġdebuted":57085,"-Pro":57086,"Ġcrises":57087,".addView":57088,"ëıĻ":57089,"-door":57090,"Ġmonet":57091,"Ġmillis":57092,"Ġvier":57093,"InternalEnumerator":57094,"Ġadmins":57095,"ĠLair":57096,"zin":57097,"getQuery":57098,"umbles":57099,"LIMIT":57100,"ĠVig":57101,"_song":57102,"":57415,"Ġpasado":57416,"thank":57417,"_Delete":57418,"ĠBrighton":57419,",unsigned":57420,"ä½ľèĢħ":57421,"Ġaspirations":57422,"-how":57423,"Rose":57424,"=((":57425,"_needed":57426,"_plural":57427,">ĊĊ":57545,"Ġsurfaced":57546,"ĠìłĢìŀ¥":57547,"platz":57548,"ĉemail":57549,"ceptors":57550,"\">(":57551,"Ġepile":57552,"读":57553,"ĠDebt":57554,"åijĬ":57555,"NOP":57556,"\"https":57557,":j":57558,"FormItem":57559,"_LICENSE":57560,".getDouble":57561,"ĠAgenda":57562,"ĉfinally":57563,"(filters":57564,"(av":57565,"ç¾İ":57566,"APER":57567,"Ġlava":57568,"еÑĢж":57569,"))))ĊĊ":57570,"Ġfaulty":57571,"_nm":57572,"Ġtrava":57573,"(Bitmap":57574,"Ġspeeding":57575,">').":57576,"Ġscreened":57577,"_roll":57578,"ĠMacBook":57579,"ĠAUD":57580,"Ġdiagnose":57581,".Generate":57582,"Ġ^^":57583,"Ġstrs":57584,"[Test":57585,"Ġransom":57586,"ĠDHCP":57587,"elden":57588,"Ġinterpretations":57589,"()].":57590,"flatMap":57591,"ĠlineHeight":57592,"_mount":57593,"ĠWizards":57594,"Ġsluts":57595,"ehler":57596,"odal":57597,"Ġmilitia":57598,"å²":57599,"earned":57600,"Ġmisery":57601,"intval":57602,"fund":57603,"Ġhides":57604,"Ġdiarr":57605,"ĠWesley":57606,"Ġxmm":57607,"Ġquem":57608,"ĠArabs":57609,"ifth":57610,"ategorized":57611,"Disposable":57612,"Pure":57613,"_NOTIFY":57614,"snippet":57615,"ĠGarrett":57616,".running":57617,".weights":57618,"Ġ(--":57619,"Ġinvariant":57620,"äºĭä»¶":57621,"ĠAllowed":57622,"dirs":57623,"Ġpassions":57624,"Ġlad":57625,"ĠFlush":57626,"menus":57627,":block":57628,"Ġcompra":57629,".chomp":57630,"allocator":57631,"Ġcurated":57632,"ĠKnowing":57633,"ĠPatterson":57634,"Ġtelah":57635,"'ex":57636,"Ġdoomed":57637,"Ġphilanth":57638,"otty":57639,".styles":57640,"Owned":57641,"Ġallergies":57642,"=params":57643,"ocese":57644,"itelist":57645,"ĠSending":57646,"bef":57647,"orrar":57648,"ĠNão":57649,"ĠFargo":57650,"ĠLub":57651,"ĠCombined":57652,"_given":57653,"ĉĉĉĉĉĠĠĠĠ":57654,"Ġreconciliation":57655,"Patterns":57656,"azard":57657,"Ġbiomass":57658,"ĠHouses":57659,"respuesta":57660,"cco":57661,"/topics":57662,"ĠYuk":57663,"Ġweakened":57664,"_calendar":57665,"Ġmulheres":57666,"ĠMarl":57667,"Ġsine":57668,"ĠTil":57669,"ĠSouls":57670,"ĠDeutsche":57671,"ĠFOLLOW":57672,"Ġpipelines":57673,"ĠBeverly":57674,"_DIPSETTING":57675,"\"#":57676,"ĠProto":57677,".big":57678,"ĠSavings":57679,"ĠTanz":57680,"jun":57681,"ĠGamma":57682,"ĠSadd":57683,"Ġadvisors":57684,"Ġroast":57685,"Ġunters":57686,"udies":57687,"_lon":57688,"-pointer":57689,"ĠElementRef":57690,"\\Builder":57691,"exampleInput":57692,".webdriver":57693,"dataType":57694,"ĠQuite":57695,"ĠCeltics":57696,"uil":57697,"-defense":57698,"bish":57699,"ĠUIWindow":57700,"ĠSuddenly":57701,".hot":57702,".reason":57703,"Ġgör":57704,"AMD":57705,".Multi":57706,"authenticated":57707,"regions":57708,";(":57709,"аÑĢам":57710,"ĠKirby":57711,"$route":57712,"PRECATED":57713,"ĠDurham":57714,"owo":57715,"ĠPerforms":57716,"Ġdisregard":57717,"nst":57718,"ĠPols":57719,"ĠgetP":57720,"\"]:":57721,"-colored":57722,"(Keys":57723,"ĠAlleg":57724,"_modify":57725,"_loading":57726,"strained":57727,"Ġatroc":57728,"_phr":57729,"":58721,"ceph":58722,".DateTimePicker":58723,".\";ĊĊ":58724,"ĠTie":58725,",item":58726,"Ġmenn":58727,"Gas":58728,"ocha":58729,"_virtual":58730,"Ġmasterpiece":58731,"_sequences":58732,"LTE":58733,"ĠSubmission":58734,"Caller":58735,"$\\":58736,"Sport":58737,"agus":58738,"ConstraintMaker":58739,"Ġcoloc":58740,"Ġwig":58741,"ĠУ":58742,"ĉArray":58743,"Looks":58744,"ĠGTA":58745,".steps":58746,"atchewan":58747,"_ranges":58748,"extAlignment":58749,"ĠBrennan":58750,"Ġabstraction":58751,"ulerAngles":58752,".misc":58753,"Ġantibodies":58754,"Ġexponential":58755,"ĠCHANNEL":58756,"expense":58757,"'y":58758,"Ġdetectives":58759,"Ġpurported":58760,"YSTEM":58761,"Ġradioactive":58762,"ĠLatina":58763,".Encoding":58764,".TAG":58765,"xin":58766,"Degree":58767,"uracion":58768,"prices":58769,"ĠReferentialAction":58770,"Ġrarity":58771,"Ġpiles":58772,"gende":58773,"_projects":58774,"_globals":58775,".startTime":58776,"Ġ구":58777,"SECTION":58778,"_publish":58779,"Fault":58780,"DDL":58781,"_prior":58782,"Mom":58783,"Ġthicker":58784,"Ġsequelize":58785,"Ġessentials":58786,"stras":58787,"intr":58788,">(()":58789,".management":58790,"eil":58791,"éĹŃ":58792,"Aware":58793,".City":58794,"ĠArbit":58795,"_DM":58796,"_keyboard":58797,"LObject":58798,"-webpack":58799,"ĠNewport":58800,"ĠprincipalColumn":58801,"legant":58802,"Ġpallet":58803,"Ġfracture":58804,"Ġgmail":58805,".Meta":58806,"Above":58807,".KeyEvent":58808,"jit":58809,"_macro":58810,"_PUSH":58811,"ứ":58812,"/controller":58813,"åĬłè½½":58814,"Ġsuperficial":58815,"exterity":58816,"Ġmensagem":58817,"Wind":58818,"iston":58819,".openapi":58820,"иÑĢов":58821,"ĠSerializer":58822,"uctive":58823,"Ġzar":58824,"Places":58825,".Static":58826,"Ba":58827,"Ġinadvert":58828,"ĠIndonesian":58829,"_IPV":58830,"(horizontal":58831,"ĠgetTitle":58832,"idepress":58833,"ĠConsoleColor":58834,"ipers":58835,"$out":58836,"Ġfestive":58837,"Ġevenings":58838,".GetData":58839,"uitka":58840,"ĠManuals":58841,"ussed":58842,"_Max":58843,".Chat":58844,"ĠAircraft":58845,"=com":58846,"FOUND":58847,"apro":58848,"Ġtreasures":58849,"_alive":58850,"Ġgadget":58851,"eking":58852,"ButtonDown":58853,"Browsable":58854,".PERMISSION":58855,"PASSWORD":58856,"ĠHASH":58857,"fé":58858,"\\TestCase":58859,"LOSS":58860,"others":58861,",J":58862,"Ġasshole":58863,"werk":58864,"Ġmã":58865,".ie":58866,"evil":58867,"kontakte":58868,"////////////////////////////////////////////////////////////////////////////////Ċ":58869,"=sys":58870,"ĉlock":58871,"--;ĊĊ":58872,"_FUN":58873,"FillColor":58874,"óa":58875,"prend":58876,"Ġcompressor":58877,"Mother":58878,"ĠArcher":58879,".goto":58880,"Ġwürde":58881,"Ġbamboo":58882,"ï¼İ":58883,"ĠTrees":58884,"Ġbumper":58885,"Ġsausage":58886,"ĠElasticsearch":58887,"Ġhorizontally":58888,"ĠGul":58889,"Immutable":58890,"Ġloser":58891,"Ġaborted":58892,"-demo":58893,"ĠHatch":58894,"Ġunde":58895,"Ġprocesso":58896,"-call":58897,"Income":58898,"åĥ":58899,"_returns":58900,"'].\"'":58901,"(sw":58902,"CBS":58903,"amilies":58904,"ĠYourself":58905,"ĠHolt":58906,".MON":58907,"à§ĩ":58908,"ÑĪе":58909,"anon":58910,"ĠFontAwesome":58911,"producer":58912,"jr":58913,"Ġmau":58914,"ĉinter":58915,"Ġdishonest":58916,"Ġmagna":58917,"ĠCollective":58918,"Ġvraiment":58919,"Ġchoix":58920,"stay":58921,"Ġwelding":58922,"rising":58923,",min":58924,"ĠFate":58925,"glob":58926,"RGBA":58927,"Ġdette":58928,"Ven":58929,"Ġembarrassment":58930,".DELETE":58931,"gregar":58932,"-render":58933,"(bucket":58934,"\">ĊĊĊ":58935,".waitKey":58936,"Busy":58937,"Ġdifferentiation":58938,"ĠCST":58939,".Constant":58940,"ĠlineNumber":58941,"(matches":58942,"Ġwebsocket":58943,"Ġbarred":58944,"Ġpuedes":58945,"Mono":58946,"CORE":58947,"IID":58948,"ĠĠĠĠčĊčĊ":58949,"Ġpúblico":58950,"leaning":58951,"Ġcleansing":58952,"Ġcris":58953,"ĠDevils":58954,"_SETTING":58955,"untary":58956,".);Ċ":58957,"ĊĠĠĠĊ":58958,"[curr":58959,"tsy":58960,"ĠAlexis":58961,"ritel":58962,"Ġpetroleum":58963,".preprocessing":58964,"matter":58965,"ForResult":58966,"-license":58967,"Ġtravellers":58968,"ĠDispatcher":58969,"ennifer":58970,"Ġdigestive":58971,"PED":58972,"hibition":58973,"MASConstraintMaker":58974,"ĠWatt":58975,"Benef":58976,".setView":58977,"dto":58978,"TEE":58979,"ĠPelosi":58980,"_EXTRA":58981,"Ġmedals":58982,"xhr":58983,"forecast":58984,"Ġnargin":58985,"ouns":58986,"-fill":58987,"_CURSOR":58988,"Ġsupervised":58989,"Ġturf":58990,"ĠEdgar":58991,"POSITION":58992,"ĠcategoryId":58993,"âī":58994,"_ER":58995,"á»§a":58996,"Shown":58997,".ll":58998,"_POLICY":58999,"(),'":59000,"ĠPrev":59001,"ĠStringField":59002,"ĉGlobal":59003,"assed":59004,"Throughout":59005,"ostringstream":59006,".awtextra":59007,"Ġslopes":59008,"ĠSequential":59009,"Ġgiorn":59010,"Ġzelf":59011,"Ġversatility":59012,"leneck":59013,".cgi":59014,"Ġdoubling":59015,"ĠBangkok":59016,"Ġbuurt":59017,"Ġusuário":59018,"studio":59019,"Ġjeunes":59020,"Ġmuted":59021,"Ġips":59022,"_fraction":59023,"&&(":59024,"Ġstunt":59025,"');?>čĊ":59049,"Ġevapor":59050,"bable":59051,"ĠPRICE":59052,"Ġæ³":59053,"lucent":59054,"Ġvamp":59055,"ĠTechnician":59056,"Ġuniqueness":59057,"Mes":59058,"urban":59059,".parametrize":59060,"ĠReplay":59061,"Sessions":59062,"embr":59063,"-Americans":59064,"_PROXY":59065,"Ġpian":59066,"Ġtrie":59067,"ĠDestructor":59068,"GameState":59069,"ĠIMF":59070,"chin":59071,"Ġporte":59072,"ĠSwal":59073,"åŁİ":59074,"Substring":59075,"iming":59076,"/Library":59077,"Ġfrightened":59078,"writes":59079,"Ġrecursos":59080,"arResult":59081,"_INITIALIZ":59082,"ĠBadge":59083,"_crc":59084,"Eight":59085,"ĠDISTINCT":59086,"Ġthro":59087,"@Xml":59088,"ĠLegendary":59089,"-twitter":59090,"_easy":59091,"Ġ+++":59092,"(DATA":59093,".Locale":59094,"Ġkä":59095,"Ġnurt":59096,"Ġcruis":59097,"_ios":59098,"Ġsensing":59099,"_Line":59100,"ĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":59101,"pong":59102,"oleon":59103,"Ġwildcard":59104,"ç͍æĪ·åIJį":59105,"Ġbegging":59106,"Rod":59107,"ĠÃİ":59108,"_CELL":59109,"Researchers":59110,".selector":59111,"_ing":59112,"Ġaspiring":59113,"Ġimmortal":59114,"Ġymin":59115,"_robot":59116,"Ġplur":59117,"BTC":59118,"ĠDID":59119,"Ġpiercing":59120,"*u":59121,"_DEFINED":59122,"ĠThi":59123,"itaire":59124,"(media":59125,"-ons":59126,"Ġchefs":59127,"Ġ\"*.":59128,"/AP":59129,"Ġrazor":59130,"ĠsearchData":59131,"Ġ=&":59132,"ĠãĢĤ":59133,"Ġmourn":59134,"tingham":59135,"Ġoli":59136,"ĠVernon":59137,"_RS":59138,"ŀæĢ§":59139,"Ġfácil":59140,"angen":59141,"celain":59142,"Ġail":59143,"lest":59144,"ĠQCOMPARE":59145,"gain":59146,"Ġε":59147,"ĠKob":59148,"ĠFault":59149,"_configs":59150,"ç»ĵæŀľ":59151,".+":59152,"calar":59153,"(colors":59154,"Mul":59155,"_ART":59156,"Ġexperimenting":59157,"ermen":59158,"ĠAnglo":59159,".FixedSingle":59160,"Sea":59161,"Ġctxt":59162,".slider":59163,"Collapse":59164,"Grey":59165,"Ġfld":59166,"-proof":59167,".capacity":59168,"getParent":59169,"ĠCompliance":59170,"Ġburgl":59171,"-rec":59172,"Ġoverwritten":59173,"MU":59174,"Ġrouters":59175,"ĉModel":59176,"Ġfantasies":59177,"avian":59178,"_prec":59179,"ĠScandin":59180,"Ġ//<":59181,"/oct":59182,"Ġceremonies":59183,"Months":59184,"undy":59185,"Ġqued":59186,"ĠNou":59187,"ĠVibr":59188,".rgb":59189,"Ġcitrus":59190,"Ġbraces":59191,"-uppercase":59192,"getTable":59193,"Ġdopo":59194,"ĠKerr":59195,"_CHILD":59196,"-cloud":59197,"ĉMatrix":59198,"Ġgardening":59199,"Sing":59200,"almost":59201,"Requirements":59202,"uguay":59203,"(Property":59204,"subscriber":59205,"FAST":59206,"reaction":59207,"(lp":59208,")})Ċ":59209,"`).":59210,".wallet":59211,"_exchange":59212,".Maximum":59213,"ĠVerb":59214,"âĶģ":59215,"()<":59216,"ï¼ĽĊ":59217,"ROT":59218,"CARD":59219,"ubit":59220,"{@":59221,"_kel":59222,"ĠTooltip":59223,"MySQL":59224,"MainActivity":59225,"arf":59226,"Ġmalign":59227,"Ġseinen":59228,"apist":59229,"Ġ<%":59230,"MethodImpl":59231,"Mil":59232,"ĠMick":59233,".depend":59234,">&":59267,"ĉok":59268,"-low":59269,".usuario":59270,"nested":59271,"XB":59272,"OURS":59273,".BorderColor":59274,"Ġbrow":59275,"ĠÐķ":59276,"corr":59277,"ĠRedskins":59278,".getTag":59279,".getTransaction":59280,"Ġstigma":59281,"hardt":59282,"ĠPlayerPrefs":59283,"alsy":59284,"ucson":59285,"Languages":59286,"ĠOlivia":59287,"Ġtac":59288,"Ġbli":59289,"Ġcaval":59290,"Ġconsolidated":59291,"Ġperil":59292,"Ġdele":59293,"Ġformulated":59294,"Ġhighways":59295,".spawn":59296,"==$":59297,"ĠNiet":59298,"Ġveggies":59299,"ypo":59300,"-rule":59301,"ĠVie":59302,"/epl":59303,"Ġenfants":59304,"stringLiteral":59305,"Ġtoughest":59306,"buyer":59307,"Ġcovariance":59308,"Ġili":59309,"ĠSophie":59310,"ĠBAB":59311,"Ġ\"),":59312,"ĠUk":59313,"currentIndex":59314,"_userdata":59315,".codec":59316,"ĠPunjab":59317,"ĠSNP":59318,"lol":59319,"advance":59320,"Ġcomfy":59321,"JsonIgnore":59322,"Ġfashionable":59323,"ĠICON":59324,"Ġora":59325,"ĠPricing":59326,"E":59384,"tering":59385,"/screens":59386,"Ġheightened":59387,"аÑĢÑĤ":59388,"Authorities":59389,"_bbox":59390,"ünst":59391,".fontSize":59392,"ĠBOOLEAN":59393,"divide":59394,"ĠSloven":59395,"ucer":59396,"ÙĴ":59397,"stub":59398,"Ġnavigating":59399,":animated":59400,"_NOW":59401,"_vect":59402,"}{Ċ":59403,"@(":59404,"Ġtelecom":59405,"Ġcontracting":59406,"ĠAssange":59407,"Ġextracting":59408,"Ġgrö":59409,"cobra":59410,".DIS":59411,"Ġcrab":59412,"Ġtwitch":59413,"Ġverts":59414,"Ġrejects":59415,"ĉformat":59416,"Ġregeneration":59417,".Sys":59418,"solve":59419,"ĉdialog":59420,"shi":59421,"meter":59422,"(best":59423,"validators":59424,"Ġonwards":59425,"Ġguru":59426,"Ġmoderator":59427,"owied":59428,"experiment":59429,"rub":59430,"Ġmqtt":59431,"ĠCaucas":59432,"Ġnationalism":59433,"Ġmange":59434,"ĉImGui":59435,"/Edit":59436,"Ġinh":59437,"Ġintellig":59438,"erokee":59439,"ĉexport":59440,"Ġdiscriminate":59441,"subtract":59442,"ĠMoodle":59443,"enser":59444,"ĠGuides":59445,"RAP":59446,"-hot":59447,"_grp":59448,".picture":59449,"XA":59450,"ĠinitView":59451,"_Comm":59452,"Ġoverdose":59453,"Ġ+ĊĊ":59454,"ĠSilent":59455,"shows":59456,"Ġinterpolate":59457,"Formation":59458,"Ġbisc":59459,"markets":59460,"(SC":59461,"Ze":59462,"ĠNetworking":59463,"Ġadrenal":59464,"ĠGuns":59465,"eteor":59466,"Declared":59467,"orgetown":59468,"Ġkarena":59469,"/password":59470,"_addresses":59471,"ITERAL":59472,"Buzz":59473,"ĠConway":59474,"(case":59475,"PWD":59476,"heiro":59477,"(act":59478,"**čĊ":59479,"());ĊĊĊ":59480,"Ġanv":59481,"Ġ..ĊĊ":59482,"(MenuItem":59483,"(mail":59484,"_sections":59485,"ĉnet":59486,"Ġplut":59487,"Ġwrench":59488,"/object":59489,"ĠIst":59490,"ĠVIS":59491,"/pub":59492,"alten":59493,"Ġguitars":59494,"Ġantibiotic":59495,"ï¼ĸ":59496,"¹":59497,"Ġ\"+\"":59498,"formula":59499,"Ġbabes":59500,"ĠPrompt":59501,"Ġenim":59502,"/player":59503,"ĉref":59504,"ĠbyÄĩ":59505,"Ġconsumes":59506,"ĠHast":59507,"ĠTao":59508,"Ġ'))Ċ":59509,"Ġclam":59510,"Ġthighs":59511,"Ġmotif":59512,"ApiOperation":59513,"ĠWL":59514,"getC":59515,"ĉflags":59516,"ointments":59517,"Ġeconomical":59518,"needle":59519,"xls":59520,"practice":59521,"utzer":59522,"timeofday":59523,"-output":59524,"ĠfindById":59525,"ĠBuddy":59526,"ÐŀÑĤ":59527,"Seven":59528,"ĠBark":59529,"Ġenvoy":59530,"_algorithm":59531,"åĪ©":59532,"Ġballistic":59533,"ç§»":59534,"rades":59535,"ĉdoc":59536,"roducing":59537,"ĠEating":59538,"Unmount":59539,"/dataTables":59540,"_bonus":59541,"Ġlitt":59542,"pps":59543,")localObject":59544,"perf":59545,"ĠHelvetica":59546,"shutdown":59547,"/ml":59548,".tokens":59549,"ĠHardcore":59550,",row":59551,"/bg":59552,"Scaler":59553,"âĢĶas":59554,"_logits":59555,"âĢĻint":59556,"ĉApp":59557,"Implicit":59558,".Fprintf":59559,"ETO":59560,"Ġterra":59561,"Ġpossessing":59562,".rstrip":59563,",),":59564,"=yes":59565,"ĠStripe":59566,"?=":59567,"neutral":59568,".good":59569,"Ġkennen":59570,"ĠSung":59571,"fault":59572,"ystatechange":59573,"Canadian":59574,"','\".$":59575,"ĠMits":59576,"ænd":59577,"ĠSTRUCT":59578,"ĠURLWithString":59579,"ĠCompass":59580,"Ġ--ĊĊ":59581,"ĠNSLayoutConstraint":59582,"|min":59583,"-adjust":59584,"Ġrebuilt":59585,"LIGHT":59586,"/se":59587,"-mount":59588,"vpn":59589,"validated":59590,"(QObject":59591,"Ġignition":59592,"ĠChargers":59593,"RYPTO":59594,"]initWithFrame":59595,"ĠFluid":59596,"Ġcadre":59597,"Ġnominations":59598,"Neill":59599,"ĠHou":59600,"Ġcurrents":59601,"_gene":59602,"(inp":59603,"Paris":59604,"zÄĻ":59605,"aggregate":59606,"Ġassoc":59607,"weeted":59608,"errat":59609,"âĢĵĊĊ":59610,"Ġ'/',Ċ":59611,"fixture":59612,"ĠHighest":59613,"ambient":59614,"Ġchmod":59615,"Ġconte":59616,"Ġsensual":59617,"Ġgarment":59618,"zers":59619,"ĠPowered":59620,"domains":59621,"Reward":59622,"iomanip":59623,"Ġcockpit":59624,"outfile":59625,"Ġbuiltin":59626,"Ġinsisting":59627,".vars":59628,"zipcode":59629,"Ġ����":59630,"fails":59631,"Ġconsolidation":59632,"_oid":59633,"Planet":59634,"Ġ=\",":59635,"ĉel":59636,"UILT":59637,"ätz":59638,"afari":59639,"ĠMcCl":59640,"Timeline":59641,"Esta":59642,"Ġfram":59643,"YE":59644,"Ġcerebral":59645,"OfMonth":59646,"ĠPregn":59647,"ĠклаÑģÑģ":59648,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":59649,"ĠFres":59650,"Approved":59651,".Special":59652,"ĠProtestant":59653,"Ġallergy":59654,"_pcm":59655,"ĉCopyright":59656,"ĠsuperClass":59657,"\"strconv":59658,"ĠMohamed":59659,"Ġ'//":59660,"ForeColor":59661,"Arthur":59662,"ĠJungle":59663,"Ġveins":59664,"Sad":59665,"Ġbackups":59666,"ĠOpinion":59667,"ût":59668,"Ġintermitt":59669,"odyn":59670,"ĠChristina":59671,"Ġandre":59672,"Ġevacuation":59673,"palette":59674,"horse":59675,"ĠResident":59676,"ĠHassan":59677,".Nil":59678,"Ġaisle":59679,"ĠGrowing":59680,"Ġbloginfo":59681,"/sql":59682,"_ioctl":59683,"Scaling":59684,"ĠMonad":59685,"_cpp":59686,"ĠHutch":59687,"ĠAppleWebKit":59688,"Expense":59689,"_JOB":59690,"Ġpointless":59691,"FromBody":59692,"antal":59693,"Ġdepicting":59694,"ĠCELL":59695,"Ġrefin":59696,"ĠCNC":59697,"ì¹ĺ":59698,"_dimensions":59699,"ĠSAN":59700,"Ġaft":59701,"Ġfootsteps":59702,"ccoli":59703,"_PHONE":59704,"/math":59705,"-kind":59706,"ĠMeans":59707,"ichael":59708,".guna":59709,"Ġinauguration":59710,"-driving":59711,"(delete":59712,"ĠtotalCount":59713,"_MC":59714,".Extension":59715,"Commercial":59716,"ĠzIndex":59717,"$":59849,"Ġebay":59850,"Ġcaptive":59851,"pliant":59852,"ĠCalculates":59853,"olta":59854,"esting":59855,"_revision":59856,"Ġmús":59857,"+m":59858,"\",\"\",\"":59859,"WHAT":59860,"Ġcompassionate":59861,"harga":59862,"[random":59863,"Ġmodulo":59864,"(sn":59865,"Ġoccupations":59866,"////Ċ":59867,"ĉboard":59868,"ĠBalk":59869,"wiÄħ":59870,"ĠWifi":59871,".Profile":59872,":maj":59873,"ĉmat":59874,"LOCKS":59875,"(jButton":59876,"Ġ('$":59877,"Mur":59878,"æĮī":59879,"bble":59880,"Ġfrog":59881,"-hide":59882,"Ġbroadcaster":59883,"à¸ŀ":59884,"haled":59885,"Ġamusing":59886,"_predictions":59887,"_intr":59888,"Ġeagle":59889,"аÑĤелÑĮ":59890,"ĠgetList":59891,"psilon":59892,"Ġcharacterization":59893,"ARDS":59894,"Ġrelocation":59895,"Ġrulers":59896,"PAY":59897,"ĠDefinitely":59898,"_Action":59899,"Ġclosures":59900,"Ġfactual":59901,"odynamic":59902,"Ġprecautions":59903,"niej":59904,"ĠParties":59905,"ĠSubaru":59906,"Ġcousins":59907,"arbeit":59908,".money":59909,"gunta":59910,"(and":59911,"getitem":59912,".StylePriority":59913,"Ġslid":59914,"singleton":59915,"Ġgarn":59916,"ĠPAS":59917,"Ġdazz":59918,"aż":59919,"Ġbogus":59920,"ĠMog":59921,"Ġrivalry":59922,"isol":59923,"Ġlandmarks":59924,"ñas":59925,"Bern":59926,"ĠSachs":59927,"Ġ\")ĊĊ":59928,"Ġhostility":59929,"_mex":59930,"mere":59931,"Mot":59932,"pictureBox":59933,"Defense":59934,"Ġaffidavit":59935,"otherwise":59936,".directory":59937,"_UnityEngine":59938,"-blog":59939,".skin":59940,"phem":59941,"Apellido":59942,"erchant":59943,"[class":59944,"Ġwart":59945,".\"[":59946,"aleur":59947,"/back":59948,"ĠĠĠĠĉĠĠĠ":59949,"Ġprecipitation":59950,"Ġobstruction":59951,"ĠpObj":59952,"Ġrupt":59953,"UCKET":59954,"aye":59955,"æİĴ":59956,"gx":59957,"Ġecl":59958,"Ġsecrecy":59959,"/Header":59960,"ĠLesb":59961,"Ġlei":59962,"ĠBulletin":59963,"Ġgiveaway":59964,".Home":59965,"_ROOM":59966,"\"W":59967,"Ġcowork":59968,"_ra":59969,"ĠCycling":59970,"ĠPaw":59971,"Ġpupil":59972,"/arch":59973,"ĠFileUtils":59974,"é¦ĸ":59975,"rsp":59976,"Ġfreedoms":59977,"ĠLear":59978,"}`).":59979,"Ġbowls":59980,"/block":59981,"_logging":59982,"Ġmethane":59983,"Ġhorns":59984,"Ġwonderfully":59985,"Ġalterations":59986,"Ġexile":59987,"lsen":59988,"_pause":59989,"_LANGUAGE":59990,"ĠUSDA":59991,"_mysql":59992,"_AMOUNT":59993,"ĠLIFE":59994,"Ġyoungsters":59995,"Ġriots":59996,"[E":59997,"Ġunforgettable":59998,",},Ċ":59999,"Disposed":60000,"ĠAssassin":60001,"UNG":60002,"ĠNewsp":60003,"UserService":60004,":aload":60005,"+',":60006,"Ġsettlers":60007,"Ġscreams":60008,"Ġinconvenience":60009,".Rotate":60010,"Ġjars":60011,"ĠPuzzle":60012,"Ġmest":60013,"arsi":60014,"ĠSharma":60015,"|(":60016,".ds":60017,"ĠSacred":60018,"_evt":60019,"Ġexpresses":60020,"Ġhoch":60021,"ĠDuch":60022,".calls":60023,"thr":60024,"ĠSheffield":60025,".AlertDialog":60026,"Ġradically":60027,"Ġtrous":60028,"Ġprevailing":60029,"ĠWWII":60030,"âĢĻn":60031,"ensely":60032,"ĠYesterday":60033,"ĠSirius":60034,"Ġkillers":60035,"ĠFFT":60036,"Ġoval":60037,"'):čĊ":60038,"Ġìłķë³´":60039,"ourage":60040,"ĠCheckbox":60041,"Workbook":60042,".defer":60043,"_floor":60044,"Ġcouncill":60045,"Ġnorske":60046,"moil":60047,"orea":60048,"Ġmarketed":60049,"_SUR":60050,"xAA":60051,"Ġstained":60052,"eut":60053,"ĠMeng":60054,"Ġieee":60055,".extern":60056,"egie":60057,"Ġrapp":60058,"ĠPyongyang":60059,"'class":60060,"Mob":60061,"ĠinitialValue":60062,"_wave":60063,"Ġjab":60064,"Ġmasculine":60065,"Ġamplifier":60066,"Ġtty":60067,"PathComponent":60068,"_xt":60069,"ĠGFP":60070,"/sec":60071,"ĉdispatch":60072,"markdown":60073,"ĠSchn":60074,"bole":60075,"··":60076,"mousemove":60077,"ĠerrMsg":60078,"Ġasign":60079,"_mono":60080,"ToSelector":60081,"ĠZu":60082,"(Rect":60083,"ĠErrorCode":60084,"latin":60085,"angible":60086,"vtk":60087,"CGSize":60088,"Pokemon":60089,"Ġclassmates":60090,"Ġattracts":60091,"ĠTatto":60092,"ultan":60093,"ológ":60094,"Ġhalted":60095,"न":60096,"ĠKart":60097,"Ġue":60098,"_InitStructure":60099,"TestClass":60100,"ĠAirbnb":60101,"_\",":60102,"Ġcharcoal":60103,"Ġipc":60104,"ĠStretch":60105,".glide":60106,"latesAutoresizingMaskIntoConstraints":60107,"Ġpotion":60108,"ITTLE":60109,"Ġcountert":60110,"_hd":60111,"prepared":60112,"Ads":60113,"ĠVampire":60114,"robots":60115,".CreateIndex":60116,"StatusLabel":60117,"Ġtucked":60118,"afür":60119,"Ut":60120,"Ġsweater":60121,"_FN":60122,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĉ":60123,"ataka":60124,"Ġeyebrows":60125,"acoes":60126,"uden":60127,".LinearLayoutManager":60128,"Ġsway":60129,"Ġmultin":60130,"())))Ċ":60131,"ĠNSUInteger":60132,"ĠMyBase":60133,"Partner":60134,"utschen":60135,"ĠCater":60136,".setBackgroundColor":60137,"Ġaccomplishment":60138,"_problem":60139,".dtd":60140,"ĠpageNumber":60141,"Ġjackets":60142,"Ġcropped":60143,"uels":60144,"ĠHep":60145,"Ġcapped":60146,"*Math":60147,"_callbacks":60148,"Ġpubb":60149,"ĠBrunswick":60150,".respond":60151,"[\"_":60152,"Ġbedding":60153,"hythm":60154,"OX":60155,"(speed":60156,"Ġpesticides":60157,"Ġ-------":60158,".Blue":60159,"Ġnoodles":60160,"ĠGoes":60161,"Ġsaver":60162,"oxy":60163,"_completion":60164,"ĠSwinger":60165,"ĠgetDate":60166,"Ġminded":60167,"integration":60168,"ĠLotus":60169,"(stop":60170,"(',');Ċ":60171,"Ġfloods":60172,"ĠWorkflow":60173,"Ġerupted":60174,"Macro":60175,"ĠSauce":60176,"ĠeventName":60177,"\\Input":60178,"Breaking":60179,"ĉwhen":60180,"_pw":60181,"INDER":60182,"ĠWellness":60183,"Ġvoxel":60184,"ĠMell":60185,"ĠMEDIA":60186,"SENS":60187,"ĠFunds":60188,"ĠMild":60189,"Ċ":60198,"Ġtempting":60199,"Ġtestament":60200,"Ġbible":60201,"Ġconsulted":60202,"ĠIndexError":60203,"è¨ĺ":60204,"Ġkeypad":60205,"izzo":60206,"(ok":60207,"Ġwhatsapp":60208,"ĠRemoteException":60209,"Ġteamed":60210,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":60211,"»,":60212,"ĠgetTime":60213,"diag":60214,"issy":60215,"Ġhed":60216,"Ġknots":60217,"jom":60218,"Ġfunnel":60219,"-mails":60220,"Ġexporting":60221,"ĠVL":60222,"ĠKarn":60223,"ĠBuddhism":60224,"ĠAllan":60225,"_RADIUS":60226,"Ġwording":60227,"ĠForget":60228,"ĠCorona":60229,"iphy":60230,"Ġlimburg":60231,"uggy":60232,"ĠUserRepository":60233,"imin":60234,"(ele":60235,"Ġlabelled":60236,"社":60237,"ĠHerman":60238,".qq":60239,"Ġ\"));Ċ":60240,"ieber":60241,".Translate":60242,"ryn":60243,"Ġdesenv":60244,"umd":60245,"Simply":60246,"ĉmode":60247,"Rpc":60248,"ĠValencia":60249,"Ġstaffers":60250,"Ġselv":60251,"ĠSpike":60252,"Ġdelic":60253,"Ġeru":60254,"_DT":60255,"Judge":60256,"á»ķ":60257,"ĠBasin":60258,".mutable":60259,"\"url":60260,"Ġtariff":60261,"ĠSleeve":60262,"Ġflare":60263,".dropout":60264,"Ġbrides":60265,")),čĊ":60266,"_constraints":60267,"destruct":60268,"Outline":60269,"Ġdisappears":60270,"_locked":60271,"ĠNSLocalizedString":60272,"cke":60273,"ĉnull":60274,"adresse":60275,"Ġtopping":60276,"ĠJoker":60277,"bishop":60278,"ноÑģÑĤÑĮ":60279,"andering":60280,"_amp":60281,"=time":60282,"_Space":60283,"_PULL":60284,"'=":60285,"Ġantiqu":60286,"Ġcach":60287,"___ĊĊ":60288,"ONES":60289,"оÑı":60290,"Ġunread":60291,".policy":60292,"oooooooo":60293,"룬":60294,"Ġusted":60295,"ĠRece":60296,"Ġallem":60297,"ãĥ¼ãĤ¹":60298,"ĠThoughts":60299,"veillance":60300,"istrate":60301,"_lane":60302,"Ġfamed":60303,".GetName":60304,"Ġsmoother":60305,"ĠQualified":60306,"azers":60307,"_geo":60308,"Fax":60309,"ĠMinds":60310,"ĠRaises":60311,"Ġtranscripts":60312,"Conversation":60313,"Ġremarked":60314,"ëĤĺ":60315,"dling":60316,"Ġdeploying":60317,"ĠsharedApplication":60318,"Ġkp":60319,"FontAwesomeIcon":60320,"_dummy":60321,"reiben":60322,"ĠJaneiro":60323,"Directions":60324,".getBean":60325,"sass":60326,"Ġcommanders":60327,"vation":60328,"errorCode":60329,"ĠAlloy":60330,".localized":60331,"Ðij":60332,"Ġdishwasher":60333,"ĠSoup":60334,"Nu":60335,"_Default":60336,"Ġuneven":60337,"Ġ/>\";Ċ":60338,"-Based":60339,"Ġseamlessly":60340,"-null":60341,"ĠXC":60342,"Ġstew":60343,"(delay":60344,"ATORS":60345,"ĠWheeler":60346,"\"H":60500,"east":60501,".air":60502,"âĢľBut":60503,"ObjectContext":60504,"successfully":60505,"_land":60506,"Ġfolds":60507,"_COORD":60508,"Ġsubpo":60509,".getAddress":60510,"instr":60511,"Materials":60512,"ÑĥÑģÑĤ":60513,"deposit":60514,"-last":60515,"_GRAY":60516,"=find":60517,"Ġmutant":60518,"Ġlesbienne":60519,"letcher":60520,"ROUGH":60521,"ureka":60522,".capture":60523,"Ġenn":60524,"Ġ([[":60525,"ĠFlu":60526,"ĠtaskId":60527,"ĠHussein":60528,".folder":60529,"Ġausterity":60530,"ISTRATION":60531,"_Impl":60532,"注æĦı":60533,"Ġdecree":60534,"-chat":60535,"Ġimplication":60536,"Ġguesses":60537,"ulkan":60538,"Analytics":60539,".plus":60540,"COMMAND":60541,"ели":60542,"»ĊĊ":60543,"_SITE":60544,"ĠequalTo":60545,"SupportFragmentManager":60546,"ĠRecording":60547,"å®ĮæĪIJ":60548,"Ġbaggage":60549,"Ġpitchers":60550,"ĠEh":60551,"oque":60552,"ĉcnt":60553,"Ġ=>$":60554,"/foo":60555,"IRA":60556,"ĠSatellite":60557,"borah":60558,"Ġ}}\"Ċ":60559,"ĠEnds":60560,"ĠSpray":60561,",param":60562,".Chrome":60563,"*q":60564,"thought":60565,"ibrated":60566,"Ġthieves":60567,"Ġbeneficiaries":60568,"Entered":60569,"ottesville":60570,"Ġveterin":60571,"ByID":60572,"quipe":60573,"umption":60574,"-unit":60575,"ExecutionContext":60576,"@s":60577,"ĠGiov":60578,".ToolTip":60579,"_friend":60580,"(attributes":60581,"Ġdumping":60582,"ĠJC":60583,"_DOCUMENT":60584,"ĠArmour":60585,"(insert":60586,".HorizontalAlignment":60587,"ĠQed":60588,"ãģĦãģ¾ãģĻ":60589,"/git":60590,"ĠYYYY":60591,"ĠCardiff":60592,"Ġapa":60593,"organic":60594,"ĠWhereas":60595,"ĠæĿ":60596,"ĠMia":60597,"Ġdemolition":60598,"Ġscars":60599,"Ġpai":60600,"Ġretries":60601,"Ġrq":60602,"ĠDenis":60603,"(Utils":60604,"Ġalleviate":60605,"ĠPIC":60606,"idue":60607,"Ġacknowledging":60608,"Ġ//////////////////////////////////":60609,"ç¡®å®ļ":60610,"Ä«":60611,"\\Json":60612,".binary":60613,"Ġxtype":60614,"signals":60615,"ĠAppearance":60616,"&r":60617,"}s":60618,"Ci":60619,"ĠIllum":60620,"porate":60621,"hog":60622,"ĠindexOf":60623,"\\Command":60624,"_parallel":60625,"ĠSherlock":60626,"íĥ":60627,"Ġ\"\")čĊ":60628,"////////////////////////////////////////////////////////////////////////////////////////////////":60629,"Ġcriticize":60630,"ĠSoap":60631,"ĠMatcher":60632,"Ġgrilled":60633,"*T":60634,"Ġadore":60635,"ulling":60636,"Ġjedoch":60637,"_refs":60638,"leanup":60639,"ĠJAXB":60640,"Ġroses":60641,"ĠLiam":60642,"sizei":60643,"Ġgetchar":60644,"Ġtarde":60645,"-tooltip":60646,"Ġqualifier":60647,"ĠIntermediate":60648,"_Window":60649,"ĠMalta":60650,"Disconnect":60651,"ewhere":60652,"Campo":60653,"Ġirrational":60654,"ledo":60655,"ĠDN":60656,"ARGV":60657,"Ġoutro":60658,"Ġthirteen":60659,"Joseph":60660,"MAR":60661,"/gl":60662,"Jess":60663,"ĠPsychiat":60664,"ĠpaddingBottom":60665,"-loop":60666,"/fonts":60667,"_seen":60668,"Teams":60669,"ReactDOM":60670,"(man":60671,"(xpath":60672,".getSimpleName":60673,">(*":60674,"ĠPvt":60675,"Ġelders":60676,"Ġpies":60677,".userAgent":60678,"-region":60679,"ĠGreeks":60680,"(fragment":60681,"stu":60682,"Ġcouncils":60683,"Ġstamina":60684,"ĠGoddess":60685,"西":60686,"Ġphilosophers":60687,"Ġpersone":60688,"ĠLose":60689,"ĠCLR":60690,"ĠDocs":60691,"Ġsoak":60692,"ĠHOLDER":60693,"Ġbells":60694,"hashCode":60695,"RATE":60696,"_WEIGHT":60697,"inous":60698,"endra":60699,"ophobic":60700,"Ġprose":60701,"Ġfinely":60702,"/oauth":60703,"(space":60704,"adge":60705,"ĠMama":60706,"ĠstringBuffer":60707,"Ġstint":60708,"Ġmisma":60709,"Ġvillains":60710,"ĠCrimea":60711,"Ġdiploma":60712,"ĠпоÑģл":60713,"ĠBea":60714,"(join":60715,"Ġíķ´":60716,"CHAT":60717,"pering":60718,"ĠCros":60719,"Ġmonkeys":60720,"Ġpreds":60721,"yla":60722,",,,":60723,"Ġvibrator":60724,"ĠNU":60725,"åħĪ":60726,"fant":60727,"zet":60728,"Ġbietet":60729,"unft":60730,"sworth":60731,".Flow":60732,"Ġpsyched":60733,"ĠContinental":60734,">t":60735,"Ġquilt":60736,".UP":60737,"Ġexpansive":60738,"Dispose":60739,"(language":60740,"Caps":60741,"_ZONE":60742,"Ġrecycle":60743,"ĠManaged":60744,"currentColor":60745,".broadcast":60746,"signIn":60747,".prom":60748,"llu":60749,"ueblo":60750,"Ġpunches":60751,"Ġautomat":60752,"Ġassigning":60753,"ĠcreateUser":60754,"ĠAllied":60755,"Ġconductor":60756,"Ĥ¨":60757,"Ġsaddle":60758,"Ġdni":60759,"omedical":60760,"-West":60761,"PositiveButton":60762,"Ġitalic":60763,"?[":60764,"(trigger":60765,"Ġelephants":60766,"\":\"\",\"":60767,"Ġcaliber":60768,"rafted":60769,"digits":60770,"Ġmarshal":60771,"milliseconds":60772,"markers":60773,"mom":60774,"/place":60775,"Ġholistic":60776,":t":60777,"#,":60778,"Ġboto":60779,"Ġnausea":60780,"ĠShooting":60781,"itech":60782,"ĠtextStatus":60783,"())Ċ":61004,"ADDRESS":61005,"BST":61006,"etzt":61007,"ĠQgs":61008,"Sense":61009,"ExceptionHandler":61010,"ĠChu":61011,".getOwnProperty":61012,"Ġexercised":61013,"iotic":61014,"ĠReleases":61015,"Ġpinterest":61016,"olie":61017,"isoft":61018,"Ġsequencing":61019,"Ġpadre":61020,"]));čĊ":61021,"(radius":61022,".med":61023,"ainties":61024,".ObjectModel":61025,"Ġemple":61026,"Ġseguro":61027,"Stars":61028,"Ġqualitative":61029,"lemn":61030,"á»±":61031,">\").":61032,"Ġgx":61033,"-cert":61034,"ĠASTM":61035,"Ġfullname":61036,"Ġtelemetry":61037,"ĠCambodia":61038,"_ul":61039,"ĠClare":61040,"CUSTOM":61041,"QC":61042,"ĠUns":61043,"ĠHTTPS":61044,"ĠParkinson":61045,"ancybox":61046,"','.":61047,"Tue":61048,".getLast":61049,"Ġabi":61050,"Äħd":61051,"Ast":61052,"ĠEditing":61053,".Unity":61054,"jmp":61055,"Ġmats":61056,"ĠsharedPreferences":61057,"Captain":61058,".pageSize":61059,"Ġrtl":61060,"Ġanmeld":61061,"RuntimeObject":61062,"Ġdemande":61063,"(\";":61064,"seite":61065,"-headed":61066,"ĠKra":61067,"ĠFONT":61068,"`\\":61069,"ClassNotFoundException":61070,".avg":61071,"atical":61072,"Aj":61073,"Ġpermitting":61074,"Proj":61075,"ERRQ":61076,"Ġcreampie":61077,"ĠBuyer":61078,"-modules":61079,"ĠSundays":61080,"|`Ċ":61081,"Ġdaytime":61082,"Ġ+(":61083,"Ġglitch":61084,"ĠOperand":61085,"Ġtoxins":61086,"inya":61087,"DNS":61088,"ĠSas":61089,"Cake":61090,"ĠNationals":61091,".addTo":61092,"Ġsinking":61093,"Ġcomprehension":61094,"Ġscor":61095,"agements":61096,"Ġtard":61097,"Ġmarching":61098,"ĠMTV":61099,"Ġsane":61100,"CreateInfo":61101,"ắ":61102,"ĠendIndex":61103,"ĉlayout":61104,"ĠåIJį":61105,"SITE":61106,"ĠTHERE":61107,"Ġ[{'":61108,"opathic":61109,"Ġtransmitter":61110,"/body":61111,"Ġpund":61112,"ĠClosing":61113,"Ġsetattr":61114,"Ġbounded":61115,"Atlas":61116,"suming":61117,"(times":61118,"parer":61119,"ynom":61120,"feit":61121,"Ġfrem":61122,"-leg":61123,"ĠBras":61124,">#":61125,"Ġì¶ľëł¥":61126,"ĠINSTANCE":61127,"ĠCouch":61128,"_hosts":61129,"likelihood":61130,".Marker":61131,"ĠMasks":61132,"Ġcereal":61133,"utilities":61134,"Ġelemental":61135,"Ġdistorted":61136,"inactive":61137,"cry":61138,"WL":61139,"UPPORTED":61140,".Throws":61141,"/schema":61142,"serie":61143,".\"',":61144,"ĠBenedict":61145,"-picker":61146,"iggs":61147,"ĠPirate":61148,"åij¨æľŁ":61149,"ĠThema":61150,"ĠSouthampton":61151,"ĠarrayWith":61152,"ĠPaula":61153,"Ġpredictor":61154,"-Ass":61155,".userid":61156,"Ġperi":61157,"Ġexaggerated":61158,"urate":61159,"arseille":61160,"ĠConcent":61161,"ĠPik":61162,"Ġ@_;ĊĊ":61163,"Ġformations":61164,"Ġdenomin":61165,"\"/>.Ċ":61166,"endedor":61167,"Ġpancre":61168,"Ġamt":61169,"ĠonResume":61170,"onDelete":61171,"ĠBCH":61172,")(\"":61173,"movement":61174,"Ġpotassium":61175,"":69726,"ĠPPC":69727,"isz":69728,"akeFromNib":69729,"ĠDisp":69730,"ĠAthletics":69731,"Ġnightclub":69732,"GOOD":69733,".setGeometry":69734,"+[":69735,"/send":69736,"Ġbinaries":69737,"Ġráp":69738,":req":69739,"-consuming":69740,"ertime":69741,"UPDATED":69742,"_nullable":69743,"VIN":69744,"ulia":69745,"cyan":69746,"Ġmisunderstanding":69747,"orical":69748,"degrees":69749,"Leading":69750,".AR":69751,"ickest":69752,"Nuevo":69753,"uforia":69754,"Ġgoodies":69755,"Ġfores":69756,"()<<\"":69757,"ademic":69758,"ActionCreators":69759,"servername":69760,"(nt":69761,"dbContext":69762,"Ġairborne":69763,"Ġexhibitions":69764,"cele":69765,"Ġtela":69766,"":69782,".setPreferredSize":69783,"ĠMID":69784,"ĠAless":69785,"Ġhorsepower":69786,"Ġatm":69787,"ĠPackaging":69788,"Ġciphertext":69789,"RequestMethod":69790,"Ġbeiden":69791,"è£":69792,"ĠPOW":69793,".WriteHeader":69794,"director":69795,"-but":69796,"ãģłãģķãģĦ":69797,"incer":69798,"_dn":69799,"!!!!!":69800,"Ġmanufactures":69801,".TextUtils":69802,"Ġconsciously":69803,"Ġbounced":69804,"culture":69805,"ĠSpar":69806,"ĠPiper":69807,".press":69808,"-owner":69809,"Ġevaluator":69810,"ĠSTREAM":69811,".PictureBoxSizeMode":69812,"Ġsugars":69813,"ScreenWidth":69814,"ĠnextState":69815,"Ġivory":69816,"Ġbrunch":69817,"density":69818,"_OW":69819,"ĠCoronavirus":69820,"ĠCFR":69821,"bak":69822,"\\Category":69823,"æķ°ç»Ħ":69824,"Ġinvokevirtual":69825,"}()Ċ":69826,"Ġsujet":69827,"-marker":69828,"isdigit":69829,"ĠMobil":69830,"ĠJsonRequestBehavior":69831,"_REMOTE":69832,".existsSync":69833,"Ġriches":69834,".presenter":69835,"ĠglColor":69836,"Ġhanya":69837,"Ġfortress":69838,"Ġflashed":69839,"viz":69840,"requently":69841,"buat":69842,"$con":69843,">|":69844,".Func":69845,"Ġhumorous":69846,"uem":69847,".ZERO":69848,"ĠSTL":69849,"ĠBuk":69850,"/sample":69851,"ĠGros":69852,"Recipes":69853,"Ġinflated":69854,"Ġswung":69855,":F":69856,"Facing":69857,".Theme":69858,"ник":69859,"Ġsplendid":69860,"ĠrequestId":69861,".CenterScreen":69862,"/autoload":69863,"embedded":69864,"_depart":69865,"ĠPorts":69866,"à¹ĥ":69867,"айд":69868,"discussion":69869,"_consum":69870,"Ġscouts":69871,"Ġcolabor":69872,".Stage":69873,".nano":69874,"eldorf":69875,"Ġgemacht":69876,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":69877,"Ġpolicymakers":69878,"_PKT":69879,",Th":69880,"oky":69881,"_UID":69882,"Ping":69883,"Ġorchest":69884,"Ġoptics":69885,"uhan":69886,"ĠXOR":69887,"Ġespañol":69888,"ĠAdidas":69889,"rng":69890,"mans":69891,".vstack":69892,"Ġgetaway":69893,"Ġhierarchical":69894,"anoia":69895,"ĠBitmapFactory":69896,"realm":69897,"ĉap":69898,"_apps":69899,"-divider":69900,".drawer":69901,"ĠHARD":69902,"'];?>Ċ":69903,"-packed":69904,"æ²»":69905,"_STRUCTURE":69906,"[Y":69907,"iParam":69908,"(eq":69909,"Ġencompasses":69910,"Ġ\\ĊĊ":69911,"->[":69912,"&utm":69913,"groupon":69914,"strate":69915,"DY":69916,"omorphic":69917,"':[":69918,"Ġgravitational":69919,"ĠMicha":69920,"ĠTencent":69921,"Ġcoached":69922,"ì¶ľ":69923,"ÑĥменÑĤ":69924,"/mobile":69925,"MouseDown":69926,"bud":69927,"ĠYas":69928,"ĠProviders":69929,"NZ":69930,"ĉreport":69931,"errmsg":69932,"ĠimagePath":69933,"acterial":69934,"ĠManga":69935,"wicklung":69936,"(usuario":69937,"\"));čĊčĊ":69938,"/***":69939,"Ġorganise":69940,"Indexed":69941,"_QUAL":69942,"(PyObject":69943,"Ġsurrendered":69944,"POCH":69945,"ĠNOTES":69946,"\\\\\"":69947,"-job":69948,"Ġseventy":69949,"####Ċ":69950,"ĠManor":69951,"Ġdownright":69952,"Ġtimeframe":69953,"insurance":69954,"checker":69955,"ĠSECRET":69956,"Ġechoes":69957,"ĠCarmen":69958,".setHorizontalAlignment":69959,"ĠisChecked":69960,"ĠTOR":69961,"_nn":69962,"('(":69963,"FetchRequest":69964,"ĠPrinted":69965,"Fluid":69966,"ĠSTACK":69967,"GES":69968,"aigned":69969,"igor":69970,".Unknown":69971,"CBC":69972,"ĠCarlson":69973,".URI":69974,"Ġplight":69975,"/start":69976,"ĠPersonnel":69977,"ĠPREFIX":69978,",**":69979,"Ġlimite":69980,"_heat":69981,"%ï¼Į":69982,"ĠDonne":69983,"getNode":69984,"ĠScientology":69985,"Ġcomet":69986,"Ġwenig":69987,"Aside":69988,"ĠMPEG":69989,"'?":69990,"variably":69991,".endDate":69992,"Ġuncont":69993,"ĠScores":69994,"ĠLoginForm":69995,".generated":69996,",ch":69997,"-mar":69998,"ĠNed":69999,"ĠeventId":70000,"+p":70001,"ĠSIN":70002,"/reset":70003,".REACT":70004,"ĠMessi":70005,"_RANK":70006,".writeFile":70007,"Ġcripp":70008,"esthetic":70009,"ERSIST":70010,"Ġreimbursement":70011,"CurrentValue":70012,"Ġunin":70013,"DownLatch":70014,"ĠpaddingRight":70015,"Ġstocked":70016,"/'.":70017,"Ġrepayment":70018,"trak":70019,"/backend":70020,"Ġизмен":70021,"CSR":70022,"Ġpreventive":70023,"Ġpantalla":70024,"_trim":70025,"Pedido":70026,"hospital":70027,"Ġmanageable":70028,"routeParams":70029,"textures":70030,"......ĊĊ":70031,"Ġsélection":70032,"NameValuePair":70033,"Ġpollut":70034,"Modes":70035,"ĠLaud":70036,"jay":70037,"ĠUrs":70038,"Ġsigner":70039,"ĠJJ":70040,"ĠCherokee":70041,"_EXISTS":70042,"Ġdwar":70043,"Ġ($('#":70044,"Ġreef":70045,">{$":70046,"ĠBaylor":70047,"ĠModelState":70048,"-_":70049,"ĠStructures":70050,"Ġsouvent":70051,"Specify":70052,"(pipe":70053,"Ġfracking":70054,"ĠGPA":70055,"Ġbele":70056,"ĉĉĉĉĉĉĉĠĠĠ":70057,"ĠMinority":70058,"Ġtud":70059,"Ġopenness":70060,"ĠIllustrated":70061,"Ġoxidation":70062,"ĠNK":70063,"ĉUpdate":70064,"ĠEMS":70065,"ĠTeddy":70066,"Ġgenerals":70067,"ĉMat":70068,"Ġradios":70069,"ĠAntique":70070,"conomy":70071,"ĠSquadron":70072,")','":70073,"声":70074,"Ġyoure":70075,"ĠMainPage":70076,"Ġbehaviours":70077,"enght":70078,"(@\"%@\",":70079,"Ġtestcase":70080,"ĠCompilation":70081,"Ġflavours":70082,"ĠExtend":70083,"illator":70084,"Ġcoh":70085,"Ġspline":70086,"ĠKG":70087,"-pay":70088,"Ġcommunism":70089,"ĠBusinesses":70090,"ocking":70091,".MaxLength":70092,"assandra":70093,"quiring":70094,"adden":70095,"ĠJeb":70096,"_fault":70097,"[file":70098,"Ġprominence":70099,"disciplinary":70100,"âĢĶthey":70101,"_extent":70102,"ĠVIC":70103,"Ġentails":70104,".partner":70105,"Ġhippoc":70106,"League":70107,"çĶ·":70108,"wipe":70109,"-spinner":70110,"Ġsalute":70111,"ĠSurgical":70112,"(outputs":70113,"worked":70114,"[strlen":70115,"appointed":70116,"ĠHeg":70117,"ĠACPI":70118,"([^":70119,"uala":70120,"_tol":70121,"ĠRit":70122,".Payment":70123,"kowski":70124,"Ġwalmart":70125,"requirements":70126,"ĠFINSEQ":70127,"_BACKGROUND":70128,"ĠOsborne":70129,"(errorMessage":70130,"Reporting":70131,"Ġauctions":70132,"Ġcombos":70133,"ĠNoticed":70134,"_oct":70135,"Ġprimero":70136,"taire":70137,"_hr":70138,"Ġмод":70139,"Ġcontradictory":70140,"=\"@":70141,"achines":70142,"(optarg":70143,"ĠPenguin":70144,"ĠAbbas":70145,"Ġsublime":70146,"Ġpageable":70147,"ĠDefensive":70148,"Ġdistinctly":70149,"ĠAutomatically":70150,"Understanding":70151,"EqualityComparer":70152,"gota":70153,"Ġ\"::":70154,"Ġpulver":70155,"ĠBattles":70156,"Ġunparalleled":70157,"TCHA":70158,"Ġconstrued":70159,"-aff":70160,"Ġprecursor":70161,"-lfs":70162,"Ġmaduras":70163,"ĠDaisy":70164,"ĠArbeits":70165,".Management":70166,"ĉIn":70167,"Ġrobes":70168,"Ġspéc":70169,"âĢľ(":70170,"Ġmaternity":70171,"extent":70172,"ĠSpacer":70173,"DidAppear":70174,"ĉus":70175,".getRequestDispatcher":70176,"(cols":70177,"Ġplummet":70178,"ìħ":70179,"Ġ{ĊĊĊĊ":70180,"érica":70181,"ĠSizes":70182,".enum":70183,".Highlight":70184,"Ġ!!}ĊĊĊ":70193,"Wenn":70194,"Ġclimax":70195,"Ġcrem":70196,"_that":70197,"[â̦":70198,"_domains":70199,"_REPLY":70200,"Ġcompleta":70201,"VEST":70202,"_particle":70203,"Ġsop":70204,"Ġfatalities":70205,"implify":70206,"ĠSKF":70207,"Ġinfusion":70208,"ĠJavier":70209,"Ġballet":70210,"Ġamigo":70211,".want":70212,"Ġcollagen":70213,"ĠLawyer":70214,".Statement":70215,".rt":70216,"baar":70217,"EndPoint":70218,"ĠBek":70219,"SHIP":70220,"Ġpatriarch":70221,"ĠAunt":70222,"_TM":70223,"ĠmÃŃn":70224,"Ġmastered":70225,"WXYZ":70226,"Ġespos":70227,"=logging":70228,"Ġrighteousness":70229,"torrent":70230,"Ġbst":70231,"_CHAIN":70232,"Ġoutskirts":70233,"(rotation":70234,"Ġ'.')":70235,"igrants":70236,"+lsi":70237,"ĠCCTV":70238,"_PHASE":70239,".azure":70240,"_Process":70241,"vae":70242,"ĠTropical":70243,"ĠAnkara":70244,"imageView":70245,"_RUNNING":70246,"Ġ*)__":70247,"ến":70248,"(cli":70249,"scatter":70250,"Ġsche":70251,"Registrar":70252,"Ġairing":70253,"Ġpyplot":70254,"isión":70255,"/customer":70256,"Ġsimplement":70257,"Ġclassy":70258,"ĠDWC":70259,"ĠBashar":70260,"ĠDEVELO":70261,"ĠVick":70262,"avail":70263,"ĠHö":70264,"_extend":70265,"drFc":70266,".isNotBlank":70267,"Ġplais":70268,"|}Ċ":70269,"Ġpornofil":70270,"labs":70271,"Ġhaus":70272,"Ġoriginating":70273,"Ġsurrounds":70274,"ĠQUAL":70275,"meg":70276,"/logger":70277,"[obj":70278,"Ġirresponsible":70279,"ĠPublicKey":70280,"HONE":70281,":'/":70282,"ibox":70283,"ĠFVector":70284,"|{Ċ":70285,"ataloader":70286,"hawks":70287,"HDR":70288,"Ġescalation":70289,"ĠPodsDummy":70290,"elite":70291,"Ġpresup":70292,"Cached":70293,">G":70294,".optimizer":70295,"ĠVisible":70296,"´Ģ":70297,"Ġnen":70298,"Ġpcs":70299,"ĠIdle":70300,"[Any":70301,"Ġkeyboards":70302,"ĠCOMPONENT":70303,"Ġtitanium":70304,"(mut":70305,"ĠLedger":70306,"Ġprosperous":70307,"etrofit":70308,"_LL":70309,"_patient":70310,"Ġpdata":70311,"Ġkontakte":70312,"Swipe":70313,"Ġcheerful":70314,"ĠHonduras":70315,"\"][$":70316,"Ġhemorrh":70317,"\":\"+":70318,"Ġleasing":70319,"Ġinstalls":70320,"ĠPax":70321,"ĠLogistics":70322,"Ġkinetic":70323,"ĠPhon":70324,"_movement":70325,"ĉbytes":70326,"Ġcinco":70327,"ĠMadness":70328,"\")+":70329,"ĠJE":70330,"_ij":70331,"SceneManager":70332,"ĠBust":70333,"ptest":70334,"aea":70335,"Ġbesser":70336,"ÃŃg":70337,"дин":70338,"(tasks":70339,"(\"(\"":70340,"setType":70341,"(outfile":70342,"ĉreset":70343,"ĠARC":70344,"Ġmúsica":70345,"ĠShelf":70346,"ĠminY":70347,"pch":70348,"Ġweiber":70349,"issor":70350,"Ġtrouve":70351,"ĉButton":70352,"Ġregenerated":70353,"Å£i":70354,"imachinery":70355,"blocking":70356,".dataTables":70357,"_frac":70358,"ĠAdvantage":70359,".visitMethod":70360,"éĩįæĸ°":70361,"Ġextrapol":70362,"Ġteasing":70363,"ĠHitch":70364,"ĠGeek":70365,"ESCO":70366,"Ġwich":70367,"ĉax":70368,"_decor":70369,"ĠscreenWidth":70370,"ĠSophia":70371,"Forgot":70372,".uni":70373,"ĠVenture":70374,"_collision":70375,"Ġlawmaker":70376,"(Edit":70377,"blers":70378,"ĠgetNext":70379,"âĢĶyou":70380,"MediaPlayer":70381,"ĠHorde":70382,"ĠCongressman":70383,"observations":70384,"ĉproperty":70385,"Ġ<--":70386,"CreatedAt":70387,"ubyte":70388,"Ġquarantine":70389,"Ġdistressed":70390,"_APB":70391,"ĠGoodman":70392,"ãĤ«":70393,"Ġrecomend":70394,"_PRINTF":70395,"DONE":70396,"Bindable":70397,"rstrip":70398,"centaje":70399,"ĠUnexpected":70400,"ĠSCHOOL":70401,"ĠProfessionals":70402,"ĠGPUs":70403,"Lesson":70404,"Exclusive":70405,"Ġatrav":70406,"ĠDank":70407,"ĠLawyers":70408,"ĠWalton":70409,">[]":70410,"Ġaloud":70411,"=\"../../../":70412,"Ġdebating":70413,"ĠAVG":70414,"_VOL":70415,"/cgi":70416,".deg":70417,":g":70418,".Infof":70419,"MeasureSpec":70420,".song":70421,"mtree":70422,"ulls":70423,"Jordan":70424,"ĠCovers":70425,"Ġattributable":70426,"Ġjedis":70427,"iatrics":70428,"Ġrotterdam":70429,"Ġmeld":70430,"ĠContentType":70431,"Ġmantle":70432,"Ġalice":70433,"_duplicate":70434,"/Internal":70435,"Ġfilesize":70436,"ĉfire":70437,"rese":70438,"ondere":70439,"Ġfamiliarity":70440,"ĠCrest":70441,"Ġkarma":70442,"Ġtorino":70443,"Ġmesa":70444,"/temp":70445,"Ġchir":70446,"ĠOverflow":70447,"Ġtenemos":70448,"unik":70449,"NEXT":70450,"Alle":70451,"Ġnxt":70452,"Mart":70453,"Ġatl":70454,"Ġperiodo":70455,"_you":70456,"Ġ})).":70457,"intestinal":70458,".AdapterView":70459,"Ġhesitant":70460,"Ġcomparatively":70461,".UInt":70462,"(viewModel":70463,"Ġsangat":70464,"ĠResponsive":70465,"ĠZack":70466,"âħ":70467,"JAVA":70468,"ĠFuller":70469,"ĠâĿ¤":70470,".Consumer":70471,"Ġank":70472,"Ġreactors":70473,"fuck":70474,"_rat":70475,"ĠsessionFactory":70476,"_backward":70477,"Ġscrambled":70478,"ĉth":70479,"Ġinsensitive":70480,"Ġchamps":70481,"Ġnginx":70482,"Ġconhec":70483,"ĠJasper":70484,".fm":70485,"StrictEqual":70486,"achsen":70487,"-Nov":70488,"lassen":70489,".integration":70490,"(lbl":70491,"Compose":70492,"ĠFon":70493,"Ãļ":70494,"Gratis":70495,"ĠLime":70496,"ĠAdapterView":70497,"Ġpoisoned":70498,"anchors":70499,"设计":70500,"']?>\"":70501,"Ġprocur":70502,"Italy":70503,".MONTH":70504,"ĠLUA":70505,"ĠLithuania":70506,"ĠHeads":70507,"_CHUNK":70508,"ĠPUSH":70509,"AspectRatio":70510,"Ġweg":70511,"Ġvids":70512,"ĠWein":70513,"ĉINT":70514,"sessionId":70515,"Industry":70516,"Ġdenounced":70517,"JKLM":70518,"ĠVanessa":70519,".Identifier":70520,"propri":70521,"Ġиг":70522,"Ġtécn":70523,"Ġmosaic":70524,"StreamReader":70525,"-Th":70526,"forth":70527,"Ġadherence":70528,"bate":70529,"Ġknights":70530,"sounds":70531,"Ġsalle":70532,"OMET":70533,"ãĤ¹ãĥĪ":70534,"-tm":70535,"ĠRhe":70536,".FileOutputStream":70537,"åĪĨç±»":70538,"ĠENG":70539,"holiday":70540,"ĠCongratulations":70541,")(Ċ":70542,"Ġaggregates":70543,"HOOK":70544,"ewire":70545,"Senator":70546,"Ġembeddings":70547,"epy":70548,"(COM":70549,"Ġrobber":70550,"äter":70551,"wang":70552,"_teacher":70553,"Ġresentment":70554,"Ġlettuce":70555,"erreur":70556,"(ic":70557,"ĠTactical":70558,"ĠContracts":70559,"Ġmænd":70560,"Ġsitios":70561,"Ġbastante":70562,"Ġnuevos":70563,"ĉNdrFc":70564,"ĠprivateKey":70565,"ucch":70566,"MMdd":70567,"Ġè¾ĵåĩº":70568,"umba":70569,"@foreach":70570,":\");ĊĊ":70571,"Ġslippery":70572,"ĠKeystone":70573,"Ġpioneering":70574,"_triangle":70575,"(\"Ċ":70576,"ĉĉĉĉĉĉĉĉĠĠ":70577,"ĠIntervention":70578,"SCI":70579,"ĠcJSON":70580,"Ġterminating":70581,"ë¹Ħ":70582,"Ġbabys":70583,"Subset":70584,"Ġë¡":70585,"Ġseulement":70586,"Ġmuestra":70587,"Entre":70588,"以ä¸Ĭ":70589,"ngo":70590,"\"bytes":70591,"QRST":70592,"Ġypos":70593,"persona":70594,"ĠDeploy":70595,"cee":70596,"Ġà®":70597,".goal":70598,"Ġhabitats":70599,"ĠisAdmin":70600,"Ġexploiting":70601,"Ġventil":70602,"ĠBalls":70603,"اب":70604,"Ġmindfulness":70605,"(kwargs":70606,"Ġresembling":70607,"Ġchoir":70608,"ĠonBackPressed":70609,"ĠSECURITY":70610,"/gtest":70611,"Ġjustices":70612,"ĠintegerValue":70613,"blah":70614,"ĠAim":70615,"_finalize":70616,"keh":70617,"ĠComplexity":70618,"Ġaugust":70619,"getElementsByTagName":70620,"Ġpreach":70621,"Ġpronunciation":70622,"ĠTrash":70623,"-percent":70624,"_PRIV":70625,"ĠHunts":70626,"ĠCurse":70627,"uellen":70628,"Ġheavyweight":70629,"Xi":70630,"ĉselected":70631,"ĠMcCoy":70632,"å¼Ĥ常":70633,"|=Ċ":70634,"ĠBattlefield":70635,"ItemImage":70636,"Ġdeductions":70637,"ĠElemental":70638,"());//":70639,"ĠBurk":70640,"})čĊčĊ":70641,"swift":70642,"/function":70643,"Usually":70644,"_St":70645,"_feats":70646,"ĠIsValid":70647,"Ġzad":70648,"ImageContext":70649,"Ġclassname":70650,"Ġdonner":70651,"Ġ-->ĊĊĊ":70652,"Ġmotorcycles":70653,"+'/'+":70654,"ĠsetBackground":70655,"\\CMS":70656,".AllArgsConstructor":70657,"ĠLexington":70658,".examples":70659,"ĠPurs":70660,"PushMatrix":70661,"Ġ==============================================================":70662,".addTarget":70663,"pora":70664,"Fullscreen":70665,"Ġgoof":70666,"hlen":70667,"äge":70668,"ĠCURL":70669,"ĠInteresting":70670,"Ġretrieves":70671,"_Obj":70672,"inness":70673,"-----ĊĊ":70674,".tsv":70675,"(IM":70676,"ĠBraves":70677,"_ISR":70678,"osti":70679,"á»ĵ":70680,"ĠExterior":70681,"ĠCourtney":70682,"Ġresidues":70683,"Tier":70684,".*;čĊčĊ":70685,":black":70686,"webView":70687,"\"path":70688,"Ġmasa":70689,"]!='":70690,"ĠMatching":70691,"dur":70692,"Jvm":70693,"=context":70694,"_RING":70695,"Ġproponents":70696,"ĠQStringLiteral":70697,"Ġinflate":70698,"\">čĊ":70931,"_COST":70932,"ilinear":70933,"ĠWorkspace":70934,"Ġspel":70935,"agogue":70936,"ĠMillennium":70937,"ĠPopulate":70938,"Ġnid":70939,".parseColor":70940,"Solar":70941,"ĠGad":70942,"Ġì¤ij":70943,"ĠKamp":70944,"ĉrm":70945,"Ġbenz":70946,"ĠHonestly":70947,"Ġelectrode":70948,"ĠPrairie":70949,"ĠPROFILE":70950,"ĠOriental":70951,"ĠOLED":70952,"/copyleft":70953,"awaii":70954,"(products":70955,")\\<":70956,"-created":70957,".ManyToMany":70958,"\"How":70959,"ĠвÑĭп":70960,"Ġmitochondrial":70961,"_testing":70962,"(created":70963,"ĠgetField":70964,"_EVAL":70965,"].\"":70966,"ĠFSM":70967,"ĠRita":70968,"ĠåıĤæķ°":70969,"Ġcôt":70970,"ĠInsight":70971,"ĉmysqli":70972,"_timing":70973,"IDO":70974,")))))Ċ":70975,"COVERY":70976,".imag":70977,"CDF":70978,"lust":70979,"ickt":70980,"_FP":70981,".','":70982,"gcc":70983,"Ġkurz":70984,"_pwm":70985,"Ġodpowied":70986,"ĠBarrier":70987,"/***************************************************************************Ċ":70988,"pak":70989,"-Israel":70990,"ĠRutgers":70991,"ĠselectedItem":70992,"ĠRamirez":70993,"Farm":70994,"Ġcalendars":70995,"gzip":70996,"Ġblockbuster":70997,"ĠPlymouth":70998,"çľĮ":70999,"responses":71000,".DialogInterface":71001,"-grand":71002,"ĠgetSource":71003,"Ġdejtings":71004,"Ġtieten":71005,"Ġcondemnation":71006,"Ġcontinuar":71007,".MockMvc":71008,"/english":71009,"ĠMediaPlayer":71010,"computed":71011,"ĠClippers":71012,"(delegate":71013,".Slf":71014,"Ġë¡ľ":71015,"ĠTide":71016,"Ġihrem":71017,"ĠWan":71018,"ÑĥÑİÑī":71019,"}><":71020,"Discussion":71021,"Ġwatts":71022,"-minus":71023,"ĠJuliet":71024,"éĽħ":71025,"Ġconcluding":71026,"andscape":71027,"Ġúltima":71028,"ĠDERP":71029,"ĠsignUp":71030,"ĠSecondly":71031,"WAIT":71032,"lds":71033,".callbacks":71034,"(hour":71035,"imators":71036,"volent":71037,"AAF":71038,"edriver":71039,"ĠMathematic":71040,"'":71042,"{j":71043,"_ABORT":71044,"Ether":71045,"Ġeducator":71046,"Ġprecaution":71047,"Ġfingertips":71048,"getVar":71049,"camatan":71050,"-debug":71051,"ĠRAF":71052,"[arg":71053,"Ġraced":71054,"Ġtsunami":71055,".flink":71056,"Ġglyc":71057,"uko":71058,"ĠMultiply":71059,"Ġredistribution":71060,"AGO":71061,"ĠRoutine":71062,"Ġopr":71063,"(lower":71064,"ĠFunktion":71065,".dk":71066,"Ġegt":71067,"_BASIC":71068,"syscall":71069,"ĠLSD":71070,"ĠDuplicate":71071,"_sell":71072,"ĠerrorHandler":71073,"_ips":71074,"Ġerv":71075,"annie":71076,"(resourceName":71077,"Ġbottled":71078,"Ġcrawling":71079,"egment":71080,".setTag":71081,"Ġrss":71082,"ĠQuarry":71083,"_exact":71084,".jwt":71085,"ĠBoards":71086,"opi":71087,"Ġnasal":71088,"ĠXYZ":71089,".ud":71090,"Northern":71091,"Ġactivating":71092,"edx":71093,"ovah":71094,"Ġindx":71095,"AlertDialog":71096,"Ġtienes":71097,"annya":71098,"_pan":71099,"(decimal":71100,".Dict":71101,"Ġsubsidiaries":71102,"ProductName":71103,"Few":71104,"dato":71105,"odied":71106,"-under":71107,"Ġê²ĥ":71108,"çīĪæľ¬":71109,"atism":71110,"[Math":71111,".'<":71112,"(infile":71113,"Ġdenotes":71114,"$class":71115,"_SECURITY":71116,"Ġsewage":71117,"melon":71118,"(Character":71119,"/github":71120,"Ġglaring":71121,".Guid":71122,"_sparse":71123,"ĠMargin":71124,"_dns":71125,"Ġmeiner":71126,"Ġleftist":71127,"ĉloc":71128,"abytes":71129,"Ġequipments":71130,"expo":71131,"ĠSomerset":71132,"EK":71133,"æį¢":71134,"Ġlecturer":71135,"Ġmemiliki":71136,"æł¸":71137,"ç´ł":71138,"pron":71139,":pointer":71140,"borrow":71141,"ĠProtective":71142,"_cf":71143,"ĠÐķÑģли":71144,"bpp":71145,"';ĊĊĊĊ":71146,"aturally":71147,"_NAV":71148,"Ġpeptide":71149,">d":71150,"Ġifstream":71151,"_FACTORY":71152,"');//":71153,"joined":71154,"mong":71155,"Ġtimespec":71156,"Ġdestabil":71157,"Ġautop":71158,"-limit":71159,"publication":71160,"ĠDenn":71161,".Memory":71162,"(skb":71163,"ĠAnaheim":71164,"_RETURNTRANSFER":71165,"oueur":71166,"(_('":71167,"legt":71168,"istingu":71169,"ĉpriv":71170,"Ġredirects":71171,"Mt":71172,"Ġalleen":71173,"ĠPointF":71174,"Ġomin":71175,"Ġcitt":71176,"ĠTage":71177,"ĠWalls":71178,"á»ī":71179,"Ġoccupying":71180,"xBF":71181,"rangle":71182,"Ġrelational":71183,"-org":71184,"Ġjpg":71185,"-derived":71186,"Ġmalfunction":71187,"ĠBenson":71188,"(scroll":71189,"ĠXD":71190,"Holy":71191,"(commands":71192,"Ġtipping":71193,"Ġprimitives":71194,"Ġsexle":71195,"CallCheck":71196,"ĠMASTER":71197,"_TEAM":71198,".setRequestHeader":71199,"_specs":71200,"Ġserge":71201,".Master":71202,"Ġims":71203,".SpringBootTest":71204,"paypal":71205,"ĠWANT":71206,".Inst":71207,"ĠCarpet":71208,"Ġwrongly":71209,"($('.":71210,"Ġbild":71211,".Roll":71212,"ĠUrb":71213,"-can":71214,"ãģıãģłãģķãģĦ":71215,"oliberal":71216,"čĊčĊ":71610,"ĠMahm":71611,"}\";ĊĊ":71612,"Ġdq":71613,"ĠPublishers":71614,"ĠAmpl":71615,"ĠDanielle":71616,"Ġtern":71617,"èµ·":71618,"noÅĽÄĩ":71619,"ein":71620,"ĠAsyncStorage":71621,"unger":71622,"rouw":71623,"Ġscissors":71624,"/assert":71625,".bucket":71626,"/archive":71627,"_Man":71628,"Ġintoler":71629,"Ġ()=>":71630,"ĠÐĴÑĭ":71631,"Ġsai":71632,".xy":71633,".\"čĊ":71634,"Ġurinary":71635,"esub":71636,"ISTICS":71637,"Ġκ":71638,"Ġcompliments":71639,"ĠtypingsJapgolly":71640,"ihar":71641,"Expansion":71642,"ĠServing":71643,"_students":71644,"ĠXBOOLE":71645,"(il":71646,"Ġì²ĺ":71647,"Ġjó":71648,"(tol":71649,"(JS":71650,"ĉCG":71651,"ĠDRAW":71652,"twig":71653,"Ġoat":71654,"_smooth":71655,"ĠCSL":71656,"Ġosob":71657,"Ġensuing":71658,"Ġbanker":71659,"ĠBackpack":71660,"_ping":71661,"Ġwishlist":71662,"=ax":71663,"ĉĠĠĠĊ":71664,"Disney":71665,"steady":71666,"\">%":71667,"Ġprophets":71668,"ĠZX":71669,"Ġminimalist":71670,".PLAIN":71671,"Seattle":71672,".ordinal":71673,"ĠPIPE":71674,"Ġretorna":71675,"Ġjugador":71676,"ĠBret":71677,"ĠâĶľ":71678,"Ġplush":71679,"ULATOR":71680,"Sorting":71681,".gridy":71682,"ectomy":71683,"_activ":71684,"rack":71685,"Interactive":71686,"ĠAntarctica":71687,"Ġvengeance":71688,"enso":71689,"_known":71690,"upplier":71691,".Modules":71692,"ĠConnectionState":71693,"éļIJèĹı":71694,"@FindBy":71695,"Ġplacer":71696,"\\model":71697,"<()>":71698,".isSuccessful":71699,"-good":71700,"bz":71701,"ĠDraco":71702,"Assistant":71703,"-extra":71704,"аблиÑĨ":71705,"Ġhypocrisy":71706,"Ġtst":71707,"ĠAgr":71708,"$txt":71709,"Ġlogistic":71710,"licensed":71711,"ĠHof":71712,"Ġtat":71713,"(iv":71714,"Ġintoxic":71715,"postId":71716,"_strike":71717,"Ġhumiliation":71718,"pcodes":71719,"\"sync":71720,"(recipe":71721,"+N":71722,"rente":71723,"ĉClient":71724,"ycopg":71725,"ĠZurich":71726,"ĠProfiles":71727,"Countries":71728,"Ġpict":71729,"Ġrollout":71730,"requencies":71731,"Ġpatched":71732,"Ġcartridges":71733,"Ġshading":71734,"Jar":71735,"Ġsalvage":71736,"ĠTaxes":71737,"Ġstandby":71738,"aporan":71739,"Eigen":71740,".angular":71741,"ĠNested":71742,"享":71743,"ĠisVisible":71744,"ĠDwight":71745,"_BRANCH":71746,".Delay":71747,"Ġkend":71748,"Ġfacilitated":71749,".flatMap":71750,"Ġsanta":71751,"ĉSend":71752,"/messages":71753,"ĠofType":71754,"ĉswap":71755,"#plt":71756,"ĠTurks":71757,"NES":71758,"Ġprogressively":71759,"ĠResidence":71760,"ĠTREE":71761,"Ġnoen":71762,"dio":71763,"Ġnelle":71764,"Ġsogar":71765,"itti":71766,"weekly":71767,"Ġambiguity":71768,"_Settings":71769,"Ware":71770,".neo":71771,"_DST":71772,"Ġæĸ¹":71773,"prep":71774,"lobby":71775,"@email":71776,"/movie":71777,"Ġfunkc":71778,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":71779,"ÂŃs":71780,"Ġguardians":71781,"-pos":71782,"Ġconfiguring":71783,"ĠCPS":71784,"ĠDeus":71785,"Ġvidéos":71786,"_empresa":71787,"Ġslapped":71788,"',Ċ":71820,"_XDECREF":71821,"ĠBuzzFeed":71822,"_MARGIN":71823,"PLOY":71824,".small":71825,"ĠmimeType":71826,"Ġholog":71827,"ĉcamera":71828,"lias":71829,"Ġsuspense":71830,"odynam":71831,"bau":71832,"Ġgraveyard":71833,"_named":71834,"\":\"'":71835,"Ġ************************************************":71836,"ĠgameOver":71837,"ĠLENGTH":71838,"ĉscreen":71839,"ĠdoInBackground":71840,"_dependencies":71841,"Ġrtc":71842,"/up":71843,"_ROM":71844,"Hall":71845,"Ġdeficiencies":71846,"(te":71847,"'#":71848,"_equiv":71849,"Ġpreorder":71850,"ĠAxe":71851,"омÑĥ":71852,".sendFile":71853,"Ġfilt":71854,"ĠLimits":71855,"ĠCavaliers":71856,".discount":71857,"âĨIJ":71858,"ĠWit":71859,"QRSTUV":71860,"Ġij":71861,"Ġtegen":71862,"Ġ:\",":71863,"difficulty":71864,"punkt":71865,"ĠEmails":71866,"chlor":71867,"(fun":71868,".Uint":71869,"ĠStall":71870,"_verified":71871,"uD":71872,"FileType":71873,"Ġpleasures":71874,"Ġjudiciary":71875,"Ġsham":71876,"ipur":71877,"_PLUS":71878,"offers":71879,"(foo":71880,"_GT":71881,"ĉcore":71882,"ENTION":71883,"ĠLiberation":71884,"CommandLine":71885,"_department":71886,".Ar":71887,"_neighbor":71888,"ĠSubmitted":71889,"ĠĊ":96121,"Ġdroits":96122,"Ġhomosexuals":96123,"Ġabduction":96124,"ĉwidget":96125,"$headers":96126,"ĠDAR":96127,"Ġfla":96128,"threat":96129,"Ġlouis":96130,".GetProperty":96131,"\"Just":96132,"(frames":96133,"ryo":96134,"profession":96135,"|i":96136,"íķ´ìĦľ":96137,"(sv":96138,"Ġunrecognized":96139,"Ionic":96140,"Fashion":96141,"ScreenState":96142,"ĠIncoming":96143,"NotNil":96144,"Ġsyncing":96145,"emie":96146,"Ġthermo":96147,"_procs":96148,"Ġinconsistency":96149,"religious":96150,".mj":96151,"Ġpersonn":96152,"Ġmomentos":96153,"orarily":96154,"ĠæĬ":96155,"_neurons":96156,"Illustr":96157,"imoto":96158,"ilik":96159,"ĠWoj":96160,"Trading":96161,"Ġappare":96162,"Ġentreprises":96163,"achat":96164,"Ġ¬":96165,"Ġneigh":96166,"BUTTONDOWN":96167,"ĠMaher":96168,"aghan":96169,"-hash":96170,"\"f":96171,"Ġclientele":96172,".addButton":96173,"ĉSP":96174,"Qi":96175,"Ġgrated":96176,"POSITE":96177,":>":96178,"ĠHowell":96179,"ĠComparative":96180,"ĠISC":96181,"ÂŃi":96182,"Ocean":96183,"Davis":96184,"ĠFilme":96185,"Wins":96186,"ĠJIT":96187,"occer":96188,"ĠCorm":96189,"ENCHMARK":96190,"rchive":96191,"icação":96192,"Ġmata":96193,"Ġchildbirth":96194,"ĠOptionally":96195,"Ens":96196,"Ġxhttp":96197,"Ġelucid":96198,"_OscInitStruct":96199,"))):Ċ":96200,"Ġintuit":96201,"ĠDonate":96202,"Ġcorrelates":96203,">Delete":96204,"Ġequipe":96205,"Ġboca":96206,"Ġinflatable":96207,"erah":96208,"ĠDateTimeKind":96209,"Ġcalves":96210,"\\Lib":96211,"Ġemlrt":96212,"ĠTrilogy":96213,"ĠPanc":96214,"ĠDuis":96215,"ĠpelÃŃcula":96216,"WARDS":96217,"_DETECT":96218,"-sectional":96219,"dhcp":96220,"ForRow":96221,"-destruct":96222,"ĠPresenter":96223,"/slick":96224,",on":96225,"ĠCitadel":96226,"loggedin":96227,"_subtype":96228,"Ġsigue":96229,"Ġcuring":96230,"ĠFirewall":96231,"Ġfluorescence":96232,"ĠItalians":96233,"иÑĤÑģÑı":96234,".getStyle":96235,"InSeconds":96236,"jie":96237,"-Smith":96238,"Ġxlink":96239,"Ġsubmissive":96240,"онÑĤ":96241,"arbonate":96242,"ĠFaul":96243,"_goals":96244,"ĠCommissioners":96245,"chartInstance":96246,"_POSTFIELDS":96247,"Ġmedial":96248,"Ġmanos":96249,"Ġdelt":96250,"svm":96251,".Apis":96252,"ephy":96253,"Ġasympt":96254,"ĠappDelegate":96255,"Ġimprobable":96256,"cka":96257,"simd":96258,"/Error":96259,".âĢĵ":96260,"ĠPTS":96261,"deer":96262,"Ġsina":96263,"magnitude":96264,"IDADE":96265,"']}'":96266,"Ġmayores":96267,"ĉcomment":96268,"/console":96269,"\"@":96270,"volt":96271,".sell":96272,"ĠMacy":96273,"Ġmelod":96274,"Ġimágenes":96275,"_chg":96276,"Ġinout":96277,"idente":96278,")'),Ċ":96279,"dni":96280,".blob":96281,"Ġtypography":96282,"Ġeerie":96283,"_OID":96284,"pesan":96285,"ajan":96286,"Ġchopping":96287,"Ġbluff":96288,"adf":96289,"_bases":96290,".Formatter":96291,"Ġ\\%":96292,"ĠPageInfo":96293,"Carrier":96294,"ĠCalibration":96295,"como":96296,"-bodied":96297,"Ġfinancier":96298,"ĠINA":96299,".ERR":96300,"Ġhoodie":96301,"ĠSanity":96302,"guarded":96303,".opendaylight":96304,"ISMATCH":96305,"Highlights":96306,"ünk":96307,"aniem":96308,"angered":96309,"assignments":96310,"Ġregistrado":96311,"ĠUPPER":96312,"ampilkan":96313,"ashire":96314,"ĠNikola":96315,"ĠCFL":96316,"ĠHDC":96317,"Ġpoids":96318,"ĠIPs":96319,"Ġpreventative":96320,"ipsoid":96321,"ifix":96322,".camel":96323,".ga":96324,"Volumes":96325,"-ste":96326,"Yahoo":96327,"_sibling":96328,"Highest":96329,"optgroup":96330,"Ġkvinna":96331,"âĢĿãĢĤĊĊ":96332,"ĠAppliances":96333,"Ġ\"><":96334,"')\")Ċ":96335,"htt":96336,"ĠIdentified":96337,"Ġpencils":96338,"ĠmemberId":96339,"ĠappendString":96340,".loadData":96341,"ĠmockMvc":96342,"Ġjub":96343,"ĠSlut":96344,"ĠTaipei":96345,"statt":96346,"Polit":96347,"Ġpartager":96348,"DidChange":96349,"Increases":96350,")}.":96351,"ĠBaba":96352,"_CLIP":96353,"[unit":96354,"ĠклÑİÑĩ":96355,"Ġalcuni":96356,"ĠLola":96357,"Ġclinging":96358,"@PostMapping":96359,"(concat":96360,"Ġssid":96361,"ĠFauc":96362,"okit":96363,"ĠRecorded":96364,"ález":96365,"($('<":96366,".assertIsNot":96367,"Ġkali":96368,"Volt":96369,"Ġwarmly":96370,"Ġscares":96371,"getti":96372,"führt":96373,"_does":96374,".EMAIL":96375,"imations":96376,"Ġspringfox":96377,"ĠDecom":96378,"arcy":96379,"Ġglitches":96380,"ĠMoff":96381,"ĠVoll":96382,".between":96383,"Ġcoorden":96384,"ĠParticularly":96385,"GBP":96386,"Ġsemble":96387,"Eastern":96388,"_MSB":96389,"]){čĊ":96390,"morgan":96391,"ĠEVAL":96392,"dere":96393,"HOUSE":96394,"moire":96395,"istique":96396,"_lstm":96397,"-commit":96398,"ysterious":96399,"Ġtwink":96400,"-thumbnails":96401,"enÃŃ":96402,":'',":96403,"Ġblackout":96404,"ĠFloors":96405,"Ġsofas":96406,"Ġoui":96407,"leshoot":96408,"ĠRaq":96409,"-abs":96410,"Ġkra":96411,"Mining":96412,"shaft":96413,".setColumns":96414,"Clazz":96415,"PRETTY":96416,".playlist":96417,"éĸ¢":96418,"-Saharan":96419,"MING":96420,"ĉbl":96421,"è®®":96422,"jf":96423,"DOCKER":96424,"hopefully":96425,"(ignore":96426,"ĠUsersController":96427,"ĠMitarbeiter":96428,"ĠLES":96429,"Hamilton":96430,"-metadata":96431,"ĠKK":96432,"iktig":96433,"Ġwollte":96434,"egrator":96435,"]bool":96436,",current":96437,"ĠvalueType":96438,"Ġexcavation":96439,"oland":96440,"Ġverv":96441,"/filepath":96442,"AuthProvider":96443,"Ġprocrast":96444,"ĉULONG":96445,"_MEMBERS":96446,"Ġuplift":96447,"ĠAutonomous":96448,"Ġartworks":96449,"ĠOutreach":96450,"Ġpore":96451,"Homepage":96452,"DialogTitle":96453,"ĠGenerating":96454,"PARSE":96455,"Ġsemanas":96456,"Ġhumano":96457,"JSGlobalScope":96458,"Ġvolte":96459,"Ġbella":96460,"(isinstance":96461,"Ġplc":96462,"\\Catalog":96463,"Ġesteemed":96464,"鼷":96465,"(suffix":96466,"Ġsweeps":96467,"ĉORDER":96468,"Ġdoivent":96469,"ĠSwarm":96470,"ĠCompiled":96471,"getPage":96472,"ADR":96473,".RichTextBox":96474,"ĠNaming":96475,"agged":96476,"ĠGANG":96477,"rasing":96478,"odeled":96479,"Ġgala":96480,"ĠJSName":96481,"ddf":96482,"Ġillust":96483,"ĠLansing":96484,"[port":96485,"-death":96486,"Ġdinheiro":96487,"ĠEighth":96488,"Ġbian":96489,"stÃ¥":96490,"Ġversión":96491,"ĠLinearGradient":96492,"ĠHarding":96493,".*)":96494,"eczy":96495,"$header":96496,"ĠvÃ¥r":96497,"Unchecked":96498,"Ġkoje":96499,"ĠPaladin":96500,"())),":96501,"Giving":96502,"()})Ċ":96503,"Ġdips":96504,"Friendly":96505,"Ġportrays":96506,"Ġhelium":96507,"Ġinsurgency":96508,"_expiry":96509,"ĠstringByAppendingString":96510,"Ġaantal":96511,"slope":96512,"mast":96513,".getInteger":96514,"Ġ########################":96515,"_PIPELINE":96516,"Ġdensely":96517,"Ġmutating":96518,"midi":96519,"ĠSeit":96520,"ayne":96521,"NOWLED":96522,"ĠDesmond":96523,"ĠFName":96524,"ĠNairobi":96525,"\\Context":96526,"Ġcalcular":96527,"-den":96528,"Ġcott":96529,"]):čĊ":96530,"ĠRecommendation":96531,"ĠRolex":96532,"ĠvalidationResult":96533,".pat":96534,"ĠnÃły":96535,"ĠRestClient":96536,"ĠGPI":96537,"ĠAsheville":96538,"ĠOSP":96539,"ĠPERMISSION":96540,"ÐĶаÑĤа":96541,"/notification":96542,"Knight":96543,"_Word":96544,"ĠBender":96545,"ranking":96546,"Ġpartida":96547,"_reservation":96548,"ÌĢ":96549,"ĠmName":96550,"Ġgetch":96551,"Ġborr":96552,"Ġdiligent":96553,"Discuss":96554,"æŃ£åľ¨":96555,"apeake":96556,"ioned":96557,"-Nazi":96558,".cum":96559,"ĠKron":96560,"=$('#":96561,"/single":96562,"Ġerotisch":96563,"ĠVib":96564,"Ġratified":96565,"Ġconcerted":96566,"ĠREGARD":96567,"Ġdobr":96568,".DriverManager":96569,"'r":96570,"Portable":96571,"ĉsuite":96572,"Ġrelaciones":96573,"ĠDop":96574,"emploi":96575,"DOB":96576,"Ġcrumbs":96577,"Ġxls":96578,"_Application":96579,"(':',":96580,"Ġ------------------------------------------------------------------------Ċ":96581,"mse":96582,"Ġberk":96583,"ĠReturnValue":96584,"ĠBelly":96585,"Ġcamar":96586,"ĠPeek":96587,"elsing":96588,"Ġnotifies":96589,"ĠTristan":96590,"ĠGAR":96591,"emme":96592,"ĠElevated":96593,"_CSV":96594,"(chalk":96595,"Ġtwenties":96596,"ĠSearchResult":96597,"=search":96598,"ĠMixing":96599,"ýt":96600,"Ġrecruiter":96601,"ĠIDEOGRAPH":96602,"ĠAgo":96603,"(Operation":96604,"$values":96605,"Ġworldly":96606,"ĠRosenberg":96607,"ĠConfigureServices":96608,">*Ċ":96705,"Ġsnork":96706,"_opacity":96707,"ĠinitWithNibName":96708,"iado":96709,"AAC":96710,"Ġ]).":96711,";z":96712,"_paragraph":96713,"Ġnoses":96714,"stands":96715,"ifr":96716,"_mE":96717,"Iraq":96718,".Predicate":96719,"enaire":96720,"]]];Ċ":96721,"Ġunidad":96722,"Ġretirees":96723,"_hello":96724,"Ġmodele":96725,"ĠUITableViewController":96726,"fwrite":96727,"_numero":96728,"_visited":96729,"Ġrecebe":96730,"(Notification":96731,"Fantastic":96732,"_submenu":96733,"ĠPEM":96734,"ĠCupertino":96735,"approximately":96736,"classed":96737,".ReadString":96738,"Ġdomicile":96739,"_PW":96740,"Ġballpark":96741,"ĠKale":96742,"contra":96743,"_favorite":96744,"/of":96745,"Quite":96746,"ĠOTA":96747,"Ġaccelerometer":96748,"didn":96749,"|^":96750,"ĠRohingya":96751,"ivicrm":96752,"annabin":96753,"обÑĭÑĤи":96754,"orado":96755,"')+":96756,"Haunted":96757,",ID":96758,"(UIAlertAction":96759,"urv":96760,"_bel":96761,"ĠMexicans":96762,"/terms":96763,"ĠPainter":96764,"InputLabel":96765,"ĠVinci":96766,"ĠRosie":96767,"\\uc":96768,"":96929,"_gs":96930,"Ġcompil":96931,"nard":96932,"-exc":96933,"Ġrhyme":96934,"Ġbutto":96935,"says":96936,"antasy":96937,"ë¸":96938,"ĠcittÃł":96939,"Ġcheg":96940,"TimeString":96941,"Ġpositivity":96942,"ĠDabei":96943,"Ġwang":96944,"Ġescre":96945,"\"c":96946,"ĉvideo":96947,"ĠRanked":96948,".strings":96949,">>>(":96950,"ĠинÑĤеÑĢ":96951,"Ġresta":96952,"[:,:":96953,"Ġrendre":96954,"Ġdeser":96955,"Jos":96956,"Ġdisruptions":96957,"ĠопеÑĢ":96958,"sampling":96959,"suppress":96960,"ĠcontainerView":96961,"ĠSeamless":96962,"Ġairy":96963,"Ġonload":96964,".WindowManager":96965,"ĠPLA":96966,"braco":96967,".setPositiveButton":96968,"Ġpdu":96969,"Ġgsi":96970,"ĠCli":96971,"_gradients":96972,"Ñıд":96973,"ĠWhisper":96974,"cstdint":96975,"Ġläng":96976,"Ġformulations":96977,"énom":96978,"ournemouth":96979,"[$_":96980,"Ġordinarily":96981,".setUsername":96982,"Ġfaculties":96983,"MITTED":96984,"/values":96985,"Ġweir":96986,"ĠApt":96987,"MZ":96988,"ĉcf":96989,"ucken":96990,"ĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉ":96991,"defense":96992,"[iVar":96993,"ĠBusinessException":96994,"Selectors":96995,"(coordinates":96996,"ĠResets":96997,"ĠDrinks":96998,"oleans":96999,"(stypy":97000,"_IOC":97001,".xxx":97002,"ĠSlater":97003,"ĠBelize":97004,"Ġ/************************************************************************":97005,"addin":97006,"_episodes":97007,"Ġischem":97008,"legalArgumentException":97009,"Danny":97010,"Ġpared":97011,".codehaus":97012,"ĠAssy":97013,"ĉRect":97014,"âŀ":97015,".lista":97016,"ĠваÑĪ":97017,"Ġvets":97018,"HWND":97019,"isoner":97020,"Ġxo":97021,"Ġorally":97022,"ĠStmt":97023,".rnn":97024,"ĠDPI":97025,"ĠStrikes":97026,".setViewportView":97027,"ĠèĩªåĬ¨çĶŁæĪIJ":97028,"YELLOW":97029,"GLenum":97030,"partners":97031,"ĠImplicit":97032,"Ġtako":97033,"âĢĻelle":97034,"Ġermög":97035,"totalCount":97036,"Gil":97037,"ĉwork":97038,"Ġpratic":97039,"inati":97040,"abies":97041,"ĠSkinner":97042,"Ġspirited":97043,"Ġpancreatic":97044,"Ġhdf":97045,"'em":97046,"Ġpsychosis":97047,"olicit":97048,"Ġ\"{\"":97049,"_atual":97050,"Ġélect":97051,"TEAM":97052,"Ġdak":97053,"ĠSWAT":97054,".FragmentManager":97055,"Ġprovisioning":97056,"lifetime":97057,"_EXTENSIONS":97058,"ĠCASCADE":97059,"Ġ![":97060,"(KP":97061,"Ġvem":97062,"ĠInterracial":97063,"']},Ċ":97064,"spacer":97065,"_kv":97066,"Warehouse":97067,"RDD":97068,"_fsm":97069,".StretchImage":97070,",Yes":97071,"ĠRefugee":97072,"ĠBringing":97073,"Ġválido":97074,".intersection":97075,"Ġspooky":97076,"_portal":97077,"Ġmoth":97078,"ĠZodiac":97079,"ĠSOCIAL":97080,"MimeType":97081,"']}}":97200,"_Blue":97201,"Ġbotanical":97202,"Ġfrags":97203,"Ġfamilial":97204,"-du":97205,"Ġseizing":97206,"(blocks":97207,".rd":97208,".checkNotNull":97209,"Ġmiser":97210,"Ġmaxx":97211,"ĠKnee":97212,"ViewItem":97213,"InnerHTML":97214,"Danger":97215,"((__":97216,"Ġprzypad":97217,"createUrl":97218,"**,":97219,"ĠDecorating":97220,"ATEGY":97221,"?>/":97222,".Designer":97223,"hexdigest":97224,"ĠEverywhere":97225,"alleries":97226,".TEXTURE":97227,".Blocks":97228,"zell":97229,"Ġpreço":97230,"Suddenly":97231,"inputEmail":97232,"(sync":97233,".bd":97234,"golden":97235,">');":97236,"ĠDickinson":97237,">>(Ċ":97238,"ĠQUEUE":97239,"ĠgetColumn":97240,"ĠSAND":97241,".piece":97242,"licer":97243,"Flutter":97244,"ĠgetVersion":97245,"ĠresourceId":97246,"ogl":97247,"ÅĤaw":97248,".Branch":97249,"ĉweb":97250,"Ġframerate":97251,"PPP":97252,"Ġfray":97253,"CNT":97254,"Ġinformatie":97255,"']čĊčĊ":97256,"neas":97257,"HeaderCode":97258,"Ġæ¸":97259,"Ġtrg":97260,"rawtypes":97261,"Honda":97262,"Ġmarketer":97263,"ĠrequestData":97264,"ĠPg":97265,"ĉnot":97266,"ĠpageInfo":97267,"Ġaktuellen":97268,"ãģķãĤĵ":97269,"ĠAMS":97270,"pushViewController":97271,"ĉAL":97272,"Ġvests":97273,"produce":97274,"-même":97275,"ĠRahman":97276,"Funny":97277,"EZ":97278,"_Valid":97279,"Ġsquadron":97280,"Ġlash":97281,"Ġirm":97282,"iasco":97283,"ĠParan":97284,"Ġpetites":97285,"ĠDecay":97286,"Ġuninitialized":97287,"privileged":97288,"Ġmbedtls":97289,"å¤ĩ注":97290,"Ġ^.":97291,"Ġecstatic":97292,"Detroit":97293,"Ġparten":97294,"Ġsouvenir":97295,".getLogin":97296,"моÑĤÑĢ":97297,"enção":97298,"ĠmÃŃnimo":97299,"ĠAccessed":97300,"rió":97301,"Mic":97302,"ĠVocal":97303,".SetString":97304,"Ġmensajes":97305,"åĢį":97306,"Ġattravers":97307,"ĠAph":97308,"Ġ');čĊ":97309,"ünde":97310,"Ġenchanted":97311,"ĠRootState":97312,"ĠCLOSED":97313,"ĉĉĉĉĉĉĉĉčĊ":97314,"Ġcaliente":97315,"orris":97316,"Ġphysicists":97317,"hwnd":97318,"_vi":97319,"Ġrápido":97320,"Ġcapitalized":97321,"edBy":97322,"Ġmachining":97323,"Ġhubby":97324,"ĠStacy":97325,".Bus":97326,"drink":97327,"Hur":97328,"Ġpropia":97329,"UnitTest":97330,"Ġmisconception":97331,"__));Ċ":97332,"/dc":97333,"ĠMayweather":97334,"_mC":97335,".createFrom":97336,"ĠQPainter":97337,"ropsych":97338,"innitus":97339,"ayas":97340,"Ġgeg":97341,"(dw":97342,"Ġusado":97343,"Ġtrickle":97344,"Ġannihil":97345,"ĠPasta":97346,"Ġ++Ċ":97347,"(ExpectedConditions":97348,".postValue":97349,"icap":97350,"ĠDonetsk":97351,"_soup":97352,"-publish":97353,"ĠPb":97354,"mentions":97355,"ACCEPT":97356,".Pull":97357,",âĢĻâĢĻ":97358,"Ġretarded":97359,"_ATOM":97360,"ĠTerminator":97361,"-court":97362,"ĠCLLocationCoordinate":97363,"Ġreverence":97364,"ĠSSC":97365,"utely":97366,"ĠWON":97367,"ĠGSL":97368,"frei":97369,".getLongitude":97370,"ĠopenFileDialog":97371,".Butter":97372,"-important":97373,"_MANY":97374,"ĠGong":97375,"âĢľHow":97376,"Ġgorge":97377,"=msg":97378,"ĠEzek":97379,"createCommand":97380,":checked":97381,"Ġinfographic":97382,".WEST":97383,"Dirs":97384,"Ġguarda":97385,"Ġbeetle":97386,"Loading":97460,"_mA":97461,".getRandom":97462,"blings":97463,"Ġcheeses":97464,"tti":97465,".âĢ¢":97466,"ĠBurgess":97467,"enderit":97468,".',čĊ":97469,"(\"\"+":97470,"acb":97471,"%p":97472,"indexed":97473,"_predicate":97474,"nesia":97475,"Ġbied":97476,"ĠCIT":97477,"(Pos":97478,"_radi":97479,"ä»·æł¼":97480,"Biz":97481,"ĠAdolescent":97482,"Ġviên":97483,"cycl":97484,"_Cancel":97485,"Ġconclusive":97486,"Ġappellate":97487,"informatics":97488,"SJ":97489,"Ġelective":97490,"roleId":97491,"Fetcher":97492,"ĉCommand":97493,"(\"(%":97494,"Ġfart":97495,"ILA":97496,"getBlock":97497,"AUSE":97498,"Ġдан":97499,"ĠArte":97500,"Ġnotifying":97501,"Ġgele":97502,".same":97503,"ĠRegel":97504,"ĠBaÅŁ":97505,".creation":97506,"ĠVN":97507,"_community":97508,"Ġunsustainable":97509,"SEX":97510,"ĠgridSize":97511,"rescia":97512,"aversable":97513,"(',')[":97514,"ĠPhelps":97515,"á»ķi":97516,"ANCELED":97517,"-IS":97518,".runners":97519,"ĠStokes":97520,".Produ":97521,"Ġwhipping":97522,"_acquire":97523,"Ġinvestigación":97524,"fried":97525,".copyWith":97526,"ĠHardcover":97527,"-Se":97528,"áŀ¶áŀ":97529,"invitation":97530,"lesai":97531,"ĠDorm":97532,"ĠÑģпиÑģка":97533,"Ġconcatenated":97534,"ophil":97535,"Ġthinker":97536,"/fontawesome":97537,"ĠLeopard":97538,"Ġ\"/\");Ċ":97539,"Ġresiduals":97540,"ĠMicrowave":97541,"Ġconforme":97542,"throp":97543,"Ġdisemb":97544,"ĠOMG":97545,"ĠDiscipline":97546,"ĠAcrobat":97547,"/repository":97548,"dfa":97549,"_MED":97550,"bufio":97551,"Ġméthode":97552,"_HOLD":97553,"iasi":97554,"_legacy":97555,")ččĊ":97556,"æ£Ģ":97557,"GetProcAddress":97558,"Ġyay":97559,"otence":97560,"orderid":97561,"-tw":97562,"Ġdearly":97563,"Incoming":97564,"/il":97565,"Ġneurop":97566,"ucz":97567,");čččĊ":97568,"ĠInnovative":97569,"Ġprofund":97570,"igmat":97571,"SelectionMode":97572,"relevant":97573,".GO":97574,"Ġbruises":97575,"Ġsach":97576,"odef":97577,"Ġreimb":97578,"/desktop":97579,"-spot":97580,"undance":97581,"Entropy":97582,"\\core":97583,"Ġsuger":97584,"ĠMvc":97585,"ĠGNOME":97586,"_indx":97587,"ĠYYSTYPE":97588,"ĠMatlab":97589,"ĠCIF":97590,"Ġ*))":97591,"ĠproductList":97592,"ĠAlright":97593,"acemark":97594,"ÑĤив":97595,"modification":97596,"international":97597,"Ġhomers":97598,"Ġdicts":97599,"ĠQFont":97600,".SQLite":97601,"Ġtransplantation":97602,"ĠMessageBoxButton":97603,"ĠElves":97604,"']])Ċ":97605,"(QIcon":97606,"Ġcinemas":97607,"COORD":97608,"-China":97609,"Ġkhẩu":97610,"æĪijçļĦ":97611,"Ġskulls":97612,"Ġpainstaking":97613,"fce":97614,".XRLabel":97615,"Ġspecifier":97616,"Ġpreferring":97617,"/activity":97618,"(Photo":97619,"ált":97620,".lot":97621,"''.":97622,"annonce":97623,".googlecode":97624,"-pdf":97625,"ĠPoke":97626,"_ACL":97627,"Ġendowed":97628,"discover":97629,".omg":97630,"Ġwoodland":97631,".Magic":97632,"Ġvolont":97633,"NotAllowed":97634,"Ġchave":97635,"BMW":97636,"','=',":97637,"ĠSIX":97638,"æĪij们":97639,"Ġkosher":97640,"Ġaspiration":97641,"intl":97642,"_refptr":97643,"'+Ċ":97644,"mentor":97645,".club":97646,"WindowState":97647,".ARR":97648,"Ġzza":97649,"ĠmessageType":97650,".equ":97651,"Thor":97652,"Ġinjust":97653,"Ġgums":97654,"ĠborderSide":97655,"/////":97656,"ĠTransmit":97657,"Ġbufsize":97658,"Ġhak":97659,"Ġellas":97660,"RANDOM":97661,"ĉmc":97662,"Ġpea":97663,"eko":97664,"documento":97665,"Ġhysteria":97666,"Ġarenas":97667,"Ġgunmen":97668,"Ġmike":97669,"Ġimpunity":97670,"atisation":97671,"_Zero":97672,"_COMPANY":97673,"ĠGors":97674,"ĠuseClass":97675,"(redis":97676,"ĠRUNNING":97677,"ĠBair":97678,"velte":97679,"Ġ','.":97680,"аÑĤÑĮÑģÑı":97681,"öst":97682,"encodeURIComponent":97683,"_restrict":97684,"Ġdecals":97685,"ĠPedido":97686,"Ġaltercation":97687,"Displays":97688,"ĠApplicants":97689,"CUS":97690,"Textarea":97691,"ĠAngola":97692,".future":97693,"ĠUSHORT":97694,"Ġsuppressing":97695,"Ġsetzen":97696,"APolynomial":97697,"Ġtoch":97698,"Ġhallmark":97699,"Ġ$$$":97700,"ĠCHARSET":97701,".rpm":97702,"ĠDich":97703,"--------------------":97704,"_parm":97705,"è¿ĺ":97706,"acciones":97707,"hait":97708,"WARDED":97709,"_routing":97710,"ĠNOM":97711,"Ġenclave":97712,"ĠLotto":97713,"ĉfr":97714,"complexContent":97715,"ĠBallard":97716,"kube":97717,"/win":97718,".getColumnModel":97719,"_REPLACE":97720,"HeaderValue":97721,"Ġestudiantes":97722,"Ġapis":97723,"Ġbpm":97724,"ĠTypeName":97725,"AndGet":97726,"rita":97727,"Plans":97728,">Note":97729,"Ġfetisch":97730,"Ġtoned":97731,"_goto":97732,"onsense":97733,"Ġmolds":97734,"Ġinfiltration":97735,"ĠGuerrero":97736,"ubbo":97737,"cki":97738,"($(\".":97739,"_activities":97740,"(changes":97741,"ĠofApp":97742,"ĠKepler":97743,"ĠDemp":97744,"ĠContinent":97745,".Ticks":97746,"ĠUnsigned":97747,"ĠJahres":97748,"Ġfreshmen":97749,"ĠArchived":97750,"ĠкоÑĤоÑĢÑĭй":97751,"Ġ'::":97752,"Tutorial":97753,"Cc":97754,"ĠtableLayoutPanel":97755,"fromJson":97756,".levels":97757,"_transient":97758,"Ġendorsing":97759,"ĠDIC":97760,"lauf":97761,"Ġshred":97762,"_EMIT":97763,"ificantly":97764,"ALA":97765,"/proto":97766,"Ġnarrowing":97767,"Utc":97768,"Factors":97769,"Ġsentient":97770,"æŀIJ":97771,"lixir":97772,"ĠCROSS":97773,"meteor":97774,"Ġgroin":97775,"Ġmdb":97776,"ĠRotterdam":97777,"Ġcomida":97778,"ĠOpCode":97779,"ĠDefaultValue":97780,"PermissionsResult":97781,"Ġheterogeneous":97782,"Ġmoot":97783,"Ġdeceived":97784,"-independent":97785,"ĠObjectOutputStream":97786,"Ġoverpower":97787,".dup":97788,"Ġldb":97789,"Ġdomestically":97790,"Ġbestellen":97791,"Ġlov":97792,"ĠContractors":97793,"Triangles":97794,"Ġfodder":97795,"Ġfilmes":97796,"ä¼ģ":97797,"Ġrevolver":97798,"StartupScript":97799,"/validation":97800,"ĠResourceType":97801,"iÅŁ":97802,"ĠLaz":97803,"fef":97804,"Ġlstm":97805,"{*":97806,".attachment":97807,".hits":97808,"ewith":97809,"DOG":97810,"Alabama":97811,"Ġmediums":97812,".mContext":97813,"-cols":97814,"åıĭ":97815,".notice":97816,"Ġattn":97817,"ĠPacking":97818,"ĠLn":97819,"_COMPLEX":97820,"/Users":97821,".savetxt":97822,"ĠRounds":97823,"?,?,?,?,":97824,"Ġingl":97825,"ĠROC":97826,"_female":97827,"ĠStard":97828,"]];":97829,"Ġwrestlers":97830,"Ġtorrents":97831,"Ġsinh":97832,"ĊĊ":97833,"ë³µ":97834,"sense":97835,"however":97836,".Physics":97837,"Infrastructure":97838,"ĠSacr":97839,"Fel":97840,"ĠDISTRIBUT":97841,"éments":97842,"ĠValidates":97843,"############################################################":97844,"Ġ|/":97845,"Ġesl":97846,"Ġréseau":97847,"ĠBip":97848,"BYTES":97849,"_WATER":97850,"Turning":97851,"ELS":97852,"Ġjuxtap":97853,"Ġlesbische":97854,"ých":97855,"(Unknown":97856,"Neo":97857,"@JsonProperty":97858,"Ġalumnos":97859,"ĠRaqqa":97860,"imei":97861,".getBounds":97862,".MouseEventHandler":97863,"#######":97864,"GenericType":97865,"/cms":97866,"Ġturno":97867,"Ġмин":97868,"Ġfolklore":97869,"ĠEvo":97870,"Ġconductivity":97871,"Ġleben":97872,"Ġgearbox":97873,"-vs":97874,"ĠÏĨ":97875,"Ġdrinkers":97876,"Ġconexao":97877,"ĠTeeth":97878,"ĠgetArguments":97879,"ĠRAT":97880,"entious":97881,"Educ":97882,"+W":97883,"ĠInstitutional":97884,"ĠBord":97885,"isEqual":97886,"(pwd":97887,"Ġignited":97888,"ĠRousse":97889,"Ġimpactful":97890,"ĠMalk":97891,"Ġgeral":97892,"ĠPivot":97893,"Ġazt":97894,"Ġcsvfile":97895,"ĠRope":97896,"ĠSOLUTION":97897,"ĠArbitrary":97898,"Ġletto":97899,".MouseAdapter":97900,"Ġ}}}":97901,"ĠSailor":97902,"dera":97903,"Putting":97904,"Ġconcentrates":97905,"ĠauthDomain":97906,"âĢĿçļĦ":97907,"-finals":97908,",strlen":97909,"Muon":97910,"ĠOrdinary":97911,"firefox":97912,"ĠLaTeX":97913,"ĠHund":97914,"engineering":97915,"/blue":97916,"edTextBox":97917,"(\"\");":97918,"ĠCDDL":97919,"kept":97920,"ĠGetString":97921,"Kir":97922,"()='":97923,"ĠOCD":97924,"antium":97925,"$menu":97926,"ĠAppalachian":97927,"Secretary":97928,"ë¥ĺ":97929,"ีย":97930,"Semantic":97931,"Ġ*[":97932,"estone":97933,"ungkin":97934,"MaxY":97935,"-tone":97936,"\"};čĊ":97937,"_Part":97938,"ĊĊ":98140,"Lic":98141,"ĠMirage":98142,"ĠAssemblyFileVersion":98143,"TeV":98144,"ĠValueEventListener":98145,"-solving":98146,"Tho":98147,"roulette":98148,"_WP":98149,"Ġuninterrupted":98150,"ĠfieldType":98151,".Typed":98152,"Ġamour":98153,"Ġmockery":98154,"(vol":98155,"ĠSubcommittee":98156,"ĠRuf":98157,"erox":98158,":UIButtonTypeCustom":98159,"ĠBlur":98160,"Ġwykon":98161,"nces":98162,"ASHBOARD":98163,"!!\");Ċ":98164,"Ġmurderers":98165,".daily":98166,"ĠDIAG":98167,"jing":98168,"Ġdolphin":98169,"Ġlòng":98170,"Ġbö":98171,"ĠVocabulary":98172,".StObject":98173,"')\">":98174,"Ġzun":98175,"Ġscrimmage":98176,"tréal":98177,"ĠLig":98178,"[vi":98179,"Cole":98180,"Ġfrosting":98181,".Players":98182,"-translate":98183,"Feels":98184,"=\\\"/":98185,".ButterKnife":98186,"Ġ?>;Ċ":98187,"Ġavi":98188,"innie":98189,".Failure":98190,"Ġspindle":98191,"ConfigurationException":98192,"_hop":98193,"Ġposição":98194,"ĠAwait":98195,"UIImagePickerController":98196,"ĉday":98197,"Ġgenom":98198,"Cab":98199,"ĠÑĢезÑĥлÑĮÑĤаÑĤ":98200,"ORIGINAL":98201,"Ġejaculation":98202,"(tcp":98203,"SECOND":98204,"Ġtonic":98205,"ĠListBox":98206,"ĠĉĉĊ":98207,"()>Ċ":98208,"Ġquatre":98209,"ượng":98210,"withErrors":98211,".Maybe":98212,",â̦":98213,"tokenId":98214,"_UNDEF":98215,"Ġfreshness":98216,"ĠAmendments":98217,".mapbox":98218,".CV":98219,"(blog":98220,"_gettime":98221,".quest":98222,"sparse":98223,"Ġresale":98224,"Ġenthusiastically":98225,"ĠProstitutas":98226,"Wa":98227,"Cargo":98228,".Parcelable":98229,"SENSOR":98230,"ĠRyu":98231,"Laughs":98232,"_Native":98233,"/pg":98234,"ysts":98235,"Ġphotoc":98236,"ç®Ģ":98237,"adopt":98238,".species":98239,"conciliation":98240,"Adjusted":98241,".FirebaseAuth":98242,"uttle":98243,"ordination":98244,"Ġmunch":98245,"ĠStake":98246,".ping":98247,"anker":98248,"(QStringLiteral":98249,"Ġsubscript":98250,"ĠĠĉĊ":98251,"ĠMCC":98252,"_Cmd":98253,"sexy":98254,"iou":98255,"ĠMANY":98256,"Ġnanny":98257,"TRAIN":98258,"Ġflourishing":98259,"ĠWatches":98260,"ĠQMap":98261,"ĠFerm":98262,"Ġwasm":98263,"ĠAbed":98264,"_UD":98265,"ĠGlasses":98266,"+v":98267,"Attend":98268,".Chain":98269,"Ġdecency":98270,"ĠSupplementary":98271,"hunter":98272,"-txt":98273,"Ġ\"}\";Ċ":98274,".setWindowTitle":98275,"(\"":98377,"Ġmascara":98378,"(Profile":98379,"åĬŁèĥ½":98380,"imité":98381,"Ġwildfires":98382,"-ROM":98383,".isOn":98384,"(groupId":98385,"Repair":98386,"accumulate":98387,"Ġ<\",":98388,"Ġhandwritten":98389,"Ġacheter":98390,"ĠMGM":98391,"ĠIrma":98392,"->{_":98393,"gee":98394,"criminal":98395,"Ġèĭ¥è¦ģ":98396,"Ġmomentarily":98397,"\")!=":98398,"_lit":98399,"ĠexpiresIn":98400,".\").":98401,"éķ¿åº¦":98402,"Ġfrække":98403,"vlc":98404,"Ġorbs":98405,"),$":98406,"Ġventured":98407,"/>\\":98408,"charm":98409,"Nuitka":98410,"eldig":98411,"atonin":98412,"Witness":98413,"-lat":98414,"ĠsetHidden":98415,"Ġrelics":98416,"Ġconsulate":98417,".IGNORE":98418,"\"After":98419,"ĠsetAddress":98420,"Ġbesteht":98421,"Ġ'')ĊĊ":98422,".xaxis":98423,"Ġserão":98424,"Ġmisled":98425,"_UNIFORM":98426,"ĠVIA":98427,"incr":98428,"Ġzenith":98429,"Ġviscosity":98430,"Ġthinly":98431,".getSharedPreferences":98432,".ErrorCode":98433,"\"),\"":98434,"ĠMillionen":98435,"Ġ/>)Ċ":98436,"ScrollIndicator":98437,"-seeking":98438,"ĠPOLITICO":98439,"asca":98440,"_rl":98441,"Navig":98442,"(fullfile":98443,"Ġsolitude":98444,"Ġjuven":98445,"Ġhauling":98446,"ĠMacros":98447,"ĠGry":98448,"Ġexercitation":98449,"ĠATTACK":98450,"TickCount":98451,"Ġrites":98452,"Ġdoe":98453,"ParticleSystem":98454,"Ġslu":98455,"WindowText":98456,"ĠClassName":98457,"Ġslander":98458,"ĉPort":98459,"jong":98460,"?a":98461,".Dial":98462,"âĢĶat":98463,"$objPHPExcel":98464,"Ġsoar":98465,"ENN":98466,"appeared":98467,"Ġquotid":98468,"emachine":98469,"Ġnip":98470,"Ġmicrotime":98471,"ĠAlma":98472,";!":98473,"------------------------------------------------------------------------------------------------":98474,"ĠPassage":98475,"Ġdumpsters":98476,"ĠExclude":98477,"Ġsuggestive":98478,"ĠCircularProgressIndicator":98479,"_clr":98480,"ArrayType":98481,"ILLA":98482,"ElapsedTime":98483,"Driven":98484,"ĠresourceName":98485,"ĠGarrison":98486,"serir":98487,"-ahead":98488,"Ġpinnacle":98489,"ĠEspresso":98490,"Sparse":98491,"Ġassays":98492,"ĠGirlfriend":98493,"imid":98494,"]='\\":98495,"ONGLONG":98496,"Ġportraying":98497,"Lane":98498,"Ġbúsqueda":98499,"Ġreinforcements":98500,"ĠSpreadsheet":98501,"ĠArrayCollection":98502,",arr":98503,"lightbox":98504,"icana":98505,"<\"":98506,"builders":98507,"Kid":98508,"ĠMatSnackBar":98509,"EXPR":98510,"odcast":98511,"ĠFoundations":98512,"Ġinds":98513,"='${":98514,"Fizz":98515,"-functional":98516,"(workspace":98517,"Ġstemmed":98518,"_patches":98519,"ĠJarvis":98520,"READING":98521,"Ġdisrespectful":98522,"ĠQDom":98523,"Ġ${Ċ":98524,"estatus":98525,"Reached":98526,"!.ĊĊ":98527,"ILT":98528,"ĠNDEBUG":98529,"ĠCourage":98530,"birthdate":98531,"ĠTing":98532,"Ġutilizado":98533,"ánchez":98534,"Outdoor":98535,"Ġhandguns":98536,"RefCount":98537,"ÉĻ":98538,"romo":98539,"Ġtts":98540,".She":98541,"ĠPane":98542,"ãĢij,ãĢIJ":98543,"ĠIOCTL":98544,"/black":98545,"inscription":98546,"Ġbiopsy":98547,"ĠTimeInterval":98548,".TestCheck":98549,"ĠGUIStyle":98550,"ĠCapability":98551,"ĠBeitrag":98552,"donnees":98553,"Treatment":98554,".backup":98555,"Ġsignings":98556,"ĠBoca":98557,"drm":98558,".MAIN":98559,"Ġgoede":98560,"ĠMarkup":98561,"GREE":98562,"ĠBaseService":98563,".Creator":98564,"Ġjails":98565,"ĠKahn":98566,"IpAddress":98567,"ACHI":98568,"Ġinhibited":98569,"Ġ@$_":98570,"ĠAssass":98571,"Ġenviado":98572,"Heroes":98573,"ÐŁÐµÑĢ":98574,"ĠMaven":98575,".ls":98576,"Ġive":98577,"|RF":98578,"ĠresizeMode":98579,"Ġrumpe":98580,"_attachments":98581,"TU":98582,"Ġtactile":98583,"Attempting":98584,"Ġrobin":98585,"yaw":98586,"Ġmercenaries":98587,"ĠHabitat":98588,"enddate":98589,"Ġoxy":98590,"ĉRandom":98591,"ohon":98592,"IsNull":98593,"ĠValidationResult":98594,"ãĥļ":98595,"umbed":98596,"ppv":98597,"Ġarp":98598,"ichick":98599,"_rnn":98600,"ĠTFT":98601,"TexImage":98602,"\"On":98603,"ĠSampler":98604,"topl":98605,"Ġjane":98606,"yling":98607,"ĠUNICODE":98608,"TabIndex":98609,"<{Ċ":98610,"suspend":98611,"uvian":98612,",application":98613,"олиÑĩеÑģÑĤво":98614,"yat":98615,"ezier":98616,"ĠCHUNK":98617,"ĠAdler":98618,"/Add":98619,"ĠKeyValue":98620,"Ġsposób":98621,"Sampling":98622,"chers":98623,"_AMD":98624,"Ru":98625,".MustCompile":98626,"Nation":98627,"Assoc":98628,"Managing":98629,"ĠEngl":98630,"_GB":98631,"Ġsuccinct":98632,"Ġdisliked":98633,"ĠIke":98634,"Bulletin":98635,"_ARCHIVE":98636,"Proposal":98637,"Ġjogging":98638,".CREATED":98639,"Ġchol":98640,"è£ħ":98641,"Į¨":98642,"-push":98643,"Ġreserva":98644,"corev":98645,"ètre":98646,"THR":98647,"Ġincompetence":98648,"Ġcharisma":98649,"æĦŁ":98650,"Ġ\"==":98651,"BTN":98652,"ĠLocator":98653,"ivet":98654,"('.')Ċ":98655,"ĠforIndexPath":98656,"ôme":98657,"Ġcapacit":98658,"waters":98659,"ĠWRONG":98660,"hoa":98661,"ĠMIPS":98662,"Ġemiss":98663,"ĠJacqueline":98664,"(cmp":98665,"Ġeens":98666,"Leo":98667,".timing":98668,"CLUSION":98669,"Ġ(\"-":98670,"åĵĪ":98671,".kode":98672,"ĠUndert":98673,"Ġbewild":98674,"ĠEssen":98675,".hd":98676,"Ġrenegot":98677,"Ġmower":98678,"Ġlsp":98679,"Ġpenchant":98680,"Ġmanoe":98681,"Ġagli":98682,"Ġrecal":98683,"ĠOPERATION":98684,"(^)(":98685,"Ġν":98686,"ĠScoped":98687,"Ġ@\"Ċ":98688,"=label":98689,"[loc":98690,"Intl":98691,"ĠNz":98692,"tablet":98693,".ColumnName":98694,"ĠscreenSize":98695,"DBus":98696,"cooked":98697,"-registration":98698,"âĢľOne":98699,"-non":98700,"ĠwiÄĻc":98701,"Ġcosta":98702,".addTab":98703,".conditions":98704,"ĠHess":98705,"MEMORY":98706,"ĠAvalanche":98707,"()}}Ċ":98708,"Ġtriplet":98709,"Ġlabyrinth":98710,"ĠNodeList":98711,"ĠNYT":98712,"Ġyeni":98713,"dff":98714,".HtmlControls":98715,"AVIS":98716,"/Math":98717,"Ġmemcmp":98718,"اء":98719,"оÑģÑĮ":98720,"crap":98721,"(pages":98722,"Ġlxml":98723,"ĠQDateTime":98724,"_tcb":98725,"Ġopenid":98726,"Ġsynaptic":98727,"ĠMDMA":98728,"(slug":98729,"igmatic":98730,"enor":98731,"Ġcramped":98732,"GOP":98733,"ŃIJ":98734,".isFile":98735,"ĠDifferential":98736,"Ġ=\"\";Ċ":98737,"ĉĉĉĠĠĠĠĉ":98738,"ĠCooke":98739,"ĉUFUNCTION":98740,"Ġperseverance":98741,"RelativeLayout":98742,"IMPORTANT":98743,"Ġexon":98744,"Ġон":98745,"ibase":98746,"(CONT":98747,"novation":98748,"ä½ķ":98749,"[sub":98750,"AdminController":98751,"HTTPHeader":98752,"crear":98753,"ĠNIR":98754,"ĠDropDownList":98755,"Ġvalide":98756,"Ġdehydration":98757,".']":98758,"(WIN":98759,"Ġ...\\":98760,"Ġphotoshop":98761,"ĉInit":98762,"_cou":98763,"ĠtimeZone":98764,"darwin":98765,"romatic":98766,"NavigationItemSelectedListener":98767,"brates":98768,"]--;Ċ":98769,"Ġtragedies":98770,"ĠPediatrics":98771,"SMART":98772,"-API":98773,"ĠMessageLookup":98774,"ĉvo":98775,"Ġprejudices":98776,"ĠmA":98777,"Ups":98778,"ĠMISSING":98779,"ĉad":98780,"Cream":98781,"ĠTb":98782,"ĠMona":98783,"_ghost":98784,"ĉtypes":98785,"Emb":98786,"ĠDocumentary":98787,"');ĊĊĊĊ":98788,"Ġlup":98789,"_Reference":98790,"ĠBATCH":98791,"Ġintertwined":98792,"":98915,"Ġfoyer":98916,"'utilisation":98917,"ĠMüller":98918,"ĠFetish":98919,"ĠdefaultManager":98920,"Ġbacktrack":98921,"Bah":98922,"Explicit":98923,"_ASCII":98924,"ĠmActivity":98925,"(Msg":98926,"Ġê²Į":98927,"ĠTERMS":98928,"ĠAngie":98929,"HSV":98930,"ĠMosque":98931,".Names":98932,"íĬ¼":98933,"reste":98934,"_parms":98935,"Ġgaping":98936,"Ġcropping":98937,"DataFrame":98938,"Ġresponsiveness":98939,"_undo":98940,"_tran":98941,".terminate":98942,"Ġitaliane":98943,"Ġwalkthrough":98944,"Ġattractiveness":98945,"де":98946,"_STS":98947,"_learn":98948,"Ġchocolates":98949,"ierarchical":98950,"-thinking":98951,"Ġ)))":98952,"ishments":98953,".Logf":98954,"ĠTMZ":98955,"ĠCanary":98956,"foil":98957,"ĠVaccine":98958,".vx":98959,"ĠSurround":98960,"Intermediate":98961,"Ġiov":98962,"vais":98963,"';\";Ċ":98964,"ï½ŀĊĊ":98965,"éĢģæĸĻ":98966,"â̦it":98967,"Seats":98968,"Clar":98969,"Wars":98970,"ĠHutchinson":98971,"ĠHasan":98972,"!')ĊĊ":98973,"ĠRichie":98974,"cheiden":98975,"($('":98976,"York":98977,"Ġlids":98978,"Ġalphanumeric":98979,"ĠGlock":98980,".shapes":98981,"Ġsparking":98982,"_epsilon":98983,"uplicated":98984,".dirty":98985,"])==":98986,"ĠìľĦì¹ĺ":98987,"Ġscn":98988,"Ġ/****************************************************************":98989,"_PREVIEW":98990,"_HC":98991,"ielding":98992,"fgets":98993,"ĠAddison":98994,"ĠproductService":98995,"-figure":98996,"(retval":98997,"zano":98998,"Ġautob":98999,"ĉsd":99000,"_numer":99001,"ĠSetLastError":99002,"ĠFior":99003,"ificance":99004,"Untitled":99005,"Ġinfield":99006,"Ġ{}));Ċ":99007,"Ġspac":99008,"Ġrookies":99009,"(describing":99010,"ngen":99011,"ிà®":99012,".rdf":99013,".Mutex":99014,"Ġkneeling":99015,"ĠQE":99016,"setMax":99017,"ReadStream":99018,"Ġventas":99019,"sut":99020,"cmpeq":99021,".WriteAllText":99022,"ĠExperienced":99023,"$__":99024,"Ġkaum":99025,"ĠLIS":99026,"Ġdocumentos":99027,"_HEALTH":99028,"icontains":99029,"Ġartisans":99030,"OWNER":99031,"Ġblinked":99032,"getDisplay":99033,"Ġtoen":99034,"ĠrowNum":99035,"Ġavril":99036,"Ġinvis":99037,"ĠKear":99038,"toBeInTheDocument":99039,"apur":99040,"Ġracked":99041,"ĠMcMaster":99042,"_ATTRIB":99043,"Haz":99044,"Ġfactura":99045,"/ts":99046,"ĠÑĢазмеÑĢ":99047,"Ġzf":99048,"Ġshortfall":99049,".fasta":99050,"ĠCONSTANT":99051,".managed":99052,"gems":99053,"SharedPointer":99054,"Ġblurry":99055,"brightness":99056,"(components":99057,"Ġ...\"ĊĊ":99058,"SELL":99059,"ĠIllustrator":99060,".getChannel":99061,"Ġtrouvé":99062,"ysters":99063,"Ġvois":99064,"ĠLinden":99065,"Ġemojis":99066,"Ġbrawl":99067,"ĠMSR":99068,"ĠElo":99069,"ĠCroatian":99070,"PopupMenu":99071,"Lewis":99072,".JWT":99073,"Ġastonished":99074,"Bush":99075,"(itemId":99076,"Ġdetachment":99077,"ĠEncore":99078,"å°Ķ":99079,"Ġrekl":99080,"Ġcram":99081,")$/":99082,".getHost":99083,"_recommend":99084,"-HT":99085,"_calibration":99086,"Authenticate":99087,".firebaseapp":99088,"UNIX":99089,"ĉCamera":99090,"ĠHEAP":99091,"Ideal":99092,".office":99093,"Ġgoofy":99094,"(Symbol":99095,"Ġjouer":99096,"_partitions":99097,"Ġrapidement":99098,"ĠGNUNET":99099,"idUser":99100,"Ġsupervise":99101,"(Contact":99102,"AWN":99103,"ãģĺ":99104,"Ġnaam":99105,"Ġaust":99106,"åľ¨çº¿":99107,"_softmax":99108,"AllowAnonymous":99109,"ammable":99110,"ROUTE":99111,"*D":99112,"Ġaden":99113,"ĠCristina":99114,"ĠCristiano":99115,"Ġbloodstream":99116,"subclass":99117,"_persona":99118,"CHILD":99119,"-know":99120,"ĠnavigationOptions":99121,"ĠZukunft":99122,"ĠPixar":99123,"Tyler":99124,"Ġunderworld":99125,"Ġsincerity":99126,"Ġdispenser":99127,"Ġkter":99128,"idders":99129,".addNode":99130,"-checked":99131,"Ġkeyst":99132,"ĠWTO":99133,".signals":99134,"Ġadventurer":99135,"ĠPang":99136,"\\R":99137,"=pos":99138,"Ġdispensaries":99139,"ĠCloset":99140,"(\"{\\\"":99141,"ideon":99142,"Ġnécessaire":99143,"()\"Ċ":99144,"_RECEIVED":99145,"Ġrésultats":99146,"Ġmoden":99147,"ĠIcelandic":99148,";d":99149,".allowed":99150,"(newUser":99151,"Ġmerciless":99152,".WaitFor":99153,"Ġdaycare":99154,"ĠConveyor":99155,"çĸ":99156,"ð¬":99157,"çĥ":99158,"çĹ":99159,"çł":99160,"èĦ":99161,"é²":99162,"å¦":99163,"çĿĢ":99164,"å¾Ī":99165,"éħ":99166,"çĭ":99167,"éª":99168,"æĤ":99169,"é¥":99170,"èħ":99171,"æĥ³":99172,"å¨":99173,"é¹":99174,"çĤ":99175,"åĴ":99176,"çĮ":99177,"è´¨":99178,"æ¢":99179,"æ°Ķ":99180,"ð«":99181,"æķĻ":99182,"çŁ":99183,"åĦ":99184,"åıijå±ķ":99185,"åĪĽ":99186,"èij":99187,"æħ":99188,"åŀ":99189,"åģļ":99190,"æĪĺ":99191,"æIJ":99192,"强":99193,"æ·±":99194,"åĩł":99195,"ç¿":99196,"å©":99197,"èŀ":99198,"å§Ķ":99199,"åIJĦ":99200,"èİ":99201,"é¸":99202,"éº":99203,"åıĹ":99204,"èģĮ":99205,"åĺ":99206,"æ½":99207,"é£İ":99208,"èIJ¥":99209,"åħļ":99210,"èľ":99211,"éĤ£":99212,"é¢Ĩ":99213,"çij":99214,"é³":99215,"æľ¯":99216,"ä»Ģ":99217,"æĪ¿":99218,"ç²¾":99219,"åª":99220,"éĨ":99221,"太":99222,"èĤ¡":99223,"èĽ":99224,"åħī":99225,"æŀģ":99226,"åĬŀ":99227,"èĵ":99228,"çĺ":99229,"å´":99230,"åĹ":99231,"èĬ±":99232,"çłĶ":99233,"å¿«":99234,"å¸Ī":99235,"è¶Ĭ":99236,"è§Ĥ":99237,"æ¤":99238,"æ¦":99239,"çŀ":99240,"èĤ²":99241,"çα":99242,"çϽ":99243,"ä¸ĸ":99244,"ä»Ģä¹Ī":99245,"çľ¼":99246,"å³":99247,"èĴ":99248,"æĵ":99249,"被":99250,"å¹²":99251,"çĹħ":99252,"士":99253,"çĴ":99254,"è¸":99255,"æ¾":99256,"å·¥ä½ľ":99257,"让":99258,"çĥŃ":99259,"è¾ĥ":99260,"åĦ¿":99261,"åĬ©":99262,"积":99263,"ç³":99264,"çĵ":99265,"ç£":99266,"åĤ":99267,"è¹":99268,"èļ":99269,"å·±":99270,"çϾ":99271,"åĬ¿":99272,"èµĽ":99273,"æ¨":99274,"æ¿":99275,"èĸ":99276,"æĿij":99277,"带":99278,"å¢ĥ":99279,"æĬ¤":99280,"éŃ":99281,"å«":99282,"èĩªå·±":99283,"æµİ":99284,"ä½İ":99285,"åĮ»":99286,"éĺ²":99287,"åĨľ":99288,"èĨ":99289,"çĨ":99290,"é«":99291,"åĨĽ":99292,"æĪı":99293,"åįĩ":99294,"æĸ¯":99295,"ä½ı":99296,"èIJ½":99297,"åħ»":99298,"èĩ´":99299,"çĬ":99300,"çĩ":99301,"çħ":99302,"èĶ":99303,"ä¼ģä¸ļ":99304,"åĽ¢":99305,"æīį":99306,"æł¡":99307,"åĩĨ":99308,"å¥ĩ":99309,"åī¯":99310,"é¼":99311,"æ¼Ķ":99312,"马":99313,"èµ°":99314,"ç¥ŀ":99315,"åħĭ":99316,"æľĽ":99317,"æ²¹":99318,"è¾¹":99319,"åįĥ":99320,"å¾Ģ":99321,"åĪĩ":99322,"æ©":99323,"ç¶":99324,"åĻ":99325,"éĻħ":99326,"çīĮ":99327,"社ä¼ļ":99328,"游æĪı":99329,"æĸ½":99330,"çħ§":99331,"æİ§":99332,"满":99333,"è¯Ĩ":99334,"éĩįè¦ģ":99335,"è¶³":99336,"çķĻ":99337,"ç»Ĩ":99338,"åįı":99339,"éĢĤ":99340,"æĩ":99341,"æ§":99342,"éĦ":99343,"èĿ":99344,"å¸Ĥåľº":99345,"ç»ıæµİ":99346,"ä¹ł":99347,"æĸĩåĮĸ":99348,"éļ¾":99349,"ä¹IJ":99350,"åĨ³":99351,"欢":99352,"è§ī":99353,"åĽŃ":99354,"åħ´":99355,"åħħ":99356,"举":99357,"æī¹":99358,"èķ":99359,"æĬĬ":99360,"æĬĢæľ¯":99361,"ç©¶":99362,"第ä¸Ģ":99363,"便":99364,"åĵį":99365,"çİ©":99366,"åĿļ":99367,"èŀį":99368,"åįĬ":99369,"åĸľ":99370,"å±Ĥ":99371,"离":99372,"ä»ħ":99373,"éŁ":99374,"åij³":99375,"念":99376,"åŃ£":99377,"ç´§":99378,"ä¹ħ":99379,"é¤":99380,"éŀ":99381,"è¤":99382,"åĢĻ":99383,"åĨµ":99384,"çŁ³":99385,"åģ¥":99386,"æĢİ":99387,"å®Ŀ":99388,"è¡Ģ":99389,"åŁŁ":99390,"æĹ©":99391,"çŁ¥éģĵ":99392,"è´Ł":99393,"åįļ":99394,"å·´":99395,"亲":99396,"å±ŀ":99397,"严":99398,"äºī":99399,"å¯Ł":99400,"èº":99401,"ç°":99402,"建设":99403,"产ä¸ļ":99404,"åIJĥ":99405,"åŃ©":99406,"æĹħ":99407,"æł¹":99408,"æĿIJ":99409,"ä¼Ĺ":99410,"éļı":99411,"å®ĺ":99412,"åºķ":99413,"彩":99414,"å¯Į":99415,"温":99416,"åį«":99417,"åī§":99418,"çĽĬ":99419,"æĬĹ":99420,"è´¢":99421,"纪":99422,"æĨ":99423,"çĶŁæ´»":99424,"红":99425,"çĶŁäº§":99426,"è¿ľ":99427,"éĴ±":99428,"åĶ®":99429,"群":99430,"çıŃ":99431,"楼":99432,"éĩĩ":99433,"èīº":99434,"å±ħ":99435,"åģĩ":99436,"è°Ī":99437,"æĻļ":99438,"é¬":99439,"èĪª":99440,"害":99441,"èĹ":99442,"çį":99443,"åµ":99444,"çİĭ":99445,"康":99446,"èİ·":99447,"ç»Ń":99448,"äºļ":99449,"é£Ł":99450,"åİĭ":99451,"æĭĽ":99452,"èĮĥ":99453,"许":99454,"åĽ´":99455,"é½":99456,"éĻį":99457,"纳":99458,"åĵª":99459,"æķĻèĤ²":99460,"å·²ç»ı":99461,"å¾·":99462,"æŀĹ":99463,"å®īåħ¨":99464,"é¾Ļ":99465,"大家":99466,"éĿĴ":99467,"åºľ":99468,"æ²³":99469,"åı¤":99470,"èį¯":99471,"åĿĩ":99472,"æĻº":99473,"乡":99474,"çķ¥":99475,"åĨ·":99476,"ç¦ı":99477,"室":99478,"ç»´":99479,"æī¿":99480,"å±Ĭ":99481,"è¯ī":99482,"åĪ»":99483,"èŁ":99484,"æª":99485,"å°±æĺ¯":99486,"è¿Ļ个":99487,"ä¸Ńå¿ĥ":99488,"ä¸ĸçķĮ":99489,"åŁİå¸Ĥ":99490,"éĿŀ常":99491,"åĪĴ":99492,"åıĮ":99493,"æĢİä¹Ī":99494,"åΰäºĨ":99495,"æľĥ":99496,"åı²":99497,"ä¾Ĩ":99498,"å¾ĭ":99499,"å¥ĸ":99500,"ç»Ī":99501,"åªĴ":99502,"å®ģ":99503,"课":99504,"èģĮä¸ļ":99505,"åħį":99506,"æµĭ":99507,"æĢ¥":99508,"æķij":99509,"çĭ¬":99510,"èѦ":99511,"é¤IJ":99512,"æĦ¿":99513,"è´«":99514,"çĸij":99515,"åļ":99516,"她":99517,"åıĪ":99518,"åĽłä¸º":99519,"ä¸įæĺ¯":99520,"å¤Ł":99521,"æĸ¹éĿ¢":99522,"éķĩ":99523,"äºĴ":99524,"éħĴ":99525,"讲":99526,"çĸĹ":99527,"æĺ¥":99528,"æ¹ĸ":99529,"å¤ľ":99530,"责任":99531,"人æ°ij":99532,"åħ°":99533,"çŁŃ":99534,"æķħ":99535,"åĩı":99536,"æĻ®":99537,"亮":99538,"ä¾Ŀ":99539,"åį°":99540,"éĿĻ":99541,"åĢĭ":99542,"å¾ģ":99543,"åIJ¸":99544,"缺":99545,"æĶ»":99546,"åĩĢ":99547,"åħ¸":99548,"åĽº":99549,"访":99550,"ç¹":99551,"çĢ":99552,"æıIJä¾Ľ":99553,"ç»ĩ":99554,"å¾Īå¤ļ":99555,"çłĶç©¶":99556,"è·Ł":99557,"主è¦ģ":99558,"æĥħåĨµ":99559,"çŃĸ":99560,"æŃ»":99561,"大åѦ":99562,"æĶ¿åºľ":99563,"å½±åĵį":99564,"ä¹°":99565,"åħŃ":99566,"éĻ©":99567,"åħ«":99568,"æŁIJ":99569,"è´¨éĩı":99570,"åįł":99571,"å·®":99572,"æĽ´å¤ļ":99573,"æľĭ":99574,"éĿ©":99575,"宣":99576,"çł´":99577,"è½»":99578,"座":99579,"æĺ¾":99580,"稳":99581,"è´µ":99582,"èĥĮ":99583,"èī¯":99584,"çĸ«":99585,"æ¯Ĵ":99586,"ä¹İ":99587,"åĢŁ":99588,"è¿·":99589,"çŃĶ":99590,"æ¿Ģ":99591,"åij¼":99592,"äºĨä¸Ģ":99593,"è¶£":99594,"ä¼´":99595,"ä¼Ļ":99596,"è¼":99597,"ð¬Ń":99598,"åĽ½å®¶":99599,"æ´»åĬ¨":99600,"çİ°åľ¨":99601,"ç§ijæĬĢ":99602,"åį¡":99603,"ä¸įåIJĮ":99604,"个人":99605,"è®°èĢħ":99606,"ä¸įæĸŃ":99607,"éĹ»":99608,"ä¹Ŀ":99609,"èijĹ":99610,"综":99611,"ä¸ĥ":99612,"æłij":99613,"æľĭåıĭ":99614,"åįĸ":99615,"伤":99616,"æ²Ļ":99617,"åĸĦ":99618,"å¥Ĺ":99619,"è½®":99620,"ç©¿":99621,"è¡¥":99622,"ä¸Ģå®ļ":99623,"çªģ":99624,"çĿ£":99625,"追":99626,"å¨ģ":99627,"åı¦":99628,"åĽ°":99629,"æŀ¶":99630,"ç»Ŀ":99631,"æķ£":99632,"æİ¢":99633,"æ´Ĺ":99634,"临":99635,"ä¼¼":99636,"è´¸":99637,"丰":99638,"æĺ¯ä¸Ģ":99639,"ç«ŀ":99640,"è¿İ":99641,"èģļ":99642,"è«":99643,"æįŁ":99644,"æī§":99645,"驾":99646,"è¿Ŀ":99647,"è¥":99648,"èł":99649,"ä»ĸ们":99650,"æĹ¶åĢĻ":99651,"å®ĥ":99652,"人åijĺ":99653,"è¿Ļæł·":99654,"å·¥ç¨ĭ":99655,"åĪĽæĸ°":99656,"åŃ©åŃIJ":99657,"å¸Į":99658,"éĥ¨åĪĨ":99659,"éĵ¶":99660,"代表":99661,"é¦Ļ":99662,"帮":99663,"æİ¨è¿Ľ":99664,"çĽĺ":99665,"积æŀģ":99666,"éĥ¨éŨ":99667,"åŁ¹":99668,"æŃ¦":99669,"ä¸įä¼ļ":99670,"çŃij":99671,"éĢĻ":99672,"çݩ家":99673,"æĭ¿":99674,"åİĤ":99675,"æ¯Ľ":99676,"çģµ":99677,"æŃĮ":99678,"绿":99679,"å¦Ī":99680,"缼":99681,"é¦Ĩ":99682,"顺":99683,"èĦ¸":99684,"å°¼":99685,"丽":99686,"奥":99687,"éģĩ":99688,"è¯į":99689,"å°ģ":99690,"ä¸Ŀ":99691,"好çļĦ":99692,"æĭħ":99693,"èĦ±":99694,"æģ¶":99695,"åİļ":99696,"åĬ³":99697,"缣":99698,"æĬĺ":99699,"åı¥":99700,"æĢĢ":99701,"æŁĵ":99702,"书记":99703,"åĨł":99704,"é²ľ":99705,"æ¦Ĥ":99706,"éļIJ":99707,"å¹ħ":99708,"èµŀ":99709,"å¹ķ":99710,"æ¥Ń":99711,"éģĹ":99712,"åΤ":99713,"èĺ":99714,"å¶":99715,"æĬķèµĦ":99716,"è¡Įä¸ļ":99717,"äºij":99718,"çݯå¢ĥ":99719,"åѦçĶŁ":99720,"åIJĪä½ľ":99721,"åģ¥åº·":99722,"é£ŀ":99723,"ä¸ĢæŃ¥":99724,"ä¸Ģ缴":99725,"åıijçĶŁ":99726,"éĺ¿":99727,"é¢Ĩ导":99728,"åĸľæ¬¢":99729,"åºĶ该":99730,"çĤº":99731,"è®Ń":99732,"æĿĢ":99733,"港":99734,"交éĢļ":99735,"éĺ¶":99736,"éĴ¢":99737,"令":99738,"å°½":99739,"æ¯į":99740,"è¡£":99741,"ç²ī":99742,"é¡¶":99743,"ä¹Łä¸į":99744,"æĬĵ":99745,"èĭ¦":99746,"幸":99747,"礼":99748,"第ä¸ī":99749,"大çļĦ":99750,"éģİ":99751,"çĥŁ":99752,"éģ¿":99753,"ä»į":99754,"åºĨ":99755,"æĢķ":99756,"è°¢":99757,"çĽĸ":99758,"å°Ħ":99759,"éľ²":99760,"æĸĹ":99761,"çĬ¶":99762,"åѸ":99763,"æ¯ķ":99764,"å·¨":99765,"çŁ¿":99766,"çļĩ":99767,"å¸Ń":99768,"çĹĩ":99769,"æī¬":99770,"å»¶":99771,"ä¾§":99772,"æ·¡":99773,"çļĦä¸Ģ":99774,"ç¶²":99775,"æ´ģ":99776,"ç¸":99777,"è§Ī":99778,"çѹ":99779,"ç§ĺ":99780,"è¯Ĭ":99781,"çı¾":99782,"èªī":99783,"毫":99784,"ð¨":99785,"åį´":99786,"æĪIJ为":99787,"èĥ½åĬĽ":99788,"é»Ħ":99789,"æĹħ游":99790,"èά":99791,"æ¯Ķè¾ĥ":99792,"èµ·æĿ¥":99793,"äºĨè§£":99794,"èĩªçĦ¶":99795,"ä¸Ģ次":99796,"åŁºæľ¬":99797,"æĽ¾":99798,"综åIJĪ":99799,"èıľ":99800,"è§īå¾Ĺ":99801,"第äºĮ":99802,"è·ij":99803,"æ³¢":99804,"åĢĴ":99805,"ç¡Ģ":99806,"åħµ":99807,"èįī":99808,"çͳ":99809,"çͰ":99810,"æĤ£":99811,"è§Ħå®ļ":99812,"èĥľ":99813,"èµĦ产":99814,"梦":99815,"æľĿ":99816,"è¿ĻéĩĮ":99817,"夫":99818,"æĮ¥":99819,"ä½Ľ":99820,"å®Ī":99821,"鼶":99822,"æĸ¼":99823,"ç¯ĩ":99824,"å²Ľ":99825,"åĵ¥":99826,"éŃĶ":99827,"ä¸įåΰ":99828,"æīĺ":99829,"åºĬ":99830,"欧":99831,"èį£":99832,"æ±ĩ":99833,"æī©":99834,"åģı":99835,"å¢Ļ":99836,"讯":99837,"å©ļ":99838,"æĥł":99839,"æ´ĭ":99840,"å®ľ":99841,"润":99842,"æħ¢":99843,"éĢı":99844,"宽":99845,"顾":99846,"ç´¯":99847,"污":99848,"çĪĨ":99849,"ç§Ł":99850,"æĥĬ":99851,"涨":99852,"饰":99853,"éĺµ":99854,"饮":99855,"æļĸ":99856,"åºŁ":99857,"æĹĹ":99858,"éļĶ":99859,"ç¶ĵ":99860,"åĭĻ":99861,"實":99862,"éĢĶ":99863,"æī«":99864,"çĥĪ":99865,"鼻":99866,"åĪij":99867,"éĹľ":99868,"éĹª":99869,"å¥ĭ":99870,"åĤ¨":99871,"缩":99872,"ä¾µ":99873,"å¬":99874,"ð¬¶":99875,"åĽ½éĻħ":99876,"ç»Ħç»ĩ":99877,"ä¸ĵä¸ļ":99878,"åıijçݰ":99879,"å¸ĮæľĽ":99880,"ç»ıèIJ¥":99881,"åı«":99882,"æĿ¥è¯´":99883,"éļľ":99884,"ä»»ä½ķ":99885,"交æĺĵ":99886,"éĩįçĤ¹":99887,"çļ®":99888,"ç»į":99889,"æ´¾":99890,"ç§ijåѦ":99891,"åºĶç͍":99892,"建çŃij":99893,"èĤī":99894,"æĶ¹éĿ©":99895,"åŁºç¡Ģ":99896,"æ±ī":99897,"åĩºæĿ¥":99898,"è¿Ļä¹Ī":99899,"åĪļ":99900,"åĿIJ":99901,"ä¸įä»ħ":99902,"ä¼ļè®®":99903,"éĿł":99904,"åªĴä½ĵ":99905,"æ°¸":99906,"åĨ²":99907,"èĭı":99908,"央":99909,"çζ":99910,"åłĤ":99911,"å®ŀéĻħ":99912,"è¡Ĺ":99913,"ç«¥":99914,"éĺħ":99915,"äºĭæĥħ":99916,"åİŁåĽł":99917,"éħ¸":99918,"以æĿ¥":99919,"娱":99920,"宫":99921,"åĿĹ":99922,"绩":99923,"éĩİ":99924,"ä¸įå¾Ĺ":99925,"ä¼łå¥ĩ":99926,"硬":99927,"åİħ":99928,"æĹ¢":99929,"ç»ĥ":99930,"èĦij":99931,"å¼±":99932,"æİĮ":99933,"è´´":99934,"æĮĤ":99935,"åħ³éĶ®":99936,"å°ļ":99937,"é¥Ń":99938,"åºĦ":99939,"çϼ":99940,"åľĭ":99941,"æİĪ":99942,"个æľĪ":99943,"äºĪ":99944,"å¸ģ":99945,"è·Ŀ":99946,"æ²ī":99947,"竣":99948,"åĨ¬":99949,"æĬ½":99950,"éĨĴ":99951,"å¼Ł":99952,"触":99953,"èģĺ":99954,"è±Ĩ":99955,"æļ´":99956,"åijĬè¯ī":99957,"豪":99958,"èµ¢":99959,"è·¨":99960,"è³ĩ":99961,"çΏ":99962,"æĬ±":99963,"浪":99964,"麻":99965,"仪":99966,"è¡¡":99967,"奶":99968,"çģ¾":99969,"èµ¶":99970,"èĤ¥":99971,"å§IJ":99972,"åĢº":99973,"éľĩ":99974,"订":99975,"æ¬Ĭ":99976,"ç·":99977,"å»ī":99978,"ä¿Ĺ":99979,"å¿ĺ":99980,"å¦ĩ":99981,"ç¼ĵ":99982,"åŃķ":99983,"漫":99984,"è£ģ":99985,"çĩĥ":99986,"é»ĺ":99987,"çī¢":99988,"çĪ·":99989,"æĬµ":99990,"宾":99991,"æľīä¸Ģ":99992,"迹":99993,"è¿«":99994,"è²Į":99995,"æľīçļĦ":99996,"ð¬ĺ":99997,"è¿ĺæĺ¯":99998,"æīĢ以":99999,"ä¹Łæĺ¯":100000,"è¿ĻäºĽ":100001,"对äºİ":100002,"åIJ§":100003,"缮åīį":100004,"èĩªå·±çļĦ":100005,"èĥ½å¤Ł":100006,"å¦Ĥä½ķ":100007,"æľºæŀĦ":100008,"åıªæĺ¯":100009,"ç½ijç«Ļ":100010,"åħ¨éĿ¢":100011,"为äºĨ":100012,"å¼Ģåıij":100013,"æĸ°éĹ»":100014,"éĩijèŀį":100015,"ç»§":100016,"客æĪ·":100017,"ä¸Ģèµ·":100018,"èĮ¶":100019,"åħ³æ³¨":100020,"æ°´å¹³":100021,"åİĨåı²":100022,"å¢ŀéķ¿":100023,"é±":100024,"åŁºéĩij":100025,"åºŃ":100026,"åı¶":100027,"ä¿ĥ":100028,"鼨":100029,"æ¶Īè´¹":100030,"èι":100031,"çŁ¥è¯Ĩ":100032,"æĪĺçķ¥":100033,"ç»ıéªĮ":100034,"å³°":100035,"æĽ²":100036,"èĦļ":100037,"åĨ°":100038,"å¤ı":100039,"å½Ĵ":100040,"ç¬Ķ":100041,"èĻij":100042,"çͲ":100043,"åľĪ":100044,"è¯Ĺ":100045,"é½IJ":100046,"容æĺĵ":100047,"çłĶåıij":100048,"骨":100049,"纸":100050,"è·µ":100051,"æĹ§":100052,"çķ¶":100053,"åΏ":100054,"è´·":100055,"åı¬":100056,"ç§ĭ":100057,"æ¶²":100058,"è¡ĮæĶ¿":100059,"çĮ®":100060,"èĤ¤":100061,"éĢIJ":100062,"è¶ĬæĿ¥":100063,"è¶ĬæĿ¥è¶Ĭ":100064,"æĦıè§ģ":100065,"èĪŀ":100066,"åīĤ":100067,"æ¶ī":100068,"ç¨ĭ度":100069,"åħ¬åħ±":100070,"械":100071,"æľ«":100072,"纯":100073,"åͱ":100074,"æ´²":100075,"æĬ¢":100076,"æ¤į":100077,"å¿Ļ":100078,"ä¼°":100079,"å¼¹":100080,"æ³ī":100081,"æľĢ大":100082,"è¶ĭ":100083,"å·§":100084,"ç¦ģ":100085,"æī¶":100086,"åį±":100087,"çıł":100088,"çĨŁ":100089,"æĭľ":100090,"主ä¹ī":100091,"æĿĤ":100092,"éĻĦ":100093,"éģį":100094,"æIJŃ":100095,"æĮ¯":100096,"å¤ļå¹´":100097,"æķ¬":100098,"æijĦ":100099,"纷":100100,"å¼ĥ":100101,"湿":100102,"å¨ĺ":100103,"æ¡£":100104,"é©¶":100105,"æľĹ":100106,"æ®ĸ":100107,"æ¦ľ":100108,"åĵ¡":100109,"ä¸Ģä½ĵ":100110,"æŁ¥çľĭ":100111,"ç¹ģ":100112,"æµĵ":100113,"åħ¬å®ī":100114,"æ½ľ":100115,"è´¯":100116,"éªĹ":100117,"æIJľ":100118,"å·¡":100119,"è¬":100120,"éĬ":100121,"å§Ķä¼ļ":100122,"æĤł":100123,"åī©":100124,"æıŃ":100125,"åŃ£åº¦":100126,"ð«ĺ":100127,"ð¬¬":100128,"ä´":100129,"ðª":100130,"ä½Ĩæĺ¯":100131,"éĥ½æĺ¯":100132,"å¹³åı°":100133,"åŃ¦ä¹ł":100134,"åĵģçīĮ":100135,"ä¸Ķ":100136,"è¿Ļç§į":100137,"æĶ¿çŃĸ":100138,"æĭ¬":100139,"认为":100140,"ä¸Ģèά":100141,"æłĩåĩĨ":100142,"æĶ¯æĮģ":100143,"模å¼ı":100144,"åħ³ç³»":100145,"çļĦæĺ¯":100146,"è¿Ļä¸Ģ":100147,"ä¸įè¦ģ":100148,"çĶļ":100149,"ç²¾ç¥ŀ":100150,"æĭ¥":100151,"åĪ©ç͍":100152,"ä¿ĿæĬ¤":100153,"ä½ľç͍":100154,"èĭ¥":100155,"åĽ½åĨħ":100156,"ä»ĭç»į":100157,"ä¸Ģä¸ĭ":100158,"å·¥ä¸ļ":100159,"缮æłĩ":100160,"æľĢåIJİ":100161,"ä»·å̼":100162,"å°į":100163,"éĵģ":100164,"è°ģ":100165,"ç»ĵæŀĦ":100166,"éĽª":100167,"æĻºèĥ½":100168,"ä¼łç»Ł":100169,"ä½ĵèĤ²":100170,"çĶŁæĢģ":100171,"æĭį":100172,"æİª":100173,"åĨľä¸ļ":100174,"çī¹èī²":100175,"è§Ħ模":100176,"æĹ¶ä»£":100177,"è¿ĩç¨ĭ":100178,"éĴĪ":100179,"æĿ¾":100180,"åĶIJ":100181,"åĮ»çĸĹ":100182,"çģ¯":100183,"åζéĢł":100184,"æł¸å¿ĥ":100185,"ä¸įåı¯":100186,"ç³»åĪĹ":100187,"åIJī":100188,"åľ£":100189,"åĢij":100190,"ä½³":100191,"æĿ¥çľĭ":100192,"æ¯ĶèµĽ":100193,"ä¸ĭæĿ¥":100194,"åĩºäºĨ":100195,"å¹²éĥ¨":100196,"微信":100197,"å½ĵåľ°":100198,"åį·":100199,"åį«çĶŁ":100200,"ä¼Ł":100201,"çĸ«æĥħ":100202,"è°·":100203,"åĩłä¸ª":100204,"éĺ´":100205,"çĶŁçī©":100206,"å°¤":100207,"ä¼Ĭ":100208,"èĤ¯":100209,"éĿ¢ç§¯":100210,"åĪĽéĢł":100211,"æı¡":100212,"åľĨ":100213,"æĻĵ":100214,"æĪIJäºĨ":100215,"åĩ¡":100216,"çĸ¾":100217,"ç«ŀäºī":100218,"讨":100219,"主é¢ĺ":100220,"é²ģ":100221,"迪":100222,"ä¿Ħ":100223,"æĢª":100224,"並":100225,"èĻļ":100226,"æ½®":100227,"çĥ§":100228,"è̳":100229,"æ±ł":100230,"éĢĤåIJĪ":100231,"æł¹æľ¬":100232,"åĬłçĽŁ":100233,"ç͵è§Ĩ":100234,"æ··":100235,"ç¼ĺ":100236,"çªĹ":100237,"çĬ¯":100238,"æĥ¯":100239,"æĦıä¹ī":100240,"åĬŀæ³ķ":100241,"ä¼ij":100242,"æ»ij":100243,"åĭĩ":100244,"æķ¢":100245,"寻":100246,"è¦Ĩ":100247,"éĢĥ":100248,"ç»ıçIJĨ":100249,"åĿı":100250,"æ³½":100251,"ä¹ĺ":100252,"åĪº":100253,"å±ı":100254,"é¡¿":100255,"亡":100256,"éĤĢ":100257,"åħ¼":100258,"åĭ¤":100259,"æ®ĭ":100260,"æĺł":100261,"æ¯ķä¸ļ":100262,"æĪª":100263,"è·Į":100264,"å£ģ":100265,"åı¦ä¸Ģ":100266,"羣å®ŀ":100267,"磨":100268,"è¯ļ":100269,"å¿ħè¦ģ":100270,"æģĭ":100271,"æĩĤ":100272,"å¾Ĵ":100273,"è°ĵ":100274,"æķı":100275,"æĻ¨":100276,"èĥ¸":100277,"æĭ¼":100278,"å¦Ļ":100279,"诸":100280,"èģĬ":100281,"æĤī":100282,"麼":100283,"åĩŃ":100284,"èĪĴ":100285,"æ¶Ĥ":100286,"è¿ģ":100287,"沿":100288,"å¡ij":100289,"æĽ¿":100290,"æ¾³":100291,"å¿į":100292,"èĢĹ":100293,"龸":100294,"åĩłå¹´":100295,"åĪĬ":100296,"èĦī":100297,"èħIJ":100298,"æ¡Į":100299,"çºł":100300,"æ»ļ":100301,"æĤ²":100302,"åĨĴ":100303,"妹":100304,"çķħ":100305,"纵":100306,"æijĩ":100307,"夺":100308,"è·¯ä¸Ĭ":100309,"忽":100310,"èĸª":100311,"æģIJ":100312,"æĦıæĢĿ":100313,"å«Į":100314,"æı´":100315,"æ°§":100316,"èĢĢ":100317,"éĺ»":100318,"轨":100319,"å¹»":100320,"æįķ":100321,"åĿ¦":100322,"åĵĪåĵĪ":100323,"çĭIJ":100324,"滨":100325,"è²»":100326,"è¿Ł":100327,"人éĥ½":100328,"ç»ĺ":100329,"åı¹":100330,"çµIJ":100331,"æī°":100332,"æ»ĭ":100333,"å¥ij":100334,"åĭŁ":100335,"確":100336,"ð¦":100337,"éĽĨåĽ¢":100338,"æĿİ":100339,"å¼Ģå±ķ":100340,"æıIJåįĩ":100341,"åħ¨åĽ½":100342,"汽车":100343,"åŃ¦æł¡":100344,"æł¹æį®":100345,"è¿Ļæĺ¯":100346,"åĩºçݰ":100347,"éĻĪ":100348,"ç½Ĺ":100349,"èİ·å¾Ĺ":100350,"åĪĺ":100351,"éĶĢåĶ®":100352,"æľªæĿ¥":100353,"éľĢæ±Ĥ":100354,"å®ŀæĸ½":100355,"åĿļæĮģ":100356,"åħ¨çIJĥ":100357,"éĵ¶è¡Į":100358,"æİ§åζ":100359,"é¡»":100360,"åľ°åĮº":100361,"æīĵéĢł":100362,"çļĦè¯Ŀ":100363,"帮åĬ©":100364,"ä½ĵç³»":100365,"è¾¾åΰ":100366,"è§ĦåĪĴ":100367,"åŁ¹è®Ń":100368,"两个":100369,"æĬ¥åijĬ":100370,"åľ°æĸ¹":100371,"å®Įåħ¨":100372,"æİī":100373,"ç»ĵåIJĪ":100374,"å®£ä¼ł":100375,"æ³ķå¾ĭ":100376,"èīºæľ¯":100377,"ç͵影":100378,"說":100379,"ä¸ĢçĤ¹":100380,"è¶ħè¿ĩ":100381,"ç͵åŃIJ":100382,"æĢĿæĥ³":100383,"æķĻåѦ":100384,"éĺ¶æ®µ":100385,"åķĨä¸ļ":100386,"çµģ":100387,"åĪĽä¸ļ":100388,"æĸ¹æ¡Ī":100389,"çݰ代":100390,"æ¡¥":100391,"èIJ½å®ŀ":100392,"带æĿ¥":100393,"产çĶŁ":100394,"ç§Ģ":100395,"æ³°":100396,"ä¹±":100397,"åħ·ä½ĵ":100398,"åĸĿ":100399,"èĵĿ":100400,"å®Ĺ":100401,"åįĩ级":100402,"æ·±åħ¥":100403,"ä¿ĿéĻ©":100404,"ç®Ģåįķ":100405,"çĹĽ":100406,"稳å®ļ":100407,"è¾Ĩ":100408,"å±ŀäºİ":100409,"å·Ŀ":100410,"ä¸įå°ij":100411,"åĴ¨":100412,"ä¸ľè¥¿":100413,"å½¢å¼ı":100414,"娱ä¹IJ":100415,"æŃ£å¸¸":100416,"鸡":100417,"åħħåĪĨ":100418,"å®ŀè·µ":100419,"éĩĮéĿ¢":100420,"è·³":100421,"èĻİ":100422,"æĪIJéķ¿":100423,"æļĹ":100424,"çĿ¡":100425,"罪":100426,"çIJĨ念":100427,"æĮij":100428,"èµĦæľ¬":100429,"å¤ļå°ij":100430,"ä¸ĭéĿ¢":100431,"å¸Ŀ":100432,"åħ¬å¼Ģ":100433,"æ¸IJ":100434,"éķ·":100435,"å±ĭ":100436,"欢è¿İ":100437,"å¿ĥçIJĨ":100438,"çĤİ":100439,"æ¹¾":100440,"è®ĵ":100441,"éĤĦ":100442,"ç³ĸ":100443,"ä¹Į":100444,"åĬ±":100445,"çīĻ":100446,"èħ¿":100447,"å²Ĺ":100448,"ä¼į":100449,"æĪIJåijĺ":100450,"åŃĶ":100451,"å°ıç¼ĸ":100452,"èij£":100453,"泡":100454,"åħĪè¿Ľ":100455,"åħ§":100456,"åĺ´":100457,"è´Ŀ":100458,"è»":100459,"æIJŀ":100460,"æ³Ľ":100461,"鸣":100462,"ç½²":100463,"èĽĭ":100464,"主任":100465,"缮çļĦ":100466,"ä¹ı":100467,"æ´¥":100468,"æĪ´":100469,"ä¸¥æł¼":100470,"çħ¤":100471,"çĮ«":100472,"å͝":100473,"å°Ĭ":100474,"çĶľ":100475,"åŀĥ":100476,"åľ¾":100477,"æĭŁ":100478,"çĦ¦":100479,"é«Ķ":100480,"å®ı":100481,"æ©Ł":100482,"é©»":100483,"æĹģ":100484,"å½»":100485,"éĥ½ä¸į":100486,"æij©":100487,"ä»ĵ":100488,"ä¹³":100489,"岸":100490,"è°ĭ":100491,"大å¤ļ":100492,"çģŃ":100493,"èħ¾":100494,"æŁľ":100495,"èĪį":100496,"åħļçļĦ":100497,"å°ĺ":100498,"åįģå¹´":100499,"æĭĴ":100500,"裡":100501,"æŁĶ":100502,"å¹¼":100503,"éĶģ":100504,"ä¸ĵ项":100505,"æīİ":100506,"驾驶":100507,"ç¢İ":100508,"è¢ĭ":100509,"éĶĭ":100510,"壮":100511,"å°ĸ":100512,"çĶµæ±ł":100513,"è¿Ķ":100514,"æ¼ı":100515,"循":100516,"èıĮ":100517,"èĥĥ":100518,"è¾ħ":100519,"éĢĴ":100520,"èĥİ":100521,"éĻª":100522,"寿":100523,"å¥Ķ":100524,"çĮĽ":100525,"纹":100526,"çŁ¥åIJį":100527,"å¿Ĩ":100528,"æ¡ĥ":100529,"æ£ĭ":100530,"éĢĨ":100531,"çĤ¼":100532,"ç±į":100533,"çī§":100534,"æł·çļĦ":100535,"è¾Ľ":100536,"åłĨ":100537,"å®ŀåľ¨":100538,"ä¼ı":100539,"宿":100540,"èµı":100541,"è£Ĥ":100542,"åįĬå¹´":100543,"å̾":100544,"满æĦı":100545,"梯":100546,"æĦıåij³":100547,"åѤ":100548,"ç¥Ŀ":100549,"æĻ¶":100550,"èµĶ":100551,"åģ¿":100552,"èĦĤ":100553,"ç½ļ":100554,"ç¢į":100555,"æ²ĥ":100556,"æĵį":100557,"å´ĩ":100558,"æļĤ":100559,"è·ĥ":100560,"æIJ¬":100561,"å©Ĩ":100562,"éī":100563,"éī´":100564,"åħ´è¶£":100565,"èIJ¥ä¸ļ":100566,"è®Ĭ":100567,"èĦı":100568,"è¾Ī":100569,"å·ŀå¸Ĥ":100570,"è´«åĽ°":100571,"ç©·":100572,"ä¸Ńå°ı":100573,"æ¼Ĥ":100574,"çĻĮ":100575,"èľľ":100576,"ä¼Ļä¼´":100577,"çīµ":100578,"æĤŁ":100579,"éĻ·":100580,"èµĽåŃ£":100581,"樣":100582,"åģ¶":100583,"æĺĨ":100584,"è¢Ń":100585,"æįIJ":100586,"èī°":100587,"æĤ¬":100588,"çĶ¢":100589,"èij¡":100590,"çĽĹ":100591,"å©´":100592,"å°İ":100593,"纽":100594,"åĢ¡":100595,"æī®":100596,"è¨Ń":100597,"æĬij":100598,"ç¡ķ":100599,"è¾ĸ":100600,"éĥģ":100601,"辩":100602,"éĤ»":100603,"çݰåĩº":100604,"è¦ı":100605,"å½¹":100606,"éĺĶ":100607,"åīµ":100608,"诱":100609,"æĥij":100610,"æ·Ģ":100611,"é¢Ī":100612,"侦":100613,"æģ°":100614,"æ£Ģå¯Ł":100615,"éĨ«":100616,"çĦ¶æĺ¯":100617,"åĭĥ":100618,"èĮ«":100619,"äĵ":100620,"ð¬¸":100621,"ä½ľä¸º":100622,"çļĦ人":100623,"éĤ£ä¹Ī":100624,"ç¾İåĽ½":100625,"è¿ĺæľī":100626,"æıIJé«ĺ":100627,"èϽ":100628,"åħ·æľī":100629,"åĮħæĭ¬":100630,"æĪĸèĢħ":100631,"ä¸įè¿ĩ":100632,"ä¸Ĭæµ·":100633,"åĮ»éĻ¢":100634,"èµĦéĩij":100635,"çĶļèĩ³":100636,"åĪ¶åº¦":100637,"è§£åĨ³":100638,"èģĶç½ij":100639,"ç»§ç»Ń":100640,"建ç«ĭ":100641,"è¿Ľä¸ĢæŃ¥":100642,"æĿIJæĸĻ":100643,"ä»Ĭ天":100644,"å¿ħé¡»":100645,"åIJĦç§į":100646,"çİ°åľº":100647,"ä»ĸçļĦ":100648,"å¢ŀåĬł":100649,"é¢ĨåŁŁ":100650,"åıĤä¸İ":100651,"æĮģç»Ń":100652,"ä¹ĭä¸Ģ":100653,"çī¹åĪ«":100654,"é±¼":100655,"åħ±åIJĮ":100656,"åĬª":100657,"çİī":100658,"人们":100659,"åħĪçĶŁ":100660,"ä¼ĺåĬ¿":100661,"ä¿ĿæĮģ":100662,"ä½ľåĵģ":100663,"çīĽ":100664,"æĪIJæľ¬":100665,"æĶ¶åħ¥":100666,"åıĬæĹ¶":100667,"è´Łè´£":100668,"æİ¥åıĹ":100669,"èįIJ":100670,"åıªè¦ģ":100671,"羣çļĦ":100672,"导èĩ´":100673,"æľºåζ":100674,"è¡ĮåĬ¨":100675,"æĸ°çļĦ":100676,"å®ĮåĸĦ":100677,"为ä»Ģä¹Ī":100678,"ä¸Ń央":100679,"æĪIJç«ĭ":100680,"æĦŁè§ī":100681,"åıĺåĮĸ":100682,"åıĹåΰ":100683,"å¹¶ä¸į":100684,"åŃĻ":100685,"æĸ½å·¥":100686,"æĺİæĺ¾":100687,"è¿ĩåİ»":100688,"åıijæĮ¥":100689,"羣æŃ£":100690,"åŁºåľ°":100691,"æĺİç¡®":100692,"èĥ¡":100693,"许å¤ļ":100694,"ä¸Ģå¹´":100695,"æĸ¹åIJij":100696,"æģ©":100697,"çĽ¸ä¿¡":100698,"åľ³":100699,"详ç»Ĩ":100700,"äºĭä¸ļ":100701,"çĶŁåij½":100702,"åĴ¨è¯¢":100703,"æĸĩæĺİ":100704,"çijŀ":100705,"绿èī²":100706,"èİ«":100707,"æĦıè¯Ĩ":100708,"æĬķåħ¥":100709,"åĬłå¿«":100710,"æ¢ħ":100711,"ç¿»":100712,"å¼ĢæĶ¾":100713,"æĻ®éĢļ":100714,"åįıä¼ļ":100715,"æĪIJ绩":100716,"ä»Ļ":100717,"å¯Ĵ":100718,"è¯ģåΏ":100719,"认è¯Ĩ":100720,"丹":100721,"大éĩı":100722,"è¿ħ":100723,"åģļåΰ":100724,"设æĸ½":100725,"è´¸æĺĵ":100726,"èĥ½æºIJ":100727,"æĹ¶æľŁ":100728,"ä¸Ģ天":100729,"æ²»çIJĨ":100730,"åĺī":100731,"å®ĩ":100732,"丰å¯Į":100733,"举è¡Į":100734,"æĪIJæŀľ":100735,"èĤ¯å®ļ":100736,"çĭĹ":100737,"åĬ¨åĬĽ":100738,"森":100739,"åĩłä¹İ":100740,"åĽłç´ł":100741,"æ°ijæĹı":100742,"æ´ŀ":100743,"ç½ijåıĭ":100744,"åIJĪçIJĨ":100745,"广大":100746,"æ®Ĭ":100747,"æ´Ľ":100748,"æĿ¯":100749,"èĴĻ":100750,"ç͍äºİ":100751,"èŀįèµĦ":100752,"ç¥ĸ":100753,"æľºæ¢°":100754,"举åĬŀ":100755,"èĩªåĬ¨":100756,"åĬŀåħ¬":100757,"é»ŀ":100758,"éĽĦ":100759,"å̼å¾Ĺ":100760,"çĮª":100761,"以为":100762,"æĺĮ":100763,"è·Ŀ离":100764,"åIJ¸å¼ķ":100765,"ç»ķ":100766,"éļĨ":100767,"计ç®Ĺ":100768,"éĺŁä¼į":100769,"大ä¼ļ":100770,"å¼ķèµ·":100771,"çī¹çĤ¹":100772,"èĥ¶":100773,"å¹´è½»":100774,"æľ¬èº«":100775,"æľºåħ³":100776,"å®ĺæĸ¹":100777,"éĥij":100778,"æµĻ":100779,"è§Ĵèī²":100780,"èij£äºĭ":100781,"为主":100782,"æĹłè®º":100783,"ä¹łæĥ¯":100784,"æ¥ļ":100785,"æĭĵ":100786,"ç»Łè®¡":100787,"åħĦ":100788,"å¹¿æ³Ľ":100789,"åįĢ":100790,"污æŁĵ":100791,"è«ĭ":100792,"èĬĤ缮":100793,"伦":100794,"è¦ĨçĽĸ":100795,"èĢIJ":100796,"æī¶è´«":100797,"ç»ıåİĨ":100798,"éĩįè¦ģçļĦ":100799,"èĤ¡ä¸ľ":100800,"æĭĽèģĺ":100801,"åĽĽä¸ª":100802,"æĩī":100803,"èĥŀ":100804,"æijĨ":100805,"é«ĺéĢŁ":100806,"麦":100807,"åİŁåĪĻ":100808,"èݱ":100809,"æĽ´å¥½":100810,"éķľ":100811,"åĩĮ":100812,"åŀĥåľ¾":100813,"é̲":100814,"çģ°":100815,"éĵº":100816,"äºĭæķħ":100817,"çĶĺ":100818,"空æ°Ķ":100819,"é¾Ħ":100820,"èı²":100821,"çĵ¶":100822,"æĺ¨":100823,"æĹ¥æĬ¥":100824,"æµ®":100825,"åľ°åĽ¾":100826,"åijĪ":100827,"大åĬĽ":100828,"绪":100829,"å¸ħ":100830,"æľįåĭĻ":100831,"ä¸įéĶĻ":100832,"乡æĿij":100833,"å±¥":100834,"å¹³æĸ¹":100835,"éĹ²":100836,"æī£":100837,"ç´łè´¨":100838,"èµ´":100839,"éģŃ":100840,"èIJ¨":100841,"èĩªä¸»":100842,"éĩijå±ŀ":100843,"èī¯å¥½":100844,"两年":100845,"æ³¥":100846,"é¢ľ":100847,"精彩":100848,"ä¸Ńåįİ":100849,"æĻĭ":100850,"ä¹łè¿ij":100851,"ä¹łè¿ijå¹³":100852,"æĪĺ士":100853,"åģļçļĦ":100854,"éªij":100855,"æ»´":100856,"çĵľ":100857,"çīĪæĿĥ":100858,"èĤł":100859,"æľĥåĵ¡":100860,"çıį":100861,"種":100862,"仿":100863,"çī©ä¸ļ":100864,"åĢĭ人":100865,"妻":100866,"伸":100867,"æ±Ĺ":100868,"æĹº":100869,"çIJĨæĥ³":100870,"æij¸":100871,"è¿Ŀæ³ķ":100872,"å®Įæķ´":100873,"åݦ":100874,"è¸ı":100875,"æĸij":100876,"æ¡Ĥ":100877,"ä½ĵåζ":100878,"師":100879,"æĿĨ":100880,"殿":100881,"æ¯ģ":100882,"é¦Ī":100883,"è§Ĵ度":100884,"欣":100885,"çĥ¦":100886,"èĤº":100887,"éĩĩ访":100888,"æijĺ":100889,"æĮ¡":100890,"æ·ĺ":100891,"åħ»èĢģ":100892,"çĤ¸":100893,"è¿Ī":100894,"åİī":100895,"åĿĬ":100896,"è¾£":100897,"åĩĿ":100898,"泪":100899,"çĸı":100900,"æİĺ":100901,"åĥıæĺ¯":100902,"éĽķ":100903,"ç¼Ŀ":100904,"èį·":100905,"æį·":100906,"åł¡":100907,"åı¥è¯Ŀ":100908,"çĸ¼":100909,"æłı":100910,"éģµ":100911,"碳":100912,"å·¥åķĨ":100913,"æIJº":100914,"åĪ¥":100915,"ä¹Ļ":100916,"æĹĭ":100917,"æĥľ":100918,"ä¸Ģ大":100919,"å±Ĥ次":100920,"èµĸ":100921,"æĬ¬":100922,"æ¨Ĥ":100923,"è¯ŀ":100924,"åħĴ":100925,"篮":100926,"èĤĥ":100927,"å§¿":100928,"æĬļ":100929,"çĵ·":100930,"ç͵åĬ¨":100931,"æĸ°åĨł":100932,"æ¶µ":100933,"ç¢ij":100934,"æ·®":100935,"æĹ¨":100936,"踪":100937,"æ¸Ķ":100938,"æĦĪ":100939,"åıĶ":100940,"åįĹçľģ":100941,"義":100942,"å§Ķ书记":100943,"貸":100944,"æ¶Į":100945,"è«ĸ":100946,"èIJĦ":100947,"æıı":100948,"å¿§":100949,"辦":100950,"å¦Ĩ":100951,"æīŃ":100952,"åijµ":100953,"éģ¥":100954,"許":100955,"ä»ĩ":100956,"åįģä¸ī":100957,"åī²":100958,"èªį":100959,"èΰ":100960,"é¢ĩ":100961,"饱":100962,"çĭł":100963,"é«ĺçļĦ":100964,"çµ±":100965,"æħİ":100966,"é¢ģ":100967,"åIJĪéĢĤ":100968,"æµ´":100969,"èµĭ":100970,"æĬ¼":100971,"妥":100972,"éĻ¢éķ¿":100973,"èĢķ":100974,"辨":100975,"æħ°":100976,"åįģåĽĽ":100977,"æľµ":100978,"èĵĦ":100979,"æŀ¢":100980,"å»·":100981,"æĤĦ":100982,"涯":100983,"磩":100984,"åŃIJéĩĮ":100985,"çĬ¹":100986,"å±Ģéķ¿":100987,"éIJ":100988,"å¥ł":100989,"ä¼ļéķ¿":100990,"æĵļ":100991,"ä¸įåıĬ":100992,"åįģä¹Ŀ":100993,"欺":100994,"躺":100995,"éĺIJ":100996,"çºĮ":100997,"註":100998,"åĨĬ":100999,"èŃĺ":101000,"é«ĺçŃī":101001,"èħº":101002,"å¤ķ":101003,"ç»ij":101004,"åͤ":101005,"èķ´":101006,"çķľ":101007,"æħĭ":101008,"åıĻ":101009,"åıĥ":101010,"峡":101011,"人大":101012,"éħ¿":101013,"éģ©":101014,"奢":101015,"åı£æ°Ķ":101016,"éĮĦ":101017,"éı":101018,"åĭĺ":101019,"è´¿":101020,"éļª":101021,"éĭ":101022,"éļ¶":101023,"ð¥":101024,"ð¬£":101025,"ð£":101026,"ð«į":101027,"ð¬³":101028,"ð«ĵ":101029,"ð«Ħ":101030,"ð«Ł":101031,"ð¨±":101032,"äĹ":101033,"以åıĬ":101034,"æľīéĻIJ":101035,"åij¢":101036,"åIJĹ":101037,"çľĭåΰ":101038,"计åĪĴ":101039,"è¿Ľåħ¥":101040,"缴æİ¥":101041,"åĪĨæŀIJ":101042,"åıªæľī":101043,"设å¤ĩ":101044,"åħ¶å®ŀ":101045,"åĬłå¼º":101046,"ä¸ŃçļĦ":101047,"ä¿Ŀéļľ":101048,"èĢģå¸Ī":101049,"人æīį":101050,"å¾Ĺåΰ":101051,"é£İéĻ©":101052,"ä¸Ģç§į":101053,"空éĹ´":101054,"æĪijåĽ½":101055,"ä¹ĭåīį":101056,"ä¸ĵå®¶":101057,"æĿ¨":101058,"æĹ¥æľ¬":101059,"群ä¼Ĺ":101060,"åıĤåĬł":101061,"æķĪæŀľ":101062,"æľīåħ³":101063,"å®¶åºŃ":101064,"åĮºåŁŁ":101065,"åĬªåĬĽ":101066,"éļıçĿĢ":101067,"æĹłæ³ķ":101068,"交æµģ":101069,"è¡Į为":101070,"æ£ĢæŁ¥":101071,"æľŁéĹ´":101072,"å¦ĤæŃ¤":101073,"èĤ¡ä»½":101074,"å½ĵæĹ¶":101075,"è£ħå¤ĩ":101076,"åĩĨå¤ĩ":101077,"éħĴåºĹ":101078,"è¿IJåĬ¨":101079,"æıIJåĩº":101080,"å·¦åı³":101081,"æİªæĸ½":101082,"é£Łåĵģ":101083,"æ¶Īè´¹èĢħ":101084,"åѦéĻ¢":101085,"æĮĩ导":101086,"è¿IJèIJ¥":101087,"éĩį大":101088,"åĨľæĿij":101089,"éĢłæĪIJ":101090,"æĶ¿æ²»":101091,"éĴĪ对":101092,"æŃ£å¼ı":101093,"åıĸå¾Ĺ":101094,"éĤ£ä¸ª":101095,"éĽĨä¸Ń":101096,"åıªèĥ½":101097,"å¿«éĢŁ":101098,"身ä½ĵ":101099,"åħļåijĺ":101100,"èģĶåIJĪ":101101,"åĬĽéĩı":101102,"éĥ½æľī":101103,"æħ§":101104,"å¡Ķ":101105,"åĪ«äºº":101106,"表çݰ":101107,"æķħäºĭ":101108,"ä¸ĢåĪĩ":101109,"å°ĩ":101110,"èµĦæĸĻ":101111,"åŁ¹åħ»":101112,"éĺħ读":101113,"æľī人":101114,"èIJ¥éĶĢ":101115,"çĽijçĿ£":101116,"çݯä¿Ŀ":101117,"èĢĥèĻij":101118,"æ·±åľ³":101119,"严éĩį":101120,"èĮĥåĽ´":101121,"å§Ķåijĺ":101122,"çĽij管":101123,"ä¸ī个":101124,"è£ħä¿®":101125,"åħ¬éĩĮ":101126,"åĪĨåĪ«":101127,"çIJĨè§£":101128,"飩":101129,"åĬłå·¥":101130,"è®¤çľŁ":101131,"ä¸į好":101132,"åݻ年":101133,"éĻįä½İ":101134,"æľºä¼ļ":101135,"åįıè®®":101136,"符åIJĪ":101137,"å¢ŀ强":101138,"æĬĢèĥ½":101139,"é¦ĸåħĪ":101140,"秦":101141,"ä¸ģ":101142,"å°¾":101143,"æľīäºĨ":101144,"åľ°äº§":101145,"æ¸ł":101146,"æĸ¹ä¾¿":101147,"ç§»åĬ¨":101148,"éĢŁåº¦":101149,"å°¤åħ¶":101150,"éĢļçŁ¥":101151,"åĿĽ":101152,"éģ¿åħį":101153,"æģ¢":101154,"è´¡":101155,"èģĮå·¥":101156,"å®ŀåĬĽ":101157,"æĺ¯ä¸Ģç§į":101158,"åIJ¯åĬ¨":101159,"çĸ¾çĹħ":101160,"æĿ¥äºĨ":101161,"çĽ¸å¯¹":101162,"çݰå®ŀ":101163,"èŀįåIJĪ":101164,"åIJĮæł·":101165,"åħ¬åijĬ":101166,"ç®Ĭ":101167,"ç´«":101168,"ä¸ĭåİ»":101169,"ä¼łæĴŃ":101170,"æľĢ好":101171,"ä¼ĺè´¨":101172,"æ²Ĵ":101173,"æĮº":101174,"æĹ¦":101175,"诺":101176,"ä¸ĢåIJį":101177,"éģĵè·¯":101178,"示èĮĥ":101179,"è¿ĩæĿ¥":101180,"åIJĮåѦ":101181,"é¼ĵ":101182,"æĿŃ":101183,"æľ¬æ¬¡":101184,"åIJĮæĦı":101185,"ä¸ĸ纪":101186,"ç¾Ĭ":101187,"欲":101188,"å·¥èīº":101189,"çĵ¦":101190,"人士":101191,"æľīæīĢ":101192,"ä»İäºĭ":101193,"æľīå¾Īå¤ļ":101194,"ä¸įäºĨ":101195,"å²Ĺä½į":101196,"åıĺå¾Ĺ":101197,"åĬ³åĬ¨":101198,"å¤Ħäºİ":101199,"å¹³åĿĩ":101200,"形象":101201,"å¡ŀ":101202,"åħ±äº«":101203,"çĿĽ":101204,"åĪ©æ¶¦":101205,"æŃ£æĺ¯":101206,"å¾Ģå¾Ģ":101207,"缸æ¯Ķ":101208,"横":101209,"åĪ·":101210,"æµĻæ±Ł":101211,"大éĥ¨åĪĨ":101212,"å¤ļ个":101213,"æĤ¨çļĦ":101214,"ç͵åķĨ":101215,"å¾®åįļ":101216,"å§ĭç»Ī":101217,"çĬ¯ç½ª":101218,"æĺ¯åľ¨":101219,"ç»ĦåIJĪ":101220,"åİŁæĿ¥":101221,"æ¸ħæ¥ļ":101222,"åIJĦåľ°":101223,"æĦŁåıĹ":101224,"å½ĵä¸Ń":101225,"è¶ĭåĬ¿":101226,"æĻ¯åĮº":101227,"羣æĺ¯":101228,"ä¾ĽåºĶ":101229,"转åŀĭ":101230,"çĭĤ":101231,"èĨľ":101232,"èĭĹ":101233,"å¿ł":101234,"å¾Ī大":101235,"èĤ¡æĿĥ":101236,"ç¾İåħĥ":101237,"æİĴåIJį":101238,"åĬ¨çī©":101239,"éĶħ":101240,"墨":101241,"主å¸Ń":101242,"å¾Ī好":101243,"ç»Ŀ对":101244,"æĿľ":101245,"转载":101246,"çĴĥ":101247,"æĿijæ°ij":101248,"åIJ¨":101249,"åĽŃåĮº":101250,"é«ĺ度":101251,"çī©è´¨":101252,"è¾ī":101253,"æĹ¥å¸¸":101254,"æıĴ":101255,"ä¸īå¹´":101256,"ä½ĵçݰ":101257,"æīįæĺ¯":101258,"代çIJĨ":101259,"ä¸į管":101260,"æģĴ":101261,"åľ°ä½į":101262,"ç²®":101263,"èĸĦ":101264,"æĺİçϽ":101265,"ä¸Ģèĩ´":101266,"æĽ¼":101267,"åĵŃ":101268,"åĩ¤":101269,"åĬ²":101270,"æķĮ":101271,"æĪĺæĸĹ":101272,"主ä½ĵ":101273,"åħ¬å¸ĥ":101274,"åıĤèĢĥ":101275,"èĪªç©º":101276,"寺":101277,"åѦä¼ļ":101278,"åıįæĺł":101279,"ç¾İ丽":101280,"太éĺ³":101281,"建æĪIJ":101282,"æħ¢æħ¢":101283,"åIJĦ个":101284,"éĤ¦":101285,"ç»ĦæĪIJ":101286,"ä¸ī大":101287,"éͦ":101288,"大å¤ļæķ°":101289,"æ¦Ĥ念":101290,"éŃĤ":101291,"åħ¬çĽĬ":101292,"èįĴ":101293,"身份":101294,"æ·±åĪ»":101295,"åħ©":101296,"ç»ıåħ¸":101297,"åIJĦ项":101298,"èĻķ":101299,"è¿ĽæŃ¥":101300,"åįģäºĮ":101301,"æī§æ³ķ":101302,"æĥ³åΰ":101303,"æĦŁæŁĵ":101304,"åķĨåĬ¡":101305,"å°ıç»Ħ":101306,"èͬ":101307,"çıŃåŃIJ":101308,"åIJĮå¿Ĺ":101309,"éĿ¢ä¸´":101310,"çĤĴ":101311,"å¤ļç§į":101312,"è§ĤçĤ¹":101313,"åĵªéĩĮ":101314,"å°Ŀ":101315,"å§Ĩ":101316,"èħ¹":101317,"åŁİåĮº":101318,"太å¤ļ":101319,"çĹħæ¯Ĵ":101320,"åľ¨äºİ":101321,"æīĢè°ĵ":101322,"æĻ°":101323,"æŀĿ":101324,"æĭĸ":101325,"å®ħ":101326,"æķ´æ²»":101327,"ä½ıæĪ¿":101328,"åģ·":101329,"çĨĬ":101330,"èµģ":101331,"æ°Ľ":101332,"æł¼å±Ģ":101333,"åŁºç¡Ģä¸Ĭ":101334,"èĥĨ":101335,"åħ½":101336,"鼶åĶ®":101337,"åĿ¡":101338,"女åŃ©":101339,"æĴŀ":101340,"åħ¨åĬĽ":101341,"åĴĸ":101342,"èĤ©":101343,"çľī":101344,"èĩ³äºİ":101345,"åħļç»Ħ":101346,"ä¸Ģä»¶":101347,"æĭĨ":101348,"äºĭå®ŀ":101349,"åĤ³":101350,"æ¹ĺ":101351,"ç¶²ç«Ļ":101352,"循çݯ":101353,"åIJĮæ¯Ķ":101354,"æĭĶ":101355,"åĮ»èį¯":101356,"åħ»æ®ĸ":101357,"åĽºå®ļ":101358,"å®ŀéĻħä¸Ĭ":101359,"è®°å¾Ĺ":101360,"åĪ©äºİ":101361,"æĤ¦":101362,"æĭ³":101363,"èĤĿ":101364,"æķĪçĽĬ":101365,"該":101366,"æ°ij主":101367,"çĹĩçĬ¶":101368,"風":101369,"å¹¼åĦ¿":101370,"å§ij":101371,"æĪĴ":101372,"ä¸ĭçļĦ":101373,"渡":101374,"å¹´åºķ":101375,"è®°å¿Ĩ":101376,"åIJIJ":101377,"大å¹ħ":101378,"å¾½":101379,"åħ¬ä¼Ĺ":101380,"ä¿¡å¿ĥ":101381,"çİĽ":101382,"ä¼ļä¸Ĭ":101383,"ä¹Ķ":101384,"æijĦå½±":101385,"æ£ĭçīĮ":101386,"éĻķ":101387,"åºĶæĢ¥":101388,"æĶ¶è´¹":101389,"æİ§èĤ¡":101390,"仪å¼ı":101391,"çŀ¬":101392,"æīĢåľ¨":101393,"碰":101394,"å§ĵ":101395,"é¡Į":101396,"æĶ¯éĥ¨":101397,"使åij½":101398,"çĤī":101399,"å¯Ħ":101400,"翼":101401,"åľ°ä¸ĭ":101402,"è¾ŀ":101403,"俱":101404,"主æĮģ":101405,"è´§å¸ģ":101406,"æģ¨":101407,"èĤĮ":101408,"çĽĪ":101409,"éĶ»":101410,"å¿ĹæĦ¿":101411,"类似":101412,"æĮĸ":101413,"éĢ»":101414,"總":101415,"纪念":101416,"åķ¥":101417,"弯":101418,"åIJįåŃĹ":101419,"åģ¥èº«":101420,"çļĦå¿ĥ":101421,"驱":101422,"èĥĮåIJİ":101423,"æ³ķå¸Ī":101424,"ç²Ĵ":101425,"èĥ½éĩı":101426,"è¾°":101427,"èī³":101428,"å½¼":101429,"段æĹ¶éĹ´":101430,"åIJĪæ³ķ":101431,"æĵ¦":101432,"ç¾½":101433,"åݨ":101434,"æĪij说":101435,"äºĭåĬ¡":101436,"åĩłå¤©":101437,"åħģ":101438,"ç¼´":101439,"åįĵ":101440,"两ç§į":101441,"çĭ¬çī¹":101442,"帶":101443,"éĴ»":101444,"æĥ©":101445,"é¢ĨåħĪ":101446,"è¶³å¤Ł":101447,"壳":101448,"æĦıåij³çĿĢ":101449,"åĪĨå¸ĥ":101450,"ä¹ĥ":101451,"éģĭ":101452,"佩":101453,"è°±":101454,"çģ£":101455,"èį¡":101456,"贯彻":101457,"å¹¾":101458,"ç£ģ":101459,"åħ¸åŀĭ":101460,"åīĩ":101461,"åĨ»":101462,"æ¬ł":101463,"ä¸įä¹ħ":101464,"浦":101465,"éŃħ":101466,"å¼ĢäºĨ":101467,"使ç͍èĢħ":101468,"è¿Ļ款":101469,"å°Ī":101470,"èĦ±è´«":101471,"æĶ»åĿļ":101472,"ç®Ĺæĺ¯":101473,"ç¨Ģ":101474,"æĹłäºº":101475,"åłµ":101476,"å¥ı":101477,"éĥ½å¸Ĥ":101478,"åı¯è§ģ":101479,"ä¸įåĩº":101480,"æ·»":101481,"äºı":101482,"ç¾İ好":101483,"èĥĸ":101484,"飵":101485,"æłĩå¿Ĺ":101486,"èĬĤèĥ½":101487,"æĬ«":101488,"å°º":101489,"寸":101490,"ä¸Ģ代":101491,"é¢Ĺ":101492,"è̶":101493,"èĴ¸":101494,"åĸ®":101495,"滿":101496,"çĮľ":101497,"æµĨ":101498,"åŁĥ":101499,"åįĥä¸ĩ":101500,"èµĮ":101501,"èģ²":101502,"ä½ľé£İ":101503,"質":101504,"寨":101505,"年人":101506,"åį°è±¡":101507,"æ¡¶":101508,"æĴ¤":101509,"åįģäºĶ":101510,"æ¯ħ":101511,"沪":101512,"åĽ½æľī":101513,"大éĩıçļĦ":101514,"御":101515,"å¯ĵ":101516,"è¦ĸ":101517,"æ¼Ĥ亮":101518,"çľł":101519,"çĤŃ":101520,"é»İ":101521,"èϹ":101522,"åĪ©äºļ":101523,"èŃī":101524,"æµı":101525,"åįģåħ«":101526,"丢":101527,"è¾½":101528,"æľīä¸ĢäºĽ":101529,"æħĪ":101530,"åģľè½¦":101531,"å®ł":101532,"è§£æĶ¾":101533,"æľīå¤ļ":101534,"éĤĬ":101535,"常è§ģ":101536,"æĬ¹":101537,"纤":101538,"親":101539,"æ¡Ĩ":101540,"èİŀ":101541,"æ°§åĮĸ":101542,"è¿Ļä»¶":101543,"åĩ°":101544,"æŁ´":101545,"åıijç͵":101546,"é¼ł":101547,"转åĮĸ":101548,"å¨ĥ":101549,"æĮ¤":101550,"罩":101551,"å¯ĨåĪĩ":101552,"æĪijä¸į":101553,"é«ĺæĸ°":101554,"ä¸Ģç¯ĩ":101555,"è¿Ľç¨ĭ":101556,"è¡°":101557,"è¿ĺä¸į":101558,"çħĮ":101559,"æĸ°åįİ":101560,"èĤ¿":101561,"滩":101562,"ä¸Ģæµģ":101563,"è¯Ī":101564,"å®ŀä½ĵ":101565,"å¤ĸåĽ½":101566,"躲":101567,"èµł":101568,"覺":101569,"æ¢Ŀ":101570,"ä¸įè§ģ":101571,"è¨Ĭ":101572,"åĮ¹":101573,"åįµ":101574,"çĩ¥":101575,"æħķ":101576,"齿":101577,"å®´":101578,"饼":101579,"èij¡èIJĦ":101580,"å°ıå¿ĥ":101581,"æģ¼":101582,"éĻĮ":101583,"æĺĤ":101584,"åĥ¹":101585,"èĬĿ":101586,"æ¯ı个人":101587,"åīįæıIJ":101588,"ä½ĵä¼ļ":101589,"æ¨Ļ":101590,"æIJľçĭIJ":101591,"对åħ¶":101592,"丧":101593,"èľĤ":101594,"浸":101595,"調":101596,"åĿª":101597,"é¢ĸ":101598,"åIJį为":101599,"笼":101600,"èĪĮ":101601,"æľ¬ä¹¦":101602,"èģ¯":101603,"纺":101604,"ç®Ģ缴":101605,"éĽ¢":101606,"ç¾İçļĦ":101607,"éļ¨":101608,"é«ĺå³°":101609,"è¿Ļå®¶":101610,"åĤ¬":101611,"å°¸":101612,"ç¡ķ士":101613,"èŃ·":101614,"è°¨":101615,"æĺı":101616,"æĶ¿åįı":101617,"è¡Ķ":101618,"ç¿Ĵ":101619,"åľĴ":101620,"åĽ½æ°ij":101621,"主è§Ĵ":101622,"è£ķ":101623,"伪":101624,"åºŀ":101625,"æ°ijèIJ¥":101626,"æĥ§":101627,"ç§ĺ书":101628,"çĹķ":101629,"çϾåĪĨ":101630,"溶":101631,"æĹłçĸij":101632,"çļĦçľ¼":101633,"æĵİ":101634,"ä¼Łå¤§":101635,"å½°":101636,"åħ¬å®īå±Ģ":101637,"ç³ķ":101638,"å¼¥":101639,"åĤĻ":101640,"ä¹¾":101641,"毫ä¸į":101642,"注æĺİ":101643,"å̻":101644,"æĦī":101645,"æķ¦":101646,"馨":101647,"æĶĢ":101648,"éĢĿ":101649,"åı¯éĿł":101650,"夸":101651,"åľĺ":101652,"éĿ¢ä¸Ĭ":101653,"æĬĸ":101654,"èĦĨ":101655,"é©°":101656,"ä¼IJ":101657,"妨":101658,"å®ļäºĨ":101659,"ç³Ĭ":101660,"æŃ¡":101661,"éĥ¨éķ¿":101662,"ç§ī":101663,"èĪĨ":101664,"åĪijäºĭ":101665,"åIJµ":101666,"æ¤Ĵ":101667,"è¡ĵ":101668,"豫":101669,"èı©":101670,"åѵ":101671,"饲":101672,"就好":101673,"åłª":101674,"ä¸īè§Ĵ":101675,"åľºæ¯ĶèµĽ":101676,"ä¸įåģľ":101677,"æĵħ":101678,"åħ¨æĸĩ":101679,"æ³ģ":101680,"åѦä½į":101681,"æ±°":101682,"éłĺ":101683,"åıł":101684,"éļĽ":101685,"å¸IJ":101686,"çľĭåĩº":101687,"åĮł":101688,"å±ĢéĿ¢":101689,"æ³Į":101690,"è°Ĭ":101691,"åIJĮæľŁ":101692,"æĬķæłĩ":101693,"奴":101694,"æĿ¥çľĭçľĭ":101695,"èĦ¾":101696,"èŀº":101697,"æŃī":101698,"çĽ¯":101699,"ç¨İåĬ¡":101700,"å»Ĭ":101701,"æİ©":101702,"æħ¨":101703,"çĽ¼":101704,"èĬĴ":101705,"è®Ģ":101706,"æĮ£":101707,"èĮħ":101708,"æĸ¥":101709,"æ¤ħ":101710,"åΰæĿ¥":101711,"èijĹä½ľ":101712,"çĭ±":101713,"äºĮæīĭ":101714,"ä»İæĿ¥":101715,"çĸ²":101716,"åºĬä¸Ĭ":101717,"æĸ°æµª":101718,"æ³Ħ":101719,"å¢ŀå̼":101720,"丼":101721,"æļij":101722,"ä»İä¸ļ":101723,"æ·ĭ":101724,"å¤ļæł·":101725,"æľ´":101726,"份é¢Ŀ":101727,"æŀ£":101728,"西çľģ":101729,"æľ¬è´¨":101730,"深深":101731,"èīĩ":101732,"绵":101733,"产å̼":101734,"æ¼ł":101735,"èħ»":101736,"çŃĽ":101737,"åİĮ":101738,"æģŃ":101739,"å«Įçĸij":101740,"æĪ¶":101741,"æ»ŀ":101742,"èĨĢ":101743,"åĬ£":101744,"座è°Ī":101745,"常æĢģ":101746,"çļĦæĥħ":101747,"覽":101748,"å¯Ĥ":101749,"åĮĨ":101750,"èĩº":101751,"顯":101752,"çķı":101753,"éģ£":101754,"åįľ":101755,"çŃīå¥ĸ":101756,"責":101757,"溯":101758,"éİ":101759,"çĤ¹å¤´":101760,"èĵ¬":101761,"決":101762,"éħ¬":101763,"éģĬ":101764,"è³¼":101765,"註åĨĬ":101766,"æľ¬æĬ¥":101767,"çµķ":101768,"æ´»æĢ§":101769,"åħij":101770,"éĮ¯":101771,"åĨ¶":101772,"åĸ»":101773,"æºĸ":101774,"èĤ¢":101775,"æºĥ":101776,"æĹ¬":101777,"åīĬ":101778,"çIJĨäºĭ":101779,"å±ł":101780,"æ²§":101781,"èļĢ":101782,"鼻åŃIJ":101783,"为æŃ¢":101784,"常å§Ķ":101785,"çµĤ":101786,"éĬ·":101787,"çĭĢ":101788,"ä¾£":101789,"èĥĢ":101790,"èѰ":101791,"çĶ¨è½¦":101792,"åĻª":101793,"æŃ·":101794,"åįĶ":101795,"åι":101796,"竣æĺ¯":101797,"é©Ĺ":101798,"èIJĿ":101799,"çĻ«":101800,"çĹ«":101801,"æŃ§":101802,"å¼Ĭ":101803,"媽":101804,"çıĬ":101805,"è¡·":101806,"éľī":101807,"åŁºçĿ£":101808,"éļ±":101809,"æ°¨":101810,"绸":101811,"å°¼æĸ¯":101812,"çĥĺ":101813,"æľŁåĨħ":101814,"è°ħ":101815,"éĽĩ":101816,"éļĻ":101817,"åĸī":101818,"åī¥":101819,"çĹĺ":101820,"æĮ½":101821,"çĵ£":101822,"æ¹Ľ":101823,"樱":101824,"æ¾İ":101825,"æ¹ĥ":101826,"åĨ¬å¥¥":101827,"棵":101828,"å®°":101829,"åŀĴ":101830,"æ§ĭ":101831,"ä¾Ī":101832,"èĮĦ":101833,"åĺ¿":101834,"èıĩ":101835,"çĻĤ":101836,"åĬĥ":101837,"éį":101838,"èͽ":101839,"çŀŃ":101840,"æķŀ":101841,"ä¹ĸ":101842,"飧":101843,"è¾ľ":101844,"æĩĪ":101845,"ä½£":101846,"çŀ»":101847,"åŁĶ":101848,"èĪħ":101849,"å®ŀäºĭ":101850,"é¨":101851,"å§¥":101852,"絡":101853,"åĺ»":101854,"çķ¢":101855,"æ²ĥå°Ķ":101856,"è¿Ħ":101857,"èĤĩ":101858,"æħij":101859,"ã§":101860,"äı":101861,"ðł":101862,"ð¬ĩ":101863,"ð«Ń":101864,"ð«IJ":101865,"ã³":101866,"©½":101867,"ð«ł":101868,"ãĽ":101869,"ð¬į":101870,"é¿":101871,"ð¬Ĵ":101872,"ãĻ":101873,"ð¬¤":101874,"ð¬´":101875,"ð«ĸ":101876,"ð¤":101877,"ã¬":101878,"ä²":101879,"ð«Ķ":101880,"ð«ļ":101881,"è¦ģæ±Ĥ":101882,"ä¸ĢäºĽ":101883,"å®ŀçݰ":101884,"èĢĮä¸Ķ":101885,"åĽłæŃ¤":101886,"çͱäºİ":101887,"åħ³äºİ":101888,"çĦ¶åIJİ":101889,"æİ¨åĬ¨":101890,"ä¸Ģæł·":101891,"æĮīçħ§":101892,"è¿Ļæł·çļĦ":101893,"å½¢æĪIJ":101894,"æľīäºĽ":101895,"æĽ´åĬł":101896,"ç»ıè¿ĩ":101897,"建议":101898,"æ²»çĸĹ":101899,"ä½łä»¬":101900,"æīįèĥ½":101901,"ä¿ĥè¿Ľ":101902,"åijĺå·¥":101903,"ä½ĵéªĮ":101904,"èĪĩ":101905,"åģļ好":101906,"ä¿Ŀè¯ģ":101907,"æķ´ä¸ª":101908,"æĺ¯ä¸Ģ个":101909,"éĩĩç͍":101910,"çIJĨ论":101911,"æ¯Ķå¦Ĥ":101912,"ä¸ĬçļĦ":101913,"æİ¨èįIJ":101914,"çĶ³è¯·":101915,"天空":101916,"éĥ¨èIJ½":101917,"åįģåĪĨ":101918,"æĿ¥èĩª":101919,"ä¹ĭéĹ´":101920,"è°ĥæķ´":101921,"æ¯ı天":101922,"è°ĥæŁ¥":101923,"æĤ£èĢħ":101924,"è¿ĩç¨ĭä¸Ń":101925,"é¦Ļ港":101926,"广åijĬ":101927,"éĿ¢å¯¹":101928,"满足":101929,"éķ¿æľŁ":101930,"è§ĦèĮĥ":101931,"æķ´ä½ĵ":101932,"æĶ¹åıĺ":101933,"æĻºæħ§":101934,"å¦Īå¦Ī":101935,"å¦Ĥä»Ĭ":101936,"åIJĪåIJĮ":101937,"éĥ½ä¼ļ":101938,"åĦ¿ç«¥":101939,"åĩıå°ij":101940,"éŁ³ä¹IJ":101941,"ç»ı常":101942,"ä¸Ĭå¸Ĥ":101943,"ä¼ĺç§Ģ":101944,"çļĦéĩįè¦ģ":101945,"ä¸ĢæĿ¡":101946,"æµ·å¤ĸ":101947,"åı¦å¤ĸ":101948,"ä¸Ģå®¶":101949,"åİĭåĬĽ":101950,"大åŀĭ":101951,"çľĭçĿĢ":101952,"åĪĢ":101953,"幸ç¦ı":101954,"æİ¨å¹¿":101955,"åIJĽ":101956,"å¾IJ":101957,"æī¾åΰ":101958,"äºİæĺ¯":101959,"èĩªèº«":101960,"ä¸Ģä½į":101961,"åľŁåľ°":101962,"åĬłåħ¥":101963,"æİ¢ç´¢":101964,"æ¢ģ":101965,"主åĬ¨":101966,"å°±ä¸ļ":101967,"女æĢ§":101968,"çªģçł´":101969,"ä¸įåIJĮçļĦ":101970,"è¿IJè¾ĵ":101971,"èĩªçͱ":101972,"å±ħæ°ij":101973,"æŃ¤æ¬¡":101974,"çļĦæĹ¶éĹ´":101975,"å®¶éķ¿":101976,"ä¸Ģ个人":101977,"æ£Ģæµĭ":101978,"åĨħéĥ¨":101979,"广å·ŀ":101980,"缴æĴŃ":101981,"ä»İèĢĮ":101982,"贷款":101983,"åı¬å¼Ģ":101984,"æĶ¹éĢł":101985,"人çĶŁ":101986,"å±ķ示":101987,"æ¯ıå¹´":101988,"女人":101989,"çļĦæĸ¹å¼ı":101990,"æķĪçİĩ":101991,"å±±ä¸ľ":101992,"æ¸łéģĵ":101993,"ä¼¼ä¹İ":101994,"æ¡Īä»¶":101995,"åĪ©çĽĬ":101996,"çľĭçľĭ":101997,"å¿ĥéĩĮ":101998,"ç»´æĬ¤":101999,"å®Ŀå®Ŀ":102000,"ç½ijä¸Ĭ":102001,"论åĿĽ":102002,"å°±åı¯ä»¥":102003,"ä¸įè¶³":102004,"æģ¢å¤į":102005,"å¸ĥå±Ģ":102006,"è´¡çĮ®":102007,"ä¸ĭéĻį":102008,"æİĮæı¡":102009,"çļ®èĤ¤":102010,"å·¥åħ·":102011,"éĩįåºĨ":102012,"åĵģè´¨":102013,"æİ¨åĩº":102014,"çĶ·äºº":102015,"æī¿æĭħ":102016,"çªģåĩº":102017,"èĢĮè¨Ģ":102018,"æ²Ł":102019,"åįıè°ĥ":102020,"æĺ¯ä»Ģä¹Ī":102021,"汤":102022,"æĴij":102023,"çĭ¬ç«ĭ":102024,"çݯèĬĤ":102025,"æī©å¤§":102026,"æ´ª":102027,"æĿ°":102028,"çĽIJ":102029,"ä»ģ":102030,"æ¶īåıĬ":102031,"èĢģ人":102032,"åį³ä½¿":102033,"åįĹ京":102034,"éħįåIJĪ":102035,"鬼":102036,"çĪ¶äº²":102037,"ç½Ĺæĸ¯":102038,"å°ıåĮº":102039,"æķĻæİĪ":102040,"åĨ³çŃĸ":102041,"é¢Ħ计":102042,"æľ¬äºº":102043,"伯":102044,"竹":102045,"åΰåºķ":102046,"å¸Ĥæ°ij":102047,"åĩºåı£":102048,"éĩĩè´Ń":102049,"æĢ»ç»ĵ":102050,"æŃ¦æ±ī":102051,"åĬłå¤§":102052,"å¹¿ä¸ľ":102053,"æµģç¨ĭ":102054,"人åı£":102055,"å¦Ĥæŀľä½ł":102056,"åĩºåİ»":102057,"åĩī":102058,"åĨľæ°ij":102059,"çݰ象":102060,"åĬĽåº¦":102061,"ç»ĻäºĪ":102062,"åħļå§Ķ":102063,"è¯Ńè¨Ģ":102064,"线ä¸Ĭ":102065,"æĢİæł·":102066,"åĦ¿åŃIJ":102067,"ç¡®å®ŀ":102068,"ä¹ĭå¤ĸ":102069,"éĥ½åľ¨":102070,"èī¾":102071,"çļĦæĥħåĨµ":102072,"éĩĮçļĦ":102073,"åĽ´ç»ķ":102074,"æĽ´å¤ļçļĦ":102075,"ä¾Ŀæ³ķ":102076,"åħ¬åĽŃ":102077,"å®¶éĩĮ":102078,"æ¯į亲":102079,"ä¸įåĨį":102080,"èĭ¹":102081,"æ³ķéĻ¢":102082,"éŁ©åĽ½":102083,"缸å½ĵ":102084,"ä¸įçŁ¥":102085,"è¯Ħä¼°":102086,"ä¸įç͍":102087,"顺åĪ©":102088,"éĩįè§Ĩ":102089,"è´¢åĬ¡":102090,"ä»ĸåĢij":102091,"åıijè¡Į":102092,"ä¸ĵéŨ":102093,"åħ·å¤ĩ":102094,"å¹¶ä¸įæĺ¯":102095,"è¶³çIJĥ":102096,"éŀĭ":102097,"åıij表":102098,"æ°¸è¿ľ":102099,"èIJ¥åħ»":102100,"éħįå¥Ĺ":102101,"æķ´åIJĪ":102102,"è´º":102103,"åĽŀçŃĶ":102104,"æĶ¶çĽĬ":102105,"ä¹Łè®¸":102106,"è»Ĭ":102107,"æİ¥è§¦":102108,"æĶ»åĩ»":102109,"åĽĽå·Ŀ":102110,"æĢ§èĥ½":102111,"åĽŀåΰ":102112,"èħ°":102113,"ä¹Łæ²¡æľī":102114,"å¼Ħ":102115,"设ç«ĭ":102116,"éĺ²æİ§":102117,"æĬĢå·§":102118,"éĢļ常":102119,"è´¢æĶ¿":102120,"éĥ¨ç½²":102121,"åľºæĻ¯":102122,"æ±Łèĭı":102123,"表达":102124,"åĸ·":102125,"女åĦ¿":102126,"èζ":102127,"給":102128,"ä¼ļåijĺ":102129,"æĪĸ许":102130,"亩":102131,"举æĸ¹":102132,"天津":102133,"è¿ijå¹´":102134,"çľĭæĿ¥":102135,"æ¯Ķä¾ĭ":102136,"岩":102137,"éĵľ":102138,"çİ»":102139,"å®ŀéªĮ":102140,"æĢĿç»´":102141,"æĭħå¿ĥ":102142,"æ²Ī":102143,"身边":102144,"æ·±åĮĸ":102145,"ç²¾åĩĨ":102146,"ç§ģæľį":102147,"æ¶Īéĺ²":102148,"åİ»äºĨ":102149,"ç»Ĩèĥŀ":102150,"çIJĥéĺŁ":102151,"æĺİæĺŁ":102152,"é£Łçī©":102153,"å¾Īå¿«":102154,"è®©ä½ł":102155,"ä¿¡ç͍":102156,"å͝ä¸Ģ":102157,"åħ¶å®ĥ":102158,"çŃīæĸ¹éĿ¢":102159,"å¾ĭå¸Ī":102160,"æŃ»äº¡":102161,"æŁ³":102162,"ä¸Ģæī¹":102163,"ä¸Ĭ涨":102164,"æľºåľº":102165,"å½¢åĬ¿":102166,"æĦ¿æĦı":102167,"éĽĨä½ĵ":102168,"æĸ°åŀĭ":102169,"æįŁå¤±":102170,"æĽ¸":102171,"ä¸ĭåįĪ":102172,"æ¯ı次":102173,"æĪIJå°±":102174,"åħ¬è·¯":102175,"èĻ«":102176,"åĴ±":102177,"西å®ī":102178,"æľĢä½³":102179,"ç§ijçłĶ":102180,"å¤įæĿĤ":102181,"æľºåύ":102182,"çαæĥħ":102183,"çħ§çīĩ":102184,"å¹´é¾Ħ":102185,"è³ĩæĸĻ":102186,"ç²Ĺ":102187,"åĩĨç¡®":102188,"åĬłä¸Ĭ":102189,"åĩºçīĪ":102190,"è°IJ":102191,"å®¶å±ħ":102192,"èĥĮæĻ¯":102193,"ä¸Ģ线":102194,"äºĭ项":102195,"åĬ¨ä½ľ":102196,"祥":102197,"æĢ»ä½ĵ":102198,"æĪ¿åŃIJ":102199,"ä¹Łå°±æĺ¯":102200,"大æ¦Ĥ":102201,"é«ĺæķĪ":102202,"åIJ¹":102203,"æİĪæĿĥ":102204,"éĻĦè¿ij":102205,"æ¡Īä¾ĭ":102206,"éĹ¹":102207,"çΏçΏ":102208,"彩票":102209,"æĢĴ":102210,"举æĬ¥":102211,"æĻ®éģį":102212,"çķĻä¸ĭ":102213,"è¡£æľį":102214,"æĹłè®ºæĺ¯":102215,"åħħ满":102216,"深度":102217,"æ¡ij":102218,"æĪªèĩ³":102219,"带æĿ¥çļĦ":102220,"éϵ":102221,"æĦŁæĥħ":102222,"èµļ":102223,"åĵªäºĽ":102224,"æķ´æĶ¹":102225,"æĪIJçĨŁ":102226,"å¨ľ":102227,"é¼»":102228,"磼":102229,"çĽ¾":102230,"好好":102231,"ç¬¬åĽĽ":102232,"åĨłåĨĽ":102233,"è´¢å¯Į":102234,"æľĢ好çļĦ":102235,"车åŀĭ":102236,"éĸĢ":102237,"åį³å°Ĩ":102238,"åĪĨ为":102239,"éĿĴå²Ľ":102240,"纷纷":102241,"ä»ĬæĹ¥":102242,"平衡":102243,"å¹³æĸ¹ç±³":102244,"éĤ£ç§į":102245,"åĩºçĶŁ":102246,"éĿĴæĺ¥":102247,"人群":102248,"人工":102249,"ä¹ĭä¸ĭ":102250,"æ¹ĸåĮĹ":102251,"åľ¨æŃ¤":102252,"åįļ士":102253,"æĹ¶åĪ»":102254,"æ²³åĮĹ":102255,"æĶ¾å¼ĥ":102256,"éĢļéģĵ":102257,"森æŀĹ":102258,"çĸĨ":102259,"æķ¸":102260,"èĬ³":102261,"æīĵåĩ»":102262,"æĽ¹":102263,"åĮĸåѦ":102264,"æĥ³è±¡":102265,"ä¸ĩ人":102266,"è´¢ç»ı":102267,"åħĥç´ł":102268,"ä¼ļ计":102269,"åħ¨ä½ĵ":102270,"æĦĽ":102271,"é«ĺä¸Ń":102272,"æľºéģĩ":102273,"å£°éŁ³":102274,"æĹħè¡Į":102275,"浩":102276,"æŁ±":102277,"å°ijå¹´":102278,"åĽ½å¤ĸ":102279,"èijĹåIJį":102280,"çĶŁåŃĺ":102281,"å§ľ":102282,"带é¢Ĩ":102283,"é¢ľèī²":102284,"ä¸Ĭä¸ĭ":102285,"产ä¸ļéĵ¾":102286,"æĽ´å¥½çļĦ":102287,"å²Ń":102288,"ä¼ĺæĥł":102289,"便æĺ¯":102290,"åħ§å®¹":102291,"ä¸Ģåıª":102292,"çIJ´":102293,"梦æĥ³":102294,"ç§Łèµģ":102295,"å¼ĢåIJ¯":102296,"è´Ńçī©":102297,"åĮħåIJ«":102298,"åĪ©çİĩ":102299,"èµ·äºĨ":102300,"æľīåĬĽ":102301,"éĤ£éĩĮ":102302,"审æī¹":102303,"对æīĭ":102304,"çݰéĩij":102305,"天çĦ¶":102306,"çĽĴ":102307,"çν":102308,"å¿ħçĦ¶":102309,"åĮĸå·¥":102310,"ä¸ĵåĪ©":102311,"åķ¡":102312,"å¼Ģå¿ĥ":102313,"人ä½ĵ":102314,"éģĵ士":102315,"æĢģ度":102316,"空è°ĥ":102317,"æĭĽåķĨ":102318,"å§»":102319,"第äºĶ":102320,"æ£Ĵ":102321,"ä¸Ģç³»åĪĹ":102322,"å᱿ľº":102323,"转åıĺ":102324,"åľºæīĢ":102325,"鸣":102326,"æĪ¿éĹ´":102327,"é̼":102328,"è¯ķçĤ¹":102329,"对å¤ĸ":102330,"åĩºåı°":102331,"åľ¨è¿Ļ":102332,"åİĤå®¶":102333,"巨大":102334,"ç®Ģä»ĭ":102335,"çľĭäºĨ":102336,"åħļ建":102337,"æĮĩæĮ¥":102338,"çŁ³æ²¹":102339,"ä¸įåı¯èĥ½":102340,"èݲ":102341,"ä¸į太":102342,"åĪĽæĦı":102343,"第ä¸Ģ个":102344,"è´µå·ŀ":102345,"è¿ĩäºĨ":102346,"æľ¬æĿ¥":102347,"éģĵå¾·":102348,"çŃĶæ¡Ī":102349,"é϶":102350,"ä¸Ģè·¯":102351,"èĤĸ":102352,"æ¸ħæ´ģ":102353,"æľīæľº":102354,"åIJįåįķ":102355,"æĿ±":102356,"åij¼åIJ¸":102357,"ä¸Ī":102358,"ç¦ı建":102359,"è¯ķéªĮ":102360,"å¼ķåıij":102361,"ä¹Łæ²¡":102362,"ä¸įä½ı":102363,"çĨŁæĤī":102364,"èIJ¬":102365,"ä¸įèī¯":102366,"çłĸ":102367,"èĩ´åĬĽ":102368,"çŃ¾è®¢":102369,"åIJĬ":102370,"侯":102371,"çĺ¦":102372,"å§ijå¨ĺ":102373,"æĸ¤":102374,"妻åŃIJ":102375,"æĺ¥èĬĤ":102376,"çά":102377,"æĽĿ":102378,"çĥŃæĥħ":102379,"éķ¿æ²Ļ":102380,"èIJ¥éĢł":102381,"éħ·":102382,"éĵĿ":102383,"åŁºæľ¬ä¸Ĭ":102384,"åij¨åĽ´":102385,"ä»Ģ麼":102386,"认åı¯":102387,"åĪĨåŃIJ":102388,"ä¸Ģæĸ¹éĿ¢":102389,"è½´":102390,"å¼·":102391,"马ä¸Ĭ":102392,"éĽ¾":102393,"èĩ£":102394,"å°¿":102395,"çĶŁæĦı":102396,"å®īå¾½":102397,"ç¥ŀç»ı":102398,"åĩºå¸Ń":102399,"èį¯åĵģ":102400,"çIJĨçͱ":102401,"åįıåIJĮ":102402,"æµģåĬ¨":102403,"åıijåĬ¨":102404,"åĿļå®ļ":102405,"表æĺİ":102406,"åIJİéĿ¢":102407,"ä¹īåĬ¡":102408,"å¦ĸ":102409,"æľīåı¯èĥ½":102410,"年轻人":102411,"大éĻĨ":102412,"å²³":102413,"ä¸įèµ·":102414,"çŀ¬éĹ´":102415,"ä¸įå¾Ĺä¸į":102416,"çŃ¾çº¦":102417,"åIJĪæł¼":102418,"åħļæĶ¯éĥ¨":102419,"æµİåįĹ":102420,"便åĪ©":102421,"éļıæĹ¶":102422,"å¥ī":102423,"称为":102424,"产æĿĥ":102425,"åIJķ":102426,"çĽĨ":102427,"课åłĤ":102428,"ç·ļ":102429,"æ£ī":102430,"线ä¸ĭ":102431,"èĩªè¡Į":102432,"举æİª":102433,"åݦéŨ":102434,"èĩªä¿¡":102435,"å½±è§Ĩ":102436,"ä»Ķ":102437,"çĶŁæ´»ä¸Ń":102438,"æĿĥçĽĬ":102439,"çϽèī²":102440,"å°±ä¸į":102441,"è¿Ľå±ķ":102442,"æ¯ıæĹ¥":102443,"ä¾Ľç»Ļ":102444,"æĿĥåĪ©":102445,"æĹłæķ°":102446,"çIJĨè´¢":102447,"ä¾ĿæĹ§":102448,"ä¸ĬåįĪ":102449,"è¯ĨåĪ«":102450,"çĽĪåĪ©":102451,"çłĤ":102452,"许åı¯":102453,"åIJĮäºĭ":102454,"åĺĽ":102455,"éģ¸":102456,"çĿĢåĬĽ":102457,"éŨåı£":102458,"ä¸įå¤ļ":102459,"åħ¶æ¬¡":102460,"碧":102461,"çī©çIJĨ":102462,"åĨħå¿ĥ":102463,"çϾå§ĵ":102464,"æĢ»ç»Ł":102465,"å¹²åĩĢ":102466,"积累":102467,"åıįé¦Ī":102468,"æłijç«ĭ":102469,"社交":102470,"ç§©":102471,"åįģä¸Ģ":102472,"éĤĵ":102473,"驱åĬ¨":102474,"å±ķè§Ī":102475,"èĪĴéĢĤ":102476,"åŁºåĽł":102477,"å·®å¼Ĥ":102478,"转让":102479,"å°ıå§IJ":102480,"æł·åŃIJ":102481,"ç¿Ķ":102482,"é«ĺåħ´":102483,"å½±åĵįåĬĽ":102484,"æīĭç»Ń":102485,"缸åIJĮ":102486,"缸åºĶ":102487,"æĻĴ":102488,"è§Ģ":102489,"å¸Ĥå§Ķ":102490,"èĬ¯":102491,"å±ķçݰ":102492,"åľ°çIJĥ":102493,"éĤª":102494,"ä¸Ģå®ļçļĦ":102495,"åħģ许":102496,"ä¿¡ä»»":102497,"æīij":102498,"éĻ¢æł¡":102499,"ç®Ģç§°":102500,"åģļæ³ķ":102501,"ä¹ĭè·¯":102502,"æĹĹä¸ĭ":102503,"èħĶ":102504,"æ¶Ī失":102505,"ä¸ĸçķĮä¸Ĭ":102506,"åŁİ乡":102507,"èĪŀåı°":102508,"å¾Ī大çļĦ":102509,"绣çѹ":102510,"åħ¬å¹³":102511,"èĤ¾":102512,"çļĦ好":102513,"æ±ģ":102514,"çľ¼åīį":102515,"éĽ£":102516,"å¹½":102517,"åħ±äº§":102518,"主åĬŀ":102519,"å¤Ħç½ļ":102520,"åºĻ":102521,"éģĵçIJĨ":102522,"å¼µ":102523,"æİ¥çĿĢ":102524,"çĮİ":102525,"çģĮ":102526,"çͱæŃ¤":102527,"人åĬĽ":102528,"æµģè¡Į":102529,"ä¾ł":102530,"åı¯ä»¥è¯´":102531,"èĴĭ":102532,"å½¢æĢģ":102533,"æĹ¥åŃIJ":102534,"æ¼Ĩ":102535,"çķĻåѦ":102536,"缸éĹľ":102537,"æľĢå¤ļ":102538,"åĩŃåĢŁ":102539,"åħ¬äº¤":102540,"æĮĸæİĺ":102541,"æĿĤå¿Ĺ":102542,"主人":102543,"éļľç¢į":102544,"æł¡éķ¿":102545,"æĸ¹ä½į":102546,"ä¸ĬçıŃ":102547,"å¤ļåħĥ":102548,"èĥģ":102549,"éŃħåĬĽ":102550,"èĮĤ":102551,"åħħç͵":102552,"强大":102553,"çĥ¤":102554,"å¥ĭæĸĹ":102555,"å®ŀç͍":102556,"éĺģ":102557,"ç»ĻäºĨ":102558,"æľ¬ç§ij":102559,"æłĭ":102560,"æĭ¨":102561,"æķĻç»ĥ":102562,"éĥ½çŁ¥éģĵ":102563,"æ¯ķä¸ļçĶŁ":102564,"ç¢Ĺ":102565,"åŀĤ":102566,"讼":102567,"å®ģæ³¢":102568,"åѦèĢħ":102569,"谢谢":102570,"åŁİéķĩ":102571,"æĢİä¹ĪåĬŀ":102572,"éģĶ":102573,"æĪIJ交":102574,"æ½ľåĬĽ":102575,"åį§":102576,"æĸ°å¼Ģ":102577,"éħįå¤ĩ":102578,"主åĬĽ":102579,"åij³éģĵ":102580,"çĥĤ":102581,"é£ŀè¡Į":102582,"å«ģ":102583,"大大":102584,"ç»Ļ大家":102585,"å¤ĸéĿ¢":102586,"éĨī":102587,"åıijè¨Ģ":102588,"æĹ©é¤IJ":102589,"åIJĦèĩª":102590,"å®Ļ":102591,"èį£èªī":102592,"æĬ«éľ²":102593,"é¡ŀ":102594,"åĨħçļĦ":102595,"èĤª":102596,"è¾IJ":102597,"æ³µ":102598,"æĬĽ":102599,"æĺŁæľŁ":102600,"ä¸Ģ带":102601,"çĶŁç´ł":102602,"ç»ıéĶĢ":102603,"åĩ¶":102604,"åľ°ä¸Ĭ":102605,"åij½è¿IJ":102606,"åĵ²":102607,"ä¸Ĭåİ»":102608,"æĸĩçī©":102609,"è¯ij":102610,"æĮ¯åħ´":102611,"éķ¿æĹ¶éĹ´":102612,"ç¥Ń":102613,"åIJĪèĤ¥":102614,"è¿Ŀè§Ħ":102615,"èģª":102616,"ä½İäºİ":102617,"éĢĤå½ĵ":102618,"æľīåºı":102619,"æľ¬ç½ij":102620,"çķĻè¨Ģ":102621,"æĥ³æ³ķ":102622,"çŃ¾ç½²":102623,"å§ļ":102624,"æĢ§æł¼":102625,"èĴĻåı¤":102626,"æŁı":102627,"åŀ«":102628,"åѦåİĨ":102629,"ä»ħä»ħ":102630,"讲è¯Ŀ":102631,"éĶIJ":102632,"æĢĸ":102633,"åīª":102634,"èĭį":102635,"åIJĵ":102636,"强çĥĪ":102637,"åģ¥åħ¨":102638,"çĸ¯":102639,"åı¤ä»£":102640,"å¥Ī":102641,"ä¸įçĦ¶":102642,"乡éķĩ":102643,"æľĭåıĭ们":102644,"åĤħ":102645,"èģ½":102646,"个æĢ§":102647,"æ³ķè§Ħ":102648,"å°ıéķĩ":102649,"çĶ»éĿ¢":102650,"第åħŃ":102651,"網路":102652,"åīįæĻ¯":102653,"åIJ¬è¯´":102654,"ä¼łåªĴ":102655,"æĿ¡ä¾ĭ":102656,"åĪ«çļĦ":102657,"ä¸įæĩĤ":102658,"顾éĹ®":102659,"强度":102660,"éĺ¿éĩĮ":102661,"èµ°åĬ¿":102662,"帽":102663,"çļĦç¡®":102664,"åĮºåĪ«":102665,"éĮ¢":102666,"主管":102667,"ä¸Ģçľĭ":102668,"æĸľ":102669,"åŃĺåľ¨çļĦ":102670,"仲":102671,"åį±å®³":102672,"éĵŃ":102673,"游æĪıä¸Ń":102674,"éħ±":102675,"é¾Ļ头":102676,"人å¿ĥ":102677,"éĢĢä¼ij":102678,"æµıè§Ī":102679,"åĬ«":102680,"éĺ²æ²»":102681,"ç®Ń":102682,"å±Ī":102683,"è¾½å®ģ":102684,"壤":102685,"è¿İæĿ¥":102686,"éŀį":102687,"ç͍æĿ¥":102688,"å¤§åľ°":102689,"ä»°":102690,"éĢļ讯":102691,"å¼Ģå·¥":102692,"裤":102693,"å¦ĤåIJĮ":102694,"骤":102695,"éĺŁåijĺ":102696,"轩":102697,"ç¾İæľ¯":102698,"èĻŁ":102699,"åIJĮä¸Ģ":102700,"åľĸ":102701,"书æ³ķ":102702,"æīĵåį°":102703,"åIJ«æľī":102704,"éĽĨæĪIJ":102705,"éĹ·":102706,"å¸Ĥåľºä¸Ĭ":102707,"æĹģè¾¹":102708,"åľ°æĿ¿":102709,"产çĶŁçļĦ":102710,"粤":102711,"éĩįç»Ħ":102712,"è¡Ģæ¶²":102713,"çŃĭ":102714,"åĬŀäºĭ":102715,"常è§ģçļĦ":102716,"ä¸ĬåįĬå¹´":102717,"å±ıå¹ķ":102718,"åIJīæŀĹ":102719,"å·©":102720,"åĸľçα":102721,"ç¿ł":102722,"ä¸īç§į":102723,"æ¡Ĩæŀ¶":102724,"举èİŀ":102725,"çĶĺèĤĥ":102726,"èĬ¬":102727,"åĽ¾ä¹¦":102728,"åĩ¤åĩ°":102729,"æ°ĶåĢĻ":102730,"å°´":102731,"å°¬":102732,"两天":102733,"è¾ħ导":102734,"åĢŁæ¬¾":102735,"æĹ¥èµ·":102736,"æ´Ĵ":102737,"ä¸Ģ度":102738,"è¹Ī":102739,"æ½Ń":102740,"æīĩ":102741,"çĻľ":102742,"æĸ°åħ´":102743,"åĤ²":102744,"诸å¤ļ":102745,"è´ª":102746,"éĻ·åħ¥":102747,"èĪŁ":102748,"èĤºçĤİ":102749,"ä¸Ģæł·çļĦ":102750,"åİĺ":102751,"åľ°çIJĨ":102752,"æĬķæ³¨":102753,"éļĬ":102754,"åħīä¼ı":102755,"ä¿Ŀåģ¥":102756,"åħĶ":102757,"åħ¬åĬ¡":102758,"æīĵçł´":102759,"çĶ·åŃ©":102760,"åĬ³åĬ¡":102761,"ä½łä¼ļ":102762,"çĶ¨åľ°":102763,"溢":102764,"åıijè¾¾":102765,"èĤļ":102766,"è¿ĩäºİ":102767,"èĩĤ":102768,"éĢĻæ¨£":102769,"轻轻":102770,"ä¸Ńåħ±":102771,"åIJĦåĽ½":102772,"åĶĩ":102773,"å®ŀä¹ł":102774,"èϾ":102775,"æ§½":102776,"ä¸įä¸Ĭ":102777,"åħįçĸ«":102778,"åįłæį®":102779,"å·¥ä¼ļ":102780,"åĽĬ":102781,"èĪªå¤©":102782,"åı¯çα":102783,"æĸĹäºī":102784,"çĺ¤":102785,"å¦Ĥæľī":102786,"éĽĸ":102787,"对æĪij":102788,"åĩºç§Ł":102789,"好çľĭ":102790,"太大":102791,"æ°´åĪ©":102792,"åĬ¿åĬĽ":102793,"åħ¨æ°ij":102794,"ç½¢":102795,"èµ¢å¾Ĺ":102796,"çĶµä¿¡":102797,"车éĹ´":102798,"æĻĤåĢĻ":102799,"å°ijæķ°":102800,"éĵ¸":102801,"åħ³èģĶ":102802,"ä¸įä»ħä»ħ":102803,"为æĤ¨":102804,"åĴ¸":102805,"æľºåĬ¨":102806,"è£Ļ":102807,"åĵįåºĶ":102808,"éģł":102809,"è²·":102810,"ç©´":102811,"å¢ħ":102812,"éĶ¡":102813,"çµĦ":102814,"çģ«è½¦":102815,"è³ĩè¨Ĭ":102816,"åĨ³èµĽ":102817,"污水":102818,"èªŀ":102819,"å´Ľ":102820,"ç´§å¯Ĩ":102821,"缺å°ij":102822,"å¤ļ人":102823,"æĢ»ä¹¦è®°":102824,"éĶĪ":102825,"èijĽ":102826,"å¿ĺè®°":102827,"éĻĮçĶŁ":102828,"éķ¿å¤§":102829,"åħĪè¿ĽçļĦ":102830,"ç¡ħ":102831,"åıijæĺİ":102832,"å©´åĦ¿":102833,"æīİå®ŀ":102834,"èĽĭçϽ":102835,"ä¸ĢçϾ":102836,"缮åħī":102837,"æħĮ":102838,"åĬłæ²¹":102839,"åIJŀ":102840,"ä¸Ģ群":102841,"ä¸Ńä»ĭ":102842,"å¸ĸ":102843,"å¿Į":102844,"èģĮèĥ½":102845,"广æĴŃ":102846,"çĽijå¯Ł":102847,"ç§ĺå¯Ĩ":102848,"çĭ®":102849,"è¿ĻæĿ¡":102850,"éĢ¢":102851,"æĢ¨":102852,"åįģåħŃ":102853,"試":102854,"说åΰ":102855,"åĩĿèģļ":102856,"æĮĩ示":102857,"æ°¢":102858,"å¼ĺ":102859,"éĺĢ":102860,"æĸ©":102861,"éłħ":102862,"ä¸Ģå¼Ģå§ĭ":102863,"æİĴè¡Į":102864,"åľ¨æĪij":102865,"纪å½ķ":102866,"æĬĦ":102867,"æłª":102868,"说æ³ķ":102869,"ä¸Ńèį¯":102870,"好å¤ļ":102871,"åıªä¸įè¿ĩ":102872,"çķĻåľ¨":102873,"个å°ıæĹ¶":102874,"è®¤çŁ¥":102875,"çķ«":102876,"è§ģè¿ĩ":102877,"å°ıå¾®":102878,"ä½Ľå±±":102879,"çľ¾":102880,"讲述":102881,"梳":102882,"ç§°åı·":102883,"æĹ¥æĻļ":102884,"è¢ĸ":102885,"åķ¤":102886,"æľªç»ı":102887,"æľĢæĹ©":102888,"æī®æ¼Ķ":102889,"è¡Ģ管":102890,"纱":102891,"æĥħèĬĤ":102892,"第ä¸ĥ":102893,"æį§":102894,"ä»Ĺ":102895,"æ¿ĢçĥĪ":102896,"æĹłçº¿":102897,"ä¸į容æĺĵ":102898,"å¼Ģå¹ķ":102899,"æĸ°çĶŁ":102900,"ä¸ĵ注":102901,"èij±":102902,"åįĹæµ·":102903,"çĩŁ":102904,"èµ·ä¾Ĩ":102905,"æ´¾åĩº":102906,"åĦĴ":102907,"侨":102908,"è¼ĥ":102909,"åįļè§Ī":102910,"é̾":102911,"åĮĢ":102912,"ç»ıæµİåѦ":102913,"æ¸Ĺ":102914,"ä¿ĿèŃ·":102915,"çīº":102916,"çī²":102917,"çİ«":102918,"çij°":102919,"æľĢåIJİä¸Ģ":102920,"æĶ¿åĬ¡":102921,"æ§Ľ":102922,"èĻķçIJĨ":102923,"éļIJæĤ£":102924,"æī¿åĮħ":102925,"極":102926,"æ¡©":102927,"çĽ²":102928,"导åIJij":102929,"èĩ´å¯Į":102930,"ç¼Ĩ":102931,"æģĭçα":102932,"ä¸įåĬ¨":102933,"ç»Ļ人":102934,"å·¢":102935,"表æĥħ":102936,"举åįĹ":102937,"åĨħå¤ĸ":102938,"è¾ĪåŃIJ":102939,"åıī":102940,"åįļä¼ļ":102941,"åĬŁæķĪ":102942,"渴":102943,"屬":102944,"æİĴéϤ":102945,"éĢĽ":102946,"ä¸Ģä¼ļ":102947,"ä¸įå¼Ģ":102948,"å¼Ģå¥ĸ":102949,"é»ijé¾Ļ":102950,"é»ijé¾Ļæ±Ł":102951,"å¿«ä¸ī":102952,"度åģĩ":102953,"åĿ¤":102954,"éĤ®ä»¶":102955,"æĩĴ":102956,"ä¾Ľç͵":102957,"廣":102958,"好è¯Ħ":102959,"ç§ĺ书éķ¿":102960,"æĪĺåľº":102961,"好å¥ĩ":102962,"ä¾µæĿĥ":102963,"æĨ¾":102964,"æľĢåĪĿ":102965,"æī¹åıij":102966,"åİķ":102967,"è¼ķ":102968,"æŀ¯":102969,"ä¸ļåĨħ":102970,"è´ŃæĪ¿":102971,"ä¸įåľ¨":102972,"纪å§Ķ":102973,"æīĢéľĢ":102974,"å¸Ĥéķ¿":102975,"è³½":102976,"å¼ķæĵİ":102977,"çģµéŃĤ":102978,"éĬĢ":102979,"滤":102980,"çĿIJ":102981,"å¤ļ项":102982,"åĽŀ头":102983,"èīĺ":102984,"å¤įå·¥":102985,"éĥ¨ä»¶":102986,"ç´§ç´§":102987,"æŁIJç§į":102988,"使åħ¶":102989,"æĸ°äºº":102990,"æŀļ":102991,"æ³ķå®ļ":102992,"å·´å·´":102993,"æ¶µçĽĸ":102994,"稻":102995,"æĭ¾":102996,"æĻķ":102997,"轿":102998,"éĢļè¡Į":102999,"åĵĢ":103000,"æ³Ĭ":103001,"温馨":103002,"éĽĨèģļ":103003,"çĨĻ":103004,"åĩij":103005,"åįģä¸ĥ":103006,"æ°Ķæģ¯":103007,"æıIJä¾ĽçļĦ":103008,"æ³³":103009,"奥è¿IJ":103010,"çģ¾å®³":103011,"åĩĢåĮĸ":103012,"è·¨è¶Ĭ":103013,"åĵªæĢķ":103014,"éŁ¿":103015,"å¢ŀæ·»":103016,"çĦĬ":103017,"æ®ĭçĸ¾":103018,"ç¢Į":103019,"æĤĶ":103020,"è§ģè¯ģ":103021,"è¾ĸåĮº":103022,"å¿ĥèĦı":103023,"éļ§":103024,"åį¸":103025,"åı¯èĥ½æĢ§":103026,"æľīè¶£":103027,"åī¯ä¹¦è®°":103028,"åĮĸå¦Ĩ":103029,"ä¿Ĥ":103030,"æ£ļ":103031,"éĨĩ":103032,"带头":103033,"éłĪ":103034,"追究":103035,"æijĶ":103036,"è¿Ļéĥ¨":103037,"ä¸į论":103038,"祸":103039,"å³»":103040,"éģķ":103041,"çĶŁèĤ²":103042,"å¤ł":103043,"å¤ĸ交":103044,"è¯Ħ为":103045,"ä»İå°ı":103046,"å°ıå°ı":103047,"饿":103048,"æĴ¼":103049,"è·¨å¢ĥ":103050,"被åijĬ":103051,"åįĹå®ģ":103052,"身å¿ĥ":103053,"åĨįçĶŁ":103054,"æīĢ说":103055,"æĹ¶éĹ´åĨħ":103056,"åĪĹåħ¥":103057,"éĿĴæµ·":103058,"çα好":103059,"çªĦ":103060,"èĪĪ":103061,"è¿ĩ渡":103062,"æ¿Ł":103063,"éĽĢ":103064,"审议":103065,"åĽ½èµĦ":103066,"æŃ¥ä¼IJ":103067,"轨éģĵ":103068,"信念":103069,"ä¸īåĪĨ":103070,"çĨ¬":103071,"åѵåĮĸ":103072,"ç¼ł":103073,"éĥĬ":103074,"èĪĴæľį":103075,"纪æ£Ģ":103076,"ä¸Ģä¸ĭåŃIJ":103077,"éĽ»è©±":103078,"è²ł":103079,"éĴ¥":103080,"åĮĻ":103081,"çĹ´":103082,"è¶ģ":103083,"绣":103084,"çε":103085,"è½°":103086,"éªĦ":103087,"姨":103088,"æĭĺ":103089,"çĮ´":103090,"è®¶":103091,"è¿Ļ座":103092,"çį¨":103093,"æ·ĺæ±°":103094,"çĹħä¾ĭ":103095,"æ²Ļåıij":103096,"è§Ĩ为":103097,"头æĿ¡":103098,"å¿ħè¦ģçļĦ":103099,"åı¯è°ĵ":103100,"è¯Ŀ说":103101,"ç¯Ħ":103102,"æĹ©çĤ¹":103103,"æŀ¢çº½":103104,"羡":103105,"çĪ±åĽ½":103106,"çªģåıij":103107,"éĢĬ":103108,"æ½į":103109,"èį£èĢĢ":103110,"èŁ¹":103111,"æ¦Ĥçİĩ":103112,"å¾Īä¹ħ":103113,"æĥķ":103114,"訴":103115,"åľĨ满":103116,"çļ±":103117,"åĪĨæ³Į":103118,"åħħè¶³":103119,"çľĭæ³ķ":103120,"è¾Ł":103121,"æĭ¦":103122,"æĭ©":103123,"对åºĶ":103124,"ä¸ºæł¸å¿ĥ":103125,"èħĬ":103126,"å¤ļä¹Ī":103127,"æµij":103128,"å®ıè§Ĥ":103129,"èĦĸ":103130,"åIJĪèµĦ":103131,"çĶŁæ¶¯":103132,"å®ŀè´¨":103133,"ä¼ĺçĤ¹":103134,"çĶ¨æ°´":103135,"寿åij½":103136,"沫":103137,"åIJģ":103138,"詹":103139,"åĽ½éĺ²":103140,"å´©":103141,"åĿİ":103142,"èĨı":103143,"ä¸Ģè½®":103144,"éģĹ产":103145,"æ¹¾åĮº":103146,"ç»İ":103147,"åįķ纯":103148,"æ¾Ħ":103149,"åīįåĪĹ":103150,"身影":103151,"é»ĺé»ĺ":103152,"æįī":103153,"çĴ°":103154,"èıĬ":103155,"æĢľ":103156,"åħĭæĢĿ":103157,"æĢ»å±Ģ":103158,"çĩĥæĸĻ":103159,"ä¸ļæĢģ":103160,"åIJĦæł·":103161,"åĴ½":103162,"åĩºèī²":103163,"åĪĿå¿ĥ":103164,"åıĽ":103165,"çłĶ讨":103166,"è¡«":103167,"åİĨç¨ĭ":103168,"禽":103169,"è¶³å¤ŁçļĦ":103170,"èįĨ":103171,"çľĭå¾ħ":103172,"è´©":103173,"åĨ³å¿ĥ":103174,"裹":103175,"å¸ĪèĮĥ":103176,"åŀĦ":103177,"æĿł":103178,"åĩ¸":103179,"çĬ¹è±«":103180,"çĥŃè¡Ģ":103181,"åIJĪä¼Ļ":103182,"éħµ":103183,"èIJ½åľ¨":103184,"åįłåľ°":103185,"衬":103186,"èĵī":103187,"æĦ¤":103188,"æ¸Ĭ":103189,"åĪĨæķ°":103190,"ç¬ijçĿĢ":103191,"太平":103192,"çĤ«":103193,"æİ¨ä»ĭ":103194,"æĸ¯åĿ¦":103195,"形容":103196,"æĵĬ":103197,"æĦŁåħ´è¶£":103198,"åĨĽäºº":103199,"åĩĮæĻ¨":103200,"对çħ§":103201,"åıijçĹħ":103202,"å·¾":103203,"èĪī":103204,"檢":103205,"ç¬ijäºĨ":103206,"ç¡®è¯Ĭ":103207,"è´ŁåĢº":103208,"壮大":103209,"æĪļ":103210,"äºĴèģĶ":103211,"課":103212,"èħ¦":103213,"æĹ±":103214,"åıĹæ¬¢è¿İ":103215,"åįī":103216,"éϢ士":103217,"æ©¡":103218,"ä¸Ģ对":103219,"è¾±":103220,"æ²Ĥ":103221,"åı²ä¸Ĭ":103222,"æIJı":103223,"å´ĸ":103224,"代谢":103225,"磷":103226,"é¡ĺ":103227,"æµĩ":103228,"常ç͍":103229,"åįij":103230,"åĩºåĽ½":103231,"è¯ł":103232,"稳æŃ¥":103233,"ç»ı纪":103234,"å¤ļå¤ļ":103235,"æīĢå¾Ĺ":103236,"为主é¢ĺ":103237,"ä¸ĢåĪĨ":103238,"æł½":103239,"é¡§":103240,"纲":103241,"åĥħ":103242,"å£ĵ":103243,"åĦª":103244,"ç¿°":103245,"æİĢ":103246,"人为":103247,"媳":103248,"æ´½":103249,"èĿ¶":103250,"å¤įåħ´":103251,"ä¼ļå½±åĵį":103252,"åIJĦçķĮ":103253,"éĤ£ä¸Ģ":103254,"颤":103255,"çĢı":103256,"çĢı覽":103257,"å¯ŀ":103258,"åı¯æĢķ":103259,"åį³æĹ¶":103260,"çķ´":103261,"ä¸ĭåįĬå¹´":103262,"ç¬Ķè®°":103263,"éĻĦåĬł":103264,"çĥŃæ°´":103265,"奸":103266,"ç£ħ":103267,"æĿī":103268,"æ¸ħåįİ":103269,"éĸ±":103270,"ç°¡":103271,"å¤Ħå¤Ħ":103272,"åIJĪéĩij":103273,"æ²³æµģ":103274,"ç´°":103275,"è´ŁéĿ¢":103276,"çļĦ羣å®ŀ":103277,"åĻ¨æ¢°":103278,"èĴIJ":103279,"西äºļ":103280,"å·ħ":103281,"ç²¹":103282,"åİŁæĸĩ":103283,"æŀķ":103284,"è¡Ģåİĭ":103285,"åļ´":103286,"å¸ĺ":103287,"åĨĢ":103288,"æĮ«":103289,"çĶµè·¯":103290,"å°ıä¼Ļä¼´":103291,"èĿ´":103292,"æľĢå¿«":103293,"æĭĮ":103294,"宪":103295,"æĸ·":103296,"ç¿ħ":103297,"åĴ³":103298,"åĹ½":103299,"ç¾ŀ":103300,"èººåľ¨":103301,"èµĽè½¦":103302,"æ²IJ":103303,"éĻIJ度":103304,"为ä¸Ģä½ĵ":103305,"èĴľ":103306,"幫":103307,"æIJħ":103308,"åĭĭ":103309,"åīĸ":103310,"纳ç¨İ":103311,"éķ¿æķĪ":103312,"ç½ķ":103313,"åľ¬":103314,"ç©į":103315,"éĴ©":103316,"ç¹¼":103317,"åĽ½åľŁ":103318,"è¼ī":103319,"ä¸įå¿ĺ":103320,"èŃ¦ç¤º":103321,"çģ¿":103322,"å¿ĥå¾Ĺ":103323,"æĦļ":103324,"忽çķ¥":103325,"åĽŀäºĭ":103326,"åįłæľī":103327,"æ·Ħ":103328,"çī¡":103329,"çĽijäºĭ":103330,"ç¿¡":103331,"éĴĪ对æĢ§":103332,"çªĥ":103333,"製":103334,"èĨĿ":103335,"ç³Ł":103336,"港澳":103337,"太太":103338,"澡":103339,"ç»ĨåĮĸ":103340,"åĶ®åIJİ":103341,"å®ŀåľ¨æĺ¯":103342,"ç«£":103343,"çį²":103344,"å̾åIJij":103345,"å¼ķç͍":103346,"é¹ħ":103347,"ç¬ij容":103348,"ä¹IJè¶£":103349,"æ°ijæĶ¿":103350,"éŨæĪ·":103351,"å±ģ":103352,"迷失":103353,"éĶĮ":103354,"å°ı康":103355,"åĭī":103356,"æ³¼":103357,"ä¾ĭåŃIJ":103358,"ä¸īä½į":103359,"å»ł":103360,"èĶĵ":103361,"广éĺĶ":103362,"èĢį":103363,"èĢģèĻİ":103364,"åĭŁéĽĨ":103365,"èĦļæŃ¥":103366,"æĭ¯":103367,"åŃĹåı·":103368,"çĦ°":103369,"é¢ł":103370,"èļĤ":103371,"èļģ":103372,"飯":103373,"人æĢ§":103374,"æĴ°":103375,"åİ¢":103376,"å±ĢéĻIJ":103377,"æľªæĪIJ":103378,"åĵªåĦ¿":103379,"大åıij":103380,"ä¸įå®ļ":103381,"å¾ģæ±Ĥ":103382,"éĥµ":103383,"åĢºæĿĥ":103384,"çĪ±ä½ł":103385,"èºģ":103386,"ä»ħä¾Ľ":103387,"è¿ľå¤Ħ":103388,"éĨĽ":103389,"åĥµ":103390,"积æŀģæĢ§":103391,"æİ¡":103392,"åīįä¸ī":103393,"äºİä¸Ģä½ĵ":103394,"çŀĦ":103395,"çĿģ":103396,"沸":103397,"åħ±èµ¢":103398,"éĢĢå½¹":103399,"è´Ŀå°Ķ":103400,"æİı":103401,"æĪ²":103402,"è¡į":103403,"éĶĤ":103404,"ä¸ĩä½Ļ":103405,"ç§ijåĪĽ":103406,"æ¼Ķåͱ":103407,"欧åħĥ":103408,"æ·¡æ·¡":103409,"éĿĴå±±":103410,"èĹĿ":103411,"绽":103412,"令çīĮ":103413,"éĽĨ群":103414,"ä½ľçī©":103415,"çĢij":103416,"夯":103417,"ç½ij游":103418,"åħ«å¤§":103419,"éªļ":103420,"èªĵ":103421,"ä¼ļå±ķ":103422,"åħļåı²":103423,"æ£Ģå¯ŁéĻ¢":103424,"åĸĺ":103425,"éĺ±":103426,"èĢĮåĩº":103427,"éĢļ车":103428,"éĴĵ":103429,"æĥħ人":103430,"æ¸Ľ":103431,"ä¸Ńç§ĭ":103432,"çĪŃ":103433,"åıªåī©":103434,"æĺĶ":103435,"éĩİçĶŁ":103436,"ç¡«":103437,"èIJĿåįľ":103438,"æĬµæĬĹ":103439,"çĻ«çĹ«":103440,"éĻĢ":103441,"èĶļ":103442,"å¸ľ":103443,"满满":103444,"èı±":103445,"éļĨéĩį":103446,"æĺŁçº§":103447,"æ½ĩ":103448,"åħ¬åħĥ":103449,"è°£":103450,"æ¯Ķäºļ":103451,"æ¡ĮåŃIJ":103452,"èµ£":103453,"è²¼":103454,"æĦ¿æľĽ":103455,"顽":103456,"æ´¾éģ£":103457,"ç¥Ľ":103458,"åªļ":103459,"éĺľ":103460,"èij«":103461,"èĬ¦":103462,"æ³»":103463,"å¡Į":103464,"çĭŃ":103465,"å»īæĶ¿":103466,"å¥ijæľº":103467,"æĹĹèΰ":103468,"æĥ«":103469,"严åİī":103470,"åıĭæĥħ":103471,"å¦Ĭ":103472,"å¨ł":103473,"åĵªå®¶":103474,"èĨ¨":103475,"è¶Ł":103476,"æĮª":103477,"èĻIJ":103478,"éłģ":103479,"çŀ©":103480,"éºŁ":103481,"稣":103482,"èģĶéĢļ":103483,"åı®":103484,"çİĭèĢħ":103485,"ä¸įç¡®å®ļ":103486,"çijľ":103487,"è°İ":103488,"çī¢è®°":103489,"碼":103490,"æĬ¤èĤ¤":103491,"é¡·":103492,"çĦķ":103493,"åģļ强":103494,"éļ±ç§ģ":103495,"éļ±ç§ģæ¬Ĭ":103496,"åıĹ害":103497,"ä¸įçͱ":103498,"çĥ¹":103499,"饪":103500,"驳":103501,"ä¼½":103502,"ä¸Ŀ绸":103503,"è¥Ħ":103504,"åįģä½Ļ":103505,"éºĹ":103506,"æ¬ĬåĪ©":103507,"èģŀ":103508,"åı¤èĢģ":103509,"éģı":103510,"åIJĦå¼ı":103511,"å°±è¡Į":103512,"åħ¥å¢ĥ":103513,"çĥģ":103514,"èľĺ":103515,"èĽĽ":103516,"纬":103517,"磫":103518,"è»Ł":103519,"æ´Ĺè¡£":103520,"æĦ§":103521,"é¢Ħæ¡Ī":103522,"éľĨ":103523,"æ·±åİļ":103524,"éĺ¿æĭī":103525,"åĨĻåŃĹ":103526,"åį¦":103527,"éķĢ":103528,"æ¨¡æł·":103529,"åĤį":103530,"æIJį":103531,"èĸ¯":103532,"åłħ":103533,"åħ¬ç§¯":103534,"è¨İ":103535,"ä¼łæŁĵ":103536,"毯":103537,"çIJĨå·¥":103538,"åĨ·éĵ¾":103539,"ç«ĭæĸ¹":103540,"æ¢Ń":103541,"åľ£è¯ŀ":103542,"综èīº":103543,"çİ©ç¬ij":103544,"æĥ³ä¸įåΰ":103545,"æijĩ头":103546,"æ·¹":103547,"åģĩæĹ¥":103548,"åĢĺ":103549,"è̽":103550,"èİĵ":103551,"åŁ·":103552,"èĩªè´¸":103553,"åįĬ天":103554,"æªĶ":103555,"æ¾İæ¹ĥ":103556,"éķij":103557,"丫":103558,"éĩĮç¨ĭ":103559,"å¼ĢèįĴ":103560,"èıı":103561,"å®Ŀè´µ":103562,"èѬ":103563,"åķŁ":103564,"æŁł":103565,"檬":103566,"é©Ń":103567,"æ±Ľ":103568,"çĨĬçĮ«":103569,"èķī":103570,"éļıä¹ĭ":103571,"å±ij":103572,"è¾ĥ强":103573,"èĥ³":103574,"èĨĬ":103575,"éĿĻéĿĻ":103576,"åĴª":103577,"æĭĽåij¼":103578,"代è¨Ģ":103579,"ä¿¡ç®±":103580,"è£ħéħį":103581,"æĤį":103582,"åįķ车":103583,"èIJİ":103584,"å¤ļ彩":103585,"éϏ":103586,"ä»İ严":103587,"æ©Ħ":103588,"æ¦Ħ":103589,"éĢ®":103590,"éĩĮæĸ¯":103591,"å§¿æĢģ":103592,"太æŀģ":103593,"éĩĿ":103594,"æºī":103595,"è¿Ń":103596,"秸":103597,"ç§Ĩ":103598,"å·¥å§Ķ":103599,"æ±ķ":103600,"èģĨ":103601,"佬":103602,"ç¼ħ":103603,"ç͏":103604,"åī¯å±Ģéķ¿":103605,"éĹº":103606,"誤":103607,"è¤IJ":103608,"ä¸įéĻIJ":103609,"èħķ":103610,"åijķ":103611,"磶":103612,"åĨľå®¶":103613,"管å§Ķä¼ļ":103614,"饺":103615,"èĬľ":103616,"æ¾Ī":103617,"è©¢":103618,"å¨ģå°¼æĸ¯":103619,"ä½ķåĨµ":103620,"å°ıä¼Ļ":103621,"奢ä¾Ī":103622,"è¿Ļç¯ĩ":103623,"诵":103624,"竳ç¨ĭ":103625,"ç´Ģ":103626,"éIJĺ":103627,"éĤ¢":103628,"ç³Ļ":103629,"ç¼Ģ":103630,"ä¹Ĵ":103631,"ä¹ĵ":103632,"çī¢åĽº":103633,"åĿŀ":103634,"å¼Ī":103635,"ä¾ĭå¤ĸ":103636,"廳":103637,"è§Ħ竳":103638,"èĬĻ":103639,"篷":103640,"躯":103641,"æłĪ":103642,"åĿļå®ŀ":103643,"åŁºå»º":103644,"çĿĢçľ¼":103645,"ç·´":103646,"èij©":103647,"ç¼ļ":103648,"æ¦Ĩ":103649,"主åĭķ":103650,"ç¥Ģ":103651,"äºĴéĢļ":103652,"尤为":103653,"å®Ľ":103654,"骼":103655,"æ±²":103656,"ä¾ĥ":103657,"æĤłä¹ħ":103658,"æij§":103659,"æĭĩ":103660,"é«ĵ":103661,"éºĴ":103662,"éĻĽ":103663,"æŀ¸":103664,"æĿŀ":103665,"è´¬":103666,"å°ıé¾Ļ":103667,"åĵ®":103668,"èĵ¬åĭĥ":103669,"åĮĪ":103670,"çķľçī§":103671,"娩":103672,"个å¤ļ":103673,"æ²¥":103674,"æĺ§":103675,"çĦļ":103676,"æĬijéĥģ":103677,"çĸ¡":103678,"èĺij":103679,"éģİç¨ĭ":103680,"橱":103681,"éĿĵ":103682,"大çIJĨ":103683,"髦":103684,"åĪĨ辨":103685,"渤":103686,"çĸ¤":103687,"åĬ¨èĥ½":103688,"å¼łå®¶":103689,"ä¸ĩåįĥ":103690,"滥":103691,"饥":103692,"åºŁå¼ĥ":103693,"帳":103694,"æ¼³":103695,"è±IJ":103696,"ä»ij":103697,"å«ī":103698,"å¦Ĵ":103699,"çŀĴ":103700,"è¡ħ":103701,"çĭ¸":103702,"å¾ģç¨ĭ":103703,"éĤ¯":103704,"éĥ¸":103705,"ç¥Ī":103706,"祷":103707,"è¶´":103708,"ç»ĵæŀĦæĢ§":103709,"è§ĨåIJ¬":103710,"è¬Ŀ":103711,"çĴĢ":103712,"çĴ¨":103713,"åĩºå¤Ħ":103714,"è¯Ģ":103715,"å¾ĺ":103716,"å¾Ĭ":103717,"羨":103718,"åĸĩ":103719,"åıŃ":103720,"åĺ²":103721,"çķ¸":103722,"å¹²äºĭ":103723,"æļ§":103724,"æ²Ľ":103725,"åĦĦ":103726,"å»ĵ":103727,"åİ¿éķ¿":103728,"èĥļ":103729,"çIJ¢":103730,"çŃ·":103731,"éĩĭ":103732,"ä¾®":103733,"åIJ©":103734,"åĴIJ":103735,"åĮ¿":103736,"æĬ¬èµ·":103737,"æ³£":103738,"涤":103739,"麽":103740,"æĽĻ":103741,"åī¯éĻ¢éķ¿":103742,"åħļåĴĮ":103743,"æķ£åıij":103744,"润æ»ij":103745,"åĵº":103746,"æĥ¬":103747,"漫éķ¿":103748,"ä¸įæĩĪ":103749,"åŁł":103750,"åĹĵ":103751,"èĢģçĪ·":103752,"讽":103753,"æĪĺç»ĦåIJĪ":103754,"æ£ł":103755,"åħ¨åŁŁ":103756,"èł¢":103757,"诡":103758,"åīįçŀ»":103759,"æķĽ":103760,"ä¸Ģå°ģ":103761,"å¹Ĥ":103762,"èİĨ":103763,"è¯Ŀè¯Ń":103764,"ç»ĨåĪĻ":103765,"屿":103766,"åµĮ":103767,"éĢį":103768,"åĺ±":103769,"渲":103770,"çĥ¯":103771,"çĿ¹":103772,"é¦Ĵ":103773,"èħ¥":103774,"æĬĹåĩ»":103775,"çĿ«":103776,"èįĶ":103777,"éļİ":103778,"æ³īæ°´":103779,"è¬Ĥ":103780,"çĤ¬":103781,"åĩıæİĴ":103782,"è¸Ĭ":103783,"è·»":103784,"æ·Į":103785,"éľ¾":103786,"å¥ĩ纳":103787,"å¯Ŀ":103788,"æ¤İ":103789,"æŁ¬":103790,"æĸ¯åŁº":103791,"åħ¬ç«ĭ":103792,"è¨ĵ":103793,"é£Ļ":103794,"é©¿":103795,"åĤµ":103796,"èĽĻ":103797,"ç¯ĩ竳":103798,"åĪĨæĶ¯":103799,"ä¸Ĭå¹´":103800,"çŃĿ":103801,"缤":103802,"èĢģæĹ§":103803,"åϬ":103804,"æľ¦":103805,"èĥ§":103806,"æ¶Īè²»":103807,"æĵĶ":103808,"榴":103809,"æ¿Ĵ":103810,"糯":103811,"泸":103812,"æįĨ":103813,"ç»ļ":103814,"èµİ":103815,"çIJIJ":103816,"èµĤ":103817,"æħ®":103818,"æ²Į":103819,"çĦĻ":103820,"æĴŃæĬ¥":103821,"æ·ĩ":103822,"åĪĩåħ¥":103823,"çijķ":103824,"çĸµ":103825,"éģ´":103826,"ç¨ļ":103827,"ç©©":103828,"èŀĥ":103829,"æ£ķ":103830,"æĨ§":103831,"æĨ¬":103832,"伺":103833,"æ¯Ĺ":103834,"æįį":103835,"æĬī":103836,"ç´Ĭ":103837,"å¼Ľ":103838,"æĭŃ":103839,"æĹıèĩªæ²»":103840,"åĿ·":103841,"ç«¶":103842,"詳":103843,"è¿Ħä»Ĭ":103844,"è°´":103845,"çŀŃè§£":103846,"æŁ¿":103847,"é¢Ĭ":103848,"ç°§":103849,"çĥŁèĬ±":103850,"ä¾¥":103851,"çĿ¦":103852,"éħĿ":103853,"æ°ĵ":103854,"çIJī":103855,"å§Ĭ":103856,"æ²®":103857,"æħ·":103858,"èľķ":103859,"çijļ":103860,"éĩĩçŁ¿":103861,"åł°":103862,"åºķèķ´":103863,"èĨ³":103864,"è¾ķ":103865,"éŁŃ":103866,"åĴĻ":103867,"ç²½":103868,"åīĶ":103869,"沦":103870,"èĤ´":103871,"éķ¶":103872,"æĺ¼":103873,"è¾Ĺ":103874,"婪":103875,"åĮ®":103876,"æĸĵ":103877,"æ±¶":103878,"éĥ´":103879,"éł»":103880,"çªĴ":103881,"袱":103882,"åĽ±":103883,"èĢĺ":103884,"èļĮ":103885,"çĭĻ":103886,"çĹ¹":103887,"ç¥ī":103888,"æı®":103889,"æ·Ĩ":103890,"ç£ĭ":103891,"éĺª":103892,"æ«":103893,"ã¸":103894,"϶":103895,"ãij":103896,"ð£²":103897,"ä¢":103898,"ãŃ":103899,"ð¬¨":103900,"ð¬Ģ":103901,"ð¬®":103902,"ð¬¯":103903,"ð¬ľ":103904,"ðª¨":103905,"ð«Ĺ":103906,"ð¬Ĭ":103907,"ð¬±":103908,"ð¬Ł":103909,"äİ":103910,"ð¡":103911,"äĥ":103912,"ãł":103913,"ð©":103914,"ð©¾":103915,"ð¬º":103916,"ð¬Ļ":103917,"ãĢĶ":103918,"ãĢķ":103919,"çļĦæĹ¶åĢĻ":103920,"æľīéĻIJåħ¬åı¸":103921,"ä¹ĭåIJİ":103922,"ä¸ļåĬ¡":103923,"åķĬ":103924,"èϽçĦ¶":103925,"æĭ¥æľī":103926,"äºĴèģĶç½ij":103927,"éĤ£äºĽ":103928,"ä½łçļĦ":103929,"åĨ³å®ļ":103930,"éϤäºĨ":103931,"åĽ¢éĺŁ":103932,"åı¯æĺ¯":103933,"以åIJİ":103934,"社åĮº":103935,"çļĦéĹ®é¢ĺ":103936,"å¹¶ä¸Ķ":103937,"æķĻå¸Ī":103938,"å°±ä¼ļ":103939,"天空éĥ¨èIJ½":103940,"æľĢç»Ī":103941,"å½ĵçĦ¶":103942,"ä¹Łæľī":103943,"ç¡®ä¿Ŀ":103944,"æĥ³è¦ģ":103945,"è´Ńä¹°":103946,"人çļĦ":103947,"åIJ´":103948,"çļĦåıijå±ķ":103949,"ä¸įçŁ¥éģĵ":103950,"软件":103951,"æĪij们çļĦ":103952,"çζæ¯į":103953,"åīij":103954,"èĢĮæĺ¯":103955,"å®īæİĴ":103956,"åIJİæĿ¥":103957,"çļĦåľ°æĸ¹":103958,"èµµ":103959,"èĢĥè¯ķ":103960,"çªģçĦ¶":103961,"ä¸Ģå®ļè¦ģ":103962,"åĪ¶ä½ľ":103963,"è¯Ħä»·":103964,"åħįè´¹":103965,"è´¹ç͍":103966,"绣ä¸Ģ":103967,"çĦ¶èĢĮ":103968,"è¿Ļ次":103969,"éĿĴå¹´":103970,"人类":103971,"亦":103972,"让人":103973,"è´Łè´£äºº":103974,"éĩĩåıĸ":103975,"çļĦäºĭæĥħ":103976,"ä¹Łä¼ļ":103977,"车è¾Ĩ":103978,"æĽ´æĺ¯":103979,"强åĮĸ":103980,"æĪijåĢij":103981,"以åīį":103982,"ä¼ĺåĮĸ":103983,"å§Ķåijĺä¼ļ":103984,"åĽ°éļ¾":103985,"年度":103986,"ä½įäºİ":103987,"æĮĩåĩº":103988,"åĨῬ¡":103989,"åĬŀçIJĨ":103990,"æ¯ı个":103991,"对æĸ¹":103992,"è¿Ľè¡ĮäºĨ":103993,"æľĢé«ĺ":103994,"课ç¨ĭ":103995,"身ä¸Ĭ":103996,"æĽ¾ç»ı":103997,"åĮ»çĶŁ":103998,"å®īè£ħ":103999,"æľ±":104000,"è¿IJè¡Į":104001,"åıĮæĸ¹":104002,"æľĢ大çļĦ":104003,"æŀĦ建":104004,"è¿ŀç»Ń":104005,"çļĦå°ı":104006,"她çļĦ":104007,"çŃīçŃī":104008,"æĶ¹åĸĦ":104009,"åIJĦç±»":104010,"éģĩåΰ":104011,"æľīçĿĢ":104012,"人çī©":104013,"æĢ»æĺ¯":104014,"è¿ħéĢŁ":104015,"åζå®ļ":104016,"å®ĥ们":104017,"å®ĺç½ij":104018,"è¿ĺè¦ģ":104019,"ç»Īäºİ":104020,"æĪ¿åľ°äº§":104021,"è¯ģæĺİ":104022,"èĤ¡ç¥¨":104023,"åºĶå½ĵ":104024,"èĭ±åĽ½":104025,"è¿IJç͍":104026,"æľĢæĸ°":104027,"享åıĹ":104028,"让æĪij":104029,"æĻļä¸Ĭ":104030,"å¾ŀ":104031,"å°ı说":104032,"å°¤åħ¶æĺ¯":104033,"è®Ńç»ĥ":104034,"åħ¨å¸Ĥ":104035,"æĮijæĪĺ":104036,"æľīçĤ¹":104037,"带çĿĢ":104038,"çļĦä¸ľè¥¿":104039,"é£İæł¼":104040,"é»Ħéĩij":104041,"å¼ķ导":104042,"æŃ¤å¤ĸ":104043,"æľĢè¿ij":104044,"追æ±Ĥ":104045,"强è°ĥ":104046,"ä¹Łåı¯ä»¥":104047,"æĦŁåΰ":104048,"èĩªæĪij":104049,"çī¹åĪ«æĺ¯":104050,"æĪIJéĥ½":104051,"éĢIJæ¸IJ":104052,"å¿«ä¹IJ":104053,"ä¹ĭä¸Ń":104054,"æĬķèµĦèĢħ":104055,"ä»ĸ们çļĦ":104056,"æ°ı":104057,"å·¥ä½ľäººåijĺ":104058,"äºĨä¸Ģ个":104059,"åķ¦":104060,"ä¸ĢåĢĭ":104061,"åŁºå±Ĥ":104062,"æ²ŁéĢļ":104063,"第ä¸Ģ次":104064,"并没æľī":104065,"çļĦå·¥ä½ľ":104066,"åľ¨è¿ĻéĩĮ":104067,"æŀª":104068,"æĶ¯æĴij":104069,"æĹ¶å°ļ":104070,"æĿ¥åΰ":104071,"æĶ¶è´Ń":104072,"éĿ©åij½":104073,"æĺ¯ä¸įæĺ¯":104074,"讨论":104075,"ä¸ļ绩":104076,"å°±èĥ½":104077,"ç«ĭåį³":104078,"è¡Ĺéģĵ":104079,"åľ¨ä¸Ģèµ·":104080,"æľĪ份":104081,"é«ĺ端":104082,"å¾Īéļ¾":104083,"ä¿Ħç½Ĺæĸ¯":104084,"æīĭ段":104085,"åģļåĩº":104086,"ä¼Ĺå¤ļ":104087,"å®ŀè¡Į":104088,"æīĵå¼Ģ":104089,"游客":104090,"ä¾ĿçĦ¶":104091,"å°±åĥı":104092,"离å¼Ģ":104093,"说éģĵ":104094,"æĸ°èĥ½æºIJ":104095,"溪":104096,"äºķ":104097,"令人":104098,"ä¸Ģåľº":104099,"æĪijæĥ³":104100,"两人":104101,"èĩ³å°ij":104102,"çļĦçĶŁæ´»":104103,"æĺ¯ä¸ª":104104,"èĭ±è¯Ń":104105,"æ²Ĵæľī":104106,"æĢĿèĢĥ":104107,"éĻIJåζ":104108,"åı°æ¹¾":104109,"ä¸ĢæĹ¦":104110,"çļĦä¸Ģ个":104111,"é«ĺ级":104112,"åĬŀåħ¬å®¤":104113,"å¾·åĽ½":104114,"æĪijå°±":104115,"å®ļä½į":104116,"éĢĤåºĶ":104117,"æĮĩæłĩ":104118,"åħ¨çľģ":104119,"ä¸Ĭè¿°":104120,"å®ĥçļĦ":104121,"åĽŀå®¶":104122,"欧洲":104123,"éĵģè·¯":104124,"é¼ĵåĬ±":104125,"çļĦå½±åĵį":104126,"é«ĺæł¡":104127,"天ä¸ĭ":104128,"é«ĺè´¨éĩı":104129,"æĿŃå·ŀ":104130,"èµĦ讯":104131,"æĶ¾åľ¨":104132,"æľīä¸Ģ个":104133,"å°±è¦ģ":104134,"ä¸ĬéĿ¢":104135,"è§£éĩĬ":104136,"éĢIJæŃ¥":104137,"尽管":104138,"æľīä»Ģä¹Ī":104139,"çļĦäºĭ":104140,"çĻ»è®°":104141,"人æ°ijå¸ģ":104142,"è§Ĥä¼Ĺ":104143,"è§Ĥå¯Ł":104144,"ç͵èĦij":104145,"çļĦåIJĮæĹ¶":104146,"ä½ľä¸ļ":104147,"宣å¸ĥ":104148,"çļĦä½ľç͍":104149,"åĽŀæĿ¥":104150,"éļ¾ä»¥":104151,"æīĢæľīçļĦ":104152,"å°ıåѦ":104153,"æıIJåīį":104154,"æ¤įçī©":104155,"åĩ¯":104156,"ä¸ĬäºĨ":104157,"å°±åľ¨":104158,"åħĪåIJİ":104159,"æīĭæľ¯":104160,"éĥŃ":104161,"éĿ¢åīį":104162,"æ¯ķ竣":104163,"äºĮæĺ¯":104164,"红èī²":104165,"éĺ³åħī":104166,"èĭ¹æŀľ":104167,"å¾Īå¤ļ人":104168,"ç»ĻæĪij":104169,"åĵ¦":104170,"çľ¼çĿĽ":104171,"éłŃ":104172,"ä¸Ģæĺ¯":104173,"åıijå±ķçļĦ":104174,"åıįåºĶ":104175,"æĪ¿å±ĭ":104176,"æľŁå¾ħ":104177,"ç§įæ¤į":104178,"æĸĩåѦ":104179,"åį³åı¯":104180,"é¦ĸ次":104181,"èĭ±éĽĦ":104182,"å¤ļ次":104183,"åĮħè£ħ":104184,"æ²³åįĹ":104185,"ä¹ĭéĹ´çļĦ":104186,"ä»įçĦ¶":104187,"åIJ¬åΰ":104188,"èij£äºĭéķ¿":104189,"è§ĦåĪĻ":104190,"ä¸Ģ份":104191,"大ä¼Ĺ":104192,"使å¾Ĺ":104193,"è¿Ľåı£":104194,"ä¸Ģçīĩ":104195,"æĢ§çļĦ":104196,"çļĦ大":104197,"æĪijæĺ¯":104198,"äºĴåĬ¨":104199,"æ°£":104200,"çļĨ":104201,"åħ¬åı¸çļĦ":104202,"ä¸Ģè¾¹":104203,"åıĬåħ¶":104204,"èī¯å¥½çļĦ":104205,"æĭĵå±ķ":104206,"å½ĵå¹´":104207,"å¹¿åľº":104208,"åģļäºĨ":104209,"åŁºäºİ":104210,"æıIJéĨĴ":104211,"åħĦå¼Ł":104212,"èĢģæĿ¿":104213,"è¿ijæĹ¥":104214,"çĬ¶åĨµ":104215,"注éĩį":104216,"åĪļåĪļ":104217,"è°ĥçłĶ":104218,"å¿ĥä¸Ń":104219,"æĬĬæı¡":104220,"éļıåIJİ":104221,"ä¸įå¤Ł":104222,"åĪĽä½ľ":104223,"ç«Ļåľ¨":104224,"缸äºĴ":104225,"çĸ«æĥħéĺ²æİ§":104226,"年代":104227,"带åĬ¨":104228,"伤害":104229,"竣çĦ¶":104230,"å¼ķè¿Ľ":104231,"累计":104232,"让æĪij们":104233,"åĽŀæĶ¶":104234,"æĬ¥åIJį":104235,"åĬ©åĬĽ":104236,"èģĶ缣":104237,"çŃĸçķ¥":104238,"åij¨è¾¹":104239,"åĭĴ":104240,"è¿ĺåľ¨":104241,"æµģéĩı":104242,"寻æī¾":104243,"ç͵åĬĽ":104244,"èιèζ":104245,"è¿ĺèĥ½":104246,"æĭħä»»":104247,"çļĦæĥħåĨµä¸ĭ":104248,"çļĦåİŁåĽł":104249,"缺ä¹ı":104250,"çIJĥåijĺ":104251,"å²ģçļĦ":104252,"çĶ·åŃIJ":104253,"å·¥èµĦ":104254,"è¿ijå¹´æĿ¥":104255,"åijĢ":104256,"æıIJä¾ĽäºĨ":104257,"她们":104258,"å®¶åħ·":104259,"çĩķ":104260,"è½»æĿ¾":104261,"æł¡åĽŃ":104262,"èĢĥæł¸":104263,"åį±éĻ©":104264,"åħļç»Ħç»ĩ":104265,"æĢ»ç»ıçIJĨ":104266,"çļĦæĸ°":104267,"çİ»çĴĥ":104268,"è¿Ļä½į":104269,"对æŃ¤":104270,"家人":104271,"çļĦè¦ģæ±Ĥ":104272,"温度":104273,"æĮĩæķ°":104274,"缴åΰ":104275,"æŃ¤æĹ¶":104276,"æ¹ĸåįĹ":104277,"éĥ½è¦ģ":104278,"ä½ľåĩº":104279,"åIJĦä½į":104280,"èĢĥçĶŁ":104281,"ä¾Ŀæį®":104282,"说è¯Ŀ":104283,"æĪijä¹Ł":104284,"å·¥åİĤ":104285,"åıĺæĪIJ":104286,"ä»ĸ人":104287,"æĪijè§īå¾Ĺ":104288,"åIJĦ级":104289,"ä¼łå¥ĩç§ģæľį":104290,"ä¸Ĭåįĩ":104291,"好åĥı":104292,"åĬłéĢŁ":104293,"äºĮåįģ":104294,"è¢ģ":104295,"è£ħ饰":104296,"éĥ½èĥ½":104297,"ä¸Ģå¼ł":104298,"åĬ¨æĢģ":104299,"å¹´çļĦ":104300,"è¿Ļå°±æĺ¯":104301,"ä¹Łè¦ģ":104302,"èµĦæł¼":104303,"æĪĺäºī":104304,"æĦŁè°¢":104305,"åŁ¹èĤ²":104306,"天æ°Ķ":104307,"女士":104308,"åı¯èĥ½ä¼ļ":104309,"çļĦ产åĵģ":104310,"ä¹Łå°±":104311,"主è¦ģæĺ¯":104312,"åĪºæ¿Ģ":104313,"ç»Ļä½ł":104314,"大æķ°æį®":104315,"åĮ»åѦ":104316,"åΤæĸŃ":104317,"ä»ĸ说":104318,"表æ¼Ķ":104319,"äºļæ´²":104320,"ä¸ĵé¢ĺ":104321,"ç«ŀäºīåĬĽ":104322,"éĤ£æł·":104323,"å±ķå¼Ģ":104324,"å¹³æĹ¶":104325,"æİ¥ä¸ĭæĿ¥":104326,"æī¿è¯º":104327,"æ³ķåĽ½":104328,"åħ³å¿ĥ":104329,"ä¼ļæľī":104330,"éĤĢ请":104331,"é¢Ħéĺ²":104332,"对æİ¥":104333,"好äºĨ":104334,"åĴ±ä»¬":104335,"çļĦæĦŁè§ī":104336,"æĢĿè·¯":104337,"éĥ½æ²¡æľī":104338,"çļĦæĸ¹æ³ķ":104339,"女åŃIJ":104340,"åı¸æ³ķ":104341,"è¿ĺä¼ļ":104342,"è¶ĬæĿ¥è¶Ĭå¤ļ":104343,"åĽłçĤº":104344,"æµ·åįĹ":104345,"人æķ°":104346,"å°Ĩä¼ļ":104347,"ä¸ļ主":104348,"é¤IJ饮":104349,"å±ħä½ı":104350,"åıijåĩº":104351,"è¿ijæľŁ":104352,"å¼ķé¢Ĩ":104353,"æľºåĻ¨äºº":104354,"åĩºæĿ¥çļĦ":104355,"çľĭè§ģ":104356,"ä¿Ĭ":104357,"让ä»ĸ":104358,"ä¸įæĥ³":104359,"å·¥ä½ľçļĦ":104360,"è¡¥åħħ":104361,"æµħ":104362,"çī¹å¾ģ":104363,"ä¸Ĭå¸Ĥåħ¬åı¸":104364,"ç¾İé£Ł":104365,"广西":104366,"æ¯ıä¸Ģ个":104367,"èIJ½åľ°":104368,"åĵģç§į":104369,"åĴĮè°IJ":104370,"å½»åºķ":104371,"é«ĺèĢĥ":104372,"æĺ¨å¤©":104373,"åīįå¾Ģ":104374,"çĽijæµĭ":104375,"çĻ¾åº¦":104376,"åľ¨ä¸ŃåĽ½":104377,"çļĦéľĢæ±Ĥ":104378,"亿ç¾İåħĥ":104379,"åŃ¦æľ¯":104380,"æĶ¶åΰ":104381,"æĿ¿åĿĹ":104382,"ä¸Ģ段":104383,"æŀĦæĪIJ":104384,"ä¼ģä¸ļçļĦ":104385,"表éĿ¢":104386,"æķ´çIJĨ":104387,"ç»ĵå©ļ":104388,"人家":104389,"åģľæŃ¢":104390,"åѦç§ij":104391,"æĺ¾å¾Ĺ":104392,"ä¼ijæģ¯":104393,"é¢ĦæľŁ":104394,"æĪĸæĺ¯":104395,"çļĦ主è¦ģ":104396,"åºĶ对":104397,"èµ°äºĨ":104398,"ä¸ŃéĹ´":104399,"èµ°è¿Ľ":104400,"åijĪçݰ":104401,"æIJŃéħį":104402,"é¹ı":104403,"æĺ¯åĽłä¸º":104404,"æĥħ绪":104405,"å®ļæľŁ":104406,"社ä¼ļ主ä¹ī":104407,"çŃī级":104408,"çŁĽçĽ¾":104409,"é£ŀæľº":104410,"èĩ³ä»Ĭ":104411,"æĶ¶éĽĨ":104412,"çļĦæķħäºĭ":104413,"åĪĩå®ŀ":104414,"å®ŀçݰäºĨ":104415,"å½¢æĪIJäºĨ":104416,"åįĹæĸ¹":104417,"ä¸ŃåѦ":104418,"æµ·æ´ĭ":104419,"åIJ¦åĪĻ":104420,"æĭįæijĦ":104421,"大åѦçĶŁ":104422,"åĩºçݰäºĨ":104423,"æĦıå¤ĸ":104424,"ä¹Łèĥ½":104425,"çļĦèĥ½åĬĽ":104426,"åĿIJåľ¨":104427,"åĪĻæĺ¯":104428,"èĢĥå¯Ł":104429,"å°Ĭéĩį":104430,"éĺ²æŃ¢":104431,"ç´§å¼ł":104432,"读书":104433,"åĩºè¡Į":104434,"å°±æľī":104435,"å±¥è¡Į":104436,"çݰ代åĮĸ":104437,"åĽ½åĬ¡":104438,"åĽ½åĬ¡éĻ¢":104439,"ç»´ä¿®":104440,"åİŁåĪĽ":104441,"æĺ¯æĮĩ":104442,"ä¼ijéĹ²":104443,"çĤ®":104444,"æĸ°æĹ¶ä»£":104445,"éĢĻåĢĭ":104446,"ä¸įæķ¢":104447,"å®Įç¾İ":104448,"ç»ĨèĬĤ":104449,"éŃı":104450,"èͬèıľ":104451,"é¢Ĩ导çıŃåŃIJ":104452,"è¶ħ级":104453,"è¡Įæĥħ":104454,"人工æĻºèĥ½":104455,"åį°åº¦":104456,"åŁºç¡Ģ设æĸ½":104457,"åıĪæĺ¯":104458,"èį¯çī©":104459,"åIJ¸æĶ¶":104460,"åį´æĺ¯":104461,"éĥİ":104462,"å¥ĸåĬ±":104463,"çļĦæľĭåıĭ":104464,"ä¿ĿçķĻ":104465,"è§Ħå¾ĭ":104466,"æĸ°çĸĨ":104467,"è¿ĺåı¯ä»¥":104468,"æİ¥è¿ij":104469,"æŃ¤åīį":104470,"æī¹åĩĨ":104471,"æĢİä¹Īæł·":104472,"çļĦä½įç½®":104473,"ä¸ĢåĿĹ":104474,"æĭĴç»Ŀ":104475,"顾客":104476,"ä¹Łåľ¨":104477,"ä¸ĢçĶŁ":104478,"éĥ¨éĺŁ":104479,"å¹´åīį":104480,"æĸ¹éĿ¢çļĦ":104481,"å°Ŀè¯ķ":104482,"羣æŃ£çļĦ":104483,"ç¦ģæŃ¢":104484,"è¿ĺ没æľī":104485,"æ°ijçĶŁ":104486,"èµ°åIJij":104487,"èĦ¸ä¸Ĭ":104488,"å½ĵ天":104489,"éĽĨåĽ¢åħ¬åı¸":104490,"çļĦä¸Ģç§į":104491,"西æĸ¹":104492,"åĽŀåºĶ":104493,"ä¸Ģ声":104494,"常常":104495,"æıIJåΰ":104496,"èħ¾è®¯":104497,"æľįè£ħ":104498,"为ä½ķ":104499,"äºijåįĹ":104500,"å°±ç®Ĺ":104501,"ä¼łæī¿":104502,"åıįèĢĮ":104503,"ä¸ĩåIJ¨":104504,"财产":104505,"å¦Ĥä¸ĭ":104506,"æĹ¥åīį":104507,"åİŁæľ¬":104508,"æľĢéĩįè¦ģçļĦ":104509,"认è¯ģ":104510,"ä¸Ģéģĵ":104511,"ä¿¡æģ¯åĮĸ":104512,"å¾ĹåΰäºĨ":104513,"é̲è¡Į":104514,"æĪijè¦ģ":104515,"éĢļä¿¡":104516,"室åĨħ":104517,"èµļéĴ±":104518,"æĶ¶èĹı":104519,"è§£åĨ³æĸ¹æ¡Ī":104520,"æĪ¿äº§":104521,"çĭ¼":104522,"æ´»åĬĽ":104523,"ç»ıæµİåıijå±ķ":104524,"çŃīå¾ħ":104525,"ä¹Łå¾Ī":104526,"åĿij":104527,"å¾Ī好çļĦ":104528,"éļ¾åº¦":104529,"ä¸įå¦Ĥ":104530,"人æ°ijæĶ¿åºľ":104531,"åĩºåıij":104532,"åīįæľŁ":104533,"æ¼Ķåijĺ":104534,"女çĶŁ":104535,"èģļçĦ¦":104536,"审计":104537,"é¢Ħæµĭ":104538,"ä¾Ŀæīĺ":104539,"äºĶå¹´":104540,"补贴":104541,"æ¸ħæĻ°":104542,"éªĤ":104543,"çľĭèµ·æĿ¥":104544,"çļĦåŃ©åŃIJ":104545,"é¢ijéģĵ":104546,"ä½ıå®ħ":104547,"éĿ¢åIJij":104548,"æľĢä½İ":104549,"æĹ¢çĦ¶":104550,"ä¸Ģå¥Ĺ":104551,"æķ°åѦ":104552,"群ä½ĵ":104553,"åĮĹ京å¸Ĥ":104554,"å±ħçĦ¶":104555,"æ°ĽåĽ´":104556,"éĢĶå¾Ħ":104557,"çļĦåŁºç¡Ģä¸Ĭ":104558,"èģĮè´£":104559,"åı¯èĥ½æĺ¯":104560,"åĨĽäºĭ":104561,"æĪIJæķĪ":104562,"åŃ©åŃIJ们":104563,"计ç®Ĺæľº":104564,"赤":104565,"产ä¸ļåıijå±ķ":104566,"巨大çļĦ":104567,"工人":104568,"çĶŁéķ¿":104569,"éĥ½åı¯ä»¥":104570,"çļĦæľºä¼ļ":104571,"èµĦè´¨":104572,"çĹĽèĭ¦":104573,"ç²īä¸Ŀ":104574,"å¢ĵ":104575,"å¹³å®ī":104576,"管éģĵ":104577,"è·ŁçĿĢ":104578,"é¥®é£Ł":104579,"åķĨå®¶":104580,"å¤ļå®¶":104581,"åı¸æľº":104582,"åºĶ该æĺ¯":104583,"éĢıéľ²":104584,"认å®ļ":104585,"è¡Įä¸ļçļĦ":104586,"çļĦä¼ģä¸ļ":104587,"æ¯ıä¸Ģ":104588,"èĮĥåĽ´åĨħ":104589,"è¾ĥ大":104590,"è´¤":104591,"å¤§èµĽ":104592,"å¤ļäºĨ":104593,"鸿":104594,"临åºĬ":104595,"åľ¨è¿Ļ个":104596,"çļĦåĨħ容":104597,"éĶĢéĩı":104598,"å¾Īå°ij":104599,"åŃŁ":104600,"ç»´æĮģ":104601,"åĴĸåķ¡":104602,"æľ¬åľ°":104603,"èī²å½©":104604,"å¹¶éĿŀ":104605,"èĢĮå·²":104606,"温æļĸ":104607,"èIJ§":104608,"æĬĵä½ı":104609,"èĢĮä¸įæĺ¯":104610,"åĸĬ":104611,"çļĦåħ³ç³»":104612,"çī©åĵģ":104613,"éĤ£æĺ¯":104614,"åĨľäº§åĵģ":104615,"è¿ĻæĹ¶":104616,"å©ļå§»":104617,"æ°´æŀľ":104618,"æĶ¶èİ·":104619,"ä»ĺåĩº":104620,"客æĪ·ç«¯":104621,"æ¼Ķåĩº":104622,"åħ¨æĸ°":104623,"è¿Ļä¹Łæĺ¯":104624,"æĺ¯çͱ":104625,"è§Ĥ念":104626,"æľī个":104627,"éĢłåŀĭ":104628,"èĥľåĪ©":104629,"ä¸īæĺ¯":104630,"è¶ħå¸Ĥ":104631,"åħļå»ºå·¥ä½ľ":104632,"æĶ¾å¿ĥ":104633,"线路":104634,"æĭĽçĶŁ":104635,"åIJĥé¥Ń":104636,"è½ī":104637,"å°½éĩı":104638,"è§ģåΰ":104639,"åIJĮæ¯Ķå¢ŀéķ¿":104640,"åįİ为":104641,"æĪijå¸Ĥ":104642,"æıIJåĩºäºĨ":104643,"æ°ijèѦ":104644,"åįļçī©":104645,"åįļçī©é¦Ĩ":104646,"è¯ļä¿¡":104647,"åīįéĿ¢":104648,"山西":104649,"è¾ħåĬ©":104650,"转移":104651,"æĽ´ä¸º":104652,"丰å¯ĮçļĦ":104653,"åį¢":104654,"å¿«éĢĴ":104655,"æĺ¾èijĹ":104656,"çī©èµĦ":104657,"åĪ°è¾¾":104658,"æľīåĪ©äºİ":104659,"åijĨ":104660,"åŃ©åŃIJçļĦ":104661,"ä¸įä½Ĩ":104662,"çłĶç©¶éĻ¢":104663,"çͳæĬ¥":104664,"æļ¨":104665,"æ°ijéĹ´":104666,"åį»":104667,"çļĦå£°éŁ³":104668,"å¸ĤåľºçļĦ":104669,"ä¸Ģåı¥":104670,"çľģ级":104671,"æĿ¥çļĦ":104672,"åĵªä¸ª":104673,"æīįä¼ļ":104674,"åĪĨéħį":104675,"èĶ¡":104676,"ä»ĸåľ¨":104677,"åħ±æľī":104678,"å¡ĺ":104679,"èĴĤ":104680,"éľį":104681,"åıĤè§Ĥ":104682,"ä¸Ī夫":104683,"ä¾ĿéĿł":104684,"æľīæĹ¶":104685,"äºĨå¾Īå¤ļ":104686,"ä¸ĸçķĮæĿ¯":104687,"å®¶æĹı":104688,"ä¸įéľĢè¦ģ":104689,"大å¸Ī":104690,"èŀįåħ¥":104691,"éĿŀæ³ķ":104692,"çĹħ人":104693,"åIJİæľŁ":104694,"大家éĥ½":104695,"ç½ijåĿĢ":104696,"åİŁæĸĻ":104697,"ä¾¿å®ľ":104698,"æ¶Ľ":104699,"ä»¿ä½Ľ":104700,"å·®è·Ŀ":104701,"åı¦ä¸Ģæĸ¹éĿ¢":104702,"产åĵģçļĦ":104703,"赫":104704,"æĥħåĨµä¸ĭ":104705,"éĴ¢éĵģ":104706,"æľ¬ç«Ļ":104707,"纳åħ¥":104708,"å·²æľī":104709,"æľī没æľī":104710,"估计":104711,"é£ĺ":104712,"æľŁè´§":104713,"åĢĭ人è³ĩæĸĻ":104714,"ä¸ĵä¸ļçļĦ":104715,"çĪĨåıij":104716,"èĩ´åĬĽäºİ":104717,"çİ°åľ¨çļĦ":104718,"æľīåĵªäºĽ":104719,"çł´åĿı":104720,"æķ°åŃĹåĮĸ":104721,"åľ°éĿ¢":104722,"é»ijèī²":104723,"å¹¼åĦ¿åĽŃ":104724,"çļĦç²¾ç¥ŀ":104725,"äºŃ":104726,"导æ¼Ķ":104727,"çݰæľī":104728,"æŃ¦åύ":104729,"èĭıå·ŀ":104730,"çİĦ":104731,"æ±Łè¥¿":104732,"延伸":104733,"论æĸĩ":104734,"è¾ĥ为":104735,"çİ©æ³ķ":104736,"é¼İ":104737,"åIJĮæŃ¥":104738,"éĩĬæĶ¾":104739,"æĽĿåħī":104740,"åĿļåĨ³":104741,"å§Ķæīĺ":104742,"å°Ĩåľ¨":104743,"äºĪ以":104744,"ä½ľæĸĩ":104745,"èĢĮåľ¨":104746,"ä¼ĺåħĪ":104747,"åĽŀåİ»":104748,"ä¿®å¤į":104749,"åĽ½åĨħå¤ĸ":104750,"çŃĸåĪĴ":104751,"åıijæĶ¾":104752,"å¿ĥæĥħ":104753,"çļĦåİĨåı²":104754,"éĿ¢è¯ķ":104755,"举åĮĹ":104756,"ä¿¡åı·":104757,"ç²®é£Ł":104758,"è¯ģ书":104759,"æŁIJäºĽ":104760,"è¿IJä½ľ":104761,"åĨ²åĩ»":104762,"çĥŃçĤ¹":104763,"æĹ¶æĹ¶":104764,"æĹ¶æĹ¶å½©":104765,"åľ°çĤ¹":104766,"ä¸Ģä½ĵåĮĸ":104767,"éļ¾é¢ĺ":104768,"æĽ°":104769,"ç«ĭåĪ»":104770,"æĺ¯éĿŀ常":104771,"åħ±åĴĮ":104772,"åħ±åĴĮåĽ½":104773,"æ¿ĢåĬ±":104774,"æľīæķĪçļĦ":104775,"å¤Ħç½®":104776,"该åħ¬åı¸":104777,"æ£ĢéªĮ":104778,"èѦæĸ¹":104779,"è´¾":104780,"äºĨä¸Ģä¸ĭ":104781,"ä»ĬåIJİ":104782,"çħ®":104783,"ç͍åĵģ":104784,"读èĢħ":104785,"æĪijåľ¨":104786,"åĽŀå¤į":104787,"ä¸Ģ座":104788,"è¿ĺ没":104789,"å®ļåζ":104790,"没æĥ³åΰ":104791,"夹":104792,"ä¼łéĢĴ":104793,"ä¸Ģ款":104794,"强大çļĦ":104795,"çļĦè¡Į为":104796,"å¤ı天":104797,"åıijåĬ¨æľº":104798,"é¢ĨåŁŁçļĦ":104799,"å®ŀéªĮ室":104800,"ä¸ĢæĬĬ":104801,"æĺ¯ä¸ºäºĨ":104802,"éĻķ西":104803,"æĭħä¿Ŀ":104804,"è¾¾æĪIJ":104805,"è¦ģæĺ¯":104806,"æĺİ天":104807,"ç»Ļä»ĸ":104808,"建ç«ĭäºĨ":104809,"ä¸įè¡Į":104810,"ä¸Ńæĸĩ":104811,"åľ°è¯´":104812,"åIJİçļĦ":104813,"çĽijæİ§":104814,"é̏":104815,"æĢ»éĥ¨":104816,"æľ¬æĸĩ":104817,"鹿":104818,"æĻ¯è§Ĥ":104819,"çļĦ缮æłĩ":104820,"èĽĩ":104821,"åĨ¯":104822,"ä¸ŃåĮ»":104823,"æķĪåºĶ":104824,"产éĩı":104825,"åŃĿ":104826,"è´¦æĪ·":104827,"è¿Ŀåıį":104828,"èij£äºĭä¼ļ":104829,"äº¬ä¸ľ":104830,"责任ç¼ĸè¾ij":104831,"åķıé¡Į":104832,"çαå¿ĥ":104833,"èŃ¦å¯Ł":104834,"é¤IJåİħ":104835,"å¸ĤæĶ¿åºľ":104836,"天天":104837,"æĸ°é²ľ":104838,"éĥijå·ŀ":104839,"è¶ħè¶Ĭ":104840,"å½Ń":104841,"çŁ¥è¯Ĩ产æĿĥ":104842,"åĽŀå¿Ĩ":104843,"路线":104844,"å»īæ´ģ":104845,"éĿĴå°ijå¹´":104846,"åıĸå¾ĹäºĨ":104847,"çľĭåΰäºĨ":104848,"馬":104849,"ç²¾åĵģ":104850,"åľ°éĵģ":104851,"æĮģæľī":104852,"ä¸ĭäºĨ":104853,"æľīæĹ¶åĢĻ":104854,"ä¸Ģ人":104855,"æĴĴ":104856,"ä»Ķç»Ĩ":104857,"èĢģåħ¬":104858,"äºĭå®ŀä¸Ĭ":104859,"èģĶèµĽ":104860,"ä¾ĽåºĶéĵ¾":104861,"é¢Ħç®Ĺ":104862,"åζéĢłä¸ļ":104863,"å®īåħ¨çĶŁäº§":104864,"俱ä¹IJ":104865,"俱ä¹IJéĥ¨":104866,"çļĦæł¸å¿ĥ":104867,"æīĵç®Ĺ":104868,"å½±çīĩ":104869,"æIJŃ建":104870,"ä¹Łä¸įä¼ļ":104871,"æĭħå½ĵ":104872,"å±ĤéĿ¢":104873,"åѦåijĺ":104874,"临æĹ¶":104875,"缸ç»ĵåIJĪ":104876,"对æ¯Ķ":104877,"ä»ĸæĺ¯":104878,"æĸ°åĮº":104879,"è¿Ľåİ»":104880,"çϾ年":104881,"ä¿©":104882,"尽快":104883,"ç͵åŃIJåķĨåĬ¡":104884,"æĽ´æľī":104885,"æ¸ħçIJĨ":104886,"åı¦ä¸Ģ个":104887,"åĤ»":104888,"ä»Ģä¹Īæł·çļĦ":104889,"æĺ¯æľĢ":104890,"åij¨å¹´":104891,"å¾Ī容æĺĵ":104892,"åĽ¢ç»ĵ":104893,"ç´Ħ":104894,"æĹ©å·²":104895,"çļĦåıĺåĮĸ":104896,"éľŀ":104897,"æĹ¥ä¸ĬåįĪ":104898,"失åİ»":104899,"ä¸Ńåľĭ":104900,"çļĦä¸ĢäºĽ":104901,"å°ıåŃ©":104902,"ä¸ĭè·Į":104903,"éĶ»çĤ¼":104904,"éij":104905,"éij«":104906,"å¿ĹæĦ¿èĢħ":104907,"èĤ¡å¸Ĥ":104908,"èµĽäºĭ":104909,"许åı¯è¯ģ":104910,"åı¯æĮģç»Ń":104911,"åijĬè¯īè®°èĢħ":104912,"éĢ»è¾ij":104913,"å¼ķåħ¥":104914,"çļĦè¿ĩç¨ĭä¸Ń":104915,"è§Ĩè§ī":104916,"èĩªæ²»åĮº":104917,"è¯ģæį®":104918,"è£ħç½®":104919,"第ä¸īæĸ¹":104920,"å¹´æĿ¥":104921,"å¹¿ä¸ľçľģ":104922,"带æĿ¥äºĨ":104923,"éķ¿æ±Ł":104924,"访éĹ®":104925,"å·®ä¸įå¤ļ":104926,"æĺ¯æĪij":104927,"éģŃéģĩ":104928,"æĬĵ好":104929,"é«ĺè¾¾":104930,"å¹¶åľ¨":104931,"èĩªè§ī":104932,"ä¾ĽåºĶåķĨ":104933,"æĥħæĦŁ":104934,"ä½ıäºĨ":104935,"çļĦèģĮä¸ļ":104936,"çļĩå¸Ŀ":104937,"西éĥ¨":104938,"åĴĮå¹³":104939,"çļĦåĬĽéĩı":104940,"汪":104941,"åħħåĪĨåıijæĮ¥":104942,"æĬķè¯ī":104943,"èµ·åΰ":104944,"äºĴ缸":104945,"æ¾³éŨ":104946,"æİ¥åΰ":104947,"æ°´æ³¥":104948,"模åŀĭ":104949,"ä¸ĢåįĬ":104950,"ç§©åºı":104951,"æĪijä»¬åľ¨":104952,"æī¿è®¤":104953,"ä¸Ģéĥ¨åĪĨ":104954,"åįłæ¯Ķ":104955,"å¦ĩ女":104956,"ç²ĺ":104957,"äºĨè§£åΰ":104958,"ä¸Ģå®ļä¼ļ":104959,"åIJĦ大":104960,"èµ°åĩº":104961,"为大家":104962,"é«ĺéĵģ":104963,"åı¯ä»¥åľ¨":104964,"ä½Ĩåľ¨":104965,"çĶŁæĢģçݯå¢ĥ":104966,"èı¯":104967,"çļĦä»·æł¼":104968,"麻çĥ¦":104969,"æ¿Ģåıij":104970,"éĤ£å°±":104971,"çļĦæł·åŃIJ":104972,"为æŃ¤":104973,"å¤©åľ°":104974,"çļĦ缮çļĦ":104975,"åĢºåΏ":104976,"å·²ç¶ĵ":104977,"åĽĽå¤§":104978,"åIJĮæĹ¶ä¹Ł":104979,"å½¼æŃ¤":104980,"æĭ¿åΰ":104981,"åIJ«éĩı":104982,"åįģ大":104983,"éļ¾éģĵ":104984,"å¼Ĺ":104985,"ä¸Ģ段æĹ¶éĹ´":104986,"çħ§é¡¾":104987,"æķ°æį®æĺ¾ç¤º":104988,"æĪIJ为äºĨ":104989,"èµ°åΰ":104990,"æľ¬åħ¬åı¸":104991,"ç»Ī端":104992,"ä¹Łä¸įæĺ¯":104993,"头åıij":104994,"大约":104995,"é£İæĻ¯":104996,"æ¶ĪèĢĹ":104997,"å®¡æŁ¥":104998,"äºīåıĸ":104999,"æ³ķæ²»":105000,"äºĭçī©":105001,"ç¼ĵè§£":105002,"æĥ¨":105003,"缸åºĶçļĦ":105004,"çļĦæķĪæŀľ":105005,"åıįå¤į":105006,"åıijçĶŁäºĨ":105007,"éĢĻäºĽ":105008,"ç»ĥä¹ł":105009,"åݨæĪ¿":105010,"å¼Ģæĭĵ":105011,"欣èµı":105012,"夫妻":105013,"ä¸įä¸Ģæł·":105014,"产èĥ½":105015,"èĬ¯çīĩ":105016,"è¦ģç´ł":105017,"åıį对":105018,"çİĩåħĪ":105019,"è´§çī©":105020,"æĹ¥ç͵":105021,"ä½ľå®¶":105022,"æĶ¹è¿Ľ":105023,"æĪIJåĪĨ":105024,"åĽłèĢĮ":105025,"åĩıèĤ¥":105026,"æ½ĺ":105027,"å±±ä¸ľçľģ":105028,"åĬĿ":105029,"åŁĭ":105030,"æŃ¦è£ħ":105031,"æ±ĩæĬ¥":105032,"ä¸Ģ个æľĪ":105033,"çĥŃéŨ":105034,"大éģĵ":105035,"æ´»åĭķ":105036,"éĥ½å¾Ī":105037,"çĶµæ¢¯":105038,"ç´§æĢ¥":105039,"åĢºåĬ¡":105040,"客æľį":105041,"ä¸Ģéĥ¨":105042,"ä½łæĺ¯":105043,"çݰçĬ¶":105044,"æŃ£ç¡®çļĦ":105045,"ä¹ĭå¤Ħ":105046,"ç¼ĸåζ":105047,"ä½łåı¯ä»¥":105048,"çŃīåľ°":105049,"èİī":105050,"对è¯Ŀ":105051,"æ·ĺå®Ŀ":105052,"è°ĥèĬĤ":105053,"æİĴæĶ¾":105054,"åºĵåŃĺ":105055,"ç´ļ":105056,"çļĦä¼ĺåĬ¿":105057,"æĿĥå¨ģ":105058,"以ä¸ĭç®Ģç§°":105059,"ä¸Ģ项":105060,"èģļéĽĨ":105061,"ä¼łç»ŁçļĦ":105062,"æ··åIJĪ":105063,"è¿Ļä¸ĢçĤ¹":105064,"ä¸Ģçľ¼":105065,"æĹłéĻIJ":105066,"èİ·å¾ĹäºĨ":105067,"éĢīæīĭ":105068,"åζåĵģ":105069,"åįıä½ľ":105070,"çĭ¬çī¹çļĦ":105071,"ä¸Ģ级":105072,"è¿Ļ个éĹ®é¢ĺ":105073,"æĸĮ":105074,"æĺ¯æĪij们":105075,"æķĮ人":105076,"æ¸ħæ´Ĺ":105077,"ä¸ĢçĽ´åľ¨":105078,"å°ıç±³":105079,"çļĦè¿ĩç¨ĭ":105080,"åľ¨åĮĹ京":105081,"ä¸ĢæĶ¯":105082,"æĹ©ä¸Ĭ":105083,"æĸĩèīº":105084,"ç¦ıåĪ©":105085,"é£Łç͍":105086,"æĦŁåĬ¨":105087,"åħ¨ç¨ĭ":105088,"æĶ¯åĩº":105089,"æĸ°å»º":105090,"å¸ķ":105091,"æĺ¾çĦ¶":105092,"羣çļĦæĺ¯":105093,"æĸ°éĹ»ç½ij":105094,"èĥ½åIJ¦":105095,"åįıåĬ©":105096,"亲èĩª":105097,"å¾Īæľī":105098,"çϼå±ķ":105099,"æĦı大":105100,"æĦı大åĪ©":105101,"ç͵ç½ij":105102,"æĹ¥çĽĬ":105103,"çĨ±":105104,"èĤĮèĤ¤":105105,"çĶ·æĢ§":105106,"ç»Ħ建":105107,"çŃīéĹ®é¢ĺ":105108,"æ¶ĪéϤ":105109,"æĬ¤çIJĨ":105110,"å¡ijæĸĻ":105111,"ä¹Įåħĭ":105112,"ä¹Įåħĭåħ°":105113,"åķĨæłĩ":105114,"çIJ³":105115,"æĸ°æīĭ":105116,"çļĦçī¹çĤ¹":105117,"åĴ¬":105118,"å½ĵä¸ĭ":105119,"设计å¸Ī":105120,"èµĶåģ¿":105121,"第åįģ":105122,"æĻºèĥ½åĮĸ":105123,"å¼ĢåıijåĮº":105124,"åı¯ä»¥éĢļè¿ĩ":105125,"åħ±äº§åħļ":105126,"åİī害":105127,"ç쵿´»":105128,"æĹ¶åħī":105129,"éĥ¨ä½į":105130,"人æĸĩ":105131,"è¿ĽæĿ¥":105132,"ä¹ĭæīĢ以":105133,"ä¸īåįģ":105134,"çļĦåѦçĶŁ":105135,"éĺ²æĬ¤":105136,"åĽ½äº§":105137,"æ·±åľ³å¸Ĥ":105138,"éĤ£å°±æĺ¯":105139,"åΰä½į":105140,"çľĹ":105141,"çľĹæĻ®":105142,"å®ŀæĹ¶":105143,"åı°çģ£":105144,"èĢĮä¸į":105145,"æĮĩå®ļ":105146,"åĿĿ":105147,"èħIJè´¥":105148,"çī¹å®ļ":105149,"å¢ŀéĢŁ":105150,"æłĩçѾ":105151,"æĪ¿ä»·":105152,"æĦģ":105153,"贯彻èIJ½å®ŀ":105154,"æĢ§è´¨":105155,"çłĶç©¶çĶŁ":105156,"ç¾İ容":105157,"æī¹è¯Ħ":105158,"究竣":105159,"人åĬĽèµĦæºIJ":105160,"éĸĭå§ĭ":105161,"åĽŀå½Ĵ":105162,"èIJ¥åķĨ":105163,"èIJ¥åķĨçݯå¢ĥ":105164,"ä¸ŃåĽ½äºº":105165,"çļĦåŁºæľ¬":105166,"è¯Ŀé¢ĺ":105167,"æłĩåĩĨåĮĸ":105168,"西èĹı":105169,"åĭ¾":105170,"çļĦ设计":105171,"ç®ĢåįķçļĦ":105172,"å¤įåζ":105173,"æ¸IJæ¸IJ":105174,"以å¤ĸ":105175,"èģĶåĬ¨":105176,"两次":105177,"æĢ§åĴĮ":105178,"æĽ´å¤§":105179,"çļĦåIJįåŃĹ":105180,"飦":105181,"ä½łè¦ģ":105182,"å¢ĥå¤ĸ":105183,"æĹ©æľŁ":105184,"åĪĿæŃ¥":105185,"è´¦åı·":105186,"害æĢķ":105187,"æĺ¨æĹ¥":105188,"åĪļæīį":105189,"ç¥ŀç§ĺ":105190,"ç²¾å¿ĥ":105191,"æµģéĢļ":105192,"åħ¨æĸ¹ä½į":105193,"以å¾Ģ":105194,"ä¹Łå°Ĩ":105195,"æĺ¯ä¸ŃåĽ½":105196,"åĽ½å®¶çº§":105197,"å°ĨåĨĽ":105198,"æijĬ":105199,"æľĢ为":105200,"第ä¸ĢæĹ¶éĹ´":105201,"æ¶Īæ¯Ĵ":105202,"å°Ĩäºİ":105203,"å¨ģèĥģ":105204,"èĭ±æĸĩ":105205,"æīĭä¸Ń":105206,"çIJĥè¿·":105207,"è§Ĥçľĭ":105208,"离å©ļ":105209,"æľ¬åľŁ":105210,"åĪĨæķ£":105211,"æĻ´":105212,"è¦ģ注æĦı":105213,"浪费":105214,"管æİ§":105215,"åĩºåĶ®":105216,"æĢ»è£ģ":105217,"ä¸Ģéĺµ":105218,"å¨ĩ":105219,"äºĶ个":105220,"å½ĵåĪĿ":105221,"çºłçº·":105222,"ä¸ĵç͍":105223,"å¤ĩæ¡Ī":105224,"åĪĿæľŁ":105225,"å®ĥæĺ¯":105226,"åĮºåĿĹ":105227,"åĮºåĿĹéĵ¾":105228,"大è¿ŀ":105229,"è¿Ļç±»":105230,"åıĺæĪIJäºĨ":105231,"éĤĦæĺ¯":105232,"åįļ客":105233,"çı¾åľ¨":105234,"ä¸Ģæĸ¹":105235,"å®ĮæĪIJäºĨ":105236,"è¿Ļ个æĹ¶åĢĻ":105237,"åħ¨å¹´":105238,"ä¸Ĭ线":105239,"ç½IJ":105240,"ç«ŀèµĽ":105241,"åĩºçīĪ社":105242,"åĵ¥åĵ¥":105243,"寫":105244,"å¾Ĺ以":105245,"èĬ±åĽŃ":105246,"äºĨèµ·æĿ¥":105247,"èĦ±è´«æĶ»åĿļ":105248,"çļĦåİŁåĪĻ":105249,"讲解":105250,"æ¶ĪåĮĸ":105251,"æįŁå®³":105252,"æļĤæĹ¶":105253,"å¾ĹçŁ¥":105254,"éĢĤç͍":105255,"éŨåºĹ":105256,"解读":105257,"æĻ®åıĬ":105258,"人æ°ijæ³ķéĻ¢":105259,"åī¯ä¸»ä»»":105260,"å¿ĥçģµ":105261,"è¯ĬæĸŃ":105262,"ç¾İ女":105263,"æŁ¯":105264,"年以æĿ¥":105265,"æ´»è·ĥ":105266,"åĢŁåĬ©":105267,"åħ±å»º":105268,"è¯ī讼":105269,"æĶ¾æĿ¾":105270,"çªĹåı£":105271,"ä¼ģæ¥Ń":105272,"åĬłæĭ¿":105273,"åĬłæĭ¿å¤§":105274,"ä¹°äºĨ":105275,"主æµģ":105276,"æĩĤå¾Ĺ":105277,"å°Ĩåħ¶":105278,"éĢıæĺİ":105279,"å·¥ä½ľä¸Ń":105280,"èĤ¡ä»·":105281,"æ¡£æ¡Ī":105282,"没æľīä»»ä½ķ":105283,"åijĬçŁ¥":105284,"å¹´åĪĿ":105285,"æĹ¥ä¸ĭåįĪ":105286,"åİĤåķĨ":105287,"èĬĤå¥ı":105288,"主导":105289,"è£Ŀ":105290,"åħ³éĶ®è¯į":105291,"èģĬ天":105292,"åĨĻä½ľ":105293,"æĶ¹éĿ©å¼ĢæĶ¾":105294,"æľīæľĽ":105295,"éĢļæĬ¥":105296,"èIJĮ":105297,"æĢ»é¢Ŀ":105298,"çŁŃæľŁ":105299,"ä¸Ģçķª":105300,"çĶŁæ´»çļĦ":105301,"åĮĸçļĦ":105302,"æĺ¥å¤©":105303,"è¿Ļåľº":105304,"æĸ°å¼Ģä¼łå¥ĩ":105305,"æĺ¯è¦ģ":105306,"å°ļæľª":105307,"åıĺæĽ´":105308,"ä¸Ģåij¨":105309,"客è§Ĥ":105310,"æĹ¥èĩ³":105311,"é¹°":105312,"çݲ":105313,"å°ĨæĿ¥":105314,"客人":105315,"åıĺéĿ©":105316,"说äºĨ":105317,"åİŁçIJĨ":105318,"èģĮåĬ¡":105319,"åıĪæľī":105320,"ä¸Ģåı¥è¯Ŀ":105321,"æĦŁåıĹåΰ":105322,"ç¬ĶèĢħ":105323,"ç§»æ°ij":105324,"西åįĹ":105325,"ä¹ĥèĩ³":105326,"æŃ£è§Ħ":105327,"åĪĿä¸Ń":105328,"çĬ¬":105329,"å½ĵäºĭ":105330,"å½ĵäºĭ人":105331,"æĪij们è¦ģ":105332,"åħ¥åı£":105333,"éĤ£æĹ¶":105334,"æľīéĻIJ责任":105335,"å°ij女":105336,"è¿Ļä¹Īå¤ļ":105337,"åĪĨåħ¬åı¸":105338,"å®ĩå®Ļ":105339,"çļĦéĢīæĭ©":105340,"å§IJå§IJ":105341,"åıijèµ·":105342,"è»į":105343,"æĽ´å¥½åľ°":105344,"éĻĨç»Ń":105345,"æľ¬æľįåĭĻ":105346,"å«©":105347,"èµ¶ç´§":105348,"èĦĤèĤª":105349,"第äºĮ天":105350,"æĪijä¼ļ":105351,"两ä½į":105352,"æķ²":105353,"åħ¬å®īæľºåħ³":105354,"ç§ijæĬĢåĪĽæĸ°":105355,"尺寸":105356,"è¾IJå°Ħ":105357,"å®ĹæķĻ":105358,"转æį¢":105359,"åĩºçİ°åľ¨":105360,"ä¸Ģé¢Ĺ":105361,"æľŁéĻIJ":105362,"åIJĮåѦ们":105363,"åĮĹæĸ¹":105364,"ä½łå°±":105365,"ä¸Ģ带ä¸Ģè·¯":105366,"èĢģå©Ĩ":105367,"游æĪıçݩ家":105368,"çļĦç»ĵæŀľ":105369,"è¡¥åģ¿":105370,"å¤ĸè´¸":105371,"对å¾ħ":105372,"ç»´çĶŁç´ł":105373,"ç»ıéĶĢåķĨ":105374,"è¿ĺå°Ĩ":105375,"åŃIJ女":105376,"æĽ´é«ĺ":105377,"ä¸į大":105378,"éī´å®ļ":105379,"让ä»ĸ们":105380,"æīĢè°ĵçļĦ":105381,"æŃ»äºĨ":105382,"帮æī¶":105383,"åĵ²åѦ":105384,"以ä¸ĬçļĦ":105385,"çļĦåħ³éĶ®":105386,"æĹ©å°±":105387,"æĬ¥ä»·":105388,"éģµå®Ī":105389,"æī©å¼ł":105390,"æĺ¯å¾Ī":105391,"å¼ĢéĢļ":105392,"æĸ°åĬł":105393,"æĸ°åĬłåĿ¡":105394,"ç¿»è¯ij":105395,"询éĹ®":105396,"é¸Ń":105397,"ä½ĵåĨħ":105398,"两个人":105399,"çι":105400,"éľľ":105401,"乡æĿijæĮ¯åħ´":105402,"çĿ¡è§ī":105403,"å®ĺåijĺ":105404,"åĪĽå§ĭ":105405,"åĪĽå§ĭ人":105406,"ä¼Ĺ人":105407,"åį³ä¾¿":105408,"çĸ«èĭĹ":105409,"ä¼ģä¸ļå®¶":105410,"渣":105411,"ç²¾åĬĽ":105412,"å¤ĸéĥ¨":105413,"èģªæĺİ":105414,"è¿Ļä¹Ł":105415,"å½ķåıĸ":105416,"åĨ²çªģ":105417,"åħ¨èº«":105418,"åŃ£èĬĤ":105419,"忽çĦ¶":105420,"çļĦæĢģ度":105421,"åĤ¨å¤ĩ":105422,"ä¿Ŀåħ»":105423,"çļĦæĥ³æ³ķ":105424,"ä¸Ĭæµ·å¸Ĥ":105425,"æIJºæīĭ":105426,"çļĦä¿¡æģ¯":105427,"åķĨåľº":105428,"çļĦæĢĿæĥ³":105429,"æĿĥåĬĽ":105430,"毫æĹł":105431,"æĢĢåŃķ":105432,"硬件":105433,"åĨħèĴĻåı¤":105434,"æİ¢è®¨":105435,"åħ»çĶŁ":105436,"çļĦ表çݰ":105437,"空ä¸Ń":105438,"æģIJæĢĸ":105439,"å¾Īé«ĺ":105440,"ç»ıæµİ社ä¼ļ":105441,"ä¸ĬæĿ¥":105442,"å»¶ç»Ń":105443,"éĩįå¤į":105444,"éĺ²èĮĥ":105445,"çļĦå½¢å¼ı":105446,"æľĪåºķ":105447,"èĢģ年人":105448,"绿åĮĸ":105449,"å±±åĮº":105450,"æĭ¿åĩº":105451,"æĹħ客":105452,"æĽ´æį¢":105453,"åħ¬ä¸»":105454,"èĬĤ约":105455,"åħ¨åİ¿":105456,"åĽŀæĬ¥":105457,"çIJĨæĢ§":105458,"çĸ¯çĭĤ":105459,"æ¶īå«Į":105460,"åī§æĥħ":105461,"åĨ¬åŃ£":105462,"åIJİç»Ń":105463,"è¿Ļæĺ¯ä¸Ģ个":105464,"æ¼Ķ讲":105465,"ä¸Ģå±Ĥ":105466,"æľīåħ³éĥ¨éŨ":105467,"æĹłå¥Ī":105468,"ç§įç±»":105469,"缸åħ³çļĦ":105470,"æĪĸèĢħæĺ¯":105471,"æī¶æĮģ":105472,"å¤ļæķ°":105473,"çļĦä½ľåĵģ":105474,"ä¸ĭä¸ĢæŃ¥":105475,"å¸ĪåĤħ":105476,"é«ĺéĢŁåħ¬è·¯":105477,"好åıĭ":105478,"ä¼ĺç§ĢçļĦ":105479,"è¿ĽäºĨ":105480,"æģIJæĢķ":105481,"äºĨåIJ§":105482,"大è§Ħ模":105483,"çļĦä¸ĸçķĮ":105484,"æĢĢçĸij":105485,"å··":105486,"åħ´å¥ĭ":105487,"æĪ°":105488,"æĿijéĩĮ":105489,"æľĭåıĭåľĪ":105490,"åĨ¬å¤©":105491,"ä¸Ńåįİ人æ°ij":105492,"åįıåķĨ":105493,"è¯ĦéĢī":105494,"æĹŃ":105495,"å¢ŀåĬłäºĨ":105496,"åıĹ伤":105497,"ä¸ĢèĤ¡":105498,"便æį·":105499,"ä¸ij":105500,"鹤":105501,"å¤ĸè§Ĥ":105502,"å·¥ç¨ĭå¸Ī":105503,"åĴĮåħ¶ä»ĸ":105504,"è¿Ļå°±":105505,"ä¸Ńå°ıä¼ģä¸ļ":105506,"西åĮĹ":105507,"åĽ½æľīä¼ģä¸ļ":105508,"èĭ¥æĺ¯":105509,"åı¯æĥľ":105510,"çĶŁæĹ¥":105511,"åĩ½":105512,"ä¹°åįĸ":105513,"ç¥Ŀç¦ı":105514,"人æ°ij群ä¼Ĺ":105515,"åħīæĺİ":105516,"åħ¬å¯ĵ":105517,"æĺ¯è°ģ":105518,"æĪijçŁ¥éģĵ":105519,"è¯Ńæĸĩ":105520,"æķıæĦŁ":105521,"ä¸įéĶĻçļĦ":105522,"æĿ¥è®²":105523,"æ³¢åĬ¨":105524,"çļĦ第ä¸Ģ":105525,"åľ°éľĩ":105526,"åľ¨åħ¨åĽ½":105527,"骨干":105528,"å®īç½®":105529,"å®¶ç͵":105530,"ä¸İæŃ¤":105531,"ä¸İæŃ¤åIJĮæĹ¶":105532,"åıĹçģ¾":105533,"çĥŃ线":105534,"çļĦæĬĢæľ¯":105535,"æµĭéĩı":105536,"ä¾Ŀèµĸ":105537,"ä¸ŃåĽ½çļĦ":105538,"ç̧":105539,"è¾ĥé«ĺ":105540,"踩":105541,"ä¼ļåľ¨":105542,"建éĢł":105543,"导èĪª":105544,"æĥ³èµ·":105545,"åħ¨ä¸ĸçķĮ":105546,"建æĿIJ":105547,"ç¯Ģ":105548,"çļĦåŁºç¡Ģ":105549,"èĩªåĬ¨åĮĸ":105550,"åīįåIJİ":105551,"çĿ¡çľł":105552,"æİ¨è¡Į":105553,"æį®äºĨè§£":105554,"ä»Ģä¹ĪæĹ¶åĢĻ":105555,"ä¸įåĸľæ¬¢":105556,"çħ¤çĤŃ":105557,"éĤ£ä¹Īå¤ļ":105558,"å¸ĤåľºåĮĸ":105559,"ä¸į管æĺ¯":105560,"ç«ĭåľº":105561,"éĥ½æ²¡":105562,"课é¢ĺ":105563,"æĪij们å°Ĩ":105564,"è¿ĩçļĦ":105565,"åĨįåĬłä¸Ĭ":105566,"çξ":105567,"身æĿIJ":105568,"çͷ女":105569,"è¿ľè¿ľ":105570,"çĶ·çĶŁ":105571,"èĩªèº«çļĦ":105572,"è´Łæĭħ":105573,"çϾä¸ĩ":105574,"西çıŃ":105575,"西çıŃçīĻ":105576,"åĩĢåĪ©æ¶¦":105577,"澳大":105578,"澳大åĪ©äºļ":105579,"ä¸įåİ»":105580,"æī¿åıĹ":105581,"楼çĽĺ":105582,"å¢ĥåĨħ":105583,"æ··åĩĿ":105584,"æ··åĩĿåľŁ":105585,"æĢĿæĥ³æĶ¿æ²»":105586,"å¸ĤåĮº":105587,"æĭĽæłĩ":105588,"åĽ¢ä½ĵ":105589,"è¿Ľåº¦":105590,"åĨĽéĺŁ":105591,"åıįå¼¹":105592,"äºĨä¸ĢäºĽ":105593,"æİ¥å¾ħ":105594,"çļĦåŃ¦ä¹ł":105595,"éħįéĢģ":105596,"é£Łåĵģå®īåħ¨":105597,"æĽ¿ä»£":105598,"æĺ¯ä»¥":105599,"éĢļç͍":105600,"çłĶç©¶æīĢ":105601,"ç¦ħ":105602,"æīĶ":105603,"éļĶ离":105604,"ä¸ĩå¹³æĸ¹ç±³":105605,"çļĦè§Ħå®ļ":105606,"ç»ĻæĪij们":105607,"æ¿Ģåħī":105608,"ä¼ļåĩºçݰ":105609,"çŁŃä¿¡":105610,"ç©¿çĿĢ":105611,"æ²Īéĺ³":105612,"æķĻæĿIJ":105613,"éĺ²çĸ«":105614,"ä¼ĺèī¯":105615,"约å®ļ":105616,"æĪijçľģ":105617,"åħ¬æ°ij":105618,"é쏿ĵ":105619,"é쏿ĵĩ":105620,"å·²æĪIJ为":105621,"ä¸įå¿ħ":105622,"ç¥ĸåĽ½":105623,"å¹¶æľª":105624,"åľŁå£¤":105625,"å¾®ç¬ij":105626,"äºĭä¸ļåįķä½į":105627,"çļĦ游æĪı":105628,"åħ¬ç¤º":105629,"åIJĪçIJĨçļĦ":105630,"çªĿ":105631,"æ°Ķ象":105632,"å®¶ä¸Ń":105633,"äº®çĽ¸":105634,"å᫿ĺŁ":105635,"è®°è½½":105636,"è§Ĩéĩİ":105637,"åľ°åĮºçļĦ":105638,"ä½Ĩä»ĸ":105639,"èĤĮèĤī":105640,"äºıæįŁ":105641,"åĬŀåѦ":105642,"ä¸Ģè¡Į":105643,"è¯ŀçĶŁ":105644,"åıijå¸ĥçļĦ":105645,"çļĦæľįåĬ¡":105646,"çļĦçłĶç©¶":105647,"åij¨æľ«":105648,"产ä¸ļåĽŃ":105649,"é«ĺ温":105650,"æĪIJåĬŁçļĦ":105651,"æŃ¥éª¤":105652,"åŃĺåĤ¨":105653,"åŃIJåħ¬åı¸":105654,"让她":105655,"ä¸Ńæľī":105656,"åĺī宾":105657,"妮":105658,"æĺİå¹´":105659,"äºĨåIJĹ":105660,"äºīè®®":105661,"æĪĪ":105662,"ä¸Ģæľ¬":105663,"ç¾İ丽çļĦ":105664,"ä½łè¯´":105665,"大人":105666,"æĶ»çķ¥":105667,"ä¸įæľĥ":105668,"å¾ħéģĩ":105669,"ä¸Ģè¾Ĩ":105670,"çīĪæĿĥæīĢæľī":105671,"æ°ijä¼Ĺ":105672,"åĬŁå¤«":105673,"å±ķä¼ļ":105674,"大èĦij":105675,"æ¯ıæľĪ":105676,"å°ı麦":105677,"æµĻæ±Łçľģ":105678,"çļĦæīĢæľī":105679,"ä¸ĭæ»ij":105680,"èĵĿèī²":105681,"è¦ģæĥ³":105682,"åѦçĶŁçļĦ":105683,"å½ĵä½ł":105684,"ä½ľæĪĺ":105685,"家乡":105686,"å¤ļåIJį":105687,"é«ĺäºİ":105688,"åĿļ强":105689,"è¿ŀéĶģ":105690,"åIJİæŀľ":105691,"人äºĭ":105692,"ç´ħ":105693,"æ¿ĢåĬ¨":105694,"è¿ĽæĶ»":105695,"ç©Ĩ":105696,"ä¸ĺ":105697,"让èĩªå·±":105698,"以æŃ¤":105699,"夫人":105700,"å¼Ģ设":105701,"æ°Ķè´¨":105702,"鸡èĽĭ":105703,"çĦ¡æ³ķ":105704,"åIJĥäºĨ":105705,"åĪĨåĪ«ä¸º":105706,"èģĶåIJĪåĽ½":105707,"å½ĵ代":105708,"å¦Ĥæŀľæĺ¯":105709,"è¿ľç¨ĭ":105710,"åĸĤ":105711,"è®°ä½ı":105712,"æ¸ħåįķ":105713,"åIJĪä½ľä¼Ļä¼´":105714,"åİ»åģļ":105715,"æķħéļľ":105716,"模æĭŁ":105717,"å¸ĪçĶŁ":105718,"åīįæĿ¥":105719,"ç͵è§Ĩåī§":105720,"çĥŃçα":105721,"éľ²åĩº":105722,"é«ĺå±Ĥ":105723,"ç͵åύ":105724,"纪å¾ĭ":105725,"å¼ĢåıijåķĨ":105726,"éķ¿å®ī":105727,"è½½ä½ĵ":105728,"çļĦå°±æĺ¯":105729,"被人":105730,"åıĹçIJĨ":105731,"篮çIJĥ":105732,"èİİ":105733,"交ç»Ļ":105734,"æľªæĿ¥çļĦ":105735,"两大":105736,"åIJķå¸ĥ":105737,"çŃī人":105738,"çļĦæĹ¥åŃIJ":105739,"åIJĪä½ľç¤¾":105740,"æĮijéĢī":105741,"åŃĺæ¬¾":105742,"ç³»ç»ŁçļĦ":105743,"æĬĬå®ĥ":105744,"没æľīä»Ģä¹Ī":105745,"ä»İæŃ¤":105746,"ä¸ŃåįĪ":105747,"çĸ¼çĹĽ":105748,"å·©åĽº":105749,"浪漫":105750,"缸åħ³éĥ¨éŨ":105751,"éķ¿åŁİ":105752,"纤维":105753,"ä¸ĬéŨ":105754,"çĪĨçĤ¸":105755,"èµ·çĤ¹":105756,"çļĦéĢļçŁ¥":105757,"èĢĮæĿ¥":105758,"çļĦèĢģ":105759,"æīĭéĩĮ":105760,"è¯ŃéŁ³":105761,"è¾Ľèĭ¦":105762,"æ±Łèĭıçľģ":105763,"ç͍äºĨ":105764,"身份è¯ģ":105765,"æľīåĬ©":105766,"æľīåĬ©äºİ":105767,"çī©èģĶç½ij":105768,"åĩºéŨ":105769,"å¼ŁåŃIJ":105770,"æĥ¹":105771,"è¿Ļä»¶äºĭ":105772,"æĪij们åı¯ä»¥":105773,"çļĦçĶŁåij½":105774,"æľīä¸Ģç§į":105775,"åºĹéĵº":105776,"åıĮæīĭ":105777,"çļĦæ¶Īæģ¯":105778,"èĢIJå¿ĥ":105779,"å°´å°¬":105780,"éĤ£å¤©":105781,"é¦ĸæī¹":105782,"æĺ¯ä¸Ģå®¶":105783,"人æ°Ķ":105784,"åıįæŃ£":105785,"æĪijåĴĮ":105786,"å®łçī©":105787,"ä¸į对":105788,"寻æ±Ĥ":105789,"çĽ¸ä¼¼":105790,"åľ¨ç¾İåĽ½":105791,"åı«åģļ":105792,"åĹİ":105793,"ç«ĭè¶³":105794,"ç͍éĢĶ":105795,"åħĨ":105796,"大æ°Ķ":105797,"åIJijä¸Ĭ":105798,"ä»ĸå°±":105799,"é¡¹çĽ®å»ºè®¾":105800,"èĭ¥å¹²":105801,"æĺ¯æľī":105802,"æ¿Ģæĥħ":105803,"çļĦæĦıä¹ī":105804,"æĺŃ":105805,"严éĩįçļĦ":105806,"å¯ĨéĽĨ":105807,"èĪŀè¹Ī":105808,"èį£èİ·":105809,"èİ·æĤī":105810,"æ±ŁåįĹ":105811,"åģĩå¦Ĥ":105812,"æĪ·å¤ĸ":105813,"线索":105814,"ç§ģ人":105815,"转åŀĭåįĩ级":105816,"çļĦä»·å̼":105817,"åįķçĭ¬":105818,"èĢģçϾå§ĵ":105819,"å°įæĸ¼":105820,"åĽ½éĻħåĮĸ":105821,"ä¼°å̼":105822,"æľįåĬ¡ä¸ļ":105823,"èĩŃ":105824,"æİīäºĨ":105825,"è§£åĨ³äºĨ":105826,"ä¹Łä¸įèĥ½":105827,"åħ¹":105828,"æĸ¯çī¹":105829,"æķħæĦı":105830,"è¿ĩ度":105831,"èĬĤæĹ¥":105832,"çϽçĻľ":105833,"çϽçĻľé£İ":105834,"ç»§æī¿":105835,"äºĨä¸įå°ij":105836,"äºĮ人":105837,"è§ģéĿ¢":105838,"æĥ³æĥ³":105839,"å¤įåIJĪ":105840,"康å¤į":105841,"åİ¿åŁİ":105842,"åľ¨åĽ½åĨħ":105843,"åľºåľ°":105844,"é϶çĵ·":105845,"è¿Ļ项":105846,"çľ¼ä¸Ń":105847,"糸":105848,"æĦŁè§īåΰ":105849,"æŀľçĦ¶":105850,"æĶ¾åħ¥":105851,"约æĿŁ":105852,"æİĴæŁ¥":105853,"车主":105854,"çļĦæĦıæĢĿ":105855,"æĸ°åŁİ":105856,"æĥ³çĿĢ":105857,"éģĤ":105858,"èĮ¶åı¶":105859,"ä¹°æĪ¿":105860,"åĨľæĪ·":105861,"é«ĺæīĭ":105862,"çİīç±³":105863,"æĸ°åĨłèĤºçĤİ":105864,"çħ§æĺİ":105865,"æĮĩåįĹ":105866,"踢":105867,"æķijæı´":105868,"æĻ¯çĤ¹":105869,"ç¨İæĶ¶":105870,"çļĦæīĭ":105871,"æŃ£å¥½":105872,"è¦ģæĬĬ":105873,"éļıæĦı":105874,"åħ¶å®ŀæĺ¯":105875,"ç»Ļèĩªå·±":105876,"è°ĪåΤ":105877,"æ¯ı天éĥ½":105878,"æĢģåĬ¿":105879,"é¢Ħ约":105880,"åİĨåı²ä¸Ĭ":105881,"å®Ŀè´Ŀ":105882,"åīįè¿Ľ":105883,"ä¹Łå°±æĺ¯è¯´":105884,"çļĦæĦıè§ģ":105885,"åı£ç½©":105886,"åİĺç±³":105887,"èĬ±è´¹":105888,"ä½ĵèĤ²æĬķæ³¨":105889,"åħ¬ä¼Ĺåı·":105890,"èijĹåIJįçļĦ":105891,"å¼ĢæĪ·":105892,"æĭįåįĸ":105893,"å²ģæľĪ":105894,"åĨħæ¶µ":105895,"å®Įæķ´çļĦ":105896,"é«ĺåİĭ":105897,"åħ¬åĬ¡åijĺ":105898,"使ç͍çļĦ":105899,"çĶŁäº§çº¿":105900,"妹妹":105901,"走访":105902,"æĺ¯åı¯ä»¥":105903,"åľ¨å®¶":105904,"æļ´åĬĽ":105905,"æ³°åĽ½":105906,"è´¨çĸij":105907,"ä¸įéģİ":105908,"天çĦ¶æ°Ķ":105909,"缺çĤ¹":105910,"å°ıåŀĭ":105911,"ä¸įä»ħæĺ¯":105912,"é»ijæļĹ":105913,"梨":105914,"æĸĩæĹħ":105915,"è¦ģæľī":105916,"ä¸Ńå±±":105917,"çļĦæķ°æį®":105918,"å¾Ĺå¾Ī":105919,"以便":105920,"对ä»ĸ":105921,"åĬłä»¥":105922,"çϼçı¾":105923,"设å®ļ":105924,"èĤļåŃIJ":105925,"éĿĸ":105926,"å¥īçĮ®":105927,"ä¸įåıĺ":105928,"åı£ç¢ij":105929,"åľ¨åĵªéĩĮ":105930,"ä½IJ":105931,"è¿Ļ两个":105932,"çļĦæĸ¹åIJij":105933,"æŀ«":105934,"äºĮ次":105935,"çīĩåĮº":105936,"éłIJ":105937,"ç£Ĭ":105938,"æĭ¿çĿĢ":105939,"å·²ç»ıæĪIJ为":105940,"ä¹ĭä¸Ĭ":105941,"å®ĹæĹ¨":105942,"奶奶":105943,"é«ĺæĸ°åĮº":105944,"社æľĥ":105945,"è·Łè¸ª":105946,"æľįåĬ¡ä¸Ńå¿ĥ":105947,"æī¯":105948,"æīĭæĮĩ":105949,"礼çī©":105950,"宿èĪį":105951,"ç͍å¿ĥ":105952,"æıIJé«ĺäºĨ":105953,"亮çĤ¹":105954,"ä¸įæĦ¿æĦı":105955,"æĴѿ;":105956,"å¤ļå°ijéĴ±":105957,"没ä»Ģä¹Ī":105958,"æķ°åįģ":105959,"æĢ»çĽij":105960,"çļĦåŁİå¸Ĥ":105961,"æī¾åΰäºĨ":105962,"åĨħåľ°":105963,"åΰçİ°åľ¨":105964,"æĪĺæĸĹåĬĽ":105965,"åİŁå§ĭ":105966,"åĥ§":105967,"åĢĴæĺ¯":105968,"æľĢåħ·":105969,"è´«åĽ°æĪ·":105970,"éĢģåΰ":105971,"级åĪ«":105972,"åĩºèµĦ":105973,"æĪªæŃ¢":105974,"ç§įåŃIJ":105975,"èĥ½ä¸įèĥ½":105976,"幸è¿IJ":105977,"èĸĩ":105978,"项éĵ¾":105979,"æĮĤçīĮ":105980,"ä¸Ģ樣":105981,"ä¹ĺ客":105982,"èIJ½åIJİ":105983,"ä½ĨæĪij":105984,"æĹ©åľ¨":105985,"åĬ¨æ¼«":105986,"å¹³çŃī":105987,"å¯¹ä½ł":105988,"ä¸įæĢķ":105989,"å¤ĸçķĮ":105990,"å¤ļå¹´æĿ¥":105991,"é¦ĸ个":105992,"æ²³åįĹçľģ":105993,"æĪĸåħ¶ä»ĸ":105994,"éķľå¤´":105995,"åįĹæĺĮ":105996,"ä¸ĢéĿ¢":105997,"éĢłæĪIJçļĦ":105998,"å´Ķ":105999,"çŃĴ":106000,"æķĻèĤ²éĥ¨":106001,"åľ°åŁŁ":106002,"æĺĨæĺİ":106003,"å·´é»İ":106004,"æīĭ游":106005,"ä¸ĢæĹ¶":106006,"çłį":106007,"顶级":106008,"åħ±è®¡":106009,"åİŁæ²¹":106010,"è¾īçħĮ":106011,"说æĺ¯":106012,"æĸ°åįİ社":106013,"ç»ıåİĨäºĨ":106014,"ä¸įæŃ¢":106015,"è¦ģä¹Ī":106016,"èĢħçļĦ":106017,"æĢ»æĬķèµĦ":106018,"è¡Įé©¶":106019,"ä¸Ĭå¸Ŀ":106020,"年纪":106021,"çIJ¼":106022,"ä¼łè¯´":106023,"ç²¾èĭ±":106024,"æĸ¹éĴĪ":106025,"æ±Łæ¹ĸ":106026,"æĪIJçĤº":106027,"æĢ»éĩı":106028,"æĬķæĶ¾":106029,"åĬ¨çĶ»":106030,"èŤ":106031,"ç͵æºIJ":106032,"éĴĻ":106033,"åIJĮè¡Į":106034,"æĻ®éĢļçļĦ":106035,"åĽ¾ä¹¦é¦Ĩ":106036,"è¯ĪéªĹ":106037,"æħĪåĸĦ":106038,"è¿Ļ份":106039,"主æĮģ人":106040,"å°±è¿Ļæł·":106041,"èĢĮæĪIJ":106042,"èĩªè¡Į车":106043,"ä¸ŃåĽ½çī¹èī²":106044,"èĤ¿çĺ¤":106045,"åIJ¾":106046,"å¼Łå¼Ł":106047,"åıĹçĽĬ":106048,"éĢīæĭ©äºĨ":106049,"æĺİæĺ¾çļĦ":106050,"æĬ¥èĢĥ":106051,"ç¬ijéģĵ":106052,"éĽĸçĦ¶":106053,"温å·ŀ":106054,"éĿŀæ´²":106055,"ç§įç§į":106056,"åıĤåĬłäºĨ":106057,"è´§è¿IJ":106058,"éļı便":106059,"就没æľī":106060,"縣":106061,"央è§Ĩ":106062,"ç©¿è¶Ĭ":106063,"çļĦçݰ象":106064,"åĩłæ¬¡":106065,"çļĦé£İéĻ©":106066,"æŃĮæĽ²":106067,"æľ¬å±Ĭ":106068,"å¹´åĨħ":106069,"ä¸įè¶ħè¿ĩ":106070,"è¿ĩå¤ļ":106071,"å¿ħé¡»è¦ģ":106072,"ç»ĵ论":106073,"åĢŁéī´":106074,"ç¥ŀå¥ĩ":106075,"æľŁæľĽ":106076,"ä¸ĵ享":106077,"éĿŀ常éĩįè¦ģ":106078,"æĦıè¯Ĩåΰ":106079,"åIJĪå¹¶":106080,"æĬĬèĩªå·±":106081,"å¥Ĺè£ħ":106082,"éŃĶæ³ķ":106083,"å¤ıåŃ£":106084,"ä¸įåĥı":106085,"å¢ĥçķĮ":106086,"æĥĬåĸľ":106087,"æľīä¸Ģ天":106088,"çĦ¦çĤ¹":106089,"æĪij认为":106090,"åħ°å·ŀ":106091,"ç͵æ°Ķ":106092,"èģĶç³»æĪij们":106093,"ç§ijæĻ®":106094,"她说":106095,"çļĦæĸĩ竳":106096,"å¥ĩæĢª":106097,"åıĭ好":106098,"饮æĸĻ":106099,"çļĦæĶ¯æĮģ":106100,"çŃĶåºĶ":106101,"éĩįéĩı":106102,"çij¶":106103,"åĩıè½»":106104,"ç§ijåѦ家":106105,"巴西":106106,"éĩijèŀįæľºæŀĦ":106107,"åħļå§Ķ书记":106108,"貸款":106109,"ç²¾èĩ´":106110,"ä»İæľª":106111,"åį°åĪ·":106112,"åĽŀ顾":106113,"é¦ĸéĥ½":106114,"åıijèĤ²":106115,"éĹ®éģĵ":106116,"è¾¾åΰäºĨ":106117,"å¿įä¸įä½ı":106118,"æīįæľī":106119,"æįIJèµł":106120,"ä½ĽæķĻ":106121,"ä¸įæ¸ħ":106122,"éĺŁéķ¿":106123,"缸åıį":106124,"æĬ¥èѦ":106125,"大åħ¨":106126,"æ¬§çĽŁ":106127,"帮å¿Ļ":106128,"çļĦæĻĤåĢĻ":106129,"缮å½ķ":106130,"足以":106131,"èī°éļ¾":106132,"ä»ĸä¹Ł":106133,"å·¥ä½ľèĢħ":106134,"头èĦij":106135,"缺éĻ·":106136,"æĪIJç«ĭäºĨ":106137,"å°±å¼Ģå§ĭ":106138,"认åIJĮ":106139,"é»Ħèī²":106140,"çĹħæĥħ":106141,"覺å¾Ĺ":106142,"è¿Ļ两":106143,"ä¿¡ä»°":106144,"åľĭå®¶":106145,"ä¸įä»ħä»ħæĺ¯":106146,"çĭ¬å®¶":106147,"èάçļĦ":106148,"æĿIJè´¨":106149,"æµ·ä¸Ĭ":106150,"çĤºäºĨ":106151,"æľºåĬ¨è½¦":106152,"缸å½ĵäºİ":106153,"å¤ļåħĥåĮĸ":106154,"æĽ´å¤§çļĦ":106155,"èĽ®":106156,"åģĩæľŁ":106157,"å¼ıçļĦ":106158,"交éĢļè¿IJè¾ĵ":106159,"çľģå§Ķ":106160,"ä¸įç®Ĺ":106161,"æĶ¾ä¸ĭ":106162,"éĹ¯":106163,"äººåľ¨":106164,"港åı£":106165,"æĹ¨åľ¨":106166,"åij½ä»¤":106167,"æŁIJ个":106168,"平稳":106169,"åıªå¥½":106170,"人人":106171,"äºŀ":106172,"äºĮç»´":106173,"äºĮç»´çłģ":106174,"æŀģ为":106175,"åĪ«å¢ħ":106176,"åħ¶ä½Ļ":106177,"大äºĭ":106178,"主管éĥ¨éŨ":106179,"æĹłéĶ¡":106180,"éŵ":106181,"éģŃåΰ":106182,"说è¿ĩ":106183,"ä¸ºä½ł":106184,"è§£çŃĶ":106185,"éªĮæĶ¶":106186,"çļĦç»ıéªĮ":106187,"åĮ¹éħį":106188,"çģ«ç®Ń":106189,"豪åįİ":106190,"æŁIJæŁIJ":106191,"çļĦæĹ¶ä»£":106192,"书éĿ¢":106193,"æģĴ大":106194,"å»¶éķ¿":106195,"ä¸ĢåIJĮ":106196,"æľªèĥ½":106197,"交æį¢":106198,"çĶ¢åĵģ":106199,"çŃīåΰ":106200,"åĪĨ离":106201,"æīĵç͵è¯Ŀ":106202,"å¹²çĩ¥":106203,"è¾ĥå¤ļ":106204,"å¤ļå¹´çļĦ":106205,"èĥĮæĻ¯ä¸ĭ":106206,"为ä¾ĭ":106207,"æijĺè¦ģ":106208,"å´Ľèµ·":106209,"æŃ¤åĪ»":106210,"æľīæľºä¼ļ":106211,"æĿ¡æ¬¾":106212,"é¢Ĩ导å°ıç»Ħ":106213,"çļĦ身ä½ĵ":106214,"åįķä¸Ģ":106215,"央è¡Į":106216,"ä¸įæĸŃæıIJé«ĺ":106217,"ä»·å̼è§Ĥ":106218,"èĬ½":106219,"èIJį":106220,"æ³ķå¾ĭæ³ķè§Ħ":106221,"ä¸įéĶĪ":106222,"ä¸įéĶĪéĴ¢":106223,"åĩºäºİ":106224,"èĻļæĭŁ":106225,"æį®æĤī":106226,"çĥ¦æģ¼":106227,"åħ¨æĸ°çļĦ":106228,"æī«æıı":106229,"çĻ»éĻĨ":106230,"èīºæľ¯å®¶":106231,"çļĦé£Łçī©":106232,"çļĦåŃĺåľ¨":106233,"客åİħ":106234,"æĪij们就":106235,"æŁ¥çľĭæĽ´å¤ļ":106236,"è¯Ħ审":106237,"å¸Ĥåł´":106238,"è¬Ľ":106239,"巨头":106240,"ä¸ŃåĽ½ç»ıæµİ":106241,"äºĨèĩªå·±çļĦ":106242,"åĨ³è®®":106243,"çĽijçĿ£ç®¡çIJĨ":106244,"æĬķ票":106245,"åĨį度":106246,"è¡ĮçĤº":106247,"注åħ¥":106248,"ä½ľä¸ºä¸Ģ个":106249,"æ¯ı个人éĥ½":106250,"åįķåħĥ":106251,"è¦ģçŁ¥éģĵ":106252,"被称为":106253,"ä¹ĭéĻħ":106254,"è§£éϤ":106255,"丸":106256,"溫":106257,"ä¸īæĺŁ":106258,"é²ľæĺİ":106259,"ä¹Łéĥ½":106260,"æĹ¶æľº":106261,"åĩºæīĭ":106262,"æĥħå½¢":106263,"åķĨè´¸":106264,"éĢī举":106265,"对èĩªå·±":106266,"çĶŁåĬ¨":106267,"åħĭæľį":106268,"个ä½ĵ":106269,"èĭij":106270,"稱":106271,"大åݦ":106272,"æĺ¯å¯¹":106273,"åĪ©æģ¯":106274,"è¿IJåĬ¨åijĺ":106275,"åĮĸè§£":106276,"åīįæ²¿":106277,"æĦŁæģ©":106278,"æĢ»ä¹ĭ":106279,"é«ĺæĸ°æĬĢæľ¯":106280,"åĿĩ为":106281,"åħ¨åĮº":106282,"æ°Ķæ°Ľ":106283,"åı¯ä»¥è¯´æĺ¯":106284,"ä½ı宿":106285,"åħļåijĺå¹²éĥ¨":106286,"åĹ¯":106287,"è·µè¡Į":106288,"çļĦä¸ĵä¸ļ":106289,"èĢĥéªĮ":106290,"èķ¾":106291,"åħ¬åŃIJ":106292,"çļĦçĬ¶æĢģ":106293,"æ½®æµģ":106294,"ä¿¡æīĺ":106295,"è´¼":106296,"åIJĦæĸ¹":106297,"æķijåĬ©":106298,"éĿŀ常çļĦ":106299,"æ¡¥æ¢ģ":106300,"åħ¬æĸ¤":106301,"ä¼¼çļĦ":106302,"çľĭ好":106303,"å±Ģéĥ¨":106304,"å®īéĿĻ":106305,"éħįä»¶":106306,"常è§Ħ":106307,"å¼Ģ车":106308,"第äºĮ次":106309,"ä¸Ĭ级":106310,"åıĤèµĽ":106311,"å®¶å±ŀ":106312,"强åĬ¿":106313,"åľ¨ä»ĸ":106314,"åIJijåīį":106315,"ä¹ĭåľ°":106316,"éĥ¡":106317,"è¡Įç¨ĭ":106318,"èѦåijĬ":106319,"è§Ħå®ļçļĦ":106320,"åķĨåŁİ":106321,"äºĶ大":106322,"æķĻ室":106323,"åįģè¶³":106324,"æīĢä»¥åľ¨":106325,"å°Ĩç»§ç»Ń":106326,"çŃīæĸ¹å¼ı":106327,"å®¶ä¼ģä¸ļ":106328,"交ä»ĺ":106329,"çĤ¹è¯Ħ":106330,"ç»ĵç®Ĺ":106331,"ä¹Łåı¯":106332,"å¤ĸæ±ĩ":106333,"è¿Ļç§įæĥħåĨµ":106334,"æİĪäºĪ":106335,"å¸ĥç½®":106336,"æĪIJç«ĭäºİ":106337,"é¢ĦèѦ":106338,"管çIJĨ人åijĺ":106339,"å©ļ礼":106340,"ç»ĵæĿŁåIJİ":106341,"åħ¥éĢī":106342,"æĹłæ¯Ķ":106343,"åĴĮåıijå±ķ":106344,"çϽéħĴ":106345,"çİ©åħ·":106346,"ä¸ĩç¾İåħĥ":106347,"çļĦæĪIJ绩":106348,"æĭįçħ§":106349,"èĢĥèĻijåΰ":106350,"ä¼ģä¸ļåıijå±ķ":106351,"äºĨ个":106352,"çĶŁæ°Ķ":106353,"çļĦ女人":106354,"äºĶåįģ":106355,"çĪ·çĪ·":106356,"纽约":106357,"éĥ½è¢«":106358,"ä¸Ĭ课":106359,"çĽ¡":106360,"ä¼łç»ŁæĸĩåĮĸ":106361,"æ½ľåľ¨":106362,"åıijå°Ħ":106363,"ä¸Ģ身":106364,"éĺ²å®Ī":106365,"åĪ®":106366,"é¢ĺ缮":106367,"åľ¨åĨħçļĦ":106368,"ç¾İ好çļĦ":106369,"è¿ĻéĩĮçļĦ":106370,"ä¸Ģä¸Ŀ":106371,"人åĿĩ":106372,"å̡坼":106373,"身åIJİ":106374,"æī©å±ķ":106375,"大éŨ":106376,"就被":106377,"è¯¥é¡¹çĽ®":106378,"æŀ¶æŀĦ":106379,"ä¸Ģåı£":106380,"ä¿¡æģ¯æĬĢæľ¯":106381,"å¼Ģä¸ļ":106382,"æĶ¶åıĸ":106383,"ç½ij页":106384,"æĶ¯æı´":106385,"å°ģéĹŃ":106386,"å¡ijéĢł":106387,"大èĥĨ":106388,"å¿«éĢŁåıijå±ķ":106389,"çľĭä¼¼":106390,"æ¸Ŀ":106391,"è¿Ļæł·ä¸Ģ个":106392,"模åĿĹ":106393,"注æĦıåΰ":106394,"çł´è§£":106395,"èĩªä»İ":106396,"åijµåijµ":106397,"ä¹ĭå¾Į":106398,"ä¹ĭæĹħ":106399,"è·ŁæĪij":106400,"æ³ķ人":106401,"æİĴè¡Įæ¦ľ":106402,"åĿļå®Ī":106403,"好å¤Ħ":106404,"çŁ³å¤´":106405,"å¹¶å°Ĩ":106406,"èα":106407,"æŃĩ":106408,"两岸":106409,"å¤ļä¹ħ":106410,"象å¾ģ":106411,"个æĢ§åĮĸ":106412,"çļĦè§Ĵ度":106413,"å¸Ĩ":106414,"ç¦ıå·ŀ":106415,"æŁ¥å¤Ħ":106416,"ä¸¤åĽ½":106417,"åIJ¸å¼ķäºĨ":106418,"é¦ĸå¸Ń":106419,"大åĵ¥":106420,"é¤Ĭ":106421,"涨å¹ħ":106422,"éĢīç͍":106423,"許å¤ļ":106424,"èIJ½æĪ·":106425,"åĵĪå°Ķ":106426,"åĵĪå°Ķ滨":106427,"åģļä»Ģä¹Ī":106428,"以åħį":106429,"é¾į":106430,"æĹłéľĢ":106431,"åΰåºķæĺ¯":106432,"æĢ¡":106433,"åijĬè¯īä½ł":106434,"éĺ²æ°´":106435,"è¿ĻæĹ¶åĢĻ":106436,"欢ä¹IJ":106437,"转åIJij":106438,"è¿Ļä¸ªåľ°åĽ¾":106439,"åħ¥é©»":106440,"èįīåİŁ":106441,"æĹ¶ä»£çļĦ":106442,"åıĺåĬ¨":106443,"åĬłå¼ºå¯¹":106444,"åģ¶å°Ķ":106445,"å®ĪæĬ¤":106446,"æ°Ķ温":106447,"人éĹ´":106448,"æľĿé²ľ":106449,"ç»ıè´¹":106450,"åĽŃæŀĹ":106451,"å·¥åľ°":106452,"è§Ħæł¼":106453,"åĩłåįģ":106454,"è¯ķåĽ¾":106455,"å¦ĥ":106456,"éĤ£æĹ¶åĢĻ":106457,"å¼ĺæī¬":106458,"ä¸ļçķĮ":106459,"çļĦéĢŁåº¦":106460,"ä¼ļä¸įä¼ļ":106461,"èIJ¥æĶ¶":106462,"å°ıå¾®ä¼ģä¸ļ":106463,"çľĭè¿ĩ":106464,"æĬĬä»ĸ":106465,"éģµå¾ª":106466,"è¿Ļè¾¹":106467,"没æľī人":106468,"壶":106469,"æ¹ĸåįĹçľģ":106470,"æŀģåħ¶":106471,"çļĦ人çĶŁ":106472,"ä»ĸè¿ĺ":106473,"转åĮĸ为":106474,"èµ°è¿ĩ":106475,"æĬ±çĿĢ":106476,"çīĽå¥¶":106477,"ä¸ĩ亩":106478,"å¿ĥæĢģ":106479,"æĹ¥å¸¸çĶŁæ´»":106480,"ä½ĵæ£Ģ":106481,"æĻĥ":106482,"çŃīé¢ĨåŁŁ":106483,"æĩī該":106484,"åı¯ä»¥çľĭåΰ":106485,"æī¾ä¸įåΰ":106486,"èĢģå¹´":106487,"æĬĬæĪij":106488,"积åĪĨ":106489,"梳çIJĨ":106490,"绳":106491,"çļĦæĶ¿æ²»":106492,"å¸ĿåĽ½":106493,"éĻªä¼´":106494,"æ´Ľéĺ³":106495,"åħ¬æŃ£":106496,"å¼Ģåı£":106497,"çī¹èī²çļĦ":106498,"åĽ°å¢ĥ":106499,"ä¸Ĭæľī":106500,"ç«ĭä½ĵ":106501,"æīĵå·¥":106502,"åķ¤éħĴ":106503,"åľ¨éĤ£éĩĮ":106504,"éĤ£è¾¹":106505,"个åĪ«":106506,"ä¸Ģå®ļæĺ¯":106507,"çļĦéĩįè¦ģæĢ§":106508,"ä¸»å¼ł":106509,"åĴĮæľįåĬ¡":106510,"ä¸Ĭç½ij":106511,"è¡¥åĬ©":106512,"åıªéľĢ":106513,"弦":106514,"éģ®":106515,"åĬĽäºī":106516,"度è¿ĩ":106517,"èij¬":106518,"é¡¿æĹ¶":106519,"éĦī":106520,"纺ç»ĩ":106521,"åľ°åĿĹ":106522,"ä¿¡ç͍åį¡":106523,"ç½ļ款":106524,"åijĬè¯īæĪij":106525,"éĽĻ":106526,"书çĶ»":106527,"è¨Ńè¨Ī":106528,"æĢ»ä¼ļ":106529,"åΤåĨ³":106530,"ä¿¡èªī":106531,"个èĤ¡":106532,"平常":106533,"æĢİ麼":106534,"ä½ĵçİ°åľ¨":106535,"é»Ħæ²³":106536,"åĽĽå·Ŀçľģ":106537,"羣缸":106538,"åIJĦé¡¹å·¥ä½ľ":106539,"åĬ¨åijĺ":106540,"å³°ä¼ļ":106541,"ä¸ĢæľŁ":106542,"æľīä¸Ģå®ļçļĦ":106543,"é«ĺ度éĩįè§Ĩ":106544,"ç¹ģèį£":106545,"åıijçݰäºĨ":106546,"ç½ij红":106547,"æīĭæ³ķ":106548,"å®¶åĽŃ":106549,"仪åύ":106550,"è¾ĥä½İ":106551,"çļĦå®īåħ¨":106552,"æ¡IJ":106553,"ä»ĺ款":106554,"æĬijåζ":106555,"åįĵè¶Ĭ":106556,"æŃ£éĿ¢":106557,"åĵij":106558,"强åζ":106559,"ä»Ĭ天çļĦ":106560,"æĪĺèĥľ":106561,"楼å¸Ĥ":106562,"æĭ¿ä¸ĭ":106563,"é¢ľå̼":106564,"举éĥ¨":106565,"çłĶåζ":106566,"çļĦæĪĺçķ¥":106567,"åľ¨ä¸Ģ个":106568,"ä¸ī人":106569,"å®ĮäºĨ":106570,"æĸ°æĬĢæľ¯":106571,"ç»ıæµİæķĪçĽĬ":106572,"å¯Įæľī":106573,"澳洲":106574,"åĬ©çIJĨ":106575,"é¢Ĩåıĸ":106576,"è°Ń":106577,"çĩĥçĥ§":106578,"ç´łåħ»":106579,"éĤĦæľī":106580,"è¿ĽèĢĮ":106581,"ä»Ģä¹Īæĺ¯":106582,"çłĶç©¶ä¸Ńå¿ĥ":106583,"éĢĤç͍äºİ":106584,"æİ¥æĶ¶":106585,"å¤±æľĽ":106586,"äºĮ级":106587,"éĹ´çļĦ":106588,"åİŁæłĩé¢ĺ":106589,"èªįçĤº":106590,"æį¡":106591,"对çĿĢ":106592,"对éĿ¢":106593,"ä¸ŃåİŁ":106594,"éĵĥ":106595,"çĶŁäº§çļĦ":106596,"åıijå¸ĥä¼ļ":106597,"士åħµ":106598,"è¿Ļåı¥è¯Ŀ":106599,"缴纳":106600,"ä¸Ģ个个":106601,"åѸçĶŁ":106602,"çĸijéĹ®":106603,"交èѦ":106604,"示èĮĥåĮº":106605,"天使":106606,"åľ¨ä¸Ĭæµ·":106607,"åIJĮæĻĤ":106608,"è½»æĺĵ":106609,"å͝ä¸ĢçļĦ":106610,"çĥŃéĹ¹":106611,"ä¹IJè§Ĥ":106612,"çļĦ身份":106613,"åĸĦäºİ":106614,"大åİħ":106615,"èĤ¯å®ļæĺ¯":106616,"éĺ²çģ«":106617,"å¤ĸåĩº":106618,"æį®è¯´":106619,"é¡¹çĽ®çļĦ":106620,"ä¸Ģåı°":106621,"èĻļåģĩ":106622,"ä¸Ģç¬Ķ":106623,"ç«ĭæ³ķ":106624,"严èĤĥ":106625,"æī¿åĬŀ":106626,"åįģåĩł":106627,"çļĦ空éĹ´":106628,"æľ¬ç½ijç«Ļ":106629,"åģļå¾Ĺ":106630,"ä¿Ŀ温":106631,"æľĪåĪĿ":106632,"åľ¨ç½ijä¸Ĭ":106633,"åIJĦæĸ¹éĿ¢":106634,"ä¸ī天":106635,"交æĺĵæīĢ":106636,"è§£æŀIJ":106637,"åħļä¸Ń央":106638,"è¿Ľåĩºåı£":106639,"åĴĮ社ä¼ļ":106640,"次æķ°":106641,"ä¹ĭå®¶":106642,"维度":106643,"æ´¾åĩºæīĢ":106644,"产çĶŁäºĨ":106645,"带æľī":106646,"å¾Ī强":106647,"æľīäºĽäºº":106648,"å¹´åIJİ":106649,"äºĨ许å¤ļ":106650,"å¯Ĩ度":106651,"åŃ¦æľŁ":106652,"çıłæµ·":106653,"æľĢå¤ļçļĦ":106654,"è¾¹ç¼ĺ":106655,"容éĩı":106656,"第äºĮ个":106657,"ä¸Ģ缴æĺ¯":106658,"ä¸įç¦ģ":106659,"æŃ²":106660,"ä»ĭç»įäºĨ":106661,"ä¼ĺéĽħ":106662,"æ¯Ķè¼ĥ":106663,"èģĮä½į":106664,"温æŁĶ":106665,"æľīéĴ±":106666,"æľĢé«ĺçļĦ":106667,"åįļè§Īä¼ļ":106668,"ä¸įæĪIJ":106669,"éĶĻäºĨ":106670,"è¯ģçĽij":106671,"è¯ģçĽijä¼ļ":106672,"æĪIJ人":106673,"åĿĩåĮĢ":106674,"æľīåĪ©":106675,"è¶ĬåįĹ":106676,"æīĵäºĨ":106677,"好åIJĥ":106678,"系統":106679,"è·Łéļı":106680,"çļĦåľ°ä½į":106681,"æŃ£å¦Ĥ":106682,"ç¨įå¾®":106683,"åį°åıij":106684,"åĪĽç«ĭ":106685,"é£İåħī":106686,"å°ĨæĪIJ为":106687,"ä¸įé«ĺ":106688,"é¢ijç¹ģ":106689,"设æľī":106690,"ä¼ŀ":106691,"æĭĨéϤ":106692,"å½±åĥı":106693,"æ¸ĹéĢı":106694,"å¹´å¼Ģå§ĭ":106695,"ç½ijæĺĵ":106696,"è¦ģåģļ":106697,"ç͵åĬ¨è½¦":106698,"羣å¿ĥ":106699,"æµ·åĨĽ":106700,"ä¼łæĿ¥":106701,"å·®åĪ«":106702,"è°¨æħİ":106703,"çĥŁåı°":106704,"åįĥå¹´":106705,"è¯ģå®ŀ":106706,"çIJª":106707,"çļĦåħ·ä½ĵ":106708,"åΰå¤Ħ":106709,"ä¸įå®ľ":106710,"èľĢ":106711,"èĥ½åĬĽåĴĮ":106712,"çīºçī²":106713,"çļĦéĴ±":106714,"大éĺŁ":106715,"é¦ĸè¦ģ":106716,"ä¸įæĦ¿":106717,"çİ«çij°":106718,"人æ°ijç½ij":106719,"è¿ĺæĺ¯è¦ģ":106720,"åĽĽå¹´":106721,"æįŁä¼¤":106722,"çļĦåģļæ³ķ":106723,"éĿĪ":106724,"è¡Ķæİ¥":106725,"åIJĪæĪIJ":106726,"没人":106727,"éĹ¨æ§Ľ":106728,"ä¿¡è´·":106729,"çļĦ缸åħ³":106730,"举é£İ":106731,"社ä¿Ŀ":106732,"ä¸ĭ游":106733,"åĿĹéĴ±":106734,"è¿ĩåIJİ":106735,"çļĦåºĶç͍":106736,"饶":106737,"é¢ģåıij":106738,"ä¸Ģå¤Ħ":106739,"åįİå¤ı":106740,"为ä¼ģä¸ļ":106741,"åıªä¼ļ":106742,"侵害":106743,"çļĦåĬŁèĥ½":106744,"åѸç¿Ĵ":106745,"ä¸Ńåįİæ°ijæĹı":106746,"åıijå¸ĥäºĨ":106747,"è¿İæİ¥":106748,"æĪijèĩªå·±":106749,"è¿ĺéľĢè¦ģ":106750,"太éĺ³èĥ½":106751,"åİ»ä¸ĸ":106752,"æĺ¯ä½ł":106753,"åIJĪåĬĽ":106754,"ç»ĺçĶ»":106755,"åı°åĮĹ":106756,"çĿ£ä¿ĥ":106757,"åĮĹéĥ¨":106758,"æľīå¤ļå°ij":106759,"å¾Īéĩįè¦ģ":106760,"åĪĴåĪĨ":106761,"åı·çº¿":106762,"æĶ¾å¤§":106763,"ä¼ļ被":106764,"èİ·å¥ĸ":106765,"ä¹ĭåĨħ":106766,"失åİ»äºĨ":106767,"çݩ家们":106768,"éĩĩéĽĨ":106769,"壹":106770,"å®¶ä¼Ļ":106771,"çϽ天":106772,"åĽłä¸ºä»ĸ":106773,"社ä¼ļæ²»çIJĨ":106774,"å¼ĢåĪĽ":106775,"ç͵ç¼Ĩ":106776,"æĸ°ä¸Ģ代":106777,"å¹¶è´Ń":106778,"就已ç»ı":106779,"çļĦ社ä¼ļ":106780,"éϤéĿŀ":106781,"åı¯ä»¥ç͍":106782,"å©ī":106783,"æ¯Ķè¾ĥ好":106784,"å®ŀä¸ļ":106785,"åĪĽåĬŀ":106786,"æıIJèµ·":106787,"é»ĥ":106788,"ä½ıåľ¨":106789,"å¸ĤæĶ¿":106790,"éĿ¢ä¸´çļĦ":106791,"èĥ½åľ¨":106792,"çŁŃçŁŃ":106793,"çľŁäºº":106794,"æĺİæĺİ":106795,"èµĦåĬ©":106796,"çļĦä¸įåIJĮ":106797,"å°ıæľĭåıĭ":106798,"é¢ĺæĿIJ":106799,"ç¾İåij³":106800,"æĺŁåº§":106801,"ä¸įä¸Ģæł·çļĦ":106802,"çľĭä¸Ĭåİ»":106803,"ä¸Ģæł¹":106804,"广å·ŀå¸Ĥ":106805,"åıijçĶŁçļĦ":106806,"é«ĺç§ijæĬĢ":106807,"ä¸Ģè¾ĪåŃIJ":106808,"交åıī":106809,"ä½ĵ系建设":106810,"åĽłä¸ºæĪij":106811,"çıįæĥľ":106812,"ä¸ĬåѦ":106813,"æĪĺæľ¯":106814,"æŃ¤ç±»":106815,"交å¾Ģ":106816,"æĮīæij©":106817,"人们çļĦ":106818,"åħ¶å¯¦":106819,"åİŁæĿIJæĸĻ":106820,"æ¸´æľĽ":106821,"缸å¤Ħ":106822,"微微":106823,"æ®·":106824,"ä¹ĺåĿIJ":106825,"å¼Ģå±ķäºĨ":106826,"é«ĺåĵģè´¨":106827,"æĹłäººæľº":106828,"ä¸įæĺ¯å¾Ī":106829,"çļĦæĬķèµĦ":106830,"èĬĤçľģ":106831,"èĩī":106832,"ç²¾éĢī":106833,"çļĦæłĩåĩĨ":106834,"åįĹéĥ¨":106835,"认è¯Ĩåΰ":106836,"å¹³éĿĻ":106837,"èĹ¥":106838,"æī«é»ij":106839,"æī«é»ijéϤ":106840,"æī«é»ijéϤæģ¶":106841,"éĢĻ種":106842,"建çŃijéĿ¢ç§¯":106843,"ç¡®ç«ĭ":106844,"管çIJĨåĬŀæ³ķ":106845,"æĦıå¿Ĺ":106846,"丨":106847,"让åŃ©åŃIJ":106848,"æķijçģ¾":106849,"å½ĵä»Ĭ":106850,"çģ«çģ¾":106851,"åIJĦéĥ¨éŨ":106852,"ä¾µçĬ¯":106853,"æ¯ıåij¨":106854,"æı½":106855,"ä¸Ģ次æĢ§":106856,"åħ¶ä»ĸ人":106857,"éĶĻè¿ĩ":106858,"ä¸İåħ¶":106859,"åĭĩæ°Ķ":106860,"çĩĥæ°Ķ":106861,"é¦ĸå±Ĭ":106862,"æľį饰":106863,"ç²¥":106864,"å®Įæ¯ķ":106865,"å°±æĬĬ":106866,"åĬŀäºĭå¤Ħ":106867,"ä¸Ģä¼ļåĦ¿":106868,"离ä¸įå¼Ģ":106869,"å¦ĤæŀľæĤ¨":106870,"ä»ĵåºĵ":106871,"导å¸Ī":106872,"åIJĪéĢĤçļĦ":106873,"毫米":106874,"å®īåħ¨æĢ§":106875,"ä¾Ŀçħ§":106876,"产ä¸ļåĮĸ":106877,"ä½łçľĭ":106878,"羣çļĦå¾Ī":106879,"åѤçĭ¬":106880,"éĺ²å¾¡":106881,"å¾Īç®Ģåįķ":106882,"é£İæ°´":106883,"ä½Ĩä¹Ł":106884,"æİ¨åĩºäºĨ":106885,"æ°ijèIJ¥ä¼ģä¸ļ":106886,"çłģ头":106887,"å¤įæĿĤçļĦ":106888,"ç»ĦæĪIJéĥ¨åĪĨ":106889,"åħħ满äºĨ":106890,"è¿ijåĩłå¹´":106891,"çľģæĶ¿åºľ":106892,"æľīå¿ħè¦ģ":106893,"éϳ":106894,"ä¹ĭç±»":106895,"ä¹ĭç±»çļĦ":106896,"æĢ§ä»·":106897,"æĢ§ä»·æ¯Ķ":106898,"åķĨåºĹ":106899,"å¸Ĥå̼":106900,"人æīįåŁ¹åħ»":106901,"æ·±åıĹ":106902,"管çIJĨå±Ģ":106903,"æģIJæĥ§":106904,"ä»ħæľī":106905,"æĬµè¾¾":106906,"æµ·åħ³":106907,"èµĭäºĪ":106908,"äºĭåĦ¿":106909,"ä»·éĴ±":106910,"æīĭä¸Ĭ":106911,"èĩªå¾ĭ":106912,"åħ³çα":106913,"享æľī":106914,"éģĹæĨ¾":106915,"å¾Īå¿«å°±":106916,"æĽ´å¿«":106917,"æłĩè¯Ĩ":106918,"åºĨç¥Ŀ":106919,"ä¹Łå¥½":106920,"ä¸įæĺĵ":106921,"æĪijå¾Ī":106922,"æĶ¹éĿ©åıijå±ķ":106923,"å¤ĸåľ°":106924,"æĬµæĬ¼":106925,"è¯Ĺ人":106926,"åİķæīĢ":106927,"æĸ°åªĴä½ĵ":106928,"èĸĽ":106929,"è°Īè¯Ŀ":106930,"ä¸Ģå®ļç¨ĭ度":106931,"èµ°åľ¨":106932,"æľĢ强":106933,"åĬŁçİĩ":106934,"åħ±è¯Ĩ":106935,"大桥":106936,"ä¸ĭæĸ¹":106937,"å¤ĸèµĦ":106938,"碱":106939,"å·¡è§Ĩ":106940,"æ¹ĸåĮĹçľģ":106941,"个çϾåĪĨ":106942,"个çϾåĪĨçĤ¹":106943,"çļĦ责任":106944,"çļĦåĵģçīĮ":106945,"åĬ©æİ¨":106946,"åĪĽéĢłäºĨ":106947,"ä»»èģĮ":106948,"å¿«æį·":106949,"æĿijåºĦ":106950,"åİ»çľĭ":106951,"æīįèĥ½å¤Ł":106952,"層":106953,"æĪijå®¶":106954,"æĺ¯ä¸Ģ款":106955,"ç¾ħ":106956,"åĨ°éĽª":106957,"æŀģ大":106958,"çģ¯åħī":106959,"éĨĭ":106960,"ä¸İåħ¶ä»ĸ":106961,"æıIJåĩºçļĦ":106962,"éĿłè¿ij":106963,"è°ĥåĬ¨":106964,"å°½åı¯èĥ½":106965,"åıijåĬĽ":106966,"ç»Ļ她":106967,"éĢĤéĩı":106968,"è·¨åĽ½":106969,"åħĪè¡Į":106970,"æĸ°æĿIJæĸĻ":106971,"ä½ľäºĨ":106972,"满äºĨ":106973,"ä¸į满":106974,"çļĦçľ¼çĿĽ":106975,"çľĭå¾Ĺ":106976,"è¿Ļä¸Ģ次":106977,"é½IJåħ¨":106978,"çļĦä¸Ģéĥ¨åĪĨ":106979,"ä¸Ļ":106980,"æ¸ħæĸ°":106981,"說æĺİ":106982,"身边çļĦ":106983,"æīĢæľī人":106984,"å½°æĺ¾":106985,"è±¹":106986,"åį¿":106987,"è¿IJ转":106988,"æĮĩå¼ķ":106989,"å¸Ĥåħ¬å®īå±Ģ":106990,"åıĤå±ķ":106991,"ä¹ĭæĹ¶":106992,"éĩijèŀįæľįåĬ¡":106993,"èµĦæľ¬å¸Ĥåľº":106994,"èĥ½è®©":106995,"å¿ĺäºĨ":106996,"天åłĤ":106997,"æ¯Ķå¦Ĥ说":106998,"éĬĢè¡Į":106999,"èĽĭç³ķ":107000,"çĶ©":107001,"æł¸å®ŀ":107002,"æĻ®äº¬":107003,"ä¼ĺç¾İ":107004,"åı£èħĶ":107005,"漫çĶ»":107006,"çľ¼éĩĮ":107007,"äºĨä¸ĭæĿ¥":107008,"æĪijä»¬ä¹Ł":107009,"ä¾į":107010,"为ä¸Ńå¿ĥ":107011,"å¥ĩ迹":107012,"éĿĴçĿIJ":107013,"æĪªèĩ³çĽ®åīį":107014,"åĩºä¾Ĩ":107015,"æĢ»åħ¬åı¸":107016,"弥补":107017,"ç®Ĺæ³ķ":107018,"å·¥ä½ľå®¤":107019,"æīĢ以æĪij":107020,"æ°´åĪĨ":107021,"æīĢå±ŀ":107022,"ä¸į说":107023,"ä½Ĩæĺ¯åľ¨":107024,"è¦ģåİ»":107025,"åĪĽä¸ļèĢħ":107026,"ä¸įæ¸ħæ¥ļ":107027,"åĽĽåij¨":107028,"æĺ¯ä»İ":107029,"çļĦæł¹æľ¬":107030,"çģ¶":107031,"æ¯Ľæ³½":107032,"æ¯Ľæ³½ä¸ľ":107033,"æµ·åı£":107034,"åĽĽåįģ":107035,"ä¹Łè¢«":107036,"èģ·":107037,"ä¸Ģæīĭ":107038,"绩æķĪ":107039,"çļĦçĶ·äºº":107040,"书ç±į":107041,"ä¸ĢèĦ¸":107042,"大äºİ":107043,"鼶éĥ¨ä»¶":107044,"åħ³æĢĢ":107045,"平米":107046,"æļ´éľ²":107047,"å¾Ĺå¤ļ":107048,"ä¸ī级":107049,"æľ¬åij¨":107050,"两èĢħ":107051,"对ä¸ŃåĽ½":107052,"åıªè§ģ":107053,"欧ç¾İ":107054,"å¦Ĥæŀľæľī":107055,"å·²ç»ıæĺ¯":107056,"çľĭå®Į":107057,"çģ«éĶħ":107058,"èµIJ":107059,"ä¸Ģéģį":107060,"æĦŁåĨĴ":107061,"ç»ĵå±Ģ":107062,"ä»ĵåĤ¨":107063,"å®ŀåľ°":107064,"å̻ç»ıçIJĨ":107065,"ä¹Łä¸įçŁ¥éģĵ":107066,"碰åΰ":107067,"åIJĪ计":107068,"客æĪ·çļĦ":107069,"ç½Ĺ马":107070,"æĦīå¿«":107071,"é£Ľ":107072,"çĥŃçĥĪ":107073,"伦æķ¦":107074,"åĮ»ä¿Ŀ":107075,"éĺ¿éĩĮå·´å·´":107076,"åĨį说":107077,"ä¸ºåŁºç¡Ģ":107078,"çĶŁäº§ç»ıèIJ¥":107079,"è¿ĻäºĽäºº":107080,"åĪĹ车":107081,"æ²³åĮĹçľģ":107082,"è¿Ļ段":107083,"æ´»åĬ¨ä¸Ń":107084,"å©·":107085,"çĶŁçIJĨ":107086,"ä¸ŃåĽ½äººæ°ij":107087,"éĦĤ":107088,"åIJ¬åıĸ":107089,"å¤įä¹ł":107090,"æľīçĽĬ":107091,"æĶ¶æĭ¾":107092,"å¾Īåı¯èĥ½":107093,"ç½ijç»ľæ¸¸æĪı":107094,"们çļĦ":107095,"èµĭèĥ½":107096,"éļ¾å¾Ĺ":107097,"åĪĨæīĭ":107098,"羣è¯ļ":107099,"åħ¬åı¸åľ¨":107100,"åĿĩè¡¡":107101,"åı£åij³":107102,"çīµå¤´":107103,"ä¸ĢèάçļĦ":107104,"轿车":107105,"çŃīäºİ":107106,"æ²īé»ĺ":107107,"æĪijéĥ½":107108,"å°ıç¨ĭåºı":107109,"ä¸Ģåī¯":107110,"æī¿è½½":107111,"åľ°è´¨":107112,"çķĮéĿ¢":107113,"çĶµæľº":107114,"çĦ¦èĻij":107115,"éĶĢåĶ®é¢Ŀ":107116,"æĸ°è½¦":107117,"ä¸Ĭ游":107118,"主æ¼Ķ":107119,"éļIJç§ģ":107120,"åıijå±ķæĪĺçķ¥":107121,"çļĦåĬªåĬĽ":107122,"å¼Ģåħ³":107123,"è§£åĨ³éĹ®é¢ĺ":107124,"çĿ£å¯¼":107125,"对æĬĹ":107126,"å¾Īå¤ļ人éĥ½":107127,"æĹłæķĪ":107128,"产åĵģè´¨éĩı":107129,"å®īå¿ĥ":107130,"åįİ人":107131,"ä¸į符åIJĪ":107132,"èĩªå®¶":107133,"éĺµå®¹":107134,"çļĦåIJĦç§į":107135,"çļĦçIJĨ念":107136,"çļĦæĸĩåĮĸ":107137,"为èĩªå·±":107138,"山水":107139,"游泳":107140,"éľĩèį¡":107141,"çĶŁæ´»æĸ¹å¼ı":107142,"è¿ľç¦»":107143,"çŁ³åĮĸ":107144,"æŃ¤äºĭ":107145,"æĺ¯çľŁçļĦ":107146,"çļĦæ¯Ķä¾ĭ":107147,"ç͍ç͵":107148,"奥è¿IJä¼ļ":107149,"ä¿Ŀå®ī":107150,"èĽĭçĻ½è´¨":107151,"çļĦå¿ĥçIJĨ":107152,"å·«":107153,"åı·çłģ":107154,"æ°Ķä½ĵ":107155,"åıijæĶ¹":107156,"åıijæĶ¹å§Ķ":107157,"åĮ»å¸Ī":107158,"æ¶ĤæĸĻ":107159,"æĺĬ":107160,"å¸Ĥ级":107161,"ä¸ĸçķĮçļĦ":107162,"åĪĨåĪ«æĺ¯":107163,"çł´äº§":107164,"ä¸ĢæĿ¯":107165,"æĭīå¼Ģ":107166,"å¹³åĩ¡":107167,"çļĦåıijçĶŁ":107168,"åĬ¨æīĭ":107169,"ä¸ĢçĽ´ä»¥æĿ¥":107170,"æīĭå·¥":107171,"éĩĮéĿ¢çļĦ":107172,"æĹłåħ³":107173,"ä»ĭåħ¥":107174,"èµ°ä¸Ĭ":107175,"å°±æĺ¯è¦ģ":107176,"å¹´éĹ´":107177,"åĩºçı¾":107178,"å½±éŁ¿":107179,"å¹ħ度":107180,"éĽģ":107181,"éģĵåħ·":107182,"缮çļĦåľ°":107183,"åIJİèĢħ":107184,"ä¸Ĭæ¼Ķ":107185,"äºĨåĩł":107186,"æ®ĭçĸ¾äºº":107187,"å¿Ļç¢Į":107188,"æĺ¯åIJ¦æľī":107189,"并对":107190,"ä¼ļ导èĩ´":107191,"æ°´åºĵ":107192,"ç»Ĩèĩ´":107193,"åIJİæĤĶ":107194,"å¿ĥæĢĿ":107195,"åģļäºĭ":107196,"åİĤæĪ¿":107197,"çĿ¿":107198,"è¿IJèIJ¥åķĨ":107199,"头éĥ¨":107200,"çļĦè§Ĵèī²":107201,"æĺ¯ä»ĸ":107202,"æĹ¢æľī":107203,"å°ıæĹ¶åĢĻ":107204,"强åĬ²":107205,"主æĴŃ":107206,"åħ¨åĽ½åIJĦåľ°":107207,"æįı":107208,"æįŁåĿı":107209,"åķĨä¼ļ":107210,"ä¿Ŀç½Ĺ":107211,"çľģå¸Ĥ":107212,"éļ§éģĵ":107213,"æľīä¸įå°ij":107214,"è¦ģåľ¨":107215,"å»ºè®¾é¡¹çĽ®":107216,"ç³ĸå°¿":107217,"ç³ĸå°¿çĹħ":107218,"æĿ¡ä»¶ä¸ĭ":107219,"ä¼ĺè´¨çļĦ":107220,"é¦ĸåıij":107221,"å½ĵæĹ¶çļĦ":107222,"丰çͰ":107223,"大çĽĺ":107224,"缸继":107225,"å®ģå¤ı":107226,"åħ¥ä½ı":107227,"æĪijè¿ĺ":107228,"åħĭæĸ¯":107229,"å®ļä»·":107230,"å¹³æĸ¹åħ¬éĩĮ":107231,"çļĦçŁ¥è¯Ĩ":107232,"æĪij们ä¼ļ":107233,"åħĥå®Ŀ":107234,"ä½ĵéĩį":107235,"è³£":107236,"对æĪij们":107237,"çŁ³å®¶":107238,"çŁ³å®¶åºĦ":107239,"ç²¾åįİ":107240,"å½¢çĬ¶":107241,"åıĹåΰäºĨ":107242,"修订":107243,"ç¾İåľĭ":107244,"é«ĺæ¸ħ":107245,"çľ¼éķľ":107246,"è§īå¾Ĺèĩªå·±":107247,"带ç»Ļ":107248,"åͮ价":107249,"éĹ¨ç¥¨":107250,"åŃķå¦ĩ":107251,"ç͵è§Ĩåı°":107252,"åıijä½ľ":107253,"çļĦåij³éģĵ":107254,"éķ¿è¿ľ":107255,"åħ¬åħ±æľįåĬ¡":107256,"æŃ£å¸¸çļĦ":107257,"æľīè¿ĩ":107258,"é£İæĥħ":107259,"æ¯Ķéĩį":107260,"åIJ»":107261,"管çIJĨå·¥ä½ľ":107262,"综åIJο̧":107263,"已被":107264,"说起":107265,"æİĴæ°´":107266,"ä¸įæĸŃåľ°":107267,"æĥħæĢĢ":107268,"è¾ĵéĢģ":107269,"è¿ĩæķı":107270,"çļĦåı¯èĥ½æĢ§":107271,"æľįç͍":107272,"æľī许å¤ļ":107273,"å§Ķåī¯ä¹¦è®°":107274,"åĮĸå¦Ĩåĵģ":107275,"æļĤåģľ":107276,"æĬķèµĦ人":107277,"çıŃ级":107278,"说çĿĢ":107279,"åįĹåĮĹ":107280,"åĪĨè¡Į":107281,"çıłå®Ŀ":107282,"寶":107283,"å¢ŀå¤ļ":107284,"被åĬ¨":107285,"ç®ĬçļĦ":107286,"éĹľä¿Ĥ":107287,"çļĦèĦ¸":107288,"æĥŁ":107289,"ä¸įä¸Ģå®ļ":107290,"ç¶Ń":107291,"çģ«çĪĨ":107292,"ç§Łéĩij":107293,"çŀ§":107294,"éĩį建":107295,"è·ª":107296,"ä¸Ģ種":107297,"çļĦåIJĪä½ľ":107298,"å®īæħ°":107299,"ä»įæĺ¯":107300,"ä¸ĵä¸ļåĮĸ":107301,"è°ĥè§£":107302,"ä¸į妨":107303,"éĢĻæĺ¯":107304,"å¿ħéłĪ":107305,"ä¼ĬæľĹ":107306,"å¾ĹäºĨ":107307,"æľįåĬ¡å¹³åı°":107308,"姬":107309,"åħĪéĶĭ":107310,"çİĭåŃIJ":107311,"çļĦä¸ĢåĪĩ":107312,"æĢ»çIJĨ":107313,"åĵ¼":107314,"çªij":107315,"çļĦå¿ĥæĥħ":107316,"çļĦéĩį大":107317,"çijŁ":107318,"ä¸Ģç¬ij":107319,"åıijå±ķä¸Ń":107320,"åģ¥åº·åıijå±ķ":107321,"åĵģçīĮçļĦ":107322,"禮":107323,"ä½Ļ人":107324,"ä»Ĭ年以æĿ¥":107325,"æķ°çłģ":107326,"çѾè¯ģ":107327,"åİ»æī¾":107328,"åŁºéĩijä¼ļ":107329,"æĬ±æĢ¨":107330,"æŃ£å½ĵ":107331,"çıŃåŃIJæĪIJåijĺ":107332,"ä¸įåIJĪæł¼":107333,"åζå®ļäºĨ":107334,"ç¼ĵæħ¢":107335,"åĪ¶çº¦":107336,"æłı缮":107337,"å¸Ĥåľºç»ıæµİ":107338,"ç»ĦæĪIJçļĦ":107339,"严峻":107340,"æĹ¥è®¯":107341,"ä¸ĢçĤ¹çĤ¹":107342,"æĺ¯æĢİä¹Ī":107343,"çļĦçħ§çīĩ":107344,"éĺ»æŃ¢":107345,"模ç³Ĭ":107346,"缸":107347,"éģķåıį":107348,"æIJ¬è¿ģ":107349,"éĩijéĴ±":107350,"彬":107351,"ä¸įå®ī":107352,"æĪĺçķ¥åIJĪä½ľ":107353,"å¡«åĨĻ":107354,"讲究":107355,"åħħåĪĨåĪ©ç͍":107356,"èĥ½å¤ł":107357,"èij¡èIJĦéħĴ":107358,"éĩĩç͍äºĨ":107359,"åľ¨ä»Ĭå¹´":107360,"ä¸Ńå°ıåѦ":107361,"åľ¨æĦı":107362,"çļĦåİĭåĬĽ":107363,"ä¸į幸":107364,"åζèį¯":107365,"åı¯ä»¥è®©":107366,"被è¯Ħ为":107367,"ç»ĨèıĮ":107368,"æĪıåī§":107369,"åįĬ导":107370,"åįĬ导ä½ĵ":107371,"è§Ĩè§Ĵ":107372,"åĸľæŃ¡":107373,"å¾ģæĶ¶":107374,"è°ĭåĪĴ":107375,"æŀģ大çļĦ":107376,"çĤ¹èµŀ":107377,"è®°èĢħä»İ":107378,"两åIJį":107379,"èĩªåĬ©":107380,"èµ·æŃ¥":107381,"æĬ¤å£«":107382,"å®Ŀ马":107383,"太åŃIJ":107384,"å°ıå°ıçļĦ":107385,"温æ³ī":107386,"åĩºç§Łè½¦":107387,"ç§ŁæĪ¿":107388,"两家":107389,"éľĩæĴ¼":107390,"ç§īæī¿":107391,"ä¸Ģä»¶äºĭ":107392,"çĥĪ士":107393,"å®ĺåħµ":107394,"转身":107395,"ä¹IJåĽŃ":107396,"çĻĮçĹĩ":107397,"模èĮĥ":107398,"æĦ£":107399,"è¿ĩåİ»çļĦ":107400,"代价":107401,"çļĦæ¦Ĥ念":107402,"åĩłçϾ":107403,"è´µéĺ³":107404,"æĭħå¿§":107405,"éĢĤå®ľ":107406,"çݯå¢ĥä¿ĿæĬ¤":107407,"çĥ«":107408,"ä½łæĥ³":107409,"æŃ¤åIJİ":107410,"ä½łä¹Ł":107411,"çįİ":107412,"éϤæŃ¤":107413,"éϤæŃ¤ä¹ĭå¤ĸ":107414,"è°ĥ度":107415,"ç§ij缮":107416,"æīĢ说çļĦ":107417,"åĬĩ":107418,"忽è§Ĩ":107419,"ä¸ī次":107420,"ä¸ĢæĹ¥":107421,"åŀĤ缴":107422,"ç«ŀæĬĢ":107423,"éĿ¢åĮħ":107424,"大æĪĺ":107425,"æIJºå¸¦":107426,"å¦Ĥæŀľæ²¡æľī":107427,"åħ»æĪIJ":107428,"åĩºè¡Ģ":107429,"çα好èĢħ":107430,"æīĵéĢļ":107431,"èµ·è¯ī":107432,"åijĪçݰåĩº":107433,"æŃĮæīĭ":107434,"åľ¨å¤ĸ":107435,"é¢Ĩ导干éĥ¨":107436,"åĨ¥":107437,"èĪĨ论":107438,"æıIJåıĸ":107439,"éĺ¿å°Ķ":107440,"æľĽçĿĢ":107441,"ä¸īäºļ":107442,"財":107443,"åĪ·æĸ°":107444,"æĻļæĬ¥":107445,"è¿ĺæľīä¸Ģ个":107446,"åĨ°ç®±":107447,"ç½ijçĤ¹":107448,"åĩºåħ·":107449,"强çĥĪçļĦ":107450,"æĪijçĽ¸ä¿¡":107451,"å¸ĮæľĽèĥ½":107452,"çīĻ齿":107453,"äºĭå®ľ":107454,"ä¸ļåĨħ人士":107455,"ä»£æĽ¿":107456,"åıĺå½¢":107457,"éĽ²":107458,"è°ĥæİ§":107459,"åĪĽæĸ°åĪĽä¸ļ":107460,"æĭĨè¿ģ":107461,"æł¸æŁ¥":107462,"éĢĹ":107463,"åħ¥åѦ":107464,"æĦıåIJij":107465,"æıĽ":107466,"ä¸ĭ次":107467,"ä¼łè¾ĵ":107468,"ä»ĸä»¬åľ¨":107469,"èĢĮä¸Ķè¿ĺ":107470,"æĹ¥åľ¨":107471,"æķĻè®Ń":107472,"æ´»çĿĢ":107473,"çļĦæľīæķĪ":107474,"å¤įå·¥å¤į":107475,"å¤įå·¥å¤į产":107476,"æĺ¯ä¸Ģä»¶":107477,"çŃīçĿĢ":107478,"復":107479,"åĭĩæķ¢":107480,"éģŃåıĹ":107481,"å¥Ķé©°":107482,"讲座":107483,"说å®Į":107484,"ç»Ļåĩº":107485,"è°¦":107486,"è¯ĬçĸĹ":107487,"çĽ²çĽ®":107488,"客è¿IJ":107489,"å°±è¿ŀ":107490,"å¼Ģåħĥ":107491,"å¼Ģåħĥæ£ĭçīĮ":107492,"ä¸įæĸŃæıIJåįĩ":107493,"ç͍æĪ·çļĦ":107494,"æĴķ":107495,"ä¾Ľæ°´":107496,"ç¶ĵæ¿Ł":107497,"ä¸ŃåĮ»èį¯":107498,"èģĶæĥ³":107499,"åħ¬äº¤è½¦":107500,"èĪªçıŃ":107501,"æĬĢè¡ĵ":107502,"å¼ķèµ·çļĦ":107503,"å°¹":107504,"èµĦæ·±":107505,"åĽ½èµĦå§Ķ":107506,"èĺŃ":107507,"é¼»åŃIJ":107508,"éĹ½":107509,"æİĴéĺŁ":107510,"è§Ĥåħī":107511,"éģĹåĿĢ":107512,"ä¸ľäº¬":107513,"é¥ŃåºĹ":107514,"ä¸įæĸŃçļĦ":107515,"å°±æĺ¯ä¸Ģ个":107516,"éķ¿ä¹ħ":107517,"çļĦè§ĤçĤ¹":107518,"娶":107519,"æĪijçİ°åľ¨":107520,"çķ°":107521,"å¾Ĺåĩº":107522,"å¿ħå®ļ":107523,"ä¸įåıĹ":107524,"åıªéľĢè¦ģ":107525,"åĽ°æī°":107526,"ç§ijåѦæĬĢæľ¯":107527,"çīĽèĤī":107528,"è¾ĥé«ĺçļĦ":107529,"è·ijæŃ¥":107530,"æ²¾":107531,"èı©èIJ¨":107532,"æľĢå¾Į":107533,"ä¿Ŀå¯Ĩ":107534,"æ²»å®ī":107535,"éĤ±":107536,"常è¯Ĩ":107537,"èĦ¸èī²":107538,"åĮĹ大":107539,"æ±ĩèģļ":107540,"æijĨèĦ±":107541,"é¾Ļ头ä¼ģä¸ļ":107542,"女åıĭ":107543,"çŃīå·¥ä½ľ":107544,"ä¸Ńç¾İ":107545,"èģĮåľº":107546,"èĦijè¢ĭ":107547,"åĨĻçļĦ":107548,"饲æĸĻ":107549,"åĬ³åĬ¨åĬĽ":107550,"屯":107551,"æĮģèĤ¡":107552,"åĽ¾åĥı":107553,"è¿ĩåİ»äºĨ":107554,"貨":107555,"è¾²":107556,"éĹ®æĪij":107557,"è·Łä½ł":107558,"çĶŁæŃ»":107559,"审ç¾İ":107560,"é¢Ĺç²Ĵ":107561,"ä¸Ńæĸ¹":107562,"åĬłçĥŃ":107563,"æĹħè¡Į社":107564,"çϼçĶŁ":107565,"ä¸įåłª":107566,"åĤ·":107567,"æ¥ł":107568,"åĬŀæ¡Ī":107569,"æŁĦ":107570,"æĹ¢æĺ¯":107571,"å¤ĦåĪĨ":107572,"羣å®ŀçļĦ":107573,"æĬ¥çº¸":107574,"å¸Īçζ":107575,"å®īå¾½çľģ":107576,"åī¯ä¸»å¸Ń":107577,"ä¹ĭéģĵ":107578,"导弹":107579,"åŃ¦æł¡çļĦ":107580,"åŁİå¸ĤçļĦ":107581,"è°Īåΰ":107582,"æ¢Ĺ":107583,"å¹³éĿ¢":107584,"说ä»Ģä¹Ī":107585,"é¢ijçİĩ":107586,"éķ¿ä¸īè§Ĵ":107587,"çļĦåĪ©çĽĬ":107588,"黨":107589,"è±ĨèħIJ":107590,"å®ŀéĻħæĥħåĨµ":107591,"æŀĹä¸ļ":107592,"纪æ£ĢçĽijå¯Ł":107593,"ä½ıéĻ¢":107594,"çļĦæķ´ä½ĵ":107595,"åīįè¡Į":107596,"æĮ¨":107597,"çħ¤çŁ¿":107598,"å̻è£ģ":107599,"å°ıåIJĥ":107600,"æŀģ端":107601,"å©Ĩå©Ĩ":107602,"çݰ货":107603,"è¯ĹæŃĮ":107604,"éĴ¥åĮĻ":107605,"缩çŁŃ":107606,"ä½Ĩè¿Ļ":107607,"æĸ°åĵģ":107608,"è¿Ļ对":107609,"çŁ¥åIJį度":107610,"å¿ĹæĦ¿æľįåĬ¡":107611,"大å±Ģ":107612,"è¡¡éĩı":107613,"ä½ĵçݰäºĨ":107614,"æ¡ĥèĬ±":107615,"åIJ¸å¼ķåĬĽ":107616,"åł¤":107617,"æĵħéķ¿":107618,"åĴĴ":107619,"çĽ¸æľº":107620,"ä¸Ģç«Ļ":107621,"ä¸Ģç«Ļå¼ı":107622,"æľĢç¾İ":107623,"æ°¸ä¹ħ":107624,"çļĦéĥ¨åĪĨ":107625,"åĪĨå·¥":107626,"å·¥ç¨ĭ建设":107627,"æIJŃè½½":107628,"æ°´ä¸Ń":107629,"èĮ¨":107630,"çļĦæĵįä½ľ":107631,"ç»Łæ²»":107632,"çķħéĢļ":107633,"åħļçļĦåįģ":107634,"輸":107635,"測":107636,"ç¾İè§Ĥ":107637,"ä¸įåĪ©":107638,"åıįæĢĿ":107639,"éªĦåĤ²":107640,"æłĩçļĦ":107641,"æĿĢ人":107642,"éĺ¿å§¨":107643,"é£ŁæĿIJ":107644,"åIJĥçļĦ":107645,"åIJİåĨį":107646,"çŁ£":107647,"两侧":107648,"æ¸ħæ°´":107649,"è¿ĽçIJĥ":107650,"å¼Ģå§ĭäºĨ":107651,"åIJ¬äºĨ":107652,"çĦĬæİ¥":107653,"磮":107654,"å¨Ł":107655,"为人":107656,"éĢģç»Ļ":107657,"åĨĴéĻ©":107658,"æķ·":107659,"ç»ĪæŃ¢":107660,"æīįçŁ¥éģĵ":107661,"è¿IJæ°Ķ":107662,"éĢļé£İ":107663,"æĥĬè®¶":107664,"ç§ijåѦéĻ¢":107665,"æıIJéĹ®":107666,"太åİŁ":107667,"缸åIJĮçļĦ":107668,"ä»ķ":107669,"èģĸ":107670,"æĥħæ³ģ":107671,"é¢Ĩ导人":107672,"åĩºæĿ¥äºĨ":107673,"沿线":107674,"éϽ":107675,"æĦŁè¦º":107676,"ä»įåľ¨":107677,"æ©Ļ":107678,"约为":107679,"åĸĿéħĴ":107680,"ç͍èį¯":107681,"ä¸ĭä¸Ģ":107682,"æ³ķå®ĺ":107683,"顺åºı":107684,"åģļä¸Ģ个":107685,"åĭ¢":107686,"æŃª":107687,"ç͵ç«ŀ":107688,"ä¼´éļıçĿĢ":107689,"ä¹ĭåĬĽ":107690,"ä¹ĭ人":107691,"äºij计ç®Ĺ":107692,"åĪ«äººçļĦ":107693,"ç§ijåѦåıijå±ķ":107694,"第åħ«":107695,"å¹²æī°":107696,"女ç¥ŀ":107697,"è¿Ļæł·åģļ":107698,"å¤Ħåľ¨":107699,"æ°´è´¨":107700,"éķ¿æĺ¥":107701,"å¸ĤåľºéľĢæ±Ĥ":107702,"ç»´æĿĥ":107703,"èĢ³æľµ":107704,"æĸĩåĮĸçļĦ":107705,"奶ç²ī":107706,"ä¼łè¾¾":107707,"æīĭæľºçīĪ":107708,"æĽ¾åľ¨":107709,"äºĮæľŁ":107710,"åİŁåĽłæĺ¯":107711,"æºIJ头":107712,"åıĪèĥ½":107713,"裸":107714,"æĬĢæľ¯åĪĽæĸ°":107715,"æĸĩåĮĸæĹħ游":107716,"åıij票":107717,"年级":107718,"ä½łä¸į":107719,"ä¹ĭå¿ĥ":107720,"æķ°çϾ":107721,"åIJijå¾Ģ":107722,"èĢģå®¶":107723,"åľĭéļĽ":107724,"çļĦé«ĺ度":107725,"æľĿéĺ³":107726,"æ¸ħéϤ":107727,"èĩªæľī":107728,"书ä¸Ń":107729,"游æĪıè£ħå¤ĩ":107730,"ä¸ĩå¤ļ":107731,"驾驶åijĺ":107732,"ä½łçŁ¥éģĵ":107733,"åĽ½åºĨ":107734,"é£ŁåłĤ":107735,"æİ¥åı£":107736,"æĢ»æķ°":107737,"åħ¶ä»ĸçļĦ":107738,"çĶŁåij½çļĦ":107739,"ä½łåľ¨":107740,"çļĦ缮åħī":107741,"è¿Ļæĸ¹éĿ¢":107742,"éĥ½è¯´":107743,"çĸĹæ³ķ":107744,"åĭĩ士":107745,"åľ¨åħ¨çIJĥ":107746,"ä¿ĿéĻ©åħ¬åı¸":107747,"çĿ£æŁ¥":107748,"åĸĦèī¯":107749,"表彰":107750,"è¹²":107751,"路段":107752,"æľĥåĵ¡è¦ı":107753,"æľĥåĵ¡è¦ıç¯Ħ":107754,"æĪ·åŀĭ":107755,"ä¿ĥ使":107756,"修建":107757,"é«ĺæ°´å¹³":107758,"åģļåĩºäºĨ":107759,"ä¸»åľº":107760,"è¡Įèµ°":107761,"空çϽ":107762,"æľī人说":107763,"è¿Ļ个ä¸ĸçķĮ":107764,"åIJįä¹ī":107765,"å®Įç¾İçļĦ":107766,"羡æħķ":107767,"åıĬåħ¶ä»ĸ":107768,"åı¯ç͍":107769,"æĭIJ":107770,"è¾ĥ大çļĦ":107771,"æĬĢæľ¯åĴĮ":107772,"å°¼äºļ":107773,"çĻ¾è´§":107774,"æıī":107775,"éĢīè´Ń":107776,"éĺŁåıĭ":107777,"ä¼łæĦŁ":107778,"ä¼łæĦŁåύ":107779,"åıªè¦ģä½ł":107780,"为ä»Ģä¹Īè¦ģ":107781,"ä¸ĵ注äºİ":107782,"ä½Ļé¢Ŀ":107783,"åħ¸åŀĭçļĦ":107784,"缮åīįå·²":107785,"æ¬²æľĽ":107786,"èģĶ绾":107787,"æµģä¼ł":107788,"çļĦå®¶åºŃ":107789,"åı·åı¬":107790,"çıįè´µ":107791,"ä¼Łå¤§çļĦ":107792,"éī´äºİ":107793,"è·Łä»ĸ":107794,"产çī©":107795,"ä¸įå·²":107796,"è¿Ŀæ³ķè¡Į为":107797,"头ä¸Ĭ":107798,"åĪĨè§£":107799,"åı¯ä»¥çľĭåĩº":107800,"æł¡åĮº":107801,"åŃĹä½ĵ":107802,"ä¿®çĤ¼":107803,"çĶļèĩ³æĺ¯":107804,"微信åħ¬ä¼Ĺ":107805,"åıĸ代":107806,"èIJ¥ä¸ļæĶ¶åħ¥":107807,"æ½įåĿĬ":107808,"ä½łèĥ½":107809,"社ä¼ļä¿Ŀéļľ":107810,"æ¯ĶèµĽä¸Ń":107811,"污水å¤ĦçIJĨ":107812,"夫å¦ĩ":107813,"ä¸Ģå¹ħ":107814,"沿海":107815,"åı£æĦŁ":107816,"ä½Ĩåį´":107817,"å½ĵæĹ¥":107818,"çļĦæľĢ大":107819,"æ¯ıä¸Ģä½į":107820,"没äºĭ":107821,"çī¹åĪ¥":107822,"å¼ĢåѦ":107823,"è·¯éĿ¢":107824,"å¿ĥçIJĨåѦ":107825,"æĶ¾ç½®":107826,"éĩįåºĨå¸Ĥ":107827,"ä½łèĩªå·±":107828,"æ¶Īè´¹èĢħçļĦ":107829,"ä¸Ģæ³¢":107830,"èѦæĥķ":107831,"åį§å®¤":107832,"注å°Ħ":107833,"é£İ鼨":107834,"沿çĿĢ":107835,"åijĬ訴":107836,"表çݰåĩº":107837,"åĽĽæĺ¯":107838,"åı¤åħ¸":107839,"æĽ´éĩįè¦ģçļĦ":107840,"好äºĭ":107841,"çľ¼æ³ª":107842,"æ¨ĵ":107843,"审åΤ":107844,"碰æĴŀ":107845,"车ç«Ļ":107846,"è¿Ľåħ¥äºĨ":107847,"éĽĨåIJĪ":107848,"æł¼å¤ĸ":107849,"宾é¦Ĩ":107850,"æĶ¯ä»ĺå®Ŀ":107851,"她æĺ¯":107852,"æĺ¯å¦Ĥä½ķ":107853,"人次":107854,"çļĦæĪIJåĬŁ":107855,"æĹłåĬĽ":107856,"æµ·æĭĶ":107857,"æĺ¥åŃ£":107858,"éĥ½ä¸įä¼ļ":107859,"çŃīå¤ļç§į":107860,"ä¸Ģ个å°ı":107861,"åģľè½¦åľº":107862,"è®©æĽ´å¤ļ":107863,"è¿ĻçĤ¹":107864,"æĪIJåĵģ":107865,"éĴī":107866,"éģĩè§ģ":107867,"çıŃ主任":107868,"æĦıæĦ¿":107869,"çļĦåIJĮåѦ":107870,"游è§Ī":107871,"åİĭ缩":107872,"åľ¨ä¼łå¥ĩ":107873,"å¼¹æĢ§":107874,"æĹ¥åĨħ":107875,"ç¦ı建çľģ":107876,"è§ĴèIJ½":107877,"åĪĨå¼Ģ":107878,"ä¼ļ让":107879,"å¤ĸåĽ´":107880,"çĨŁæĤīçļĦ":107881,"çĨĶ":107882,"ä¸ĩè¾Ĩ":107883,"å¤ľéĹ´":107884,"车身":107885,"ä¸ŃæľŁ":107886,"å®ĮåĸĦçļĦ":107887,"åĵģç±»":107888,"åıĭè°Ĭ":107889,"éĢīæĭĶ":107890,"éªij士":107891,"彦":107892,"çļĦçľĭæ³ķ":107893,"åĽ½çİĭ":107894,"è¾£æ¤Ĵ":107895,"åıijå¸ĥæĹ¶éĹ´":107896,"åı¤åŁİ":107897,"éļıæľº":107898,"ç«ĸ":107899,"å¼Ģè¾Ł":107900,"ä¼ĹçĶŁ":107901,"没åĬŀæ³ķ":107902,"åįĥéĩĮ":107903,"æĿ¥æºIJäºİ":107904,"çļĦæĿĥåĪ©":107905,"æ¯ĶåĪĨ":107906,"满æĦıçļĦ":107907,"ä¿®è¡Į":107908,"åĿł":107909,"大海":107910,"èݹ":107911,"åĩºèº«":107912,"è«ĩ":107913,"åħ³èĬĤ":107914,"åIJį人":107915,"éľĢè¦ģ注æĦı":107916,"æĹ©æĻ¨":107917,"å¤ĸåįĸ":107918,"åıĪè¦ģ":107919,"æ¶īæ¡Ī":107920,"çĶ³è¯·äºº":107921,"éĻĦè¿ijçļĦ":107922,"åĬłå¿«æİ¨è¿Ľ":107923,"æĸ°å¹´":107924,"大è¡Ĺ":107925,"ä¸Ģé»ŀ":107926,"èĭıå®ģ":107927,"æĤĦæĤĦ":107928,"èĦ¾æ°Ķ":107929,"å¸ĮèħĬ":107930,"éļıåį³":107931,"æķ¢äºİ":107932,"å®ŀè·µä¸Ń":107933,"æĺ¯æ²¡æľī":107934,"æľīè¶£çļĦ":107935,"æĿ¥èĩªäºİ":107936,"è£ģåΤ":107937,"女åŃ©åŃIJ":107938,"èĩ³åħ³":107939,"èĩ³åħ³éĩįè¦ģ":107940,"æĻºåĬĽ":107941,"èµ°åĩºåİ»":107942,"çŁŃæĿ¿":107943,"å¤§åĽ½":107944,"çļĦ认è¯Ĩ":107945,"å¹´å¤ľ":107946,"åĨįåΰ":107947,"åIJĮæł·çļĦ":107948,"å¯Ĩå°ģ":107949,"å¤ĸ交éĥ¨":107950,"çĶŁæķĪ":107951,"æĤ¨åı¯ä»¥":107952,"ä½łåĢij":107953,"è¿ĩå¹´":107954,"å¼ĵ":107955,"è¡ĮæĿİ":107956,"æ¯Ķèµ·":107957,"身é«ĺ":107958,"è¿Ļ个人":107959,"ä¸Ńå¤ĸ":107960,"éģĵæŃī":107961,"çĽ¯çĿĢ":107962,"亲åŃIJ":107963,"éŸ":107964,"çϽäºij":107965,"èĦĸåŃIJ":107966,"ä¸ĢåĪĩéĥ½":107967,"æ·ij":107968,"è°ľ":107969,"åģ¶çĦ¶":107970,"éĿłè°±":107971,"é«ĺ管":107972,"ä¸ĭåıij":107973,"æĶ¾åΰ":107974,"ç±»åĪ«":107975,"ä¸ĭåĪĹ":107976,"æ··ä¹±":107977,"åIJĪæ³ķæĿĥçĽĬ":107978,"çݯçIJĥ":107979,"æľīæķĪåľ°":107980,"åķĨæĪ·":107981,"æ¹ĸ人":107982,"海岸":107983,"æĬķ产":107984,"两个æľĪ":107985,"éĥ½éĿŀ常":107986,"å¢ŀ强äºĨ":107987,"æĿ¥åΰäºĨ":107988,"åī©ä½Ļ":107989,"æĤ¨çļĦåŃ©åŃIJ":107990,"æµģæ°´":107991,"æŃ£ä¹ī":107992,"天çĮ«":107993,"åģļè¿ĩ":107994,"ä½ķæĹ¶":107995,"æĪijåİ»":107996,"çľģ份":107997,"å¥ĸéĩij":107998,"该å¦Ĥä½ķ":107999,"ä¸ĭçıŃ":108000,"åģ¶åĥı":108001,"æijĨæĶ¾":108002,"æĸ°æ¨¡å¼ı":108003,"æĬķè³ĩ":108004,"è·¯åı£":108005,"åĨľæ°ijå·¥":108006,"大åѸ":108007,"ä»¶äºĭ":108008,"æł¹æľ¬ä¸į":108009,"æµĵ度":108010,"æµĵåİļ":108011,"è½®èĥİ":108012,"æĪ¿ä¼ģ":108013,"éĿŀ常好":108014,"ä»İä¸Ń":108015,"äººæł¼":108016,"ç¿ģ":108017,"æĹ¶éĹ´åĴĮ":108018,"è¿Ļä¸įæĺ¯":108019,"åΏåķĨ":108020,"æĥĬ人":108021,"åύå®ĺ":108022,"åĩĨåĪĻ":108023,"æĥħæĻ¯":108024,"æĽ´é«ĺçļĦ":108025,"åѦ家":108026,"泡沫":108027,"åľ°æĸ¹æĶ¿åºľ":108028,"å°±çŁ¥éģĵ":108029,"åij¼åIJģ":108030,"ç»ıè´¸":108031,"èĬ±éĴ±":108032,"æľīä¸Ģ次":108033,"æĦŁæħ¨":108034,"ä¸Ģåįĥ":108035,"å¤ľæĻļ":108036,"詹å§Ĩ":108037,"詹å§Ĩæĸ¯":108038,"è¦ģéĹ»":108039,"ç»Ĵ":108040,"æºIJäºİ":108041,"çļĦè´¨éĩı":108042,"注æĦıäºĭ项":108043,"æħ¢æĢ§":108044,"稳å®ļçļĦ":108045,"建设åĴĮ":108046,"æĻ¯è±¡":108047,"éĩıåĮĸ":108048,"çļĦ話":108049,"è¯Ħ级":108050,"æºľ":108051,"红åĮħ":108052,"éĢļéģİ":108053,"社ä¼ļ责任":108054,"æĸ°äº§åĵģ":108055,"åĨ·éĿĻ":108056,"çľĭä¸įåΰ":108057,"èģĶéĤ¦":108058,"éŃĦ":108059,"çļĦåīįæıIJ":108060,"çļĦåīįæıIJä¸ĭ":108061,"è¾ĥ好":108062,"çļĦæĦŁæĥħ":108063,"客æĪ·æıIJä¾Ľ":108064,"çĭ¬èĩª":108065,"å¢ŀæĶ¶":108066,"æĸĩçĮ®":108067,"æĭ¼åij½":108068,"管çIJĨåĴĮ":108069,"æµģåĬ¨æĢ§":108070,"åħ¨å®¶":108071,"ä¸Ĭæĸ¹":108072,"æİ¨åĩºçļĦ":108073,"ä¸īåĽ½":108074,"ä¸Ģ个æĺ¯":108075,"æĸ°ä¸Ģè½®":108076,"æĸĩåĮĸéģĹ产":108077,"殺":108078,"大湾åĮº":108079,"éĥ½éľĢè¦ģ":108080,"çļĦå®ŀéĻħ":108081,"ç·Ĭ":108082,"大å¥ĸ":108083,"åħīèĬĴ":108084,"便äºİ":108085,"çļĦ表æĥħ":108086,"æ¼Ķç»İ":108087,"红åĨĽ":108088,"å½ĵæĪij":108089,"æ²»æĦĪ":108090,"é¢Ŀ度":108091,"éĿľ":108092,"ä»»ä½ķ人":108093,"è¡Ĺ头":108094,"çĸ¯":108095,"çĸ¯æĭī":108096,"åĮ»çĸĹæľºæŀĦ":108097,"ç»ĻåŃ©åŃIJ":108098,"è§Ħ磩":108099,"è£ľ":108100,"çļĦ身影":108101,"ä¸ĵæłı":108102,"æĿ¥ä¸´":108103,"童年":108104,"å¤įèĭı":108105,"è¨Ĥ":108106,"åŀĭåı·":108107,"åĽ¾æ¡Ī":108108,"ç®ĢåİĨ":108109,"æĭ±":108110,"èį·åħ°":108111,"ä»»æĦı":108112,"æī¿æİ¥":108113,"è¿Ļæīį":108114,"客车":108115,"æľĿçĿĢ":108116,"éłħ缮":108117,"åı°é£İ":108118,"çļĦæĪ¿åŃIJ":108119,"éªı":108120,"æĿ±è¥¿":108121,"éģĹä¼ł":108122,"è¶Ĭå¤ļ":108123,"äºĨä»ĸçļĦ":108124,"ä¸Ĭåij¨":108125,"管çIJĨåĪ¶åº¦":108126,"失ä¸ļ":108127,"çĶ·åıĭ":108128,"æİ¥ç§į":108129,"å¨ģåIJį":108130,"çĴ°å¢ĥ":108131,"åıijçĶŁåľ¨":108132,"ä¸ªåĽ½å®¶":108133,"åĪĽæĸ°åıijå±ķ":108134,"æĶ¹åıĺäºĨ":108135,"åģ¥åº·çļĦ":108136,"å̼å¾Ĺä¸Ģ":108137,"å̼å¾Ĺä¸ĢæıIJ":108138,"åĽ¢ä¼Ļ":108139,"åģĩ设":108140,"åı°ä¸Ĭ":108141,"è§ĦèĮĥåĮĸ":108142,"éĻªåIJĮ":108143,"座æ¤ħ":108144,"åı¯æĢľ":108145,"åħĭæĢĿ主ä¹ī":108146,"æ³ķå¾ĭ责任":108147,"ä¸Ģé¡¿":108148,"æĬ¬å¤´":108149,"为éĩįçĤ¹":108150,"è¿ľæ´ĭ":108151,"éĢıè¿ĩ":108152,"åħ¨çIJĥåĮĸ":108153,"è¶£åij³":108154,"票æĪ¿":108155,"æ¯ı人":108156,"åIJĦç§įåIJĦæł·":108157,"äºĨåĩºæĿ¥":108158,"ç»Ŀ对æĺ¯":108159,"ä¸ĭå±ŀ":108160,"ä¸ĢåıĮ":108161,"è¿ĻåĿĹ":108162,"æĬĹçĸ«":108163,"è¦ģçĤ¹":108164,"å½¢æĪIJçļĦ":108165,"æĪijçľĭ":108166,"ä¸ĩéĩĮ":108167,"èĢĥçłĶ":108168,"为åħ¶":108169,"æ°ij宿":108170,"å¤ļä½į":108171,"大èĩ´":108172,"ä»ĺè´¹":108173,"åħ¥æīĭ":108174,"å±ħå®¶":108175,"æīĢåľ¨åľ°":108176,"人身":108177,"è¿ĩå¾Ĺ":108178,"è¯ķè¯ķ":108179,"访è°Ī":108180,"åĬłéĩį":108181,"å°±ä¸įä¼ļ":108182,"çĶŁäº§ä¼ģä¸ļ":108183,"åĽŀåĽ½":108184,"åºķ线":108185,"èµ¶åΰ":108186,"æĶ¯éĺŁ":108187,"æĪij们éĥ½":108188,"éĤ®æĶ¿":108189,"缴èĩ³":108190,"éĴ¢çIJ´":108191,"åħľ":108192,"çłĶ讨ä¼ļ":108193,"æľĪ亮":108194,"åĿļæĮģ以":108195,"åħ¬å®īéĥ¨":108196,"éĴ¢ç®¡":108197,"å°ıçϽ":108198,"ç½®ä¸ļ":108199,"èģĭ":108200,"书åĨĻ":108201,"æĿı":108202,"éħįæĸ¹":108203,"èĢĮåıĪ":108204,"çijŀ士":108205,"çķĮçļĦ":108206,"èĢģ大":108207,"æĪIJçĨŁçļĦ":108208,"å¹²ä»Ģä¹Ī":108209,"ä¸ĵ项æĸĹäºī":108210,"çŃīå¤ļ个":108211,"èĦ±ç¦»":108212,"ä¸ī个æľĪ":108213,"çłĶç©¶åijĺ":108214,"æĹĭ转":108215,"æŀģèĩ´":108216,"åħįè´£":108217,"åħį责声æĺİ":108218,"å¾Īå¤ļçݩ家":108219,"车ä¸Ĭ":108220,"交äºĴ":108221,"å·²æĺ¯":108222,"ä¸Ģå°ı":108223,"çļĦéĩįçĤ¹":108224,"èĬ±äºĨ":108225,"ä¸įæĺİ":108226,"æľīåħ³è§Ħå®ļ":108227,"çĬ¹å¦Ĥ":108228,"羸":108229,"寡":108230,"çļĦè¡£æľį":108231,"åĮħ裹":108232,"身åŃIJ":108233,"å¸ĪèĮĥ大åѦ":108234,"äºĭåħĪ":108235,"线æĿ¡":108236,"æ³ķåζ":108237,"åħ»æĬ¤":108238,"稳å®ļæĢ§":108239,"éĤµ":108240,"åŀĦæĸŃ":108241,"é¡į":108242,"èĢĥåı¤":108243,"æĿłæĿĨ":108244,"èĭıèģĶ":108245,"æ°´ç͵":108246,"åħ·ä½ĵçļĦ":108247,"æ¿Ģæ´»":108248,"æĪijæł¡":108249,"åĪļå¼Ģå§ĭ":108250,"åĩ¸æĺ¾":108251,"禾":108252,"åħ¼èģĮ":108253,"éĢıéģİ":108254,"åľ¨æ¸¸æĪıä¸Ń":108255,"社ä¼ļåıijå±ķ":108256,"好çİ©":108257,"å¹»æĥ³":108258,"ä¸į代表":108259,"注æĦıåĬĽ":108260,"æ£į":108261,"ç͍æīĭ":108262,"ç¾İ人":108263,"许å¤ļ人":108264,"å¾Īæĺ¯":108265,"çļĦçłĶåıij":108266,"æīĵåĩº":108267,"åIJĪä¼Ļ人":108268,"ä¸Ģå¤ľ":108269,"ç¼ĵç¼ĵ":108270,"ä¿®æŃ£":108271,"æĦŁçŁ¥":108272,"ç»Ī身":108273,"æ¿Ģç´ł":108274,"çݯå¢ĥä¸ĭ":108275,"次ä¼ļè®®":108276,"ç»ıæµİå¢ŀéķ¿":108277,"æīĽ":108278,"åıijéħµ":108279,"åĪĨæŀIJå¸Ī":108280,"åľ¨æľªæĿ¥":108281,"主è¦ģæľī":108282,"ä¸ĢåŃ£åº¦":108283,"çļĦ说æ³ķ":108284,"ä»İæĿ¥æ²¡æľī":108285,"货车":108286,"缩å°ı":108287,"太è¿ĩ":108288,"æķĪåĬĽ":108289,"ä¸įä¸ĭ":108290,"æĬķ稿":108291,"èį¯ä¸ļ":108292,"ç»Ħéķ¿":108293,"ç«ĻçĤ¹":108294,"å¾Īåĸľæ¬¢":108295,"éIJµ":108296,"åĬ¿å¤´":108297,"æ¼ıæ´ŀ":108298,"æĦ¤æĢĴ":108299,"åħħå®ŀ":108300,"åĪĽä¸ļæĿ¿":108301,"çĪª":108302,"æľªå¿ħ":108303,"åºķéĥ¨":108304,"å¾ĹåĪĨ":108305,"人æ°ijåĮ»éĻ¢":108306,"äºĮæīĭæĪ¿":108307,"å·²ç»ı被":108308,"大楼":108309,"æĸ°æĪ¿":108310,"辦æ³ķ":108311,"ç͍åĬĽ":108312,"æĭĵ宽":108313,"åĨħåľ¨":108314,"æĴŃåĩº":108315,"饰æ¼Ķ":108316,"ä¹Łè®©":108317,"ä½ľçĤº":108318,"çī©ä¸ļ管çIJĨ":108319,"åį´ä¸į":108320,"为ä¸ŃåĽ½":108321,"å±ĢåĬ¿":108322,"ä¸įèĤ¯":108323,"æľĢæĸ°çļĦ":108324,"åı¯ä»¥éĢīæĭ©":108325,"æĺ¾çݰ":108326,"å°±ç®Ĺæĺ¯":108327,"åľ¨æł¡":108328,"é¾Ł":108329,"两æĿ¡":108330,"çļĦå®ŀåĬĽ":108331,"è¶Ĭ好":108332,"å¥¹åľ¨":108333,"å¿łè¯ļ":108334,"ä¹ŁéľĢè¦ģ":108335,"游æĪıæĵįä½ľ":108336,"è¶ħåĩº":108337,"å¦Ĥæŀľä¸į":108338,"æīĢåľ¨çļĦ":108339,"ä½łè¿ĺ":108340,"以åĨħ":108341,"æľīä¸Ģå®ļ":108342,"åı¯è¾¾":108343,"è·ijåΰ":108344,"åīĽ":108345,"建ç«ĭåģ¥åħ¨":108346,"æķ´è½¦":108347,"åīįæĸ¹":108348,"éĹ´æİ¥":108349,"çѹå¤ĩ":108350,"çĸ²åĬ³":108351,"离å¼ĢäºĨ":108352,"æ±Ŀ":108353,"éĿ¢éĥ¨":108354,"ä¹ĭåīįçļĦ":108355,"åıĺ为":108356,"å¦Ĥæŀľè¯´":108357,"对ä»ĺ":108358,"åĿĩåı¯":108359,"被åijĬ人":108360,"ç²¾ç¾İ":108361,"èģļä¼ļ":108362,"çĿ̥̿":108363,"è°·æŃĮ":108364,"ä¸Ģåı·":108365,"红åĪ©":108366,"ä¼łå¥ĩ游æĪı":108367,"å»ĸ":108368,"è´ŀ":108369,"ä¹°åΰ":108370,"éŃļ":108371,"ä½ĵè´¨":108372,"å°ijäºĨ":108373,"æ³īå·ŀ":108374,"åIJŁ":108375,"ç»Ŀä¸į":108376,"é»ijæģ¶":108377,"é»ijæģ¶åĬ¿åĬĽ":108378,"ä¸Ĭæĺł":108379,"çļĦè¯Ŀé¢ĺ":108380,"ä¸ĩ人次":108381,"ä¸ĸéĹ´":108382,"ç͍工":108383,"贯穿":108384,"å®ĿçŁ³":108385,"ä½łå¥½":108386,"åĪĩåī²":108387,"å¼ºåĽ½":108388,"åĽŀèIJ½":108389,"æ°´æĻ¶":108390,"模仿":108391,"洪水":108392,"éĢĻ麼":108393,"åįģä¸īäºĶ":108394,"ä½ij":108395,"éĻĦä»¶":108396,"çļĦå¢ŀéķ¿":108397,"éĻĦå±ŀ":108398,"çݰ已":108399,"å¸®ä½ł":108400,"éĩijçīĮ":108401,"é«ĺåİŁ":108402,"åľ¨å®¶éĩĮ":108403,"éĺ²èħIJ":108404,"ç¡®å®ŀæĺ¯":108405,"宣讲":108406,"天æīį":108407,"ç»ıèIJ¥ç®¡çIJĨ":108408,"éĶħçĤī":108409,"åIJĪä¸Ģ":108410,"è§Ĥèµı":108411,"éķ¿è¾¾":108412,"主ä¹īæĢĿæĥ³":108413,"éĤ£éº¼":108414,"é£İäºij":108415,"为主çļĦ":108416,"æļijåģĩ":108417,"æĮģä¹ħ":108418,"å¼Ĥåľ°":108419,"å¼ĢéŨ":108420,"模æĿ¿":108421,"æī¹æ¬¡":108422,"ä¸į便":108423,"天çĶŁ":108424,"åĩłä¸ªæľĪ":108425,"ä¸ĵç§ij":108426,"åı¦æľī":108427,"åħ¬å¸ĥçļĦ":108428,"æĩ·":108429,"åľºåIJĪ":108430,"çļĦå¿ĥæĢģ":108431,"è¿ĺ好":108432,"å®ŀæĪĺ":108433,"èĢģå¸ĪçļĦ":108434,"åħ©åĢĭ":108435,"åı¯åľ¨":108436,"éĤ£ä½į":108437,"å¥łå®ļäºĨ":108438,"ä¿ĥéĶĢ":108439,"æı´åĬ©":108440,"ä¸ĩçī©":108441,"æĥħæĬ¥":108442,"é¦ĸåħĪè¦ģ":108443,"æĸĩåĮĸåĴĮ":108444,"éĥ½å·²ç»ı":108445,"ä¸Ĭä¸ĸ纪":108446,"åĨľåľº":108447,"大æī¹":108448,"æĺİçϽäºĨ":108449,"çļĦæĪIJéķ¿":108450,"çļĦæ¯ĶèµĽ":108451,"失误":108452,"åģļæĪIJ":108453,"ä»Ĭ天å°ıç¼ĸ":108454,"é¢Ĩè¢ĸ":108455,"æıIJåįĩäºĨ":108456,"å¾IJå·ŀ":108457,"ä»įæľī":108458,"è¿ĩ滤":108459,"å¹½é»ĺ":108460,"çĥŃéĩı":108461,"ä¸Ģé¦ĸ":108462,"æ¼Ĥ亮çļĦ":108463,"åĩłç§į":108464,"åĢ¡è®®":108465,"å°±åı¯ä»¥äºĨ":108466,"æİĴåĪĹ":108467,"éĩįéĩį":108468,"ä¼ģä¸ļåĴĮ":108469,"ä¸ĵå±ŀ":108470,"çħİ":108471,"亲æĪļ":108472,"çϾåĪĨä¹ĭ":108473,"稿件":108474,"è¿ĺå¾Ĺ":108475,"人åĵ¡":108476,"äºī夺":108477,"æĽ´å®¹æĺĵ":108478,"大èĩªçĦ¶":108479,"鼻èħ¦":108480,"太空":108481,"åľ°å¤Ħ":108482,"夢":108483,"ä»ĸ对":108484,"å¿ħå°Ĩ":108485,"ä¸įå½ĵ":108486,"严谨":108487,"åĩºåľº":108488,"å·²ç»ıæľī":108489,"é¢ĨåĨĽ":108490,"é«ĺæ¡£":108491,"ä¸ĢæīĢ":108492,"æłĹ":108493,"让åѦçĶŁ":108494,"æĽ¹æĵį":108495,"æŁIJä¸Ģ":108496,"伸åĩº":108497,"èĬ±åįī":108498,"æ¸ħéĨĴ":108499,"èģĶç³»æĸ¹å¼ı":108500,"åĪĨå±Ģ":108501,"èħ³":108502,"æ©¡èĥ¶":108503,"éķ¿å¾Ĺ":108504,"ç»¿åľ°":108505,"è¢į":108506,"çļĦèīºæľ¯":108507,"女æľĭåıĭ":108508,"ä¸Ńè¶ħ":108509,"离åŃIJ":108510,"å¤ļæł·åĮĸ":108511,"éĺ³åı°":108512,"ä½İ碳":108513,"ä¸Ģç±»":108514,"çŃīæĸ¹éĿ¢çļĦ":108515,"å¾Ĺ好":108516,"模åħ·":108517,"ä¸ĩ亿":108518,"çķĻæĦı":108519,"临æ²Ĥ":108520,"å°ijéĩı":108521,"çľĭåIJij":108522,"ç»ıèIJ¥èĢħ":108523,"çķĻä¸ĭäºĨ":108524,"åĿıäºĨ":108525,"åijĬåĪ«":108526,"羣çIJĨ":108527,"ç¼´è´¹":108528,"æĬĬä½ł":108529,"çļĦä»»åĬ¡":108530,"æĪij对":108531,"ä¹°åħ¥":108532,"çĻ»ä¸Ĭ":108533,"æľī两个":108534,"ä¸Ģ头":108535,"æĵįæİ§":108536,"åħ¨è¦ĨçĽĸ":108537,"çĿĢæīĭ":108538,"å¢ĻéĿ¢":108539,"å¤ļæĸ¹":108540,"åı¯çαçļĦ":108541,"ä¹Łåı¯èĥ½":108542,"æľĢæľī":108543,"è¿ĻäºĽéĥ½æĺ¯":108544,"æĥ¡":108545,"å®®":108546,"å¾Īå°ı":108547,"éĹ®é¢ĺæĺ¯":108548,"åĿĩæľī":108549,"å¾ģéĽĨ":108550,"说åĩº":108551,"æľīæĦı":108552,"é¢Ĥ":108553,"æī¬å·ŀ":108554,"åķĨä¸ļ模å¼ı":108555,"çĶŁèĤĸ":108556,"æįIJ款":108557,"å²Ĥ":108558,"ç¾İæĻ¯":108559,"è¿ĺ羣":108560,"æĭ¥æĬ±":108561,"身ä½ĵåģ¥åº·":108562,"æ·±å¤Ħ":108563,"çľ¼ç¥ŀ":108564,"çļĦ形象":108565,"ä¼ĺè¶Ĭ":108566,"å½ĵæĪIJ":108567,"åĮºåĪĨ":108568,"åİ»éϤ":108569,"注å®ļ":108570,"å§IJ妹":108571,"åĮºåĨħ":108572,"é©ļ":108573,"æļĹ示":108574,"æĺİ亮":108575,"æħ°éĹ®":108576,"å¸Ĥåľºä»½é¢Ŀ":108577,"çĮªèĤī":108578,"çļĦèµĦéĩij":108579,"åİĨç»ı":108580,"å§ĭç»ĪåĿļæĮģ":108581,"çĶŁæľº":108582,"ä¸į顾":108583,"éĩijåĪļ":108584,"大声":108585,"éĻķ西çľģ":108586,"é²į":108587,"åĨľä¸ļåĨľæĿij":108588,"æľī害":108589,"éŨè¯Ĭ":108590,"æ¯ıä¸Ģ次":108591,"çļĦåĽłç´ł":108592,"é¢Ŀå¤ĸ":108593,"åݿ级":108594,"çļĩåIJİ":108595,"åĽ½ä¼ģ":108596,"é¦ĸéĢī":108597,"ç¼ĸåĨĻ":108598,"æĭ¿èµ·":108599,"åģ·åģ·":108600,"ä¸İä¸ŃåĽ½":108601,"åįĸå®¶":108602,"ç»Ļä»ĸ们":108603,"ç¥ŀè¯Ŀ":108604,"åŃ¸æł¡":108605,"æĪijä¸Ģ缴":108606,"çŁ¥éģĵäºĨ":108607,"åįĴ":108608,"åĴĮåľ°åĮº":108609,"ä»Ģä¹Īéĥ½":108610,"çͻ家":108611,"æľ¬çĿĢ":108612,"ä½ĻåIJį":108613,"审çIJĨ":108614,"ä¸ĢåIJij":108615,"åıijå±ķè¶ĭåĬ¿":108616,"åĮºéĹ´":108617,"注åĨĮèµĦæľ¬":108618,"çIJ¦":108619,"ä¸įåı¯ä»¥":108620,"çļĦåĦ¿åŃIJ":108621,"å̼çıŃ":108622,"ä¸¥æł¼çļĦ":108623,"å®ŀä½ĵç»ıæµİ":108624,"æľīæĿĥ":108625,"æĪijåıĪ":108626,"éĵ¶æ²³":108627,"ç«ĭ马":108628,"æĿĢäºĨ":108629,"åĮħ容":108630,"管家":108631,"身é«Ķ":108632,"éĵħ":108633,"å°ıåŃIJ":108634,"管çIJĨç³»ç»Ł":108635,"æľīçļĦ人":108636,"é£İç͵":108637,"æĻºèĥ½åζéĢł":108638,"精确":108639,"æĭĽåķĨå¼ķ":108640,"æĭĽåķĨå¼ķèµĦ":108641,"äºĮæīĭ车":108642,"åİ¿å§Ķ":108643,"èīºäºº":108644,"å¥ķ":108645,"è¿İæĿ¥äºĨ":108646,"ç»ĵæĿŁäºĨ":108647,"çļĦä¼łç»Ł":108648,"æĭ¼æIJı":108649,"奥迪":108650,"çĸijæĥij":108651,"ä¹ĭæĹ¥èµ·":108652,"æłĩå¿ĹçĿĢ":108653,"åľ°åįĢ":108654,"è¯łéĩĬ":108655,"åĪ°æľŁ":108656,"åħ¨éĥ½":108657,"çŁŃæļĤ":108658,"æĺ¯æĪijåĽ½":108659,"æĪijå·²ç»ı":108660,"æ»´æ»´":108661,"天èµĭ":108662,"对她":108663,"åį«çĶŁéĹ´":108664,"çĶŁäº§åŁºåľ°":108665,"æĹ¥è®°":108666,"çļĦæķĻåѦ":108667,"åĵĩ":108668,"æ°ijäºĭ":108669,"è¿ĺåİŁ":108670,"æīĭä¸ŃçļĦ":108671,"çļĦèī¯å¥½":108672,"æ·«":108673,"ä¸Ńåħ±ä¸Ń央":108674,"åĪĥ":108675,"åĵĦ":108676,"åľ¨ä»ĸçļĦ":108677,"å°Īæ¥Ń":108678,"åľºéĿ¢":108679,"éĤ»å±ħ":108680,"çĹĴ":108681,"å¦Ħ":108682,"å¤ĸç§ij":108683,"ä¸įéĢĤ":108684,"举åĬŀçļĦ":108685,"éĤ¹":108686,"åħļçļĦ建设":108687,"çĻ¼è¡¨":108688,"è·¨çķĮ":108689,"æ²īæ·Ģ":108690,"大çīĩ":108691,"è¶Ĭé«ĺ":108692,"å°Ĩæĺ¯":108693,"è§īéĨĴ":108694,"åĤ¨åŃĺ":108695,"å¢ŀ大":108696,"ä¸į让":108697,"æķ´å½¢":108698,"å¹³åı°ä¸Ĭ":108699,"åĩłä½į":108700,"è¯īæ±Ĥ":108701,"好ä¸į好":108702,"åľį":108703,"æĸĩæľ¬":108704,"é̲åħ¥":108705,"ç´į":108706,"æł¹æĵļ":108707,"èįīæ¡Ī":108708,"åħŃ个":108709,"åĭ¿":108710,"åζæĪIJ":108711,"饮水":108712,"æ°¸æģĴ":108713,"èĩªæĿĢ":108714,"åı¸é©¬":108715,"éļ¾çĤ¹":108716,"为æĪij们":108717,"å¼§":108718,"åī©ä¸ĭçļĦ":108719,"åĩĨå¤ĩ好":108720,"çļĦæľĢä½³":108721,"èģĶåIJĪä¼ļ":108722,"æĤ£èĢħçļĦ":108723,"æĪijä¸įçŁ¥éģĵ":108724,"ä¸ĭä¸Ģ个":108725,"åıijå±ķæĸ¹åIJij":108726,"笨":108727,"æīĢ以æĪij们":108728,"åĨĻäºĨ":108729,"éĢłæĪIJäºĨ":108730,"æ²Ļæ¼ł":108731,"çŃĽéĢī":108732,"çģ¾åĮº":108733,"ä¸Ĭçľĭ":108734,"éħ¶":108735,"æ»ļåĬ¨":108736,"éļ¾åħį":108737,"åIJīåĪ©":108738,"ä¸Ģä¸Ģ":108739,"ç²¾å¯Ĩ":108740,"伸æīĭ":108741,"礼仪":108742,"åħ¨æĺ¯":108743,"è¶Ĭ大":108744,"ä¸Ńæłĩ":108745,"åıĸåĨ³":108746,"åıĸåĨ³äºİ":108747,"éĢĶä¸Ń":108748,"讨åİĮ":108749,"æīĭåĨĮ":108750,"第ä¹Ŀ":108751,"åŃĶåŃIJ":108752,"çĦ¶å¾Į":108753,"ä¸Ģåħ±":108754,"æµ·æĬ¥":108755,"款å¼ı":108756,"æķ´å¤©":108757,"è¾¹çķĮ":108758,"路边":108759,"æĻĭ级":108760,"åIJIJæ§½":108761,"çļĦåħ³æ³¨":108762,"æĪij没æľī":108763,"å°±æĺ¯åľ¨":108764,"缮çļĦæĺ¯":108765,"åį³ä½¿æĺ¯":108766,"é¡¶å°ĸ":108767,"å·²ç»ıåľ¨":108768,"å®īåħ¨éļIJæĤ£":108769,"æłĩæĿĨ":108770,"åįĹéĢļ":108771,"ä¼ļ对":108772,"座ä½į":108773,"èµ¢å¾ĹäºĨ":108774,"åİŁæĿ¥çļĦ":108775,"身为":108776,"书åºĹ":108777,"è¢Ńåĩ»":108778,"ä»ĬæĻļ":108779,"以èī²":108780,"以èī²åĪĹ":108781,"æĬĸéŁ³":108782,"åį´æ²¡æľī":108783,"丧失":108784,"çļĦå±ĢéĿ¢":108785,"åįģåĽĽäºĶ":108786,"çŃī缸åħ³":108787,"æ±ĩæĢ»":108788,"å¤ĸ表":108789,"为æ°ij":108790,"éľĩæĥĬ":108791,"å¥Ĺè·¯":108792,"çĬ¯ç½ªå«Įçĸij":108793,"å°Ĩ以":108794,"çİĩé¢Ĩ":108795,"éħĴåIJ§":108796,"è¡Įä¸ļåıijå±ķ":108797,"å¹´èĩ³":108798,"åύæĿIJ":108799,"åĴĮæĬĢæľ¯":108800,"æľĢå°ı":108801,"è¿Ļä¸ĢåĪĩ":108802,"èģĮç§°":108803,"å½ĵä½ľ":108804,"æİĢèµ·":108805,"åĴĭ":108806,"ä¸Ńéĥ¨":108807,"æīĭèĩĤ":108808,"ç½¢äºĨ":108809,"媳å¦ĩ":108810,"æ´½è°Ī":108811,"æĹ¶ä»£ä¸ŃåĽ½":108812,"人çĶŁçļĦ":108813,"æŀģéĻIJ":108814,"ç¦Ħ":108815,"åĮºæĶ¿åºľ":108816,"æľ¬éĴ±":108817,"礼åĵģ":108818,"çļĦéĤ£ä¸ª":108819,"ä¾¦æŁ¥":108820,"太å¤ļçļĦ":108821,"å®ŀæĸ½æĸ¹æ¡Ī":108822,"é«ĺæłĩåĩĨ":108823,"æĮĩæĮ¥éĥ¨":108824,"å̾æĸľ":108825,"çī¹èī²ç¤¾ä¼ļ":108826,"çµIJæŀľ":108827,"éĴ»çٳ":108828,"ç§»æ¤į":108829,"çī¹ç§į":108830,"èĩªæĦ¿":108831,"æĭľçĻ»":108832,"åįķ身":108833,"åį´åıĪ":108834,"åĪ¥äºº":108835,"åIJĪè§Ħ":108836,"æľºç͵":108837,"çĦı":108838,"å½ĵåīįä½įç½®":108839,"ä¹°å®¶":108840,"åIJĪ约":108841,"èĤ©èĨĢ":108842,"为åĩĨ":108843,"å®¶è£ħ":108844,"çļĦçĥŃæĥħ":108845,"éĿŀéģĹ":108846,"çļĦéŃħåĬĽ":108847,"åİŁåijĬ":108848,"社ä¼ļåIJĦçķĮ":108849,"ä¹°çļĦ":108850,"å¤ļåIJĥ":108851,"éĽķå¡ij":108852,"èµ·ä¹ī":108853,"åĬłåī§":108854,"éĤ£ä¸ĢåĪ»":108855,"å°Ĩè¿Ľä¸ĢæŃ¥":108856,"æ¡ĤæŀĹ":108857,"æĽ´å¼º":108858,"对ä¼ģä¸ļ":108859,"æĹłæĦı":108860,"ä¹łè¿ijå¹³æĸ°":108861,"æµģ失":108862,"微软":108863,"çĽ¸å¯¹äºİ":108864,"座è°Īä¼ļ":108865,"主èIJ¥ä¸ļ":108866,"主èIJ¥ä¸ļåĬ¡":108867,"ç§ģåĭŁ":108868,"å±ķ示äºĨ":108869,"常æĢģåĮĸ":108870,"è²´":108871,"符åı·":108872,"å¹´è½»çļĦ":108873,"å°±éľĢè¦ģ":108874,"ä¹ŁæĽ¾":108875,"çļĦæĥħ绪":108876,"è¾¾æłĩ":108877,"èĩ¨":108878,"ä½įå±ħ":108879,"ä»ħ为":108880,"é¦ĸå®¶":108881,"éĺ´éĺ³":108882,"ä¸įåĨįæĺ¯":108883,"åĽłä¸ºå®ĥ":108884,"ä¼ģä¸ļåľ¨":108885,"çĺ¾":108886,"åIJ¬è§ģ":108887,"åİŁæľī":108888,"åζè£ģ":108889,"å¯Ĥå¯ŀ":108890,"éĢļè¿ĩ对":108891,"æ»ijéĽª":108892,"è¿Ļå¼ł":108893,"çļĦçIJĨè§£":108894,"æĸ°ä¸ŃåĽ½":108895,"è¿ĻåĦ¿":108896,"ä½İä»·":108897,"æĥ³è¿ĩ":108898,"çļĦä¿¡å¿ĥ":108899,"建çŃijçī©":108900,"çļĦé¢ľèī²":108901,"ä¸įåºĶ该":108902,"æĹłçĸijæĺ¯":108903,"å¼ķèµ·äºĨ":108904,"åħ¨åijĺ":108905,"æĿ°åĩº":108906,"è¿Ļæĺ¯æĪij":108907,"誰":108908,"èĺĩ":108909,"éĺµåľ°":108910,"åħħå̼":108911,"çŁ¿ä¸ļ":108912,"çĿĢä»ĸ":108913,"信访":108914,"ä¸ĩè¾¾":108915,"æij©æĵ¦":108916,"å¼Ģ端":108917,"èı²å¾ĭ":108918,"èı²å¾ĭ宾":108919,"车åŃIJ":108920,"æľ¬èº«çļĦ":108921,"çģ«è½¦ç«Ļ":108922,"常å·ŀ":108923,"为代表":108924,"为代表çļĦ":108925,"广ç͵":108926,"亲人":108927,"åı³æīĭ":108928,"éĽĨè£ħ":108929,"éĽĨè£ħç®±":108930,"çļĦåį°è±¡":108931,"æ©Łæľĥ":108932,"åĮĨåĮĨ":108933,"åħīç͵":108934,"大æĸ¹":108935,"è¿ĺæľª":108936,"åΩ好":108937,"ç»Ŀ大å¤ļæķ°":108938,"åľ¨è¿Ļç§į":108939,"ä¸Ģç»Ħ":108940,"æĸ°èĤ¡":108941,"转åıij":108942,"æ³ķåºŃ":108943,"æĹłæīĢ":108944,"éģĵè·¯ä¸Ĭ":108945,"çŁ¿å±±":108946,"èijī":108947,"æĶ¶åĽŀ":108948,"ç§°ä¹ĭ":108949,"ç§°ä¹ĭ为":108950,"æıŃéľ²":108951,"åı£å²¸":108952,"åIJ¼":108953,"å¿ĥæĥ³":108954,"çļĦ梦æĥ³":108955,"éĽ¯":108956,"ä¹ĭåĪĿ":108957,"å¥ĸ项":108958,"订éĺħ":108959,"èĵĿ天":108960,"åĿ¦åħĭ":108961,"ç«ĭæ¡Ī":108962,"èģĶæīĭ":108963,"ä½Ĩæĺ¯æĪij":108964,"帮æĪij":108965,"ä»ħ代表":108966,"说æĪij":108967,"çļĦè¶ĭåĬ¿":108968,"æ¯Ķè¾ĥ大":108969,"èµ°å»Ĭ":108970,"éĩįçĤ¹é¡¹çĽ®":108971,"èµĮåľº":108972,"åIJįçīĩ":108973,"æĦŁåı¹":108974,"åľ¨åľ°ä¸Ĭ":108975,"åıijçĥŃ":108976,"èĮĥçķ´":108977,"çļĦéģĵè·¯":108978,"éĩijèī²":108979,"ä»ĸåıĪ":108980,"ä¼ļ产çĶŁ":108981,"æ°ijåĽ½":108982,"å®ĺæĸ¹ç½ijç«Ļ":108983,"æĶ¶çĽĬçİĩ":108984,"çļĦåΰæĿ¥":108985,"çļĦåĬŀæ³ķ":108986,"æĶ¹åζ":108987,"ä¸ĩç§ij":108988,"ä¸įäºĪ":108989,"è¿ĻäºĽéĹ®é¢ĺ":108990,"çαä¸Ĭ":108991,"çIJĥåľº":108992,"责令":108993,"æİĪ课":108994,"åľ¨é¦Ļ港":108995,"ç»Ĩèħ»":108996,"å¤ļä¸ĩ":108997,"åIJĮå¹´":108998,"大使":108999,"æĸĭ":109000,"ä¹Łä¸º":109001,"æĥłå·ŀ":109002,"åIJī祥":109003,"çͰåĽŃ":109004,"åĽ½å®¶éĺŁ":109005,"éĩįçĶŁ":109006,"åľ¨åħ¶":109007,"é¦Ļåij³":109008,"è´Łèį·":109009,"亲åĪĩ":109010,"èĩªè±ª":109011,"没éĶĻ":109012,"åĽłä¸ºåľ¨":109013,"æĺŁæĺŁ":109014,"éĤij":109015,"è¿ĺæľīå¾Īå¤ļ":109016,"æij©æīĺ":109017,"æij©æīĺ车":109018,"æŃ¥è¡Į":109019,"管çIJĨä½ĵç³»":109020,"èĦļä¸ĭ":109021,"éģİåİ»":109022,"æ±īè¯Ń":109023,"对ä¸įèµ·":109024,"çļĦç»ıåİĨ":109025,"åıĬ缸åħ³":109026,"ä¸įå°ij人":109027,"éĩįç£ħ":109028,"åĬ³åĬ¨èĢħ":109029,"大åĬĽåıijå±ķ":109030,"æĢİä¹Īåģļ":109031,"çĭĹçĭĹ":109032,"举åįĹäºļ":109033,"åĭĩäºİ":109034,"åħ¬éĸĭ":109035,"çĵ·çłĸ":109036,"åıĤçħ§":109037,"广æĴŃç͵è§Ĩ":109038,"举åĬ¨":109039,"æ±Łè¥¿çľģ":109040,"æķĪèĥ½":109041,"å͝æľī":109042,"éĿ¢è²Į":109043,"èĩªåĬ¨é©¾é©¶":109044,"æ¦ľåįķ":109045,"å½ĵæĪij们":109046,"仲è£ģ":109047,"æľ¨æĿIJ":109048,"ç±³åħ°":109049,"çϽéĵ¶":109050,"çļĦ人éĥ½":109051,"å°±åĥıæĺ¯":109052,"æŃ¥åħ¥":109053,"åįłç͍":109054,"åĩ»è´¥":109055,"让大家":109056,"ä¼ļè®©ä½ł":109057,"åİ¿æĶ¿åºľ":109058,"è¦ģç͍":109059,"çŃīå½¢å¼ı":109060,"åįĩé«ĺ":109061,"责任æĦŁ":109062,"å¤ĩç͍":109063,"ä»ĸ认为":109064,"æ¸ħåįİ大åѦ":109065,"ä»ĸèĩªå·±":109066,"éĸ±è®Ģ":109067,"太平æ´ĭ":109068,"éĶģå®ļ":109069,"çŃĨ":109070,"è¿Ļçīĩ":109071,"æī§æĶ¿":109072,"è¿ĶåĽŀæIJľçĭIJ":109073,"å°±æŃ¤":109074,"éģĩåΰäºĨ":109075,"å¼Ģå¹ķå¼ı":109076,"管çIJĨéĥ¨éŨ":109077,"å§¿åĬ¿":109078,"设æĥ³":109079,"åĽĽåŃ£":109080,"æĬĢæľ¯äººåijĺ":109081,"å·®çĤ¹":109082,"è¾ŀèģĮ":109083,"èĢģ師":109084,"çļĦæĦŁåıĹ":109085,"ä¹ŁéĿŀ常":109086,"å¹´ä¸ĬåįĬå¹´":109087,"æĢªçī©":109088,"èĮĥæĸĩ":109089,"æĪĺå½¹":109090,"åIJ«ä¹ī":109091,"åħ¨è¿ĩç¨ĭ":109092,"èĢĮéĿŀ":109093,"éĢļ讯åijĺ":109094,"è¿Ļæł·æīįèĥ½":109095,"æľºç»Ħ":109096,"è£ı":109097,"çķ¶çĦ¶":109098,"èµĮåįļ":109099,"åIJĦæľī":109100,"å·¥ä½ľæľºåζ":109101,"äºĭåIJİ":109102,"åī§éĻ¢":109103,"å±ĬæĹ¶":109104,"åĺ´éĩĮ":109105,"主线":109106,"ä¸ĢåľĪ":109107,"主è¦ģåİŁåĽł":109108,"å°¸ä½ĵ":109109,"åĮ»çĸĹåĻ¨æ¢°":109110,"ä½łæĢİä¹Ī":109111,"ä½Ĩçͱäºİ":109112,"æĹ¶ç©º":109113,"çĶ·æľĭåıĭ":109114,"çĶľèľľ":109115,"é«ĺåľ°":109116,"æĻĸ":109117,"èĴIJéĽĨ":109118,"åĩĿèģļåĬĽ":109119,"å¤ĩåıĹ":109120,"æĸĩåĪĽ":109121,"马æĿ¥":109122,"马æĿ¥è¥¿äºļ":109123,"æŁ´æ²¹":109124,"使人":109125,"æķĻä¼ļ":109126,"ç§ĭ天":109127,"æĺİçıł":109128,"åħŃåįģ":109129,"çݯå¢ĥä¸Ń":109130,"æ¸ħæĻ¨":109131,"积æŀģåıĤä¸İ":109132,"å·ħå³°":109133,"ä¸ºæľŁ":109134,"çѾåŃĹ":109135,"æĦŁæ¿Ģ":109136,"ç§ĭåŃ£":109137,"æĿijåŃIJ":109138,"æ¢ħ西":109139,"æļ´éĽ¨":109140,"çĶŁæ´»åľ¨":109141,"çªĹæĪ·":109142,"æģ¶åĬ£":109143,"纯粹":109144,"åľ¨æİ¥åıĹ":109145,"没èĥ½":109146,"è¡Į人":109147,"åĭº":109148,"æĭ¨æīĵ":109149,"ä½ľåĩºäºĨ":109150,"çļĦ主é¢ĺ":109151,"æľªä¾Ĩ":109152,"ä¸ŃæľĢ":109153,"æ¾ľ":109154,"é«ĺè¡Ģåİĭ":109155,"åħ´èµ·":109156,"æŃ£èĥ½éĩı":109157,"åŁ¹è®ŃçıŃ":109158,"æİ¥åħ¥":109159,"çĦ¶åIJİåĨį":109160,"åѦçĶŁä»¬":109161,"é¢ĨåħĪçļĦ":109162,"çģ«çĥŃ":109163,"ä¸ĵèģĮ":109164,"æĪĸèĢħ说":109165,"建è¨Ń":109166,"é»ı":109167,"对åħ¬åı¸":109168,"çľīçļĦ":109169,"åħīèį£":109170,"å½ĵåľº":109171,"éĿ¢åŃIJ":109172,"èµĦ产管çIJĨ":109173,"æĹ¶æľŁçļĦ":109174,"çŀİ":109175,"åįİ举":109176,"åıĪä¸Ģ次":109177,"èĥİåĦ¿":109178,"å®ļçĤ¹":109179,"头çĹĽ":109180,"æ¶²ä½ĵ":109181,"æĺ¯ä¸Ģä½į":109182,"帽åŃIJ":109183,"å¹´èµ·":109184,"ä¸įä½İäºİ":109185,"è¾ĥå°ij":109186,"éĿ¢ä¸´çĿĢ":109187,"å±Ĥå±Ĥ":109188,"èĿ´èĿ¶":109189,"èī°èĭ¦":109190,"éĺ¿æł¹":109191,"éĺ¿æł¹å»·":109192,"æ¦Ĥæĭ¬":109193,"请éĹ®":109194,"èµ·åºĬ":109195,"å±Ģå±Ģéķ¿":109196,"稳åģ¥":109197,"å¦ĤæŀľæĪij们":109198,"éħĴç²¾":109199,"æĪ·åı£":109200,"æĦŁæĤŁ":109201,"æĪij们éľĢè¦ģ":109202,"æĬĢèīº":109203,"èĩªåªĴä½ĵ":109204,"è¿ĽåĮĸ":109205,"æ¿ĢçĥĪçļĦ":109206,"ä½ĵ温":109207,"èļķ":109208,"èĩ´è¾ŀ":109209,"宪æ³ķ":109210,"ä¸ĢçŃīå¥ĸ":109211,"çĵ¶é¢Ī":109212,"æĥłæ°ij":109213,"èµ°è·¯":109214,"çݰ任":109215,"åķĨéĩı":109216,"ä¸ĭ车":109217,"åĪł":109218,"責任":109219,"èŀįåIJĪåıijå±ķ":109220,"ç´łæĿIJ":109221,"油价":109222,"åģļ人":109223,"çŀª":109224,"æĶ¹éĿ©åĪĽæĸ°":109225,"çļĦåĮºåĪ«":109226,"è·¨å¢ĥç͵åķĨ":109227,"æ¶īåıĬåΰ":109228,"æīĺ管":109229,"æĪijè¿ĺæĺ¯":109230,"åĿIJæłĩ":109231,"ç½ij讯":109232,"å½ĵåľ°çļĦ":109233,"追溯":109234,"åľŁè̳":109235,"åľŁè̳åħ¶":109236,"åºķä¸ĭ":109237,"åĩłåįģå¹´":109238,"ç©¿è¿ĩ":109239,"çĶŁæĢģæĸĩæĺİ":109240,"æİ¨èĸ":109241,"æİ¨èĸ¦":109242,"éłĨ":109243,"åĴ³åĹ½":109244,"åĪĨæĪIJ":109245,"çĹķ迹":109246,"æĪ·ç±į":109247,"éĥ½ä¸įèĥ½":109248,"æĻļä¼ļ":109249,"åĢ©":109250,"ä½ĵåĬĽ":109251,"è¿Ļ个èģĮä¸ļ":109252,"æĹłå½¢":109253,"åıªæĥ³":109254,"è¿Ľåıĸ":109255,"æĿ̿ѻ":109256,"èĦĬ":109257,"äºijåįĹçľģ":109258,"æľªçŁ¥":109259,"ç¾İèģĶ":109260,"ç¾İèģĶåĤ¨":109261,"å¤ĸå½¢":109262,"诱æĥij":109263,"çĽ£":109264,"è¡Į使":109265,"åłĨ积":109266,"çĨŁç»ĥ":109267,"éĺIJè¿°":109268,"æľĢ大éĻIJ度":109269,"å·¡æŁ¥":109270,"夺åĨł":109271,"ä¼ģä¸ļæĸĩåĮĸ":109272,"çĭ®åŃIJ":109273,"ä¿Ŀå®Ī":109274,"ä¸ºæł¸å¿ĥçļĦ":109275,"æī©æķ£":109276,"åζéĢłåķĨ":109277,"æŁĶ软":109278,"为ä¸Ģä½ĵçļĦ":109279,"游çİ©":109280,"çĶŁçĹħ":109281,"幫åĬ©":109282,"åͱæŃĮ":109283,"æīįåı¯ä»¥":109284,"宽æĿ¾":109285,"è¦ģæ¯Ķ":109286,"æĺ¯æĢİæł·":109287,"çģ°èī²":109288,"çİĭåĽ½":109289,"æIJħæĭĮ":109290,"计éĩı":109291,"åij¨åĽ´çļĦ":109292,"æĻºèĥ½æīĭæľº":109293,"常åĬ¡":109294,"常åĬ¡åī¯":109295,"é©´":109296,"å°Ĩè¿ij":109297,"寻常":109298,"ä¸ŃåĽ½å¸Ĥåľº":109299,"容åύ":109300,"å±±ä¸Ĭ":109301,"èĥĮåIJİçļĦ":109302,"亲å¯Ĩ":109303,"æīĢ以说":109304,"éİ®":109305,"çļĦçIJĨçͱ":109306,"大åŁİå¸Ĥ":109307,"常年":109308,"æĹħ游ä¸ļ":109309,"å°±æĺ¯è¿Ļæł·":109310,"åĨįæĿ¥":109311,"é«ĺä½į":109312,"åĨħ饰":109313,"æŀĦéĢł":109314,"ä¸Ģèµ·æĿ¥":109315,"çͳè«ĭ":109316,"å·²ç»ıå¼Ģå§ĭ":109317,"çļĦåĬ¨ä½ľ":109318,"被迫":109319,"éģįå¸ĥ":109320,"åīĸæŀIJ":109321,"å°ıäºĭ":109322,"å¿ĥä¸ŃçļĦ":109323,"ä½ĵåζæĶ¹éĿ©":109324,"çļĩå®¶":109325,"æķĻåłĤ":109326,"åIJĥå®Į":109327,"åĽ½æ°ijåħļ":109328,"æĺİç¡®äºĨ":109329,"åıijå±ķè§ĦåĪĴ":109330,"第ä¸ĢæŃ¥":109331,"å¾Ĺèµ·":109332,"åľ¨åĵª":109333,"çļĦè·¯ä¸Ĭ":109334,"é»Ķ":109335,"çķ¶æĻĤ":109336,"大åĬĽæĶ¯æĮģ":109337,"åıĮéĩį":109338,"çŁ¥éģĵèĩªå·±":109339,"åIJĪä½ľåįıè®®":109340,"æ°ĶåĬ¿":109341,"éķ¿æķĪæľºåζ":109342,"ç½ķè§ģ":109343,"åĽŀæĿ¥äºĨ":109344,"ä»ĸä¼ļ":109345,"ä¸Ńæĸ°":109346,"ä¸Ńæĸ°ç½ij":109347,"çļĦåķĨåĵģ":109348,"èµłéĢģ":109349,"決å®ļ":109350,"å¸ĤåľºçĽij管":109351,"çķĻåѦçĶŁ":109352,"ç͵åİĭ":109353,"äºļ马":109354,"äºļ马éĢĬ":109355,"è¿ĺæĺ¯æ¯Ķè¾ĥ":109356,"ä¿ĥè¿ĽäºĨ":109357,"æµģåħ¥":109358,"æijĦåĥı":109359,"æijĦåĥı头":109360,"æıIJåıĬ":109361,"åıijæİĺ":109362,"æī¾åĩº":109363,"æ¢Ŀä»¶":109364,"ç¹¼çºĮ":109365,"æĪijåĸľæ¬¢":109366,"å¥İ":109367,"æ¦ľæł·":109368,"å¼ĢèĬ±":109369,"æ²īéĩį":109370,"åŁºåĩĨ":109371,"ä»ħä»ħæĺ¯":109372,"轨éģĵ交éĢļ":109373,"åĶIJå±±":109374,"çŃīä¸Ģç³»åĪĹ":109375,"ä¸įè¿ĩæĺ¯":109376,"åŃĺåľ¨çĿĢ":109377,"èĬ±çĶŁ":109378,"夷":109379,"ç»Īç©¶":109380,"ä¹Łæĺ¯ä¸Ģ个":109381,"åįģåŃĹ":109382,"èĸªéħ¬":109383,"伤å¿ĥ":109384,"æĺ¥ç§ĭ":109385,"åĨ·åį´":109386,"ç²¾çģµ":109387,"çļĦåľ°åĽ¾":109388,"æ¯Ķçī¹":109389,"æ¯Ķçī¹å¸ģ":109390,"æĢ§åĪ«":109391,"ä½Ļä¸ĩåħĥ":109392,"ä¸įå¿ĺåĪĿå¿ĥ":109393,"å¿ĥçĸ¼":109394,"æĽ²çº¿":109395,"é«ĺä½İ":109396,"è¦ıå®ļ":109397,"æĻ¯èī²":109398,"è¦ģ说":109399,"åħ¬åı¸å°Ĩ":109400,"æ¶²åİĭ":109401,"è¿Ŀ约":109402,"åİļ度":109403,"åºŀ大çļĦ":109404,"è¿ĺæĺ¯å¾Ī":109405,"é¦ĸåħĪæĺ¯":109406,"çµ²":109407,"åĬ¡å®ŀ":109408,"並ä¸Ķ":109409,"å¢ŀè¿Ľ":109410,"ç»Ħç»ĩå¼Ģå±ķ":109411,"èµ·æĿ¥äºĨ":109412,"è¾ĥå°ı":109413,"导游":109414,"ä¸¤åľ°":109415,"ç¿ĺ":109416,"çģ¿çĥĤ":109417,"é£İéĩĩ":109418,"æĶ¯çº¿":109419,"æĶ¯çº¿ä»»åĬ¡":109420,"娱ä¹IJåľĪ":109421,"天津å¸Ĥ":109422,"åĮħåĽ´":109423,"æľ¬èµĽåŃ£":109424,"éĩįè¦ģ讲è¯Ŀ":109425,"åıĮåIJij":109426,"åįİ丽":109427,"éͤ":109428,"åĦ¿å¥³":109429,"åįĸåĩº":109430,"ä¾Ĩ說":109431,"ä»ĭç»įä¸Ģä¸ĭ":109432,"åIJ¦è®¤":109433,"åĭĿ":109434,"æĻ®éĢļ人":109435,"çļĦåĬ¨åĬĽ":109436,"涨åģľ":109437,"åŁºéĩij管çIJĨ":109438,"ä¸Ģ个éĩįè¦ģ":109439,"è¿IJæ²³":109440,"çħŀ":109441,"è´¢æĶ¿éĥ¨":109442,"è¡Įä¸ļåįıä¼ļ":109443,"éĥ½å°Ĩ":109444,"è¨Ģ论":109445,"ä¸ĭä¾Ĩ":109446,"墨西":109447,"墨西åĵ¥":109448,"åĽłä¸ºä»ĸ们":109449,"æĢİä¹ĪåĽŀäºĭ":109450,"åĬłå¤§å¯¹":109451,"èĬŃ":109452,"çīĮåŃIJ":109453,"ä¼ļ使":109454,"妹åŃIJ":109455,"ç«Ļéķ¿":109456,"å¿ħå¤ĩ":109457,"æłijæľ¨":109458,"æģ¶æĦı":109459,"æ²³éģĵ":109460,"å¯Įè£ķ":109461,"ç¹ģåįİ":109462,"ä»£è¡¨åĽ¢":109463,"æµij身":109464,"é¦ĸä½į":109465,"èĪªç©ºåħ¬åı¸":109466,"éĽ»å½±":109467,"ä¸ĵè¾ij":109468,"æ°´æºIJ":109469,"ä¸Ńæ¯Ĵ":109470,"並ä¸į":109471,"èĢĮåİ»":109472,"éĥĿ":109473,"äºİæŃ¤":109474,"æĸĩåĮĸ建设":109475,"èĤ¯å®ļä¼ļ":109476,"å¸ĮæľĽå¤§å®¶":109477,"æııåĨĻ":109478,"ä½İè°ĥ":109479,"æĸ°åħ´äº§ä¸ļ":109480,"æ·Ħåįļ":109481,"æĶ¾å¼Ģ":109482,"çļĦæĢ§æł¼":109483,"çĸ¾çĹħçļĦ":109484,"æķ´é¡¿":109485,"线ä¸Ĭ线ä¸ĭ":109486,"éĢī项":109487,"çļĦ认åı¯":109488,"æķ´é½IJ":109489,"çĶļä¹Ī":109490,"çľģåĨħ":109491,"åı¤äºº":109492,"æ°ijä¿Ĺ":109493,"çī¡ä¸¹":109494,"éŨçªĹ":109495,"éĤ£æł·çļĦ":109496,"çĽijäºĭä¼ļ":109497,"ç¿¡ç¿ł":109498,"禹":109499,"åįĥä¸ĩä¸įè¦ģ":109500,"æĶ¶ç¼©":109501,"çļĦæĸĩåŃĹ":109502,"åĴĮå°ļ":109503,"æĮĩ令":109504,"åħ±äº§åħļåijĺ":109505,"çļĦçĪ¶äº²":109506,"å®Įå·¥":109507,"åĬ¡å·¥":109508,"马æĭī":109509,"马æĭīæĿ¾":109510,"æµĭè¯Ħ":109511,"å²ļ":109512,"ä¸įåģļ":109513,"ä¸ĥå¹´":109514,"åĿĩä»·":109515,"主è§Ĥ":109516,"å¾Īä¸įéĶĻ":109517,"èĤ¡ä¸ľå¤§ä¼ļ":109518,"äºĶä¸Ģ":109519,"é£İåIJ¹":109520,"å¼Ģéĩĩ":109521,"è¿Ļä¹Ī大":109522,"èĥ½çľĭåΰ":109523,"èĢĥè¯Ħ":109524,"åį³ä¾¿æĺ¯":109525,"çݰ代åĨľä¸ļ":109526,"æ¯Ķè¾ĥé«ĺ":109527,"è¦ģçľĭ":109528,"没äºĨ":109529,"解決":109530,"çݯæ¯Ķ":109531,"åĨ²åĬ¨":109532,"æ·±å¤ľ":109533,"åĩłåįĥ":109534,"ä¿ı":109535,"ç½ijæ°ij":109536,"就没":109537,"ä»ĸ表示":109538,"éĩıåŃIJ":109539,"æĹ©é¤IJåĬłçĽŁ":109540,"åįĬå²Ľ":109541,"æIJŀç¬ij":109542,"ä¸ĬæĬ¥":109543,"審":109544,"é¢Ħ订":109545,"èľĤèľľ":109546,"æŁ¥æī¾":109547,"ä¼ĹæīĢ":109548,"ä¼ĹæīĢåij¨":109549,"ä¼ĹæīĢåij¨çŁ¥":109550,"æĹ©æĹ¥":109551,"åıijæī¬":109552,"åĴĮ个人":109553,"åĬłåħ¥äºĨ":109554,"åĸ®ä½į":109555,"åĪĨæĺİ":109556,"第ä¸Ģæī¹":109557,"ç¾İåĨĽ":109558,"æĿĢæīĭ":109559,"éŨå¤ĸ":109560,"åķĨåľĪ":109561,"ä¸ĢåĪ»":109562,"çļĦçľ¼ç¥ŀ":109563,"éľĦ":109564,"äºĽä»Ģä¹Ī":109565,"åĬłæ·±":109566,"æ¯ıä½į":109567,"å¸ĤéĿ¢ä¸Ĭ":109568,"åıĶåıĶ":109569,"çļĦéĤ£ç§į":109570,"粤港澳":109571,"è´´å¿ĥ":109572,"æĸĩåĮĸ产ä¸ļ":109573,"红æĹĹ":109574,"åĺīåħ´":109575,"æĶ¶çĽĺ":109576,"å®ĮæĪIJåIJİ":109577,"ä¼ģä¸ļ管çIJĨ":109578,"纵横":109579,"ä¸įä¿¡":109580,"æĪIJéĥ½å¸Ĥ":109581,"æ´Ĺ澡":109582,"举è¡ĮçļĦ":109583,"çĶ¢çĶŁ":109584,"ç©¿ä¸Ĭ":109585,"åĪļ好":109586,"åħī线":109587,"æīĵæŀ¶":109588,"è¿Ļæľ¬ä¹¦":109589,"åĶ®åIJİæľįåĬ¡":109590,"åĩłåĪĨ":109591,"ä¸Ĭ次":109592,"ä¸įåĪĨ":109593,"产åIJİ":109594,"éģ¿å¼Ģ":109595,"ç»Īæŀģ":109596,"代表大ä¼ļ":109597,"æ¼ĶæĬĢ":109598,"åĽŀè´Ń":109599,"åŃ¦è´¹":109600,"éĺ»ç¢į":109601,"ä¸Ģ大æī¹":109602,"竣工":109603,"åĨ³å®ļäºĨ":109604,"ä½Ĩå¦Ĥæŀľ":109605,"ç͵æµģ":109606,"ä¸Ŀ毫":109607,"èĥ½å¤Łåľ¨":109608,"éĶĢåĶ®æĶ¶åħ¥":109609,"åľ¨åŃ¦æł¡":109610,"æ°´åĩĨ":109611,"è§Ĩ线":109612,"èĩªåľ¨":109613,"åķĨä¸ļéĵ¶è¡Į":109614,"为äºĨ让":109615,"çį²å¾Ĺ":109616,"çݩ家æľĭåıĭ":109617,"éĿ¢èĨľ":109618,"åĪĨåī²":109619,"åī§æľ¬":109620,"ç«Ń":109621,"说å¾Ĺ":109622,"æĥ³çŁ¥éģĵ":109623,"çļĦ人çī©":109624,"èĮħåı°":109625,"åIJĮä¸Ģ个":109626,"æķ°æį®ä¸Ńå¿ĥ":109627,"çĶĦ":109628,"åĸľæĤ¦":109629,"ä¸ĭæĿ¥çļĦ":109630,"å®ļåIJij":109631,"æŀģåħ·":109632,"çļĦåľŁåľ°":109633,"éĤ£åĢĭ":109634,"æijĦåħ¥":109635,"äºĨæĪijçļĦ":109636,"马路":109637,"åħ¨ç¤¾ä¼ļ":109638,"è®®æ¡Ī":109639,"å±ĭåŃIJ":109640,"åIJįåı«":109641,"åĮª":109642,"åľ¨å¤ĸéĿ¢":109643,"åįİåįĹ":109644,"åıijè´§":109645,"å¯ĴåĨ·":109646,"é«ĺçŃīæķĻèĤ²":109647,"详ç»ĨçļĦ":109648,"ä¸ªé¡¹çĽ®":109649,"çĶŁäº§åĬĽ":109650,"æĹ¶å¸¸":109651,"å°±æľĥ":109652,"ä¸ĩèĤ¡":109653,"éĻĮçĶŁäºº":109654,"æııç»ĺ":109655,"å½ĵçĦ¶æĺ¯":109656,"æĭīåĬ¨":109657,"éĵ¾æĿ¡":109658,"æī£éϤ":109659,"ä¸Ģ缴éĥ½":109660,"å°ıåŃ©åŃIJ":109661,"伤åı£":109662,"第äºĮå±Ĭ":109663,"è´Ńç½®":109664,"çļĩ马":109665,"æĹłèģĬ":109666,"表åĨ³":109667,"诸å¦Ĥ":109668,"åĵįèµ·":109669,"é£İæļ´":109670,"ä¸ĢæµģçļĦ":109671,"ç·¨":109672,"è§£æĶ¾åĨĽ":109673,"室å¤ĸ":109674,"å°±è¿Ļä¹Ī":109675,"å³¶":109676,"æīĢæľī人éĥ½":109677,"æIJľç´¢å¼ķæĵİ":109678,"çļĦæĪIJæľ¬":109679,"åħļæĶ¿":109680,"åıijè¡Į人":109681,"çļĦäºĭå®ŀ":109682,"对该":109683,"åıĹæįŁ":109684,"ä¿Ħä¹Į":109685,"é²ľèĬ±":109686,"åĨľèį¯":109687,"æŀģéĢŁ":109688,"æĢ¥æĢ§":109689,"两ä¼ļ":109690,"ä¸ĢèάæĿ¥è¯´":109691,"æµ·é²ľ":109692,"åĨĪ":109693,"çĶ¨äºº":109694,"çĶ¨äººåįķä½į":109695,"åĢª":109696,"åĦªæĥł":109697,"æł¹æºIJ":109698,"åĽ¢è´Ń":109699,"ç¾İæ´²":109700,"ä¸ĭè¡Į":109701,"å¹´æľ«":109702,"èľ¡":109703,"è¯ģä»¶":109704,"åľ¨æĪijåĽ½":109705,"ä¸įåºĶ":109706,"æĮīæĹ¶":109707,"åłªç§°":109708,"åľºä¸Ĭ":109709,"å¹²éĥ¨èģĮå·¥":109710,"æľīå¾Ī大çļĦ":109711,"æķ°åŃĹç»ıæµİ":109712,"æ¼Ķç»ĥ":109713,"æį®ç»Łè®¡":109714,"å¾ĢæĿ¥":109715,"广åijĬæľįåĬ¡":109716,"çļĦè·Ŀ离":109717,"æŃ¸":109718,"è¨Ģè¯Ń":109719,"被èªī":109720,"被èªī为":109721,"åĭī强":109722,"å°Ĭæķ¬":109723,"ä¸ĩ亿åħĥ":109724,"ä¸ŃåĽ½åĽ½éĻħ":109725,"å¹²é¢Ħ":109726,"年产":109727,"èĢķåľ°":109728,"èĮİ":109729,"å᳿ĺ¯":109730,"æĺ¨æĻļ":109731,"æĪIJ为ä¸Ģ个":109732,"çºłæŃ£":109733,"åij½åIJį":109734,"é¢ģå¸ĥ":109735,"çĮľæµĭ":109736,"ä¿ĿèŃ·æĶ¿çŃĸ":109737,"æĭ¢":109738,"活泼":109739,"çŃīéĥ¨éŨ":109740,"åѦåΰ":109741,"å¢ŀå̼ç¨İ":109742,"èĪªçº¿":109743,"åĨ¤":109744,"åįģåĩłå¹´":109745,"æİ§èĤ¡èĤ¡ä¸ľ":109746,"ä¸ĢéŨ":109747,"ä¸ªå·¥ä½ľ":109748,"ä¸ªå·¥ä½ľæĹ¥":109749,"æĸ°è¥¿":109750,"æĸ°è¥¿åħ°":109751,"论è¯ģ":109752,"ä»Ĩ":109753,"åı¦å¤ĸä¸Ģ个":109754,"æĶ¹ç¼ĸ":109755,"严ç¦ģ":109756,"åĸľå¥½":109757,"个人信æģ¯":109758,"满æĦı度":109759,"åĵ¨":109760,"å¸ĪèµĦ":109761,"æĶ¹ä¸º":109762,"ç«ŀäºī对æīĭ":109763,"åĩºçĤī":109764,"åķĨ人":109765,"大æ£ļ":109766,"æĮĩ导ä¸ĭ":109767,"å¦ĩç§ij":109768,"輪":109769,"æīģ":109770,"åIJĮæĹ¶è¿ĺ":109771,"å¹¶éĢļè¿ĩ":109772,"æĪĺéĺŁ":109773,"èĶĵå»¶":109774,"ä¿ŀ":109775,"éĢĤå½ĵçļĦ":109776,"åīįè¾Ī":109777,"åĵģåij³":109778,"æ¹¿åľ°":109779,"æĪIJåŀĭ":109780,"ä¸įåıªæĺ¯":109781,"æĥ©ç½ļ":109782,"åĩºåı°äºĨ":109783,"çݩ游æĪı":109784,"æīįåıijçݰ":109785,"åºĶèģĺ":109786,"å¤ĸæĿ¥":109787,"åįłé¢Ĩ":109788,"å±ķæľĽ":109789,"å«Ĥ":109790,"港èĤ¡":109791,"æ¡Įä¸Ĭ":109792,"æĶ¯æŁ±":109793,"çļĦæĥħå½¢":109794,"广éĺĶçļĦ":109795,"æĶ¯è¡Į":109796,"å´©æºĥ":109797,"æľĪä¸Ń":109798,"æľĪä¸ŃæĹ¬":109799,"ç»įåħ´":109800,"临è¿ij":109801,"æĬ¤æłı":109802,"æļ®":109803,"åįķèģĮä¸ļ":109804,"è¾¹å¢ĥ":109805,"æĹ¥çħ§":109806,"ä¸ĢåłĨ":109807,"缴å¾Ħ":109808,"åħ±åIJĮä½ĵ":109809,"æĸ°åįİç½ij":109810,"æīĵ好":109811,"ç͵åĬ¨æ±½è½¦":109812,"ä¸įæĺİçϽ":109813,"éĢĻ裡":109814,"çĽĽå¤§":109815,"çİĭæľĿ":109816,"åĨįä¸Ģ次":109817,"åĬŀåħ¬åİħ":109818,"è´¨æĬ¼":109819,"åIJĪåĩ»":109820,"人们对":109821,"éĽ¶é£Ł":109822,"éĥ½ä¸įçŁ¥éģĵ":109823,"çļĦè¯Ńè¨Ģ":109824,"åĭŁéĽĨèµĦéĩij":109825,"åĬ¨èĦī":109826,"彤":109827,"è¿Ļåĩłå¹´":109828,"çŁŃè§Ĩé¢ij":109829,"太é«ĺ":109830,"常å§Ķä¼ļ":109831,"åĬłçıŃ":109832,"éĩįå¿ĥ":109833,"åªĴä½ĵæĬ¥éģĵ":109834,"没æ³ķ":109835,"éĹ»åIJį":109836,"çĥŃ度":109837,"å¹¿æ³ĽçļĦ":109838,"åħŃ大":109839,"çī©ä½ĵ":109840,"ä¸į该":109841,"é¢ĺ主":109842,"精彩çļĦ":109843,"ä¸ºè¿Ľä¸ĢæŃ¥":109844,"èĻŀ":109845,"åĽºçĦ¶":109846,"è´µå·ŀçľģ":109847,"çºłç»ĵ":109848,"代çIJĨ人":109849,"æ³ķå®ļ代表":109850,"åı¦ä¸Ģç§į":109851,"ä¸įåIJ«":109852,"æĭ¯æķij":109853,"ä¼ļç»Ļ":109854,"è¯Ĺè¯į":109855,"åIJĮç±»":109856,"å¾Ĺä¸įåΰ":109857,"æĬĵç´§":109858,"以åħ¶":109859,"åħ¥åħļ":109860,"è¿ĺåı¯":109861,"æľŁåĪĬ":109862,"å¾Īå¤ļæĹ¶åĢĻ":109863,"æĹ¥åIJİ":109864,"åħ¬çº¦":109865,"ä¸Ģ举":109866,"æ¯Ķè¾ĥå¤ļ":109867,"éĩijæ²Ļ":109868,"æįŀ":109869,"æİĴåĩº":109870,"æŃ¦æľ¯":109871,"ä¸įæĸ·":109872,"ä¸ŃèĢĥ":109873,"ä¿¡èµĸ":109874,"ä»İä¸ļ人åijĺ":109875,"çģ«çĦ°":109876,"éĨĴæĿ¥":109877,"ä½İ温":109878,"éĢ¾æľŁ":109879,"åĬ±å¿Ĺ":109880,"éħ¥":109881,"åı¯è°ĵæĺ¯":109882,"è¿ĻæĦıåij³çĿĢ":109883,"é¢łè¦Ĩ":109884,"åĮĹ京大åѦ":109885,"ä¸ĵ线":109886,"åıĬ以ä¸Ĭ":109887,"訪":109888,"èĢĮåIJİ":109889,"çŁ¥ä¹İ":109890,"ä¸Ģ对ä¸Ģ":109891,"å¨ĥå¨ĥ":109892,"çģ¾éļ¾":109893,"åħ¨å±Ģ":109894,"æīĢå¾Ĺç¨İ":109895,"å®ŀæĥł":109896,"èļĤèļģ":109897,"ä¹ŁçŁ¥éģĵ":109898,"温åĴĮ":109899,"èIJ½ä¸ĭ":109900,"åŀĭä¼ģä¸ļ":109901,"åĨįä¹Ł":109902,"ä¾ĽçĥŃ":109903,"é«ĺæ½®":109904,"çĢı覽åύ":109905,"çļĦ巨大":109906,"åħĪ天":109907,"å¹´ä¸ŃåĽ½":109908,"类似çļĦ":109909,"çIJĨäºĭä¼ļ":109910,"空éĸĵ":109911,"ç쵿ĦŁ":109912,"åĬĽæ°Ķ":109913,"带ä¸Ĭ":109914,"ä¸į好æĦıæĢĿ":109915,"æľīä½ķ":109916,"å·²åľ¨":109917,"åıĸåĩº":109918,"è¿Ŀæ³ķçĬ¯ç½ª":109919,"åŃ¦ä¹łè´¯å½»":109920,"åľ°å¸¦":109921,"楼梯":109922,"çŃīæĥħåĨµ":109923,"ä»İåīį":109924,"çļĦä¹łæĥ¯":109925,"ç³Łç³ķ":109926,"å°±èĥ½å¤Ł":109927,"è©ķ":109928,"ä¸Ģå¾ĭ":109929,"æĮ«æĬĺ":109930,"åİŁæĸĩåľ°åĿĢ":109931,"å½ĵå±Ģ":109932,"ä¸įéĢļ":109933,"æķ°åįĥ":109934,"éĺŁä¼į建设":109935,"æĹ¶èĬĤ":109936,"åģļèµ·":109937,"çļĦè®°å¿Ĩ":109938,"ç½ij绾å®īåħ¨":109939,"åĩ¡æĺ¯":109940,"æ°¯":109941,"éĽķåĪ»":109942,"åŁĥåıĬ":109943,"æĪijåı¯ä»¥":109944,"çĽijçIJĨ":109945,"æĽ´åħ·":109946,"åŁİ管":109947,"èĭ¯":109948,"åı¥åŃIJ":109949,"èĭ¥æľī":109950,"ä»İæĿ¥ä¸į":109951,"缸åħ³è´Łè´£":109952,"å®īåħ¨æĦŁ":109953,"æĽ´è¦ģ":109954,"çļĦæĥħæĦŁ":109955,"çī¢çī¢":109956,"è¾ĥ好çļĦ":109957,"æ°®":109958,"ç¬ijè¯Ŀ":109959,"车å±ķ":109960,"ä¹ĭç¾İ":109961,"ç®Ģ约":109962,"ç±»åŀĭçļĦ":109963,"èĢģåĮĸ":109964,"çľĭä½ł":109965,"è¿ĩåĪĨ":109966,"éŨåīį":109967,"ä¸ĢéĹ´":109968,"æĥ³åİ»":109969,"åªĽ":109970,"åľŁè±Ĩ":109971,"åıĪç§°":109972,"ä¸Ńä¿¡":109973,"åŃĺéĩı":109974,"马äºij":109975,"èĩ´ä½¿":109976,"åħĪåīį":109977,"èĢģåŃIJ":109978,"æīĵæī®":109979,"æ¯ķä¸ļäºİ":109980,"æ¯ķä¸ļåIJİ":109981,"ç¾İ好çĶŁæ´»":109982,"å·¥ä¸ļä¼ģä¸ļ":109983,"就好äºĨ":109984,"èħIJèļĢ":109985,"çıįçıł":109986,"åΰè¿ĻéĩĮ":109987,"æīĢéľĢçļĦ":109988,"è¿Ļæĺ¯åĽłä¸º":109989,"çIJĨæĥ³çļĦ":109990,"å·®å¼ĤåĮĸ":109991,"é®":109992,"é®®":109993,"äºļ太":109994,"æĹłç©·":109995,"æıIJçݰ":109996,"ä¸ĵä¸ļæĬĢæľ¯":109997,"çĶ¢æ¥Ń":109998,"åѦåŃIJ":109999,"ç§ijå¹»":110000,"åįłåľ°éĿ¢ç§¯":110001,"ä¸įåĩĨ":110002,"æľªæĪIJ年人":110003,"æĶ¶å½ķ":110004,"è¿ĺ款":110005,"éĴ¢çŃĭ":110006,"æ¼¢":110007,"å¾ĹæĦı":110008,"综åIJĪä½ĵ":110009,"æŀģé«ĺ":110010,"åįķè¯į":110011,"é«ĺæķĪçļĦ":110012,"骨头":110013,"æī§çĿĢ":110014,"缼ä¸ĸ":110015,"模çī¹":110016,"æĽ´èĥ½":110017,"ç»ĿæľĽ":110018,"对åºĶçļĦ":110019,"æ¨Ĭ":110020,"æĸ°ä¸ī":110021,"æĸ°ä¸īæĿ¿":110022,"æģ°æģ°":110023,"åIJįå®¶":110024,"æł¸å¿ĥæĬĢæľ¯":110025,"个å°ı":110026,"æĢİä¹Īä¼ļ":110027,"说ä¸įå®ļ":110028,"西çĵľ":110029,"åĵİ":110030,"ç¢Ł":110031,"å¿ħä¸įåı¯":110032,"å¿ħä¸įåı¯å°ij":110033,"ä¹ĭéĸĵ":110034,"åĪĨ管":110035,"交éĢļäºĭæķħ":110036,"å¼ĢåĬŀ":110037,"å¾ģæ±ĤæĦıè§ģ":110038,"亨":110039,"鼻åŃIJéĥµ":110040,"鼻åŃIJéĥµä»¶":110041,"ä¿¡æģ¯æľįåĬ¡":110042,"ä½łè§īå¾Ĺ":110043,"缴è§Ĥ":110044,"å·²å®ĮæĪIJ":110045,"åĪĨä¼ļ":110046,"åĽŀåįĩ":110047,"éļ»":110048,"好人":110049,"äºĨè§£ä¸Ģä¸ĭ":110050,"å᫿µ´":110051,"æľĢçα":110052,"åºŀ大":110053,"客æĪ¿":110054,"çijŀåħ¸":110055,"éĥ½ä¸įæĺ¯":110056,"館":110057,"èĹī":110058,"çļĦåIJĦ项":110059,"ä¸ºçĽ®æłĩ":110060,"çļĦè®¤çŁ¥":110061,"å½±åĵįåĬĽçļĦ":110062,"å¤¸å¼ł":110063,"佩æĪ´":110064,"æ±ĩçİĩ":110065,"çļĦçαæĥħ":110066,"æĺ¥é£İ":110067,"æĺ¯æĪijçļĦ":110068,"樹":110069,"åįĬå°ıæĹ¶":110070,"å±±åİ¿":110071,"山西çľģ":110072,"èĢĮè¿Ļ":110073,"æĽ´å¤ļä¿¡æģ¯":110074,"è¿ĺæľīä¸ĢäºĽ":110075,"ç²¾ç»ĨåĮĸ":110076,"ç¾İåѦ":110077,"çͱæĸ¼":110078,"ä»ħä¾ĽåıĤèĢĥ":110079,"å¾Īé«ĺçļĦ":110080,"åıłåĬł":110081,"è¿Ļä¹Ī说":110082,"å±ķåĩº":110083,"åĽĽå¤Ħ":110084,"ä¸ĩå®¶":110085,"æĭĽåĭŁ":110086,"çļĦ强大":110087,"æĤ£æľī":110088,"å°ıäºİ":110089,"ä¹Łè®¸æĺ¯":110090,"对èĩªå·±çļĦ":110091,"èģĮä¸ļæķĻèĤ²":110092,"æĿ¥è¿Ľè¡Į":110093,"档次":110094,"æīĵèµ¢":110095,"éĥ½æľīçĿĢ":110096,"庸":110097,"è¯Ńæ°Ķ":110098,"çͲéĨĽ":110099,"空åĨĽ":110100,"车åĨħ":110101,"åĽłä¸ºä½ł":110102,"å®ŀæķĪ":110103,"æĥħä¾£":110104,"åıijè¾¾åĽ½å®¶":110105,"éķľåŃIJ":110106,"æ¯įå©´":110107,"ä½Ĩæĺ¯ä»ĸ":110108,"积æŀģæİ¨è¿Ľ":110109,"大å¹ħ度":110110,"çļĦ女åĦ¿":110111,"é¤IJæ¡Į":110112,"åIJ¬å¾Ĺ":110113,"çļĦ积æŀģæĢ§":110114,"好åIJ§":110115,"æĹ¥æ¶Īæģ¯":110116,"æľīä»»ä½ķ":110117,"æ¯Ĵåĵģ":110118,"æĹ©çĤ¹åĬłçĽŁ":110119,"第ä¸Ģ天":110120,"å°½åĬĽ":110121,"æłĸ":110122,"主æīĵ":110123,"æĺ¯ä¸ĢåIJį":110124,"çĪĨæĸĻ":110125,"äºĭä¸ļåıijå±ķ":110126,"å¾®åķĨ":110127,"äºİä¸Ģä½ĵçļĦ":110128,"çĶŁçĮª":110129,"èĩªçĦ¶èµĦæºIJ":110130,"çŀĦåĩĨ":110131,"è§Ħ模åĮĸ":110132,"å¹¶ä¸İ":110133,"èĤ¥èĥĸ":110134,"å®¶ç͍":110135,"大çĪ·":110136,"é¢ĦåijĬ":110137,"æĿ¥åģļ":110138,"éĺ³åİ¿":110139,"æŀĦçŃij":110140,"é¢ģå¥ĸ":110141,"åİĨåı²æĸĩåĮĸ":110142,"æľįåĭĻæĪĸ":110143,"æĢ»åĨ³èµĽ":110144,"åıijåŀĭ":110145,"æĪij羣çļĦ":110146,"æĽ¦":110147,"åıĤä¼ļ":110148,"èĦĨå¼±":110149,"åĩĨåħ¥":110150,"èħ¹éĥ¨":110151,"åı¸ä»¤":110152,"æĤ²åī§":110153,"天ä¸Ĭ":110154,"åı£ä¸Ń":110155,"ä¸ĩ个":110156,"åѦä¸ļ":110157,"æıIJåĢ¡":110158,"两边":110159,"大èĤ¡ä¸ľ":110160,"åı¤éķĩ":110161,"è¡Ģç³ĸ":110162,"çļĦç¨ĭ度":110163,"æ£īèĬ±":110164,"åIJİåı°":110165,"å°±åĮ»":110166,"æķ´æķ´":110167,"èĴ²":110168,"çĽĪåĪ©èĥ½åĬĽ":110169,"ç±½":110170,"èĦ«":110171,"çľĭéĩį":110172,"å®¶éķ·":110173,"èģĺç͍":110174,"èµĽéģĵ":110175,"åīįèĢħ":110176,"建èѰ":110177,"å¾ĭå¸ĪäºĭåĬ¡":110178,"èīºæľ¯åĵģ":110179,"æľīèĩªå·±çļĦ":110180,"åIJ¦å®ļ":110181,"ç¤¾åĽ¢":110182,"åij¨äºĶ":110183,"带åΰ":110184,"å·¥ä½ľä¼ļè®®":110185,"èĤ¡æľ¬":110186,"å¤ĸåĮħ":110187,"å®¶åħ¬åı¸":110188,"çĽijçĭ±":110189,"èĪĬ":110190,"åIJįæł¡":110191,"西æ¹ĸ":110192,"è¶ħè¿ĩäºĨ":110193,"åįĹå±±":110194,"ç»Ħä»¶":110195,"å̼å¾Ĺ注æĦı":110196,"æĮ£æīİ":110197,"äºĭ迹":110198,"ç¶ĵçĩŁ":110199,"ç§ij室":110200,"好åIJĹ":110201,"æ¤ħåŃIJ":110202,"åľĪåŃIJ":110203,"ä½Ĩ她":110204,"æµģçķħ":110205,"åIJĦèĩªçļĦ":110206,"èģĮåijĺ":110207,"è¡įçĶŁ":110208,"åħ¨åľº":110209,"æĴ¤éĶĢ":110210,"åį´è¢«":110211,"å®ģéĿĻ":110212,"åīįæīĢ":110213,"åīįæīĢæľª":110214,"åīįæīĢæľªæľī":110215,"主ä¸ļ":110216,"åĮĹç¾İ":110217,"è¯Ħå®ļ":110218,"åĵģå°Ŀ":110219,"大家éĥ½åľ¨":110220,"主å¸ħ":110221,"ç»Ĩå¿ĥ":110222,"ä¿¡æģ¯æĬ«éľ²":110223,"çļĦç«ŀäºī":110224,"éĢĻæ¨£çļĦ":110225,"ç§ijåĪĽæĿ¿":110226,"éĩĩæijĺ":110227,"票æį®":110228,"éĢIJå¹´":110229,"èĭ±è¶ħ":110230,"è¡Įä¸ļåĨħ":110231,"人寿":110232,"åIJİåĭ¤":110233,"å¦ĤæĦı":110234,"ç¬Ķè¯ķ":110235,"æ·¡æ·¡çļĦ":110236,"ä¸įèĪĴæľį":110237,"ä½ĵ积":110238,"ä¹Łä¸įè¦ģ":110239,"éĿ¢æĸĻ":110240,"æł·æľ¬":110241,"ç¥ģ":110242,"æĮīè§Ħå®ļ":110243,"大æ¦Ĥæĺ¯":110244,"æĥħåĨµè¿Ľè¡Į":110245,"åIJĦåįķä½į":110246,"çļĦç¬ij容":110247,"åĩºèī²çļĦ":110248,"代表æĢ§":110249,"çļĦç¾İ好":110250,"éĴ¦":110251,"å¾®çĶŁçī©":110252,"è¶Ĭæĺ¯":110253,"æĸ¹åı¯":110254,"å¹²èĦĨ":110255,"éģĬæĪ²":110256,"çļĦåħ´è¶£":110257,"éĹ®è´£":110258,"åĽłä¸ºæĪij们":110259,"èĢĥéĩı":110260,"çĶŁçĶŁ":110261,"éĺ»åĬĽ":110262,"ä¸įåħģ许":110263,"æıIJè®®":110264,"åĩıæĮģ":110265,"åıªæĺ¯ä¸Ģ个":110266,"æĪijæĬĬ":110267,"åıijçݰèĩªå·±":110268,"å¢ŀå¹ħ":110269,"å¦į":110270,"èĹĿè¡ĵ":110271,"ä¸Ģ家人":110272,"åĪĨ级":110273,"çļĦæķ°éĩı":110274,"è½®èŀįèµĦ":110275,"çŃīåĽłç´ł":110276,"大夫":110277,"èģĺ请":110278,"é£İæľº":110279,"绽æĶ¾":110280,"ä»»ä½ķä¸Ģ个":110281,"éłĤ":110282,"éĺ¶çº§":110283,"æĬĬ她":110284,"è¿ĽåĨĽ":110285,"èĥ½åģļåΰ":110286,"åŁ¹è®ŃæľºæŀĦ":110287,"çĸĻ":110288,"ç«¥è¯Ŀ":110289,"æĮĩ导æĦıè§ģ":110290,"éĺ®":110291,"æ·±åħ¥æİ¨è¿Ľ":110292,"ä¸»æľº":110293,"æ¸Ķä¸ļ":110294,"ä¸įæľį":110295,"æµĵéĥģ":110296,"è¡Ĺä¸Ĭ":110297,"ä¾Ŀ次":110298,"æĹ¶æ®µ":110299,"梵":110300,"çļĦåĸľçα":110301,"å¾Īéķ¿":110302,"åĪĿ级":110303,"æŀľæĸŃ":110304,"æĬ¢æķij":110305,"é¼ĵèĪŀ":110306,"ä¾ĽéľĢ":110307,"æ·±åħ¥å¼Ģå±ķ":110308,"产ä¸ļéĽĨ群":110309,"åĻªéŁ³":110310,"åIJ¬çĿĢ":110311,"æ·±åĪ»çļĦ":110312,"å¿įåıĹ":110313,"ç͵ç£ģ":110314,"强èĢħ":110315,"æ»ĭåij³":110316,"æĽ¼èģĶ":110317,"åı¯ä»¥çĽ´æİ¥":110318,"大米":110319,"æŃ·åı²":110320,"æĶ¿åĬ¡æľįåĬ¡":110321,"åħ¬å¼ı":110322,"社群":110323,"éģĵ士èģĮä¸ļ":110324,"ä¹ĭæĥħ":110325,"æµ·æ°´":110326,"æ¼Ķå¥ı":110327,"åºĹéĩĮ":110328,"迹象":110329,"åıijå±ķçIJĨ念":110330,"é«ĺ空":110331,"åij¨åĪĬ":110332,"åĽŀåΰäºĨ":110333,"ä¸įéĢĤåIJĪ":110334,"åłµå¡ŀ":110335,"åĬĪ":110336,"æ°´ä¸Ĭ":110337,"çĢijå¸ĥ":110338,"纳ç¨İ人":110339,"çĩĥæ²¹":110340,"å·¥ç¨ĭé¡¹çĽ®":110341,"峡谷":110342,"æľīéĴĪ对æĢ§":110343,"åľĨå½¢":110344,"æľ¬å¸Ĥ":110345,"è¿Ļè¯Ŀ":110346,"管çIJĨèĢħ":110347,"ç¡®è¯ĬçĹħä¾ĭ":110348,"æĬĬæīĭ":110349,"彩èī²":110350,"ä¸Ĭåīį":110351,"夯å®ŀ":110352,"ç¾ĬèĤī":110353,"å¾Ģå¹´":110354,"æĵħèĩª":110355,"迷人":110356,"èĪªæ¯į":110357,"ç²¾ç»Ĩ":110358,"åľ¨æĪijçļĦ":110359,"åĪĽæĬķ":110360,"麦åħĭ":110361,"æľĪç»ı":110362,"åĮĹæµ·":110363,"ä¹ĭæĺŁ":110364,"åı¶åŃIJ":110365,"å¸Ĥåľºç«ŀäºī":110366,"è¿Ļäºĭ":110367,"åıĥèĪĩ":110368,"äº§åľ°":110369,"åĶī":110370,"åķĨåĵģæĪ¿":110371,"èĪªè¿IJ":110372,"ä¼ĺå¼Ĥ":110373,"ä»ĸ们æĺ¯":110374,"éĽ¨æ°´":110375,"è¯įæ±ĩ":110376,"åĨľçͰ":110377,"欧éĺ³":110378,"çŁŃ线":110379,"管ç½ij":110380,"æł¹åŁº":110381,"åıªæľīä¸Ģ个":110382,"éŀĭåŃIJ":110383,"å¸Ĥå§Ķ书记":110384,"åĪ»æĦı":110385,"è¡Į车":110386,"åıĪ被":110387,"åı¯éĿłæĢ§":110388,"è´±":110389,"ä»»åij½":110390,"åºĶåľ¨":110391,"å°±å¾Ĺ":110392,"æľįåĬ¡ä½ĵç³»":110393,"æĶ¿æĿĥ":110394,"åıijè¨Ģ人":110395,"è¿ĩå¾Ģ":110396,"两åıª":110397,"èĻ½è¯´":110398,"éĢģä¸Ĭ":110399,"ä»Ģä¹Īäºĭ":110400,"æķ£æĸĩ":110401,"æİĮæİ§":110402,"èĸĦå¼±":110403,"ä¸ĭéĿ¢å°±":110404,"主è¦ģåĨħ容":110405,"å¾Īéĩįè¦ģçļĦ":110406,"就说":110407,"çϽèī²çļĦ":110408,"éĤ£ä¸ªæĹ¶åĢĻ":110409,"ç»ı纪人":110410,"çļĦæ¯į亲":110411,"ç¬Ķè®°æľ¬":110412,"åºķå±Ĥ":110413,"è¿ij代":110414,"解说":110415,"è²łè²¬":110416,"æľĢ大åĮĸ":110417,"åķĨéĵº":110418,"æł¡åıĭ":110419,"æ²ģ":110420,"ä¸įåĩºæĿ¥":110421,"éĻ·éĺ±":110422,"ç¨ħ":110423,"åħ¬å¸ĥäºĨ":110424,"åĩĢå̼":110425,"çĽ¸å¯¹è¾ĥ":110426,"笼":110427,"æł¸ç®Ĺ":110428,"åįİ侨":110429,"æĢ¥æķij":110430,"æĮºå¥½":110431,"åħĴç«¥":110432,"äºĮèĥİ":110433,"åĩºèĩª":110434,"åĿŁ":110435,"æīĭä¸ĭ":110436,"屡":110437,"åĪĽéĢłæĢ§":110438,"ä¸¥æł¼æĮīçħ§":110439,"åĨįåİ»":110440,"举缣":110441,"人æµģ":110442,"äºĨä¸Ģ声":110443,"å°ıæĹ¶åīį":110444,"è´µæĹı":110445,"éľĸ":110446,"ä¹Łæĺ¯éĿŀ常":110447,"é̱":110448,"çľĭäºĨçľĭ":110449,"ç¹ģæ®ĸ":110450,"èĩ³æŃ¤":110451,"é¢Ħå¤ĩ":110452,"å¾Īæĺİæĺ¾":110453,"æ¼Ķèīº":110454,"åĿIJçĿĢ":110455,"ä¿ĦåĨĽ":110456,"åľ¨è¿ĩåİ»":110457,"ä¹ĭäºĭ":110458,"æĬĵèİ·":110459,"åĿIJä¸ĭ":110460,"çͱä¸ŃåĽ½":110461,"ä¹Łå¼Ģå§ĭ":110462,"çŃĶå¤į":110463,"åŀĥåľ¾åĪĨç±»":110464,"éĴĵé±¼":110465,"åIJĦ種":110466,"缸éģĩ":110467,"ä¸įåģľçļĦ":110468,"æī¹éĩı":110469,"éĩįè¦ģä½ľç͍":110470,"å§Ķå±Ī":110471,"åħŃå¹´":110472,"ä¸ĥåįģ":110473,"ä¹ĭæĪĺ":110474,"é£İéĻ©ç®¡çIJĨ":110475,"éŁ³æ¨Ĥ":110476,"è¡ĮæĶ¿å¤Ħç½ļ":110477,"æľ¬äºĭ":110478,"æĴ°åĨĻ":110479,"èģļåIJĪ":110480,"éĢĤæĹ¶":110481,"æIJ¬å®¶":110482,"ç¢İçīĩ":110483,"çĽĽå®´":110484,"ç®Ģæ´ģ":110485,"åı¬éĽĨ":110486,"ç®ĢåĮĸ":110487,"åĮĹ京æĹ¶éĹ´":110488,"第ä¸īå±Ĭ":110489,"æĿ¥åĽŀ":110490,"常ç͍çļĦ":110491,"京津":110492,"京津åĨĢ":110493,"梦幻":110494,"è¯ķè¡Į":110495,"æľºåºĬ":110496,"åΰæľĢåIJİ":110497,"åĬ©æīĭ":110498,"åĪĨ彩":110499,"åĩºåĵģ":110500,"åĪ¹è½¦":110501,"åIJ¯åıij":110502,"ä¾§éĿ¢":110503,"æ¯ıå½ĵ":110504,"缸åħ³è§Ħå®ļ":110505,"ä¸ĸ人":110506,"è´Ń车":110507,"å¿ĥ缮":110508,"å¿ĥ缮ä¸Ń":110509,"äºĶéĩij":110510,"è¿ĺè®°å¾Ĺ":110511,"ä¾ĿçĦ¶æĺ¯":110512,"æıIJæ¡Ī":110513,"ç͵åķĨå¹³åı°":110514,"åģļåΰäºĨ":110515,"æĿľç»Ŀ":110516,"å®īåįĵ":110517,"ä¸ĸçķĮåIJĦåľ°":110518,"åīįéĢĶ":110519,"æ´ĹåĩĢ":110520,"å¥ĭåĬĽ":110521,"åŁİå¸Ĥ建设":110522,"å¤ļåĬŁèĥ½":110523,"ä¼ļéĢłæĪIJ":110524,"åıijå¸ĥä¼ļä¸Ĭ":110525,"究竣æĺ¯":110526,"åĪĨ红":110527,"çŁ¥èŃĺ":110528,"éĿ¢æĿ¿":110529,"æĹłå£°":110530,"æĢ¥éľĢ":110531,"å¤±çľł":110532,"çΏå¦Ī":110533,"äºĤ":110534,"åħ¨æĻ¯":110535,"ç»ıåħ¸çļĦ":110536,"åī§ä¸Ń":110537,"é¢Ĩ导ä¸ĭ":110538,"åħļåĨħ":110539,"åħ¥ä¾µ":110540,"æĭīæĸ¯":110541,"ä¸Ģå¹ķ":110542,"åĬłä¹ĭ":110543,"èĤĨ":110544,"èĭ±æł¼":110545,"èĭ±æł¼åħ°":110546,"å·§åħĭ":110547,"å·§åħĭåĬĽ":110548,"ä¸Ģå¿ĥ":110549,"èģĤ":110550,"å¾Ģå¾Ģæĺ¯":110551,"管çIJĨå±Ĥ":110552,"çĻ»åħ¥":110553,"建ç«ĭèµ·":110554,"å»ºåĽ½":110555,"åŃIJ宫":110556,"åºĶä»ĺ":110557,"æİ¢ç©¶":110558,"第ä¸Ģä½į":110559,"ä½Ļå®¶":110560,"çŃīæ´»åĬ¨":110561,"æīĢèĩ´":110562,"è¾ĥå¿«":110563,"æĺ¯éĿŀ":110564,"æıIJåIJį":110565,"äºĮèĢħ":110566,"åıªåī©ä¸ĭ":110567,"åħ¶ä¸ŃåĮħæĭ¬":110568,"ç¼ĸç¨ĭ":110569,"çł´ç¢İ":110570,"ä¸Ń举":110571,"å·¥ä½ľæĬ¥åijĬ":110572,"çѾåIJį":110573,"éħĴä¸ļ":110574,"çŁ¥æĻĵ":110575,"çĥŃå¿ĥ":110576,"éĿŀåĩ¡":110577,"èIJ¥ä¸ļæī§":110578,"èIJ¥ä¸ļæī§çħ§":110579,"人大代表":110580,"ä¸Ģ个æĸ°çļĦ":110581,"å¨ģæµ·":110582,"éĤ£äºº":110583,"涨价":110584,"æ¶ĪçģŃ":110585,"éļ¾å¿ĺ":110586,"ç¶ĵé©Ĺ":110587,"åı£è¢ĭ":110588,"ç³»æķ°":110589,"æĸĩä¸Ń":110590,"好转":110591,"æĸ°éĽ¶åĶ®":110592,"讲述äºĨ":110593,"å¼ĢçĽĺ":110594,"çķĻç»Ļ":110595,"æħ¢æħ¢çļĦ":110596,"æĤ²ä¼¤":110597,"æľ¬æľŁ":110598,"äºĨå¤ļå°ij":110599,"è¿Ļ让":110600,"åIJĮçŃī":110601,"æ¸ħæĺİ":110602,"个åŁİå¸Ĥ":110603,"æºĸåĤĻ":110604,"åĩłä¹İæĺ¯":110605,"强åĬĽ":110606,"俯":110607,"水稻":110608,"åĽºå®ļçļĦ":110609,"æł¸åĩĨ":110610,"说æľį":110611,"顯示":110612,"è¿Ļå¥Ĺ":110613,"æĻºæħ§åŁİå¸Ĥ":110614,"å±ĭé¡¶":110615,"ä¸įæĿ¥":110616,"çĶŁé²ľ":110617,"çŁ¥æĥħ":110618,"æĬķ身":110619,"åijĬè¯īæĪij们":110620,"ä¸īåĽĽ":110621,"ä¸ĩä¸Ģ":110622,"è¾Ĩ车":110623,"为ä¹ĭ":110624,"åΰæĹ¶åĢĻ":110625,"è¿Ļæīįæĺ¯":110626,"åIJįçīĮ":110627,"åºŁæ°´":110628,"åݻ年åIJĮæľŁ":110629,"å¹´éĻIJ":110630,"éģĭåĭķ":110631,"åıĮçľ¼":110632,"è¦ģç´§":110633,"对çŃĸ":110634,"åľºé¦Ĩ":110635,"çϾç§ij":110636,"è¶Ĭéĩİ":110637,"å¯ĮåIJ«":110638,"大å¤ļæķ°äºº":110639,"æľĢå°ij":110640,"åı¬åͤ":110641,"åħ¸èĮĥ":110642,"åĨľæľº":110643,"æŃ£æĸĩ":110644,"åºĶç͍äºİ":110645,"æ·±èĢķ":110646,"ä¿Ń":110647,"ä»Ģä¹Īä¸ľè¥¿":110648,"å¥Ĺé¤IJ":110649,"å½ĵéĢī":110650,"å·¦æīĭ":110651,"è°ĥçIJĨ":110652,"æĻļé¤IJ":110653,"éļ¾åħ³":110654,"åĩŃè¯ģ":110655,"çĪ±äºº":110656,"æĮĩè´£":110657,"è´£ç¼ĸ":110658,"çļĦä¸Ģ款":110659,"éĵ²":110660,"åįģ个":110661,"èĢ»":110662,"æľįåĬ¡åķĨ":110663,"åľ°çĭ±":110664,"è¿ŀå¿Ļ":110665,"åĽ°æĥij":110666,"çļĵ":110667,"ä¸įåIJĥ":110668,"çİ°åľ¨å·²ç»ı":110669,"çĽĺçĤ¹":110670,"ä¸įåģľåľ°":110671,"管çIJĨ模å¼ı":110672,"è¿Ļ段æĹ¶éĹ´":110673,"椰":110674,"礼åĮħ":110675,"æµģ转":110676,"æī«çłģ":110677,"éĽĨä¸Ńåľ¨":110678,"æ±ĤåĬ©":110679,"åįĬ个":110680,"å¿«éĢŁå¢ŀéķ¿":110681,"å¾Ģä¸ĭ":110682,"è¯ĦåĪĨ":110683,"å°±æĥ³":110684,"åķĨåĬ¡éĥ¨":110685,"æľīéĹ®é¢ĺ":110686,"èİ·åĪ©":110687,"æ¯ĽçĹħ":110688,"æĦŁåºĶ":110689,"è̧":110690,"åĪĨæŃ§":110691,"åĨī":110692,"æĪij们çİ°åľ¨":110693,"è¦ģåĬłå¼º":110694,"å·§å¦Ļ":110695,"èŀºæĹĭ":110696,"åĪĩæį¢":110697,"çĭĦ":110698,"顺çķħ":110699,"å°¤åħ¶æĺ¯åľ¨":110700,"èĬĿ麻":110701,"éļ¾è¿ĩ":110702,"æĹĹå¸ľ":110703,"å¤įåį°":110704,"å¤įåį°ä»¶":110705,"å¿ħéľĢ":110706,"对å¤ĸå¼ĢæĶ¾":110707,"éļ¾åıĹ":110708,"åİŁæĿ¥æĺ¯":110709,"ç®ĹäºĨ":110710,"é«ĺå±±":110711,"离èģĮ":110712,"çµĦç¹":110713,"çµĦç¹Ķ":110714,"å±ģèĤ¡":110715,"çϾ家":110716,"éģĩä¸Ĭ":110717,"æĺĶæĹ¥":110718,"ä¸į容":110719,"çĽij管éĥ¨éŨ":110720,"主æĦı":110721,"æµģåŁŁ":110722,"è·Įå¹ħ":110723,"èĩ³ä¸Ĭ":110724,"åĪ«è¯´":110725,"æĺ¯æ¯Ķè¾ĥ":110726,"å®ıè§Ĥç»ıæµİ":110727,"å¸Ĥåľºä¸»ä½ĵ":110728,"污æŁĵçī©":110729,"æķijæ²»":110730,"丰æĶ¶":110731,"åŃĺæĶ¾":110732,"åĩĦ":110733,"éĩijå±±":110734,"æį¢äºĨ":110735,"ä¸ĵ人":110736,"éĹľæĸ¼":110737,"æĹ¢è¦ģ":110738,"åĽ½è¶³":110739,"éļĭ":110740,"åıįåĩ»":110741,"起身":110742,"åħĪæĺ¯":110743,"å¸ĮæľĽèĥ½å¤Ł":110744,"åĪ¶è®¢":110745,"åºĹéĿ¢":110746,"åĸĢ":110747,"æķĻä½ł":110748,"éĻ῏©":110749,"åĬĽæ±Ĥ":110750,"ä¸īçϾ":110751,"çī©ä»·":110752,"丢失":110753,"å¢Ļä¸Ĭ":110754,"éĥ¨ä»½":110755,"æł·æĿ¿":110756,"ä¹ĭæĦı":110757,"ç½ijå°ıç¼ĸ":110758,"ä¸ĸä¸Ĭ":110759,"è°ĥè¯ķ":110760,"污æŁĵéĺ²æ²»":110761,"å½±éĻ¢":110762,"å®Įåħ¨åı¯ä»¥":110763,"éĢļåħ³":110764,"ä¹īåĬ¡æķĻèĤ²":110765,"没æľīåĬŀæ³ķ":110766,"èĢ¿":110767,"妳":110768,"æĹłæĥħ":110769,"å¾ĹçĽĬ":110770,"å¾ĹçĽĬäºİ":110771,"æľŁçĽ¼":110772,"娱ä¹IJåľº":110773,"çͲæĸ¹":110774,"ä¸Ģæ±½":110775,"çŰ":110776,"çĸijä¼¼":110777,"æĸ°æµªå¾®åįļ":110778,"强è¡Į":110779,"å½ĵä»ĸ":110780,"èĥº":110781,"ç͍æĪ·æıIJä¾Ľ":110782,"åĮºå§Ķ":110783,"æĦ¿æĻ¯":110784,"æĬĺæī£":110785,"失踪":110786,"è¿«åĪĩ":110787,"åŃĹæ¯į":110788,"åĴ¯":110789,"èªįèŃĺ":110790,"ä»Ģä¹ĪæĦıæĢĿ":110791,"çĽĴåŃIJ":110792,"å½ķéŁ³":110793,"建设工ç¨ĭ":110794,"ä¸ļä½Ļ":110795,"å®ŀ践活åĬ¨":110796,"çľŁç©º":110797,"çĤĸ":110798,"åľ¨è·¯ä¸Ĭ":110799,"主è¦ģåĮħæĭ¬":110800,"该æĢİä¹Ī":110801,"æĢ»æľī":110802,"æĢ§æĦŁ":110803,"æ°ijèĪª":110804,"å¼ĢåºĹ":110805,"欺éªĹ":110806,"çªģåĩ»":110807,"缺失":110808,"æī§ä¸ļ":110809,"åľ°éģĵ":110810,"å¹¶æĹł":110811,"æ°ijåĬŀ":110812,"ç»Ħç»ĩçĶŁæ´»":110813,"æĪijå¦Ī":110814,"è¨ĺèĢħ":110815,"管åζ":110816,"æī¾ä¸ª":110817,"èĹ»":110818,"çĤİçĹĩ":110819,"äºĴåĬ©":110820,"æµıè§Īåύ":110821,"çݩ家æĿ¥è¯´":110822,"éĻįä½İäºĨ":110823,"è£Ķ":110824,"æĮ£éĴ±":110825,"åķĨæľº":110826,"æĶ¹è£ħ":110827,"æµģ浪":110828,"æĶ¿æ³ķ":110829,"èĢģ头":110830,"çĶŁäº§åĴĮ":110831,"ç©Ĺ":110832,"亲çα":110833,"亲çαçļĦ":110834,"å±¥èģĮ":110835,"åŁİéĩĮ":110836,"ç»ĨåĪĨ":110837,"åĬ³åĬ¨åIJĪåIJĮ":110838,"åľ¨æĹ¥æľ¬":110839,"å¨ģå°Ķ":110840,"åį«è§Ĩ":110841,"éĢ£çµIJ":110842,"çĿĢéĩį":110843,"æĬĺ磨":110844,"åĽ¾ä¸º":110845,"çľ·":110846,"å·¥åºı":110847,"æĵģ":110848,"æĵģæľī":110849,"ç½ijç«Ļåľ°åĽ¾":110850,"çļĦä¸Ģ大":110851,"ç»Ħç»ĩå®ŀæĸ½":110852,"æĬĽå¼ĥ":110853,"åĴĮæĶ¯æĮģ":110854,"æ³ķåĪĻ":110855,"浪潮":110856,"çݰæľīçļĦ":110857,"åĩłçİĩ":110858,"为客æĪ·":110859,"åįģä¸ĩ":110860,"è¹Ħ":110861,"çªģåĩºéĹ®é¢ĺ":110862,"åıĥåĬł":110863,"éĥ½ä¼ļæľī":110864,"缤":110865,"è°ģéĥ½":110866,"æīĭåĬ¨":110867,"çĽ´è¾¾":110868,"çĤ¹å¤ļ":110869,"éĺ¶å±Ĥ":110870,"ä¸įä½³":110871,"éĤ£æ®µ":110872,"滨海":110873,"æĺ¯åĽ½åĨħ":110874,"æĪijå¸ĮæľĽ":110875,"åIJĽåŃIJ":110876,"è§ĤéŁ³":110877,"åģļé¥Ń":110878,"æ±½è»Ĭ":110879,"åħ³ç¨İ":110880,"çľ¼åīįçļĦ":110881,"æ°´éĿ¢":110882,"èĢ³æľº":110883,"追踪":110884,"æİ¨éĢģ":110885,"éĴ±åĮħ":110886,"æģ¶å¿ĥ":110887,"æµ·åŁŁ":110888,"å·į":110889,"å¼ĢæĿ¥":110890,"表æĢģ":110891,"仪表":110892,"å¹³åİŁ":110893,"åįģå¤ļå¹´":110894,"ä¹ŁæĹłæ³ķ":110895,"åħ¼é¡¾":110896,"è¡£æŁľ":110897,"æł½åŁ¹":110898,"æĪ¿æºIJ":110899,"设ç«ĭäºĨ":110900,"ä¸ĩåIJį":110901,"æķ°é¢Ŀ":110902,"è¦ģåĿļæĮģ":110903,"åIJīæŀĹçľģ":110904,"请èģĶç³»":110905,"ç»ıåİĨè¿ĩ":110906,"çļĦæľ¬è´¨":110907,"åħ¥éŨ":110908,"æľ¬æ¡Ī":110909,"çİĩè¾¾åΰ":110910,"åı°éĺ¶":110911,"éĴŀ":110912,"æĪijèĥ½":110913,"èݲèĬ±":110914,"éĴł":110915,"ä¸Ģäºĭ":110916,"åİŁæľīçļĦ":110917,"æ¯ıåĢĭ":110918,"æ¯Ķäºļ迪":110919,"æ£ĭçīĮ游æĪı":110920,"ä¸įä¼ļæľī":110921,"å½ĴæĿ¥":110922,"äºĶçϾ":110923,"è¿ĩé«ĺ":110924,"éĽ·è¾¾":110925,"ä¸Ģèµ·åİ»":110926,"æķĻ导":110927,"å°±è¯Ĭ":110928,"å°±å¾Ī":110929,"ä¸įåIJĮäºİ":110930,"俺":110931,"å¸ĸåŃIJ":110932,"æĶ¿åįıå§Ķåijĺ":110933,"çĸ«æĥħå½±åĵį":110934,"åĪĨè£Ĥ":110935,"为ä»Ģä¹Īä¼ļ":110936,"äºĶæĺŁ":110937,"å°ijåĦ¿":110938,"æĬ¢éĻ©":110939,"梦è§ģ":110940,"è®°èĢħéĩĩ访":110941,"山路":110942,"æĪij个人":110943,"æ²Ļ滩":110944,"è¹Ń":110945,"æĶ¹è®Ĭ":110946,"æĸ°åŀĭåĨł":110947,"æĸ°åŀĭåĨłçĬ¶":110948,"åĮ»æĬ¤":110949,"åĮ»æĬ¤äººåijĺ":110950,"æµ·å°Ķ":110951,"åħ³äºİæĪij们":110952,"éϤå¤ĸ":110953,"åºļ":110954,"宣åijĬ":110955,"ä¸īåįĥ":110956,"榨":110957,"ç§ijæĬĢ大åѦ":110958,"ä¸ĥåħ«":110959,"顺åºĶ":110960,"çΏçΏå¦Īå¦Ī":110961,"éĢīåıĸ":110962,"åī§çĥĪ":110963,"乡æĿijæĹħ游":110964,"积æŀģæİ¢ç´¢":110965,"表çݰ为":110966,"å¾Īæ¸ħæ¥ļ":110967,"大åĨĽ":110968,"æĿ¥ç͵":110969,"å¥ĹæĪ¿":110970,"çݰè¡Į":110971,"享åıĹåΰ":110972,"çľĭçĤ¹":110973,"åĽºå®ļèµĦ产":110974,"以人为":110975,"ä»¥äººä¸ºæľ¬":110976,"ä¸įå®Į":110977,"éĻį鼨":110978,"åģļçļĦäºĭæĥħ":110979,"å¹¶äºİ":110980,"顽强":110981,"è̏":110982,"åĺ´å·´":110983,"缸åħ³ä¿¡æģ¯":110984,"æĪij没":110985,"æĪĺçķ¥æĢ§":110986,"æĢĿ念":110987,"åĪĺå¤ĩ":110988,"åĬ©æĶ»":110989,"é£İè²Į":110990,"éĿ¢å¯¹éĿ¢":110991,"积æŀģå¼Ģå±ķ":110992,"çĸĹæķĪ":110993,"çľĭ书":110994,"缺åı£":110995,"åĽ½æ°ijç»ıæµİ":110996,"使ç͍æĿĥ":110997,"éģ¥è¿ľ":110998,"å¡«è¡¥":110999,"第ä¸ī人":111000,"åįĬå¤ľ":111001,"æŃ¦æ±īå¸Ĥ":111002,"æĪijåıijçݰ":111003,"ä¼ĺæĥłæĶ¿çŃĸ":111004,"é£İåı£":111005,"å°±ä¸įèĥ½":111006,"为主è¦ģ":111007,"æµģåĩº":111008,"å´ĩæĭľ":111009,"å¹¶ä¸įèĥ½":111010,"é«ĺä¸ī":111011,"ä¸ĸçķĮä¸ĬæľĢ":111012,"æĥ³å¿ħ":111013,"åħ¶æīĢ":111014,"åĢĻéĢī":111015,"åĢĻéĢī人":111016,"ä¸įçα":111017,"åī¯ä½ľç͍":111018,"人æ°ijæĹ¥æĬ¥":111019,"æĪijä¸įæĺ¯":111020,"å®ŀçī©":111021,"ç͵åİĤ":111022,"ä¹Łç®Ĺæĺ¯":111023,"æľīéĹľ":111024,"æľīèĥ½åĬĽ":111025,"æĮĤåľ¨":111026,"çľ¼ä¸ĭ":111027,"约翰":111028,"å°ıåѦçĶŁ":111029,"èµ·åΰäºĨ":111030,"工夫":111031,"åIJĮå¿ĥ":111032,"åĿ¦è¨Ģ":111033,"çłĮ":111034,"åıijæĮ¥äºĨ":111035,"èģĮä¸ļéģĵå¾·":111036,"è¿ĻäºĽå¹´":111037,"念头":111038,"èĢģé¼ł":111039,"åħ¨èµĦ":111040,"åħ¨èµĦåŃIJ":111041,"ä¸Ģåij³":111042,"å¤ļä¸ĩåħĥ":111043,"æł¼æľĥ":111044,"éķ¿éĢĶ":111045,"带走":111046,"èĭ±å¯¸":111047,"æĸĩä½ĵ":111048,"对ä»ĸ们":111049,"åĵŃäºĨ":111050,"å¡«æĬ¥":111051,"çīĪæĿĥ声æĺİ":111052,"çĶµçº¿":111053,"è´Ńçī©ä¸Ńå¿ĥ":111054,"饱满":111055,"ä½İ头":111056,"强迫":111057,"ä¿Ŀæ´ģ":111058,"欧åĨł":111059,"缸è¿ŀ":111060,"认è´Ń":111061,"ç쫿ĺŁ":111062,"é«ĺå°Ķ":111063,"é«ĺå°Ķ夫":111064,"èij«èĬ¦":111065,"æłĩ注":111066,"çļĦçIJĨæĥ³":111067,"æł¸éħ¸":111068,"æł¸éħ¸æ£Ģæµĭ":111069,"åĬī":111070,"ä¸Ģèάæĺ¯":111071,"æĢĿç´¢":111072,"轨迹":111073,"çĥŃ带":111074,"éĻ£":111075,"åĩĨç¡®æĢ§":111076,"æĪ´çĿĢ":111077,"åľ¨çĶŁæ´»ä¸Ń":111078,"æīĢèĥ½":111079,"æľ¯åIJİ":111080,"å¸¦ä½ł":111081,"ç¥ł":111082,"æ®ĭéħ·":111083,"ä¹Łåıªæĺ¯":111084,"çͳè´Ń":111085,"举åĬŀäºĨ":111086,"æľīæĦıä¹ī":111087,"æĹºçĽĽ":111088,"åľ¨ç¶²":111089,"åľ¨ç¶²è·¯ä¸Ĭ":111090,"å¾Ī大ç¨ĭ度":111091,"管è¾ĸ":111092,"çĸ«æĥħæľŁéĹ´":111093,"触æij¸":111094,"éĺ¶æ®µæĢ§":111095,"ä¼ļè§īå¾Ĺ":111096,"çļĦçĶ»éĿ¢":111097,"æİ¥åıĹäºĨ":111098,"表达äºĨ":111099,"éĤĵå°ı":111100,"éĤĵå°ıå¹³":111101,"åħļé£İ":111102,"åħļé£İå»īæĶ¿":111103,"åķĨåѦéĻ¢":111104,"åħijæį¢":111105,"é£Łåĵģèį¯åĵģ":111106,"éĿŀ常好çļĦ":111107,"çľ¯":111108,"纳米":111109,"åĬ¨æijĩ":111110,"åĽŀéģ¿":111111,"çľĭèijĹ":111112,"款项":111113,"åħ«å¹´":111114,"åģļ个":111115,"æĸĩæ¡£":111116,"éĩijèŀįç§ijæĬĢ":111117,"åħ¶ä¸Ńæľī":111118,"äºĨä¸Ģç³»åĪĹ":111119,"æĹĹèΰåºĹ":111120,"ç§°èµŀ":111121,"éĽ¢éĸĭ":111122,"åζåĨ·":111123,"å®¶éŨåı£":111124,"åįģå¤ļ":111125,"ä¼´ä¾£":111126,"çľĭçĹħ":111127,"æĭīçĿĢ":111128,"æīĴ":111129,"çĸ²æĥ«":111130,"å°ijæķ°æ°ijæĹı":111131,"åĽ¾å½¢":111132,"è½§":111133,"å¢ŀéĩı":111134,"饲åħ»":111135,"çģ«å±±":111136,"æ¯ı个æľĪ":111137,"ä½ľä¸ºä¸ĢåIJį":111138,"è½´æī¿":111139,"æĸĩ书":111140,"ç¼ķ":111141,"åħ·ä½ĵæĥħåĨµ":111142,"çĹĽçĤ¹":111143,"缴éĶĢ":111144,"å¡Ĭ":111145,"ä¹Łæľĥ":111146,"çĥŃæ½®":111147,"å¹³æ°ij":111148,"æ¼Ķåͱä¼ļ":111149,"æķĻçłĶ":111150,"éĢĥéģ¿":111151,"ä¸Ģè´¯":111152,"å°±è¶Ĭ":111153,"å®ŀå®ŀåľ¨":111154,"å®ŀå®ŀåľ¨åľ¨":111155,"ä¹łè¿ijå¹³æĢ»":111156,"溺":111157,"å¿ĥåºķ":111158,"éķ¿å¾ģ":111159,"媽媽":111160,"第ä¸ī次":111161,"åĩºæ¼Ķ":111162,"çĭĢæ³ģ":111163,"å°Ķæĸ¯":111164,"代çIJĨåķĨ":111165,"çĨı":111166,"çļĦ对象":111167,"ç͵éĩı":111168,"è¡ĮåĪĹ":111169,"åĽ½äºº":111170,"è·ijäºĨ":111171,"åįĶåĬ©":111172,"èIJ¥è¿IJ":111173,"å¸ĪåħĦ":111174,"榮":111175,"æĥ³åĥı":111176,"æĢ§å¼º":111177,"ç§ijåѦçłĶç©¶":111178,"å»¶å®ī":111179,"ä¸¥æł¼èIJ½å®ŀ":111180,"é¢Ĩä¼ļ":111181,"çĽ¸å·®":111182,"路人":111183,"çĶ«":111184,"æľīä»·å̼":111185,"æľīä»·å̼çļĦ":111186,"ç¾İåĽ¢":111187,"æ°ij主çĶŁæ´»":111188,"æĪijæīį":111189,"ç¾İåĽ½äºº":111190,"æ°Ķåij³":111191,"åıįå°Ħ":111192,"çļĦåĨ³å¿ĥ":111193,"大è±Ĩ":111194,"交代":111195,"è¿Ľåĩº":111196,"åıįæĬĹ":111197,"æĮĩçļĦæĺ¯":111198,"ä»·ä½į":111199,"è¿Ľé©»":111200,"ä¸ĬçϾ":111201,"ä½įåĪĹ":111202,"ä¸ŃåĽ½ä¼ģä¸ļ":111203,"çļĦ好å¤Ħ":111204,"主ç¼ĸ":111205,"汽油":111206,"ä½ĨæĪij们":111207,"æĢİä¹Īçľĭ":111208,"é»Ħå±±":111209,"å¤ļåªĴä½ĵ":111210,"åIJİåį«":111211,"èİ·å¾ĹæĽ´å¤ļ":111212,"åĬ¡å¿ħ":111213,"为å¥ijæľº":111214,"é¦ĸ饰":111215,"ä¸ĩåįļ":111216,"è¶ĬæĿ¥è¶Ĭ大":111217,"ä¸ĵ项è¡ĮåĬ¨":111218,"å¥ĭè¿Ľ":111219,"ä»įçĦ¶æĺ¯":111220,"è´¨æĦŁ":111221,"å¦Ĥæŀľä¸įæĺ¯":111222,"ç«Ļèµ·æĿ¥":111223,"ä¹¾éļĨ":111224,"åı¯æĢķçļĦ":111225,"å¯Įè´µ":111226,"æ¸ħç®Ĺ":111227,"åIJijä¸ĭ":111228,"åĢļ":111229,"çļĦçŃĶæ¡Ī":111230,"èιä¸Ĭ":111231,"çļĦ羣å®ŀæĢ§":111232,"çŃīåĬŁèĥ½":111233,"åĸľåī§":111234,"å¨ģåĬĽ":111235,"æĸ°é¢ĸ":111236,"æł¸ç͵":111237,"æĬ¥éĶĢ":111238,"æķħ乡":111239,"ä¼´éļı":111240,"éŀŃ":111241,"å¦Ĭå¨ł":111242,"åĪĨåĮĸ":111243,"æľīå¾Ī大":111244,"æĢİä¹Ī说":111245,"æĻĤ代":111246,"产åĩº":111247,"ä»ĭç»į说":111248,"å¤ĦçIJĨåύ":111249,"èĨ¨èĥĢ":111250,"åī¯å¸Ĥéķ¿":111251,"çļĦ妻åŃIJ":111252,"æł·åĵģ":111253,"åIJĮæ¯Ķä¸ĭéĻį":111254,"åħĥå·¦åı³":111255,"ç͍èĩªå·±çļĦ":111256,"é«ĺéĽĦ":111257,"æĺ¥æĻļ":111258,"ä¹Łæľīå¾Īå¤ļ":111259,"çľ¼çIJĥ":111260,"æķ£æŃ¥":111261,"ä»ĸ们éĥ½":111262,"第ä¸Ģå®¶":111263,"åĬŀ好":111264,"å®īéĺ²":111265,"ä¸Ģä¸ĩ":111266,"åľ¨éĩĮéĿ¢":111267,"éŁ³é¢ij":111268,"åı£åı·":111269,"ä¸Ģè¶Ł":111270,"ç¦ıçī¹":111271,"é³ŀ":111272,"æĥĬèī³":111273,"æĸ°å¨ĺ":111274,"绿èī²åıijå±ķ":111275,"ä¸Ńå¼ı":111276,"ä¹Łåıªæľī":111277,"çݰ身":111278,"åı¯ä¾Ľ":111279,"æ¯ıä¸Ģ个人":111280,"第ä¸īèĢħ":111281,"åľ°å½¢":111282,"éĴ¢ç»ĵæŀĦ":111283,"çĽijçĿ£æ£ĢæŁ¥":111284,"åı«æĪij":111285,"èĩ´æķ¬":111286,"æ´Ĺæīĭ":111287,"ä¸ĭè°ĥ":111288,"康çĨĻ":111289,"æĪIJ交éĩı":111290,"ä¹ŁæĪIJ为":111291,"åħīæ»ij":111292,"å®Įæķ´æĢ§":111293,"çģ¼":111294,"ç¶²éłģ":111295,"éķ¿å¯¿":111296,"éģ©ç͍":111297,"çļĦä¸Ģ项":111298,"çŀ©çĽ®":111299,"æĬĬèĩªå·±çļĦ":111300,"éĵ¶è¡Įåį¡":111301,"å°±å¿ħé¡»":111302,"ç¾İçϽ":111303,"éŀįå±±":111304,"æľ¬é¢Ĩ":111305,"ä¸Ģç¢Ĺ":111306,"æīĵæ³ķ":111307,"æĤ¨å¥½":111308,"对åŃ©åŃIJ":111309,"æĬ¥éģĵç§°":111310,"ä¼łåĩº":111311,"大èĩ£":111312,"ç¬ĭ":111313,"çĽı":111314,"é¾ļ":111315,"çĽ´çº¿":111316,"æĻºåºĵ":111317,"ç§Łè½¦":111318,"é£İåij³":111319,"çľĭä¸Ģä¸ĭ":111320,"æİ¨éĶĢ":111321,"éĥ¨éĥ¨éķ¿":111322,"è´¨éĩıåĴĮ":111323,"åĪĬçĻ»":111324,"å·¥ä¸ļåĮĸ":111325,"çİĩ为":111326,"鼶件":111327,"硬åĮĸ":111328,"ä¸Ĭåįĥ":111329,"ç»ıéªĮå̼":111330,"å¹³è¡Į":111331,"声éģĵ":111332,"æľįåĬ¡è´¨éĩı":111333,"çĶŁçĶ¢":111334,"æľĢ容æĺĵ":111335,"ä¸Ģæŀļ":111336,"å¹´æĬ¥":111337,"åħ¬ç½ij":111338,"åħ¬ç½ijå®ī":111339,"åħ¬ç½ijå®īå¤ĩ":111340,"çļĦèĥ½éĩı":111341,"å®ŀéĻħè¡ĮåĬ¨":111342,"è¦ģä¸įè¦ģ":111343,"æĹ¥æľ¬äºº":111344,"èĢ¶ç¨£":111345,"ç¼ĸåī§":111346,"æ¶©":111347,"åį°å°¼":111348,"ä¸Ĭä¸ĭ游":111349,"åĩłåı¥":111350,"ä¸Ńéĵģ":111351,"ç°¡åĸ®":111352,"èĩªå¸¦":111353,"çĶŁäºİ":111354,"ä¸Ģåı£æ°Ķ":111355,"åĭ¤å¥ĭ":111356,"éĻįä»·":111357,"å±ķçݰäºĨ":111358,"å¸ĥæĭī":111359,"ä¼ļéĢīæĭ©":111360,"çļĦç»ıåħ¸":111361,"好æľĭåıĭ":111362,"车éģĵ":111363,"æķ´åĢĭ":111364,"åľĵ":111365,"éķ¿æľŁä»¥æĿ¥":111366,"æĬķå½±":111367,"çļĩåĨł":111368,"è¿ĩ大":111369,"åijĬè¯īä»ĸ":111370,"ä¼ģä¸ļæıIJä¾Ľ":111371,"æĬ½è±¡":111372,"éĢĤ度":111373,"çļĦ女åŃ©":111374,"èµ·ä¼ı":111375,"çļĦåĬŁæķĪ":111376,"ä¸ĵ项æķ´æ²»":111377,"åı¯éĢļè¿ĩ":111378,"ä¸įåIJĮç¨ĭ度":111379,"å¼Ĥè®®":111380,"åĩĢèµĦ产":111381,"åijĹ":111382,"ä»Ģä¹Īåij¢":111383,"å·¡éĢ»":111384,"è¸ıä¸Ĭ":111385,"ä½Ĩå®ĥ":111386,"精度":111387,"管å±Ģ":111388,"第ä¸ĢåIJį":111389,"åĨħåŃĺ":111390,"æijĨåľ¨":111391,"åī©ä¸ĭ":111392,"主ä½ĵ责任":111393,"çĤ¹åįĬ":111394,"以èĩ³äºİ":111395,"åħ»èĢģä¿ĿéĻ©":111396,"æĦŁåıĹåΰäºĨ":111397,"çŁ¥åIJįçļĦ":111398,"å¯Į豪":111399,"妥åĸĦ":111400,"åŃĻåŃIJ":111401,"éĵĤ":111402,"说èĩªå·±":111403,"让æĤ¨":111404,"æķ°æİ§":111405,"çļĦçľ¼åħī":111406,"注éĶĢ":111407,"çļĦçģµéŃĤ":111408,"è¿ĺä¸įéĶĻ":111409,"éĹ®ä»ĸ":111410,"èĩªä¸»çłĶåıij":111411,"èĵĭ":111412,"ç´«èī²":111413,"åĽ½å®¶å®īåħ¨":111414,"è¾½å®ģçľģ":111415,"ä¹Łæ¯Ķè¾ĥ":111416,"ç¾İèĤ¡":111417,"ä¸įç¡®å®ļæĢ§":111418,"å¿ĥ头":111419,"æĪ³":111420,"级åĪ«çļĦ":111421,"论述":111422,"çļĦåĽŀçŃĶ":111423,"ä¿Ŀè¯ģéĩij":111424,"çŃīè¡Įä¸ļ":111425,"幸ç¦ıæĦŁ":111426,"æŃ§è§Ĩ":111427,"æľºç¥¨":111428,"派人":111429,"èĩ´åij½":111430,"åĺ´è§Ĵ":111431,"æĸ°éĹ»ä¸Ńå¿ĥ":111432,"æĶ¾å¼ĥäºĨ":111433,"å®ľå±ħ":111434,"åĨĻä¸ĭ":111435,"éĹ®çŃĶ":111436,"è¿ĻéĩĮæĺ¯":111437,"å¤ļåľ°":111438,"åĮºåŁŁåĨħ":111439,"åĸ°":111440,"çľĭä»ĸ":111441,"æī§æ³ķ人åijĺ":111442,"åĬ¨æľº":111443,"éŁ³åĵį":111444,"çļĦåij½è¿IJ":111445,"é¡¶éĥ¨":111446,"åĵŁ":111447,"éĥ½æľĥ":111448,"æīĵéĢłæĪIJ":111449,"æĦıåĽ¾":111450,"çļĸ":111451,"åĢĴåħ¥":111452,"å·´èIJ¨":111453,"åĬ©åѦ":111454,"å¤įåı¤":111455,"åIJ¯ç͍":111456,"åĽ½éĻħå¸Ĥåľº":111457,"åĤ¨èĥ½":111458,"é»ijé¾Ļæ±Łçľģ":111459,"ä¹ĺ车":111460,"è¿IJåĬ¨ä¼ļ":111461,"ä¿ĿåĪ©":111462,"çŁ³æĿIJ":111463,"çµ®":111464,"çĤĴä½ľ":111465,"çļĦä¿¡ä»»":111466,"å°±æĪIJäºĨ":111467,"åı¯è§Ĥ":111468,"çļĩä¸Ĭ":111469,"è¿Ļåĩłå¤©":111470,"ä¸ĢéĶ®":111471,"åĨ·åĨ»":111472,"ä¿Ŀåį«":111473,"æł¸æ¡ĥ":111474,"åIJĪä½ľåħ³ç³»":111475,"éĢģåĩº":111476,"æĹĹä¸ĭçļĦ":111477,"åľ¨ä¹İ":111478,"为广大":111479,"åįĪé¤IJ":111480,"ä¸ĵ访":111481,"æĪĸå°Ĩ":111482,"éĿĴå²Ľå¸Ĥ":111483,"å¥Ķè·ij":111484,"æĹ¥æĬ¥éģĵ":111485,"å¥ijåIJĪ":111486,"æĸ°æĺ¥":111487,"ä¸įå°ıå¿ĥ":111488,"两ä¸ī":111489,"æĦıæĢĿæĺ¯":111490,"åĨ·èĹı":111491,"çļĦçĹĩçĬ¶":111492,"æĢ§åij½":111493,"è¶ħæłĩ":111494,"å¯Ĩ碼":111495,"ç§ijæĬĢèĤ¡ä»½":111496,"äºĨä¸Ģæī¹":111497,"çĿ£å¯Ł":111498,"åªĴä»ĭ":111499,"å°Ħæīĭ":111500,"ä¿®åħ»":111501,"çīĩåĪ»":111502,"éĢĤåIJĪèĩªå·±":111503,"åıªè¦ģæĺ¯":111504,"åIJĥè¿ĩ":111505,"éĩijéĵ¶":111506,"缴å±ŀ":111507,"åѦéĹ®":111508,"åİĭåζ":111509,"çªĹå¤ĸ":111510,"æĶ¶åΰäºĨ":111511,"åħ¨åĽ½äººå¤§":111512,"ä½Ĩæĺ¯å¯¹äºİ":111513,"åľ¨æķ´ä¸ª":111514,"çļĦèĥĮåIJİ":111515,"åĩıå°ijäºĨ":111516,"åıįèħIJ":111517,"åıįèħIJåĢ¡":111518,"åıįèħIJåĢ¡å»ī":111519,"æĹ·":111520,"åĪĨæľŁ":111521,"åľ¨æ·±åľ³":111522,"æīĵçĿĢ":111523,"æī«ä¸Ģ":111524,"æī«ä¸Ģæī«":111525,"æĶ¿åºľéĥ¨éŨ":111526,"æİ¥è¿ŀ":111527,"å±ŀäºİèĩªå·±":111528,"åŃIJå¼¹":111529,"åIJĮæł·æĺ¯":111530,"æĢ»åħ±":111531,"车ä¼ģ":111532,"æ¢ĵ":111533,"åħ¬é¡·":111534,"åıij声":111535,"éĴĽ":111536,"èµ°åĬ¿åĽ¾":111537,"主èIJ¥":111538,"åĸĶ":111539,"æķ°æį®åĪĨæŀIJ":111540,"ä¸įè¿ľ":111541,"æľīåIJį":111542,"æľīåIJįçļĦ":111543,"åģ¿è¿ĺ":111544,"å¾Īä½İ":111545,"è®ĵ人":111546,"èĿī":111547,"é«ĺè´µ":111548,"å°ij许":111549,"æ°Ł":111550,"å¹¢":111551,"亲æĥħ":111552,"è¿Ļä»¶äºĭæĥħ":111553,"ç͍é¤IJ":111554,"缸åħ³æĸ°éĹ»":111555,"å°±åºĶ该":111556,"ç»ĪçĤ¹":111557,"æĺ¯å¤ļå°ij":111558,"çĻ»åľº":111559,"è¯ķ管":111560,"è¯ķ管婴åĦ¿":111561,"åģļ大":111562,"åģļ大åģļ强":111563,"çļĦä¾ĭåŃIJ":111564,"åħ«ä¸ª":111565,"æĺİæĹ¥":111566,"çĤ³":111567,"èµ°åİ»":111568,"éģº":111569,"墩":111570,"ä½ĵä¼ļåΰ":111571,"åĴı":111572,"ä¸ĭè¾¾":111573,"å¤įåıij":111574,"追éĢIJ":111575,"æīĵåĵį":111576,"çļĦéļ±ç§ģæ¬Ĭ":111577,"åħ·æľīä¸Ģå®ļ":111578,"è¿Ļä¹Īå¤ļå¹´":111579,"æłijæŀĹ":111580,"æľĢéķ¿":111581,"åIJĮèĥŀ":111582,"åħīæ³½":111583,"åŁŁåIJį":111584,"æĮĩåIJij":111585,"åıĹ害èĢħ":111586,"æłijèĦĤ":111587,"æľīå¤ļ大":111588,"大éĿ¢ç§¯":111589,"æĹłç¼Ŀ":111590,"æĶ¹æŃ£":111591,"æĽ´å¤ļçļĦæĺ¯":111592,"æľŁæľ«":111593,"æŃ¼":111594,"ä¹īä¹Į":111595,"éĤ£ä½ł":111596,"çļĦ第ä¸Ģ个":111597,"èĮµ":111598,"å°§":111599,"èį«":111600,"ä¸įä»ħåı¯ä»¥":111601,"æ¶Įçݰ":111602,"æĢ»éĿ¢ç§¯":111603,"æĸ°éĹ»åıijå¸ĥ":111604,"æ°ijç͍":111605,"就读":111606,"æīĵè´¥":111607,"å¤ĸè¯Ń":111608,"æĪij们ä¸Ģèµ·":111609,"é¢Ħå®ļ":111610,"çĥ¹é¥ª":111611,"æľĢ主è¦ģ":111612,"æľĢ主è¦ģçļĦ":111613,"çīĮçħ§":111614,"åĽłåħ¶":111615,"ä½İä¸ĭ":111616,"ä¼ļåIJĮ":111617,"è§ģè§£":111618,"éĹ´éļĶ":111619,"æķĻç¨ĭ":111620,"å°ī":111621,"å¸Ĥä¸Ńå¿ĥ":111622,"åħ³éĶ®æĺ¯":111623,"æµ·åįĹçľģ":111624,"çī¹åĪ«æĺ¯åľ¨":111625,"ä¸ŃåĽ½å¤§éĻĨ":111626,"åħħè¶³çļĦ":111627,"æĹ¢èĥ½":111628,"åĤ³çµ±":111629,"çijľä¼½":111630,"åħ¥åĽ´":111631,"æħ¢æħ¢åľ°":111632,"æĬ¥éħ¬":111633,"æī¹å¤į":111634,"å·¥ä¸ļåĽŃåĮº":111635,"ä¸İåıijå±ķ":111636,"èĥ¸éĥ¨":111637,"åľ¨ç½ij绾":111638,"åľ¨ç½ij绾ä¸Ĭ":111639,"交è°Ī":111640,"æĽ´æĶ¹":111641,"åįłæľīçİĩ":111642,"ä¸Ŀ绸ä¹ĭè·¯":111643,"è¡Ľ":111644,"çłĶåΤ":111645,"åĪª":111646,"åĪªéϤ":111647,"è¿Ļåıª":111648,"çļĦæ°Ķæģ¯":111649,"åĬłå·ŀ":111650,"éĴ§":111651,"çIJĨäºĭéķ¿":111652,"ä¸ĸå®¶":111653,"æµģè¡ĮçļĦ":111654,"å¾Īæľīåı¯èĥ½":111655,"们éĥ½":111656,"ç»ıèIJ¥æ¨¡å¼ı":111657,"è¡Įä¸ļä¸Ń":111658,"éĢļçŁ¥ä¹¦":111659,"åij½é¢ĺ":111660,"æľ¬ç¶²ç«Ļ":111661,"æ²Ļçī¹":111662,"åıijåħī":111663,"é«ĺä»·":111664,"å·²çĦ¶":111665,"åıĮåįģä¸Ģ":111666,"ä¸Ĭè¯ī":111667,"ç¿ħèĨĢ":111668,"è¿Ļä¸Ģå¹´":111669,"大ä¼ļä¸Ĭ":111670,"éĩī":111671,"å®Įåħ¨æĺ¯":111672,"å¾Ĺ太":111673,"ä¸ĢèĪ¬äºº":111674,"è¿ĺç®Ĺ":111675,"æĬĺåıł":111676,"æĬķæľº":111677,"çĤ¹çĩĥ":111678,"çݰéĩijæµģ":111679,"åħĶåŃIJ":111680,"ç½ijæł¼":111681,"æİ¥è¿ĩ":111682,"ä¾Ľè´§":111683,"éĺ´å½±":111684,"åİŁåħĪ":111685,"æį£":111686,"左侧":111687,"åħĭæĭī":111688,"æīĵåį¡":111689,"ç§ijæ¯Ķ":111690,"æ±ĩéĽĨ":111691,"åľ°çIJĨä½įç½®":111692,"è¯Ħå§Ķ":111693,"ç»ĵåIJĪèµ·æĿ¥":111694,"è¿Ľåħ¥åΰ":111695,"åı¯è¡Į":111696,"åı¯è¡ĮæĢ§":111697,"让å®ĥ":111698,"åĪ¶åº¦æĶ¹éĿ©":111699,"çĶĺèĤĥçľģ":111700,"åĵĹ":111701,"åģıåģı":111702,"è¡£çī©":111703,"ç¥Ŀè´º":111704,"æºIJèĩª":111705,"å¹¶ä¸į代表":111706,"åĽ½åº¦":111707,"好åĿı":111708,"æĿĸ":111709,"æĿŃå·ŀå¸Ĥ":111710,"湿度":111711,"鲸":111712,"åįļ彩":111713,"æ³°å±±":111714,"æĿijèIJ½":111715,"æĸ°èģŀ":111716,"èĤĭ":111717,"åı¤èĢģçļĦ":111718,"çļĦç§ĺå¯Ĩ":111719,"ä¸Ģ个éĹ®é¢ĺ":111720,"éģıåζ":111721,"åįĥ亿":111722,"è¿ĩ硬":111723,"å°Ħåĩ»":111724,"èĩªçĦ¶æĺ¯":111725,"产åĮº":111726,"çĤ¹çĤ¹å¤´":111727,"åı¯ä»¥å¸®åĬ©":111728,"说å®ŀ":111729,"说å®ŀè¯Ŀ":111730,"æĪijåıªæĺ¯":111731,"ä¹ĭä½Ļ":111732,"åIJĮæĹ¶ä¹Łæĺ¯":111733,"ä¸ŃåĽ½éĺŁ":111734,"建æĪIJåIJİ":111735,"ä¹IJè§Ĩ":111736,"åij¨å²ģ":111737,"èį¯åºĹ":111738,"éĩijåįİ":111739,"严éĩįå½±åĵį":111740,"è´¨åľ°":111741,"æĹħéģĬ":111742,"åħµåύ":111743,"æķĻèĤ²æķĻåѦ":111744,"离åİ»":111745,"åIJĦå¼ıåIJĦæł·":111746,"ä»ĭç´":111747,"ä»ĭç´¹":111748,"å¼Ģ头":111749,"å°Ĩèĩªå·±çļĦ":111750,"åIJ¬åĬĽ":111751,"ä¿¡æģ¯ç³»ç»Ł":111752,"ä»İæł¹æľ¬":111753,"ä»İæł¹æľ¬ä¸Ĭ":111754,"æİĮ声":111755,"欢åĸľ":111756,"å±ķåĮº":111757,"åķ¸":111758,"太å¤ļäºĨ":111759,"éĹ²ç½®":111760,"èĥ¡èIJĿåįľ":111761,"å§Ķå®£ä¼ł":111762,"å§Ķå®£ä¼łéĥ¨":111763,"åįĹéĺ³":111764,"å·ŀåĮº":111765,"ä¸İæĹ¶":111766,"ä¸İæĹ¶ä¿±":111767,"ä¸İæĹ¶ä¿±è¿Ľ":111768,"å«Įçĸij人":111769,"èī¯å¿ĥ":111770,"头顶":111771,"è´¢æĬ¥":111772,"ä½Ľæ³ķ":111773,"å¾µ":111774,"åİŁä»¶":111775,"åĭŀ":111776,"çĶ·ç¯®":111777,"å¤ĸåĽ½äºº":111778,"è¿Ŀ纪":111779,"æī¾äºĨ":111780,"æįķæįī":111781,"缸è¯Ĩ":111782,"æIJľéĽĨ":111783,"çļĦä¼Łå¤§":111784,"ä¸īç»´":111785,"å°±è¡ĮäºĨ":111786,"çĭIJæľĪ":111787,"çĭIJæľĪå±±":111788,"å¸ĮæľĽéĢļè¿ĩ":111789,"èĢĮ对äºİ":111790,"éĿ¢å°į":111791,"åĨĽåĽ¢":111792,"è¡ĹåĮº":111793,"æĤ¬æĮĤ":111794,"便ç§ĺ":111795,"æľīä¸ĢçĤ¹":111796,"ä¼ļè®®ä¸Ĭ":111797,"ä¸ĭæīĭ":111798,"廣åijĬ":111799,"äºĶè¡Į":111800,"çŃīåĢĻ":111801,"ç´§ç´§åĽ´ç»ķ":111802,"æĭ¿äºĨ":111803,"æ¡ĮéĿ¢":111804,"ç¥ŀæĥħ":111805,"éĽĦåİļ":111806,"çŀ³":111807,"楼ä¸ĭ":111808,"彪":111809,"äºĭåıij":111810,"åĨįè§ģ":111811,"é¤ĺ":111812,"é¢ĦåĶ®":111813,"åİ»çľĭçľĭ":111814,"æĪij们åºĶ该":111815,"ä¸īå®¶":111816,"æµĬ":111817,"ä¹IJéĺŁ":111818,"çľĭä¸įè§ģ":111819,"èĦijåŃIJ":111820,"æĮģæľīçļĦ":111821,"çϽèıľ":111822,"éĹªçĥģ":111823,"åĸĿæ°´":111824,"æİ§åĪ¶ç³»ç»Ł":111825,"ä¸ĵåĮº":111826,"æľĿå»·":111827,"æĪijå¿ĥéĩĮ":111828,"å±ķåİħ":111829,"èľĺèĽĽ":111830,"åĨ»ç»ĵ":111831,"粪":111832,"åºIJ":111833,"åIJij社ä¼ļ":111834,"åĨ³çŃĸéĥ¨ç½²":111835,"çŁŃæľŁåĨħ":111836,"æĸ°ä¸ļæĢģ":111837,"æľĶ":111838,"æĹ¶æĬ¥":111839,"使ä¹ĭ":111840,"åĽłåŃIJ":111841,"åıĤä¸İèĢħ":111842,"çļĦ年轻人":111843,"æīĭ表":111844,"å°ģéĶģ":111845,"为ä»Ģä¹Īä¸į":111846,"åIJ¸çĥŁ":111847,"æ¯Ĵç´ł":111848,"åĪijæ³ķ":111849,"磫æŃ£":111850,"身æĹģ":111851,"åİŁè°ħ":111852,"çĽijæĬ¤":111853,"æŃ¤å¤Ħ":111854,"éĻĤ":111855,"æŀľå®ŀ":111856,"åĮ»çĸĹæľįåĬ¡":111857,"ä¸įåIJĪçIJĨ":111858,"æIJŀ好":111859,"çļĦèĦļæŃ¥":111860,"å¤ĸå¥Ĺ":111861,"ç¶ĵéģİ":111862,"æĶ¾ç¼ĵ":111863,"åģľçķĻ":111864,"æĺŁçIJĥ":111865,"çļĦä¸ĢéĿ¢":111866,"åĩłä½ķ":111867,"è½®åĽŀ":111868,"æ¯Ľå·¾":111869,"ä¿®çIJĨ":111870,"ä¸įçŁ¥ä¸į":111871,"ä¸įçŁ¥ä¸įè§ī":111872,"æķ´ä¸ªäºº":111873,"æ¯ģçģŃ":111874,"åı°å·ŀ":111875,"使çĶ¨å¯¿åij½":111876,"é»ijçϽ":111877,"æij¸ç´¢":111878,"é¼łæłĩ":111879,"éĿ©æĸ°":111880,"麵":111881,"ä¸ĵéĹ¨ä¸º":111882,"å¾Īå¤ļæľĭåıĭ":111883,"å·¥ä½ľç»Ħ":111884,"åIJĪå½±":111885,"çĤºä»Ģ麼":111886,"æŀģ度":111887,"çļĦè¿ĽæŃ¥":111888,"å½ĵä¹ĭ":111889,"å½ĵä¹ĭæĹł":111890,"å½ĵä¹ĭæĹłæĦ§":111891,"è´´è¿ij":111892,"尺度":111893,"åľ¨çİ°åľº":111894,"éĻį临":111895,"åħ»èĢģéĩij":111896,"ç£ķ":111897,"åı¯ä»¥ä½¿":111898,"管çIJĨæ°´å¹³":111899,"æľ¬æĬ¥è®°èĢħ":111900,"æ³ķ令":111901,"åį¡è½¦":111902,"ä¸ľæµ·":111903,"å¤ļéĩį":111904,"åħ¶éĹ´":111905,"ç´Ļ":111906,"éĩįå¤§é¡¹çĽ®":111907,"æ±Ĺæ°´":111908,"ç»Ħå§Ķä¼ļ":111909,"ä¿¡æģ¯åħ¬å¼Ģ":111910,"ä¸į论æĺ¯":111911,"ä¸ĢåIJ¬":111912,"èĴ¸æ±½":111913,"æıŃç§ĺ":111914,"è¶ħéģİ":111915,"触åıij":111916,"婦":111917,"åħ³èģĶ交æĺĵ":111918,"å°±ç»Ļ大家":111919,"好ä¹ħ":111920,"åĢŁè´·":111921,"游æĪıè§Ĵèī²":111922,"å¼ĢåIJ¯äºĨ":111923,"æİł":111924,"åħļçļĦåįģä¹Ŀ":111925,"ä¸ĭ鼨":111926,"çŁŃæĹ¶éĹ´åĨħ":111927,"å¯ħ":111928,"导åħ¥":111929,"å·¥ä½ľç»ıéªĮ":111930,"ä¹Łåıªèĥ½":111931,"鼷éľĨ":111932,"è·Łè¿Ľ":111933,"åį¡éĢļ":111934,"é¢ĩæľī":111935,"æľºä½ĵ":111936,"æĪĺ士èģĮä¸ļ":111937,"女主":111938,"ä½ĵåĪ¶æľºåζ":111939,"è¶³åįı":111940,"èĪĴéĢĤçļĦ":111941,"åĢŁåı£":111942,"æī¹åΤ":111943,"æķ°å̼":111944,"諾":111945,"éĺ¿æĭī伯":111946,"åĺİ":111947,"æħ¶":111948,"达人":111949,"å¼Ģæ°´":111950,"å¤§éĽ¨":111951,"温室":111952,"ä½İè¿·":111953,"ä»įæĹ§":111954,"éªĹåŃIJ":111955,"亲å±ŀ":111956,"çIJĨæĻº":111957,"æľ¬åŁºéĩij":111958,"å¨ħ":111959,"åĨĻåŃĹæ¥¼":111960,"å¢Ļå£ģ":111961,"宵":111962,"èϽçĦ¶æĺ¯":111963,"顺çĿĢ":111964,"åħ«åį¦":111965,"åķĨç͍":111966,"ä¸į失":111967,"è¿·èĮ«":111968,"顺便":111969,"æļijæľŁ":111970,"æ¬ºè´Ł":111971,"é¢ijé¢ij":111972,"è¯¥æł¡":111973,"æĸĻçIJĨ":111974,"æ·±æĥħ":111975,"åīįéĶĭ":111976,"ä¿ĿèŃī":111977,"èģĮä¸ļçĶŁæ¶¯":111978,"åħ¬å¼Ģåıij":111979,"åħ¬å¼Ģåıijè¡Į":111980,"åħ¥æĪ·":111981,"éłĵ":111982,"å̾åIJ¬":111983,"éŃģ":111984,"æĦīæĤ¦":111985,"åĽŀåIJĪ":111986,"åħ¨åĬĽä»¥":111987,"åħ¨åĬĽä»¥èµ´":111988,"åĥ¹å̼":111989,"èĥ½åĬĽå¼º":111990,"ç»ıå¼Ģ":111991,"ç»ıå¼ĢåĮº":111992,"è¿ľæĸ¹":111993,"çļĦéģĵçIJĨ":111994,"缴åįĩ":111995,"缴åįĩæľº":111996,"为主é¢ĺçļĦ":111997,"ç»ĻæĤ¨":111998,"è¿ĺæĥ³":111999,"æ¯ĶæĪij":112000,"åĨľçī§":112001,"æµ·åºķ":112002,"çŃ¾è®¢äºĨ":112003,"对äºİæĪij们":112004,"æĹ¶è®¸":112005,"éĶ®çĽĺ":112006,"å®ŀéĻħæİ§åζ":112007,"çļĦæ¨¡æł·":112008,"åıįæĺłäºĨ":112009,"代åĬŀ":112010,"åĮ»ç͍":112011,"éĽĨç»ĵ":112012,"åıijå±ķåīįæĻ¯":112013,"æĮĩçĿĢ":112014,"åįİåĮĹ":112015,"è¿Ļåĩłä¸ª":112016,"åIJįæ°Ķ":112017,"åĤįæĻļ":112018,"èĩªåıij":112019,"æ³¢åħ°":112020,"大åĬĽæİ¨è¿Ľ":112021,"èĩªç§°":112022,"èįĨå·ŀ":112023,"æIJį害":112024,"äºĨä¸Ģåı¥":112025,"æľĢåĪĿçļĦ":112026,"éĩijèŀįå᱿ľº":112027,"æĢĢ念":112028,"è¡Įåĭķ":112029,"女æİĴ":112030,"ä¸įè§£":112031,"ä¼łéĶĢ":112032,"转载请":112033,"饰åĵģ":112034,"åıªä¸º":112035,"ä¸İä¼Ĺ":112036,"ä¸İä¼Ĺä¸įåIJĮ":112037,"èĥ½èĢĹ":112038,"èı©æıIJ":112039,"è¿ij两年":112040,"è¿Ķ乡":112041,"马ä¸Ĭå°±":112042,"äºĮçŃīå¥ĸ":112043,"水管":112044,"æ³ķåѦ":112045,"çģŃçģ«":112046,"大å§IJ":112047,"åij¨è½¬":112048,"æľīæľŁ":112049,"æľīæľŁå¾Ĵ":112050,"æľīæľŁå¾ĴåĪij":112051,"å°įæĸ¹":112052,"ç¥ŀèī²":112053,"æ²¹èĦĤ":112054,"ä¸īçĤ¹":112055,"ä¸įåĪ©äºİ":112056,"äºĭä¸ļéĥ¨":112057,"å°±è·Ł":112058,"å¼ĢæĶ¯":112059,"å°ı女åŃ©":112060,"åħ±åIJĮåĬªåĬĽ":112061,"çĶļèĩ³è¿ĺ":112062,"è¿ĻåIJį":112063,"è¿Ļç¬Ķ":112064,"çݯåį«":112065,"æľīç§į":112066,"è§ĨåĬĽ":112067,"çĨŁçŁ¥":112068,"åħ¬ç§¯éĩij":112069,"æ¶Īéĺ²å®īåħ¨":112070,"é¢ĩ为":112071,"大èħ¿":112072,"éĿ¶":112073,"çķĪ":112074,"æľįåĬ¡åĮº":112075,"å¼Ģåĩº":112076,"深度èŀįåIJĪ":112077,"æĹłå¿§":112078,"æŁ¥éĺħ":112079,"ç»Īç»ĵ":112080,"ä¿Ŀç¨İ":112081,"è¨İè«ĸ":112082,"å½ĵåģļ":112083,"è·³èĪŀ":112084,"寧":112085,"女çİĭ":112086,"è®°èĢħåľ¨":112087,"åħ¨äº§ä¸ļéĵ¾":112088,"è´¯éĢļ":112089,"åħ´ä¸ļ":112090,"éĻįåΰ":112091,"å°ģéĿ¢":112092,"åħ¨éĿ¢æİ¨è¿Ľ":112093,"奶èĮ¶":112094,"éĢīåĿĢ":112095,"äºĨä¸Ģåľº":112096,"åIJĮä¼´":112097,"议论":112098,"æIJĵ":112099,"诸èijĽ":112100,"诸èijĽäº®":112101,"å¹²åĺĽ":112102,"æµģæĦŁ":112103,"ä¸ĵä¸ļçŁ¥è¯Ĩ":112104,"ç͵ç«Ļ":112105,"åĩıå¼±":112106,"åĩºåħ¥":112107,"åIJĦçľģ":112108,"éĿŀ常é«ĺ":112109,"åľ°æ¯¯":112110,"åıijæĸĩ":112111,"çĦī":112112,"çĥ§çĥ¤":112113,"å£ģ纸":112114,"æģ¶åĮĸ":112115,"èĬ¸":112116,"èĥĸåŃIJ":112117,"çĩĴ":112118,"çľģéĴ±":112119,"çĻ¾å¼º":112120,"çIJĨ工大åѦ":112121,"éĴ¢æĿIJ":112122,"åĽ½æľīèµĦ产":112123,"æĪĺæľº":112124,"æ³Ħéľ²":112125,"åIJİéĿ¢çļĦ":112126,"æ°´èµĦæºIJ":112127,"æ¢ħèĬ±":112128,"åĨĻçĿĢ":112129,"ä¹ĭ声":112130,"æĹłåı¯":112131,"æĺİæľĿ":112132,"ç«ĭæĸ¹ç±³":112133,"ç·£":112134,"æĶ¾è¿ĩ":112135,"ç¦ıçͰ":112136,"å¾Ĺä½ı":112137,"åıĹä¼Ĺ":112138,"ä¸Ń级":112139,"çĹħåıĺ":112140,"ä¸Ģçŀ¬éĹ´":112141,"æĿĥéĩį":112142,"人æĢ§åĮĸ":112143,"åĮ»çĸĹåį«çĶŁ":112144,"ä¸įåΰä½į":112145,"æĻºèĥ½å®¶å±ħ":112146,"饮ç͍":112147,"æ¼Ķåıĺ":112148,"é«ĺç´łè´¨":112149,"ä¹Ļæĸ¹":112150,"åģľçķĻåľ¨":112151,"èİ·æī¹":112152,"ç©¿æ¢Ń":112153,"å®¢åľº":112154,"æĮ½åĽŀ":112155,"京åŁİ":112156,"çĶŁåij½åĬĽ":112157,"實éļĽ":112158,"çĩĪ":112159,"åĨįçݰ":112160,"çݰå®ŀä¸Ń":112161,"æľīä¿¡å¿ĥ":112162,"çĸıéĢļ":112163,"åĺ´åĶĩ":112164,"鼷éĶĭ":112165,"èıľåįķ":112166,"éħ¯":112167,"è¶ħé«ĺ":112168,"å¾Īé«ĺåħ´":112169,"çĶŁæ®ĸ":112170,"éĢłä»·":112171,"误åĮº":112172,"æĨĭ":112173,"好æ¶Īæģ¯":112174,"å´Ń":112175,"以èĩ´":112176,"å¼Ģçİ©ç¬ij":112177,"çĽijè§Ĩ":112178,"å·¡å¯Ł":112179,"å¾·å·ŀ":112180,"æĹ©æĹ©":112181,"éĹªç͵":112182,"æĪªåĽ¾":112183,"åı¯ä»¥æł¹æį®":112184,"æīĭèīº":112185,"æİ¥è½¨":112186,"ç§įæĹı":112187,"æĢĢéĩĮ":112188,"åİ»åĮ»éĻ¢":112189,"ä¸ĢäºĮ":112190,"å¼ĢéĺĶ":112191,"åĩıéĢŁ":112192,"ä½Ĩä»İ":112193,"éĢĻä¸Ģ":112194,"åĩıåħį":112195,"主é¢ĺæķĻèĤ²":112196,"å¼Ģ工建设":112197,"蹦":112198,"æľĪ饼":112199,"ä¸ĭæ²ī":112200,"å°Ĭ严":112201,"éĻĩ":112202,"å®ŀæľ¨":112203,"å»łåķĨ":112204,"声称":112205,"èĢĥåľº":112206,"å¸ĥé²ģ":112207,"èĩªæĿ¥":112208,"èĩªæĿ¥æ°´":112209,"éĴ¾":112210,"年以ä¸Ĭ":112211,"大åıĶ":112212,"ä»ĸå·²ç»ı":112213,"åħ¨æĿij":112214,"èģĶç³»ç͵è¯Ŀ":112215,"为导åIJij":112216,"åΤå¤Ħ":112217,"对éĺµ":112218,"缮æ¨Ļ":112219,"åIJįé¢Ŀ":112220,"客æ°Ķ":112221,"横åIJij":112222,"çŃīåĨħ容":112223,"åĩłçĤ¹":112224,"è°Ī论":112225,"ä¸įä¹ı":112226,"å±ķçݰåĩº":112227,"è¾ĥéķ¿":112228,"éĢĨ转":112229,"å°ıæĻĤ":112230,"æĺ¯å¤ļä¹Ī":112231,"æľ¬æľĪ":112232,"è¿ijè§Ĩ":112233,"æĪIJç«ĭ以æĿ¥":112234,"代表çĿĢ":112235,"æĬ¥å¤į":112236,"æĪıæĽ²":112237,"è¨ŃåĤĻ":112238,"åħ¥èĤ¡":112239,"å¾ģæľį":112240,"é«ĺåĩº":112241,"èĪŀåı°ä¸Ĭ":112242,"å¿ĥåĬ¨":112243,"两çĤ¹":112244,"缸çķ¶":112245,"èĻĽ":112246,"主页":112247,"åĩłå®¶":112248,"æĹłä¸į":112249,"åįıå®ļ":112250,"æĸIJ":112251,"å¯ĵæĦı":112252,"åħ¨çº¿":112253,"æįķé±¼":112254,"åı¯ä»¥ä»İ":112255,"æľīè¿Ļæł·çļĦ":112256,"æģ¶éŃĶ":112257,"åĮħåŃIJ":112258,"æģ¤":112259,"å¼Ģå¥ĸç»ĵæŀľ":112260,"ä¸įæŃ»":112261,"èĹį":112262,"å¼¯æĽ²":112263,"海峡":112264,"éĶĢæ¯ģ":112265,"çļĦçĭ¬çī¹":112266,"示æĦı":112267,"ä¸įèĥ½åĨį":112268,"èĥ½æĬĬ":112269,"éĺ²çº¿":112270,"ä¸įå°ijäºİ":112271,"æ±Ģ":112272,"çļĦéĤ£ä¸Ģ":112273,"羣æĥħ":112274,"åŀ®":112275,"被æīĵ":112276,"åĽ½å®ī":112277,"ç¾İå¦Ļ":112278,"è¿Ļåĩł":112279,"åĩºéģĵ":112280,"æľįåĬ¡äºİ":112281,"æĪIJæŀľè½¬åĮĸ":112282,"æīįåįİ":112283,"天é¹ħ":112284,"åĩłä¸ªäºº":112285,"åĢĺèĭ¥":112286,"èĢ½è¯¯":112287,"æĬĹæĪĺ":112288,"è¡ĮéĬ·":112289,"æĿ¥è¢Ń":112290,"åĢŁéĮ¢":112291,"èįīèİĵ":112292,"ä¸¥æł¼æī§è¡Į":112293,"举è¡ĮäºĨ":112294,"å¤ĸç±į":112295,"已达":112296,"æĿijåħļæĶ¯éĥ¨":112297,"è¡Ŀ":112298,"éĻįèĩ³":112299,"æµ·éĩı":112300,"é¤IJé¦Ĩ":112301,"æĢ¥å¿Ļ":112302,"æ·±è¿ľ":112303,"å¾Ģè¿Ķ":112304,"ç¨İåĬ¡å±Ģ":112305,"å¹¿æ³ĽåºĶç͍":112306,"è®®åijĺ":112307,"æĹłæķĮ":112308,"çľ¼åħī":112309,"çĥŃè¡Ģä¼łå¥ĩ":112310,"æŃIJ":112311,"äºĨäºĽ":112312,"è¿ĿèĥĮ":112313,"è¿Ļæĺ¯ä¸Ģç§į":112314,"ä¸į稳å®ļ":112315,"大家åĪĨ享":112316,"表çı¾":112317,"åīįåįģ":112318,"è·¯è¿ĩ":112319,"æĴ©":112320,"åIJĮæĥħ":112321,"ä¹łä¿Ĺ":112322,"åıijè´¢":112323,"åºĶæľīçļĦ":112324,"æĿİæŁIJ":112325,"èĤĽ":112326,"马åħĭ":112327,"éĢļåijĬ":112328,"巨人":112329,"ä¸ĢåĽ¢":112330,"éĢĻæ¬¡":112331,"ä¸įäºĨè§£":112332,"æĸ½è¡Į":112333,"èij¡èIJĦçīĻ":112334,"åıĺå¾ĹæĽ´åĬł":112335,"æı£":112336,"åĪĽæĸ°èĥ½åĬĽ":112337,"çķħéĶĢ":112338,"表æī¬":112339,"æ¯ĶåĪ©":112340,"æ¯ĶåĪ©æĹ¶":112341,"åĮ»çĸĹä¿ĿéĻ©":112342,"æĵį纵":112343,"伤亡":112344,"æµİå®ģ":112345,"åıĺäºĨ":112346,"æľ¬æ¬¡æ´»åĬ¨":112347,"åľŁè±ª":112348,"æĥ³åĬŀæ³ķ":112349,"æĺķ":112350,"å½ĵæĻļ":112351,"åĩºå±Ģ":112352,"çĥŃè®®":112353,"è°Īè°Ī":112354,"æĻĭåįĩ":112355,"åĬ¿å¿ħ":112356,"çϻ山":112357,"éĤ£åĦ¿":112358,"åIJĥåΰ":112359,"ä¹ĭåŁİ":112360,"å¿«æĿ¥":112361,"æ¹Ľæ±Ł":112362,"第ä¸ī个":112363,"åħ¨éĿ¢æıIJåįĩ":112364,"å¥ĸåѦ":112365,"å¥ĸåѦéĩij":112366,"æĬķåħ¥ä½¿ç͍":112367,"é½IJé²ģ":112368,"åı¯ä»¥æĬĬ":112369,"åĴĮä»ĸçļĦ":112370,"è´ŃæĪ¿èĢħ":112371,"æŃ£å¼ıåIJ¯åĬ¨":112372,"åįİæ¶¦":112373,"ä¸įæĸŃå®ĮåĸĦ":112374,"éĴ¢æĿ¿":112375,"累积":112376,"满èĦ¸":112377,"åĽĽæĸ¹":112378,"è´¢çī©":112379,"ä»ĸ们ä¼ļ":112380,"å¤ıæĹ¥":112381,"éĤ£ä¸ªäºº":112382,"éĿłçĿĢ":112383,"çĤ¹äºĨ":112384,"çĤ¹äºĨçĤ¹å¤´":112385,"æ©ĭ":112386,"åıĪ好":112387,"åıĪ好åıĪ":112388,"åıĪ好åıĪå¿«":112389,"éĺµéĺµ":112390,"å°ģ建":112391,"æľ¬çͰ":112392,"çī©ä¸ļæľįåĬ¡":112393,"èĩªè´¸åĮº":112394,"åIJı":112395,"便åĪ©åºĹ":112396,"åĽ½å®¶æłĩåĩĨ":112397,"éĿ¢ç²ī":112398,"èī°è¾Ľ":112399,"æĶ»åħ³":112400,"æīĵåĮħ":112401,"车éĺŁ":112402,"人éĢī":112403,"åı¯ä¸įæĺ¯":112404,"äºĮåįģå¹´":112405,"åIJįå¸Ī":112406,"æµ¦ä¸ľ":112407,"åħ¬è¯ģ":112408,"è¿IJéĢģ":112409,"æĺ¯æľĢ好çļĦ":112410,"æŁĶåĴĮ":112411,"çİĭæŁIJ":112412,"çĹħæĪ¿":112413,"åĨ¶éĩij":112414,"ä¸Ģä»¶äºĭæĥħ":112415,"åį¤":112416,"åı¯æİ§":112417,"çīŁ":112418,"æĭĤ":112419,"å·²äºİ":112420,"人éĢł":112421,"çĶŁçī©åĮ»èį¯":112422,"ä½ĵçݰåĩº":112423,"èĤ²åĦ¿":112424,"èĢģå®ŀ":112425,"åľĸçīĩ":112426,"諸":112427,"ç´¯äºĨ":112428,"æĦŁåħ´è¶£çļĦ":112429,"åĽ¾çīĩæĿ¥æºIJ":112430,"ä¹Łæĺ¯ä¸Ģç§į":112431,"æ¾İæ¹ĥæĸ°éĹ»":112432,"æĹ¶è¡¨ç¤º":112433,"åħīè¾ī":112434,"æĬ¥åºŁ":112435,"å²ģæĹ¶":112436,"éħ®":112437,"æ£Ģä¿®":112438,"åıĺéĢŁ":112439,"åıĺéĢŁç®±":112440,"åľ¨èģĮ":112441,"éı¡":112442,"æįĤ":112443,"çĿ£åĬŀ":112444,"æ°¸ä¸į":112445,"åģļä¸ĢäºĽ":112446,"åİĨæĹ¶":112447,"å·¥ç¨ĭæľºæ¢°":112448,"æģ°å½ĵ":112449,"å°±åľ¨äºİ":112450,"ç§°åij¼":112451,"éĢļ常æĺ¯":112452,"æł·å¼ı":112453,"åij¨ä¸Ģ":112454,"èĭ±éķij":112455,"åĿĩ线":112456,"ä¼łéĹ»":112457,"ç͍æĪ·ä½ĵéªĮ":112458,"èµŀåIJĮ":112459,"骨æĬĺ":112460,"为主ä½ĵ":112461,"æ±Łå±±":112462,"æ¸ħæľĿ":112463,"æĶĢåįĩ":112464,"ä¸įçĽ¸ä¿¡":112465,"éĿ´":112466,"æŃ¦åĬŁ":112467,"åĭ¤åĬ³":112468,"æĿ¥æī¾":112469,"å°ĨæĮģç»Ń":112470,"丫头":112471,"æ¨Ļæºĸ":112472,"裴":112473,"深深çļĦ":112474,"åŃķèĤ²":112475,"è§ĦåĪĴ建设":112476,"æ¸ħçν":112477,"ç²¾åĩĨæī¶è´«":112478,"æīĵçł´äºĨ":112479,"è¿Ļä¸Ģ天":112480,"å·¥ä½ľæĢ»ç»ĵ":112481,"æĹħç¨ĭ":112482,"举èIJ¥":112483,"æĶ¾å°Ħ":112484,"æľīåĩłä¸ª":112485,"éĿŀçī©è´¨":112486,"åIJĥå¾Ĺ":112487,"åŨ":112488,"ä¼ļåıijçĶŁ":112489,"篮æĿ¿":112490,"å¼Ģå°ģ":112491,"麻å°Ĩ":112492,"èııæ³½":112493,"ä¸įåIJĪ":112494,"ç³»åĪĹ产åĵģ":112495,"èѬå¦Ĥ":112496,"ç¾İèªī":112497,"èĩªå·±åĸľæ¬¢":112498,"交æĺĵä¸Ńå¿ĥ":112499,"åIJĪåͱ":112500,"使æĪij":112501,"åĥıç´ł":112502,"带éĺŁ":112503,"ä½Ĩ对äºİ":112504,"æĬĬè¿Ļ个":112505,"èĤĿèĦı":112506,"åįķ纯çļĦ":112507,"æĶ»åĿļæĪĺ":112508,"缼ä¼ļ":112509,"åijµæĬ¤":112510,"æªĢ":112511,"èµ¶ä¸Ĭ":112512,"æ¥Ĭ":112513,"ä¹ħäºĨ":112514,"ç¡Ŀ":112515,"çŃĶé¢ĺ":112516,"ä¿ĿæĮģçĿĢ":112517,"è§ģè¯Ĩ":112518,"çĤ¹åĦ¿":112519,"åįĬ个æľĪ":112520,"æ»ĩ":112521,"浸泡":112522,"ä¼łéĢģ":112523,"åľ¨å¸Ĥåľºä¸Ĭ":112524,"ä¹ĭ乡":112525,"çī¹éķ¿":112526,"éĽŀ":112527,"èªł":112528,"身å¤Ħ":112529,"æŁłæª¬":112530,"身穿":112531,"çľģåħ¬å®ī":112532,"çľģåħ¬å®īåİħ":112533,"åıĻåĪ©äºļ":112534,"åĩłåĪĨéĴŁ":112535,"人åĢij":112536,"åľ°æ®µ":112537,"èĩªåѦ":112538,"ä¹Łè¶ĬæĿ¥è¶Ĭ":112539,"èģĮæĿĥ":112540,"æĸ§":112541,"èĩ»":112542,"å½Ĵ纳":112543,"驾é©Ń":112544,"éĥ¨åĪĨåľ°åĮº":112545,"没æľīæĥ³åΰ":112546,"æĴĩ":112547,"ä¹Įé²ģ":112548,"ä¹Įé²ģæľ¨":112549,"ä¹Įé²ģæľ¨é½IJ":112550,"èĤ²äºº":112551,"çļĦæŃ¥ä¼IJ":112552,"å»¶æľŁ":112553,"æ²¹æ°Ķ":112554,"åģļå®Į":112555,"åľ£åľ°":112556,"丰åİļ":112557,"宽带":112558,"åı¯éĿłçļĦ":112559,"åºŃéĻ¢":112560,"åŃľ":112561,"å°ı康社ä¼ļ":112562,"å®īåħ¨ç®¡çIJĨ":112563,"年第":112564,"æİĴ污":112565,"èĥĮåĮħ":112566,"å®¶ä½ı":112567,"åħ¶å®ŀå°±æĺ¯":112568,"ä¼ļè§ģ":112569,"帮åĬ©ä¼ģä¸ļ":112570,"ç½ijè´Ń":112571,"æĺ¯ä¸įä¼ļ":112572,"飯åºĹ":112573,"æŃ»åİ»":112574,"åħįçĸ«åĬĽ":112575,"æľķ":112576,"åĸĿäºĨ":112577,"轻微":112578,"个æľĪåĨħ":112579,"ç»ĦåĽ¢":112580,"åĴĮå®ĮåĸĦ":112581,"鸽":112582,"æıIJéĢŁ":112583,"西å®īå¸Ĥ":112584,"ä¸Ńå¿ĥ主任":112585,"æĹ¶éĹ´ä¸º":112586,"æľŁæĿĥ":112587,"è¶ķ":112588,"ä¸įä»ħè¦ģ":112589,"æľįä»İ":112590,"é¡ĺæĦı":112591,"ä¸įå°ı":112592,"ä¸įå°ıçļĦ":112593,"ç°ĩ":112594,"窦":112595,"åĪĩæĪIJ":112596,"åĵĪåĪ©":112597,"å¤©çľŁ":112598,"ä¸Ģ次次":112599,"éĩijå¸ģ":112600,"æĢİä¹Īèĥ½":112601,"ç½ijè´·":112602,"ä¼ļ计å¸Ī":112603,"çŁŃ缺":112604,"对æłĩ":112605,"åıĺå¾ĹæĽ´":112606,"åīįåĩłå¤©":112607,"éĺ²æ±Ľ":112608,"彩èϹ":112609,"åĵģä½į":112610,"è¡¨æł¼":112611,"严å¯Ĩ":112612,"æ¯ĽåĪ©çİĩ":112613,"çļĦåį±å®³":112614,"å½ķåζ":112615,"æ°´åĬ¡":112616,"èĥ½å¤Łè®©":112617,"å¹³æĿ¿":112618,"ä¹³æĪ¿":112619,"è¸ıå®ŀ":112620,"é¦ĸåĪĽ":112621,"é¦Ļèķī":112622,"æĬ¥è¡¨":112623,"ä¸ĢæĬ¹":112624,"åĩºçĶŁäºİ":112625,"è²»ç͍":112626,"åĩºè®©":112627,"åIJĪæ³ķæĢ§":112628,"å°¼åħĭ":112629,"åĨ°åĨ·":112630,"é¦Ļæ°Ķ":112631,"åı·ç§°":112632,"èµ·çłģ":112633,"åŁİåİ¿":112634,"çİ©èĢį":112635,"ä¸ĬéĻIJ":112636,"ä¼ļ议精ç¥ŀ":112637,"æĹģè¾¹çļĦ":112638,"便ä¼ļ":112639,"æıŃæĻĵ":112640,"çİ©æĦı":112641,"éĽªå±±":112642,"åIJijçĿĢ":112643,"ä½ĵèĤ²åľ¨çº¿":112644,"说æĺİ书":112645,"åĮĸèĤ¥":112646,"åħļç»Ħ书记":112647,"åĬ¨äºº":112648,"ä¹ĭæīĢ":112649,"æľĪèĩ³":112650,"æľĢå¿«çļĦ":112651,"èĬĤåģĩæĹ¥":112652,"ä¸ĵåľº":112653,"èĢĥä¸Ĭ":112654,"çªŁ":112655,"é²ľè¡Ģ":112656,"è¾ĥ强çļĦ":112657,"æĤĦçĦ¶":112658,"å¤ļä¸ªåĽ½å®¶":112659,"çªĹå¸ĺ":112660,"æŀģå¤§åľ°":112661,"ä¸įç͍æĭħå¿ĥ":112662,"è¿Ļä¹Īåģļ":112663,"åĥ¹æł¼":112664,"ç¾İ丽乡æĿij":112665,"å°ıæĹ¶åĨħ":112666,"ç´§è¿«":112667,"大çģ«":112668,"èĥ³èĨĬ":112669,"æĵįä½ľç³»ç»Ł":112670,"æ®ĭçķĻ":112671,"åĨĻåĩº":112672,"ç¦ģå¿Į":112673,"åĬłçĽŁåºĹ":112674,"è¿ijçϾ":112675,"便åı¯":112676,"æķ´æĶ¹æİªæĸ½":112677,"éĩĩ访æĹ¶":112678,"åĶIJ代":112679,"æ·±åĮĸæĶ¹éĿ©":112680,"çŁ¢":112681,"éĥ½åĸľæ¬¢":112682,"è¶ĬæĿ¥è¶Ĭé«ĺ":112683,"èĬ±æľµ":112684,"头çĸ¼":112685,"å®ī康":112686,"å¢ŀéķ¿çİĩ":112687,"çľ¼çľĭ":112688,"å°±æĺ¯ä¸ºäºĨ":112689,"èĢĮ导èĩ´":112690,"åĬłå¿«å»ºè®¾":112691,"èĬ±æł·":112692,"åĨħå¿ĥçļĦ":112693,"æĺĨå±±":112694,"è³ĩæºIJ":112695,"åĽŀåΰ家":112696,"èıĬèĬ±":112697,"æ°´éĩı":112698,"å¾ģä¿¡":112699,"è¡ĮæĶ¿åĮº":112700,"ä¹ĥæĺ¯":112701,"æĬķèµĦé¡¹çĽ®":112702,"å«ģç»Ļ":112703,"ç¥ŀåľ£":112704,"稳":112705,"æľ¬æĿ¥å°±":112706,"éĢIJä¸Ģ":112707,"èģĮä¸ļæĬĢæľ¯":112708,"ä¸įèī¯ä¿¡æģ¯":112709,"æīĺè¿IJ":112710,"åIJ¯ç¤º":112711,"ä¹ĭåħ§å®¹":112712,"飶":112713,"奢åįİ":112714,"æıŃ示":112715,"æĪIJ为ä¸ŃåĽ½":112716,"æ¶Īè´¹åĵģ":112717,"åħ¬ç͍":112718,"æIJŀå®ļ":112719,"è¯·ä½ł":112720,"æŁļ":112721,"åĨħè¡£":112722,"ä½Ĩä»ĸ们":112723,"ä¿Ŀ湿":112724,"该åİ¿":112725,"饱åĴĮ":112726,"æİ¨åIJij":112727,"èµĦæĸĻæĺ¾ç¤º":112728,"ä¸įå½±åĵį":112729,"人人éĥ½":112730,"åıijå±ķ壮大":112731,"åħ»èĢģæľįåĬ¡":112732,"çĶŁæ´»æ°´å¹³":112733,"åIJĦåİ¿":112734,"ä½łéľĢè¦ģ":112735,"说çļĦæĺ¯":112736,"å¤ĸåªĴ":112737,"æŃ¤äºº":112738,"次è¦ģ":112739,"追赶":112740,"åºĶ该å¦Ĥä½ķ":112741,"æĹ¥åĩĮæĻ¨":112742,"çķ¥æľī":112743,"éĥ½æĥ³":112744,"游ä¹IJ":112745,"è¿Ļ款游æĪı":112746,"平淡":112747,"æĺ¯ä¸ĢåĢĭ":112748,"å¤ĩèĢĥ":112749,"åζæŃ¢":112750,"ä¸Ģå®ļèĥ½":112751,"å¾Ĵå¼Ł":112752,"以çĤº":112753,"åįĥåħĥ":112754,"äºĶåħŃ":112755,"迪士":112756,"迪士尼":112757,"éĺ³æĢ§":112758,"åĨ¬å¥¥ä¼ļ":112759,"å°±æĺ¯åĽłä¸º":112760,"æĮĤéĴ©":112761,"æ¦ĤåĨµ":112762,"åıªè¦ģæľī":112763,"æ²¹çĶ»":112764,"åľ°æłĩ":112765,"ä¸Ĭè°ĥ":112766,"产ä¸ļåĽŃåĮº":112767,"åħ«åįģ":112768,"棱":112769,"æ¶²æĻ¶":112770,"æĿijå§Ķä¼ļ":112771,"çŃ¾çº¦ä»ªå¼ı":112772,"è¿Ļåħ¶ä¸Ń":112773,"åĨĻéģĵ":112774,"示èĮĥåŁºåľ°":112775,"éĩİçĶŁåĬ¨çī©":112776,"鼻åŃIJä¿¡ç®±":112777,"åĽ½éĻħè´¸æĺĵ":112778,"人æĿĥ":112779,"ä¿Ŀ管":112780,"èĭ¥æĤ¨":112781,"åİĭæĬij":112782,"黼":112783,"åľ°çľĭçĿĢ":112784,"éϰ":112785,"ä¸Ģå¹´å¤ļ":112786,"ä»İ容":112787,"ä¸ŃæĸŃ":112788,"å¯Łè§ī":112789,"移交":112790,"é͝":112791,"æĪĸ许æĺ¯":112792,"ç¶ł":112793,"两项":112794,"æľĢåĸľæ¬¢":112795,"æľĢåĸľæ¬¢çļĦ":112796,"å¤ľéĩĮ":112797,"åIJĮä»ģ":112798,"åĪĽæĸ°é©±åĬ¨":112799,"è°ģèĥ½":112800,"飾":112801,"åħīåѦ":112802,"åİĦ":112803,"èĦ±é¢ĸ":112804,"èĦ±é¢ĸèĢĮåĩº":112805,"迦":112806,"æĺ¯ä¸įåı¯èĥ½":112807,"窥":112808,"èĥ½æ»¡è¶³":112809,"宽度":112810,"伦çIJĨ":112811,"åı¯ä»¥èİ·å¾Ĺ":112812,"转ä¼ļ":112813,"å±±æĿij":112814,"éĵºè®¾":112815,"åĩºåĩ»":112816,"æĸĩåĮĸèīºæľ¯":112817,"ä¼ļ议室":112818,"æŃĮ声":112819,"æ»Ķ":112820,"èIJİ缩":112821,"æľįåĬ¡åijĺ":112822,"åıij表äºĨ":112823,"æĸ¼æĺ¯":112824,"æĺİç¡®è§Ħå®ļ":112825,"ç»´å¥ĩ":112826,"水产":112827,"æĬķä¿Ŀ":112828,"éĺ´éģĵ":112829,"èµ¶å¿«":112830,"夺å¾Ĺ":112831,"ä¸ĭåįķ":112832,"çµģåħ¬åı¸":112833,"çݯç»ķ":112834,"å½Ī":112835,"ä½ľé£İ建设":112836,"æĹħ游æĻ¯åĮº":112837,"æľīæĽ´å¤ļçļĦ":112838,"丰å¯Įå¤ļ彩":112839,"çIJĨ财产åĵģ":112840,"åĩºå·®":112841,"ä»İ严治":112842,"ä»İ严治åħļ":112843,"çĽ¸å¹²":112844,"æ»ĭ润":112845,"主åĬŀæĸ¹":112846,"åī§åľº":112847,"æ»ļçIJĥ":112848,"æ©Ħæ¦Ħ":112849,"èĩªä¸»åĪĽæĸ°":112850,"éĢļå¾Ģ":112851,"æł¼å°Ķ":112852,"çļĦä¼ĺçĤ¹":112853,"èĥĮä¸Ĭ":112854,"çªľ":112855,"çĪĨåĩº":112856,"å¹³æķ´":112857,"ä¸ĢèĦļ":112858,"åħ¨ä½ĵåijĺå·¥":112859,"éĻIJå®ļ":112860,"åŁİéķĩåĮĸ":112861,"æ·³":112862,"éĢ®æįķ":112863,"è¡ĮåĬ¨è®¡åĪĴ":112864,"æīĵå¾Ĺ":112865,"åİļéĩį":112866,"纪å½ķçīĩ":112867,"åĿļä¿¡":112868,"央ä¼ģ":112869,"åĨįä¹Łä¸į":112870,"天涯":112871,"åıĤèĢĥèµĦæĸĻ":112872,"æľīæ¯Ĵ":112873,"åIJ¸çº³":112874,"è¶Ĭåıij":112875,"éĩįè¦ģæĦıä¹ī":112876,"åĽ½éĺ²éĥ¨":112877,"è¿Ļ个è¡Įä¸ļ":112878,"æĻ®æŁ¥":112879,"å¼ĤæĢ§":112880,"å»¶è¿Ł":112881,"å°ıå¹ħ":112882,"èĥħ":112883,"综åIJĪæ²»çIJĨ":112884,"æŃ£æĺ¯åĽłä¸º":112885,"产ä¸ļç»ĵæŀĦ":112886,"çłĶç©¶æĬ¥åijĬ":112887,"åģľä¸ĭ":112888,"éķ¿èĢģ":112889,"éĩĿå°į":112890,"åįĹ京å¸Ĥ":112891,"çģĮæºī":112892,"转è¿IJ":112893,"欺è¯Ī":112894,"éĢłåģĩ":112895,"åĪĨå¸ĥå¼ı":112896,"æĦŁè§¦":112897,"æĪijå½ĵæĹ¶":112898,"åıijè§ī":112899,"åĽ¾çº¸":112900,"æĶ¹èī¯":112901,"çĭłçĭł":112902,"åĨ²åĪº":112903,"æĸ°äº¬":112904,"æĸ°äº¬æĬ¥":112905,"ç¥ŀåύ":112906,"秸ç§Ĩ":112907,"çĪº":112908,"å°Ĩè¿İæĿ¥":112909,"工信":112910,"工信éĥ¨":112911,"éĻIJéĩı":112912,"æŃ¢æįŁ":112913,"åѦä¼ļäºĨ":112914,"åįİ缼":112915,"åįİçĽĽé¡¿":112916,"å¾Įä¾Ĩ":112917,"ä¸ĭéĿ¢æĺ¯":112918,"ä¸ĭéĿ¢æĺ¯å°ı":112919,"æIJ¬è¿IJ":112920,"ç¾İæľ¯é¦Ĩ":112921,"æ¸ħåĩī":112922,"å¤ļå¹´åīį":112923,"è©ŀ":112924,"åįĥç±³":112925,"表述":112926,"æ±ŁéŨ":112927,"åĬłæ²¹ç«Ļ":112928,"æľ¬èĥ½":112929,"导读":112930,"åĽ´è§Ĥ":112931,"å¹¶åIJij":112932,"åŁºæľ¬æĥħåĨµ":112933,"æīĵå¼ĢäºĨ":112934,"è¿Ļä¸ī个":112935,"æ±ķ头":112936,"强æľīåĬĽ":112937,"强æľīåĬĽçļĦ":112938,"è¿Ľåľº":112939,"ä¹Ŀæ±Ł":112940,"çIJĥæĺŁ":112941,"好çľĭçļĦ":112942,"大æĪ·":112943,"湯":112944,"å¥ĩå¦Ļ":112945,"ä¹IJåύ":112946,"æĪijçļĦå¿ĥ":112947,"çľī头":112948,"åĨľä¸ļçĶŁäº§":112949,"ç¼ĸçłģ":112950,"åŁºç¤":112951,"åŁºç¤İ":112952,"天æĸĩ":112953,"åĢĭ人è³ĩè¨Ĭ":112954,"åİ»è¿ĩ":112955,"èģĨåIJ¬":112956,"æĶ¾åģĩ":112957,"ä¸įåħ·å¤ĩ":112958,"æ·Ģç²ī":112959,"大佬":112960,"åħ¨å¤©":112961,"åħ¨éĿ¢å»ºæĪIJ":112962,"éļIJå½¢":112963,"ç¼ħç͏":112964,"åIJ³":112965,"è¡ĮæĶ¿æī§æ³ķ":112966,"åŁİåł¡":112967,"èİ«æĸ¯":112968,"èİ«æĸ¯ç§ij":112969,"æīĢæľīæĿĥ":112970,"éĽĨåľĺ":112971,"å±Ģåī¯å±Ģéķ¿":112972,"åĩłä¹İ没æľī":112973,"æ´ģåĩĢ":112974,"ç͵影èĬĤ":112975,"åŃ©ç«¥":112976,"æīĢåģļçļĦ":112977,"æ¸ħ代":112978,"æĸ°çīĪ":112979,"éĵĿåIJĪéĩij":112980,"为æĬĵ":112981,"为æĬĵæīĭ":112982,"åΤå®ļ":112983,"çī¹äº§":112984,"æīĭæ©Ł":112985,"ä¸įåı¯æĪĸ":112986,"ä¸įåı¯æĪĸ缺":112987,"å¸Ĥåľºè§Ħ模":112988,"åĿ¯":112989,"åĮ»åѦéĻ¢":112990,"å¿«è¦ģ":112991,"èĮľ":112992,"æĬĺèħ¾":112993,"äºĨè¿ĩæĿ¥":112994,"æĬ¥åijĬæľŁåĨħ":112995,"çī©ç§į":112996,"ç»Łè®¡å±Ģ":112997,"æī©å»º":112998,"æ¶ħ":112999,"责任人":113000,"éĺİ":113001,"è¯Ħè®®":113002,"å¾Ģäºĭ":113003,"æīĢ示":113004,"æķ´æ´ģ":113005,"éĹºèľľ":113006,"æĹħéĢĶ":113007,"å®ŀè®Ń":113008,"ä¹ĭç§°":113009,"巴士":113010,"éĢŁåº¦å¿«":113011,"ä¸įä»ħå¦ĤæŃ¤":113012,"å®Ŀè´µçļĦ":113013,"åºŁçī©":113014,"河水":113015,"æİ¥çº³":113016,"ç²¾æ¹Ľ":113017,"åħ¶æ¬¡æĺ¯":113018,"顺德":113019,"åħ¬åħ±åį«çĶŁ":113020,"è¤IJèī²":113021,"ä¸įæĥľ":113022,"æĬĢæľ¯æľįåĬ¡":113023,"æİ·":113024,"æ±ĤèģĮ":113025,"ä¸ī峡":113026,"æĬķåħ¥åΰ":113027,"太åIJİ":113028,"åIJ¯åĬ¨ä»ªå¼ı":113029,"缴æİ¥å½±åĵį":113030,"æĸ°æ¬¾":113031,"个乡éķĩ":113032,"çĻ¾äº¿":113033,"庫":113034,"ä¹ŁæŃ£æĺ¯":113035,"åı¶çīĩ":113036,"æľĢæĹ©çļĦ":113037,"æĪĺ绩":113038,"å·¥æľŁ":113039,"æĻļæľŁ":113040,"è¿Ļæł·è¯´":113041,"è¯įè¯Ń":113042,"ä¾Ħ":113043,"æķ£çĥŃ":113044,"éĽĨæĪIJçĶµè·¯":113045,"åIJįè¯į":113046,"æĻºåķĨ":113047,"æĭ¥åłµ":113048,"çĭĤ欢":113049,"è¿Ļèά":113050,"浴室":113051,"åijķåIJIJ":113052,"æľªæĿ¥åıijå±ķ":113053,"ä¸īä½įä¸Ģä½ĵ":113054,"åªĴé«Ķ":113055,"ä¸įå¾Ĺ转载":113056,"åĽłä¸ºå¥¹":113057,"æĺ¾ç¤ºå±ı":113058,"ä¾Ľæļĸ":113059,"éĨ«éĻ¢":113060,"æľīæĦıæĢĿ":113061,"æľīæĦıæĢĿçļĦ":113062,"娱ä¹IJåŁİ":113063,"åįµå·¢":113064,"åĪĽéĢłåĬĽ":113065,"竳èĬĤ":113066,"人大常å§Ķ":113067,"èĢĮçİ°åľ¨":113068,"å¤ĸå©Ĩ":113069,"å¢ŀæĮģ":113070,"äºĶåįĥ":113071,"èĢģå¸Ī们":113072,"æ´ĽæĿī":113073,"æ´ĽæĿī磶":113074,"æİĮæı¡äºĨ":113075,"ä¸ŃåĽ½æĸĩåĮĸ":113076,"æĸ°æĶ¿":113077,"主è¦ģç͍äºİ":113078,"åıijçĥ§":113079,"类似äºİ":113080,"åĮĹæŀģ":113081,"æĪij们认为":113082,"弥漫":113083,"åħ¨çIJĥç»ıæµİ":113084,"é¢IJ":113085,"ä¸Ģèµ·è£ħä¿®":113086,"æĶĴ":113087,"æĭīèIJ¨":113088,"帶ä¾Ĩ":113089,"åĨ·æ°´":113090,"ä¸īåĨľ":113091,"æĿ¿æĿIJ":113092,"è¿ŀè¿ŀ":113093,"éĵ®":113094,"ç»ıèIJ¥çIJĨ念":113095,"山顶":113096,"å¾Īæĥ³":113097,"çĺ«":113098,"å§ĭç»Īä¿ĿæĮģ":113099,"åľ¨å¹¿å·ŀ":113100,"ä¸įåIJĮæĦı":113101,"åıĺåİĭ":113102,"åıĺåİĭåύ":113103,"产éĶĢ":113104,"表éĿ¢ä¸Ĭ":113105,"æīĢ以ä»ĸ":113106,"ç»ıéªĮ丰å¯Į":113107,"éĥ¨å§Ķ":113108,"åħµåĽ¢":113109,"æīĢè¿°":113110,"æķ¦çħĮ":113111,"ç»ıèIJ¥èĮĥåĽ´":113112,"åı£è¯Ń":113113,"失信":113114,"æ¯ı个人çļĦ":113115,"æīĭæĮģ":113116,"æģIJæħĮ":113117,"åł¡åŀĴ":113118,"é¦ħ":113119,"éĵ¸éĢł":113120,"æĭ¿åĩºæĿ¥":113121,"æİ¢æµĭ":113122,"大家ä¸Ģèµ·":113123,"奧":113124,"å®ŀè´¨æĢ§":113125,"å°ıåĦ¿":113126,"èĩºåįĹ":113127,"èĩºåįĹå¸Ĥ":113128,"å¼ĢåıijèĢħ":113129,"åı¯æł¹æį®":113130,"ç®±åŃIJ":113131,"饺åŃIJ":113132,"å¿ĻçĿĢ":113133,"æĿ¥ä¸įåıĬ":113134,"çĽ¸ä¼ł":113135,"åĽ½ç½ij":113136,"èħ¹æ³»":113137,"è¿ĻéĩĮæľī":113138,"é£İæĻ¯åĮº":113139,"åıĤä¿Ŀ":113140,"æŃ»èĢħ":113141,"æĪ´ä¸Ĭ":113142,"æ©Łæ§ĭ":113143,"è¯ķéªĮåĮº":113144,"ä¼łæİĪ":113145,"æµ·è¾¹":113146,"泪水":113147,"缸åħ³åĨħ容":113148,"éĥijå·ŀå¸Ĥ":113149,"åħijçݰ":113150,"两åij¨":113151,"èĬľæ¹ĸ":113152,"ç͵åŃIJä¿¡æģ¯":113153,"红å¤ĸ":113154,"æĹħ游å±Ģ":113155,"å¾Ģå¾Ģä¼ļ":113156,"è¿ħçĮĽ":113157,"ä¼łçľŁ":113158,"æ¸ħæ¾Ī":113159,"å°±è¿ij":113160,"微信群":113161,"ç³»åĪĹæ´»åĬ¨":113162,"ç»ı常ä¼ļ":113163,"è§Ĥæµĭ":113164,"å¿ĥå¾Ĺä½ĵä¼ļ":113165,"éĻĪåĪĹ":113166,"åĮĹæĸĹ":113167,"è«®":113168,"諮詢":113169,"è¿ĺæĺ¯ä¼ļ":113170,"æµĭç®Ĺ":113171,"æĺŁç©º":113172,"宽容":113173,"çī©ä¸ļåħ¬åı¸":113174,"æĪĴæĮĩ":113175,"å¸ħæ°Ķ":113176,"ä¸ĢæŃ¥æŃ¥":113177,"åħ±é¸£":113178,"åĨ³ä¸į":113179,"æİ¥ç®¡":113180,"å¦ĩèģĶ":113181,"æ¯Ķåĸ»":113182,"é²ģè¿ħ":113183,"æĮģçºĮ":113184,"çĽ¸äº²":113185,"å¨ģå°¼æĸ¯äºº":113186,"ç«ĭ项":113187,"åĪĿå§ĭ":113188,"èĩªåζ":113189,"è¿Īè¿Ľ":113190,"ä¸Ĭæ±½":113191,"å®ıä¼Ł":113192,"æł¹æľ¬æ²¡æľī":113193,"æĸ°åĨłçĹħæ¯Ĵ":113194,"åĵªç§į":113195,"康åħ»":113196,"è¡°èĢģ":113197,"å½ķåĥı":113198,"é«Ķé©Ĺ":113199,"ç»ijå®ļ":113200,"é¢Ŀ头":113201,"äºĶæľĪ":113202,"èĬ±å¼Ģ":113203,"ä¸Ģ线åŁİå¸Ĥ":113204,"åĪ°åľº":113205,"æĬķéĻį":113206,"çĹĺçĹĺ":113207,"åıĹä¸įäºĨ":113208,"æīİæł¹":113209,"æĽ´ä½ķåĨµ":113210,"æĬ½æŁ¥":113211,"åĩºè·¯":113212,"审议éĢļè¿ĩ":113213,"ä¸įåĥħ":113214,"èī²è°ĥ":113215,"çϾä½Ļ":113216,"èĤłéģĵ":113217,"æ·±åİļçļĦ":113218,"马åĬĽ":113219,"æĹ©æĻļ":113220,"æŃĮèĪŀ":113221,"éĺ²æĻĴ":113222,"æľĢåIJİä¸Ģ个":113223,"樱èĬ±":113224,"å°ıä¼ĻåŃIJ":113225,"åľ¨å½ĵåľ°":113226,"å°ıä¼Ļ伴们":113227,"èµ·æºIJ":113228,"åħ¨åªĴä½ĵ":113229,"ç°½":113230,"éħ±æ²¹":113231,"æĹłè®ºå¦Ĥä½ķ":113232,"裤åŃIJ":113233,"åģľäº§":113234,"ä¸įçͱå¾Ĺ":113235,"çīµå¼ķ":113236,"ä¼łåĬ¨":113237,"ä¹Ŀé¾Ļ":113238,"åĬłåĽº":113239,"ä¹Łä¸įæķ¢":113240,"æĬĢæľ¯æĶ¯æĮģ":113241,"ä¸Ĭå²Ĺ":113242,"ç»ıéªĮåĴĮ":113243,"æł¼æŀĹ":113244,"åIJ¸éĻĦ":113245,"æľªæĪIJå¹´":113246,"奢ä¾Īåĵģ":113247,"追æį§":113248,"好ä¸į容æĺĵ":113249,"èķ´åIJ«":113250,"ä¿Ŀå®ļ":113251,"æĬ¥ä¸ļ":113252,"æµ·åĨħå¤ĸ":113253,"ä½łçİ°åľ¨":113254,"æ²¹èĢĹ":113255,"è´¨éĩı管çIJĨ":113256,"æ½ľæ°´":113257,"ä¸½æ±Ł":113258,"转åħ¥":113259,"è¿Ļä¹Īä¹ħ":113260,"æĺİ代":113261,"责任åζ":113262,"éĩįå·¥":113263,"大巴":113264,"触åıĬ":113265,"èµ·åĪĿ":113266,"大å¦Ī":113267,"æĸ¯å¡Ķ":113268,"åĨĽå·¥":113269,"书éĻ¢":113270,"峨":113271,"æİ¨çIJĨ":113272,"è¿Ļç¯ĩæĸĩ竳":113273,"è¿ģç§»":113274,"åľ¨åIJĮä¸Ģ":113275,"ç»Ĩç»Ĩ":113276,"åīĬå¼±":113277,"书æĪ¿":113278,"ç¶ĵ常":113279,"è¯ķé¢ĺ":113280,"æĤ£ä¸Ĭ":113281,"çĻ«çĹ«çĹħ":113282,"åĨ²æ´Ĺ":113283,"å¤ĸæı´":113284,"åħĭåζ":113285,"åįģæľĪ":113286,"åģļä¸įåΰ":113287,"ç¾İåĮĸ":113288,"å¦ĤæľŁ":113289,"è¿ĺéľĢ":113290,"å¤©åºľ":113291,"å°±æĦıåij³çĿĢ":113292,"çļĦç¡®æĺ¯":113293,"éªĹå±Ģ":113294,"å°ıç»ĦèµĽ":113295,"è©©":113296,"ä¹Ŀå¹´":113297,"æĻĵå¾Ĺ":113298,"çłĶ究人åijĺ":113299,"大éħĴåºĹ":113300,"ç§ijåѸ":113301,"åħŃåIJĪ":113302,"çķĮå®ļ":113303,"车载":113304,"å¼ĢçĿĢ":113305,"毫æĹłçĸij":113306,"毫æĹłçĸijéĹ®":113307,"è¿IJç»´":113308,"ç¦ģåĮº":113309,"èĦ±èIJ½":113310,"讲å¸Ī":113311,"产ä¸ļåŁºåľ°":113312,"é«ĺæĢ§èĥ½":113313,"åħī彩":113314,"çݰéĺ¶æ®µ":113315,"åĩ¿":113316,"è¾ĥå·®":113317,"饮çĶ¨æ°´":113318,"éĸĭçϼ":113319,"ç½ijåIJ§":113320,"çĮ´åŃIJ":113321,"æŃ¦æŀĹ":113322,"å®īåİ¿":113323,"ä¸įåı¯æĢĿ":113324,"ä¸įåı¯æĢĿè®®":113325,"éĬ·åĶ®":113326,"è´«ç©·":113327,"为åķ¥":113328,"éºĵ":113329,"å¹¾åĢĭ":113330,"è§Ħ模以ä¸Ĭ":113331,"æıļ":113332,"è¢«åĽ°":113333,"缺å¸Ń":113334,"å¿«é¤IJ":113335,"æĬ¢åįł":113336,"æĻŁ":113337,"å¤įæ´»":113338,"æľ¬æĬ¥è®¯":113339,"åĪĽä¸ĭ":113340,"海滩":113341,"éĩı产":113342,"å¦Ĥä½ķåİ»":113343,"车ä½į":113344,"å¯ĩ":113345,"äºĮåįģåĽĽ":113346,"ç»ıæµİæįŁå¤±":113347,"éħįå¥Ĺ设æĸ½":113348,"åŁºæľ¬éĿ¢":113349,"äºī论":113350,"就好åĥı":113351,"çłĶç©¶æĪIJæŀľ":113352,"éĻĪè¿°":113353,"æīĵåĬ¨":113354,"ä¸ĭå·´":113355,"ç§ĴéĴŁ":113356,"对人ä½ĵ":113357,"æĬĢæľ¯çłĶåıij":113358,"åİŁåŃIJ":113359,"æĺ¯ä¸Ģ项":113360,"äºĨä¸Ģ份":113361,"æĮĩçͲ":113362,"ç͍éĩı":113363,"è¿ĺä¸įå¤Ł":113364,"æĶ¿åºľéĩĩè´Ń":113365,"çŁ¥è¯ĨçĤ¹":113366,"ä¸ŃåĽ½æ¢¦":113367,"å¾Īå¼Ģå¿ĥ":113368,"礼è²Į":113369,"éĿŀ常å¤ļ":113370,"éĿŀ常å¤ļçļĦ":113371,"åĽļ":113372,"æĹħé¦Ĩ":113373,"å°½æĥħ":113374,"æŃĮåͱ":113375,"æ²Ļé¾Ļ":113376,"车åİ¢":113377,"客æµģ":113378,"åģıå·®":113379,"积累äºĨ":113380,"æ¡Ķ":113381,"çĶ»çĶ»":113382,"ä¹ŁåºĶ该":113383,"åºĶç͍ç¨ĭåºı":113384,"èĥĥèĤł":113385,"以å¾Į":113386,"豪å®ħ":113387,"æ·±åĬłå·¥":113388,"缴è¨Ģ":113389,"åĮĸçŁ³":113390,"åĽ½éģĵ":113391,"ä¸ĥ个":113392,"ä»İèĢĮ使":113393,"èĤłèĥĥ":113394,"æĹ¥è¶ĭ":113395,"çζåŃIJ":113396,"ç·©":113397,"æĭĽçīĮ":113398,"产å¦ĩ":113399,"çķªèĮĦ":113400,"æĪijéĻ¢":113401,"建çŃijå·¥ç¨ĭ":113402,"å±ķè§Īä¼ļ":113403,"å®¶éķ¿ä»¬":113404,"åĨľä½ľçī©":113405,"æĹ¥å¤ľ":113406,"æĶ»æĵĬ":113407,"è§Ħéģ¿":113408,"èĪŁå±±":113409,"便æ°ij":113410,"åħ«åŃĹ":113411,"ä¸įæĽ¾":113412,"æĶ¯éħį":113413,"çĨ¬å¤ľ":113414,"人é¡ŀ":113415,"ç´ĢéĮĦ":113416,"ç»ıèIJ¥æ´»åĬ¨":113417,"大涨":113418,"å¸Ĥå§Ķ常å§Ķ":113419,"åĪĨéIJĺ":113420,"ä¸Ģ个èģĮä¸ļ":113421,"çĹħåĽł":113422,"è¿Ļ对äºİ":113423,"ä¸įå¾Ĺä¸į说":113424,"åıijçĶµæľº":113425,"æľīæīĢ帮åĬ©":113426,"缮æłĩä»»åĬ¡":113427,"åĽłåľ°":113428,"åĽłåľ°åζ":113429,"åĽłåľ°åĪ¶å®ľ":113430,"å°Ĩè¾¾åΰ":113431,"ç²Ĺç³Ļ":113432,"ç¨³åĽº":113433,"å«£":113434,"çİ°åľ¨å¾Īå¤ļ":113435,"ä¸ĸçķĮ级":113436,"å¼łæŁIJ":113437,"çĤ¹ç¼Ģ":113438,"èijµ":113439,"社ä¼ļç»Ħç»ĩ":113440,"å¾ĢåIJİ":113441,"åĬłæģ¯":113442,"åĻªå£°":113443,"æľīåħ´è¶£":113444,"为æĤ¨æıIJä¾Ľ":113445,"æ²¹æ¼Ĩ":113446,"ç¬¬åĽĽå±Ĭ":113447,"çļĩ宫":113448,"ä¹Ĵä¹ĵ":113449,"ä¹Ĵä¹ĵçIJĥ":113450,"éļ¨èijĹ":113451,"éģ©åIJĪ":113452,"åįĹéĿŀ":113453,"æĵ´":113454,"西æ´ĭ":113455,"åĬłå¯Ĩ":113456,"æĪIJåĬŁä¸¾åĬŀ":113457,"åı£æ°´":113458,"æĪIJ年人":113459,"æīĢæıIJä¾ĽçļĦ":113460,"éļĶå£ģ":113461,"åľ¨äº¬":113462,"å½ĵåľ°æĹ¶éĹ´":113463,"çŃīåIJĦç§į":113464,"é£İæ°Ķ":113465,"å±ĭéĩĮ":113466,"ä¸ĢåŃĹ":113467,"çļĦæĹ¶éĹ´éĩĮ":113468,"åĺ¿åĺ¿":113469,"快讯":113470,"ä¸Ńåľº":113471,"ä¸Ģçĵ¶":113472,"æ»ķ":113473,"é¢Ĩè·ij":113474,"好èݱ":113475,"好èݱåĿŀ":113476,"没åħ³ç³»":113477,"åĩºå¢ĥ":113478,"ä¸įæĺ¯ä¸Ģ个":113479,"éĥ½æĺ¯éĿŀ常":113480,"éľĩåĬ¨":113481,"èİ·èĥľ":113482,"åįļå¼Ī":113483,"æĬļåħ»":113484,"对ç«ĭ":113485,"æľįåĬ¡æľºæŀĦ":113486,"è°£è¨Ģ":113487,"社ä¼ļç§ijåѦ":113488,"åIJ¬è¯´è¿ĩ":113489,"æī³":113490,"æīĵ磨":113491,"åı£æľį":113492,"好åĥıæĺ¯":113493,"以åıĬåħ¶ä»ĸ":113494,"çī¹è´¨":113495,"亲è¿ij":113496,"ä¸Ģç»ı":113497,"æ¶Ŀ":113498,"éŃĶæľ¯":113499,"éģĵ路交éĢļ":113500,"è§Ħ模æľĢ大":113501,"å®ŀæĸ½æĦıè§ģ":113502,"ä¹ŀ":113503,"ä¸Ģä¸ĸ":113504,"åŁ·è¡Į":113505,"è±Ĩçĵ£":113506,"åĪĹ为":113507,"æķħ宫":113508,"çĶŁåij½åij¨æľŁ":113509,"ä¸īç§įèģĮä¸ļ":113510,"详ç»Ĩä»ĭç»į":113511,"å®Įå¤ĩ":113512,"å²©çŁ³":113513,"éļıæīĭ":113514,"飲":113515,"æķĪæŀľåĽ¾":113516,"ç§ĭåĨ¬":113517,"åĬŁå¾·":113518,"è§Ħ竳åĪ¶åº¦":113519,"æĹ¥æ¸IJ":113520,"æīĢéľĢè¦ģ":113521,"æīĢéľĢè¦ģçļĦ":113522,"å²Ľä¸Ĭ":113523,"åĩºåľŁ":113524,"åĽ¾æĸĩ":113525,"ç§ijæĬĢè¿ĽæŃ¥":113526,"éĢļèĥĢ":113527,"èĢģ太太":113528,"èĭĹæľ¨":113529,"éĵ¶å·Ŀ":113530,"å¸IJ篷":113531,"éĿŀè¦ģ":113532,"éħįç͵":113533,"å¤Ħå¢ĥ":113534,"èĤ¡æĿĥæĬķèµĦ":113535,"ä¸Ģ缴åΰ":113536,"åĿĩçͱ":113537,"æĬĹæĹ¥":113538,"æį®ä»ĭç»į":113539,"ä½łåĸľæ¬¢":113540,"åĪĽæĸ°åŀĭ":113541,"åıĺè¿ģ":113542,"è§Ĩå¯Ł":113543,"å®Įåħ¨æ²¡æľī":113544,"åħĥæĹ¦":113545,"åı¯ä¿¡":113546,"åı¦è¡Į":113547,"æĿij级":113548,"åħ¥åľº":113549,"æIJŃæ¡£":113550,"ä¹ŁåĽłæŃ¤":113551,"æį¢æĪIJ":113552,"ä¸įè´Ł":113553,"äºĨ大éĩıçļĦ":113554,"éģĶåΰ":113555,"å¸Ĥåİ¿":113556,"å¹´è¼ķ":113557,"å¿«æīĭ":113558,"å¸Įå°Ķ":113559,"èĩªèIJ¥":113560,"éĽªèĬ±":113561,"æIJģ":113562,"çľ¼ç§ij":113563,"æŃ£ç¢º":113564,"çļĦå§¿æĢģ":113565,"åĿļå®ŀçļĦ":113566,"æĮĩ纹":113567,"æªĶæ¡Ī":113568,"ç½®äºİ":113569,"佩æľį":113570,"豪éŨ":113571,"åĵĴ":113572,"æģ°å¥½":113573,"æª¢æŁ¥":113574,"åĪĿè¡·":113575,"大åĶIJ":113576,"约ä¼ļ":113577,"èĴ¸åıij":113578,"çѹåĪĴ":113579,"å¹´ç»Ī":113580,"è¡Įæ¥Ń":113581,"åħ±éĿĴ":113582,"åħ±éĿĴåĽ¢":113583,"ä¼ļå¼ķèµ·":113584,"ä¸Ńç§ij":113585,"ä¸Ńç§ijéĻ¢":113586,"æĮ¯åĬ¨":113587,"åį´åıijçݰ":113588,"ä¸įåĬ¨äº§":113589,"èĮ¹":113590,"æĪ¿éĹ´éĩĮ":113591,"è´§å¸ģæĶ¿çŃĸ":113592,"æ²»çĻĤ":113593,"æħİéĩį":113594,"å¡ŀå°Ķ":113595,"åĽ½ç±į":113596,"åĽłæŀľ":113597,"çŃīçī¹çĤ¹":113598,"山谷":113599,"ä¸ĭè¼ī":113600,"è®ĵæĪij":113601,"饮éħĴ":113602,"è¿Ļ个游æĪı":113603,"ç»Ŀ大éĥ¨åĪĨ":113604,"åĴ¨è¯¢æľįåĬ¡":113605,"干活":113606,"è®®ä¼ļ":113607,"æ¦Ĥè¿°":113608,"åĪĨåĮº":113609,"æŃ»åIJİ":113610,"ç«ĻçĿĢ":113611,"主è¦ģé¢Ĩ导":113612,"åIJĮåŁİ":113613,"大æłij":113614,"对åѦçĶŁ":113615,"社ä¼ļä¿ĿéĻ©":113616,"å¢ŀèµĦ":113617,"主人åħ¬":113618,"å®£ä¼łæķĻèĤ²":113619,"æĸĩåĮĸ交æµģ":113620,"客æĪ¶":113621,"çŁ¥åIJįåĵģçīĮ":113622,"æ»ŀåIJİ":113623,"äºĴè¡¥":113624,"æĦŁäºº":113625,"åī¿":113626,"åIJİ代":113627,"äºī龸":113628,"æķĻèĤ²åٹè®Ń":113629,"éĿĻèĦī":113630,"ä¹ıåĬĽ":113631,"说åĩºæĿ¥":113632,"çİĭèĢħèį£èĢĢ":113633,"åĢ«":113634,"åįĩèµ·":113635,"éķģ":113636,"åĩºæ¸¸":113637,"éĢļè¡Įè¯ģ":113638,"å·¥ä½ľå²Ĺä½į":113639,"åĮłå¿ĥ":113640,"æĭ¿æĿ¥":113641,"æ´Ĺè¡£æľº":113642,"æĪijä¸įæĥ³":113643,"é¢Ħè§ģ":113644,"æ¼Ķ示":113645,"ä¸ĢçĽ´æ²¡æľī":113646,"è·Łå¥¹":113647,"对çħ§æ£ĢæŁ¥":113648,"ç°¿":113649,"ä¸ĵå¿ĥ":113650,"è®®äºĭ":113651,"åīį端":113652,"åį¡å°Ķ":113653,"è¨Ńå®ļ":113654,"设置äºĨ":113655,"å©ļ纱":113656,"åľ¨åĽ½å¤ĸ":113657,"åı³ä¾§":113658,"è³¼çī©":113659,"å¥ĩèij©":113660,"å¢ŀåĬłå̼":113661,"好è¿IJ":113662,"åĽ½éĻħæľºåľº":113663,"ä¸ĭç§°":113664,"缮åīį为æŃ¢":113665,"ç¥ŀä»Ļ":113666,"å®ĥåı¯ä»¥":113667,"æ¾Ħæ¸ħ":113668,"èĥ½ä½¿":113669,"游åĩ»":113670,"游åĩ»éĺŁ":113671,"åĩ¹":113672,"ä¸įè¦ģåĨį":113673,"åĨ³èĥľ":113674,"åĨ³æĪĺ":113675,"æĭ½":113676,"缼åħ¸":113677,"å¾Īå¥½åľ°":113678,"æľĢç¾İçļĦ":113679,"åĥļ":113680,"å·´åŁº":113681,"å·´åŁºæĸ¯åĿ¦":113682,"æľĢéĢĤåIJĪ":113683,"é«ĺèģĮ":113684,"ä¿Ŀå§Ĩ":113685,"æİĪæ¬Ĭ":113686,"说åΰè¿ĻéĩĮ":113687,"æİ¨å¼Ģ":113688,"çİĩè¾¾":113689,"ä¸īåĪĨä¹ĭä¸Ģ":113690,"管çIJĨä¸Ńå¿ĥ":113691,"交æ±ĩ":113692,"森æŀĹåħ¬åĽŃ":113693,"å¾Ģä¸Ĭ":113694,"éªijè¡Į":113695,"æį®æŃ¤":113696,"纽带":113697,"ç»ŀ":113698,"ä¸īæĸ¹":113699,"æĦıä¹īä¸ĬçļĦ":113700,"æİ¨è¿Ł":113701,"å¤ļæł·æĢ§":113702,"æĥ³èµ·äºĨ":113703,"æİĴåIJį第":113704,"å·¨é¢Ŀ":113705,"æĿŁç¼ļ":113706,"å®īå®ļ":113707,"äºĭ實":113708,"çļĦæĦ¿æľĽ":113709,"è£ħå¤ĩåζéĢł":113710,"人å±ħ":113711,"人å±ħçݯå¢ĥ":113712,"å¿ĺè®°äºĨ":113713,"该游æĪı":113714,"楼ä¸Ĭ":113715,"å¼Ģä¼ļ":113716,"æģ³":113717,"åıĭæĥħéĵ¾æİ¥":113718,"ç¡Ĵ":113719,"ç»ĻäºĪäºĨ":113720,"åģı好":113721,"åĵī":113722,"交éĢļå®īåħ¨":113723,"éĽĮ":113724,"æ²»çĹħ":113725,"è§īå¾Ĺå¾Ī":113726,"衬衫":113727,"å¿ĥæĦ¿":113728,"æ´ŀå¯Ł":113729,"æ°ijæ£Ģå¯ŁéĻ¢":113730,"æıIJçĤ¼":113731,"è¦ģè¿Ľä¸ĢæŃ¥":113732,"驾车":113733,"æĻ®æĥł":113734,"æķĸ":113735,"ç¦ıéŁ³":113736,"éĢģè¾¾":113737,"è§ĦåĪĴ设计":113738,"æīĭå¥Ĺ":113739,"å®īä¿Ŀ":113740,"è¿ĺä¸įå¦Ĥ":113741,"åīįè¿°":113742,"æłĩè®°":113743,"ç´§æİ¥çĿĢ":113744,"æ§IJ":113745,"æ·±æ·±åľ°":113746,"满满çļĦ":113747,"æĺ¥è¿IJ":113748,"æĹ¥äº§":113749,"çαæĬ¤":113750,"åħ¨æĹ¥":113751,"åħ¨æĹ¥åζ":113752,"转åĬ¨":113753,"ç¥Ńç¥Ģ":113754,"ä¹°ä¸ľè¥¿":113755,"å¯¹æľªæĿ¥":113756,"æ¶Ī失äºĨ":113757,"åļ´éĩį":113758,"ä¸īæĿ¡":113759,"éħ¸å¥¶":113760,"éĽĨåĽ¢èĤ¡ä»½":113761,"西路":113762,"åıªå¾Ĺ":113763,"éĢģåİ»":113764,"çĭłæĬĵ":113765,"åĪ©ç͍çİĩ":113766,"ä¸ĭåij¨":113767,"å¥ĭæĪĺ":113768,"æĺ¥èĬĤæľŁéĹ´":113769,"è´Łè´£ä»»":113770,"æĺĤè´µ":113771,"尾巴":113772,"ç¯ĩæĸĩ竳":113773,"åħ®":113774,"è®ĬæĪIJ":113775,"å¹¹":113776,"çĻ»éĮĦ":113777,"ä½Ī":113778,"å·¥åĮł":113779,"åĵªæĢķæĺ¯":113780,"åıįåĵį":113781,"ç§ĥ":113782,"åĩºè½¨":113783,"æĹ¥åĨĽ":113784,"åIJįèªī":113785,"æķıéĶIJ":113786,"æľįåĬ¡æ°´å¹³":113787,"çħ§å°Ħ":113788,"ä¼Ĭæĭī":113789,"ä¼Ĭæĭīåħĭ":113790,"åĨħéĺģ":113791,"èĬĴæŀľ":113792,"ä¸ĩåĪĨ":113793,"éĢĢæ¬¾":113794,"缴æĴŃéĹ´":113795,"æĭ¿åΰäºĨ":113796,"å°İèĩ´":113797,"空æ°Ķä¸Ń":113798,"客æĪ·æľįåĬ¡":113799,"è¿IJåĬ¿":113800,"ç»ĵçŁ³":113801,"ä¸įå¿ħè¦ģçļĦ":113802,"èĥ¶åĽĬ":113803,"çIJĨä¼ļ":113804,"æĬ½åĩº":113805,"空æ°Ķè´¨éĩı":113806,"æ¯ķ竣æĺ¯":113807,"åĨ·æ¼ł":113808,"ä¸Ģå¦Ĥ":113809,"ä¸Ģå¦ĤæĹ¢":113810,"ä¸Ģå¦ĤæĹ¢å¾Ģ":113811,"æĤ£çĹħ":113812,"åĬłæĮģ":113813,"èµŀåĬ©":113814,"é«®":113815,"åij½ä¸Ń":113816,"æĦıä¹īä¸Ĭ":113817,"ä¸įèĪį":113818,"å쬦":113819,"æīĵæī«":113820,"æĺŁåħī":113821,"æĸŃè£Ĥ":113822,"åħ¨å¥Ĺ":113823,"è£ģå®ļ":113824,"马åħĭæĢĿ":113825,"骨骼":113826,"ä¸Ģè·¯ä¸Ĭ":113827,"å®ļæĹ¶":113828,"å·¥ç¨ĭæĬĢæľ¯":113829,"å½¼å¾Ĺ":113830,"æ±²åıĸ":113831,"ä¸Ģè§Ī":113832,"åIJµæŀ¶":113833,"ä¿Ĺç§°":113834,"æłªæ´²":113835,"åºŁæĹ§":113836,"è¡ĮæĺŁ":113837,"åıijçĶŁåıĺåĮĸ":113838,"é¦ĸä»ĺ":113839,"åįģåĪĨéĩįè¦ģ":113840,"æĬĬè¿ĻäºĽ":113841,"ç¥ŀå·ŀ":113842,"æıIJä¾ĽåķĨ":113843,"楷":113844,"å±İ":113845,"çĬ¶åħĥ":113846,"åŁİå¢Ļ":113847,"çľĭä¸Ģçľĭ":113848,"çĶŁäº§èĥ½åĬĽ":113849,"åŁºæľ¬ä¸Ĭéĥ½":113850,"æīĵæī°":113851,"åĪĿ次":113852,"åĩºç¤º":113853,"åħ¶ä¸Ńä¸Ģ个":113854,"çĶŁæĢģç³»ç»Ł":113855,"æīĭæİĮ":113856,"æµİåįĹå¸Ĥ":113857,"åľĭåħ§":113858,"æŃ£å̼":113859,"å¹¾ä¹İ":113860,"æİ¨èįIJéĺħ读":113861,"è¿Ń代":113862,"è°ĥä¾ĥ":113863,"饮åĵģ":113864,"å¢Ļä½ĵ":113865,"åıĺçݰ":113866,"äºĨ好":113867,"äºĨ好åĩł":113868,"ä¸įçķĻ":113869,"çβ":113870,"å°½æĹ©":113871,"æŃ£åľ¨è¿Ľè¡Į":113872,"åĩºéĻ¢":113873,"æĿĢ害":113874,"æıIJ款":113875,"åıijå±ķ空éĹ´":113876,"åīį身":113877,"ä¸įæĸŃå¢ŀ强":113878,"æ·±å±Ĥ次":113879,"容纳":113880,"éĤ£ä»½":113881,"å·¥ä½ľæķĪçİĩ":113882,"æľ¬åĽ½":113883,"失èIJ½":113884,"æŃ£åĽłä¸º":113885,"èĬĤæ°´":113886,"ä¸ĭä¸Ģ代":113887,"çłĶåıijä¸Ńå¿ĥ":113888,"ä¸įçIJĨ":113889,"å®Į好":113890,"ä¿ĿæĬ¤åĮº":113891,"ç»ĵæŀĦè°ĥæķ´":113892,"å¥łå®ļ":113893,"宣称":113894,"éĺ»æĮ¡":113895,"æĴ¤ç¦»":113896,"ä¸įæĸ¹ä¾¿":113897,"åĴķ":113898,"ç¬ijäºĨç¬ij":113899,"çݯå¢ĥ污æŁĵ":113900,"ä½ıæĪ·":113901,"ç»Ŀç¼ĺ":113902,"éϤå°ĺ":113903,"é«ĺå°ļ":113904,"æĢİä¹Īåı¯èĥ½":113905,"éĿ¢èī²":113906,"åķĨæ¥Ń":113907,"çĸ¹":113908,"èµĦæºIJä¼ĺåĬ¿":113909,"è¾ĸåĮºåĨħ":113910,"èĢĢçľ¼":113911,"æij§æ¯ģ":113912,"ä¸ĸçķĮç»ıæµİ":113913,"å¼ķæĿ¥":113914,"ä¸ĢåĪĻ":113915,"æĭĩæĮĩ":113916,"æĬµå¾¡":113917,"éĽį":113918,"åĩĨå¤ĩå·¥ä½ľ":113919,"çıłä¸īè§Ĵ":113920,"ç¨ĢåľŁ":113921,"èİ·å¾ĹæĦŁ":113922,"æĪIJåĬŁçİĩ":113923,"ç½ij约":113924,"ç½ij约车":113925,"èĦIJ":113926,"æķ¬ä¸ļ":113927,"éĩijä»·":113928,"ç²¾é«ĵ":113929,"买车":113930,"åħ³åı£":113931,"åĨįå¤ļ":113932,"æŀģåĵģ":113933,"åIJĦå®¶":113934,"举æĬ¥ç͵è¯Ŀ":113935,"èļĬ":113936,"æĸ¹å½¢":113937,"ç§ijæĬĢæĪIJæŀľ":113938,"æľĢ好æĺ¯":113939,"éĹ®åĢĻ":113940,"红éħĴ":113941,"åĽĽç§į":113942,"ç¿Ĵæħ":113943,"ç¿Ĵæħ£":113944,"åŀ¦":113945,"éĤ£åıª":113946,"é¢ĨæĤŁ":113947,"çľ¼éĥ¨":113948,"æ³°å®ī":113949,"ä»»æľŁ":113950,"磨æįŁ":113951,"æĽ¿æį¢":113952,"åħ¸ç¤¼":113953,"符åIJĪæĿ¡ä»¶":113954,"è¿ĺæľīä»Ģä¹Ī":113955,"åħ±äº«åįķ车":113956,"åı¯åĪĨ为":113957,"åŃ£åIJİ":113958,"åŃ£åIJİèµĽ":113959,"举èİŀå¸Ĥ":113960,"å¿ĥæĦı":113961,"æīŃæĽ²":113962,"ä½ľä¸ºä¸Ģç§į":113963,"è¿Ļéĥ¨åĪĨ":113964,"åıĤä¸İåΰ":113965,"ç½ijçIJĥ":113966,"實çı¾":113967,"ç»Ħè£ħ":113968,"åIJijå¤ĸ":113969,"å·¥ä½ľæĸ¹æ¡Ī":113970,"åįģæĿ¡":113971,"課ç¨ĭ":113972,"颤æĬĸ":113973,"åĵ©":113974,"éĤ®å¯Ħ":113975,"亢":113976,"åħįè²»":113977,"秤":113978,"åºĶæĢ¥ç®¡çIJĨ":113979,"åĽĽäºĶ":113980,"éºĴéºŁ":113981,"å¾ĴæŃ¥":113982,"è¨ĺå¾Ĺ":113983,"çĴIJ":113984,"æĺ¯åIJ¦ä¼ļ":113985,"æĦıè§ģåıįé¦Ī":113986,"éļ¾æĢª":113987,"çªį":113988,"交æİ¥":113989,"两åįĥ":113990,"æĩīç͍":113991,"æľŁéĸĵ":113992,"æIJ¬åΰ":113993,"è®®é¢ĺ":113994,"碧æ¡Ĥ":113995,"碧æ¡ĤåĽŃ":113996,"åģļçĶŁæĦı":113997,"éĻĽä¸ĭ":113998,"è·ĭ":113999,"èĢģ人家":114000,"带åĽŀ":114001,"æŀ¸æĿŀ":114002,"è¡Įéķ¿":114003,"åĨħ容ç®Ģä»ĭ":114004,"梢":114005,"æĮĩæİ§":114006,"éĩįçĹĩ":114007,"ç½ijåıĭ们":114008,"çı¾ä»£":114009,"类产åĵģ":114010,"å¥Ķæ³¢":114011,"渺":114012,"ç²īç¢İ":114013,"è¿Ļåıªæĺ¯":114014,"æ£Ģå¯Łæľºåħ³":114015,"é½Ĭ":114016,"æĪ¿ç§Ł":114017,"å¾·æĭī":114018,"å²ģ以ä¸Ĭ":114019,"纯åĩĢ":114020,"åĪĨå¸ĥåľ¨":114021,"èĥ½å¾Ĺåΰ":114022,"ä¸įå°½":114023,"ç«ŀä»·":114024,"çļĦ带é¢Ĩ":114025,"çļĦ带é¢Ĩä¸ĭ":114026,"ä¸Ńè᝿ĿIJ":114027,"æĿijéķĩ":114028,"ä¸įåı¯éģ¿åħį":114029,"éľ²å¤©":114030,"å°ıå§ijå¨ĺ":114031,"çī©ä»¶":114032,"èijĹä½ľæĿĥ":114033,"æĭĺçķĻ":114034,"éĥ½è§īå¾Ĺ":114035,"æĽ²æĬĺ":114036,"æ·»åĬłåīĤ":114037,"åı¬åĽŀ":114038,"æīİå®ŀæİ¨è¿Ľ":114039,"æĬĦè¢Ń":114040,"åĮĸ身":114041,"缴èIJ¥":114042,"ä¹Łå¸ĮæľĽ":114043,"èį£èªīç§°åı·":114044,"åįĸç»Ļ":114045,"æľīä¸įåIJĮçļĦ":114046,"å¥ĩçī¹":114047,"éĥ½è®¤ä¸º":114048,"å¦ŀ":114049,"æĪIJéķ¿ä¸º":114050,"辩æĬ¤":114051,"主æķĻç»ĥ":114052,"æ³ķå¸ĪèģĮä¸ļ":114053,"æ¤įåħ¥":114054,"索尼":114055,"åIJ¬è¿ĩ":114056,"ä¹łæĥ¯äºĨ":114057,"夺åıĸ":114058,"éŁĵ":114059,"æľ¬è´¨ä¸Ĭ":114060,"æİ¥åĬĽ":114061,"äºij端":114062,"è¦ģåģļ好":114063,"è·¯çģ¯":114064,"åįıåIJĮåıijå±ķ":114065,"æľīå¾ħ":114066,"æ°´åŁŁ":114067,"æIJľçĭIJé¦ĸ页":114068,"è´¨éĩıå®īåħ¨":114069,"åįģäºĮäºĶ":114070,"åĵ®åĸĺ":114071,"èĵ¬åĭĥåıijå±ķ":114072,"åIJį声":114073,"身亡":114074,"çİĭåºľ":114075,"åİŁåĪĻä¸Ĭ":114076,"çĥĺå¹²":114077,"éģĹæ¼ı":114078,"éĿ¢çĽ®":114079,"åĽ½ä¼ļ":114080,"ä¸Ģ缴éĥ½æĺ¯":114081,"æľīä¸Ģä½į":114082,"éħįæľī":114083,"éĻªçĿĢ":114084,"ä¼ģåĽ¾":114085,"æĮīä¸ĭ":114086,"èĵĿåĽ¾":114087,"æ©ĺ":114088,"大å¤ļæĺ¯":114089,"辩论":114090,"æĹĭå¾ĭ":114091,"æĬ¥éĢģ":114092,"æĿ¡è§Ħå®ļ":114093,"åĬ¨éĿĻ":114094,"åĮĪ奴":114095,"æĭľè®¿":114096,"ä¸ĢåĪĢ":114097,"ä»ĸçŁ¥éģĵ":114098,"主æĿĥ":114099,"ä»ĸæĽ¾":114100,"æĴŃç§į":114101,"å£ģåŀĴ":114102,"çī¢è®°ä½¿åij½":114103,"åľ¨è¿Ļæĸ¹éĿ¢":114104,"æīĭèħķ":114105,"æĶ¯æŀ¶":114106,"ä¾Ĩèĩª":114107,"éĩįå¡ij":114108,"å¤ļå±Ĥ次":114109,"ä»ĭè´¨":114110,"éĿ¢åŃĶ":114111,"潮湿":114112,"åİ¿åŁŁ":114113,"游æĪıå½ĵä¸Ń":114114,"å£ŀ":114115,"åĪĹåĩº":114116,"èµĽåĮº":114117,"å¤ļåįĬ":114118,"éĩįçĤ¹å·¥ä½ľ":114119,"æĪij们å¿ħé¡»":114120,"æŁıæŀĹ":114121,"é²ģèĥ½":114122,"æĸ½å±ķ":114123,"åIJĦåĮº":114124,"åħįç¨İ":114125,"èµĽåIJİ":114126,"æľĢéĩįè¦ģ":114127,"ä¸Ģ个好çļĦ":114128,"è¿Ŀæ³ķè¿Ŀè§Ħ":114129,"äºĨè§£æĽ´å¤ļ":114130,"æķ¬è¯·":114131,"ç¬ijçĿĢ说":114132,"ä¸įæĸŃåıijå±ķ":114133,"æijĦå½±å¸Ī":114134,"以éĺ²":114135,"çĤ¸å¼¹":114136,"声åĵį":114137,"ç¤ģ":114138,"æĩ¿":114139,"èĪĨæĥħ":114140,"èĩªçĶ±è´¸æĺĵ":114141,"æķıæį·":114142,"ä¸ī大éĺ¶æ®µ":114143,"èĭĶ":114144,"æĹºåŃ£":114145,"ä¸į满æĦı":114146,"微信åı·":114147,"修为":114148,"çł´è£Ĥ":114149,"éĢĥ离":114150,"æ¯ıèĤ¡":114151,"è¾¾ä¸įåΰ":114152,"æ¯ıå¹´éĥ½":114153,"çģ¯ç¬¼":114154,"æŃ¤åŁºç¡Ģä¸Ĭ":114155,"åĥı个":114156,"åĪĨ娩":114157,"æĻ¾":114158,"ä¸įèĩ³äºİ":114159,"红线":114160,"误解":114161,"ä¸ľè·¯":114162,"æ·®å®ī":114163,"产åѦ":114164,"产åѦçłĶ":114165,"è»ĭ":114166,"è»ĭçĹħ":114167,"åīįæıIJæĺ¯":114168,"æ¯ıä¸Ģ天":114169,"ä¸ĥ大":114170,"æłijåı¶":114171,"èµ°å¾Ĺ":114172,"è¿Ļ两ç§į":114173,"æİıåĩº":114174,"æİIJ":114175,"é¢Ĩ导èĢħ":114176,"ä¸Ģæľµ":114177,"个å¤ļæľĪ":114178,"ä¸Ńåħ³":114179,"ä¸Ńåħ³æĿij":114180,"课åłĤæķĻåѦ":114181,"大åĴĸ":114182,"éģĭç͍":114183,"è¯ļæĦı":114184,"ç»ĦåĽ¾":114185,"è¯ķçĿĢ":114186,"ä¹Ķæ²»":114187,"è¿ĺä¸įæĺ¯":114188,"æľīæĽ´å¥½çļĦ":114189,"åIJİå¤ĩ":114190,"æĸ°çĶŁåĦ¿":114191,"æ°Ķè¡Ģ":114192,"æ²¥éĿĴ":114193,"å±ıéļľ":114194,"æ¥ŃåĭĻ":114195,"æĪij以为":114196,"éķ¿çĽ¸":114197,"èĢģçΏ":114198,"éķĩæ±Ł":114199,"æľºæ¢°è®¾å¤ĩ":114200,"ä½Ĩæĺ¯å¦Ĥæŀľ":114201,"åĿļå®ļä¸į":114202,"åĿļå®ļä¸įç§»":114203,"åĨ²éĶĭ":114204,"ç®Ģ缴æĺ¯":114205,"åĤ¨èĵĦ":114206,"纯ç͵åĬ¨":114207,"漫æŃ¥":114208,"举起":114209,"æģ¶æĢ§":114210,"è¨ĺéĮĦ":114211,"èģĮèĥ½éĥ¨éŨ":114212,"åħ¨éķ¿":114213,"鼻è¦ĸ":114214,"ä¹³èħº":114215,"ä½ķå¤Ħ":114216,"æ¶Īæŀģ":114217,"æŃ£å¤Ħäºİ":114218,"å®īå®ģ":114219,"æĪIJéķ·":114220,"åıĻè¿°":114221,"æºĥçĸ¡":114222,"ä½Ĩçİ°åľ¨":114223,"女æĺŁ":114224,"å©´å¹¼åĦ¿":114225,"æĬķèŀįèµĦ":114226,"éĹ®éĹ®":114227,"æıŃå¼Ģ":114228,"è¯ı":114229,"åIJįå½ķ":114230,"èĺijèıĩ":114231,"åIJĬé¡¶":114232,"æ¹ĸåĮº":114233,"åįĸåľº":114234,"建ç¯":114235,"建ç¯ī":114236,"èݽ":114237,"åIJ¬åIJ¬":114238,"ç«ŀäºīä¼ĺåĬ¿":114239,"åĩºä»»":114240,"æľī两ç§į":114241,"æ©±æŁľ":114242,"褪":114243,"è¯ķåį·":114244,"ç»ıæµİæĬĢæľ¯":114245,"æ·±å±Ĥ":114246,"éĩįè¦ģåĨħ容":114247,"é£İæİ§":114248,"çĬ¶æĢģä¸ĭ":114249,"éĥ¨éĸĢ":114250,"广汽":114251,"è§Ĥæij©":114252,"éģĹçķĻ":114253,"转账":114254,"æĮģä»ĵ":114255,"æĢ»è®¡":114256,"åľĺéļĬ":114257,"æĪ¿ä¸ľ":114258,"éĺĢéŨ":114259,"åħ¬åħ³":114260,"åħ³åĪĩ":114261,"èĤĺ":114262,"æķ¸æĵļ":114263,"ä¸īåįģå¹´":114264,"è§ģè¯ģäºĨ":114265,"å±Ĩ":114266,"çģ°å°ĺ":114267,"æ¦ľé¦ĸ":114268,"è¦ĨçĽĸçİĩ":114269,"ä»Ļ女":114270,"çĶŁäº§æĢ»":114271,"çĶŁäº§æĢ»å̼":114272,"æĪ¿è´·":114273,"æ±ŁåĮº":114274,"åħħçĶµæ¡©":114275,"çϾåIJĪ":114276,"確èªį":114277,"转移åΰ":114278,"éĥ½æĹłæ³ķ":114279,"纪念é¦Ĩ":114280,"çŃ¾ç½²äºĨ":114281,"å¹¶ä¸įå¤ļ":114282,"æĮł":114283,"ä¸į太好":114284,"ä¸ĸ代":114285,"误导":114286,"é«ĺ峰论åĿĽ":114287,"åħ¼å®¹":114288,"龸æ°Ķ":114289,"æĿ¥è®¿":114290,"æīĢ带æĿ¥çļĦ":114291,"æĺ¯ä¸Ģéĥ¨":114292,"æĻļé¥Ń":114293,"åİĨ代":114294,"åIJ¦åīĩ":114295,"ä¹ħä¹ħ":114296,"æľīæķĪæľŁ":114297,"诱åıij":114298,"æĢ»èµĦ产":114299,"æľ¬èº«å°±æĺ¯":114300,"çĶŁäº§åİĤå®¶":114301,"æĹ¶é«¦":114302,"èĢIJç͍":114303,"ä»İå°ıå°±":114304,"æĿ¡çº¦":114305,"èĭ±åĭĩ":114306,"ä¿Ĺè¯Ŀ说":114307,"寺åºĻ":114308,"å¿ĥçIJĨåģ¥åº·":114309,"ä»Ģä¹Īäºĭæĥħ":114310,"æ±īåŃĹ":114311,"çķĻä½ı":114312,"åįĹè·¯":114313,"ä¸ī项":114314,"丢äºĨ":114315,"æĥ³åΰäºĨ":114316,"çѹéĽĨ":114317,"éĻĦåĬłå̼":114318,"西è£ħ":114319,"ä¹ĭä½ľ":114320,"åģļçļĦäºĭ":114321,"çķ¶æĤ¨":114322,"çķ¶æĤ¨åľ¨":114323,"é¦ĸ款":114324,"ä¸įåľ¨ä¹İ":114325,"å·¥ç¨ĭæĸ½å·¥":114326,"éļIJéļIJ":114327,"åıĺ身":114328,"沿éĢĶ":114329,"æĤłæĤł":114330,"ä¿Ŀæļĸ":114331,"çĶŁæ´»åŀĥåľ¾":114332,"渤海":114333,"æŃ¦ä¾ł":114334,"女主è§Ĵ":114335,"举ä¾ĭ":114336,"æ·¨":114337,"çϽé¢Ĩ":114338,"è£ĻåŃIJ":114339,"è¿Ķè¿ĺ":114340,"è¿Īåĩº":114341,"é¾ĻéŨ":114342,"ç»ıæµİä½ĵ":114343,"æĶ¶å®ĺ":114344,"çķĮéĻIJ":114345,"è·³åĩº":114346,"åįĩå̼":114347,"绵éĺ³":114348,"çĸ¤çĹķ":114349,"çľĭæ¸ħ":114350,"æĭĴçµķ":114351,"è¥Ħéĺ³":114352,"课å¤ĸ":114353,"åŃIJåŃĻ":114354,"æŃĮè¯į":114355,"æĪIJåIJį":114356,"溶液":114357,"åĦĴå®¶":114358,"åķĨä¸ļåĮĸ":114359,"辨åĪ«":114360,"å¤ļè¾¾":114361,"ç½ijåºĹ":114362,"ä¹Ŀ大":114363,"ä¹Ŀ大精ç¥ŀ":114364,"æŃ¤ä¸¾":114365,"è¿ŀè½½":114366,"ä¸ĢåĢĭ人":114367,"è³½":114368,"æ¶µçĽĸäºĨ":114369,"è¦ıåĬĥ":114370,"åĽ½æĥħ":114371,"åį«çĶŁåģ¥åº·":114372,"积æŀģåĵįåºĶ":114373,"æĭĻ":114374,"åζåĬ¨":114375,"æĥ³è±¡åĬĽ":114376,"çļĦä¹IJè¶£":114377,"å¼łå®¶çķĮ":114378,"å´İ":114379,"éĩįåŀĭ":114380,"å¤ĸå¢Ļ":114381,"æĶ¾åѦ":114382,"è®¤çľŁåŃ¦ä¹ł":114383,"è´¬å̼":114384,"æ³ķæ¡Ī":114385,"æĬ¤èĤ¤åĵģ":114386,"éĻ·åħ¥äºĨ":114387,"请æĤ¨":114388,"åŀ¢":114389,"æķĻèĤ²èµĦæºIJ":114390,"交æĺĵå¹³åı°":114391,"æĹ¶è£ħ":114392,"ä¼łæŁĵçĹħ":114393,"æ¹ĸæ³Ĭ":114394,"èµĦ管":114395,"åݨå¸Ī":114396,"éĹľéį":114397,"éĹľéįµ":114398,"åĵĪåĵĪåĵĪ":114399,"çĽĹçªĥ":114400,"çĶľç¾İ":114401,"åºĦåĽŃ":114402,"缮åīįå·²ç»ı":114403,"è¾¹ä¸Ĭ":114404,"çģ«èĬ±":114405,"æĬ¥è®°èĢħ":114406,"æģĭæĥħ":114407,"ç´§åĩij":114408,"æ°´æµģ":114409,"è¿Ļæĺ¯æĪij们":114410,"æ³¥åľŁ":114411,"æĽ¾ä»»":114412,"æĸ¹è¨Ģ":114413,"åij¨åħŃ":114414,"åı·æ¥¼":114415,"ä¼ijåģĩ":114416,"误ä¼ļ":114417,"åĽ½åĢº":114418,"åīįå¤ķ":114419,"ä¸¤å¼ł":114420,"éĹ«":114421,"éŃĶ鬼":114422,"æĬĬæĮģ":114423,"èĬĤèĥ½çݯä¿Ŀ":114424,"æ¸ħæ´ģèĥ½æºIJ":114425,"èĤ¥æĸĻ":114426,"é«ĺé¢ij":114427,"å°±æľīäºĨ":114428,"交ä¼ļ":114429,"没éĴ±":114430,"éĽħæĢĿ":114431,"è¦ģåıĬæĹ¶":114432,"åŁ¹åħ»åѦçĶŁ":114433,"欣åĸľ":114434,"çĥŃæ°´åύ":114435,"é¾Ļæ¹ĸ":114436,"äºĮ楼":114437,"æĸ°æµªè´¢ç»ı":114438,"æĸ°åĬ¨èĥ½":114439,"èµ£å·ŀ":114440,"æĭ³å¤´":114441,"æµģåIJij":114442,"ä¹Łæĺ¯å¾Ī":114443,"åıijåĶ®":114444,"ä¸ŃåIJ«æľī":114445,"åIJĵå¾Ĺ":114446,"å·¨æĺŁ":114447,"æĹłæīĢè°ĵ":114448,"æ¯ĽåŃĶ":114449,"åħ¬åħ±äº¤éĢļ":114450,"çĤİçĥŃ":114451,"èµ·èįī":114452,"åĬłçĽŁåķĨ":114453,"说ä¸įåĩº":114454,"大åѦæ¯ķä¸ļ":114455,"å·¥ä¸ļåĽŃ":114456,"éłĺåŁŁ":114457,"åºĨåħ¸":114458,"æµģ产":114459,"èģ²éٳ":114460,"ä¼¼ä¹İæĺ¯":114461,"è´§æºIJ":114462,"æ·±åĪĩ":114463,"æ²»çĸĹæĸ¹æ³ķ":114464,"èµĦæºIJéħįç½®":114465,"ç¶²åıĭ":114466,"çĶ£":114467,"亥":114468,"èº²åľ¨":114469,"社ç§ij":114470,"è»Łé«Ķ":114471,"女è£ħ":114472,"æŃ¡è¿İ":114473,"综åIJĪå®ŀåĬĽ":114474,"æł¼å°ĩ":114475,"åħļåı²åŃ¦ä¹ł":114476,"æľĢåŁºæľ¬":114477,"æľĢåŁºæľ¬çļĦ":114478,"çľĭæľĽ":114479,"åıĹè´¿":114480,"ä¸įä»ħèĥ½":114481,"ä½ķå¿ħ":114482,"ä¸Ģ个å°ıæĹ¶":114483,"ç¾Į":114484,"æĭĽæĶ¶":114485,"çĤĴèĤ¡":114486,"æĿijå¹²éĥ¨":114487,"缸çα":114488,"æ½ľèĥ½":114489,"ä¹į":114490,"æĹ¶è¾°":114491,"欣æħ°":114492,"éĵ¶è¡Įä¸ļ":114493,"çĭŃçªĦ":114494,"éĩįçĤ¹é¢ĨåŁŁ":114495,"çݰå®ŀçĶŁæ´»":114496,"éĮ¯èª¤":114497,"æĸ°è§Ħ":114498,"滥ç͍":114499,"æĹ¶ä¸į":114500,"æĹ¶ä¸įæĹ¶":114501,"帳èĻŁ":114502,"ç¨Ģ缺":114503,"åIJij举":114504,"ä¿Ŀåģ¥åĵģ":114505,"çıŃéķ¿":114506,"äºĴåĭķ":114507,"笼罩":114508,"æ½Ľ":114509,"æļĸå¿ĥ":114510,"è½°çĤ¸":114511,"åºĨ幸":114512,"è²Įä¼¼":114513,"æĵº":114514,"èĢIJ磨":114515,"ä¸ĵä¸ļ人士":114516,"ä¸Ģèάéĥ½æĺ¯":114517,"æ¼³å·ŀ":114518,"åħ¨èĩªåĬ¨":114519,"å½ķç͍":114520,"大è·Į":114521,"æľīæķο̧":114522,"èĩªåĭķ":114523,"ä¸ī个æĸ¹éĿ¢":114524,"港åĮº":114525,"信貸":114526,"éĢļè¯Ŀ":114527,"é«ĺ涨":114528,"æ³Ħæ¼ı":114529,"éħįä¸Ĭ":114530,"åħļå·¥å§Ķ":114531,"被认为":114532,"被认为æĺ¯":114533,"ä¸įä¼ļåĨį":114534,"è°ĥåīĤ":114535,"åıĤèĤ¡":114536,"èĦ±åıij":114537,"å¿łå®ŀ":114538,"åĨħåĪĨæ³Į":114539,"ç¹ģå¿Ļ":114540,"åıĮåĪĽ":114541,"é©»æĿij":114542,"åĪĴç®Ĺ":114543,"éģİä¾Ĩ":114544,"åľ£ç»ı":114545,"èıľé¸Ł":114546,"æĭ¼å¤ļå¤ļ":114547,"ä¸ŃåĽ½æ±½è½¦":114548,"çĥŁèįī":114549,"缴æµģ":114550,"äºĨä¸Ģåı£æ°Ķ":114551,"ä½İæĪIJæľ¬":114552,"æī¾åĽŀ":114553,"èĩªåįij":114554,"總æĺ¯":114555,"æĸĩåĮĸåĪĽæĦı":114556,"天河":114557,"樱æ¡ĥ":114558,"éªijåħµ":114559,"éĩĮéĿ¢æľī":114560,"çİ®":114561,"èĥ½æī¾åΰ":114562,"éĢĥè·ij":114563,"åĪĩå°Ķ":114564,"åĪĩå°Ķ西":114565,"以ä¸ĭæĺ¯":114566,"å²³éĺ³":114567,"çļĦæ¦Ĥçİĩ":114568,"æĬµåζ":114569,"å¸ĪäºĭåĬ¡":114570,"å¸ĪäºĭåĬ¡æīĢ":114571,"åĩĨæĹ¶":114572,"屬æĸ¼":114573,"订è´Ń":114574,"åįłæį®äºĨ":114575,"ä¸ŃéĢĶ":114576,"å°ĭ":114577,"é»ij马":114578,"åİ¿åħ¬å®īå±Ģ":114579,"ä¸ĥæľĪ":114580,"èī²ç´ł":114581,"å¿ĥèĦıçĹħ":114582,"æĹ¶éĻIJ":114583,"æ¯įåħ¬åı¸":114584,"å¹ķåIJİ":114585,"ä¸Ĭæ¦ľ":114586,"å̾åIJijäºİ":114587,"纸ä¸Ĭ":114588,"æ¡ĵ":114589,"éĽĨä½ĵç»ıæµİ":114590,"æĥħå¢ĥ":114591,"è¦ģåģļåΰ":114592,"ç©į極":114593,"åıªæĢķ":114594,"æ¹ĺ西":114595,"çļ±çº¹":114596,"åħ¨åľĭ":114597,"çĦ¡è«ĸ":114598,"好æĦŁ":114599,"åįķä»·":114600,"è¿Ľç¨ĭä¸Ń":114601,"æĺĨä»ij":114602,"åĪĽå®¢":114603,"åħħæĸ¥":114604,"åħĪæĬĬ":114605,"该æĢİä¹ĪåĬŀ":114606,"åĵģå¾·":114607,"åħ¨éĿ¢åıijå±ķ":114608,"è¨ĪåĬĥ":114609,"æĢ»å·¥ä¼ļ":114610,"ä½Ľå±±å¸Ĥ":114611,"æĬĹè¡¡":114612,"å¼Ģåľº":114613,"éĴ±å¸ģ":114614,"åıĭ们":114615,"å«īå¦Ĵ":114616,"ç´¢èµĶ":114617,"è®ĬåĮĸ":114618,"æĮ¤åİĭ":114619,"æĮijè¡ħ":114620,"çŃīä¸Ģæī¹":114621,"æĿ¨æ¬¢":114622,"ä¸ĵå®¶åѦèĢħ":114623,"èĥ½è¾¾åΰ":114624,"èµ°è¿ij":114625,"è´«åĽ°åľ°åĮº":114626,"éĻIJæľŁ":114627,"ä¸į平衡":114628,"åĽ½åĨħå¸Ĥåľº":114629,"èµĽåľº":114630,"éħįèµĦ":114631,"è¦ģèĢĥèĻij":114632,"ä¸ĩåı°":114633,"æľĪæľ«":114634,"éĶ¥":114635,"åŃ«":114636,"æİ¥è§¦åΰ":114637,"åĩºäº§":114638,"æķĻåѸ":114639,"ä½ľå¼Ĭ":114640,"çļĦæľĢåIJİä¸Ģ":114641,"ä¿ĥæĪIJ":114642,"åIJ¸åıĸ":114643,"æ½ľèīĩ":114644,"被éªĹ":114645,"è¾ĵäºĨ":114646,"çĭIJçĭ¸":114647,"åįĩéĻį":114648,"è¿ĻäºĽä¸ľè¥¿":114649,"æĬķèµĦåŁºéĩij":114650,"çĶŁçī©åѦ":114651,"ç½ij绾èIJ¥éĶĢ":114652,"åIJijè®°èĢħ":114653,"èįīåľ°":114654,"æĢ¯":114655,"æľįåĬ¡èĥ½åĬĽ":114656,"éĥģéĹ·":114657,"åįķåĵģ":114658,"å¾Ĺ罪":114659,"æĺĵäºİ":114660,"个å¤ļå°ıæĹ¶":114661,"éĩįä»»":114662,"ä¸Ĭå®ĺ":114663,"æľ¬éĩij":114664,"çı¾åł´":114665,"溢价":114666,"æĺŁè¾°":114667,"æ´»åĬ¨çİ°åľº":114668,"丹麦":114669,"å¸Ŀçİĭ":114670,"æŁ¥æĺİ":114671,"åŃĺåľ¨äºİ":114672,"é¦Ļæ°´":114673,"æĬ½æ£Ģ":114674,"å®ŀéĻħä¸Ĭæĺ¯":114675,"æĸ°å¾ģç¨ĭ":114676,"è´¢åĬ¡ç®¡çIJĨ":114677,"æİĽ":114678,"åĨľåİĨ":114679,"éĥ½èĥ½å¤Ł":114680,"éĤ¯éĥ¸":114681,"çľŁå¯¦":114682,"ç»Ĭ":114683,"åĨµä¸Ķ":114684,"置身":114685,"ç¥Ī祷":114686,"çĿģå¼Ģ":114687,"æĮĩçĤ¹":114688,"å¼Ģæľº":114689,"西å®ģ":114690,"åĮĹ约":114691,"积水":114692,"åĩºåĬ¨":114693,"åıijå±ķ模å¼ı":114694,"转æĬĺ":114695,"èĢĥçĤ¹":114696,"æľīç½ijåıĭ":114697,"è´«åĽ°æĿij":114698,"æĪijä»¬çŁ¥éģĵ":114699,"åĪĨéĶĢ":114700,"å±±èĦī":114701,"æ¯ĶæĭŁ":114702,"ä¼°ç®Ĺ":114703,"æĶ¹å»º":114704,"壮è§Ĥ":114705,"ç§īæĮģ":114706,"æıª":114707,"ç¦Ģ":114708,"åĮĸåѦåĵģ":114709,"ä¸ŃåĽ½åζéĢł":114710,"ä¸Ģæŀ¶":114711,"æīįè¡Į":114712,"æĭĽå¾ħ":114713,"åıĺæį¢":114714,"åīį线":114715,"幸好":114716,"è¿Ļæł·çļĦè¯Ŀ":114717,"å¿ĥè¡Ģ管":114718,"æĢ§çĸ¾çĹħ":114719,"åħ¨èĥ½":114720,"åĪij侦":114721,"ä¿¡æģ¯åıijå¸ĥ":114722,"æĺ¾çĦ¶æĺ¯":114723,"éĿĴéĵľ":114724,"åIJĥä»Ģä¹Ī":114725,"ç͵价":114726,"æ³ķå¾ĭè§Ħå®ļ":114727,"çħ²":114728,"çĵ·åύ":114729,"èĤīç±»":114730,"æıĴåħ¥":114731,"åĹľ":114732,"è¿Łè¿Ł":114733,"ä¸ĢçĤ¹éĥ½ä¸į":114734,"è¿ĺåĮħæĭ¬":114735,"èĪįä¸įå¾Ĺ":114736,"æłĩå¿ĹæĢ§":114737,"æľĪ以æĿ¥":114738,"ç³ĸæŀľ":114739,"éĥ½åºĶ该":114740,"çݯå¢ĥåį«çĶŁ":114741,"èĪªè¡Į":114742,"éĥijéĩį":114743,"ç½ijæĬķ":114744,"åįģä½³":114745,"ç§ģä¸ĭ":114746,"æļ´è·Į":114747,"åĬłå¿«åıijå±ķ":114748,"产åĵģçłĶåıij":114749,"åĪĽéĢłåĩº":114750,"æĢ»è§īå¾Ĺ":114751,"åºķçĽĺ":114752,"èķĬ":114753,"åĩºå¸Ńä¼ļè®®":114754,"主æĿ¿":114755,"æĹ¥æĻļéĹ´":114756,"å®ĺæĸ¹å¾®åįļ":114757,"å¼ķç͍æĹ¥æľŁ":114758,"åķĻæİĪ":114759,"ç͵åŃIJ产åĵģ":114760,"è¡°éĢĢ":114761,"çķĻåŃĺ":114762,"çģ«åĬĽ":114763,"çĴ§":114764,"çļĤ":114765,"åħ¼åħ·":114766,"éĩįè¿Ķ":114767,"é¢Ĩçķ¥":114768,"åĪĩéϤ":114769,"åĨįçĶŁèĥ½æºIJ":114770,"å®ŀåľ¨å¤ª":114771,"çIJĨ论ä¸Ĭ":114772,"ä¸īå±Ĥ":114773,"ä¸ĸçķĮåIJĦåĽ½":114774,"å®ľæĺĮ":114775,"èĢ³è¾¹":114776,"宽æķŀ":114777,"æ±īæĹı":114778,"çϽçϽ":114779,"è¿ĻéĩĮéĿ¢":114780,"çĶŁæ´»ä¹łæĥ¯":114781,"èµŀèµı":114782,"çͷ士":114783,"ä¸Ńä¿Ħ":114784,"车祸":114785,"åīĤéĩı":114786,"éϤåİ»":114787,"左边":114788,"çŃijçī¢":114789,"çīĽå¸Ĥ":114790,"å®¶åĬ¡":114791,"åķĥ":114792,"ç½®æį¢":114793,"ç´«å¤ĸ":114794,"ç´«å¤ĸ线":114795,"å¾Ģåīį":114796,"åĬĽåѦ":114797,"ç´§è·Ł":114798,"缮çļĦåľ¨äºİ":114799,"ç»®":114800,"ç¥Ĥ":114801,"宣è¨Ģ":114802,"äºĮæ°§åĮĸ":114803,"äºĮæ°§åĮĸ碳":114804,"æĹłç¼ĺ":114805,"ç²¾éĢļ":114806,"診":114807,"å¼ķåıijäºĨ":114808,"æľĢåħĪ":114809,"派驻":114810,"ä¸įå¿į":114811,"æĪijçΏ":114812,"å¹´ä¸ĭåįĬå¹´":114813,"æ·ĭå·´":114814,"没éĹ®é¢ĺ":114815,"åºĹåĨħ":114816,"è·ŁæĪij说":114817,"çĶŁäº§çĶŁæ´»":114818,"è§ĤæľĽ":114819,"æ¸į":114820,"被æī§è¡Į":114821,"被æī§è¡Į人":114822,"èĪľ":114823,"æİº":114824,"ä¸Ģç§Ĵ":114825,"èįīåĿª":114826,"åij¼åĴĮ":114827,"åij¼åĴĮ浩":114828,"åij¼åĴĮ浩çī¹":114829,"人æ°ijéĵ¶è¡Į":114830,"çĦķåıij":114831,"è¯ģåĪ¸äº¤æĺĵ":114832,"çķĶ":114833,"æľºèĥ½":114834,"妾":114835,"æĻļå¹´":114836,"å·¥åķĨèģĶ":114837,"åİŁåŀĭ":114838,"è§Ĵ度çľĭ":114839,"æĬ¥ç¤¾":114840,"è¯įæĿ¡":114841,"躲éģ¿":114842,"éĩįåIJ¯":114843,"å¤ķéĺ³":114844,"èĤ¡æĿĥ转让":114845,"åľ¨ä¸Ģ":114846,"åľ¨ä¸ĢæĹģ":114847,"社ä¼ļåĮĸ":114848,"åıijå±ķåİĨç¨ĭ":114849,"æĭĸæ¬ł":114850,"使èĢħ":114851,"ä¸İåIJ¦":114852,"æĸ°å±ĢéĿ¢":114853,"ä»Ĭ天æĪij们":114854,"é½IJèģļ":114855,"对æĪij说":114856,"éĢĴ交":114857,"æľªæĽ¾":114858,"èİĬ":114859,"éĸī":114860,"亲æīĭ":114861,"è§ĴéĢIJ":114862,"æľīé»ŀ":114863,"ç¨İçİĩ":114864,"ä½İ声":114865,"é»ĺå¥ij":114866,"æĻ®æ³ķ":114867,"大ä¸ĵ":114868,"第äºĮ大":114869,"ä½ıåĿĢ":114870,"æĶ¾è¿Ľ":114871,"äºĮæĪĺ":114872,"亲身":114873,"åĽºåĮĸ":114874,"ä¸ĭ乡":114875,"åħ³éĶ®æĬĢæľ¯":114876,"åĽŀæĥ³":114877,"æĬ¥åĪĬ":114878,"æ¶ĤæĬ¹":114879,"èĹıçĿĢ":114880,"ç¥ĿæĦ¿":114881,"åįĩ温":114882,"çĶļèĩ³è¿ŀ":114883,"åħ¬åħĥåīį":114884,"ç¾İæĸ¹":114885,"è¯ļå®ŀ":114886,"æĹłåģ¿":114887,"å¥Ń":114888,"å°ıå¿ĥ翼":114889,"å°ıå¿ĥ翼翼":114890,"两æīĭ":114891,"温馨æıIJ示":114892,"ä»¿çľŁ":114893,"æĥ¶":114894,"èĥ¡åŃIJ":114895,"å·¥ä½ľç«Ļ":114896,"硬çĽĺ":114897,"ç«¿":114898,"åĤ³éĢģ":114899,"åħ¨æł¡":114900,"é²ľæ´»":114901,"çĴĢçĴ¨":114902,"ç»ĵå°¾":114903,"æį¢æĿ¥":114904,"æĪĢ":114905,"ä½İä½į":114906,"ä¸ĩåħĥ以ä¸Ĭ":114907,"åĬłåĪĨ":114908,"æİ¨ä»ĭä¼ļ":114909,"çIJĨèµĶ":114910,"å¾·å°Ķ":114911,"æĬĹè®®":114912,"æ´¼":114913,"åĸ§":114914,"åŁİéĻħ":114915,"å¾Īæ£Ĵ":114916,"人æŃ»äº¡":114917,"ä¼ļå±ķä¸Ńå¿ĥ":114918,"äºĴèģĶäºĴéĢļ":114919,"èĸĦèĨľ":114920,"éĩįé»ŀ":114921,"ç¦ģæ¯Ĵ":114922,"åĨ·ç¬ij":114923,"大家åı¯ä»¥":114924,"é¦ĸ缸":114925,"è¿ijè·Ŀ离":114926,"æµ®çݰ":114927,"ç§ĺè¯Ģ":114928,"èµ·é£ŀ":114929,"æIJ¶":114930,"羣åģĩ":114931,"æģķ":114932,"å°ıåºĹ":114933,"æ°ijçľ¾":114934,"åıijå¸ĥåħ¬åijĬ":114935,"ä¾§éĩį":114936,"å¾ĺå¾Ĭ":114937,"æĢĶ":114938,"æªIJ":114939,"æķ°çĽ®":114940,"åī¯ç§ĺ书éķ¿":114941,"两åı¥":114942,"éļIJçŀĴ":114943,"åıĮåıĮ":114944,"æīĭæĦŁ":114945,"èij¡äº¬":114946,"éģĹå¿ĺ":114947,"鬥":114948,"è¿Ļä¸ªåľ°æĸ¹":114949,"说çļĦè¯Ŀ":114950,"å·¡åĽŀ":114951,"è¿Ŀ竳":114952,"æī¾å·¥ä½ľ":114953,"æĶ¯çIJĥéĺŁ":114954,"裡éĿ¢":114955,"æĺ¾ç¤ºåĩº":114956,"èĩ³å°Ĭ":114957,"两级":114958,"åīįæ®µæĹ¶éĹ´":114959,"çĺ¦èº«":114960,"èĤ¢ä½ĵ":114961,"æ¯į親":114962,"æīĭç»Ńè´¹":114963,"汽车è¡Įä¸ļ":114964,"æİ©çĽĸ":114965,"æİ§èĤ¡éĽĨåĽ¢":114966,"åı£å¾Ħ":114967,"æĶ¿çŃĸæİªæĸ½":114968,"海绵":114969,"åħ¨éķĩ":114970,"äºĭåħ³":114971,"å¸Ńæī§è¡Į":114972,"å¸Ńæī§è¡Įå®ĺ":114973,"éĤ£æ¬¡":114974,"åı¯èĥ½åĩºçݰ":114975,"ä¸Ńå¿ĥåŁİå¸Ĥ":114976,"翻身":114977,"ä¹Łç®Ĺ":114978,"ä¾µçķ¥":114979,"åĸĩåıŃ":114980,"æ¯ı次éĥ½":114981,"è§ħ":114982,"éĻ¢éĻ¢éķ¿":114983,"å§ĭäºİ":114984,"èѦåĬ¡":114985,"è᝿ĿIJ":114986,"å±łæĿĢ":114987,"æľ¬èº«å°±":114988,"éļıæĹ¶éļı":114989,"éļıæĹ¶éļıåľ°":114990,"åĶ®åįĸ":114991,"æĹłäººé©¾é©¶":114992,"é¢ħ":114993,"åĵģ質":114994,"åĺ²ç¬ij":114995,"è·ijåİ»":114996,"åħĭéĩĮæĸ¯":114997,"çķ¸å½¢":114998,"修饰":114999,"磩éĺµ":115000,"éŁ³ä¹IJä¼ļ":115001,"æŁ³å·ŀ":115002,"齡":115003,"ä¼ļè°Ī":115004,"æŃ£çīĪ":115005,"ä¹ŁåIJĮæł·":115006,"æļ§æĺ§":115007,"è¡ĮæĶ¿éĥ¨éŨ":115008,"ä¹ĸä¹ĸ":115009,"èĤ¤èī²":115010,"æĹ¶ä»»":115011,"羣åĪĩ":115012,"æľĪä¸ĭ":115013,"æľĪä¸ĭæĹ¬":115014,"举æĸ¹è´¢å¯Į":115015,"è£ħä¿®åħ¬åı¸":115016,"éĢĢè¿ĺ":115017,"åĭĺå¯Ł":115018,"åĵ¥ä¼¦":115019,"åĵ¥ä¼¦æ¯Ķäºļ":115020,"çĭ¬ä¸Ģ":115021,"çĭ¬ä¸ĢæĹł":115022,"çĭ¬ä¸ĢæĹłäºĮ":115023,"è°ĥåij³":115024,"åİĭè¿«":115025,"åħ¨çIJĥæľĢ大":115026,"åł¡éķ¿":115027,"æĽ´ä½İ":115028,"åĪĨéĴŁåIJİ":115029,"åĽŀä¾Ĩ":115030,"åζåīĤ":115031,"åijĬè¯ī大家":115032,"çĤ¹éĴŁ":115033,"åįģä¸īå±Ĭ":115034,"åij¨åĽĽ":115035,"è¿Ļæł·ä¸Ģ":115036,"è¿Ļæł·ä¸ĢæĿ¥":115037,"èĭŁ":115038,"æľĽåİ»":115039,"æĪIJè¯Ń":115040,"å½ĵåį³":115041,"ç¬ij声":115042,"ä¹ĭåĬ¿":115043,"åĪijäºĭæ¡Īä»¶":115044,"æĮĤçĿĢ":115045,"ä½ķç§į":115046,"å°ı游æĪı":115047,"åĽ½å®¶æĪĺçķ¥":115048,"åĨ·åĨ·":115049,"å®ľå®¾":115050,"æIJºç¨ĭ":115051,"è¶ĭäºİ":115052,"åıįçľģ":115053,"常说":115054,"ä¸ĩæĪ·":115055,"åĥµå°¸":115056,"åįĥä¸ĩåĪ«":115057,"åıijçݰéĹ®é¢ĺ":115058,"åı¯çŁ¥":115059,"éŨæĪ·ç½ijç«Ļ":115060,"åģ¥åº·äº§ä¸ļ":115061,"åı³è¾¹":115062,"æµ·è¿IJ":115063,"è¿ijä¹İ":115064,"åĮ»æ²»":115065,"æĢ»ç®Ĺ":115066,"ä¸ĢåĪĨéĴŁ":115067,"æĭ§":115068,"ä¹Łæľīä¸ĢäºĽ":115069,"ä¾Ľç͵åħ¬åı¸":115070,"å»īä»·":115071,"帮ä»ĸ":115072,"æŃ¤æ¬¡æ´»åĬ¨":115073,"åıªèĥ½è¯´":115074,"èĬĭ":115075,"çīĩ段":115076,"åŃĺåľ¨éĹ®é¢ĺ":115077,"ä½łä¼ļåıijçݰ":115078,"è½®å»ĵ":115079,"ç½ijéĢļ":115080,"æ»¨æ±Ł":115081,"æİĪä¿¡":115082,"é»İæĺİ":115083,"ä¸įå±ŀäºİ":115084,"约åįł":115085,"éķ¿æ²Ļå¸Ĥ":115086,"èĥļèĥİ":115087,"åħĥä»¶":115088,"éĻĨåĨĽ":115089,"購買":115090,"æĮĩæľĽ":115091,"å®ŀä¹łçĶŁ":115092,"çī¹çĤ¹æĺ¯":115093,"çıłæ±Ł":115094,"çľĭä¸įåĩº":115095,"ä¸įè§ģäºĨ":115096,"ç¼ī":115097,"éĺµèIJ¥":115098,"åĶIJæľĿ":115099,"没å¿ħè¦ģ":115100,"åĽ½åľŁèµĦæºIJ":115101,"ç»ıæµİåѦ家":115102,"åIJĪèĤ¥å¸Ĥ":115103,"çIJ¢ç£¨":115104,"ç¡®åĪĩ":115105,"åŁİå¸Ĥåıijå±ķ":115106,"çŃ·åŃIJ":115107,"人æ°ijæľįåĬ¡":115108,"满åĪĨ":115109,"è¿·ä¿¡":115110,"ä½ľèĢħæľ¬äºº":115111,"æĸĩ竳æĿ¥æºIJ":115112,"ç«Ļç«ĭ":115113,"æŀĦæĪIJäºĨ":115114,"è¾Ľåĭ¤":115115,"è¶ħ强":115116,"éĶļ":115117,"åīįä¸īåŃ£åº¦":115118,"å°±è§īå¾Ĺ":115119,"å´ĩé«ĺ":115120,"è¶Ĭä¾Ĩ":115121,"è¶Ĭä¾Ĩè¶Ĭ":115122,"å¸ĤåľºèIJ¥éĶĢ":115123,"综åIJĪç´łè´¨":115124,"åŃļ":115125,"侮辱":115126,"äºĮåŃĹ":115127,"å·¥ä½ľä»»åĬ¡":115128,"åı²ä¸ĬæľĢ":115129,"æľĢä¼ĺ":115130,"åIJ©åĴIJ":115131,"表çϽ":115132,"èİ«åIJį":115133,"èİ«åIJįåħ¶":115134,"èİ«åIJįåħ¶å¦Ļ":115135,"å¹£":115136,"åIJĮå¿Ĺ们":115137,"建设çĶ¨åľ°":115138,"åĦĢ":115139,"éħįåģ¶":115140,"弩":115141,"åͱçīĩ":115142,"æīĭèĦļ":115143,"åħ¼ä»»":115144,"åģľæĶ¾":115145,"æŃ£å®Ĺ":115146,"æĸ°åĨľæĿij":115147,"åĤ¬çĶŁ":115148,"æīĢåŃ¦æł¡":115149,"å¿µä½Ľ":115150,"åͤéĨĴ":115151,"åħ±åĪĽ":115152,"æĭīä¸ģ":115153,"èĥĮçĿĢ":115154,"çĶŁæĢģä¿ĿæĬ¤":115155,"åı£å¤´":115156,"æĸ¹åIJijçĽĺ":115157,"調æķ´":115158,"æĭĽèģĺä¿¡æģ¯":115159,"åħ¶ä»ĸåĽ½å®¶":115160,"ç®Ģæĺĵ":115161,"åĮ¿åIJį":115162,"è¯Ħæµĭ":115163,"æĺ¯ä¸Ģ座":115164,"çīĭ":115165,"足迹":115166,"çIJĨè§£åĴĮ":115167,"æľĢåıĹ":115168,"å¿ĥè·³":115169,"çĪ¶è¦ª":115170,"éĿŀ常åĸľæ¬¢":115171,"èĭ¦éļ¾":115172,"æĬĢå¸Ī":115173,"æ°ijæĦı":115174,"æĪĺåĽ½":115175,"æĽ¿è¡¥":115176,"津贴":115177,"ä¸ŃåĽ½ä¼łç»Ł":115178,"åIJĦè¡Į":115179,"åIJĦè¡ĮåIJĦ":115180,"åIJĦè¡ĮåIJĦä¸ļ":115181,"第äºĶå±Ĭ":115182,"èį·èĬ±":115183,"æĦıèŃĺ":115184,"票价":115185,"åĪĨæµģ":115186,"æĿİçϽ":115187,"æ±ŁåĮĹ":115188,"æİĴæĸ¥":115189,"ä½ĵéĩı":115190,"åĮħåIJ«äºĨ":115191,"åĪĺæŁIJ":115192,"çݰå¦Ĥä»Ĭ":115193,"å·¥èīºåĵģ":115194,"è¿Ļç§įæĸ¹æ³ķ":115195,"åĬŀåħ¬æ¥¼":115196,"ç͵工":115197,"çħĻ":115198,"åį¡çīĩ":115199,"å¹´å¹´åºķ":115200,"ä¸ĵ项èµĦéĩij":115201,"åĮ»ç§ij":115202,"åĮ»ç§ij大åѦ":115203,"åĽŀ头çľĭ":115204,"ä¸įå±ij":115205,"èĩªé©¾":115206,"没æĶ¶":115207,"æīĵçĮİ":115208,"èĦ¸éĥ¨":115209,"åıĥèĢĥ":115210,"å°Ĩ士":115211,"è´«åĽ°äººåı£":115212,"çIJĨæĥ³ä¿¡å¿µ":115213,"é£İå°ļ":115214,"人æīįéĺŁä¼į":115215,"çij¾":115216,"æĿ¥è¿ĻéĩĮ":115217,"æ´Ĺ涤":115218,"å¹´èĸª":115219,"èĭįçϽ":115220,"ä¸ĩäºĭ":115221,"è¯¾æľ¬":115222,"åºĵéĩĮ":115223,"ç´¾":115224,"ç´¾åijĺ":115225,"èµŀç¾İ":115226,"ç©¿æĪ´":115227,"è£½ä½ľ":115228,"èµŀæĪIJ":115229,"ä¸Ģä¾§":115230,"å½ĵåľ°äºº":115231,"æĭİ":115232,"纸质":115233,"ä½Ļ个":115234,"éĶĤçĶµæ±ł":115235,"æľºåŀĭ":115236,"éĻ¢éϢ士":115237,"åģļå·¥":115238,"å¼łè´´":115239,"ç¥Ľæĸij":115240,"æ®ĸæ°ij":115241,"å¥ij约":115242,"æ¹ĺæ½Ń":115243,"æIJĸ":115244,"åŃĺè´§":115245,"交éĢļ大åѦ":115246,"è¶ģçĿĢ":115247,"æĸĩçī©ä¿ĿæĬ¤":115248,"å¤ĩæĪĺ":115249,"éĩĩ纳":115250,"åįĬæľĪ":115251,"æľĢåħ³éĶ®":115252,"æľĢåħ³éĶ®çļĦ":115253,"æİ¥éĢģ":115254,"æĶ¶åī²":115255,"åıįåĢĴ":115256,"çĥĽ":115257,"æ½Ķ":115258,"ä¼Łå¤§å¤įåħ´":115259,"çļĦè¯Ŀè¯Ń":115260,"容å¿į":115261,"å®ļéĩı":115262,"æķĹ":115263,"åĵģçīĮ形象":115264,"æīŃ转":115265,"åĽ½å®¶éĩįçĤ¹":115266,"èĨĿçĽĸ":115267,"ä¸Ģ楼":115268,"大éϏ":115269,"éĤªæģ¶":115270,"åĽŀåij³":115271,"çĮ¿":115272,"çĿ¡åīį":115273,"æĹłè¾ľ":115274,"çĹħæ¯ĴæĦŁæŁĵ":115275,"æľºæ¢°åĮĸ":115276,"çĤ¹äº®":115277,"溶解":115278,"åĩłä¹İæīĢæľī":115279,"è·ijéģĵ":115280,"ç͵è§Ĩæľº":115281,"åı¨":115282,"æijĩäºĨ":115283,"æijĩäºĨæijĩ头":115284,"èĩªè´Ł":115285,"综åIJĪåĪ©ç͍":115286,"èĩªå¦Ĥ":115287,"åİŁä¾Ĩ":115288,"ä¹Łä¸įæĥ³":115289,"èĬĤ课":115290,"è¿ĩåī©":115291,"çͲçĬ¶":115292,"çͲçĬ¶èħº":115293,"æĸ°ä¸ĸ纪":115294,"èĩªä¸»åĵģçīĮ":115295,"é«ĺå±Ĥ次":115296,"ä¸Ģè§Ĵ":115297,"è¡Įäºĭ":115298,"ç¥ĸåħĪ":115299,"å©ļåIJİ":115300,"éĹ´éļĻ":115301,"ç¼ĿéļĻ":115302,"è¿ĻæĶ¯":115303,"ä¸įæĸŃåĪĽæĸ°":115304,"å¾®åŀĭ":115305,"æĽĻåħī":115306,"享ç͍":115307,"ä¸ŃåĽ½ç§»åĬ¨":115308,"éĹŃçݯ":115309,"æī§æĦı":115310,"åıijå±ķæł¼å±Ģ":115311,"æł¸å¿ĥåĮº":115312,"éªļæī°":115313,"åħļåĴĮåĽ½å®¶":115314,"ä¸ŃåĽ½æĶ¿åºľ":115315,"帶èijĹ":115316,"ä¸ĩåįĥçĵ¦":115317,"åħ©äºº":115318,"äºİæĺ¯æĪij":115319,"åĽºä½ĵ":115320,"çªģå¦Ĥ":115321,"çªģå¦Ĥåħ¶":115322,"çªģå¦Ĥåħ¶æĿ¥":115323,"éĩĮç¨ĭç¢ij":115324,"çαç¾İ":115325,"æŁ¥éªĮ":115326,"åıĮèµ¢":115327,"éĹªåħī":115328,"楼å®ĩ":115329,"æĻı":115330,"æľīè¶³å¤ŁçļĦ":115331,"æŁĶæĢ§":115332,"ä¿¡æģ¯å®īåħ¨":115333,"管线":115334,"å¹¶ä¸įä¼ļ":115335,"åύ件":115336,"ä½łåºĶ该":115337,"çĿĢå®ŀ":115338,"æĺİæ¸ħ":115339,"æĬĹçĶŁç´ł":115340,"æīĵæŃ»":115341,"å®Įåħ¨ä¸įåIJĮ":115342,"èĬ±æ¤Ĵ":115343,"æĶ¾å®½":115344,"ä½İ端":115345,"åĽĽèĤ¢":115346,"åĮĹäº¬èµĽè½¦":115347,"éĽĨå¸Ĥ":115348,"æľªå©ļ":115349,"大å¹ħæıIJåįĩ":115350,"建çŃij设计":115351,"çĭ¬æľīçļĦ":115352,"æİ¢éĻ©":115353,"æ²³æµģåŁŁ":115354,"æħķ容":115355,"被çĽĹ":115356,"åĵºä¹³":115357,"èıģ":115358,"æĥ¬æĦı":115359,"è¶ĬæĿ¥è¶Ĭ好":115360,"广大群ä¼Ĺ":115361,"å¾·èĤ²":115362,"å¸Ĥåľºä»·æł¼":115363,"奥巴":115364,"奥巴马":115365,"èĬĤ缮ä¸Ń":115366,"两款":115367,"ä¸ĩä½Ļåħĥ":115368,"ç»´å°Ķ":115369,"çĶŁçī©ç§ijæĬĢ":115370,"åIJ¬èµ·æĿ¥":115371,"çłļ":115372,"æĭŁå®ļ":115373,"æ²¹çͰ":115374,"声èªī":115375,"建çŃijä¸ļ":115376,"éĻIJè´Ń":115377,"çīĩåŃIJ":115378,"çķľç¦½":115379,"ç½ijé¦ĸ页":115380,"ä¼Ĺçѹ":115381,"æĴŀåĩ»":115382,"åīįä¸įä¹ħ":115383,"åīįä¸ĸ":115384,"åĽĽä¸ªæĦıè¯Ĩ":115385,"æµĭç»ĺ":115386,"éĺ²ç©º":115387,"漫éķ¿çļĦ":115388,"æ²IJæµ´":115389,"æ¯Ķè¾ĥç®Ģåįķ":115390,"æµĭå®ļ":115391,"åĽŀè°ĥ":115392,"让人们":115393,"èĴĭä»ĭ":115394,"èĴĭä»ĭçŁ³":115395,"ç»ĵæĻ¶":115396,"å¢ŀæ·»äºĨ":115397,"æĿ¡è¯Ħ论":115398,"åī¯ä¼ļéķ¿":115399,"ä½ıæīĢ":115400,"ç»ĻåĩºäºĨ":115401,"è°ĥéħį":115402,"æ²ĸ":115403,"æľīç͍":115404,"æľīç͍çļĦ":115405,"ä¸ĢæĿ¡é¾Ļ":115406,"éĩİå¤ĸ":115407,"ç¼ĺåĪĨ":115408,"æ°¸è¿ľä¸įä¼ļ":115409,"æŀľæłij":115410,"大åıijå¿«ä¸ī":115411,"麻éĨī":115412,"äºijéĽĨ":115413,"åİ»åĵªéĩĮ":115414,"åħ¥å¸Ĥ":115415,"ä»»æĢ§":115416,"建档":115417,"建档ç«ĭ":115418,"建档ç«ĭåį¡":115419,"ä¸Ģ棵":115420,"社åįĢ":115421,"çĽ¸ä¼´":115422,"åļ·":115423,"å¡«åħħ":115424,"ä¸ĢæĹı":115425,"ç¾ģ":115426,"åıĸè¯ģ":115427,"èΰéĺŁ":115428,"åİĤåĮº":115429,"è¡·å¿ĥ":115430,"åıijå±ķéĺ¶æ®µ":115431,"é«ĺ强度":115432,"åĹĵåŃIJ":115433,"é¢Ĩè¡Ķ":115434,"楼主":115435,"大èĴľ":115436,"æŀķ头":115437,"粮油":115438,"é»Ħçĵľ":115439,"æĵĴ":115440,"å°ıçĭĹ":115441,"æĶ¹éĿ©å§Ķ":115442,"åįģåĪĨéĴŁ":115443,"é²ľèī³":115444,"åħ³ç¾½":115445,"çĭĢæħĭ":115446,"å®ŀç͍æĢ§":115447,"å°ijè§ģ":115448,"é£ŀæī¬":115449,"çͰéĩİ":115450,"æIJĤ":115451,"è¿Ļ个è¯į":115452,"åºĶæĢ¥é¢Ħæ¡Ī":115453,"è§Ĵ度æĿ¥çľĭ":115454,"æķ¬çķı":115455,"æ³ķå®Ŀ":115456,"åĸĦæĦı":115457,"æīĵæĸŃ":115458,"对åĨ³":115459,"çµķå°į":115460,"åĢŁæŃ¤":115461,"å¼ĢæºIJ":115462,"å°ı說":115463,"祺":115464,"å²ģ以ä¸ĭ":115465,"éĢĢå½¹åĨĽäºº":115466,"ä¸įä¹ħåīį":115467,"åĩºåİĤ":115468,"讽åĪº":115469,"æĿ¥çľĭçľĭåIJ§":115470,"éŃĶåħ½":115471,"çķĻä¸ĭæĿ¥":115472,"å±ħ室":115473,"åłħæĮģ":115474,"çľĭäºĨä¸Ģ":115475,"çľĭäºĨä¸Ģçľ¼":115476,"éĽĨåĽ¢æĹĹä¸ĭ":115477,"æĪĺæĪĺç»ĦåIJĪ":115478,"è®¤çľŁèIJ½å®ŀ":115479,"汽车产ä¸ļ":115480,"çī©çIJĨåѦ":115481,"æķµ":115482,"éĴĿ":115483,"åĽ¢éķ¿":115484,"ä¸įæĸŃæī©å¤§":115485,"èĤ©è´Ł":115486,"åıijå±ķ缮æłĩ":115487,"è³ĩéĩij":115488,"åīįç½®":115489,"ä¸ŃåĽ½åı¤ä»£":115490,"æŃ»åĪij":115491,"åħħåĪĨä½ĵçݰ":115492,"åħ³éŨ":115493,"ç¾İæĦŁ":115494,"æīĵåħ¥":115495,"æĬijéĥģçĹĩ":115496,"å°ijçĪ·":115497,"æłijæŀĿ":115498,"æ¶Īæģ¯ç§°":115499,"æ´Ľåħĭ":115500,"åį¯":115501,"è¿ĪåIJij":115502,"æİ¨åĭķ":115503,"ä»İä¸ļèĢħ":115504,"åݻ买":115505,"欢快":115506,"æĭ¥æĮ¤":115507,"马桶":115508,"æĬĬæİ§":115509,"æĶ¿åħļ":115510,"å¼łæī¬":115511,"客æłĪ":115512,"红æĺŁ":115513,"éĢģæĿ¥":115514,"åħ¨åŁŁæĹħ游":115515,"èĩªç§ģ":115516,"åįģäºĮæĿ¡":115517,"åı¹æģ¯":115518,"ä¸Ģèīĺ":115519,"ä¿Ŀè´¹":115520,"æĸ½å·¥çİ°åľº":115521,"æľī幸":115522,"ç»ŃèĪª":115523,"åı¯èĥ½æľĥ":115524,"èĥĮåıĽ":115525,"ä½£éĩij":115526,"ä¸īçŃīå¥ĸ":115527,"å¾Ī满æĦı":115528,"游æĪıåľ¬":115529,"群éĩĮ":115530,"æŀĦä»¶":115531,"åºıå¹ķ":115532,"太æ¹ĸ":115533,"æľ¨è´¨":115534,"æĻĭæ±Ł":115535,"çµĤæĸ¼":115536,"è·³è·ĥ":115537,"åĢºæĿĥ人":115538,"çŃī诸å¤ļ":115539,"æĶ¾åĩº":115540,"åħ³éĶ®æĹ¶åĪ»":115541,"æĦŁæŁĵèĢħ":115542,"é£ŀè¡Įåijĺ":115543,"èĥĨåĽº":115544,"èĥĨåĽºéĨĩ":115545,"æĬ±æŃī":115546,"åij¨äºĮ":115547,"æĸ°æĹ¶æľŁ":115548,"åĨ·éĵ¾çµģ":115549,"è¿Ļç§įæĸ¹å¼ı":115550,"该æĿij":115551,"åĽŀé¦Ī":115552,"åŁºçĿ£æķĻ":115553,"人åıĤ":115554,"æŀ¯çĩ¥":115555,"æī¹åıijå¸Ĥåľº":115556,"åħħåĪĨèĤ¯å®ļ":115557,"å¸ĤæĶ¿åįı":115558,"äºĭæ¥Ń":115559,"龸çİĭ":115560,"çĥŃæIJľ":115561,"åįģä¹Ŀ大":115562,"ä¼´æľī":115563,"ç¾İåĽ½æĢ»ç»Ł":115564,"åŁİå¸Ĥ管çIJĨ":115565,"ä¸ĭ令":115566,"èĥ¸åı£":115567,"åıªçŁ¥éģĵ":115568,"åij¨ä¸ī":115569,"ç͍æĪ¶":115570,"éѝ":115571,"å¿ĥè¡Ģ":115572,"带头人":115573,"åĮ»åĬ¡":115574,"åĮ»åĬ¡äººåijĺ":115575,"æİ§åζåύ":115576,"ä½ľåĵģåĨħ容":115577,"æĪĺåıĭ":115578,"åİĨå¹´":115579,"ä¸įåħĭ":115580,"ä¸įåħĭä¸įåıĬ":115581,"æĹ¥æŃ£å¼ı":115582,"è±IJå¯Į":115583,"ç¨İè´¹":115584,"æĹ¶æķĪ":115585,"å±ķä½į":115586,"è¡¡éĺ³":115587,"æĪ¿è²¸":115588,"çĪĨ款":115589,"ä¹IJæĦı":115590,"çͷ䏻":115591,"寬":115592,"æľĥèѰ":115593,"ä¹ĭå¤ľ":115594,"åIJĮ樣":115595,"ä¸įè¦ģ太":115596,"ä¼Ĭæĸ¯":115597,"ä¼Ĭæĸ¯åħ°":115598,"åŁºæľ¬åİŁåĪĻ":115599,"åİ»æİī":115600,"ä½İä¿Ŀ":115601,"个交æĺĵ":115602,"个交æĺĵæĹ¥":115603,"èģĬèģĬ":115604,"åĽĽä½į":115605,"åħļç»ĦæĪIJåijĺ":115606,"主è¦ģä»İäºĭ":115607,"å½±éŁ³":115608,"åĨĴåĩº":115609,"åij¼åIJ¸éģĵ":115610,"è¾¾å°Ķ":115611,"æľ¨åľ°æĿ¿":115612,"诡å¼Ĥ":115613,"çģ¯åħ·":115614,"çģ«çĥ§":115615,"è§£èĦ±":115616,"æĦĪåıij":115617,"æ¹ĸå·ŀ":115618,"é£İä¿Ĺ":115619,"æĸ°å½¢åĬ¿":115620,"æĸ°å½¢åĬ¿ä¸ĭ":115621,"è²Ŀ":115622,"èĦĵ":115623,"åĬ¨åĬĽçĶµæ±ł":115624,"é£ŀèι":115625,"飧æĢ§":115626,"åĪ©çī©":115627,"åĪ©çµ¦":115628,"ä¸į认è¯Ĩ":115629,"ç¼ĸç»ĩ":115630,"ä½ľåĿĬ":115631,"èģĮä¸ļæĬĢèĥ½":115632,"çľĭè¦ĭ":115633,"åĽ´æ£ĭ":115634,"æĺıè¿·":115635,"å½Ĵå±ŀäºİ":115636,"æĤ¬å´ĸ":115637,"éĨ«çĻĤ":115638,"å®ĭ代":115639,"åºĦæĿij":115640,"èĹķ":115641,"çĮĽçĦ¶":115642,"çĩĥæĸĻçĶµæ±ł":115643,"å®ŀä½ĵåºĹ":115644,"ä¸į足以":115645,"æĥħç·":115646,"æĥħç·Ĵ":115647,"å»ĬåĿĬ":115648,"ç͵åı°":115649,"åºĶåĬĽ":115650,"ä¸Ńå°ıåѦçĶŁ":115651,"èĥ¡åIJĮ":115652,"éī´åĪ«":115653,"åĨħç½®":115654,"乱象":115655,"æ¬ĬçĽĬ":115656,"å¼ĢæĶ¾å¼ı":115657,"åįļæĸĩ":115658,"讲课":115659,"çŃīåİŁåĽł":115660,"穷人":115661,"äº¤æĽ¿":115662,"æĬ¤çħ§":115663,"åıijå±ķæľºéģĩ":115664,"客åķĨ":115665,"åıįä¹ĭ":115666,"ç±³é¥Ń":115667,"å¹¶åıij":115668,"å¹¶åıijçĹĩ":115669,"æ±īåŃIJ":115670,"æŀľåĽŃ":115671,"对æĪijæĿ¥è¯´":115672,"åģıåIJij":115673,"æī¹ç¤º":115674,"读åIJİ":115675,"读åIJİæĦŁ":115676,"æĺİæĻº":115677,"åĽ´çĿĢ":115678,"åıį转":115679,"æĿ¨å¹Ĥ":115680,"ä¸ĵåįĸ":115681,"ä¸ĵåįĸåºĹ":115682,"åıĹéĻIJ":115683,"åºŁè¯Ŀ":115684,"æŀģå°ij":115685,"åįĪåIJİ":115686,"è¿Ľä¿®":115687,"åīĬåĩı":115688,"æľ¬ç§ijçĶŁ":115689,"ä¼ĺéĢī":115690,"åħīçħ§":115691,"åıĻäºĭ":115692,"åıĸæļĸ":115693,"åĮĹè·¯":115694,"æ¦ķ":115695,"èİĨçͰ":115696,"楼å±Ĥ":115697,"天èĬ±":115698,"天èĬ±æĿ¿":115699,"çĤľ":115700,"å·²ç»ıæľīäºĨ":115701,"è¶¾":115702,"çͳåįļ":115703,"ç͵éĺ»":115704,"åĬŁè¯¾":115705,"æŃ¥æŃ¥":115706,"éĤ£ä¹Ī容æĺĵ":115707,"æŃ¤æĸĩ":115708,"ä½°":115709,"计è¾ĥ":115710,"çīĩéĿ¢":115711,"ç͵影éĻ¢":115712,"ä¸įåħ¬å¹³":115713,"ä¸īæľŁ":115714,"æĹħ游èµĦæºIJ":115715,"å¤ļç§įå½¢å¼ı":115716,"è£Ĥç¼Ŀ":115717,"åIJİæİĴ":115718,"硬度":115719,"åĽŀæļĸ":115720,"éģĵæķĻ":115721,"è´«è¡Ģ":115722,"æ¸ħé¦Ļ":115723,"伤çĹħ":115724,"æĦı義":115725,"çļĦç¼ĺ":115726,"çļĦç¼ĺæķħ":115727,"åºĦ严":115728,"åıªæĺ¯ä¸ºäºĨ":115729,"æīĵæĬĺ":115730,"以ä¾Ĩ":115731,"滿足":115732,"çİĽä¸½":115733,"風éļª":115734,"æĸĩç§ij":115735,"éħįå¤ĩäºĨ":115736,"è¿Ľé£Ł":115737,"æ¶¡":115738,"è·¯ç¨ĭ":115739,"åı«å£°":115740,"ä¸Ńå¿ĥåŁİåĮº":115741,"æľīæīĢä¸įåIJĮ":115742,"張貼":115743,"é¢ĦæĬ¥":115744,"æľīå¤ļä¹Ī":115745,"è¿Ľè¡Įåħ¨éĿ¢":115746,"æĽ¾ç¶ĵ":115747,"ä¸ī代":115748,"å®ı大":115749,"æ¸ħæī«":115750,"éĢīåĩº":115751,"åĵªä¸Ģ个":115752,"主義":115753,"ä¾Ŀæĵļ":115754,"çļ®éĿ©":115755,"èµ¶æĿ¥":115756,"çŃĽæŁ¥":115757,"æ¨Ł":115758,"ä¿ĿèįIJ":115759,"åIJĥæĥĬ":115760,"æľĭåıĭ们对":115761,"ä»ĸæĺ¯ä¸Ģ个":115762,"åºŁæ°Ķ":115763,"æ»ħ":115764,"è´¢ç¨İ":115765,"æĿijæĿijæ°ij":115766,"èµĦäº§è´ŁåĢº":115767,"å®īå¨ľ":115768,"缮åīįåĽ½åĨħ":115769,"æĦŁè§īèĩªå·±":115770,"çµIJåIJĪ":115771,"éͦæłĩ":115772,"éͦæłĩèµĽ":115773,"æĽ´æ·±":115774,"åŁºæķ°":115775,"éħ¿éħĴ":115776,"çī¹èī²äº§ä¸ļ":115777,"åİĭå®ŀ":115778,"ä¾Ŀæ³ķ追究":115779,"æ·¡å®ļ":115780,"ç®ĢçĽ´å°±æĺ¯":115781,"å£ĵåĬĽ":115782,"æ°ijå¿ĥ":115783,"ä¸įåIJĪéĢĤ":115784,"çͱæŃ¤åı¯è§ģ":115785,"èµŀèªī":115786,"澤":115787,"åĩłå¹´åīį":115788,"åIJīä»ĸ":115789,"çł´æįŁ":115790,"è½»è½»åľ°":115791,"å²Ľå±¿":115792,"æĦıå¢ĥ":115793,"ä»Ģä¹Īåı«":115794,"åģĩè£ħ":115795,"éĢģè´§":115796,"å¹ķå¢Ļ":115797,"妥åįı":115798,"åĽ½æĹĹ":115799,"äºĨå¾Īä¹ħ":115800,"åĪĨ辨çİĩ":115801,"ç´Ķ":115802,"éĺ³åĮº":115803,"åĩŃçĿĢ":115804,"åģľè½¦ä½į":115805,"京éĥ½":115806,"éĶ£":115807,"æĵ¾":115808,"è¿ĽéŨ":115809,"åĪĺæµ·":115810,"åĽĽçº§":115811,"女足":115812,"è¡ĮæĶ¿å®¡æī¹":115813,"éģ¥æİ§":115814,"ä¸įéĮ¯":115815,"å¾Ĺå¾Ī好":115816,"ä¸ºçĽ®çļĦ":115817,"ä»įæľª":115818,"ç²¾è£ħ":115819,"éĢįéģ¥":115820,"尽头":115821,"çºłç¼ł":115822,"éłĺå°İ":115823,"æĭħè´Ł":115824,"æĪĸèĢħåħ¶ä»ĸ":115825,"åıªä¸įè¿ĩæĺ¯":115826,"åı®åĺ±":115827,"åģĩåĨĴ":115828,"æļĸæ°Ķ":115829,"çĽIJåŁİ":115830,"被è§Ĩ为":115831,"诺è´Ŀå°Ķ":115832,"ç»ĻäºĨæĪij":115833,"è¿ijåįĥ":115834,"éĩįåĽŀ":115835,"éĨĴäºĨ":115836,"çĶµè§£":115837,"忽çķ¥äºĨ":115838,"èĥĮéĥ¨":115839,"æĸĩæĺİåŁİå¸Ĥ":115840,"æºħ":115841,"è²ĵ":115842,"æĬµæĮ¡":115843,"åĸľæ¬¢åIJĥ":115844,"éĿĻéĿĻåľ°":115845,"å¾Īæ·±":115846,"åŁºç¡ĢçŁ¥è¯Ĩ":115847,"è¿ĩéĶĻ":115848,"çIJĨç§ij":115849,"交æµģåIJĪä½ľ":115850,"èĪĶ":115851,"èª¿æŁ¥":115852,"æħĪæĤ²":115853,"éĴ°":115854,"èĩ´ç͵":115855,"å®£ä¼łæ´»åĬ¨":115856,"åıĺéĩı":115857,"çļĦ人æĿ¥è¯´":115858,"æĹ¶éļĶ":115859,"ä¸įç®¡ä½ł":115860,"缸è¿ij":115861,"è´µéĩijå±ŀ":115862,"ä¹Łä¸įåı¯èĥ½":115863,"ç²īæľ«":115864,"åįĹçĵľ":115865,"çϽ马":115866,"åħīæºIJ":115867,"éĩijå¥ĸ":115868,"çĭ¬è§Ĵ":115869,"çĭ¬è§Ĵåħ½":115870,"妨ç¢į":115871,"ç»ĻåĬĽ":115872,"ä½Ĩä»į":115873,"å¼łå®¶åı£":115874,"èIJ¬åħĥ":115875,"渲æŁĵ":115876,"éķ¿å¤§äºĨ":115877,"è®°èĢħäºĨè§£":115878,"æĢĢçĿĢ":115879,"è¦ģåѦä¼ļ":115880,"游æĪı代":115881,"游æĪı代ç»ĥ":115882,"äºĮçϾ":115883,"æĦıè¯Ĩå½¢æĢģ":115884,"çݺ":115885,"计åĪĴçĶŁèĤ²":115886,"æī¾åĩĨ":115887,"åħ°èĬ±":115888,"è¿Ļ座åŁİå¸Ĥ":115889,"污泥":115890,"å®ĺæĸ¹å¾®ä¿¡":115891,"å½Ĵå±ŀ":115892,"æ°§æ°Ķ":115893,"éģİç¨ĭä¸Ń":115894,"åį°è±¡æ·±åĪ»":115895,"稳妥":115896,"çµIJæĿŁ":115897,"åŃķæľŁ":115898,"çĿĥ":115899,"åĿļåĽº":115900,"顺åĬ¿":115901,"æŀľèͬ":115902,"éĨ«å¸«":115903,"åİ®":115904,"ä¹Łæĺ¯å¦ĤæŃ¤":115905,"é¦Ĵ头":115906,"缸åĬ©":115907,"干线":115908,"ä¸Ģæľ¬ä¹¦":115909,"绥":115910,"æĮ¯å¥ĭ":115911,"èĤ¾èĦı":115912,"åĭķçī©":115913,"é£ŀè·ĥ":115914,"èıľåĵģ":115915,"å¤ļä½Ļ":115916,"å¤ļä½ĻçļĦ":115917,"éĢĿä¸ĸ":115918,"æģĭ人":115919,"å¼ĢåıijåĪ©ç͍":115920,"顺丰":115921,"éĩİå¿ĥ":115922,"æł¡å¤ĸ":115923,"æģIJé¾Ļ":115924,"éĿ¢åħ·":115925,"éķ¿è¾Ī":115926,"éļıå¤Ħ":115927,"éļıå¤Ħåı¯è§ģ":115928,"紧缺":115929,"éĩįä¸Ń":115930,"éĩįä¸Ńä¹ĭ":115931,"éĩįä¸Ńä¹ĭéĩį":115932,"奥æĸ¯":115933,"奥æĸ¯åį¡":115934,"ä¸Ģ个å¤ļ":115935,"ä¸Ģ个å¤ļæľĪ":115936,"ä¸įåı¯ç¼ºå°ij":115937,"æĸ°æł¼å±Ģ":115938,"æıIJæĮ¯":115939,"è¡Įè´¿":115940,"æ¼Ĥæµģ":115941,"èģĬåŁİ":115942,"åħ´å»º":115943,"è´¨æ£Ģ":115944,"ç§ģæľį游æĪı":115945,"æĽ´éĩįè¦ģ":115946,"è´®":115947,"çħľ":115948,"转åıĺ为":115949,"è¿Ļ两年":115950,"ä¿Ŀé²ľ":115951,"æī§æķĻ":115952,"çĥ¨":115953,"å¼Ģåıij建设":115954,"è¿IJèIJ¥ç®¡çIJĨ":115955,"误差":115956,"京åī§":115957,"å¸IJåı·":115958,"å·¥ä½ľä½ľé£İ":115959,"ä¸ĸä¿Ĺ":115960,"çϽ宫":115961,"å¤©åĽ½":115962,"å¤©åĽ½ç»§ç»Ń":115963,"å·´æĸ¯":115964,"èIJ¥åĪ©":115965,"åĵģæł¼":115966,"æĿijæ°ij们":115967,"æĪ¿è½¦":115968,"çŃīçĹĩçĬ¶":115969,"å¦Ĥå®ŀ":115970,"宸":115971,"å±Ĥ级":115972,"éĶĻè¿ĩäºĨ":115973,"ç»ĵå®ŀ":115974,"ç¬ijèĦ¸":115975,"羣å®ŀæĢ§":115976,"éĥ½å¸ĤæĬ¥":115977,"é¥Ńèıľ":115978,"åºĶ注æĦı":115979,"æĬ½çĥŁ":115980,"伪éĢł":115981,"åīįä¸Ģ天":115982,"éŃĶé¾Ļ":115983,"éŃĶé¾Ļ令çīĮ":115984,"约è°Ī":115985,"绣çѹæİ¨è¿Ľ":115986,"让ç͍æĪ·":115987,"åħ¨éĿ¢èIJ½å®ŀ":115988,"å¼Ħå¾Ĺ":115989,"è°Īæģĭçα":115990,"鸣æĪIJéķ¿":115991,"鸣æĪIJéķ¿è®°":115992,"æ´ĭæ´ĭ":115993,"çĸıæķ£":115994,"éĿ¢ç§¯çº¦":115995,"æµĵ缩":115996,"æĸ¯é¡¿":115997,"çĶŁæĢģåľĪ":115998,"æī§å¯¼":115999,"ç§»éĢģ":116000,"齿轮":116001,"æł¹æľ¬å°±ä¸į":116002,"缩åĩı":116003,"èµ°ä¸ĭåİ»":116004,"çĿ«æ¯Ľ":116005,"ä¹Łä¸įéĶĻ":116006,"åıįæĺłåĩº":116007,"èĭ¦æģ¼":116008,"缸åħ³æĶ¿çŃĸ":116009,"é«ĺ楼":116010,"ç²īèī²":116011,"æĬķèµĦé¢Ŀ":116012,"ä¸įç»ı":116013,"ä¸įç»ıæĦı":116014,"å®ģæĦ¿":116015,"èĪĮ头":116016,"æ»ĭçĶŁ":116017,"å®ģåİ¿":116018,"åīįåĪĹèħº":116019,"åĩ³":116020,"é£Łæ¬²":116021,"åıĸèĥľ":116022,"éĻ¢åŃIJ":116023,"ç´łè´¨æķĻèĤ²":116024,"滨å·ŀ":116025,"æĬ¢æĬĵ":116026,"å¼Ĥåij³":116027,"åĴļ":116028,"åĬį":116029,"宽éĺĶ":116030,"æļ´æ¶¨":116031,"æĥłåıĬ":116032,"è§Ħç¨ĭ":116033,"ä¾Ľåħ»":116034,"éĢģå¾Ģ":116035,"å±±åºĦ":116036,"举äºļ":116037,"å±ķé¦Ĩ":116038,"è§£éĶģ":116039,"æĹłè§Ĩ":116040,"éĻįèIJ½":116041,"è¿ŀäºij":116042,"è¿ŀäºij港":116043,"åıĤè°ĭ":116044,"çİĸ":116045,"ç¬ĥ":116046,"èĢĹè´¹":116047,"æī¿å¾·":116048,"社ä¼ļæķĪçĽĬ":116049,"åįĹæµ·ç½ij":116050,"åĪĽä¼¤":116051,"èIJ±":116052,"åħħæ²Ľ":116053,"ç½ijç«Ļ建设":116054,"大åºĨ":116055,"åĨįéĢł":116056,"åŃĹæł·":116057,"åħ¨æ°ijåģ¥èº«":116058,"èĮ«èĮ«":116059,"æµ®åĬ¨":116060,"åīįåı°":116061,"å¢ŀ设":116062,"éĢĽè¡Ĺ":116063,"åĢĴéĹŃ":116064,"æ³ķå¾ĭ顾éĹ®":116065,"çĸ®":116066,"çĹħçĹĩ":116067,"空åīį":116068,"请æķĻ":116069,"èĥľä»»":116070,"æĿĢèıĮ":116071,"æĪĺæĸĹæľº":116072,"ç»ĺåζ":116073,"å¤Ħæĸ¹":116074,"çªģåĽ´":116075,"çĮ«åĴª":116076,"æĬ¥åijĬæĺ¾ç¤º":116077,"ç¿Ł":116078,"çķ¶åľ°":116079,"æľĢéļ¾":116080,"纪å§Ķ书记":116081,"ä½İåİĭ":116082,"èĻļ空":116083,"è¿Ļéĥ¨ç͵影":116084,"产ä¸ļåįĩ级":116085,"è°·çα":116086,"è°·çαåĩĮ":116087,"æĬ¼éĩij":116088,"女æĸ¹":116089,"éĴ»çłĶ":116090,"æļĹæļĹ":116091,"è¿·ä½ł":116092,"æīĢè¬Ĥ":116093,"å¨ģå»ī":116094,"å¼ĢæľĹ":116095,"å²Ķ":116096,"çģ«çĤ¬":116097,"åIJĪçIJĨæĢ§":116098,"åħ¬åĬŀ":116099,"ä¼ļä¼ļéķ¿":116100,"éĺ´è°ĭ":116101,"å¼Ģå±Ģ":116102,"æĻ®éĢļè¯Ŀ":116103,"å᡿ĭī":116104,"å°ijåIJĥ":116105,"éĹªèĢĢ":116106,"æŀľæ±ģ":116107,"æī§è¡ĮåĬĽ":116108,"è°Ľ":116109,"æĬ¢åĬ«":116110,"é«ĺéĢŁåıijå±ķ":116111,"飬":116112,"åįĹæ²Ļ":116113,"é«ĺçŃīåŃ¦æł¡":116114,"æį¢ä¸ª":116115,"åı¯èĥ½åŃĺåľ¨":116116,"æĬĴ":116117,"è°±åĨĻ":116118,"被æĬĵ":116119,"æĿ¯åŃIJ":116120,"èĬĤèĥ½åĩıæİĴ":116121,"æ°ĶåĢĻåıĺåĮĸ":116122,"åĪĨåĪ¥":116123,"ä¸Ńæŀ¢":116124,"欢åij¼":116125,"åħī纤":116126,"è¿Ļ群":116127,"çľ¼çķĮ":116128,"åħ±åIJĮåıijå±ķ":116129,"çݰä»Ĭ":116130,"éĹ»è¨Ģ":116131,"çī¹èī²å°ıéķĩ":116132,"æķij人":116133,"éĻįæ°´":116134,"ä¸ĸçķĮä¸Ģæµģ":116135,"å°±é¤IJ":116136,"çŀ¥":116137,"å¤įä»ĩ":116138,"ç¾½æ¯Ľ":116139,"ç¾½æ¯ĽçIJĥ":116140,"è´©åįĸ":116141,"æºIJæ³ī":116142,"æĢ»ä½ĵè§ĦåĪĴ":116143,"åĬ¨æĦŁ":116144,"ä¸Ģ审":116145,"åĢŁéĴ±":116146,"è§ģæķĪ":116147,"èĬ±èįī":116148,"åIJĮä¸ļ":116149,"æŁ¥è©¢":116150,"åĽ½éĻħåIJĪä½ľ":116151,"ä¾ĽåĽ¾":116152,"åģ´":116153,"æłĵ":116154,"缸éĢļ":116155,"è°ĪåıĬ":116156,"è¿ĩç¨ĭå½ĵä¸Ń":116157,"é¦Ļèıĩ":116158,"åįģåĽĽæĿ¡":116159,"ä¸Ģå¼Ģå§ĭå°±":116160,"ä¸ĵåijĺ":116161,"æĺİ顯":116162,"æīĵéĢłåĩº":116163,"ä¸ĭéĿ¢æĪij们":116164,"æľºæ²¹":116165,"åı°è¯į":116166,"åŃIJå¼Ł":116167,"æľĢ常è§ģçļĦ":116168,"æĪijè®°å¾Ĺ":116169,"ç»°":116170,"æĤ¬æµ®":116171,"è¿ĺ羣æĺ¯":116172,"æĮĤåı·":116173,"åıĭåĸĦ":116174,"éĩį伤":116175,"çħ§äº®":116176,"æŃ¦èѦ":116177,"åĩºçݰéĹ®é¢ĺ":116178,"è¸Ĭè·ĥ":116179,"åľ°çIJĥä¸Ĭ":116180,"å¸Ĥ人大":116181,"åıĹ害人":116182,"å²IJ":116183,"åIJĮåѸ":116184,"éĩijèŀįå¸Ĥåľº":116185,"æľīçļĦçݩ家":116186,"å¸ĤæķĻèĤ²":116187,"å¸ĤæķĻèĤ²å±Ģ":116188,"åIJĦå¼Ĥ":116189,"ç·ļä¸Ĭ":116190,"æģº":116191,"æľī大éĩıçļĦ":116192,"åķĨæĬ¥":116193,"åįķåįķ":116194,"åħ¨é¢Ŀ":116195,"ä¾ĿæĹ§æĺ¯":116196,"好åĩłä¸ª":116197,"åĸµ":116198,"éĩįæķ´":116199,"çĶŁæ´»è´¨éĩı":116200,"æİ¢è®¿":116201,"åį°èĬ±":116202,"缼è¡Į":116203,"å¾®è§Ĥ":116204,"èĪįå¾Ĺ":116205,"åºŁå¼ĥçī©":116206,"积èĵĦ":116207,"å®ļå±ħ":116208,"æĤ¼":116209,"èĮ¸":116210,"çļĦ帮åĬ©":116211,"çļĦ帮åĬ©ä¸ĭ":116212,"亿åIJ¨":116213,"åŃĶéĽĢ":116214,"è¿ĻæĿ¡è·¯":116215,"饵":116216,"æĦĪåĬł":116217,"éķį":116218,"ä½ľæ¡Ī":116219,"èįĶæŀĿ":116220,"太å°ij":116221,"跻身":116222,"åħ¬çĽĬæ´»åĬ¨":116223,"çϽæĸij":116224,"æĬĢæľ¯æ°´å¹³":116225,"帧":116226,"æĹłçŁ¥":116227,"åºĶ该æĢİä¹Ī":116228,"éĢĢå¸Ĥ":116229,"æ¸Ń":116230,"åħ»çĮª":116231,"驼":116232,"ç¾¤å²Ľ":116233,"大åį«":116234,"ä¹ĺçĶ¨è½¦":116235,"èı²å°Ķ":116236,"è´´åIJ§":116237,"åģľä¸ĭæĿ¥":116238,"æľīæľºç»ĵåIJĪ":116239,"åĪ»èĭ¦":116240,"çļĦåľ°":116241,"çļĦåľ°æŃ¥":116242,"è¯ĬæīĢ":116243,"å¼ĢæĪĺ":116244,"èĢģçīĮ":116245,"çѹçłģ":116246,"åħ«å¤§ä»¥æĿ¥":116247,"楼æĪ¿":116248,"åŃĻæĤŁ":116249,"åŃĻæĤŁç©º":116250,"åħĴåŃIJ":116251,"第ä¸ĢæĿ¡":116252,"社交åªĴä½ĵ":116253,"æĥ³èµ·æĿ¥":116254,"大æ´ĭ":116255,"æĭ¼éٳ":116256,"è¿Ľåįļä¼ļ":116257,"è¿ĩåħ³":116258,"æ²¼":116259,"ç©¿æIJŃ":116260,"éĤ£ä¸Ģ天":116261,"çł´éŨ":116262,"æĬķæłĩ人":116263,"赢家":116264,"èĻļå¼±":116265,"æ¿ĥ":116266,"å®īæ£Ģ":116267,"客家":116268,"çĭ¬ç«ĭèij£äºĭ":116269,"æīĭåĬ¿":116270,"åīµéĢł":116271,"åľĨ满å®ĮæĪIJ":116272,"为主线":116273,"好å¥ĩå¿ĥ":116274,"é¢ĨåľŁ":116275,"çªĸ":116276,"åħ¸åŀĭæ¡Īä¾ĭ":116277,"çªģåıijäºĭä»¶":116278,"åºķæ°Ķ":116279,"头æĻķ":116280,"å®Ľå¦Ĥ":116281,"觸":116282,"æ¸ħæ·¡":116283,"åļ¼":116284,"åģľç͵":116285,"ç²īå°ĺ":116286,"éĻįä½İæĪIJæľ¬":116287,"æĶ¾æīĭ":116288,"è®°èĢħ表示":116289,"æĭĸå»¶":116290,"éªĩ":116291,"æ®ĭå¿į":116292,"çľģæķĻèĤ²":116293,"çľģæķĻèĤ²åİħ":116294,"é«ĺé¢Ŀ":116295,"éĦĻ":116296,"æ¥ŀ":116297,"åĨħç§ij":116298,"èIJ¥ä¸ļé¢Ŀ":116299,"åŁºçŁ³":116300,"æµģæ·Į":116301,"主æĹ¨":116302,"éĺIJéĩĬ":116303,"建åįİ":116304,"æĥĬåı¹":116305,"çī¢åĽºæłijç«ĭ":116306,"æĺ¯åIJ¦åŃĺåľ¨":116307,"建åĨĽ":116308,"éĽ¾éľ¾":116309,"åħ¬è®¤":116310,"åħ¬è®¤çļĦ":116311,"æ°¨åŁº":116312,"æ°¨åŁºéħ¸":116313,"åīįåĩłå¹´":116314,"åιéĤ£":116315,"æ±Łä¸ľ":116316,"å·¥æ¥Ń":116317,"ä¸ĢçĤ¹ä¹Łä¸į":116318,"修士":116319,"äºĨä¸Ģéģį":116320,"åĪģ":116321,"æ»ļæ»ļ":116322,"åĪĨæł¡":116323,"羣çα":116324,"è¡ĢèĦī":116325,"æĢ¥åī§":116326,"ä¸Ģ群人":116327,"羯":116328,"æĪIJé¾Ļ":116329,"ç²¾ç¥ŀçĹħ":116330,"缸åħ³äººåijĺ":116331,"éĿĵ丽":116332,"ä¸īåŃ£åº¦":116333,"åĪĴå®ļ":116334,"ä¸ĸçķĮ第ä¸Ģ":116335,"éĢļä¿Ĺ":116336,"åķĨä¸ļåľ°äº§":116337,"åĬŁèĥ½æĢ§":116338,"èµĦæľ¬ä¸»ä¹ī":116339,"详è§ģ":116340,"æĬĵæįķ":116341,"æĸĩæĺĮ":116342,"å®Ŀå®ī":116343,"è£ħéħįå¼ı":116344,"æºIJæºIJ":116345,"æºIJæºIJä¸įæĸŃ":116346,"çĶŁæĢķ":116347,"纵åIJij":116348,"壽":116349,"çľ¼è¢ĭ":116350,"èĤīä½ĵ":116351,"åı¤ä»Ĭ":116352,"èŀįåªĴä½ĵ":116353,"åģī":116354,"æł¼æľĥåĵ¡":116355,"çĥ·":116356,"åĬŁç͍":116357,"æīŃ磩":116358,"绿èī²éĢļéģĵ":116359,"åī§ç»Ħ":116360,"å¼±åĬ¿":116361,"è´¨éĩıéĹ®é¢ĺ":116362,"éĻIJé¢Ŀ":116363,"éªĨ":116364,"éģµä¹ī":116365,"å¯Ŀ室":116366,"æĥ³å¿µ":116367,"åł±åijĬ":116368,"ä»ħ次":116369,"ä»ħ次äºİ":116370,"èŀįåĪĽ":116371,"æĭĽèģĺä¼ļ":116372,"åºĬåŀ«":116373,"转åŀĭåıijå±ķ":116374,"ä¸ŃåĽ½çĶµä¿¡":116375,"åIJ¬è¯Ŀ":116376,"è«ĭæ±Ĥ":116377,"大éĥ¨åĪĨ人":116378,"æ´»å¾Ĺ":116379,"åĵŃæ³£":116380,"è¶Ļ":116381,"åıijçĹħçİĩ":116382,"ä¸į符":116383,"åĨĽå®ĺ":116384,"é¢Īæ¤İ":116385,"æĸ°åĨłçĸ«æĥħ":116386,"æŁ¬åŁĶ":116387,"æŁ¬åŁĶ寨":116388,"ä»»ä½ķå½¢å¼ı":116389,"人éĻħ":116390,"人éĻħåħ³ç³»":116391,"æĢ»æī¿åĮħ":116392,"å¹³åĿĩæ¯ı":116393,"æģŃåĸľ":116394,"åĦĺ":116395,"åħµé©¬":116396,"è¿Łåΰ":116397,"工伤":116398,"çīĪæĿĥå½Ĵ":116399,"çīĪæĿĥå½ĴåİŁ":116400,"æĭ¥æĬ¤":116401,"ç³Ĭæ¶Ĥ":116402,"å¹²æ¶ī":116403,"å°ijä¸įäºĨ":116404,"æĥ³æī¾":116405,"è´¹çİĩ":116406,"该éĻ¢":116407,"èŀįåĮĸ":116408,"è¿İåIJĪ":116409,"è§ĨåIJ¬èĬĤ缮":116410,"æł¼ç¶²ç«Ļ":116411,"çľīæ¯Ľ":116412,"欢è¿İ大家":116413,"å®¶åºŃæķĻèĤ²":116414,"ä¾µèļĢ":116415,"ç»Ļä½łä»¬":116416,"è¡Ģ液循çݯ":116417,"å¯Ħæīĺ":116418,"å°ĸåı«":116419,"以ä¸ĭåĩłä¸ª":116420,"è¿ĺ以为":116421,"åħ¶ä»ĸçݩ家":116422,"ç¬ijç¬ij":116423,"æīĵåIJ¬":116424,"èĩªçĦ¶ç§ijåѦ":116425,"åŁºç«Ļ":116426,"ä¹Ŀå·ŀ":116427,"ä¿Ŀ驾":116428,"ä¿Ŀ驾æĬ¤":116429,"ä¿Ŀ驾æĬ¤èĪª":116430,"æĶ¾çľ¼":116431,"çŁ¥åIJįä¼ģä¸ļ":116432,"縮":116433,"稽":116434,"æļĩ":116435,"使çĶ¨ç¶²è·¯":116436,"é¢ĦçķĻ":116437,"大象":116438,"åıijæĺİä¸ĵåĪ©":116439,"æĸĩ娱":116440,"éĢłç¦ı":116441,"湿润":116442,"éĿ¢æĿ¡":116443,"æ¶Īè´¹åįĩ级":116444,"è®Ĭå¾Ĺ":116445,"åĩłåIJį":116446,"ä»Ħ":116447,"认æ¸ħ":116448,"è¿ľæĻ¯":116449,"æıĴ座":116450,"诸侯":116451,"åıĺæĢģ":116452,"ç¦ı彩":116453,"è´§æŀ¶":116454,"失æİ§":116455,"ç§»åĬ¨ç«¯":116456,"ä¸Ĭåı¸":116457,"éĢłçº¸":116458,"å¸ĥæľĹ":116459,"çĴĩ":116460,"åı°åįĹ":116461,"åĮĹ京åĨ¬å¥¥":116462,"èĵĿçīĻ":116463,"éķ¿çŁŃ":116464,"æĬĺå°Ħ":116465,"ç»ijæŀ¶":116466,"å¯Ĵåģĩ":116467,"è½¬åŁºåĽł":116468,"æĢ¥äºİ":116469,"æŃ£åĵģ":116470,"åħħ滿":116471,"大纲":116472,"æĬĹä½ĵ":116473,"è¨ĵç·´":116474,"æĶ¶ç´§":116475,"æ¯Ķè³½":116476,"åħµåĬĽ":116477,"æľ¬æĽ¸":116478,"äºĮ代":116479,"æĢ¥è¯Ĭ":116480,"æĸĩæ¡Ī":116481,"ç»ıåķĨ":116482,"æĻ¨æĬ¥":116483,"æ£ĺ":116484,"æĢ»ä¹¦è®°åľ¨":116485,"åıĹéĤĢ":116486,"äºĶåĽĽ":116487,"å²ŃåįĹ":116488,"çαåIJĥ":116489,"åŁĥå°Ķ":116490,"å¿ĥå¢ĥ":116491,"è¦ĨçĽĸéĿ¢":116492,"å®ŀåľ¨æĺ¯å¤ª":116493,"æł¹åºķ":116494,"纷纷表示":116495,"åĹħ":116496,"éļıçĿĢæĹ¶éĹ´":116497,"åİĨåı²æĤłä¹ħ":116498,"éħī":116499,"æĢ»éĺŁ":116500,"主é¢ĺæ´»åĬ¨":116501,"éĹ®åį·":116502,"é©¿ç«Ļ":116503,"æı¡ä½ı":116504,"åı¯èĥ½å¯¼èĩ´":116505,"æ°ijéĸĵ":116506,"éĸĭåķŁ":116507,"ä½Ĩä¸įéĻIJ":116508,"ä½Ĩä¸įéĻIJäºİ":116509,"åįģéĩĮ":116510,"娥":116511,"æįŁèĢĹ":116512,"çĸı导":116513,"çݯ氧":116514,"ç¥ŀéĢļ":116515,"çαå°Ķ":116516,"çαå°Ķåħ°":116517,"æľ´å®ŀ":116518,"å¿«æĬ¥":116519,"æĶ¶åıĹ":116520,"æĪĸ許":116521,"èĥĮéĿ¢":116522,"æĸĩåĮĸä¼łåªĴ":116523,"ä¸īåĢĭ":116524,"æĶ»åĬ¿":116525,"å®ī举":116526,"å®īä¸ľå°¼":116527,"åĿĩå·²":116528,"顾èĻij":116529,"éĦŃ":116530,"è¿Ļå®¶åħ¬åı¸":116531,"åħ¬åijĬç§°":116532,"æıIJä¾Ľä¼ĺè´¨":116533,"稳æŃ¥æİ¨è¿Ľ":116534,"å¤įè¯ķ":116535,"å°Ĩé¢Ĩ":116536,"è°Īèµ·":116537,"å¨Ħ":116538,"è¿ŀ线":116539,"æ©ŁéĹľ":116540,"åºĶçĶ¨åľºæĻ¯":116541,"çĶ»åĥı":116542,"è´¢è¿IJ":116543,"ä¿Ŀéļª":116544,"çĹħçIJĨ":116545,"æ¯Ľä¸»å¸Ń":116546,"ä¸Ŀ毫ä¸į":116547,"çαå¥ĩ":116548,"çαå¥ĩèīº":116549,"ä¸ĵå®¶ç»Ħ":116550,"åij¼åͤ":116551,"éĭ¼":116552,"çģ¸":116553,"é¢ĨåħĪåľ°ä½į":116554,"æıIJæĭĶ":116555,"龸éģĵ":116556,"å±±åĿ¡":116557,"èĿİ":116558,"沸èħ¾":116559,"该项":116560,"ä»ĬçĶŁ":116561,"ä¸Ģç¯ĩæĸĩ竳":116562,"æĸ¹å¼ıè¿Ľè¡Į":116563,"é»ij客":116564,"æĶ¹åĬ¨":116565,"主é¡Į":116566,"æķ£å¸ĥ":116567,"ä»Ģä¹Īåľ°æĸ¹":116568,"åĮĸåIJĪ":116569,"åĮĸåIJĪçī©":116570,"éĿĻç͵":116571,"æĢ»æĶ¶åħ¥":116572,"å§Ķç»Ħç»ĩ":116573,"å§Ķç»Ħç»ĩéĥ¨":116574,"éĿĻæĢģ":116575,"èĢģåŃĹåı·":116576,"室åıĭ":116577,"éĥ½ä¸įæķ¢":116578,"æŀ¶åŃIJ":116579,"ç쵿ķı":116580,"审è§Ĩ":116581,"æĤ£åĦ¿":116582,"山寨":116583,"èĸªèµĦ":116584,"é©°æı´":116585,"éĥ¨åĪĨåĨħ容":116586,"好似":116587,"æĪIJåijĺåĽ½":116588,"åľ¨æĪijçľĭæĿ¥":116589,"åħ³æ³¨åº¦":116590,"éĻĪæŁIJ":116591,"è¿Ļç§įäºĭæĥħ":116592,"éĢīå®ļ":116593,"ç²¾åŃIJ":116594,"å£ģçĶ»":116595,"æ±Łæ·®":116596,"é«ĺæĺĤ":116597,"æł¼åĬĽ":116598,"輩":116599,"åѦåłĤ":116600,"æĤ¨åIJĮæĦı":116601,"ä¸ĢåĪĩéĥ½æĺ¯":116602,"潤":116603,"éĸĥ":116604,"å¸ĮæľĽèĩªå·±":116605,"ä¿ĺ":116606,"æ±Łåİ¿":116607,"æ³¾":116608,"ç§ijæķĻ":116609,"æīĵè¿Ľ":116610,"ä¸įæħİ":116611,"å¯ĴåĨ¬":116612,"æ¸Ķæ°ij":116613,"鼷æĸ¯":116614,"主宰":116615,"æĹħ游度åģĩ":116616,"ç͵åŃIJéĤ®ä»¶":116617,"æ±Ĥå©ļ":116618,"éļİæ®µ":116619,"åģ¥èº«æĪ¿":116620,"注æĺİåĩºå¤Ħ":116621,"äºĭæķħåıijçĶŁ":116622,"级以ä¸Ĭ":116623,"åŃĺæ´»":116624,"æĸ½èĤ¥":116625,"èľľèľĤ":116626,"嵩":116627,"æĮĸæİĺæľº":116628,"æĬĹæĭĴ":116629,"ä¼łå¯¼":116630,"æĺ¯ä»Ģä¹Īåij¢":116631,"ä¸Ĭå¹´åIJĮæľŁ":116632,"建åħļ":116633,"çĶŁæħĭ":116634,"ä¿Ŀä½ı":116635,"款车åŀĭ":116636,"人èĦī":116637,"éļIJèͽ":116638,"失æķĪ":116639,"éģ¿åŃķ":116640,"ç®Ģ便":116641,"è°¢è°¢ä½ł":116642,"å®Īä½ı":116643,"æĶ¾æĺł":116644,"è¨Īçķ«":116645,"çݰ代çµģ":116646,"é¤IJ廳":116647,"æķħå±ħ":116648,"大大å°ı":116649,"大大å°ıå°ı":116650,"çī¹åΫ声æĺİ":116651,"éģįåıĬ":116652,"å¿ĥçIJĨåĴ¨è¯¢":116653,"è³´":116654,"çĮ®è¡Ģ":116655,"å·²ç»ıè¾¾åΰ":116656,"æīĵæĭĽåij¼":116657,"åıĮè¾¹":116658,"ä¸Ģæĸ¹éĿ¢æĺ¯":116659,"å´ĩå°ļ":116660,"éĺ¿å¯Į":116661,"éĺ¿å¯Įæ±Ĺ":116662,"æĮģæľī人":116663,"è±ģ":116664,"é£İçŃĿ":116665,"åĬ¨èį¡":116666,"äºĨä¸Ģä¼ļ":116667,"äºĨä¸Ģä¼ļåĦ¿":116668,"ä¸ĩ象":116669,"çľĭç͵è§Ĩ":116670,"åįģä¸īæĿ¡":116671,"çĮĽçĥĪ":116672,"è¦ģä¸įçĦ¶":116673,"太æŀģæĭ³":116674,"å¼ķçĪĨ":116675,"ç»ıè¿ĩå¤ļå¹´":116676,"游æĪıéĩĮçļĦ":116677,"é¾Ļæ³ī":116678,"æłĩéħį":116679,"è®ĵä»ĸåĢij":116680,"éĢłæŀĹ":116681,"åĮºåŁŁæĢ§":116682,"亿ä¸ĩ":116683,"æĪĺçķ¥å¸ĥå±Ģ":116684,"éķĩæĶ¿åºľ":116685,"åĶ®ç¥¨":116686,"çĶŁäº§å·¥èīº":116687,"éķĩåħļå§Ķ":116688,"ä¸Ńå°ıåŀĭ":116689,"æľ¨è̳":116690,"河边":116691,"èĦ¾èĥĥ":116692,"欢è¿İæĤ¨":116693,"åıĺå¼Ĥ":116694,"缤纷":116695,"åŀĥåľ¾æ¡¶":116696,"辩è¯ģ":116697,"车åºĵ":116698,"æ¯Ķçİĩ":116699,"åħ´æĹº":116700,"详ç»ĨäºĨè§£":116701,"å®īå±ħ":116702,"çħ§æĸĻ":116703,"æĸ¹æīį":116704,"赦":116705,"åĨķ":116706,"å¥Ķèµ´":116707,"å®Ŀ鸡":116708,"åľºåĿĩ":116709,"缮åīįæŃ£åľ¨":116710,"åIJŀåϬ":116711,"è¿°èģĮ":116712,"æĩµ":116713,"å¥ĩçijŀ":116714,"ä»įå°Ĩ":116715,"èĪī辦":116716,"å·¥åķĨå±Ģ":116717,"å¡ijèĥ¶":116718,"åĬŀå®ŀäºĭ":116719,"æĸ¹æĸ¹éĿ¢":116720,"æĸ¹æĸ¹éĿ¢éĿ¢":116721,"æĸĩåĮĸèĬĤ":116722,"åħ¥èģĮ":116723,"鸥":116724,"ç©¿éĢı":116725,"ä»¥ä¹łè¿ijå¹³":116726,"åį±éļª":116727,"æľ¦èĥ§":116728,"åİĨåı²æĢ§":116729,"æķŀå¼Ģ":116730,"ä¼Ļä¼´åħ³ç³»":116731,"çŁ¿åĮº":116732,"åĽ½éĻħåľ¨çº¿":116733,"ä¼łå¥ĩéĩĮéĿ¢":116734,"è¿ijäºĽ":116735,"è¿ijäºĽå¹´":116736,"åĬ£åĬ¿":116737,"æĶ»åĩ»åĬĽ":116738,"æĻºéĢł":116739,"禧":116740,"çİĭåħĪçĶŁ":116741,"éĨ«çĶŁ":116742,"åĽĽé¡¹":116743,"å®ŀæĻ¯":116744,"åĪĿåĪĽ":116745,"å¿ĥ裡":116746,"æĻ¶ä½ĵ":116747,"交éĻħ":116748,"让æ¶Īè´¹èĢħ":116749,"课æĸĩ":116750,"æİĴæ°Ķ":116751,"å¹¶ä¸įæĦıåij³":116752,"çĽ¸å£°":116753,"第ä¸Ģå±Ĭ":116754,"åİŁèijĹ":116755,"鼾":116756,"没æľī太大":116757,"补水":116758,"çµģä¼ģä¸ļ":116759,"第äºĮæī¹":116760,"åħ¶å®ĥéĹ®é¢ĺ":116761,"æİĮéŨ":116762,"责任å¿ĥ":116763,"é¤IJåħ·":116764,"ç¾Ĭæ¯Ľ":116765,"没æľīå¿ħè¦ģ":116766,"ä¹IJåĽ¢":116767,"è¿ĽåŁİ":116768,"ä¸ĢçĤ¹åĦ¿":116769,"身形":116770,"çļ®èĤ¤çĹħ":116771,"æĺ±":116772,"å¢ŀèĩ³":116773,"è첿ĺİ":116774,"æıIJè´¨":116775,"ä½ĵèĤ²åľº":116776,"çŃ¹å»º":116777,"é¬Ĩ":116778,"车çīĮ":116779,"éļĶéŁ³":116780,"è´Łè´£åIJĮå¿Ĺ":116781,"丰ç¡ķ":116782,"ä½ĽéĻĢ":116783,"äºīåIJµ":116784,"庶":116785,"æ·¡æ°´":116786,"å°ıçĶ·åŃ©":116787,"ç§ģèĩª":116788,"åĮĸè¿Ľç¨ĭ":116789,"æĪĺ士æĿ¥è¯´":116790,"æ²¹èħ»":116791,"èĦ±è´«èĩ´å¯Į":116792,"æĹ¥å¸¸å·¥ä½ľ":116793,"交èŀį":116794,"åĨľè´¸":116795,"åĨľè´¸å¸Ĥåľº":116796,"åĵĪçĻ»":116797,"çĶµè´¹":116798,"èµĺ":116799,"åıĮèħ¿":116800,"æĵĶå¿ĥ":116801,"æĿ¥å½¢å®¹":116802,"使åij½æĦŁ":116803,"éĤ£ä¹Īç®Ģåįķ":116804,"èĬĻèĵī":116805,"åĢŁæ¬¾äºº":116806,"ç§Ģ丽":116807,"è®ĵä»ĸ":116808,"严åİīæīĵåĩ»":116809,"è³ŀ":116810,"æļ«":116811,"çħ¤æ°Ķ":116812,"çάä¸Ĭ":116813,"æ½ĩæ´Ĵ":116814,"太ä¹ħ":116815,"åij½åIJį为":116816,"è·¯çͱ":116817,"è·¯çͱåύ":116818,"驯":116819,"æıIJæĹ©":116820,"æĬĹåĩ»çĸ«æĥħ":116821,"åĩĽ":116822,"交åıĭ":116823,"éĶĢåĶ®æ¸łéģĵ":116824,"毫ä¸įçĬ¹è±«":116825,"èIJ¥åľ°":116826,"çłĶ究表æĺİ":116827,"鱼类":116828,"æį¢å±Ĭ":116829,"æİ¡åıĸ":116830,"çīĨ":116831,"缼å¼Ģ":116832,"æ²§æ¡ij":116833,"åºŃ审":116834,"ç»ıæŁ¥":116835,"åĬłå¼·":116836,"缸æ¯Ķäºİ":116837,"ä¸ĵçıŃ":116838,"ä½ĵåŀĭ":116839,"被害":116840,"被害人":116841,"æĶ¶æ¬¾":116842,"åħ·æľīèī¯å¥½":116843,"é«ĺå³°æľŁ":116844,"åģıä½İ":116845,"åĦŁ":116846,"åĨľä¸ļç§ijæĬĢ":116847,"ç®ĬæĥħåĨµ":116848,"å¦Ĥæŀľçݩ家":116849,"éķ¿çº¦":116850,"第åħŃå±Ĭ":116851,"åħ¬å¼ĢæĭĽèģĺ":116852,"åĪĩæĸŃ":116853,"迫使":116854,"çĸĹç¨ĭ":116855,"第äºĮç§į":116856,"ä¸įåħį":116857,"å¹²èѦ":116858,"çŁ³æ¦´":116859,"åĹ£":116860,"两类":116861,"çε士":116862,"åŁİ乡å±ħæ°ij":116863,"æŃ¤é¡¹":116864,"缴è¾ĸ":116865,"缴è¾ĸå¸Ĥ":116866,"åij¼åºĶ":116867,"éĴ¯":116868,"ç¦ıå¾·":116869,"æľºèº«":116870,"æĵįåľº":116871,"æ¿Ĵ临":116872,"人群ä¸Ń":116873,"èĤ¡æ°ij":116874,"åѽ":116875,"æ³ķåħ°":116876,"é¨İ":116877,"糯米":116878,"æĢ»çļĦ":116879,"æĢ»çļĦæĿ¥è¯´":116880,"åħ¸éĽħ":116881,"æĸ°éĻĪ":116882,"æĸ°éĻĪ代谢":116883,"缮çĿ¹":116884,"é¢Ħè¨Ģ":116885,"è·Įçł´":116886,"æĸ°ç¯ĩ竳":116887,"æ¯ĴæĢ§":116888,"åĸĿèĮ¶":116889,"æŁ¥èİ·":116890,"亮丽":116891,"çĶŁäº§åķĨ":116892,"æĶ¹æĪIJ":116893,"为äºĨæĽ´å¥½":116894,"深交":116895,"深交æīĢ":116896,"æİĥ":116897,"ä¹ĻèĤĿ":116898,"泸å·ŀ":116899,"åħĪè¿ĽæĬĢæľ¯":116900,"è¾ĵç»Ļ":116901,"æķ£æĪ·":116902,"æĢĿç»´æĸ¹å¼ı":116903,"åºĹ主":116904,"è°ĭæ±Ĥ":116905,"游æĪıæĬĢå·§":116906,"ä¸Ģ年级":116907,"çľ¼è§Ĵ":116908,"ä¸Ńä»ĭæľºæŀĦ":116909,"å·§åIJĪ":116910,"éĺ²çĽĹ":116911,"导è´Ń":116912,"æĪĬ":116913,"æĽ´éĢĤåIJĪ":116914,"åŁºæľ¬ä¿¡æģ¯":116915,"马ä¸ģ":116916,"åħ»æ®ĸåľº":116917,"åıįè¿ĩæĿ¥":116918,"æİ¨å´ĩ":116919,"å¯ĨåĪĩåħ³æ³¨":116920,"åŁºéĩijç»ıçIJĨ":116921,"æĮīéĶ®":116922,"åĨħéĥ¨æİ§åζ":116923,"æĪIJåijĺåįķä½į":116924,"æľ¯è¯Ń":116925,"åζæľį":116926,"åĪļéľĢ":116927,"æ£Ģç´¢":116928,"大大æıIJé«ĺ":116929,"åģ¥åº·ç®¡çIJĨ":116930,"èĩªæŃ¤":116931,"客æĪ·éľĢæ±Ĥ":116932,"丰èĥ¸":116933,"èµ·éĩį":116934,"èµ·éĩįæľº":116935,"æ¬łç¼º":116936,"æ¡ĪåŃIJ":116937,"æĥħ人èĬĤ":116938,"åħļæł¡":116939,"è¢ľ":116940,"该åī§":116941,"è¿·å¤±ä¼łå¥ĩ":116942,"ç»ļ丽":116943,"åķª":116944,"æĹłç§ģ":116945,"é̲ä¸ĢæŃ¥":116946,"第ä¸Ģ竳":116947,"åύåħ·":116948,"åĨľèµĦ":116949,"確實":116950,"åºıåĪĹ":116951,"娱ä¹IJå¹³åı°":116952,"èŀįèµĦç§Łèµģ":116953,"èµĦæºIJåħ±äº«":116954,"èģ½åΰ":116955,"æIJŀå¾Ĺ":116956,"ç»§ç»Ńä¿ĿæĮģ":116957,"åIJ¯èĴĻ":116958,"çľº":116959,"ä¸Ŀè·¯":116960,"设æĸ½å»ºè®¾":116961,"æİ¥åľ°":116962,"æİ¥åľ°æ°Ķ":116963,"第ä¸īåŃ£åº¦":116964,"åŁºè°ĥ":116965,"åıijéŁ³":116966,"社ä¼ļèµĦæľ¬":116967,"éĽĩ主":116968,"è¿ŀèĥľ":116969,"没åķ¥":116970,"廢":116971,"èµ¶èµ´":116972,"æ¼ĶåĮĸ":116973,"åı¤æĢª":116974,"çİĭçĪ·":116975,"é¢ĦåħĪ":116976,"å¼Ģåħ·":116977,"åĽŀé¦ĸ":116978,"åľ°ä¸ĭæ°´":116979,"å°ıç¼ĸä¸Ģèµ·":116980,"èµİåĽŀ":116981,"åľ°è²Į":116982,"åĪĿä¸ī":116983,"åı¯ç͍äºİ":116984,"éģĹ迹":116985,"è¿Ļæī¹":116986,"èĸªæ°´":116987,"å¿ħçĦ¶ä¼ļ":116988,"æ²½":116989,"éįĭ":116990,"第ä¸Ģéĥ¨":116991,"åĪĬçī©":116992,"å®ŀä¾ĭ":116993,"æ¸ħåĩĢ":116994,"ä¸ĬèµĽåŃ£":116995,"åĽ¾è¡¨":116996,"éĤ®è½®":116997,"åĵªè£¡":116998,"缸è§ģ":116999,"æī°ä¹±":117000,"æ¯ıæ¯ı":117001,"è¿Ļè¾ĪåŃIJ":117002,"ç¡«éħ¸":117003,"äºī缸":117004,"溯æºIJ":117005,"åĩºä¼Ĺ":117006,"çİīçŁ³":117007,"åħ±çĶŁ":117008,"æĹ¶éĹ´æ®µ":117009,"éĩįè¦ģæĮĩ示":117010,"æ¶Īè´¹éľĢæ±Ĥ":117011,"éķ¿éķ¿":117012,"éķ¿éķ¿çļĦ":117013,"å®īæĬļ":117014,"å¢ŀé«ĺ":117015,"æľ¬è½®":117016,"äº²çľ¼":117017,"é£İæ³¢":117018,"èĢģå¦Ī":117019,"æĶ¶è´¹æłĩåĩĨ":117020,"åĨħéĻĨ":117021,"æĮ¥åıij":117022,"åįĩåѦ":117023,"èĥ¸åīį":117024,"åģıè¿ľ":117025,"纯æ´ģ":117026,"æĸ½å·¥åįķä½į":117027,"身价":117028,"è´¢åĬĽ":117029,"纶":117030,"è£ħçͲ":117031,"æĺ¾ç¤ºåύ":117032,"毫åįĩ":117033,"æ·±çŁ¥":117034,"è̶ç©":117035,"è̶ç©Į":117036,"è¾ĥéĩı":117037,"åľ¨è¿ĩ渡":117038,"åľ¨è¿ĩæ¸¡æľŁ":117039,"èĮĹ":117040,"ä¸Ģ个æĺŁæľŁ":117041,"èĬ·":117042,"è´¿èµĤ":117043,"æ¿ķ":117044,"æĩĤäºĭ":117045,"ç§§":117046,"åħħå½ĵ":117047,"åĽ½ç«ĭ":117048,"èĬ±çĵ£":117049,"éĤĦè¦ģ":117050,"åħ¬åľĴ":117051,"触åĬ¨":117052,"æ³°å·ŀ":117053,"ä»Ģä¹Īæł·":117054,"æ»ĭåħ»":117055,"è¯ĦåΤ":117056,"æĮ¥æīĭ":117057,"èĦĪ":117058,"姥姥":117059,"è¿IJè´¹":117060,"æ¯ħåĬĽ":117061,"å¿ĥæĻº":117062,"ä¸įæİĴéϤ":117063,"第ä¸ī代":117064,"éĢĢè´§":117065,"æĺŁéĻħ":117066,"æ°¸åĪ©":117067,"æĬ¤åį«":117068,"çıŃ车":117069,"è¨Ģè¡Į":117070,"繪":117071,"主åĬ¨æĢ§":117072,"å·¥ç¨ĭè´¨éĩı":117073,"éĥĬåĮº":117074,"ä¸Ģæłĭ":117075,"ä½Ĩå®ŀéĻħä¸Ĭ":117076,"ä¸ī大èģĮä¸ļ":117077,"åij¼åı«":117078,"女åħĴ":117079,"è¯ģåΏæĬķèµĦ":117080,"èĢĥæħ®":117081,"çĤ«èĢĢ":117082,"治好":117083,"åĺ¶":117084,"èĥ¤":117085,"åħīä¼ıåıijç͵":117086,"åĩłæŃ¥":117087,"æīĢæīĢ":117088,"æīĢæīĢéķ¿":117089,"çħ§æł·":117090,"åĵ¥ä»¬":117091,"è¯Ľ":117092,"è¿Ļä¸ĢåĪ»":117093,"çŁ¿çī©è´¨":117094,"ä¸įå¾Ĺå·²":117095,"åIJĮ缣":117096,"ç»Ĩå¾®":117097,"è·¯èĻİ":117098,"çϾèĬ±":117099,"æ··æ²Į":117100,"ä¸Ĭæµ·è¯ģåΏ":117101,"éĢĢç¨İ":117102,"èµŀåı¹":117103,"æī®æ¼Ķ游æĪı":117104,"åIJįåĪĹ":117105,"åIJįåĪĹåīį":117106,"åIJįåĪĹåīįèĮħ":117107,"ç±³å°Ķ":117108,"ä»Ģä¹ĪåİŁåĽł":117109,"å®īåħ¨ä¿Ŀéļľ":117110,"ä¸Ģåıªæīĭ":117111,"ä¹³ä¸ļ":117112,"ä¸įçĶĺ":117113,"æĥħåķĨ":117114,"æĮ¡ä½ı":117115,"åİŁåĽłä¹ĭä¸Ģ":117116,"è¿Ļ两天":117117,"çĥĺçĦĻ":117118,"豬":117119,"ä½łä»¥ä¸º":117120,"没è§ģè¿ĩ":117121,"åĵªå®¶å¥½":117122,"åīįä»»":117123,"è¿Ľè´§":117124,"éĢĢåĽŀ":117125,"串èģĶ":117126,"èĩ³æĸ¼":117127,"åĨ°æ·ĩ":117128,"åĨ°æ·ĩæ·ĭ":117129,"æŁ¥çľĭ详æĥħ":117130,"çı¾å¯¦":117131,"æİ¨æµĭ":117132,"æİ¥æīĭ":117133,"éļ¶å±ŀäºİ":117134,"åŁİå¸Ĥ群":117135,"æĿİåħĪçĶŁ":117136,"çŁ¿æ³īæ°´":117137,"çī¹ä»·":117138,"æĽ´å¤ļ精彩":117139,"ç¨ĭå¼ı":117140,"读æĩĤ":117141,"å±ıèͽ":117142,"奥æŀĹ":117143,"奥æŀĹåĮ¹":117144,"奥æŀĹåĮ¹åħĭ":117145,"红èĸ¯":117146,"奮":117147,"å®Ŀçİī":117148,"網絡":117149,"è²§":117150,"欧å¼ı":117151,"çϽç³ĸ":117152,"èĩªçĦ¶çģ¾å®³":117153,"åijĬè¯ī她":117154,"å»ļ":117155,"çĤ¹åĩ»æŁ¥çľĭ":117156,"é£İ湿":117157,"èµĦ产éĩįç»Ħ":117158,"ä¹Łä¸įä¾ĭå¤ĸ":117159,"åįĬ个å°ıæĹ¶":117160,"åIJ¸å¼ķæĽ´å¤ļ":117161,"æĹ¶éĹ´èĬĤçĤ¹":117162,"æĶ¶çº³":117163,"åIJ¸æ¯Ĵ":117164,"èĢģ乡":117165,"çIJħ":117166,"æľĢçµĤ":117167,"åıįæĦŁ":117168,"çĶ¨å¾®ä¿¡":117169,"çĶ¨å¾®ä¿¡æī«":117170,"éĢŁçİĩ":117171,"大çĨĬçĮ«":117172,"åı¯æĥ³":117173,"åı¯æĥ³èĢĮ":117174,"åı¯æĥ³èĢĮçŁ¥":117175,"åĴ§":117176,"èµ°åħ¥":117177,"碳éħ¸":117178,"èĮĥåĨ°":117179,"èĮĥåĨ°åĨ°":117180,"被åΤ":117181,"积æŀģæİ¨åĬ¨":117182,"足足":117183,"ç²ĴåŃIJ":117184,"大å®Ĺ":117185,"大å®ĹåķĨåĵģ":117186,"ç½ij绾ç§ijæĬĢ":117187,"æĽ¼åŁİ":117188,"å·²ä¹ħ":117189,"å·²ä¹ħçļĦ":117190,"秦çļĩ":117191,"秦çļĩå²Ľ":117192,"ä»»æķĻ":117193,"å͝ç¾İ":117194,"æ·¡åĮĸ":117195,"æ¡ĤèĬ±":117196,"çŁ¥è¯ĨåĪĨåŃIJ":117197,"æĩĴå¾Ĺ":117198,"主åħ¬":117199,"设计çIJĨ念":117200,"賺":117201,"æīĢæıIJä¾Ľ":117202,"æīĢæıIJä¾Ľä¹ĭ":117203,"æĶ»åħĭ":117204,"åĤ¾":117205,"è¯Ńæ³ķ":117206,"åįĥåı¤":117207,"éĸĭæĶ¾":117208,"第ä¸ĢèĬĤ":117209,"éĤĦæ²Ĵ":117210,"éĢĥçĶŁ":117211,"æ³Ĺ":117212,"åİ¿å§Ķ书记":117213,"ä½ľèĢħæīĢæľī":117214,"çħ½":117215,"ç»ħ":117216,"æłħ":117217,"æľ´ç´ł":117218,"çijķçĸµ":117219,"åĮħåĮħ":117220,"æ°ij主åħļ":117221,"ä¸įè¿ľå¤Ħ":117222,"å¥ĩå¼Ĥ":117223,"åĺ»åĺ»":117224,"æī¼":117225,"ç¿»å¼Ģ":117226,"æĢİèĥ½":117227,"éģ´éĢī":117228,"è§£éĩĭ":117229,"å¹¼ç¨ļ":117230,"è¦ģ好好":117231,"è¶´åľ¨":117232,"ç´¢åıĸ":117233,"ç»ĪçĶŁ":117234,"åħ¨æµģç¨ĭ":117235,"éģ©çķ¶":117236,"åįıè°ĥåıijå±ķ":117237,"æĬ¥ä»ĩ":117238,"ç§ijæĬĢåĽŃ":117239,"ä»Ģä¹Īéĥ½ä¸į":117240,"æľĢåIJİä¸Ģ次":117241,"ç»Ļ人ä¸Ģç§į":117242,"æł¸å®ļ":117243,"被åĪĹåħ¥":117244,"æĦıæĥ³ä¸įåΰ":117245,"èĢĥæŁ¥":117246,"åľ¨æŃ¤ä¹ĭåīį":117247,"æīĵçIJĥ":117248,"è¶ĬæĿ¥è¶Ĭå°ij":117249,"å®ļå¾ĭ":117250,"è¡ĮæĶ¿æľºåħ³":117251,"ä½ıæĪ¿åħ¬ç§¯":117252,"å°ıå§IJå§IJ":117253,"ä¸īèı±":117254,"修补":117255,"èŀĥèŁ¹":117256,"西çͲ":117257,"æĢł":117258,"çŃīå¤ļ项":117259,"产ä¸ļéĽĨèģļ":117260,"ä»·æł¼ä¸Ĭ涨":117261,"åħ¬åħ±åľºæīĢ":117262,"è¢ĭåŃIJ":117263,"æĨ§æĨ¬":117264,"çļĦæĸ¹å¼ıæĿ¥":117265,"åĪ°è´¦":117266,"çģ½":117267,"å·´èı²":117268,"å·´èı²çī¹":117269,"æ¼Ķä¹ł":117270,"èŃ¦ç¤ºæķĻèĤ²":117271,"çķıæĥ§":117272,"å¼ķæµģ":117273,"æĶ¶æĶ¯":117274,"å±Ĥåĩº":117275,"å±Ĥåĩºä¸į":117276,"å±Ĥåĩºä¸įç©·":117277,"æijĩæ»ļ":117278,"辦çIJĨ":117279,"纵è§Ĥ":117280,"æķijæµİ":117281,"å®¶éĥ½çŁ¥éģĵ":117282,"åĮ¯":117283,"å°ı鸣":117284,"ä»»åĭĻ":117285,"计åħ¥":117286,"ç«ŀéĢī":117287,"å¼ĢèįĴæĹ¶æľŁ":117288,"åij¨æģ©":117289,"åij¨æģ©æĿ¥":117290,"交ç»ĩ":117291,"çķ¢æ¥Ń":117292,"æł¹æį®èĩªå·±":117293,"æĸ°äººçݩ家":117294,"åѵåĮĸåύ":117295,"éĩĩæļĸ":117296,"å¹³åĿĩæ°´å¹³":117297,"åħ¬å¼Ģ课":117298,"失åĪ©":117299,"伺æľį":117300,"çĬģ":117301,"忽æĤł":117302,"主è¦ģéĽĨä¸Ń":117303,"æ¤įæłij":117304,"æ¯ĹéĤ»":117305,"èĩºçģ£":117306,"åĩºåĽ½çķĻåѦ":117307,"æĬĹéľĩ":117308,"æĥ©æĪĴ":117309,"å¹´åºķåīį":117310,"åĴ¸éĺ³":117311,"æ°ijå±ħ":117312,"大çIJĨçŁ³":117313,"éĿ³":117314,"éķĸ":117315,"æ¸ħè¿ľ":117316,"è£ħè½½":117317,"èĩĢ":117318,"å½±ä¸ļ":117319,"å¼ŁåħĦ":117320,"æĤ²è§Ĥ":117321,"çĿĢçľ¼äºİ":117322,"æįįåį«":117323,"åī¥å¤º":117324,"ç¯Ĩ":117325,"å¾Īéķ¿æĹ¶éĹ´":117326,"è¥Ł":117327,"第ä¸ĢçϾ":117328,"ä¸ĢåĪĨéĴ±":117329,"æĸ°éĹ»è®°èĢħ":117330,"éķ·æľŁ":117331,"æ³ķæĪĺç»ĦåIJĪ":117332,"è°ģçŁ¥éģĵ":117333,"èħ°éĥ¨":117334,"æ±īåł¡":117335,"åħ¥çĿ¡":117336,"åįĸæİī":117337,"æ¶Īè²»èĢħ":117338,"æĥ¯ä¾ĭ":117339,"æĥ³äºĨ":117340,"æĥ³äºĨæĥ³":117341,"èĢģæĹ§å°ıåĮº":117342,"ä¼łè¨Ģ":117343,"åĪĨæķ°çº¿":117344,"æµģ泪":117345,"ç»Ħç»ĩé¢Ĩ导":117346,"äºļåĨĽ":117347,"å¢ŀå̼æľįåĬ¡":117348,"å¾¹":117349,"ä¼¶":117350,"äºĽè®¸":117351,"å¸ĥèݱ":117352,"强æĤį":117353,"宫廷":117354,"绿èĮ¶":117355,"åĮ¡":117356,"å¾ĪæŃ£å¸¸":117357,"æĺ¥å¤ı":117358,"æ¯Ļ":117359,"è¯Ħæ¯Ķ":117360,"åĩ¡äºĭ":117361,"æĬīæĭ©":117362,"åĢĴéľī":117363,"éĩį度":117364,"åįıä¼ļä¼ļéķ¿":117365,"å¿§èĻij":117366,"ä¸ĭä¸Ģç¯ĩ":117367,"沪深":117368,"æĪİ":117369,"æīĵä»Ĺ":117370,"åįĪé¥Ń":117371,"å¹´é¾Ħ段":117372,"ä¸ŃåĽ½è¶³çIJĥ":117373,"设计æĸ¹æ¡Ī":117374,"åºĶçĶ¨æŁ¥çľĭ":117375,"é¢ĦæĸĻ":117376,"åĹ¡":117377,"ç¥ĸçζ":117378,"çļĦä¸Ģåijĺ":117379,"æ´Ĺå¹²åĩĢ":117380,"åİĨåı²æĸ°":117381,"åİĨåı²æĸ°é«ĺ":117382,"çĭ¬åħ·":117383,"æħĭ度":117384,"æīĵ交":117385,"æīĵ交éģĵ":117386,"é»ĦçŁ³":117387,"çĽ¼æľĽ":117388,"çī§åľº":117389,"转弯":117390,"åįĩåįİ":117391,"åĨįä¹Łæ²¡æľī":117392,"èĭ±æīį":117393,"æĽ´åIJį为":117394,"åĢŁç͍":117395,"çºłéĶĻ":117396,"ç»Ŀ对ä¸įä¼ļ":117397,"çİĭçīĮ":117398,"çĽĨåľ°":117399,"失è°ĥ":117400,"好象":117401,"é³¥":117402,"ä¿Ŀä¿®":117403,"åĽĽä¸ªèĩªä¿¡":117404,"头çļ®":117405,"åİŁåīĩ":117406,"æĬ¥æ¡Ī":117407,"奴éļ¶":117408,"å³Ļ":117409,"è°ĥæĸĻ":117410,"ä¹Łè¨±":117411,"èIJ½åΰ":117412,"èIJ½åΰå®ŀ":117413,"èIJ½åΰå®ŀå¤Ħ":117414,"çĦļçĥ§":117415,"çĶŁæ´»çݯå¢ĥ":117416,"åºĶåıĬæĹ¶":117417,"è¶Ĭè¿ĩ":117418,"æĦŁè¬Ŀ":117419,"æĻ¯å¾·":117420,"æĻ¯å¾·éķĩ":117421,"çĬĢ":117422,"身éĤĬ":117423,"ç¨İåĬ¡æĢ»å±Ģ":117424,"åĩĢåľŁ":117425,"ä¾µåįł":117426,"åĬ¨å·¥":117427,"å¹´ä¹ĭ":117428,"å¹´ä¹ĭä¹ħ":117429,"第äºĮèĬĤ":117430,"åĬ¨çī©åĽŃ":117431,"第ä¸Ģ书记":117432,"éħļ":117433,"çĶŁäº§è®¾å¤ĩ":117434,"æŁIJç§įç¨ĭ度":117435,"åľŃ":117436,"åĩŃåĢŁçĿĢ":117437,"éĺħè§Ī":117438,"çϽæ²Ļ":117439,"æ²¹çĥŁ":117440,"çªģçł´åı£":117441,"åıĹå½±åĵį":117442,"åı¯ä»¥æĽ´å¥½":117443,"å³°å̼":117444,"æĿĤè´¨":117445,"宿è¿ģ":117446,"çĽĺæ´»":117447,"æ¿Ģèµ·":117448,"åĦ¿ç§ij":117449,"åĿIJèIJ½åľ¨":117450,"æĮªå¨ģ":117451,"æµ·å²Ľ":117452,"绣绣":117453,"éύ":117454,"ä¼ĺäºİ":117455,"å°Īå®¶":117456,"ä¸ĢéĤĬ":117457,"èIJĬ":117458,"äºĨä¸Ģåı£":117459,"æ²ĥå°Ķæ²ĥ":117460,"æŃ£å¸¸ä½¿ç͍":117461,"æĻ®éģįåŃĺåľ¨":117462,"丰满":117463,"çĶ»åį·":117464,"åºĶæĶ¶":117465,"åºĶæĶ¶è´¦":117466,"åºĶæĶ¶è´¦æ¬¾":117467,"å®Įæķ´çĥŃ":117468,"å®Įæķ´çĥŃæ¦ľ":117469,"注è§Ĩ":117470,"çĨĦ":117471,"躬":117472,"éĶĢåĶ®äººåijĺ":117473,"è¶ĭåIJij":117474,"çĦ¦æĢ¥":117475,"åįģå¹´åīį":117476,"ä¼łç»Łäº§ä¸ļ":117477,"質éĩı":117478,"åĩ¤åĩ°ç½ij":117479,"èµĦæºIJæķ´åIJĪ":117480,"æ¶Įåħ¥":117481,"æĸĩåĮĸä¼łæĴŃ":117482,"çķĮ第ä¸Ģ":117483,"æ°´æ³µ":117484,"宫殿":117485,"æİ¢å¯»":117486,"ä¿®åīª":117487,"æĦıè¦ĭ":117488,"ç´Ĭä¹±":117489,"æĽī":117490,"çĻ½è¡£":117491,"èĻİåį«":117492,"ç´§æī£":117493,"å¤Ħå¤Ħéķ¿":117494,"åĪĽå»ºå·¥ä½ľ":117495,"红æŀ£":117496,"饼干":117497,"äºĨåįĬ天":117498,"ä¼ļå½±åĵįåΰ":117499,"çĽ¸ä¿¡å¤§å®¶":117500,"èħ¾é£ŀ":117501,"å°±å¦ĤåIJĮ":117502,"ä¸ĭéĿ¢å°ıç¼ĸ":117503,"æ°ijèIJ¥ç»ıæµİ":117504,"æĻ¦":117505,"è£ħæī®":117506,"é»ijå¤ľ":117507,"常德":117508,"å·¥ä¸ļ大åѦ":117509,"æĺİçŁ¥":117510,"éĺŁåijĺ们":117511,"åIJ¬è¯¾":117512,"æ¯ıéļĶ":117513,"羣æĺ¯å¤ª":117514,"åIJĪä½ľåħ±èµ¢":117515,"çIJĨåıij":117516,"æīįå¹²":117517,"çľĭèµ·ä¾Ĩ":117518,"殿ä¸ĭ":117519,"å®īéĺ³":117520,"æīĢ产çĶŁçļĦ":117521,"éĽĩä½£":117522,"æĬ¬èµ·å¤´":117523,"æį®æĬ¥éģĵ":117524,"éļĨéĩį举è¡Į":117525,"交éĶĻ":117526,"è¶ħé¢Ŀ":117527,"åĮĸçĸĹ":117528,"é¡Ĩ":117529,"纵深":117530,"çĪ±åĽ½ä¸»ä¹ī":117531,"éĻ¢åī¯éĻ¢éķ¿":117532,"讳":117533,"羣æŃ£åģļåΰ":117534,"åѤåįķ":117535,"èĩªçĦ¶èĢĮ":117536,"èĩªçĦ¶èĢĮçĦ¶":117537,"修身":117538,"èĬ¹":117539,"æģ¯æģ¯":117540,"æģ¯æģ¯çĽ¸åħ³":117541,"é©¾æł¡":117542,"æİ©é¥°":117543,"æ³½è¿ŀ":117544,"æ³½è¿ŀæĸ¯åŁº":117545,"举æŃ¢":117546,"管çIJĨä½ĵåζ":117547,"åħ¶ä¸Ńä¹ĭä¸Ģ":117548,"æĿ¾å¼Ľ":117549,"æĭ¦æĪª":117550,"åį«åģ¥":117551,"åį«åģ¥å§Ķ":117552,"ä»İåݻ年":117553,"åĤ¢":117554,"è´Ń票":117555,"åĽ¾æłĩ":117556,"河西":117557,"æ°ijæĶ¿å±Ģ":117558,"ç§ģèIJ¥":117559,"å¤ĸåĽ½è¯Ń":117560,"干货":117561,"æĵ¦æĭŃ":117562,"åľ°ä¸Ń":117563,"åľ°ä¸Ńæµ·":117564,"æµĵæµĵ":117565,"æµĵæµĵçļĦ":117566,"å§ĭ建":117567,"å§ĭ建äºİ":117568,"ç¶ĵæŃ·":117569,"è·¯æ¼Ķ":117570,"æļ´é£İ":117571,"åŁºè¾ħ":117572,"æī¶è´«å·¥ä½ľ":117573,"ä¸Ģ缴å¤Ħäºİ":117574,"æĥħè¶£":117575,"äºĮåŃ£åº¦":117576,"åİĮæģ¶":117577,"顺åĪ©å®ĮæĪIJ":117578,"æŁ¥å°ģ":117579,"顶端":117580,"ä¸įåŃķ":117581,"ä¸Ģ大åłĨ":117582,"被æ·ĺæ±°":117583,"æĺ¯ç͍æĿ¥":117584,"æľĢåIJĪéĢĤ":117585,"äº®çľ¼":117586,"å¹¶ä¸įæĺ¯å¾Ī":117587,"ç§ijçłĶéĻ¢":117588,"ç§ijçłĶéĻ¢æīĢ":117589,"ç²Ł":117590,"é¢Īéĥ¨":117591,"é»ĺé»ĺåľ°":117592,"é«ĺä¸ŃçĶŁ":117593,"æĹıèĩªæ²»åİ¿":117594,"æķĻåŃ¦è´¨éĩı":117595,"æĪĺçģ«":117596,"åĿİåĿ·":117597,"æIJŃä¹ĺ":117598,"è¯ĹæĦı":117599,"åĪijèѦ":117600,"åĩºæ±Ĺ":117601,"åįģåħŃæĿ¡":117602,"请åıĬæĹ¶":117603,"åĨľä¸ļ大åѦ":117604,"èIJ½åı¶":117605,"æĢ»èĢĮè¨Ģ":117606,"æĢ»èĢĮè¨Ģä¹ĭ":117607,"æĿľåħ°":117608,"æĿľåħ°çī¹":117609,"éĻªä½ł":117610,"åħ¬æĬ¥":117611,"çķĻè¨ĢæĿ¿":117612,"éĺħåİĨ":117613,"ç«¶çĪŃ":117614,"ç»ĻåĪ«äºº":117615,"æĹ¥æĬ¥ç¤¾":117616,"åĿIJèIJ½":117617,"åĿIJèIJ½äºİ":117618,"éĩijåŃĹ":117619,"éĩijåŃĹå¡Ķ":117620,"åĽ¤":117621,"è¯Ŀåī§":117622,"æĮģç»Ńæİ¨è¿Ľ":117623,"æ¼ıæ°´":117624,"詳細":117625,"æĢĢæĬ±":117626,"åıĺå¹»":117627,"饥饿":117628,"éļIJ身":117629,"ä¸ªèµĽåŃ£":117630,"åĵ¡å·¥":117631,"æģ¢å¤įæŃ£å¸¸":117632,"äºĨ好å¤ļ":117633,"æĺŁå·´":117634,"æĺŁå·´åħĭ":117635,"åħīçݯ":117636,"å¸ħåĵ¥":117637,"çĻ½éĽª":117638,"ç¨įç¨į":117639,"计æıIJ":117640,"æĦĽæĥħ":117641,"éİĸ":117642,"ä¿¡éĺ³":117643,"è§Ģå¯Ł":117644,"å¦Ĥæŀľä½łæĥ³":117645,"缸æ¯Ķä¹ĭä¸ĭ":117646,"è§£å¼Ģ":117647,"æīĵåį°æľº":117648,"身躯":117649,"ç²¾ç¥ŀæĸĩæĺİ":117650,"èĤ¡æĮĩ":117651,"å¾®åĪĽ":117652,"红èĮ¶":117653,"èĩ´çĻĮ":117654,"æģ©æĸ½":117655,"èħ¿éĥ¨":117656,"大åŀĭå¤ļ人":117657,"å®īåĢį":117658,"è¾ħ导åijĺ":117659,"èĪªéģĵ":117660,"å¸ĥå°Ķ":117661,"åįĹå®ģå¸Ĥ":117662,"ä¸ĬçıŃæĹı":117663,"ä¾§ç»ĵæŀĦæĢ§":117664,"追éļı":117665,"å½ĵåľ°æĶ¿åºľ":117666,"èµ°åĩºæĿ¥":117667,"éĩijèŀįä¸ļ":117668,"ä¸Ľä¹¦":117669,"é¡¹çĽ®ç»ıçIJĨ":117670,"è¿ĩæĪ·":117671,"骨æŀ¶":117672,"è¡Ļ":117673,"ä»Ģ麽":117674,"èħĭ":117675,"è¦ģ害":117676,"åľ¨åºĬä¸Ĭ":117677,"代è¨Ģ人":117678,"並å°ĩ":117679,"åIJĦ个æĸ¹éĿ¢":117680,"è°´è´£":117681,"åħ±æĮ¯":117682,"åį³å°ĨåΰæĿ¥":117683,"èĤºçĻĮ":117684,"ä¾ĽéĶĢ":117685,"丼æŀĹ":117686,"èµĥ":117687,"åįģä½Ļå¹´":117688,"åĭĺæİ¢":117689,"飵åij³":117690,"èĭ¦ç¬ij":117691,"æľĢ大ç¨ĭ度":117692,"éĩįçĤ¹åħ³æ³¨":117693,"ä¹ĭ举":117694,"满æĢĢ":117695,"åıĹåΰ影åĵį":117696,"æĭĽæĬķæłĩ":117697,"è¡¥é½IJ":117698,"西红":117699,"è¥¿çº¢æŁ¿":117700,"鬧":117701,"è£ħåį¸":117702,"éĤ»éĩĮ":117703,"èĤĩäºĭ":117704,"æİĴæ¯Ĵ":117705,"åѤåĦ¿":117706,"鼶è·Ŀ离":117707,"å®ŀå¹²":117708,"çľĭæŁ¥çľĭ":117709,"æĶ¶è´¹ç«Ļ":117710,"ç»·":117711,"åħ¬çĽĬæĢ§":117712,"éĢĴç»Ļ":117713,"æĶ»æīĵ":117714,"æĺŁçº§éħĴåºĹ":117715,"æĺİåªļ":117716,"çį¨ç«ĭ":117717,"è¯Ŀè¯ŃæĿĥ":117718,"ä¸ĢæŃ¥ä¸ĢæŃ¥":117719,"书æ³ķå®¶":117720,"æľªç»ıæİĪæĿĥ":117721,"çŁ³èĨı":117722,"åĩŃä»Ģä¹Ī":117723,"çļĦæĹ¥":117724,"çļĦæĹ¥åŃIJéĩĮ":117725,"诱人":117726,"çϾåĪĨçϾ":117727,"èĪĪè¶£":117728,"å¼łåħĪçĶŁ":117729,"èĢģçĪ·åŃIJ":117730,"æ³¢çī¹":117731,"åŁºéĩij份é¢Ŀ":117732,"æ²Ļåıijä¸Ĭ":117733,"å¥ĭæĸĹ缮æłĩ":117734,"æ°¢èĥ½":117735,"æ²ĥå°ĶçİĽ":117736,"義åĭĻ":117737,"éŁ³ç®±":117738,"æ²ī浸":117739,"æ²īæµ¸åľ¨":117740,"èĭ±åľĭ":117741,"çģ¯çģ«":117742,"è¿Ľé¡¹":117743,"两端":117744,"ä¹Ķ丹":117745,"èĦ¸é¢Ĭ":117746,"åıijå±ķæ½ľåĬĽ":117747,"åĭķä½ľ":117748,"åĵĪä½Ľ":117749,"å®´ä¼ļ":117750,"æ§į":117751,"ç«ĭå¿Ĺ":117752,"ç¡ķ士åѦä½į":117753,"åĭĭ竳":117754,"è¿Ļåľºæ¯ĶèµĽ":117755,"æĮģå¹³":117756,"éķĢéĶĮ":117757,"èĭ±çī¹":117758,"èĭ±çī¹å°Ķ":117759,"æķĻèģĮå·¥":117760,"åĬŁåĬĽ":117761,"该æ¡Ī":117762,"ä¸Ģæ¢Ŀ":117763,"åĺīå¹´":117764,"åĺīå¹´åįİ":117765,"è¿«ä¸įåıĬ":117766,"è¿«ä¸įåıĬå¾ħ":117767,"è¿Ļ个æĹ¶ä»£":117768,"精彩æĴŃæĬ¥":117769,"人èĦ¸":117770,"人èĦ¸è¯ĨåĪ«":117771,"æ£Ģå¯Łå®ĺ":117772,"å°ıèħ¿":117773,"éĨĴ缮":117774,"åħļæĢ»":117775,"åħļæĢ»æĶ¯":117776,"æĪŁ":117777,"èĮ«çĦ¶":117778,"è±ĨæµĨ":117779,"主治":117780,"éĿĴæµ·çľģ":117781,"åĪijäºĭ责任":117782,"çł°":117783,"ä¹ĭæ¬ĬåĪ©":117784,"äºĶå®ĺ":117785,"è¿·æĥij":117786,"åħ¥åºĵ":117787,"家纺":117788,"弹簧":117789,"åįģäºĶæĿ¡":117790,"ç»Ļå®Ŀå®Ŀ":117791,"èĪªç©ºèĪªå¤©":117792,"å¾Ģå¤ĸ":117793,"å¼ķåĬĽ":117794,"çľ¼çļ®":117795,"æ¶īè¶³":117796,"æĿ¥å®¾":117797,"åľ¨çº¿è§Ĵèī²":117798,"çĥŃéĶĢ":117799,"æµģéĢĿ":117800,"泡泡":117801,"éĻįå¹ħ":117802,"è´ŁéĿ¢å½±åĵį":117803,"红楼":117804,"红楼梦":117805,"éļĶçĿĢ":117806,"侥幸":117807,"许ä¹ħ":117808,"åĴĮçĿ¦":117809,"èѽ":117810,"使ç͍èĢħæĪĸ":117811,"ä¹°åįķ":117812,"è¿´":117813,"é£İæīĩ":117814,"æķĻ師":117815,"æ¡ĮåŃIJä¸Ĭ":117816,"å¾Īæ¼Ĥ亮":117817,"åł±å°İ":117818,"第ä¸ĢåŃ£åº¦":117819,"ç©©å®ļ":117820,"æĤ²åĵĢ":117821,"çĿĢåĬĽæīĵéĢł":117822,"æĮŁ":117823,"路桥":117824,"åijIJ":117825,"åľ£è¯ŀèĬĤ":117826,"çļĩåŃIJ":117827,"ä»ĩæģ¨":117828,"éħĿéħ¿":117829,"ä¸įéĹ´":117830,"ä¸įéĹ´æĸŃ":117831,"æĮĩå°ĸ":117832,"ä¸ŃåĽ½ç½ij游":117833,"åŀ£":117834,"æĦıè§ģ建议":117835,"æ¯ħçĦ¶":117836,"亮度":117837,"èģĶè°Ĭ":117838,"å½ķåħ¥":117839,"åĦ²":117840,"å¨ĺå®¶":117841,"ç§ijå°Ķ":117842,"ä¹Łæ²¡ä»Ģä¹Ī":117843,"æł¹æį®ä¸įåIJĮ":117844,"åı¶ä¿®":117845,"å̼å®Ī":117846,"æľ«ç«¯":117847,"å΍":117848,"åĤµåĭĻ":117849,"èģ¯åIJĪ":117850,"å¥ĩå¹»":117851,"èĻļæŀĦ":117852,"é»Ħæĺı":117853,"å¹³åĿ¦":117854,"æµģæ°ĵ":117855,"æĸ°åŁºå»º":117856,"æĮ½æķij":117857,"åįİå°Ķ":117858,"åįİå°Ķè¡Ĺ":117859,"æľĢåıĹæ¬¢è¿İ":117860,"ç»Ń约":117861,"å¼Ĭ端":117862,"éŃĶæ³ķå¸Ī":117863,"éŃĶæ³ķå¸ĪåĴĮ":117864,"åħ·ä½ĵåĨħ容":117865,"çIJīçĴĥ":117866,"æī©å®¹":117867,"èĮ¶åĽŃ":117868,"主ä¹īèĢħ":117869,"ç«ĭéĿ¢":117870,"æİ¥åıĹéĩĩ访":117871,"åĩºåħ¥å¢ĥ":117872,"ç§ijåįı":117873,"éĴ³":117874,"çµIJæ§ĭ":117875,"ç»ĵæŀľæĺ¾ç¤º":117876,"åı°è´¦":117877,"å°±æĿ¥çľĭçľĭ":117878,"èĩªæķij":117879,"åıįæĩī":117880,"åİ»åĵªåĦ¿":117881,"è¿Ļé¦ĸ":117882,"è¿Ļé¦ĸæŃĮ":117883,"åIJ¬ä¼Ĺ":117884,"å¤ĸ壳":117885,"ä½ĵèĤ²é¦Ĩ":117886,"實æĸ½":117887,"èŀºä¸Ŀ":117888,"æĭīåįĩ":117889,"çĮĽåľ°":117890,"åħ¨åĽ½äººæ°ij":117891,"æĤīå°¼":117892,"æĹı群":117893,"åĽ¢åijĺ":117894,"两个å°ıæĹ¶":117895,"åľ¨çݩ家":117896,"åľ¨çݩ家ä¸Ń":117897,"çĶľçĶľ":117898,"æĬķè¡Į":117899,"åįĶæľĥ":117900,"éĻ¡":117901,"åĬłå·¥åİĤ":117902,"æ¦ĨæŀĹ":117903,"æŃ»è§Ĵ":117904,"åĨħå¹ķ":117905,"æīĢæľīæĥħèĬĤ":117906,"åĪ·åį¡":117907,"æ°´èĤ¿":117908,"èĥĥåı£":117909,"å«Įå¼ĥ":117910,"沮丧":117911,"ä¸ī年级":117912,"æ¶Ĥå±Ĥ":117913,"å¿ĥ仪":117914,"å¿ĥ仪çļĦ":117915,"å¤Ń":117916,"é¦ĸè½®":117917,"æĹłè®ºæĺ¯åħ¶":117918,"éĢıæ°Ķ":117919,"äºĮåįģäºĶ":117920,"箫":117921,"åĬŁåĬ³":117922,"çѾä¸ĭ":117923,"æ²īè¿·":117924,"æķijåij½":117925,"éĹªéĹª":117926,"åIJĥäºı":117927,"å±ķåĵģ":117928,"åį³æĹ¶åıijçĶŁ":117929,"ç¶ľ":117930,"ç¶ľåIJĪ":117931,"æłĩæĺİ":117932,"çľĭç͵影":117933,"åħ¬ç«ł":117934,"éĺ¿æ£®":117935,"éĺ¿æ£®çº³":117936,"身åĪĽéĢł":117937,"身åĪĽéĢłçļĦ":117938,"æ¸Ľå°ij":117939,"å̼å¾Ĺåħ³æ³¨":117940,"鼶åĶ®åķĨ":117941,"æįĨç»ij":117942,"è¸ıåħ¥":117943,"èĽŁ":117944,"æŁ´çº³":117945,"èĢģåħµ":117946,"绿èī²çݯä¿Ŀ":117947,"é¹Ń":117948,"éº»æľ¨":117949,"æıŃçīĮ":117950,"è¿Ļ款车":117951,"ç¾İå¾·":117952,"ç¾İå¾·åħ¬åı¸":117953,"æ¶§":117954,"è°ģçŁ¥":117955,"æ´ĭèij±":117956,"æ¯įæł¡":117957,"ä¸ĢéĹª":117958,"çͷ䏻è§Ĵ":117959,"æĹłçº¿ç͵":117960,"å±łå®°":117961,"æĺ¯éŁ©åĽ½":117962,"æĺ¯éŁ©åĽ½å¨±":117963,"容è²Į":117964,"åĿĩ使åħ¶":117965,"太快":117966,"å¹´çͱ":117967,"å¹´çĶ±çĽĽ":117968,"èĭ¦èĭ¦":117969,"åĬĽè¿ĺæĺ¯":117970,"åĬĽè¿ĺæĺ¯èĩª":117971,"æĨ©":117972,"èģ¯çµ¡":117973,"å;":117974,"åħ·æľīæĪĺ士":117975,"追éĹ®":117976,"åłĨæĶ¾":117977,"åıį驳":117978,"å®ŀäºĭæ±Ĥ":117979,"å®ŀäºĭæ±Ĥæĺ¯":117980,"åѸéĻ¢":117981,"åįģåĩłä¸ª":117982,"æķijæĬ¤":117983,"æķijæĬ¤è½¦":117984,"ç½ijç»ľä¼łæĴŃ":117985,"åįģåħ«å±Ĭ":117986,"éĥ¨åī¯":117987,"éĥ¨åī¯éĥ¨éķ¿":117988,"çĹ´è¿·":117989,"管çIJĨæĿ¡ä¾ĭ":117990,"èŀį为ä¸Ģä½ĵ":117991,"æĢ»äº§å̼":117992,"è³ĵ":117993,"ä¸ĥæĺŁ":117994,"çıŃç»Ħ":117995,"绣é¢Ĩ":117996,"请大家":117997,"éĩijéϵ":117998,"èĪħèĪħ":117999,"æµ·æ¹¾":118000,"æĸ½çŃĸ":118001,"享èªī":118002,"麥":118003,"端åįĪ":118004,"绿åŁİ":118005,"確ä¿Ŀ":118006,"å·´æĭī":118007,"åĨĴçĿĢ":118008,"æħ·æħ¨":118009,"个人è§ĤçĤ¹":118010,"ä¹Ļçĥ¯":118011,"ç¡ħè°·":118012,"éĸĭå±ķ":118013,"å°ļ书":118014,"åĿļ飧":118015,"庵":118016,"èĢģé¾Ħ":118017,"èĢģé¾ĦåĮĸ":118018,"çľ¨çľ¼":118019,"绿水":118020,"绿水éĿĴå±±":118021,"书é¦Ļ":118022,"主åĬĽåĨĽ":118023,"æīįæĺ¯çľŁæŃ£":118024,"æĬ¢åħĪ":118025,"æĪIJå°±æĦŁ":118026,"éĩįæŀĦ":118027,"éĴ¢åİĤ":118028,"æĪIJ份":118029,"èĬ±çº¹":118030,"ä¹ĭäºī":118031,"å¹²ç»Ĩèĥŀ":118032,"æĹ¢åı¯ä»¥":118033,"ç¹ģçIJIJ":118034,"æĦļèł¢":118035,"éĿŀ常æĺİæĺ¾":118036,"ä½ĵ彩":118037,"æĬĢæ³ķ":118038,"æĿĨèıĮ":118039,"å¹¿æ³Ľåħ³æ³¨":118040,"åĮĹå®ĭ":118041,"å§Ĭ妹":118042,"åįıåĬŀ":118043,"æ·®åįĹ":118044,"çĥı":118045,"æ´ĹèĦ¸":118046,"åıĹ访":118047,"åıĹ访èĢħ":118048,"éĩįè¦ģåĽłç´ł":118049,"å½±è§Ĩåī§":118050,"综èīºèĬĤ缮":118051,"èľķåıĺ":118052,"äºĮ线":118053,"äºĮ线åŁİå¸Ĥ":118054,"ä¼Ĭå§ĭ":118055,"çıĬçijļ":118056,"èĩªæŁ¥":118057,"åħ¥åĽŃ":118058,"åĩ¶æīĭ":118059,"åħ¬è¯ī":118060,"éģĩéļ¾":118061,"éĩĩçŁ¿çŃī":118062,"èĩªçIJĨ":118063,"åĸ·æ¶Ĥ":118064,"æī©åħħ":118065,"éĢıè§Ĩ":118066,"é«ĺéĢŁå¢ŀéķ¿":118067,"åĽ¾çĶ»":118068,"ç¾¹":118069,"èĤĩåºĨ":118070,"è¾ľè´Ł":118071,"èµĶä»ĺ":118072,"è·¡":118073,"åģ¥åº·æĪIJéķ¿":118074,"以ä¸ĬåѦåİĨ":118075,"åıĸå¾Ĺ以åıĬ":118076,"æ²ī积":118077,"åįģä¹Ŀå±Ĭ":118078,"缸éĹľæľįåĭĻ":118079,"æī§åĭ¤":118080,"åī¯åİ¿éķ¿":118081,"寰":118082,"åģľæ»ŀ":118083,"淹没":118084,"çŁ³çģ°":118085,"çį¸":118086,"å̦":118087,"ç¾İåªĴ":118088,"æķĻæ¡Ī":118089,"åĬłçĽĸ":118090,"åħ¬å¼ĢèµĽ":118091,"å¥łåŁº":118092,"æĺĨèĻ«":118093,"çŀħ":118094,"磷éħ¸":118095,"äºīåĪĽ":118096,"çİĭæĻĵ":118097,"ç¼ĵåĨ²":118098,"åİļåİļ":118099,"åİļåİļçļĦ":118100,"æŀ£åºĦ":118101,"ç²¾çĽĬ":118102,"ç²¾çĽĬæ±Ĥ":118103,"ç²¾çĽĬæ±Ĥç²¾":118104,"åĪĨæĶ¯æľºæŀĦ":118105,"å®ŀæĸ½ç»ĨåĪĻ":118106,"æĸ°èµĽåŃ£":118107,"總統":118108,"éĢłè¡Ģ":118109,"é¢ĩåħ·":118110,"é»ĦåŁĶ":118111,"è¡ĢèĦĤ":118112,"交éĢļå·¥åħ·":118113,"å³¥":118114,"æĹıèĩªæ²»å·ŀ":118115,"寺éĻ¢":118116,"確å®ļ":118117,"æ¦Ĥ念èĤ¡":118118,"æĦŁå®ĺ":118119,"æŁľåı°":118120,"åĶĶ":118121,"çŀŃ解並":118122,"æĢ»ä»·":118123,"åIJ¸åħ¥":118124,"æĢ¼":118125,"æĻļéĹ´":118126,"å±Ĭæ¯ķä¸ļçĶŁ":118127,"çĶŁå§ľ":118128,"éĺħ读åħ¨æĸĩ":118129,"å¾ĹåΰæľīæķĪ":118130,"æIJľæķij":118131,"åİĨæĿ¥":118132,"èŃīæĺİ":118133,"åĥ»":118134,"èĨ³é£Ł":118135,"åĦĦåħĥ":118136,"æīĵåİĭ":118137,"宾客":118138,"åķ¼":118139,"ä¸ĢçϾå¤ļ":118140,"æ·±åħ¥äººå¿ĥ":118141,"æ¢ħå·ŀ":118142,"çłĶåѦ":118143,"åħ³ä¹İ":118144,"è¼Ľ":118145,"亲åıĭ":118146,"éħįæĸĻ":118147,"æĪijçĪ±ä½ł":118148,"è´¸æĺĵæĪĺ":118149,"æľīèī²":118150,"æľīèī²éĩijå±ŀ":118151,"æįIJåĬ©":118152,"为é¦ĸ":118153,"为é¦ĸçļĦ":118154,"å¯ĮåĬĽ":118155,"çĶ·ç¥ŀ":118156,"é³³":118157,"æµĩæ°´":118158,"åIJ±":118159,"æĺİç¡®æıIJåĩº":118160,"åı¹äºĨ":118161,"åı¹äºĨåı£æ°Ķ":118162,"礼æĭľ":118163,"è¿Ļ个åIJįåŃĹ":118164,"ä¿¡å¾Ĵ":118165,"å¿Ĺ强":118166,"éĻIJæĹ¶":118167,"æĶ¶è²»":118168,"åĨľå®¶ä¹IJ":118169,"å°ıé¾ĻèϾ":118170,"èIJ½å¹ķ":118171,"æ§Ł":118172,"åѦ龸":118173,"æĪĸå¤ļ":118174,"æĪĸå¤ļæĪĸ":118175,"æĪĸå¤ļæĪĸå°ij":118176,"座è°Īä¼ļä¸Ĭ":118177,"æ¶¼":118178,"éŃĶçİĭ":118179,"å²±":118180,"é¡¶å±Ĥ":118181,"é¡¶å±Ĥ设计":118182,"èĦijåŃIJéĩĮ":118183,"éĻ¢åŃIJéĩĮ":118184,"轩è¾ķ":118185,"身å¿ĥåģ¥åº·":118186,"èħij":118187,"éĹľæ³¨":118188,"åıĤåĬłä¼ļè®®":118189,"ä¸ŃåįİæĸĩåĮĸ":118190,"追寻":118191,"å®īçĦ¶":118192,"é£Ļåįĩ":118193,"éŁŃèıľ":118194,"鸦":118195,"åĤ¨éĩı":118196,"çĶ·æĸ¹":118197,"å¤ĩ份":118198,"æijĶåĢĴ":118199,"润æ»ijæ²¹":118200,"é̼è¿ij":118201,"çͳè¯ī":118202,"鸣类":118203,"çŁ³æ²¹åĮĸå·¥":118204,"åĿļæŀľ":118205,"è¿Ļå®¶ä¼Ļ":118206,"æĭĴä¸į":118207,"羣çļ®":118208,"è·ĿéĽ¢":118209,"è¿ĺæĮº":118210,"éĽķåĥı":118211,"åĪĿæģĭ":118212,"æıIJä¾ĽæĽ´å¤ļ":118213,"æŁ¥çľĭåħ¨æĸĩ":118214,"æķ°åŃĹè´§å¸ģ":118215,"åĸīåĴĻ":118216,"åı¦ä¸Ģä½į":118217,"åĤ¬åĮĸ":118218,"åĤ¬åĮĸåīĤ":118219,"ä»İæĿ¥æ²¡":118220,"å¯ĨåĪĩ缸åħ³":118221,"éĥ¨ä¸»ä»»":118222,"产åĵģç»ıçIJĨ":118223,"並åIJĮæĦı":118224,"èIJ½åħ¥":118225,"å±ıå¹ķä¸Ĭ":118226,"åħ¬åı¸ç«łç¨ĭ":118227,"æį¢åı¥è¯Ŀ":118228,"æį¢åı¥è¯Ŀ说":118229,"ä½įæĸ¼":118230,"ä½Ķ":118231,"åĩ»æĿĢ":118232,"缸è¾ĥ":118233,"缸è¾ĥäºİ":118234,"ç²½åŃIJ":118235,"åįĹæŀģ":118236,"宫é¢Ī":118237,"è£ģåijĺ":118238,"æĺİç»Ĩ":118239,"ä»·å̼éĵ¾":118240,"åĽĽä¸ªæĸ¹éĿ¢":118241,"æĥħåĨµæĿ¥çľĭ":118242,"æĮijåīĶ":118243,"æ®ĺ":118244,"æŀģåĬĽ":118245,"çĸijéļ¾":118246,"æĬµæĬĹåĬĽ":118247,"æĢ¥éĢŁ":118248,"æĪĮ":118249,"ä½İä¼°":118250,"éĹªè¿ĩ":118251,"æģ¬":118252,"èµŀæī¬":118253,"ä»ĸå¦Ī":118254,"æĪIJ为ä¸ĢåIJį":118255,"æ´Ĺ礼":118256,"é¢Ħ计å°Ĩ":118257,"åħĪè¿Ľåįķä½į":118258,"è¼Ķ":118259,"éĢĥèĦ±":118260,"çݰåŃĺ":118261,"èĢģèĻİæľº":118262,"åįģä¸ĥæĿ¡":118263,"åı¦ä¸ĢåįĬ":118264,"温æĥħ":118265,"åī¥ç¦»":118266,"ä¸ĸè´¸":118267,"å®ĺåı¸":118268,"å¾Īå·®":118269,"éĹ´è·Ŀ":118270,"请注æĦı":118271,"åı²è¯Ĺ":118272,"åĪ©åύ":118273,"è¿IJç®Ĺ":118274,"沦为":118275,"該使ç͍èĢħ":118276,"èĮ¬":118277,"éĶ¦ç»£":118278,"åı²æĸĻ":118279,"ç쵿´»æĢ§":118280,"èģĶ社":118281,"æĹłåĬ©":118282,"æĬĹæ°§åĮĸ":118283,"èıľèĤ´":118284,"éĢłèι":118285,"æİīèIJ½":118286,"å¤įæŁ¥":118287,"åĭĥåĭĥ":118288,"åij¼å£°":118289,"給äºĪ":118290,"åIJĮäºĭ们":118291,"ç½°":118292,"è¯ķæİ¢":118293,"åħ³éĶ®åŃĹ":118294,"æįIJçĮ®":118295,"ç»Łè®¡æķ°æį®":118296,"åĪĽä½ľèĢħ":118297,"ä¸ĭåįĬ":118298,"ä¸ĭåįĬåľº":118299,"æī¿æĭħ责任":118300,"端æŃ£":118301,"ç©¿è¡£":118302,"ä¼łçIJĥ":118303,"åĬ©éķ¿":118304,"åĩ±":118305,"éķ¶åµĮ":118306,"é£ŀç¿Ķ":118307,"è¾ĵåįµ":118308,"è¾ĵåįµç®¡":118309,"ä¸ĩåħ¬éĩĮ":118310,"æİ¨å¹¿åºĶç͍":118311,"å¿«æ¨Ĥ":118312,"ç§½":118313,"èī°å·¨":118314,"åIJ¬å®Į":118315,"åĿļ硬":118316,"å¥¥åľ°":118317,"å¥¥åľ°åĪ©":118318,"é¢ĵ":118319,"èĻIJå¾ħ":118320,"ä¾Ľæ±Ĥ":118321,"éľīç´ł":118322,"伪è£ħ":118323,"ä¹¡åľŁ":118324,"åĩ¡æľ¬ç½ij":118325,"åĩ¡æľ¬ç½ij注":118326,"ä¼ĬåĪ©":118327,"è¡¡æ°´":118328,"æĽ´åĥıæĺ¯":118329,"åĪĨéĴŁå·¦åı³":118330,"è¦ı模":118331,"äºĶåĪĨéĴŁ":118332,"åºĹåĬłçĽŁ":118333,"åĽ°éĽ£":118334,"åħ³åģľ":118335,"æĢĿ绪":118336,"åĴ½åĸī":118337,"缸符":118338,"çĥ¦èºģ":118339,"æĻĤæľŁ":118340,"åijĪçı¾":118341,"è§£æķ£":118342,"诱导":118343,"éļĶçĥŃ":118344,"çĮ¶":118345,"åįĹå®ĭ":118346,"æ·±åħ¥äºĨè§£":118347,"çŃĶçĸij":118348,"æĺ¼å¤ľ":118349,"åįĥä¼ı":118350,"åĬ³åĬ¡æ´¾éģ£":118351,"红è±Ĩ":118352,"åĿıäºĭ":118353,"çĤ¹æ»´":118354,"å°±ä¸ļå²Ĺä½į":118355,"约åIJĪ":118356,"åħįéϤ":118357,"éĢĨåĬ¿":118358,"éĩįéĩijå±ŀ":118359,"å®ĺ宣":118360,"ä½İå»ī":118361,"æģ¨ä¸įå¾Ĺ":118362,"å¾Ĺ天":118363,"å¾Ĺ天çĭ¬":118364,"å¾Ĺ天çĭ¬åİļ":118365,"ä¸Ģå°ģä¿¡":118366,"æĬ½å¥ĸ":118367,"è¾Ĺ转":118368,"çķĻå®Ī":118369,"çķĻå®ĪåĦ¿ç«¥":118370,"çŃĶåį·":118371,"å·¨åŀĭ":118372,"æľĢ好ä¸įè¦ģ":118373,"æµĻæ±Łå¤§åѦ":118374,"æĨ¨":118375,"æı¡æīĭ":118376,"éĴĪç»ĩ":118377,"æİĴ骨":118378,"çĤ½":118379,"å°ģè£ħ":118380,"åįĢåŁŁ":118381,"空æ°ĶåĩĢåĮĸ":118382,"åħīå½±":118383,"åĢĴå¡Į":118384,"å§ļæĺİ":118385,"æ¤į被":118386,"åѦåīį":118387,"åѦåīįæķĻèĤ²":118388,"èĬĿåĬł":118389,"èĬĿåĬłåĵ¥":118390,"缩水":118391,"ä½Ł":118392,"åľ¨çº¿åĴ¨è¯¢":118393,"èµıæŀIJ":118394,"éĿĴèĽĻ":118395,"æĬ±ä½ı":118396,"èĮĤåIJį":118397,"åħ¨åĬĽæīĵéĢł":118398,"åįļ士åѦä½į":118399,"æ²§å·ŀ":118400,"åĻ¢":118401,"æĿĤçī©":118402,"åĪ»çĶ»":118403,"æįħ":118404,"å¾®éĩı":118405,"å¾®éĩıåħĥç´ł":118406,"ä¸ĢåĽŀäºĭ":118407,"鸡èĤī":118408,"åĪ©æ¶¦çİĩ":118409,"æīįç®Ĺ":118410,"å¾®å¦Ļ":118411,"棵æłij":118412,"贪婪":118413,"åĩıå̼":118414,"梦å¢ĥ":118415,"åı¯è§Ĩ":118416,"åı¯è§ĨåĮĸ":118417,"广大å¸Ĥæ°ij":118418,"ä¸ĵä¸ļä»İäºĭ":118419,"ç»ı纬":118420,"ç´§çĽ¯":118421,"çŁ¥å·±":118422,"è¤ļ":118423,"æĸĩåĮĸåºķèķ´":118424,"åݦéŨå¸Ĥ":118425,"临港":118426,"对åħ¶çľŁå®ŀ":118427,"岸边":118428,"è¦ĸçĤº":118429,"æĬĹçĻĮ":118430,"åĶIJå®ĩ":118431,"ä¸įå¾Ĺè¶ħè¿ĩ":118432,"å¨ģæħij":118433,"æ¡Ĩæŀ¶åįıè®®":118434,"èµ°ç§ģ":118435,"åĽ¢å§Ķ":118436,"夸大":118437,"æ¬Ħ":118438,"ç¥ŀç»ıç³»ç»Ł":118439,"æijĦå½±ä½ľåĵģ":118440,"èĬ¥":118441,"å®īåºĨ":118442,"海滨":118443,"æŀĦæĢĿ":118444,"çĮĤ":118445,"åı©":118446,"éĺIJæĺİ":118447,"éģģ":118448,"精油":118449,"ç©´ä½į":118450,"æĬ¤èº«":118451,"æĬ¤èº«ç¬¦":118452,"æĮĩå°İ":118453,"åŃĺåľ¨ä¸Ģå®ļ":118454,"å¯ĤéĿĻ":118455,"æµ·å¤ĸå¸Ĥåľº":118456,"éĿ¡":118457,"综åIJĪå¾ģ":118458,"ä¿IJ":118459,"è¨Īç®Ĺ":118460,"æĺİæľĹ":118461,"äºļè¿IJ":118462,"äºļè¿IJä¼ļ":118463,"åīįçŀ»æĢ§":118464,"åĮ®ä¹ı":118465,"产ä¸ļæī¶è´«":118466,"èĦijæµ·":118467,"èĦijæµ·ä¸Ń":118468,"åħļçļĦé¢Ĩ导":118469,"åĪĺéĤ¦":118470,"æµģæĺŁ":118471,"æĵĤ":118472,"æĶĢçĻ»":118473,"åĴĶ":118474,"ä¸Ģä¸ĭåŃIJå°±":118475,"è¯Ĭæ²»":118476,"使åĬ²":118477,"åīµä½ľ":118478,"éĵŃè®°":118479,"éĴ±è´¢":118480,"æĹ¥æĬ¥è®°èĢħ":118481,"çĥŁçģ«":118482,"èĥľè´Ł":118483,"åįļ主":118484,"ä¸ŃåĽ½èģĶéĢļ":118485,"ç½ijç«Ļé¦ĸ页":118486,"å°±å¤Ł":118487,"å°±å¤ŁäºĨ":118488,"æīijåħĭ":118489,"å±ħå§Ķä¼ļ":118490,"è°¬":118491,"å®īåħ¨äºĭæķħ":118492,"åķĨçĶ¨è½¦":118493,"循çݯç»ıæµİ":118494,"æ·¤":118495,"èĢĥè¯ģ":118496,"å®ĿèĹı":118497,"å®Įç»ĵ":118498,"çłĶåıijæĬķåħ¥":118499,"å²ij":118500,"æģŃæķ¬":118501,"离éĢĢä¼ij":118502,"水墨":118503,"å©¶":118504,"è¯Ĺåı¥":118505,"å®ģæ³¢å¸Ĥ":118506,"å¼±çĤ¹":118507,"åģľçīĮ":118508,"奶油":118509,"å¥ĩ纳河":118510,"æĨĤ":118511,"社ä¼ļå®ŀè·µ":118512,"è´Ŀ壳":118513,"çłĤæµĨ":118514,"èιåıª":118515,"宣æī¬":118516,"综åIJĪæķ´æ²»":118517,"åĤij":118518,"æ°ijæĹıæĸĩåĮĸ":118519,"éĩįçݰ":118520,"积æ·Ģ":118521,"åħ¬çĦ¶":118522,"çħī":118523,"缸èģļ":118524,"æ±¾":118525,"纹çIJĨ":118526,"çĩĥçħ¤":118527,"æŃ¤ç§į":118528,"ç¾İå¦Ĩ":118529,"åįĥçĵ¦":118530,"çIJĽ":118531,"驾驶è¯ģ":118532,"éĺ¶æ¢¯":118533,"ä¸Ŀä¸Ŀ":118534,"å¾Īå¤ļäºĭæĥħ":118535,"åħīéĺ´":118536,"èijĹä½ľæ¬Ĭ":118537,"åħ§éĥ¨":118538,"çĽ¸å¯¹æĿ¥è¯´":118539,"éĸĴ":118540,"éľĩæħij":118541,"說話":118542,"æĨij":118543,"ç«¥è£ħ":118544,"ä½ıæĪ¿åĴĮ":118545,"ä½ıæĪ¿åĴĮåŁİ":118546,"å·²ç»ıè¶ħè¿ĩ":118547,"ä¾¦å¯Ł":118548,"çŁ¿çī©":118549,"ä¾Ľå¤§å®¶":118550,"çī¹éĤĢ":118551,"ç¨ĭåºıåijĺ":118552,"çķľçī§ä¸ļ":118553,"æ°ª":118554,"çijª":118555,"åĢĴåľ¨":118556,"åĢĴåľ¨åľ°":118557,"æ¯Ģ":118558,"梯éĺŁ":118559,"æİ¥èijĹ":118560,"æĬĹèıĮ":118561,"è¤ĩ":118562,"ç¬Ļ":118563,"æ¯Ķä¸Ĭå¹´":118564,"鸡汤":118565,"åŃ¦ä¹łæĪIJ绩":118566,"æĸijæĸĵ":118567,"åħĪ导":118568,"åĪĹ举":118569,"è°ĥæŁ¥æĺ¾ç¤º":118570,"æ©«":118571,"ä¹Ŀåįģ":118572,"è°¢éŁµ":118573,"è·¨è¶Ĭå¼ı":118574,"女æĢ§æľĭåıĭ":118575,"èIJ¥åħ»ä»·å̼":118576,"å®ŀè·µç»ıéªĮ":118577,"èĭıå·ŀå¸Ĥ":118578,"çĵ¶åŃIJ":118579,"æĸ°çļĦä¸Ģ":118580,"æĸ°çļĦä¸Ģå¹´":118581,"æĺİæĻ°":118582,"å®łçα":118583,"åŃĹ第":118584,"æľĹ诵":118585,"纳æĸ¯":118586,"éĢĨè¡Į":118587,"è«ĭæĤ¨":118588,"è«ĭæĤ¨æıIJä¾Ľ":118589,"èĥ¸æĢĢ":118590,"第ä¸ĥå±Ĭ":118591,"强壮":118592,"代åŃķ":118593,"æ±¶å·Ŀ":118594,"å®¶åĸ»":118595,"å®¶åĸ»æĪ·":118596,"å®¶åĸ»æĪ·æĻĵ":118597,"èħ®":118598,"åIJ¯è¿ª":118599,"æĹłéļľç¢į":118600,"èĻķçIJĨåıĬ":118601,"æĿ¥åİĨ":118602,"å®ŀåĬ¡":118603,"ä¹Łéļıä¹ĭ":118604,"æĬĢèĥ½åٹè®Ń":118605,"åѤç«ĭ":118606,"åīģ":118607,"éĥ´å·ŀ":118608,"æĶ¶æķĽ":118609,"éł»éģĵ":118610,"èį£å¹¸":118611,"èİ«è¿ĩäºİ":118612,"æŃ¤æĻĤ":118613,"纪å§ĶçĽij":118614,"纪å§ĶçĽijå§Ķ":118615,"缸éĤ»":118616,"åı¦ä¸Ģè¾¹":118617,"çªĴæģ¯":118618,"æľīå¾Īå¤ļç§į":118619,"æ¯ıéĢ¢":118620,"éĹ®ä¸ĸ":118621,"累累":118622,"éĿĴæĺ¥æľŁ":118623,"è·¯åĨµ":118624,"åħĭèݱ":118625,"è¿Ħä»Ĭ为æŃ¢":118626,"æĥĬå¥ĩ":118627,"跨度":118628,"éħ¿éĢł":118629,"åĩĭ":118630,"è¿ijä¸īå¹´":118631,"åĨħ马":118632,"åĨħ马å°Ķ":118633,"æıį":118634,"è¿Ľå±ķæĥħåĨµ":118635,"èĮ§":118636,"æľīåºıæİ¨è¿Ľ":118637,"æĢ»åĨłåĨĽ":118638,"æĪIJ绩åįķ":118639,"éĽ»è©±åıĬ":118640,"ç´§å¯Ĩç»ĵåIJĪ":118641,"åºĬä½į":118642,"é¹Ĭ":118643,"æķ£åıijçĿĢ":118644,"åĭŁèµĦ":118645,"æ°¨éħ¸":118646,"彩ç¥ŀ":118647,"è®Ģåıĸ":118648,"éĩ῏©":118649,"ä¸ŃåŃĺåľ¨çļĦ":118650,"ç¾İéºĹ":118651,"ä¸įæĸŃå¢ŀåĬł":118652,"è½®æµģ":118653,"æİ¥åIJ¬":118654,"年产å̼":118655,"åįĥåħĭ":118656,"æĪĺåľºä¸Ĭ":118657,"çħ§é¡§":118658,"å¹²éĥ¨éĺŁä¼į":118659,"åį°ç«ł":118660,"ä¸Ģèĩ´æĢ§":118661,"è¿ŀå¤ľ":118662,"åħħè£ķ":118663,"é»ijåIJįåįķ":118664,"åĩĢæ°´":118665,"ä¸Ģ大æĹ©":118666,"åĮħ袱":118667,"çĬ¯è§Ħ":118668,"çIJĨè«ĸ":118669,"æŀģæĺĵ":118670,"骸":118671,"å¨ĺå¨ĺ":118672,"åĽ¢åľĨ":118673,"亿åħĥ以ä¸Ĭ":118674,"åĪ©ç͍æĤ¨çļĦ":118675,"带æĿ¥æĽ´å¤ļ":118676,"ä¸Ń央空è°ĥ":118677,"æľĪèĸª":118678,"çĮľæĥ³":118679,"åĪºå®¢":118680,"ä½ľæģ¯":118681,"åįķè°ĥ":118682,"äºĴåĪ©":118683,"å¦Ĥæľīä¾µæĿĥ":118684,"å°ıå·§":118685,"åįģåł°":118686,"åĵĪåĵĪåĵĪåĵĪ":118687,"è¾¹éĻħ":118688,"æłĩè¯Ń":118689,"åĪĩåħ¥çĤ¹":118690,"éĢĨè¢Ń":118691,"è¯ķåīĤ":118692,"绿è±Ĩ":118693,"è®ļ":118694,"åŁºçĿ£å¾Ĵ":118695,"壬":118696,"åħ¨æĺİæĺŁ":118697,"éĢīç§Ģ":118698,"èĪĮå°ĸ":118699,"ä¸įåIJĮç±»åŀĭ":118700,"çĥŁåĽ±":118701,"ç쵿°Ķ":118702,"åĮºç®¡å§Ķä¼ļ":118703,"åĨľåī¯":118704,"åĨľåī¯äº§åĵģ":118705,"èĶļæĿ¥":118706,"沪æĮĩ":118707,"åħ»æ®ĸæĪ·":118708,"æĸĹå¿Ĺ":118709,"é¦ĸé¢Ĩ":118710,"è¡Ģèħ¥":118711,"åĬłç´§":118712,"ä¸Ģèĩ´å¥½è¯Ħ":118713,"第ä¸īèĬĤ":118714,"æī¬å°ĺ":118715,"交éĢļæŀ¢çº½":118716,"鼶ç¢İ":118717,"é»ijæ´ŀ":118718,"çľĭä¸įæĩĤ":118719,"å±ŀå®ŀ":118720,"主åŁİåĮº":118721,"å¨Ľ":118722,"å¨Ľæ¨Ĥ":118723,"ç¬ijæĦı":118724,"èĻ¹æ¡¥":118725,"åIJĦ个çݯèĬĤ":118726,"çķ¥å¾®":118727,"èĢķèĢĺ":118728,"æľ¬åľºæ¯ĶèµĽ":118729,"æĪIJè´¥":118730,"éĢīèĤ¡":118731,"èªŀè¨Ģ":118732,"çŃĶ辩":118733,"èĩªä¹ł":118734,"棺":118735,"ä¸ĩ欧åħĥ":118736,"åģľå·¥":118737,"对åħ¶è¿Ľè¡Į":118738,"积æŀģéħįåIJĪ":118739,"ä¹¾åĿ¤":118740,"å¦ĸæĢª":118741,"èļĮåŁł":118742,"èµĦ产è¯Ħä¼°":118743,"è°ĥçļ®":118744,"éϤå¤ķ":118745,"åĽ´å¢Ļ":118746,"æľįå½¹":118747,"æ·±æ¸Ĭ":118748,"é¢Ħåζ":118749,"çĥ½":118750,"å®ī稳":118751,"建æŀĦ":118752,"çĭĻåĩ»":118753,"主åĭķ註åĨĬ":118754,"éĥ½æľīèĩªå·±":118755,"æİĴåIJį第ä¸Ģ":118756,"麻辣":118757,"çĢļ":118758,"çĥŁèĬ±çĪĨ":118759,"çĥŁèĬ±çĪĨ竹":118760,"èĩªçĦ¶ä¿ĿæĬ¤":118761,"ä»Ļå¢ĥ":118762,"为äºĨéģ¿åħį":118763,"åĨ·åºĵ":118764,"è§£æĶ¾æĢĿæĥ³":118765,"åĪĿäºĮ":118766,"ä½ĵè´´":118767,"é¦ĸå¯Į":118768,"迪æĭľ":118769,"æļĤç¼ĵ":118770,"æĶ¯æĮģåĬĽåº¦":118771,"侦æİ¢":118772,"马åĪº":118773,"åĮĹæ±½":118774,"ç¹ŀ":118775,"è°İè¨Ģ":118776,"éĢ£çºĮ":118777,"å·³":118778,"ä»»ä½ķæĹ¶åĢĻ":118779,"车èģĶç½ij":118780,"åįķ项":118781,"å¸Ńåį·":118782,"建çŃijæĿIJæĸĻ":118783,"ä¸Ńç§ĭèĬĤ":118784,"ç¡ķ士çłĶç©¶":118785,"ç§ģç«ĭ":118786,"åħļåĴĮæĶ¿åºľ":118787,"æľ¬æ¬¡äº¤æĺĵ":118788,"èººåľ¨åºĬä¸Ĭ":118789,"ç½ijåıĭè¯Ħ论":118790,"å¦Ŀ":118791,"害ç¾ŀ":118792,"åħ¬ç«ĭåĮ»éĻ¢":118793,"ä¸ŀ":118794,"çĶŁçī©è´¨":118795,"åºĶéĤĢ":118796,"æĬ½åıĸ":118797,"åĩłå¼ł":118798,"æijĺç¼ĸ":118799,"ç»ĺæľ¬":118800,"详解":118801,"强硬":118802,"æľĢåħĪè¿ĽçļĦ":118803,"æĭĽèĤ¡":118804,"æĭĽèĤ¡ä¹¦":118805,"åįĥæĸ¹":118806,"åįĥæĸ¹çϾ":118807,"åįĥæĸ¹çĻ¾è®¡":118808,"éħįéŁ³":118809,"驾çħ§":118810,"å¾ģæĪĺ":118811,"èªĵè¨Ģ":118812,"æĭľå¸Ī":118813,"æĭľå¸ĪåѦ":118814,"æĭľå¸ĪåѦèīº":118815,"æĬ±åĽ¢":118816,"ç±³ç²ī":118817,"éĿŀ常éĢĤåIJĪ":118818,"èĪªæµ·":118819,"履约":118820,"åįģåħ«æĿ¡":118821,"éĶ»éĢł":118822,"éĩįè¦ģ举æİª":118823,"åıijæĮ¥ä½ľç͍":118824,"æ·ļ":118825,"人社":118826,"人社å±Ģ":118827,"è¯ķçĤ¹å·¥ä½ľ":118828,"éĺľéĺ³":118829,"æ¡ĥåľĴ":118830,"æ°ijä¼ģ":118831,"æ´ģçϽ":118832,"贵宾":118833,"åħ¬ç¤¾":118834,"è§īæĤŁ":118835,"è®°å¿ĨåĬĽ":118836,"æľĥåĵ¡è¨»åĨĬ":118837,"æŃ¤æ¡Ī":118838,"麻çĹ¹":118839,"çıĢ":118840,"æĸ©èİ·":118841,"çĶ·åŃ©åŃIJ":118842,"å±ĢéĻIJäºİ":118843,"åĭĺæŁ¥":118844,"åIJĥ饱":118845,"èĬ¬åħ°":118846,"æ£ķèī²":118847,"ç¦ıç¥ī":118848,"çͳèĬ±":118849,"æµ·çĽĹ":118850,"èĶij":118851,"æĸĩåѸ":118852,"æ´»æĢ§çĤŃ":118853,"缴éĢļ车":118854,"è°¢éĤĢ":118855,"躺çĿĢ":118856,"åľĥ":118857,"æ¯ıæĹ¥ç»ıæµİ":118858,"åħ¬åħ±æĸĩåĮĸ":118859,"讲æķħäºĭ":118860,"å¯Łçľĭ":118861,"æĤłéĹ²":118862,"åľ°åĿª":118863,"æ¶Įçݰåĩº":118864,"é«ĺçŃīéĻ¢æł¡":118865,"èĮĦåŃIJ":118866,"éĺ²åį«":118867,"ä¾ĭè¡Į":118868,"æĺ¾éľ²":118869,"æĸ°å¸¸æĢģ":118870,"ç»Ŀä½³":118871,"å¯Įæ°ij":118872,"以人æ°ij":118873,"以人æ°ij为":118874,"éĤ¢åı°":118875,"å±ķæ¼Ķ":118876,"çϼå¸ĥ":118877,"è´Łè½½":118878,"åģı离":118879,"æ°¸éģł":118880,"éĩįè¦ģåİŁåĽł":118881,"åįıä¼ļä¼ļåijĺ":118882,"é﾿°ij":118883,"çĶŁäº§è½¦éĹ´":118884,"çģµåĬ¨":118885,"两年åīį":118886,"æĸ¹åľĨ":118887,"æ´»ä¸ĭåİ»":118888,"ä¸ĸçķĮè§Ĥ":118889,"éªĹåıĸ":118890,"ç¾İè²Į":118891,"èĥ½çľĭåĩº":118892,"çϼæı®":118893,"è§Ĥå½±":118894,"åīĥ":118895,"åIJĪèµĦåħ¬åı¸":118896,"å©§":118897,"å¹²æĹ±":118898,"åħŃ个æľĪ":118899,"尤为éĩįè¦ģ":118900,"èĤ½":118901,"ç§¦åĽ½":118902,"æīĺç¦ı":118903,"建çŃijå¸Ī":118904,"åįĩ级æĶ¹éĢł":118905,"å°ıé¢Ŀ":118906,"å°ıé¢Ŀ贷款":118907,"两个维æĬ¤":118908,"æĭįæĭį":118909,"åı¯çĸij":118910,"æį¢åıĸ":118911,"æŃ¦å£«":118912,"èµĸ以":118913,"èµĸ以çĶŁåŃĺ":118914,"æĮļ":118915,"殿åłĤ":118916,"èĩªçĦ¶çķĮ":118917,"ç£ģåľº":118918,"å¦Ĥä½ķçľĭå¾ħ":118919,"ä»ĬæĹ¥å¤´æĿ¡":118920,"è¥¿åŁŁ":118921,"èİ·è¯Ħ":118922,"é¢¨æł¼":118923,"ä¿ĦåĽ½":118924,"æīĵæĭ¼":118925,"å®£ä¼łçīĩ":118926,"å¾Īæĸ¹ä¾¿":118927,"ä¾Ľç»Ļä¾§":118928,"纪念ç¢ij":118929,"毫åħĭ":118930,"èĬ³é¦Ļ":118931,"å·¥åķĨéĵ¶è¡Į":118932,"请çĤ¹åĩ»":118933,"缪":118934,"æĹłæķ°æ¬¡":118935,"èį¯å¸Ī":118936,"èħ¸":118937,"游èīĩ":118938,"åĮ¾":118939,"å·¡èĪª":118940,"æ²»çIJĨä½ĵç³»":118941,"èIJ¥éĢłèī¯å¥½":118942,"æ··æ·Ĩ":118943,"éĢļçķħ":118944,"åĬ³ç´¯":118945,"ä»ĵä½į":118946,"å¢ŀéķ·":118947,"éļIJ约":118948,"æĿĤå¿Ĺ社":118949,"åħ»èĤ²":118950,"åı¯èĥ½åıijçĶŁ":118951,"èĢĥ試":118952,"西侧":118953,"åĬłåĢį":118954,"主æĮģåı¬å¼Ģ":118955,"çķ¢ç«Ł":118956,"éĹ®è¯¢":118957,"æµ·æ£ł":118958,"èĹ©":118959,"注æĺİæĿ¥æºIJ":118960,"æ£Ģçĸ«":118961,"请åģĩ":118962,"æĬļæij¸":118963,"èĵĦçĶµæ±ł":118964,"è·Łä¸įä¸Ĭ":118965,"çݰ代社ä¼ļ":118966,"çѹèµĦ":118967,"ä½ĵèĤ²å½©ç¥¨":118968,"延误":118969,"è¾Ľè¾£":118970,"éĿ¢å®¹":118971,"åį°è®°":118972,"çģŃ亡":118973,"ç´łé£Ł":118974,"åħ´èĩ´":118975,"éľĢè¦ģç͍":118976,"éľĢè¦ģç͍åΰ":118977,"å®Ŀå¦Ī":118978,"ç£ĭåķĨ":118979,"éļ¶å±ŀ":118980,"è´¡çĮ®åĬĽéĩı":118981,"åħ¬åħ±èµĦæºIJ":118982,"大éĺª":118983,"åĨĽè®Ń":118984,"æĤ¬å¿µ":118985,"社ä¼ļ稳å®ļ":118986,"å¹²äºĭåĪĽä¸ļ":118987,"æľīæĿ¡ä»¶":118988,"æľīæĿ¡ä»¶çļĦ":118989,"ä¸Ģå¹´ä¸Ģ度":118990,"åİ¥":118991,"强奸":118992,"豪车":118993,"æİĮæŁľ":118994,"æ°´åΩ工ç¨ĭ":118995,"峪":118996,"积æŀģä½ľç͍":118997,"æµ·æ·Ģ":118998,"æµ·æ·ĢåĮº":118999,"çĥŃæĴŃ":119000,"åĿļæĮģä¸įæĩĪ":119001,"åıĮèĦļ":119002,"绣æĪĺ":119003,"ä»»ä½ķ人éĥ½":119004,"åľ°ä¸ĭ室":119005,"åĨ¶çĤ¼":119006,"è°ħè§£":119007,"æ¸Ķèι":119008,"太éĺ³åŁİ":119009,"被æįķ":119010,"计ç®Ĺåύ":119011,"西åĮ»":119012,"èĪĴå¿ĥ":119013,"桦":119014,"éģ²":119015,"åĬij":119016,"è¨Ĺ":119017,"èݺ":119018,"åĸ¬":119019,"çĵ¯":119020,"åĺĺ":119021,"åłķ":119022,"æķĿ":119023,"åij¦":119024,"èĭŀ":119025,"æŃ¹":119026,"æĵ¬":119027,"æ£Ħ":119028,"èε":119029,"奪":119030,"çļĭ":119031,"æĶ¸":119032,"åľ©":119033,"ç¤Ļ":119034,"ç¢ĺ":119035,"éıĪ":119036,"æĦķ":119037,"ç¹³":119038,"èĺ¸":119039,"è²Ĥ":119040,"æ¼²":119041,"æij¹":119042,"æĶĿ":119043,"åŃ¢":119044,"èķŃ":119045,"騰":119046,"æ½¼":119047,"éħ°":119048,"æĴ¥":119049,"蹬":119050,"é¨Ļ":119051,"踹":119052,"éģIJ":119053,"çĺĢ":119054,"èĽ¤":119055,"æĤĸ":119056,"çĴŀ":119057,"ç£IJ":119058,"æİ°":119059,"è¾Ĭ":119060,"å¾ij":119061,"æİĸ":119062,"éģŀ":119063,"éĤ¸":119064,"éĽı":119065,"æĨİ":119066,"æľ½":119067,"çį»":119068,"ç®Ķ":119069,"褶":119070,"æļ¢":119071,"æĺµ":119072,"çıĤ":119073,"æĤ¸":119074,"åģµ":119075,"åĻľ":119076,"壯":119077,"æĴ®":119078,"æģį":119079,"å©ķ":119080,"篱":119081,"éĺĻ":119082,"çīł":119083,"è£ĺ":119084,"è³¢":119085,"éĩľ":119086,"éĵł":119087,"èİĺ":119088,"æ®Ĩ":119089,"çϏ":119090,"è´ı":119091,"ç²±":119092,"å«¡":119093,"åĨ¢":119094,"è¤Ĵ":119095,"æĩĬ":119096,"éľĵ":119097,"塵":119098,"æĭ£":119099,"å»Ł":119100,"飽":119101,"é¢Į":119102,"åļİ":119103,"æ·º":119104,"èĨł":119105,"åİŃ":119106,"åļĩ":119107,"åijĥ":119108,"çĴĭ":119109,"çѱ":119110,"æĭ·":119111,"èį§":119112,"éͰ":119113,"åѰ":119114,"èĵĵ":119115,"èĨ½":119116,"æŀī":119117,"åĸ½":119118,"çĽĶ":119119,"çŃIJ":119120,"ç¾ļ":119121,"èħĮ":119122,"辫":119123,"æ³ĵ":119124,"çͬ":119125,"èŁ²":119126,"åĸª":119127,"å¦ĵ":119128,"è¬Ģ":119129,"çĤĬ":119130,"æĽľ":119131,"æ±IJ":119132,"è´Ī":119133,"èįĢ":119134,"æĬł":119135,"碾":119136,"æ«ĥ":119137,"éŀł":119138,"èijĨ":119139,"祯":119140,"å½Ŀ":119141,"é¦į":119142,"åĮ£":119143,"æľŃ":119144,"åĿĤ":119145,"ä¿ij":119146,"èĵ®":119147,"çijĽ":119148,"æīī":119149,"èĩŁ":119150,"貫":119151,"çİ¥":119152,"æ·¼":119153,"åݲ":119154,"é³Į":119155,"å³Ń":119156,"åijĽ":119157,"é§":119158,"é§IJ":119159,"éģ·":119160,"俪":119161,"æĢĤ":119162,"è¾į":119163,"å±į":119164,"åĭģ":119165,"å¥ļ":119166,"éļħ":119167,"éĴ´":119168,"è¼Ŀ":119169,"宦":119170,"èIJĥ":119171,"çĺĭ":119172,"æĨ¶":119173,"æĤħ":119174,"è¾Ļ":119175,"åijľ":119176,"çłº":119177,"éĢŀ":119178,"æµļ":119179,"éĸ£":119180,"èĸ©":119181,"éĻĭ":119182,"çĤĻ":119183,"èªķ":119184,"丣":119185,"é¹½":119186,"ç±Į":119187,"è´°":119188,"éĭª":119189,"çľ©":119190,"æĴIJ":119191,"èĨº":119192,"éŀĺ":119193,"ç¾²":119194,"窮":119195,"ç´IJ":119196,"æ®´":119197,"纾":119198,"èºį":119199,"ç´ĭ":119200,"çĦĸ":119201,"çĶº":119202,"çī½":119203,"çĤ¯":119204,"ç¼Ķ":119205,"æ¯ĵ":119206,"嬰":119207,"梧":119208,"äºŁ":119209,"è¢ħ":119210,"çįĦ":119211,"è¿¥":119212,"æ¼¾":119213,"çĿij":119214,"績":119215,"é¦ĭ":119216,"é¤ħ":119217,"æ¹Ħ":119218,"æĺĩ":119219,"æŀŃ":119220,"èĸ°":119221,"æŁij":119222,"榻":119223,"åĻĹ":119224,"åĻ´":119225,"棣":119226,"åͧ":119227,"çĨ¹":119228,"輯":119229,"å¢Ł":119230,"é²²":119231,"æĪĽ":119232,"èī¦":119233,"èĬ®":119234,"åĺŁ":119235,"帥":119236,"å¿»":119237,"çĮĿ":119238,"寵":119239,"賦":119240,"èĽ¾":119241,"滾":119242,"çĤķ":119243,"éĵ¬":119244,"èĴ¿":119245,"éĴ¨":119246,"çĥĻ":119247,"ç²ķ":119248,"æĥ¦":119249,"溧":119250,"é¢į":119251,"éħ£":119252,"峦":119253,"ç±ģ":119254,"çĥĥ":119255,"åĨĹ":119256,"åıģ":119257,"缧":119258,"ç½µ":119259,"éĴĹ":119260,"å¬ī":119261,"è°ı":119262,"ç³§":119263,"è¾Ń":119264,"æ·¬":119265,"èŁĴ":119266,"诩":119267,"è¦ĥ":119268,"çĻĸ":119269,"é½Ĵ":119270,"çĪIJ":119271,"ç®į":119272,"ç¼İ":119273,"磺":119274,"诫":119275,"褲":119276,"æĵł":119277,"èIJ¦":119278,"çĿ¬":119279,"è°į":119280,"éĦ°":119281,"æł¾":119282,"é¡ı":119283,"縱":119284,"桨":119285,"éĨ¬":119286,"襲":119287,"讪":119288,"婺":119289,"èįŁ":119290,"åĮĿ":119291,"çĨł":119292,"èĽĬ":119293,"æ¸ļ":119294,"å´½":119295,"鲤":119296,"åķ°":119297,"åĮķ":119298,"ä¸IJ":119299,"讥":119300,"åı½":119301,"åı¼":119302,"çļ¿":119303,"è¿Ĥ":119304,"åIJĨ":119305,"å±¹":119306,"èĩ¼":119307,"讹":119308,"é©®":119309,"纫":119310,"æ±ŀ":119311,"æĬ¡":119312,"èĭĩ":119313,"åIJł":119314,"åIJŃ":119315,"åIJ®":119316,"å²ĸ":119317,"ä½ĥ":119318,"çĭĪ":119319,"åºĩ":119320,"åIJĿ":119321,"éŰ":119322,"æ±¹":119323,"忱":119324,"æĭĦ":119325,"æĭĹ":119326,"èĮī":119327,"èĭĽ":119328,"èĮģ":119329,"çŁ¾":119330,"èĻı":119331,"åij»":119332,"åĴĦ":119333,"å¿¿":119334,"èĤ®":119335,"çĭŀ":119336,"çĸŁ":119337,"çĸĻ":119338,"çĸļ":119339,"æ³ŀ":119340,"å¸ļ":119341,"å±ī":119342,"è¿¢":119343,"驹":119344,"çİ·":119345,"çıĬó":119346,"çıĬół":119347,"çıĬółĦ":119348,"çıĬółĦģ":119349,"æĮİ":119350,"æĭ´":119351,"åŀĽ":119352,"èį¤":119353,"æ®ĥ":119354,"çĽ¹":119355,"åĵĨ":119356,"è´»":119357,"毡":119358,"çĭ°":119359,"çĭ¡":119360,"æŁĴ":119361,"æģĥ":119362,"诬":119363,"è¢Ħ":119364,"诲":119365,"èļ¤":119366,"èĢĻ":119367,"åŁĤ":119368,"æįİ":119369,"æįĮ":119370,"æ¢Ĩ":119371,"éħĮ":119372,"çł¾":119373,"æ®ī":119374,"åĶł":119375,"æĻĮ":119376,"èļ£":119377,"èļª":119378,"èļĵ":119379,"鸯":119380,"åĶģ":119381,"åĶĨ":119382,"åĢĶ":119383,"èĪĢ":119384,"豺":119385,"èĥ°":119386,"鸵":119387,"鸳":119388,"é¦ģ":119389,"ç¾Ķ":119390,"æ¶£":119391,"æ¶ķ":119392,"æĤ¯":119393,"诽":119394,"è°Ĩ":119395,"ç¥Ł":119396,"绢":119397,"æįº":119398,"æį¶":119399,"æį»":119400,"æİĤ":119401,"èıł":119402,"èIJ¤":119403,"éħĹ":119404,"çľ¶":119405,"åķĦ":119406,"èļ¯":119407,"èĽĢ":119408,"åͬ":119409,"帷":119410,"éĵIJ":119411,"éĵĽ":119412,"åģİ":119413,"å¾Ļ":119414,"èĦ¯":119415,"è±ļ":119416,"çĮĸ":119417,"çĹĬ":119418,"æ¶®":119419,"æĥŃ":119420,"æĤ´":119421,"æĥĭ":119422,"è°ļ":119423,"æı©":119424,"æIJĢ":119425,"æIJĶ":119426,"æ¦Ķ":119427,"æ¤Ń":119428,"éĽ³":119429,"åĸ³":119430,"è·Ľ":119431,"èľĵ":119432,"èľĴ":119433,"é¹ĥ":119434,"éĶĦ":119435,"çĶ¥":119436,"çŃı":119437,"çĮ©":119438,"çĮ¬":119439,"çĮ¾":119440,"çĹ¢":119441,"çĹª":119442,"æĥ°":119443,"çªĺ":119444,"è°¤":119445,"éļĺ":119446,"å©¿":119447,"é¹ī":119448,"çijĻ":119449,"æĸŁ":119450,"椿":119451,"éħª":119452,"éĽ¹":119453,"åŦ":119454,"è··":119455,"è·º":119456,"è·¤":119457,"èľĪ":119458,"èľĹ":119459,"å¹Į":119460,"é¦ı":119461,"èªĬ":119462,"æ¼ĵ":119463,"è¤Ĥ":119464,"èĶĹ":119465,"èͼ":119466,"åħ¢":119467,"裳":119468,"èľ»":119469,"èĿĩ":119470,"åĺĢ":119471,"é͹":119472,"ç®ķ":119473,"箩":119474,"çĺ©":119475,"çĺŁ":119476,"æ¼±":119477,"寥":119478,"骡":119479,"æĴµ":119480,"æĴ¬":119481,"è±Į":119482,"åĺ¹":119483,"èĿł":119484,"èĿĮ":119485,"èĿĹ":119486,"èĿĻ":119487,"éķIJ":119488,"稼":119489,"ç¯ĵ":119490,"èĨĽ":119491,"鲫":119492,"çĺª":119493,"鲨":119494,"æĨĶ":119495,"ç¿©":119496,"褥":119497,"ç¼Ń":119498,"åĻ©":119499,"çĵ¢":119500,"éľİ":119501,"踱":119502,"è¹Ĥ":119503,"èŁĨ":119504,"鹦":119505,"篡":119506,"çĺ¸":119507,"窿":119508,"ç¼°":119509,"èĹIJ":119510,"è¹ĭ":119511,"èŁĭ":119512,"èŁĢ":119513,"赡":119514,"èĩĬ":119515,"é³Ħ":119516,"ç³ł":119517,"æĩ¦":119518,"åļ£":119519,"éķ°":119520,"é³į":119521,"ç°¸":119522,"çĻ£":119523,"é³ĸ":119524,"é¬ĵ":119525,"èłķ":119526,"éľ¹":119527,"èºı":119528,"黯":119529,"çĵ¤":119530,"çŁĹ":119531,"ä¹Ĥ":119532,"ä¹ľ":119533,"åħĢ":119534,"å¼ĭ":119535,"åŃij":119536,"åŃĵ":119537,"幺":119538,"äºĵ":119539,"廿":119540,"ä¸ı":119541,"åįħ":119542,"ä»ĥ":119543,"ä»ī":119544,"ä»Ĥ":119545,"åĪĪ":119546,"çĪ»":119547,"åįŀ":119548,"éĹ©":119549,"讣":119550,"夬":119551,"çĪ¿":119552,"æ¯ĭ":119553,"éĤĹ":119554,"éĤĽ":119555,"èī½":119556,"èī¿":119557,"åıµ":119558,"ä¸ķ":119559,"åĮľ":119560,"åĬ¢":119561,"åįŁ":119562,"åı±":119563,"åı»":119564,"仨":119565,"代":119566,"仡":119567,"仫":119568,"ä»ŀ":119569,"åį®":119570,"æ°IJ":119571,"çĬ°":119572,"åĪį":119573,"éĤĿ":119574,"éĤĻ":119575,"讦":119576,"è®§":119577,"讫":119578,"å°»":119579,"éĺ¡":119580,"å°ķ":119581,"å¼ģ":119582,"èĢĴ":119583,"çİİ":119584,"çİij":119585,"åľ¬":119586,"æī¦":119587,"åľª":119588,"åľ¹":119589,"æīª":119590,"åľ®":119591,"åľ¯":119592,"èĬĬ":119593,"èĬį":119594,"èĬĦ":119595,"èĬ¨":119596,"èĬij":119597,"èĬİ":119598,"èĬĹ":119599,"äºĺ":119600,"åİį":119601,"夼":119602,"æĪį":119603,"å°¥":119604,"乩":119605,"æĹ¯":119606,"æĽ³":119607,"å²Į":119608,"屺":119609,"åĩ¼":119610,"åĽ¡":119611,"éĴĩ":119612,"ç¼¶":119613,"æ°ĺ":119614,"æ°ĸ":119615,"çīĿ":119616,"ä¼İ":119617,"ä¼Ľ":119618,"ä¼¢":119619,"佤":119620,"仵":119621,"ä¼¥":119622,"ä¼§":119623,"ä¼ī":119624,"伫":119625,"åĽŁ":119626,"æ±Ĩ":119627,"åĪĸ":119628,"å¤Ļ":119629,"æĹ®":119630,"åĪİ":119631,"çĬ·":119632,"çĬ¸":119633,"èĪĽ":119634,"åĩ«":119635,"éĤ¬":119636,"饧":119637,"æ±Ķ":119638,"æ±ľ":119639,"æ±Ĭ":119640,"å¿ĸ":119641,"å¿ı":119642,"è®´":119643,"讵":119644,"è®·":119645,"èģ¿":119646,"èī®":119647,"åݾ":119648,"å¦ģ":119649,"纡":119650,"纣":119651,"纥":119652,"纨":119653,"çİķ":119654,"çİĻ":119655,"æĬŁ":119656,"æĬĶ":119657,"åľ»":119658,"åĿį":119659,"æĬĥ":119660,"ã§IJ":119661,"èĬ«":119662,"èĬ¾":119663,"èĭĪ":119664,"èĭ£":119665,"èĭĭ":119666,"èĬ¼":119667,"èĭĮ":119668,"èĭģ":119669,"èĬ©":119670,"èĬª":119671,"èĬ¡":119672,"èĬŁ":119673,"èĭĦ":119674,"èĭİ":119675,"èĭ¡":119676,"æĿĮ":119677,"æĿĵ":119678,"æĿĪ":119679,"å¿ij":119680,"åŃĽ":119681,"éĤ´":119682,"éĤ³":119683,"å¥ģ":119684,"è±ķ":119685,"å¿Ĵ":119686,"欤":119687,"轫":119688,"è¿ĵ":119689,"éĤ¶":119690,"å¿IJ":119691,"åį£":119692,"éĤº":119693,"æĹ°":119694,"åijĭ":119695,"åijĴ":119696,"åijĵ":119697,"åijĶ":119698,"åijĸ":119699,"æĹ¸":119700,"åIJ¡":119701,"èϬ":119702,"åIJ½":119703,"åIJ£":119704,"åIJ²":119705,"å¸ı":119706,"å²Ī":119707,"å²ĺ":119708,"åħķ":119709,"åĽµ":119710,"åĽ«":119711,"éĴĬ":119712,"éĴĭ":119713,"éĴĮ":119714,"è¿ķ":119715,"æ°Ļ":119716,"æ°ļ":119717,"çī¤":119718,"ä½ŀ":119719,"ä½ļ":119720,"ä½Ŀ":119721,"ä½Ĺ":119722,"å½·":119723,"ä½ĺ":119724,"ä½¥":119725,"豸":119726,"åĿĮ":119727,"èĤŁ":119728,"å¥Ĥ":119729,"åĬ¬":119730,"çĭģ":119731,"鸳":119732,"饨":119733,"饩":119734,"饫":119735,"饬":119736,"åºij":119737,"åºĭ":119738,"çĸĶ":119739,"çĸĸ":119740,"èĤĵ":119741,"éű":119742,"éĹ³":119743,"çĤĢ":119744,"æ²£":119745,"æ²ħ":119746,"æ²Ķ":119747,"沤":119748,"æ²ı":119749,"æ²ļ":119750,"汩":119751,"汨":119752,"沨":119753,"æ±´":119754,"æ²Ĩ":119755,"沩":119756,"æ³IJ":119757,"æĢĥ":119758,"æĢĦ":119759,"å¿¡":119760,"忤":119761,"忾":119762,"æĢħ":119763,"忪":119764,"æĢĨ":119765,"å¿Ń":119766,"忸":119767,"è¯Ĥ":119768,"è¯ĥ":119769,"è¯ħ":119770,"è¯ĭ":119771,"è¯Į":119772,"è¯Ĵ":119773,"éĻĤ":119774,"éĻī":119775,"妩":119776,"妪":119777,"妣":119778,"å¦Ĺ":119779,"妫":119780,"å§Ĵ":119781,"妤":119782,"åĬŃ":119783,"åĪŃ":119784,"éĤ°":119785,"çºŃ":119786,"纰":119787,"纴":119788,"çİ¡":119789,"çİŃ":119790,"çİł":119791,"çİ¢":119792,"çݦ":119793,"çĽĤ":119794,"å¿Ŀ":119795,"åĮ¦":119796,"åĿ©":119797,"æĬ¨":119798,"æĭ¤":119799,"åĿ«":119800,"æĭĪ":119801,"åŀĨ":119802,"æĬ»":119803,"åĬ¼":119804,"æĭĥ":119805,"æĭĬ":119806,"åĿ¼":119807,"åĿ»":119808,"ã§Ł":119809,"åĿ¨":119810,"åĿŃ":119811,"æĬ¿":119812,"åĿ³":119813,"èĭ·":119814,"èĭ¤":119815,"èĮı":119816,"èĭ«":119817,"èĭľ":119818,"èĭ´":119819,"èĭĴ":119820,"èĭĺ":119821,"èĮĮ":119822,"èĭ»":119823,"èĭĵ":119824,"èĮļ":119825,"èĮĨ":119826,"èĮij":119827,"èĮĵ":119828,"èĮĶ":119829,"èĮķ":119830,"èĮĢ":119831,"èĭķ":119832,"æŀ¥":119833,"æŀĩ":119834,"æĿª":119835,"æĿ³":119836,"æŀ§":119837,"æĿµ":119838,"æŀ¨":119839,"æŀŀ":119840,"æŀĭ":119841,"æĿ»":119842,"æĿ·":119843,"æĿ¼":119844,"磸":119845,"çłĢ":119846,"åγ":119847,"å¥Ħ":119848,"æ®ģ":119849,"éĥı":119850,"è½Ń":119851,"éĥħ":119852,"鸢":119853,"缱":119854,"æĺĻ":119855,"æĿ²":119856,"æĺĥ":119857,"åĴĤ":119858,"åij¸":119859,"æĺĢ":119860,"æĹ»":119861,"æĺī":119862,"çĤħ":119863,"çķĢ":119864,"èĻ®":119865,"åĴĢ":119866,"åij·":119867,"黾":119868,"åij±":119869,"åij¤":119870,"åĴĨ":119871,"åĴĽ":119872,"åij¶":119873,"åij£":119874,"åĴĿ":119875,"å²¢":119876,"岿":119877,"岬":119878,"岫":119879,"å¸Ļ":119880,"å²£":119881,"å³ģ":119882,"åĪ¿":119883,"å²·":119884,"åīĢ":119885,"å¸Ķ":119886,"å³Ħ":119887,"æ²ĵ":119888,"åĽ¹":119889,"ç½Ķ":119890,"éĴį":119891,"éĴİ":119892,"éĴı":119893,"éĴĴ":119894,"éĴķ":119895,"éĤ¾":119896,"è¿®":119897,"çī¦":119898,"竺":119899,"迤":119900,"ä½¶":119901,"ä¾ij":119902,"ä¾ī":119903,"èĩ¾":119904,"ä¾Ĺ":119905,"ä¾ı":119906,"侩":119907,"ä½»":119908,"ä½¾":119909,"侪":119910,"ä½¼":119911,"佯":119912,"侬":119913,"å¸Ľ":119914,"ä¾Ķ":119915,"å¾Ĥ":119916,"åν":119917,"éĥĦ":119918,"ç±´":119919,"çĵ®":119920,"æĪĹ":119921,"èĤ¼":119922,"äıĿ":119923,"èĤ±":119924,"èĤ«":119925,"è¿©":119926,"éĥĩ":119927,"çĭİ":119928,"çĭį":119929,"çĭĴ":119930,"åĴİ":119931,"饯":119932,"饴":119933,"åĨ½":119934,"åĨ¼":119935,"åºĸ":119936,"çĸł":119937,"çĸĿ":119938,"åħĸ":119939,"åĬ¾":119940,"ð¬ī":119941,"ð¬ī¼":119942,"çĤĺ":119943,"çĤĿ":119944,"çĤĶ":119945,"æ³Ķ":119946,"æ²Ń":119947,"æ³·":119948,"æ³±":119949,"æ³ħ":119950,"æ³ł":119951,"泺":119952,"æ³ĸ":119953,"泫":119954,"æ³®":119955,"æ²±":119956,"泯":119957,"æĢĻ":119958,"æĢµ":119959,"æĢ¦":119960,"æĢĽ":119961,"æĢı":119962,"æĢį":119963,"ã¤":119964,"ã¤ĺ":119965,"æĢ©":119966,"æĢ«":119967,"æĢ¿":119968,"å®ķ":119969,"穹":119970,"å®ĵ":119971,"è¯ĵ":119972,"è¯Ķ":119973,"è¯ĸ":119974,"è¯ĺ":119975,"æĪ¾":119976,"è¯Ļ":119977,"æĪ½":119978,"éĥĵ":119979,"è¡©":119980,"ç¥Ĩ":119981,"ç¥İ":119982,"ç¥ĩ":119983,"è¯ľ":119984,"è¯Ł":119985,"诣":119986,"诤":119987,"诧":119988,"诨":119989,"æĪķ":119990,"éĻĶ":119991,"妲":119992,"妯":119993,"å§Ĺ":119994,"å¸ij":119995,"åŃ¥":119996,"驽":119997,"èϱ":119998,"迨":119999,"ç»Ģ":120000,"ç»ģ":120001,"ç»Ĥ":120002,"é©·":120003,"驸":120004,"ç»ī":120005,"ç»Į":120006,"éªĢ":120007,"ç;":120008,"çıı":120009,"çıIJ":120010,"çıij":120011,"çݳ":120012,"顸":120013,"çıī":120014,"çıĪ":120015,"æĭ®":120016,"åŀŃ":120017,"æĮĿ":120018,"æĮŀ":120019,"åŀ¤":120020,"èµ³":120021,"è´²":120022,"åŀ±":120023,"åŀĮ":120024,"åŀ§":120025,"åŀĵ":120026,"æĮ¦":120027,"åŀł":120028,"èįļ":120029,"èįij":120030,"è´³":120031,"èįľ":120032,"èİĴ":120033,"èĮ¼":120034,"èĮ´":120035,"èĮ±":120036,"èİĽ":120037,"èįŀ":120038,"èĮ¯":120039,"èįı":120040,"èįĩ":120041,"èįĥ":120042,"èįł":120043,"èĮŃ":120044,"åŀ©":120045,"èį¥":120046,"èį¦":120047,"èį¨":120048,"èį©":120049,"åīĭ":120050,"èįª":120051,"èį¬":120052,"èį®":120053,"æŁ°":120054,"æłī":120055,"æŁĺ":120056,"æłĬ":120057,"æŁ©":120058,"æŀ°":120059,"æłĮ":120060,"æŁĻ":120061,"æŀµ":120062,"æŀ³":120063,"æŁŀ":120064,"æŁĿ":120065,"æłĢ":120066,"æŁ¢":120067,"æłİ":120068,"æŁĪ":120069,"æŁģ":120070,"æŀ·":120071,"æŁ½":120072,"åīĮ":120073,"éħĬ":120074,"éĥ¦":120075,"çĶŃ":120076,"çłĹ":120077,"çłĺ":120078,"çłĴ":120079,"æĸ«":120080,"çłŃ":120081,"çłľ":120082,"èĢ·":120083,"èĻº":120084,"æ®Ĥ":120085,"æ®ĩ":120086,"æ®Ħ":120087,"è½±":120088,"è½²":120089,"è½³":120090,"è½¶":120091,"轸":120092,"èĻ¿":120093,"æ¯ĸ":120094,"è§ĩ":120095,"å°ľ":120096,"åĵIJ":120097,"çľĦ":120098,"çľį":120099,"ðł³":120100,"ðł³IJ":120101,"éĥ¢":120102,"çľĩ":120103,"çľĬ":120104,"çľĪ":120105,"禺":120106,"åĵĤ":120107,"åĴ´":120108,"æĽ·":120109,"æĺ´":120110,"åĴ¦":120111,"åĵĵ":120112,"åĵĶ":120113,"çķİ":120114,"åij²":120115,"èĥĦ":120116,"çķĭ":120117,"çķĪ":120118,"èϼ":120119,"èĻ»":120120,"çĽħ":120121,"åĴ£":120122,"åĵķ":120123,"åīIJ":120124,"éĥ§":120125,"åĴ»":120126,"åĽ¿":120127,"åĴ¿":120128,"åĵĮ":120129,"åĵĻ":120130,"åĵļ":120131,"åĴ©":120132,"åĴ¤":120133,"åĵĿ":120134,"åĵı":120135,"åĵŀ":120136,"å³£":120137,"ç½ĺ":120138,"å³Ĵ":120139,"峤":120140,"å³ĭ":120141,"è´¶":120142,"éĴļ":120143,"éĴ¡":120144,"éĴ£":120145,"éĴ¤":120146,"éĴ«":120147,"æ°¡":120148,"çī¯":120149,"éĥľ":120150,"ç§ķ":120151,"ç§Ń":120152,"竽":120153,"ç¬Ī":120154,"俦":120155,"俨":120156,"ä¿ħ":120157,"åıŁ":120158,"åŀ¡":120159,"çī®":120160,"ä¿£":120161,"ä¿ļ":120162,"çļĪ":120163,"ä¿Ł":120164,"éĢħ":120165,"å¾ĩ":120166,"å¾ī":120167,"èĪ¢":120168,"éĥĹ":120169,"ä¿İ":120170,"éĥ¤":120171,"çΰ":120172,"éĥĽ":120173,"çĵ´":120174,"èĥ¨":120175,"èĥª":120176,"èĥĽ":120177,"èĥĤ":120178,"èĥĻ":120179,"èĥį":120180,"èĥĹ":120181,"èĥĿ":120182,"æľIJ":120183,"èĥ«":120184,"鸨":120185,"åĮį":120186,"çĭ¨":120187,"çĭ¯":120188,"é£ij":120189,"çĭ©":120190,"çĭ²":120191,"è¨ĩ":120192,"éĢĦ":120193,"æĺĿ":120194,"饷":120195,"饸":120196,"饹":120197,"åŃª":120198,"å¨Ī":120199,"庥":120200,"çĸ¬":120201,"çĸ£":120202,"çĸ¥":120203,"çĸŃ":120204,"åºł":120205,"ç«ij":120206,"é£Ĵ":120207,"éĹ¼":120208,"éĹ¾":120209,"éĹ¿":120210,"éĺĤ":120211,"ç¾ij":120212,"迸":120213,"ç±¼":120214,"éħĭ":120215,"çĤ»":120216,"çĥĢ":120217,"çĤ·":120218,"æ´±":120219,"æ´¹":120220,"æ´§":120221,"æ´Į":120222,"æµĥ":120223,"æ´ĩ":120224,"æ´Ħ":120225,"æ´Ļ":120226,"æ¶İ":120227,"æ´İ":120228,"æ´«":120229,"æµį":120230,"æ´®":120231,"æ´µ":120232,"æµĴ":120233,"æµĶ":120234,"æµķ":120235,"æ´³":120236,"æģ¸":120237,"æģĵ":120238,"æģ¹":120239,"æģ«":120240,"æģ»":120241,"æģĤ":120242,"æģª":120243,"æģ½":120244,"宥":120245,"æīĥ":120246,"衲":120247,"衽":120248,"è¡¿":120249,"è¢Ĥ":120250,"ç¥ľ":120251,"ç¥ĵ":120252,"ç¥ļ":120253,"诮":120254,"ç¥Ĺ":120255,"祢":120256,"诰":120257,"诳":120258,"鸩":120259,"æĺ¶":120260,"åĴ«":120261,"å¼Ń":120262,"çīģ":120263,"èĥ¥":120264,"éĻŁ":120265,"å§®":120266,"å¨Ĩ":120267,"å§Ŀ":120268,"å§£":120269,"å§ĺ":120270,"å§¹":120271,"羿":120272,"çĤ±":120273,"磾":120274,"ç»Ķ":120275,"éªģ":120276,"éªħ":120277,"ç»Ĺ":120278,"综":120279,"éªĪ":120280,"èĢĸ":120281,"æĮĪ":120282,"çı¥":120283,"çıĻ":120284,"顼":120285,"çı°":120286,"çı©":120287,"çı§":120288,"çı£":120289,"çıŀ":120290,"çIJ¤":120291,"çı²":120292,"æģļ":120293,"åŁķ":120294,"åŁĺ":120295,"åŁĻ":120296,"åŁļ":120297,"æĮ¹":120298,"èĢĨ":120299,"èĢĦ":120300,"åŁĴ":120301,"æįĭ":120302,"è´½":120303,"åŀ¸":120304,"æįĥ":120305,"çĽį":120306,"èį¸":120307,"èݳ":120308,"èİ´":120309,"èݪ":120310,"èİł":120311,"èİľ":120312,"èİħ":120313,"èį¼":120314,"èİ©":120315,"èį½":120316,"èݸ":120317,"èį»":120318,"èݨ":120319,"鸪":120320,"èݼ":120321,"æł²":120322,"æł³":120323,"æ¡¡":120324,"æ¡İ":120325,"æ¡¢":120326,"桤":120327,"æ¢ĥ":120328,"æłĿ":120329,"æ¡ķ":120330,"æ¡ģ":120331,"æ¡§":120332,"æ¡ħ":120333,"æłŁ":120334,"æ¡ī":120335,"æł©":120336,"éĢij":120337,"éĢĭ":120338,"å½§":120339,"鬲":120340,"è±ĩ":120341,"éħIJ":120342,"é̦":120343,"åİĿ":120344,"åѬ":120345,"çłĿ":120346,"çł¹":120347,"çł§":120348,"çł·":120349,"糣":120350,"çł¼":120351,"çł¥":120352,"çł£":120353,"åīŀ":120354,"çł»":120355,"è½¼":120356,"è½¾":120357,"è¾Ĥ":120358,"鸫":120359,"趸":120360,"é¾Ģ":120361,"鸬":120362,"èĻĶ":120363,"羬":120364,"åĶĽ":120365,"çľĻ":120366,"åĵ§":120367,"åĵ½":120368,"æĻģ":120369,"鸮":120370,"è¶µ":120371,"è¶¿":120372,"çķĽ":120373,"èļ¨":120374,"èļľ":120375,"èļį":120376,"èļĭ":120377,"èļ¬":120378,"èļĿ":120379,"èļ§":120380,"åĶ¢":120381,"åľĦ":120382,"åĶ£":120383,"åĶı":120384,"çĽİ":120385,"åĶij":120386,"å´Ĥ":120387,"å´ĥ":120388,"罡":120389,"ç½Ł":120390,"è§Ĭ":120391,"èµħ":120392,"éĴ²":120393,"éĴµ":120394,"éĴ¹":120395,"éĴº":120396,"éĴ½":120397,"éĴ¼":120398,"éĴ¿":120399,"éĵĢ":120400,"éĵĦ":120401,"éĵĨ":120402,"éĵĪ":120403,"éĵī":120404,"éĵĬ":120405,"éĵĭ":120406,"éĵĮ":120407,"éĵį":120408,"ä¥":120409,"䥽":120410,"éĵİ":120411,"æ°©":120412,"æ°¤":120413,"æ°¦":120414,"毪":120415,"èĪIJ":120416,"ç§£":120417,"ç§«":120418,"çĽī":120419,"ç¬Ħ":120420,"ç¬ķ":120421,"ç¬Ĭ":120422,"ç¬ı":120423,"ç¬Ĩ":120424,"俸":120425,"俵":120426,"åģĮ":120427,"俳":120428,"ä¿¶":120429,"å̬":120430,"åĢı":120431,"æģģ":120432,"åĢŃ":120433,"俾":120434,"åĢľ":120435,"éļ¼":120436,"éļ½":120437,"åĢĮ":120438,"åĢ¥":120439,"èĩ¬":120440,"éĥ«":120441,"å̍":120442,"è¡Ħ":120443,"é¢Ģ":120444,"å¾ķ":120445,"èĪ«":120446,"衾":120447,"èĥ¯":120448,"èĥ±":120449,"èĥ´":120450,"èĥŃ":120451,"èĦį":120452,"èĥ¼":120453,"èĦĴ":120454,"鸱":120455,"鸲":120456,"çĭ·":120457,"çĮģ":120458,"çĭ³":120459,"çĮĥ":120460,"çĭº":120461,"éĢĸ":120462,"æ¡Ģ":120463,"饽":120464,"åĩĩ":120465,"æĮĽ":120466,"亳":120467,"çĸ³":120468,"çĸ´":120469,"çĸ¸":120470,"çĸ½":120471,"çĹĪ":120472,"çĸ±":120473,"çĹĤ":120474,"çĹī":120475,"è¡®":120476,"é¢ĥ":120477,"æģ£":120478,"æĹĨ":120479,"æĹĦ":120480,"æĹĥ":120481,"éĺĥ":120482,"éĺĦ":120483,"è¨ļ":120484,"éĺĨ":120485,"æģĻ":120486,"ç²ij":120487,"çĥľ":120488,"çĥ©":120489,"çĥĬ":120490,"åī¡":120491,"éĥ¯":120492,"çĥ¬":120493,"æ¶ij":120494,"浯":120495,"æ¶ŀ":120496,"æ¶Ł":120497,"å¨ij":120498,"æ¶ł":120499,"æµŀ":120500,"æ¶ĵ":120501,"æµ¥":120502,"æ¶Ķ":120503,"æµľ":120504,"æµł":120505,"æµ£":120506,"æĤļ":120507,"æĤŃ":120508,"æĤĿ":120509,"æĤĴ":120510,"æĤĮ":120511,"æĤĽ":120512,"çªĪ":120513,"åīľ":120514,"诹":120515,"诼":120516,"è¢Ĵ":120517,"袢":120518,"诿":120519,"è°Ģ":120520,"è°Ĥ":120521,"è°Ħ":120522,"è°ĩ":120523,"å±IJ":120524,"å±Ļ":120525,"éϬ":120526,"åĭIJ":120527,"å¥ĺ":120528,"çīĤ":120529,"èļ©":120530,"éϲ":120531,"å¨Į":120532,"å¨ī":120533,"娲":120534,"娴":120535,"娣":120536,"å¨ĵ":120537,"å©Ģ":120538,"çķļ":120539,"éĢ¡":120540,"绳":120541,"éªĬ":120542,"绡":120543,"éªĭ":120544,"绦":120545,"绨":120546,"éªİ":120547,"éĤķ":120548,"鸶":120549,"å½Ĺ":120550,"èĢľ":120551,"çĦĺ":120552,"èĪĤ":120553,"çIJı":120554,"çIJĩ":120555,"麸":120556,"æı¶":120557,"åŁ´":120558,"åŁ¯":120559,"æį¯":120560,"æİ³":120561,"æİ´":120562,"åŁ¸":120563,"åŁµ":120564,"èµ§":120565,"åŁ¤":120566,"æįŃ":120567,"é̵":120568,"åŁĿ":120569,"åłĭ":120570,"åłį":120571,"æİ¬":120572,"鸷":120573,"æį½":120574,"æİĬ":120575,"åłī":120576,"æİ¸":120577,"æį©":120578,"æİ®":120579,"æĤ«":120580,"åŁŃ":120581,"åŁ½":120582,"æİĩ":120583,"æİ¼":120584,"èģĥ":120585,"èIJģ":120586,"èıĺ":120587,"åłĩ":120588,"èIJĺ":120589,"èIJĭ":120590,"èı½":120591,"èıĸ":120592,"èIJľ":120593,"èIJ¸":120594,"èIJij":120595,"棻":120596,"èıĶ":120597,"èıŁ":120598,"èIJı":120599,"èı¹":120600,"èıª":120601,"èıħ":120602,"èıĢ":120603,"èı°":120604,"èı¡":120605,"梿":120606,"æ¢ı":120607,"è§ĭ":120608,"æ¡´":120609,"æ¡·":120610,"æ£ģ":120611,"æ¡«":120612,"æ£Ĥ":120613,"åķ¬":120614,"éĥ¾":120615,"æķķ":120616,"è±ī":120617,"éĦĦ":120618,"éħŀ":120619,"ç¡İ":120620,"ç¡Ń":120621,"ç¡ĸ":120622,"ç¡Ĺ":120623,"ç¡IJ":120624,"ç¡ĩ":120625,"ç¡Į":120626,"鸸":120627,"çĵł":120628,"åĮı":120629,"åİ©":120630,"æ®Ĵ":120631,"æ®ĵ":120632,"æ®į":120633,"èµī":120634,"鼩":120635,"è¾Ħ":120636,"åłij":120637,"çľŃ":120638,"羦":120639,"åķ§":120640,"æĻ¡":120641,"æĻ¤":120642,"çľµ":120643,"åľĬ":120644,"åĸı":120645,"åķī":120646,"åĭĸ":120647,"æĻŀ":120648,"å͵":120649,"æĻĹ":120650,"åķŃ":120651,"çķ¦":120652,"趺":120653,"åķ®":120654,"è·Ħ":120655,"èļ¶":120656,"èĽĦ":120657,"èĽİ":120658,"èĽĨ":120659,"èļ°":120660,"åľī":120661,"èļ±":120662,"èĽī":120663,"èĽı":120664,"èļ´":120665,"åķģ":120666,"åķķ":120667,"åĶ¿":120668,"åķIJ":120669,"åͼ":120670,"åĶ·":120671,"åķĸ":120672,"åķµ":120673,"åķ¶":120674,"åķ·":120675,"åͳ":120676,"åͰ":120677,"åķľ":120678,"帻":120679,"å´ļ":120680,"å´¦":120681,"帼":120682,"å´®":120683,"å´¤":120684,"å´Ĩ":120685,"èµĩ":120686,"èµĪ":120687,"èµĬ":120688,"éĵij":120689,"éĵĴ":120690,"éĵĹ":120691,"éĵĻ":120692,"éĵŁ":120693,"éĵ¡":120694,"éĵ¢":120695,"éĵ£":120696,"éĵ¤":120697,"éĵ§":120698,"éĵ¨":120699,"éĵ©":120700,"éĵª":120701,"éĵ«":120702,"éĵ¯":120703,"éĵ°":120704,"éĵ±":120705,"éĵ³":120706,"éĵµ":120707,"éĵ·":120708,"çī¾":120709,"鸹":120710,"ç§¾":120711,"é̶":120712,"笺":120713,"çŃĩ":120714,"笸":120715,"笪":120716,"笮":120717,"笳":120718,"笥":120719,"笤":120720,"笳":120721,"笾":120722,"ç¬ŀ":120723,"åģ¾":120724,"åģĥ":120725,"åģķ":120726,"åģĪ":120727,"åĤĢ":120728,"åģ¬":120729,"åģ»":120730,"çļij":120731,"çļİ":120732,"鸻":120733,"å¾ľ":120734,"èΏ":120735,"èĪ»":120736,"èĪ´":120737,"èĪ·":120738,"é¾Ľ":120739,"ç¿İ":120740,"èĦ¬":120741,"èĦĺ":120742,"èĦ²":120743,"åĮIJ":120744,"çĮĹ":120745,"çĮ¡":120746,"çĮŀ":120747,"æĸĽ":120748,"çĮķ":120749,"é¦Ĺ":120750,"é¦ĥ":120751,"é¦Ħ":120752,"鸾":120753,"庹":120754,"庾":120755,"çĹĶ":120756,"çĹį":120757,"ç¿Ĭ":120758,"æĹĮ":120759,"æĹİ":120760,"袤":120761,"éĺĩ":120762,"éĺĪ":120763,"éĺī":120764,"éĺĬ":120765,"éĺĭ":120766,"éĺį":120767,"éĺı":120768,"ç¾Ł":120769,"ç²Ŀ":120770,"çĦIJ":120771,"çĦĵ":120772,"çĦĹ":120773,"æ·ħ":120774,"æ·ŀ":120775,"æ¸İ":120776,"æ¶¿":120777,"æ·ĸ":120778,"æĮ²":120779,"æ·ł":120780,"涸":120781,"æ¸ij":120782,"æ·¦":120783,"æ·Ŀ":120784,"涪":120785,"æ·Ļ":120786,"æ¶«":120787,"æ¸Į":120788,"æĤ»":120789,"æĤ±":120790,"æĥĿ":120791,"æĥĺ":120792,"æĥĨ":120793,"æĥļ":120794,"æĥĩ":120795,"æĥ®":120796,"çªķ":120797,"è°Į":120798,"æīĪ":120799,"çļ²":120800,"è°ij":120801,"è£Ĩ":120802,"袷":120803,"è£ī":120804,"è°Ĵ":120805,"è°Ķ":120806,"è°ķ":120807,"è°ĸ":120808,"è°Ĺ":120809,"è°Ļ":120810,"è°Ŀ":120811,"é̝":120812,"éĥ¿":120813,"éļĪ":120814,"ç²ľ":120815,"éļį":120816,"éļĹ":120817,"å©Ĭ":120818,"娼":120819,"å©¢":120820,"婵":120821,"èĥ¬":120822,"è¢Ī":120823,"ç¿Į":120824,"æģ¿":120825,"欸":120826,"绫":120827,"éªIJ":120828,"绯":120829,"ç»±":120830,"éªĴ":120831,"绲":120832,"éªĵ":120833,"ç»¶":120834,"绺":120835,"ç»»":120836,"绾":120837,"éªĸ":120838,"ç¼ģ":120839,"èĢł":120840,"çIJ«":120841,"çIJµ":120842,"çIJ¶":120843,"çIJ¥":120844,"çIJ¨":120845,"çIJ°":120846,"çIJ®":120847,"çIJ¯":120848,"çIJ¬":120849,"çIJļ":120850,"è¾ĩ":120851,"é¼ĭ":120852,"æı³":120853,"åłŀ":120854,"æIJ½":120855,"æı¸":120856,"æıł":120857,"åłĻ":120858,"è¶Ħ":120859,"æıĸ":120860,"é¢ī":120861,"å¡Ħ":120862,"æı¿":120863,"èĢĭ":120864,"æıĦ":120865,"èĽ©":120866,"èĽ°":120867,"å¡Ĩ":120868,"æijĴ":120869,"æıĨ":120870,"æİ¾":120871,"èģĴ":120872,"èijij":120873,"èijļ":120874,"éĿ°":120875,"éĿ¸":120876,"èij³":120877,"èijº":120878,"èij¸":120879,"èIJ¼":120880,"èij¶":120881,"èĴĮ":120882,"èijŃ":120883,"楮":120884,"棼":120885,"æ¤Ł":120886,"棹":120887,"椤":120888,"棰":120889,"èµį":120890,"æ¤ĭ":120891,"æ¤ģ":120892,"椪":120893,"æ¤IJ":120894,"é¹ģ":120895,"éħ¤":120896,"éħ¢":120897,"éħ¡":120898,"é¹Ĥ":120899,"æ®ļ":120900,"æ®Ľ":120901,"鼱":120902,"è¾ĭ":120903,"æ¤ł":120904,"è¾İ":120905,"çĿĦ":120906,"çĿĩ":120907,"çĿĥ":120908,"æĪ¢":120909,"åĸĭ":120910,"åĹĴ":120911,"åĸĥ":120912,"åĸ±":120913,"åĸ¹":120914,"æĻ·":120915,"åĸĪ":120916,"è·ĸ":120917,"è·Ĺ":120918,"è·ŀ":120919,"è·ļ":120920,"è·İ":120921,"è·ı":120922,"è·Ĩ":120923,"èĽ±":120924,"èĽ²":120925,"èĽŃ":120926,"èĽ³":120927,"èĽIJ":120928,"èĽĶ":120929,"èĽŀ":120930,"èĽ´":120931,"èĽĺ":120932,"åĸģ":120933,"åĸŁ":120934,"åķ¾":120935,"åĹĸ":120936,"åĸij":120937,"åĹŁ":120938,"åĹŀ":120939,"åĸĻ":120940,"åµĺ":120941,"åµĸ":120942,"å´´":120943,"éģĦ":120944,"è©Ī":120945,"åµİ":120946,"嵬":120947,"åµĽ":120948,"嵯":120949,"åµĿ":120950,"嵫":120951,"å¹Ħ":120952,"åµĭ":120953,"èµķ":120954,"éĵ»":120955,"éĵ¼":120956,"éĵ¿":120957,"éĶĥ":120958,"éĶĨ":120959,"éĶĩ":120960,"éĶī":120961,"éĶı":120962,"éĶij":120963,"éĶĴ":120964,"éĶĶ":120965,"éĶķ":120966,"æİ£":120967,"磬":120968,"æ°°":120969,"毳":120970,"毽":120971,"çĬĬ":120972,"çĬĦ":120973,"çĬĭ":120974,"é¹Ħ":120975,"çĬį":120976,"åµĩ":120977,"é»į":120978,"ç¨ĥ":120979,"ç¨Ĥ":120980,"çŃļ":120981,"çѵ":120982,"çŃĮ":120983,"åĤ£":120984,"åĤĪ":120985,"èĪĦ":120986,"çīį":120987,"åĤ¥":120988,"åĤ§":120989,"éģij":120990,"åĤ©":120991,"徨":120992,"åªŃ":120993,"çķ²":120994,"å¼ij":120995,"ç¿ķ":120996,"é¹Ĩ":120997,"èħĪ":120998,"èħĵ":120999,"èħĨ":121000,"èħ´":121001,"èħļ":121002,"èħ±":121003,"鱿":121004,"é²Ģ":121005,"é²Ĥ":121006,"çĮ¢":121007,"çĮ¹":121008,"çĮ¥":121009,"é£ĵ":121010,"è§ŀ":121011,"è§ļ":121012,"çĮ±":121013,"é¢İ":121014,"飧":121015,"é¦ĩ":121016,"é¦Ĭ":121017,"亵":121018,"èĦĶ":121019,"è£Ĵ":121020,"çĹ£":121021,"çŨ":121022,"çŦ":121023,"çĹŀ":121024,"çŤ":121025,"çŧ":121026,"èµĵ":121027,"竦":121028,"çĵ¿":121029,"åķ»":121030,"é¢ı":121031,"é¹ĩ":121032,"éĺij":121033,"éĺĴ":121034,"éĺķ":121035,"ç²ŀ":121036,"éģĴ":121037,"åѳ":121038,"çĦ¯":121039,"çĦľ":121040,"çĦ±":121041,"é¹Ī":121042,"渫":121043,"æ¹®":121044,"æ¹İ":121045,"æ¹ľ":121046,"æ¹į":121047,"湫":121048,"溲":121049,"æ¹Ł":121050,"æºĨ":121051,"æ¹²":121052,"æ¹Ķ":121053,"æ¹ī":121054,"渥":121055,"æ»ģ":121056,"æĦł":121057,"æĥº":121058,"æĦ¦":121059,"æĥ´":121060,"æĦĢ":121061,"æĦİ":121062,"æĦĶ":121063,"åĸ¾":121064,"å¯IJ":121065,"è°Ł":121066,"裢":121067,"è£İ":121068,"裥":121069,"祾":121070,"è°ł":121071,"è°¡":121072,"è°¥":121073,"è°§":121074,"åѱ":121075,"å¼¼":121076,"å·½":121077,"éªĺ":121078,"媪":121079,"å·¯":121080,"ç¿ļ":121081,"çļ´":121082,"éªĽ":121083,"ç¼Ĥ":121084,"ç¼ĥ":121085,"ç¼Ħ":121086,"å½ĺ":121087,"ç¼ĩ":121088,"ç¼Ī":121089,"ç¼Į":121090,"ç¼ij":121091,"ç¼Ĵ":121092,"ç¼Ĺ":121093,"飨":121094,"èĢ¢":121095,"çijģ":121096,"çijĹ":121097,"çijĦ":121098,"éģ¨":121099,"éªľ":121100,"飫":121101,"é«¡":121102,"塬":121103,"éĦ¢":121104,"è¶Ķ":121105,"è¶ij":121106,"æijħ":121107,"æijģ":121108,"èľĩ":121109,"æIJĭ":121110,"æIJª":121111,"æIJIJ":121112,"æIJĽ":121113,"æIJł":121114,"æijĪ":121115,"å½Ģ":121116,"æ¯Ĥ":121117,"æIJ¦":121118,"æIJ¡":121119,"èĵģ":121120,"æĪ¡":121121,"èĵį":121122,"éĦŀ":121123,"èĵIJ":121124,"èĵ¦":121125,"é¹ĭ":121126,"èĴ½":121127,"èĵĸ":121128,"èĵĬ":121129,"èĴ¯":121130,"èĵŁ":121131,"èĵij":121132,"èĴº":121133,"èĵł":121134,"èĴŁ":121135,"èĴ¡":121136,"èĴ¹":121137,"èĴ´":121138,"èĴĹ":121139,"èĵ¥":121140,"æ¥Ķ":121141,"æ¥Ĥ":121142,"æ¥Ŀ":121143,"楫":121144,"楸":121145,"椴":121146,"æ§Į":121147,"楯":121148,"çļĻ":121149,"æ¦Ī":121150,"æ§İ":121151,"æ¦ī":121152,"楦":121153,"楣":121154,"楹":121155,"椽":121156,"åī½":121157,"éħ©":121158,"èľĥ":121159,"ç¢Ľ":121160,"ç¢ĵ":121161,"硼":121162,"ç¢ī":121163,"ç¢ļ":121164,"ç¢ĩ":121165,"ç¢ľ":121166,"é¹Į":121167,"è¾ı":121168,"é¾ĥ":121169,"é¾ħ":121170,"訾":121171,"ç²²":121172,"çĿļ":121173,"åĹª":121174,"éŁª":121175,"åĹ·":121176,"åĹī":121177,"çĿ¨":121178,"çĿ¢":121179,"éĽİ":121180,"çĿ¥":121181,"åĹij":121182,"åĹ«":121183,"åŬ":121184,"åĹĶ":121185,"åĹĿ":121186,"æĪ¥":121187,"åĹĦ":121188,"çħ¦":121189,"æļĦ":121190,"éģ¢":121191,"æļĮ":121192,"è·¬":121193,"è·¶":121194,"è·¸":121195,"è·IJ":121196,"è·£":121197,"è·¹":121198,"èĽ¸":121199,"èľĬ":121200,"èľį":121201,"èľī":121202,"èľ£":121203,"çķ¹":121204,"èĽ¹":121205,"åĹ¥":121206,"åĹ²":121207,"åĹ³":121208,"åĹĮ":121209,"åĹį":121210,"åĹIJ":121211,"åŤ":121212,"åŵ":121213,"罨":121214,"åµĬ":121215,"åµ´":121216,"骰":121217,"éĶĹ":121218,"éĶĽ":121219,"éĶľ":121220,"éĶĿ":121221,"éĶŀ":121222,"éĶŁ":121223,"éĶ¢":121224,"é͍":121225,"éĶ©":121226,"éĶŃ":121227,"éͱ":121228,"éĽī":121229,"æ°²":121230,"çĬı":121231,"æŃĥ":121232,"ç¨ŀ":121233,"ç¨Ĺ":121234,"ç¨Ķ":121235,"çŃł":121236,"çŃ¢":121237,"çŃ®":121238,"çѲ":121239,"çīĴ":121240,"æķ«":121241,"å¾Ń":121242,"æĦĨ":121243,"èīĦ":121244,"è§İ":121245,"毹":121246,"è²Ĭ":121247,"è²ħ":121248,"è²ī":121249,"é¢Ķ":121250,"èħł":121251,"èħ©":121252,"èħ¼":121253,"èħŃ":121254,"èħ§":121255,"å¡į":121256,"媵":121257,"é²ħ":121258,"é²Ĩ":121259,"é²ĩ":121260,"é²Ī":121261,"é²ĭ":121262,"é²IJ":121263,"èĤĦ":121264,"é¹IJ":121265,"é£ķ":121266,"è§¥":121267,"éģĽ":121268,"é¦IJ":121269,"é¹ij":121270,"亶":121271,"çĺĥ":121272,"çű":121273,"çĹ¼":121274,"çĹ¿":121275,"çĺIJ":121276,"çĺģ":121277,"çĺĨ":121278,"éºĤ":121279,"æŃĨ":121280,"æĹĴ":121281,"éĺĸ":121282,"éĺĹ":121283,"ç¾§":121284,"è±¢":121285,"ç²³":121286,"çĮ·":121287,"çħ³":121288,"çħ¨":121289,"çħħ":121290,"çħĬ":121291,"çħ¸":121292,"çħº":121293,"æ»Ł":121294,"溱":121295,"æºĺ":121296,"æ¼Ń":121297,"滢":121298,"溥":121299,"溽":121300,"è£Ł":121301,"溻":121302,"溷":121303,"æ»Ĺ":121304,"滫":121305,"溴":121306,"æ»ı":121307,"æ»ĥ":121308,"滦":121309,"æºı":121310,"æ»Ĥ":121311,"æ»ĵ":121312,"æºŁ":121313,"滪":121314,"æĦ«":121315,"æħĬ":121316,"é²İ":121317,"éªŀ":121318,"çªł":121319,"窣":121320,"裱":121321,"裨":121322,"裾":121323,"裰":121324,"ç¦Ĭ":121325,"è°©":121326,"è°ª":121327,"媾":121328,"å««":121329,"媲":121330,"å«Ĵ":121331,"å«Ķ":121332,"媸":121333,"ç¼Ļ":121334,"ç¼ľ":121335,"ç¼Ľ":121336,"è¾Ķ":121337,"éªĿ":121338,"ç¼Ł":121339,"缡":121340,"ç¼¢":121341,"ç¼£":121342,"éªŁ":121343,"èĢ¥":121344,"çĴĪ":121345,"çijŃ":121346,"çįĴ":121347,"è§ı":121348,"æħĿ":121349,"å«ł":121350,"åıĨ":121351,"æij½":121352,"å¢ģ":121353,"æĴĤ":121354,"æijŀ":121355,"æĴĦ":121356,"ç¿¥":121357,"è¸ħ":121358,"æijŃ":121359,"å¢ī":121360,"å¢Ĵ":121361,"æ¦ĸ":121362,"綦":121363,"èĶ«":121364,"èĶ·":121365,"éĿº":121366,"éĿ¼":121367,"éŀħ":121368,"éĿ¿":121369,"çĶį":121370,"è͏":121371,"èĶŁ":121372,"èĶº":121373,"æĪ¬":121374,"èķĸ":121375,"èĶ»":121376,"èĵ¿":121377,"æĸ¡":121378,"é¹ķ":121379,"èĵ¼":121380,"æ¦Ľ":121381,"榧":121382,"榫":121383,"æ¦Ń":121384,"æ§Ķ":121385,"榱":121386,"æ§ģ":121387,"æ§ł":121388,"榷":121389,"åĥ°":121390,"éħ½":121391,"éħ¹":121392,"碡":121393,"碴":121394,"碣":121395,"碲":121396,"èĩ§":121397,"豨":121398,"殡":121399,"éľģ":121400,"èľļ":121401,"é¾ĩ":121402,"é¾Ī":121403,"äģ":121404,"äģĸ":121405,"çĿ½":121406,"åĺŀ":121407,"åĺĪ":121408,"åĺĮ":121409,"åĺģ":121410,"æļĿ":121411,"è¸Į":121412,"è¸ī":121413,"èľŀ":121414,"èľ¥":121415,"èľ®":121416,"èĿĪ":121417,"èľ´":121418,"èľ±":121419,"èľ©":121420,"èľ·":121421,"èľ¿":121422,"èŀĤ":121423,"èľ¢":121424,"åĺ¡":121425,"é¹Ĺ":121426,"åĺ£":121427,"åĺ¤":121428,"åĺļ":121429,"åĹ¾":121430,"åĺ§":121431,"ç½´":121432,"ç½±":121433,"å¹Ķ":121434,"å¶Ĥ":121435,"å¹Ľ":121436,"èµĻ":121437,"ç½Ĥ":121438,"骷":121439,"骶":121440,"é¹ĺ":121441,"éͲ":121442,"éĶ´":121443,"éͶ":121444,"éĶ·":121445,"é͏":121446,"é͵":121447,"éķĤ":121448,"çĬĴ":121449,"ç®IJ":121450,"箦":121451,"ç®§":121452,"箸":121453,"箬":121454,"ç®ħ":121455,"箪":121456,"箾":121457,"箢":121458,"ç®ĵ":121459,"åĥĸ":121460,"åĦĨ":121461,"åĥ³":121462,"åĥŃ":121463,"åĬģ":121464,"åĥ®":121465,"éŃĥ":121466,"éŃĨ":121467,"çĿ¾":121468,"èīĭ":121469,"éĦ±":121470,"èĨĪ":121471,"èĨij":121472,"é²ij":121473,"é²Ķ":121474,"é²ļ":121475,"é²Ľ":121476,"é²Ł":121477,"çįIJ":121478,"è§«":121479,"éĽĴ":121480,"夤":121481,"é¦ij":121482,"éĬ®":121483,"塾":121484,"çĺĮ":121485,"çĺĬ":121486,"çĺĺ":121487,"çĺĻ":121488,"æĹĸ":121489,"èĨĤ":121490,"éĺļ":121491,"éĦ¯":121492,"é²ŀ":121493,"粿":121494,"ç²¼":121495,"ç³ģ":121496,"æ§Ĭ":121497,"é¹ļ":121498,"çĨĺ":121499,"çĨ¥":121500,"æ½¢":121501,"æ¼ķ":121502,"滹":121503,"漯":121504,"æ¼¶":121505,"æ½ĭ":121506,"æ½´":121507,"漪":121508,"æ¼ī":121509,"漩":121510,"æ¾ī":121511,"æħµ":121512,"æIJ´":121513,"窨":121514,"寤":121515,"ç¶®":121516,"è°®":121517,"褡":121518,"è¤Ļ":121519,"è¤ĵ":121520,"è¤Ľ":121521,"è¤Ĭ":121522,"è°¯":121523,"è°°":121524,"è°²":121525,"å±£":121526,"é¹Ľ":121527,"嫱":121528,"å«ĸ":121529,"嫦":121530,"å«ļ":121531,"å«ĺ":121532,"é¼IJ":121533,"çŀĢ":121534,"é¹ľ":121535,"éªł":121536,"ç¼¥":121537,"缦":121538,"ç¼§":121539,"缨":121540,"骢":121541,"缫":121542,"è̦":121543,"ȩ̀":121544,"çĴľ":121545,"çĴİ":121546,"çĴģ":121547,"å¥Ń":121548,"髯":121549,"é««":121550,"æĴ·":121551,"æĴħ":121552,"èµŃ":121553,"æĴ¸":121554,"éĭĨ":121555,"æĴĻ":121556,"æĴº":121557,"å¢Ģ":121558,"èģ©":121559,"è§IJ":121560,"éŀij":121561,"èķĻ":121562,"éŀĴ":121563,"èķĪ":121564,"èķ¨":121565,"èķ¤":121566,"èķŀ":121567,"èķº":121568,"çŀ¢":121569,"èķĥ":121570,"èķ²":121571,"èµľ":121572,"æ§¿":121573,"樯":121574,"æ§Ń":121575,"æ¨Ĺ":121576,"æ¨ĺ":121577,"æ§²":121578,"éĨĮ":121579,"éĨħ":121580,"éĿ¥":121581,"éŃĩ":121582,"é¤į":121583,"ç£Ķ":121584,"ç£Ļ":121585,"éľĪ":121586,"è¾ĺ":121587,"é¾ī":121588,"é¾Ĭ":121589,"è§ij":121590,"çŀĮ":121591,"çŀĭ":121592,"çŀij":121593,"åĺŃ":121594,"åĻİ":121595,"å϶":121596,"é¢Ļ":121597,"æļ¹":121598,"åĻĺ":121599,"è¸Ķ":121600,"è¸Ŀ":121601,"è¸Ł":121602,"è¸Ĵ":121603,"踬":121604,"踮":121605,"踯":121606,"踺":121607,"è¸ŀ":121608,"èĿ½":121609,"èĿ¾":121610,"èĿ»":121611,"èĿ°":121612,"èĿ®":121613,"èŀĭ":121614,"èĿĵ":121615,"èĿ£":121616,"èĿ¼":121617,"åĺ¬":121618,"é¢ļ":121619,"åĻį":121620,"åĻĻ":121621,"åĻĮ":121622,"åĻĶ":121623,"é¢Ľ":121624,"å¹ŀ":121625,"幡":121626,"å¶Ļ":121627,"å¶Ŀ":121628,"骺":121629,"éķĬ":121630,"éķī":121631,"éķĮ":121632,"éķı":121633,"éķĴ":121634,"éķĵ":121635,"éķĶ":121636,"稷":121637,"ç®´":121638,"ç¯ij":121639,"ç¯ģ":121640,"ç¯Į":121641,"çīĸ":121642,"åĦĭ":121643,"èĻ¢":121644,"é¹ŀ":121645,"èĨĺ":121646,"é²ł":121647,"鲡":121648,"é²¢":121649,"é²£":121650,"é²¥":121651,"é²§":121652,"鲩":121653,"çįĹ":121654,"çįł":121655,"觯":121656,"é¦ĵ":121657,"é¦Ķ":121658,"麾":121659,"å»Ľ":121660,"çĺĽ":121661,"çĺ¼":121662,"çĺ¢":121663,"çĺł":121664,"é½ij":121665,"ç¾°":121666,"ð¥»":121667,"ð¥»Ĺ":121668,"ç³Į":121669,"ç³į":121670,"ç³ħ":121671,"çĨľ":121672,"çĨµ":121673,"æ¾į":121674,"æ¾Į":121675,"潸":121676,"潦":121677,"æ½²":121678,"éĭĪ":121679,"æ½Ł":121680,"潺":121681,"寮":121682,"窳":121683,"è°³":121684,"褴":121685,"è¤Ł":121686,"褫":121687,"è°µ":121688,"çĨ¨":121689,"屦":121690,"åĭ°":121691,"æĪ®":121692,"èĿ¥":121693,"缬":121694,"ç¼®":121695,"缯":121696,"骣":121697,"çķ¿":121698,"èĢ©":121699,"è̍":121700,"èĢª":121701,"çĴŁ":121702,"éĿĽ":121703,"çĴł":121704,"çĴĺ":121705,"èģ±":121706,"èŀ¯":121707,"é«»":121708,"é«Ń":121709,"髹":121710,"æĵĢ":121711,"çĶı":121712,"æĵŀ":121713,"縳":121714,"磬":121715,"é¢ŀ":121716,"èķ»":121717,"é¢Ł":121718,"èĸ¤":121719,"èĸ¨":121720,"æªł":121721,"èĸı":121722,"èĸ®":121723,"èĸľ":121724,"èĸħ":121725,"樾":121726,"æ©Ľ":121727,"æ©ĩ":121728,"樵":121729,"æªİ":121730,"橹":121731,"樽":121732,"樨":121733,"橼":121734,"墼":121735,"æ©IJ":121736,"ç¿®":121737,"éĨIJ":121738,"éĨį":121739,"éĨļ":121740,"磲":121741,"èµĿ":121742,"殪":121743,"éľı":121744,"éĮ¾":121745,"è¾ļ":121746,"éģ½":121747,"æ°ħ":121748,"çŀŁ":121749,"çŀł":121750,"çŀ°":121751,"åļĦ":121752,"åļĨ":121753,"åϤ":121754,"æļ¾":121755,"è¹Ģ":121756,"踵":121757,"踽":121758,"è¹ī":121759,"è¹ģ":121760,"èŀ¨":121761,"èŀĪ":121762,"èŀħ":121763,"èŀŃ":121764,"èŀł":121765,"èŀŁ":121766,"åϱ":121767,"åĻ«":121768,"åĻ»":121769,"åϼ":121770,"ç½¹":121771,"åľľ":121772,"ä¦":121773,"ä¦ĥ":121774,"éķĹ":121775,"éķĺ":121776,"éķļ":121777,"éķĽ":121778,"éķĿ":121779,"éķŀ":121780,"éķł":121781,"æ°ĩ":121782,"æ°Ĩ":121783,"ç©ij":121784,"ç¯Ŀ":121785,"篥":121786,"篦":121787,"篪":121788,"ç¯Ļ":121789,"çĽ¥":121790,"åĬĵ":121791,"翱":121792,"éŃī":121793,"éŃĪ":121794,"å¾¼":121795,"æŃĻ":121796,"èĨ¦":121797,"èĨĻ":121798,"é²®":121799,"é²±":121800,"é²³":121801,"é²´":121802,"é²µ":121803,"é²·":121804,"é²»":121805,"çį´":121806,"çįŃ":121807,"çį¬":121808,"éĤĤ":121809,"é¹§":121810,"廨":121811,"èµŁ":121812,"çĺ°":121813,"廪":121814,"çĺ¿":121815,"çĺµ":121816,"çĺ´":121817,"çĻĥ":121818,"çĺ³":121819,"éºĩ":121820,"éºĪ":121821,"嬴":121822,"å£ħ":121823,"ç³Ĺ":121824,"çĶij":121825,"çĩİ":121826,"çĩł":121827,"çĩĶ":121828,"çĩ§":121829,"æ¿ij":121830,"æ¿ī":121831,"æ½ŀ":121832,"æ¾§":121833,"æ¾¹":121834,"æ¾¥":121835,"æ¾¶":121836,"æ¿Ĥ":121837,"褰":121838,"窸":121839,"å¬ĸ":121840,"çĬŁ":121841,"éļ°":121842,"å¬Ĺ":121843,"颡":121844,"ç¼±":121845,"ç¼²":121846,"ç¼³":121847,"çĴ©":121848,"çĴª":121849,"èŀ«":121850,"æĵ¤":121851,"å£ķ":121852,"è§³":121853,"ç½Ħ":121854,"æĵ¢":121855,"èĸ¹":121856,"éŀ¡":121857,"éŀ¬":121858,"èĸ·":121859,"èĹĵ":121860,"èĹģ":121861,"æªĦ":121862,"檩":121863,"æĩĭ":121864,"éĨ¢":121865,"翳":121866,"ç¤ħ":121867,"磴":121868,"鹩":121869,"é¾ĭ":121870,"é¾Į":121871,"è±³":121872,"å£ij":121873,"é»»":121874,"åļı":121875,"åļħ":121876,"è¹ij":121877,"è¹Ĵ":121878,"è¹Ĭ":121879,"èŁ¥":121880,"èŀ¬":121881,"èŀµ":121882,"çĸĥ":121883,"èŀ³":121884,"èŁij":121885,"åļĵ":121886,"ç½½":121887,"ç½¾":121888,"å¶·":121889,"黾":121890,"é»Ŀ":121891,"é«ģ":121892,"é«Ģ":121893,"éķ¡":121894,"éķ¢":121895,"éķ£":121896,"éķ¦":121897,"éķ§":121898,"éķ©":121899,"éķª":121900,"éķ«":121901,"ç½ħ":121902,"ç°Į":121903,"篾":121904,"篼":121905,"ç°ĸ":121906,"ç°ĭ":121907,"é¼¢":121908,"åĦ¡":121909,"鹪":121910,"é¼¾":121911,"çļ¤":121912,"éŃį":121913,"é¾ł":121914,"ç¹ĩ":121915,"è²ĺ":121916,"éĤĪ":121917,"è²Ķ":121918,"èĩĮ":121919,"èĨ»":121920,"èĩĨ":121921,"èĩĥ":121922,"é²¼":121923,"é²½":121924,"é³Ģ":121925,"é³ĥ":121926,"é³ħ":121927,"é³ĩ":121928,"é³Ĭ":121929,"èŀ½":121930,"çĩ®":121931,"鹫":121932,"ç³ľ":121933,"縻":121934,"çĻį":121935,"éºĭ":121936,"æĩij":121937,"æ¿¡":121938,"æ¿®":121939,"æ¿ŀ":121940,"æ¿ł":121941,"濯":121942,"è¹ĩ":121943,"è¬ĩ":121944,"éĤĥ":121945,"è¥ģ":121946,"æªĹ":121947,"æĵĺ":121948,"åŃº":121949,"éļ³":121950,"嬷":121951,"èŁĬ":121952,"鹬":121953,"éįª":121954,"éıĬ":121955,"é¬Ī":121956,"é¬ĥ":121957,"çŀ½":121958,"éŀ¯":121959,"éŀ¨":121960,"éŀ«":121961,"éŀ§":121962,"éŀ£":121963,"èĹľ":121964,"èĹł":121965,"éĨª":121966,"è¹Ļ":121967,"ç¤ĵ":121968,"çĩ¹":121969,"餮":121970,"çŀ¿":121971,"æĽĽ":121972,"颢":121973,"èºĩ":121974,"è¹ļ":121975,"èŁĽ":121976,"èŁª":121977,"èŁł":121978,"èŁ®":121979,"é¹®":121980,"黳":121981,"黣":121982,"é«ħ":121983,"é«Ĥ":121984,"éķ¬":121985,"éķŃ":121986,"éķ¯":121987,"馥":121988,"ç°Ł":121989,"ç°ª":121990,"鼬":121991,"鼳":121992,"èīŁ":121993,"é³İ":121994,"é³ı":121995,"é³IJ":121996,"çĻŀ":121997,"çĻĶ":121998,"糨":121999,"蹩":122000,"éİı":122001,"éĤĭ":122002,"é¬ı":122003,"æĶī":122004,"éŀ²":122005,"éŀ´":122006,"èĹ¿":122007,"èĺ§":122008,"èĺħ":122009,"éĨ®":122010,"éĨ¯":122011,"éħĥ":122012,"éľª":122013,"éľŃ":122014,"龨":122015,"黼":122016,"åļ¯":122017,"è¹°":122018,"è¹¶":122019,"è¹½":122020,"è¹¼":122021,"è¹´":122022,"è¹¾":122023,"蹿":122024,"èłĸ":122025,"èłĵ":122026,"èŁ¾":122027,"èłĬ":122028,"黢":122029,"é«ĭ":122030,"é«Į":122031,"éķ²":122032,"ç±Ģ":122033,"é½ģ":122034,"éŃij":122035,"èī¨":122036,"é³ĵ":122037,"é³Ķ":122038,"é³ķ":122039,"é³Ĺ":122040,"é³Ļ":122041,"éıĸ":122042,"羸":122043,"ã¸Ĩ":122044,"çĢ£":122045,"çĢĽ":122046,"襦":122047,"è°¶":122048,"è¥ŀ":122049,"骥":122050,"ç¼µ":122051,"çĵĴ":122052,"æĶĺ":122053,"èĺ©":122054,"èĺĸ":122055,"éĨ´":122056,"éľ°":122057,"éħĨ":122058,"çŁį":122059,"èºħ":122060,"é¼į":122061,"å·ī":122062,"黩":122063,"黥":122064,"黪":122065,"éķ³":122066,"éķ´":122067,"é»§":122068,"çºĤ":122069,"çĴº":122070,"鼯":122071,"èĩľ":122072,"é³ľ":122073,"é³Ŀ":122074,"é³Ł":122075,"çį¾":122076,"åŃĢ":122077,"骧":122078,"çĵĺ":122079,"é¼Ļ":122080,"éĨº":122081,"礴":122082,"颦":122083,"æĽ©":122084,"é³¢":122085,"éºĿ":122086,"å¤Ķ":122087,"çĪĿ":122088,"çģı":122089,"禳":122090,"éIJ¾":122091,"ç¾¼":122092,"èł¡":122093,"è̱":122094,"é¹³":122095,"æ°į":122096,"é¥ķ":122097,"èºIJ":122098,"é«ij":122099,"éķµ":122100,"ç©°":122101,"é¥Ķ":122102,"鬻":122103,"鬣":122104,"è¶±":122105,"æĶ«":122106,"æĶ¥":122107,"颧":122108,"èºľ":122109,"é¼¹":122110,"çϝ":122111,"èł²":122112,"èł¹":122113,"èºŀ":122114,"è¡¢":122115,"çģŀ":122116,"襻":122117,"çºĽ":122118,"鬣":122119,"æĶ®":122120,"åĽĶ":122121,"é¦ķ":122122,"æĪĨ":122123,"ç΍":122124,"é½ī":122125,"äºį":122126,"å°¢":122127,"å½³":122128,"åį¬":122129,"殳":122130,"ðłĻ¶":122131,"æ¯Į":122132,"éĤĺ":122133,"æĪĭ":122134,"åľ¢":122135,"æ°ķ":122136,"ä¼ĭ":122137,"ä»Ŀ":122138,"åĨ®":122139,"æ°¿":122140,"æ±Ī":122141,"æ°¾":122142,"å¿ī":122143,"å®Ħ":122144,"ð¬£Ļ":122145,"è®±":122146,"æīŀ":122147,"åľ²":122148,"åľ«":122149,"èĬı":122150,"èĬĥ":122151,"æľ³":122152,"æľ¸":122153,"ð¨Ļ":122154,"ð¨Ļ¸":122155,"éĤ¨":122156,"åIJĴ":122157,"åIJĸ":122158,"å±¼":122159,"å±¾":122160,"辿":122161,"éĴĨ":122162,"仳":122163,"ä¼£":122164,"ä¼Ī":122165,"çĻ¿":122166,"çĶª":122167,"éĤł":122168,"çĬ´":122169,"åĨ±":122170,"éĤ¡":122171,"ð¬ĩķ":122172,"æ±ĭ":122173,"äľ":122174,"äľ£":122175,"è®»":122176,"ð¬£ŀ":122177,"åŃĸ":122178,"ð¬ĺĵ":122179,"纩":122180,"çİĴ":122181,"çİĵ":122182,"çİĺ":122183,"çİļ":122184,"åά":122185,"ð«ŃŁ":122186,"åĿľ":122187,"åĿī":122188,"æī½":122189,"ð«Ń¢":122190,"åĿĭ":122191,"æīº":122192,"ã§ij":122193,"æ¯IJ":122194,"èĬ°":122195,"èĬ£":122196,"èĭĬ":122197,"èĭī":122198,"èĬĺ":122199,"èĬ´":122200,"èĬł":122201,"ð«ĩ":122202,"ð«ĩŃ":122203,"èĬ¤":122204,"æĿķ":122205,"æĿĻ":122206,"æĿĦ":122207,"æĿ§":122208,"æĿ©":122209,"å°ª":122210,"å°¨":122211,"轪":122212,"ð«IJĦ":122213,"åĿĴ":122214,"èĬĪ":122215,"æĹ´":122216,"æĹµ":122217,"åijĻ":122218,"ãķ":122219,"ãķ®":122220,"å²į":122221,"ð«µ":122222,"𫵷":122223,"å²ł":122224,"å²ľ":122225,"åijĩ":122226,"åĨı":122227,"è§ĥ":122228,"å²Ļ":122229,"ä¼¾":122230,"ãijĩ":122231,"ä¼Ń":122232,"ä½ĸ":122233,"ä¼²":122234,"ä½ģ":122235,"é£ı":122236,"çĭĥ":122237,"éŶ":122238,"æ±§":122239,"汫":122240,"ð£²ĺ":122241,"ð£²Ĺ":122242,"æ²Ħ":122243,"æ²ĺ":122244,"ð¬ĩĻ":122245,"æ±Ń":122246,"ã³ĩ":122247,"æ²ĩ":122248,"å¿®":122249,"忳":122250,"忺":122251,"𬣡":122252,"ç¥ĥ":122253,"è¯ĩ":122254,"éĤ²":122255,"è¯İ":122256,"è¯IJ":122257,"å±ĥ":122258,"ð«¸":122259,"𫸩":122260,"å²Ĭ":122261,"éĺ½":122262,"䢺":122263,"éĺ¼":122264,"妧":122265,"å¦ĺ":122266,"ð¨ļ":122267,"ð¨ļķ":122268,"纮":122269,"驲":122270,"ð«ĺľ":122271,"纻":122272,"ð¬ĺĺ":122273,"ð«ĺĿ":122274,"纼":122275,"çݤ":122276,"çİŀ":122277,"çݱ":122278,"çİŁ":122279,"éĤ½":122280,"éĤ¿":122281,"åĿ¥":122282,"åĿ°":122283,"åĿ¬":122284,"åĿ½":122285,"å¼Ĩ":122286,"è̵":122287,"䢼":122288,"ð¦Ń":122289,"ð¦Ńľ":122290,"èĮĭ":122291,"èĭ§":122292,"èĭ¾":122293,"èĭł":122294,"æŀħ":122295,"ãŃİ":122296,"æŀĺ":122297,"æŀį":122298,"çŁ¼":122299,"磻":122300,"åĮ¼":122301,"ð¬¨Ĥ":122302,"ð¬Ģ©":122303,"ð¬Ģª":122304,"æĹ¿":122305,"æĺĦ":122306,"æĺĴ":122307,"æĺĪ":122308,"åĴī":122309,"åĴĩ":122310,"åĴį":122311,"å²µ":122312,"å²½":122313,"岨":122314,"å²ŀ":122315,"å³Ĥ":122316,"ãŁ":122317,"ãŁĥ":122318,"åĽ·":122319,"𬬩":122320,"éĴIJ":122321,"éĴĶ":122322,"éĴĸ":122323,"çī¥":122324,"ä½´":122325,"åŀĪ":122326,"ä¾ģ":122327,"ä¾¹":122328,"佸":122329,"佺":122330,"éļ¹":122331,"ãijĬ":122332,"ä¾Ĥ":122333,"ä½½":122334,"ä¾ĺ":122335,"éĥĪ":122336,"èĪł":122337,"éĥIJ":122338,"éĥĥ":122339,"æĶ½":122340,"èĤŃ":122341,"èĤ¸":122342,"èĤ·":122343,"çĭī":122344,"çĭĿ":122345,"饳":122346,"å¿ŀ":122347,"çĤĮ":122348,"çĤĨ":122349,"æ³Ļ":122350,"沺":122351,"æ³Ĥ":122352,"æ³ľ":122353,"æ³ĥ":122354,"æ³ĩ":122355,"æĢĬ":122356,"å³ĥ":122357,"穸":122358,"ç¥ĭ":122359,"ç¥Ĭ":122360,"ð«į£":122361,"𬣳":122362,"𬩽":122363,"鸤":122364,"å¼¢":122365,"弨":122366,"éĻij":122367,"𬮿":122368,"éĻİ":122369,"ð¬¯Ģ":122370,"åįº":122371,"乸":122372,"å¦Ń":122373,"å§Ī":122374,"ð«°":122375,"ð«°Ľ":122376,"迳":122377,"åıķ":122378,"𬳵":122379,"驵":122380,"𬳶":122381,"äĮ":122382,"äĮ¹":122383,"驺":122384,"ð«łĬ":122385,"ç»ĭ":122386,"ç»IJ":122387,"çłī":122388,"èĢĶ":122389,"ãĽĥ":122390,"çݶ":122391,"çıĩ":122392,"çıħ":122393,"ð¬įĽ":122394,"çıĭ":122395,"çݹ":122396,"çıĮ":122397,"çİ¿":122398,"飨":122399,"åŀļ":122400,"åŀ¯":122401,"åŀĻ":122402,"åŀ²":122403,"åŁı":122404,"åŀį":122405,"èĢĩ":122406,"é¿į":122407,"åŀİ":122408,"åŀ´":122409,"åŀŁ":122410,"åŀŀ":122411,"æĮĵ":122412,"åŀµ":122413,"åŀı":122414,"æĭ¶":122415,"èįĸ":122416,"èįģ":122417,"èįĻ":122418,"èįĽ":122419,"èĮĪ":122420,"èĮ½":122421,"èįĦ":122422,"èĮº":122423,"ð¬ľ¬":122424,"èįĵ":122425,"èĮ³":122426,"ð¦°":122427,"𦰡":122428,"èĮĽ":122429,"èįŃ":122430,"ãŃķ":122431,"æŁ·":122432,"æŁĥ":122433,"æŁĬ":122434,"æŀ¹":122435,"æłIJ":122436,"æŁĸ":122437,"éĥļ":122438,"åīħ":122439,"ä´ĵ":122440,"迺":122441,"åİĸ":122442,"çłĨ":122443,"çłij":122444,"çłĦ":122445,"èĢı":122446,"å¥ĵ":122447,"ä¶":122448,"ä¶®":122449,"è½µ":122450,"è½·":122451,"è½¹":122452,"轺":122453,"æĺº":122454,"ðª¾":122455,"𪾢":122456,"æĺ½":122457,"缷":122458,"åĴ¡":122459,"åĴº":122460,"æĺ³":122461,"æĺ£":122462,"æĺ¤":122463,"æĺ«":122464,"æĺ¡":122465,"åĴ¥":122466,"æĺª":122467,"èĻ·":122468,"èϏ":122469,"åĵĥ":122470,"å³ĺ":122471,"èĢij":122472,"å³Ľ":122473,"𪨰":122474,"å³Ĺ":122475,"å³§":122476,"帡":122477,"éĴĺ":122478,"ð«ĵ§":122479,"éĴľ":122480,"𬬮":122481,"𬬱":122482,"ð¬¬Ń":122483,"éĴª":122484,"éĴ¬":122485,"éĴŃ":122486,"磧":122487,"秬":122488,"ä¿«":122489,"èĪģ":122490,"ä¿ľ":122491,"ä¿Ļ":122492,"ä¿į":122493,"åŀķ":122494,"è¡İ":122495,"èĪ£":122496,"å¼ĩ":122497,"ä¾´":122498,"鸧":122499,"äı¡":122500,"èĥł":122501,"ð¦Ļ¶":122502,"èĥĪ":122503,"èĥ©":122504,"èĥ£":122505,"æľı":122506,"é£IJ":122507,"è¨Ħ":122508,"饻":122509,"庤":122510,"çĸ¢":122511,"çĤ£":122512,"çĤŁ":122513,"ã¶":122514,"ã¶²":122515,"æ´Ń":122516,"æ´ĺ":122517,"æ´ĵ":122518,"æ´¿":122519,"ã³ļ":122520,"æ³ļ":122521,"æµĪ":122522,"æµī":122523,"æ´¸":122524,"æ´ij":122525,"æ´¢":122526,"æ´Ī":122527,"æ´ļ":122528,"æ´º":122529,"æ´¨":122530,"æµIJ":122531,"ã³ĺ":122532,"æ´´":122533,"æ´£":122534,"æģĶ":122535,"宬":122536,"çªĢ":122537,"æīĤ":122538,"è¢Ĩ":122539,"ç¥ı":122540,"ç¥IJ":122541,"ç¥ķ":122542,"åıļ":122543,"éϧ":122544,"éĻŀ":122545,"å¨Ģ":122546,"å§ŀ":122547,"å§±":122548,"姤":122549,"å§¶":122550,"å§½":122551,"æŀ²":122552,"ç»ĸ":122553,"éªĥ":122554,"ð¬ĺ¡":122555,"𬳽":122556,"ð¬ĺ©":122557,"ð«Ħ§":122558,"å½ĸ":122559,"éªī":122560,"æģĿ":122561,"çıª":122562,"çıĽ":122563,"çı¹":122564,"çIJĬ":122565,"çݼ":122566,"çıĸ":122567,"ðªŁ":122568,"ðªŁĿ":122569,"çı½":122570,"çı¦":122571,"çı«":122572,"çıĴ":122573,"ð¬į¤":122574,"çı¢":122575,"çıķ":122576,"çıĿ":122577,"ð«Ń¼":122578,"åŁĹ":122579,"åŀ¾":122580,"åŀº":122581,"åŁĨ":122582,"åŀ¿":122583,"åŁĮ":122584,"åŁĩ":122585,"èݰ":122586,"èĮĿ":122587,"ð¬ľ¯":122588,"éĦĢ":122589,"èݶ":122590,"èİĿ":122591,"äĵĸ":122592,"èİĻ":122593,"æł»":122594,"æ¡ł":122595,"ð¬Ĥ":122596,"ð¬Ĥ©":122597,"æ¡Ħ":122598,"æ¢ł":122599,"æł´":122600,"梴":122601,"æłĴ":122602,"éħİ":122603,"éħı":122604,"ð«łĨ":122605,"çłµ":122606,"çłł":122607,"çł«":122608,"糬":122609,"ç¡ģ":122610,"æģ§":122611,"ç¿ĥ":122612,"éĥª":122613,"ð¨IJ":122614,"ð¨IJĪ":122615,"è¾Ģ":122616,"è¾ģ":122617,"ð¬Į":122618,"ð¬ĮĹ":122619,"åīķ":122620,"èµĢ":122621,"åĵ¢":122622,"æĻħ":122623,"æĻĬ":122624,"åĶĿ":122625,"åĵ³":122626,"åĵ±":122627,"åĨĶ":122628,"æĻĶ":122629,"æĻIJ":122630,"çķĸ":122631,"èļĦ":122632,"èļĨ":122633,"ð«ij":122634,"ð«ij¡":122635,"帱":122636,"å´ģ":122637,"峿":122638,"𪨶":122639,"å´Ħ":122640,"帨":122641,"å´Ģ":122642,"èµĨ":122643,"𬬸":122644,"éĴ·":122645,"𬬻":122646,"𬬹":122647,"𬬿":122648,"ð¬Ńģ":122649,"çľļ":122650,"çĶ¡":122651,"笫":122652,"åĢ»":122653,"åĢ´":122654,"èĦ©":122655,"åĢ®":122656,"åĢķ":122657,"åĢŀ":122658,"ð«¢":122659,"𫢸":122660,"åĢĵ":122661,"å̧":122662,"è¡ĥ":122663,"èĻĴ":122664,"èĪŃ":122665,"èΝ":122666,"èĪ¥":122667,"çĵŀ":122668,"鬯":122669,"鸰":122670,"èĦİ":122671,"æľĵ":122672,"èĥ²":122673,"èĻĵ":122674,"é±½":122675,"çĭ´":122676,"å³±":122677,"çĭ»":122678,"çľ¢":122679,"ð«Ĺ§":122680,"åĭį":122681,"çĹĦ":122682,"çĸ°":122683,"çĹĥ":122684,"ç«ĺ":122685,"ç¾ĸ":122686,"ç¾ĵ":122687,"æ¡Ĭ":122688,"æķī":122689,"çĥł":122690,"çĥĶ":122691,"çĥ¶":122692,"çĥ»":122693,"ð¬ĬĪ":122694,"æ¶į":122695,"浡":122696,"æµŃ":122697,"浬":122698,"æ¶Ħ":122699,"æ¶¢":122700,"æ¶IJ":122701,"æµ°":122702,"æµŁ":122703,"æµĽ":122704,"æµ¼":122705,"æµ²":122706,"æ¶ĺ":122707,"æĤĪ":122708,"æĤĥ":122709,"æĤ¢":122710,"ð¬ĴĪ":122711,"å®§":122712,"çªħ":122713,"çªĬ":122714,"çªİ":122715,"æīħ":122716,"æīĨ":122717,"袪":122718,"è¢Ĺ":122719,"袯":122720,"祧":122721,"éļº":122722,"åł²":122723,"çĸį":122724,"ð¨º":122725,"ð¨ºĻ":122726,"éĻ´":122727,"çĥĿ":122728,"çł®":122729,"ãĽļ":122730,"åĵ¿":122731,"ç¿Ģ":122732,"ç¿Ĥ":122733,"åīŁ":122734,"𬳿":122735,"ð«Ħ¨":122736,"绤":122737,"éªį":122738,"ð¬ĺ«":122739,"äĤ":122740,"äĤ®":122741,"çIJİ":122742,"çı¸":122743,"çıµ":122744,"çIJĦ":122745,"çIJĪ":122746,"çIJĢ":122747,"çıº":122748,"æİŃ":122749,"åłİ":122750,"åłIJ":122751,"åŁ¼":122752,"æİİ":122753,"åŁ«":122754,"åłĮ":122755,"æĻ¢":122756,"ð«®":122757,"ð«®ĥ":122758,"æİŀ":122759,"åŁª":122760,"壸":122761,"ãĻį":122762,"èģį":122763,"èıĿ":122764,"èIJļ":122765,"èı¥":122766,"èİ¿":122767,"äĵ«":122768,"åĭļ":122769,"äĵ¬":122770,"èIJĨ":122771,"èıĤ":122772,"èıį":122773,"èı¼":122774,"èIJ£":122775,"äĵ¨":122776,"èıī":122777,"äĵĽ":122778,"梼":122779,"梽":122780,"桲":122781,"梾":122782,"桯":122783,"梣":122784,"æ¢Į":122785,"桹":122786,"æķĶ":122787,"åİ£":122788,"ç¡Ķ":122789,"é¿İ":122790,"ç¡Ļ":122791,"ç¡ļ":122792,"ç¡Ĭ":122793,"ç¡į":122794,"åĭĶ":122795,"ä´ķ":122796,"é¾ģ":122797,"éĢ´":122798,"åĶª":122799,"åķ«":122800,"ç¿Ī":122801,"ã«":122802,"ã«°":122803,"æĻĻ":122804,"çķ¤":122805,"ð¬±ĸ":122806,"è¶¼":122807,"è·Ĥ":122808,"èĽĥ":122809,"èļ²":122810,"ð¬Ł½":122811,"èļº":122812,"åķ´":122813,"äİĥ":122814,"å´§":122815,"å´Ł":122816,"å´ŀ":122817,"å´Ĵ":122818,"å´Į":122819,"å´¡":122820,"éĵı":122821,"ð«ĵ¯":122822,"ð«Ł¹":122823,"éĵķ":122824,"ð«Ł¼":122825,"éĵĸ":122826,"éĵĺ":122827,"éĵļ":122828,"éĵŀ":122829,"éĵ¥":122830,"éĵ´":122831,"çī»":122832,"çī¿":122833,"ç¨Ĩ":122834,"笱":122835,"笯":122836,"åģ°":122837,"åģ¡":122838,"鸺":122839,"åģŃ":122840,"åģ²":122841,"åģģ":122842,"ã¿":122843,"ã¿ł":122844,"éĦħ":122845,"åģĵ":122846,"å¾Ľ":122847,"è¡Ĵ":122848,"èγ":122849,"èβ":122850,"鸼":122851,"æĤĨ":122852,"éĦĥ":122853,"çĵ»":122854,"äĿ":122855,"äĿĻ":122856,"èĦ¶":122857,"èĦŀ":122858,"èĦŁ":122859,"äı²":122860,"é±¾":122861,"çĮĩ":122862,"çĮĬ":122863,"çĮĦ":122864,"è§ĸ":122865,"ðłħ":122866,"ðłħ¤":122867,"庱":122868,"庼":122869,"庳":122870,"çĹĵ":122871,"ä´Ķ":122872,"ç««":122873,"åłĥ":122874,"éĺĮ":122875,"ç¾Ŀ":122876,"ç¾ķ":122877,"çĦĨ":122878,"çĥº":122879,"çĦĮ":122880,"æ·ı":122881,"ð¬ĩ¹":122882,"æ·Ł":122883,"æ·ľ":122884,"æ·´":122885,"æ·¯":122886,"æ¹´":122887,"æ¶´":122888,"ð¬į¡":122889,"ã¥":122890,"ã¥Ħ":122891,"æĥĽ":122892,"æĥĶ":122893,"æĤ°":122894,"æĥĻ":122895,"å¯ģ":122896,"éĢŃ":122897,"ð¬¤ĩ":122898,"ð«į¯":122899,"袼":122900,"è£Ī":122901,"祲":122902,"ð¬¤Ĭ":122903,"ð«į²":122904,"è°ŀ":122905,"èī´":122906,"弸":122907,"å¼¶":122908,"ð¬¯İ":122909,"éļĥ":122910,"å©ŀ":122911,"娵":122912,"婼":122913,"åªĸ":122914,"婳":122915,"å©į":122916,"å©Į":122917,"å©«":122918,"婤":122919,"å©ĺ":122920,"å©ł":122921,"ð¬ĺ¬":122922,"ð¬ĺŃ":122923,"ð¬´Ĥ":122924,"ð«ĺ¦":122925,"绹":122926,"ð«Łħ":122927,"ð¬ĺ¯":122928,"éªķ":122929,"ð«ĺ§":122930,"絾":122931,"çı·":122932,"çIJ²":122933,"çIJ¡":122934,"çIJŁ":122935,"çIJĶ":122936,"çIJŃ":122937,"åł¾":122938,"åł¼":122939,"æıķ":122940,"ãĻĺ":122941,"åł§":122942,"åĸĨ":122943,"åł¨":122944,"å¡ħ":122945,"åłł":122946,"çµ·":122947,"ðª£":122948,"𪣻":122949,"ð¡İ":122950,"ð¡İļ":122951,"èijľ":122952,"æĥİ":122953,"èIJ³":122954,"èijĻ":122955,"éĿ¬":122956,"èij´":122957,"èĴĩ":122958,"èĴĪ":122959,"éĦļ":122960,"èĴī":122961,"èĵĩ":122962,"èIJ©":122963,"èij°":122964,"èijİ":122965,"éĦij":122966,"èĴİ":122967,"èijĸ":122968,"èĴĦ":122969,"èIJ¹":122970,"棤":122971,"棽":122972,"棫":122973,"æ¤ĵ":122974,"æ¤ij":122975,"ð¬ĥ":122976,"ð¬ĥĬ":122977,"é¹Ģ":122978,"æ¤Ĩ":122979,"æ£ĵ":122980,"棬":122981,"棪":122982,"æ¤Ģ":122983,"æ¥Ĺ":122984,"ð¬·":122985,"ð¬·ķ":122986,"çͦ":122987,"éħ¦":122988,"è§Į":122989,"奡":122990,"çļķ":122991,"硪":122992,"欹":122993,"è©Ł":122994,"ð«IJIJ":122995,"è¾Į":122996,"æ£IJ":122997,"é¾Ĥ":122998,"ð¬¹":122999,"𬹼":123000,"黹":123001,"çīļ":123002,"çĿİ":123003,"æĻ«":123004,"æĻª":123005,"æĻ±":123006,"ð§":123007,"ð§¿":123008,"ð§¿¹":123009,"èĽij":123010,"çķ¯":123011,"æĸĿ":123012,"åĸ¤":123013,"å´¶":123014,"åµģ":123015,"ð«¶":123016,"ð«¶ĩ":123017,"å´¾":123018,"åµħ":123019,"å´¿":123020,"åµļ":123021,"ç¿Ļ":123022,"ð«ĸ®":123023,"åľĮ":123024,"åľIJ":123025,"èµij":123026,"èµĴ":123027,"é¿ı":123028,"éĵ¹":123029,"ð¬ŃĬ":123030,"éĵ½":123031,"ð¨±ĩ":123032,"ð«ĵ¶":123033,"éĶĬ":123034,"éĶį":123035,"éĶİ":123036,"ð¬Ńİ":123037,"éĶĵ":123038,"çĬĩ":123039,"é¢ĭ":123040,"ç¨Į":123041,"çŃĢ":123042,"çŃĺ":123043,"çŃľ":123044,"çŃ¥":123045,"çŃħ":123046,"åĤĥ":123047,"åĤī":123048,"ç¿Ľ":123049,"åĤĴ":123050,"åĤķ":123051,"èξ":123052,"çķ¬":123053,"ð«ĸ¯":123054,"èĦ¿":123055,"èħĺ":123056,"äIJ":123057,"äIJĥ":123058,"èħĻ":123059,"èħĴ":123060,"ð¬±Ł":123061,"é²ĥ":123062,"çĮ°":123063,"ð«Ľ":123064,"ð«ĽŃ":123065,"çĮ¯":123066,"ãº":123067,"ãºĦ":123068,"é¦ī":123069,"åĩĵ":123070,"éĦĹ":123071,"ð«·":123072,"ð«··":123073,"å»ĭ":123074,"å»Ĩ":123075,"éĦĮ":123076,"ç²¢":123077,"éģĨ":123078,"æĹIJ":123079,"𬮱":123080,"çĦŀ":123081,"ð¬Ĭ¤":123082,"欻":123083,"ð£¸":123084,"𣸣":123085,"æºļ":123086,"æºģ":123087,"æ¹Ŀ":123088,"渰":123089,"æ¹ĵ":123090,"ã´":123091,"ã´Ķ":123092,"æ¸Ł":123093,"æºł":123094,"渼":123095,"æºĩ":123096,"æ¹£":123097,"æ¹ij":123098,"æºŀ":123099,"æĦIJ":123100,"æĦĥ":123101,"æķ©":123102,"ç͝":123103,"棨":123104,"æīĬ":123105,"裣":123106,"祼":123107,"å©»":123108,"åªĨ":123109,"åªŀ":123110,"ãĽ¹":123111,"åªĵ":123112,"åªĤ":123113,"åªĦ":123114,"毵":123115,"çŁŀ":123116,"ð¬´ĥ":123117,"ð«ĺ¨":123118,"ç¼Ĭ":123119,"ç¼IJ":123120,"éªĻ":123121,"çijĥ":123122,"çijĵ":123123,"çijħ":123124,"çijĨ":123125,"ä´ĸ":123126,"çijĸ":123127,"çijĿ":123128,"çijĶ":123129,"çijĢ":123130,"ð¤§":123131,"ð¤§Ľ":123132,"çij³":123133,"çijĤ":123134,"å¶ħ":123135,"çijij":123136,"éģĺ":123137,"é«¢":123138,"å¡¥":123139,"åł½":123140,"赪":123141,"æijĽ":123142,"å¡Ŀ":123143,"æIJĴ":123144,"æIJĮ":123145,"èĴ±":123146,"èĴ¨":123147,"èĵı":123148,"èĶĢ":123149,"èĵ¢":123150,"èĵĤ":123151,"èĴ»":123152,"èĵ£":123153,"椹":123154,"楪":123155,"æ¦ĥ":123156,"æ¦ħ":123157,"æ¥Ĵ":123158,"楩":123159,"æ¦ĩ":123160,"椸":123161,"æ¥Ļ":123162,"æŃħ":123163,"ð¬ª":123164,"𬪩":123165,"ç¢ĥ":123166,"ç¢ı":123167,"ð¬ĴĶ":123168,"ç¢Ī":123169,"äĥħ":123170,"ç¡¿":123171,"éĦł":123172,"è¾Ĵ":123173,"ð¬¨İ":123174,"ð«IJĵ":123175,"é¾Ĩ":123176,"è§ľ":123177,"ä£":123178,"ä£ĺ":123179,"æļķ":123180,"é¹į":123181,"ð««":123182,"ð««ĩ":123183,"ã¬Ĭ":123184,"æļħ":123185,"è·±":123186,"èľIJ":123187,"èľİ":123188,"åµ²":123189,"èµĹ":123190,"骱":123191,"éĶĸ":123192,"ð«ĵ¹":123193,"éĶĺ":123194,"éͳ":123195,"éͧ":123196,"éĶª":123197,"ð¬Ńļ":123198,"éĶ«":123199,"éͬ":123200,"ð¬ŃĽ":123201,"ç¨ij":123202,"ç¨Ļ":123203,"äħ":123204,"äħŁ":123205,"ð¬ķ":123206,"ð¬ķĤ":123207,"çŃ»":123208,"çѼ":123209,"çѶ":123210,"çѦ":123211,"çѤ":123212,"åĤº":123213,"é¹İ":123214,"åĥĩ":123215,"èīħ":123216,"èīī":123217,"è°¼":123218,"è²Ĩ":123219,"èħ½":123220,"èħ¨":123221,"èħ¯":123222,"é²ī":123223,"é²Ĭ":123224,"é²Į":123225,"ä²Ł":123226,"ð¬¶ĭ":123227,"ð¬¶į":123228,"é²ı":123229,"éĽĬ":123230,"çĮº":123231,"é£Ķ":123232,"è§Ł":123233,"ð¦Ŀ¼":123234,"é¦Į":123235,"è£Ľ":123236,"å»Ĵ":123237,"çĺħ":123238,"éĦĺ":123239,"é¹Ĵ":123240,"éĦľ":123241,"éºĢ":123242,"éĦ£":123243,"éĺĺ":123244,"ð«Ķ¶":123245,"çħģ":123246,"çħĥ":123247,"çħ´":123248,"çħĭ":123249,"çħŁ":123250,"çħĵ":123251,"æ»ł":123252,"æºį":123253,"溹":123254,"æ»Ĩ":123255,"æ»ī":123256,"溦":123257,"溵":123258,"æ¼·":123259,"æ»§":123260,"æ»ĺ":123261,"æ»į":123262,"æĦŃ":123263,"æħ¥":123264,"æħĨ":123265,"塱":123266,"ð«ĮĢ":123267,"裼":123268,"ç¦ĭ":123269,"ç¦Ķ":123270,"ç¦ĺ":123271,"ç¦Ĵ":123272,"è°«":123273,"é¹Ķ":123274,"ð«ĸ³":123275,"æĦį":123276,"å«Ħ":123277,"媱":123278,"æĪ¤":123279,"åĭł":123280,"æĪ£":123281,"ð«ĺª":123282,"ð«ĺ¬":123283,"ç¼ŀ":123284,"è̤":123285,"çij§":123286,"ð«ŀ":123287,"ð«ŀ©":123288,"çij¨":123289,"çij±":123290,"çij·":123291,"çij¢":123292,"æĸł":123293,"æijı":123294,"å¢ķ":123295,"å¢Ī":123296,"å¢IJ":123297,"å¢ĺ":123298,"æij´":123299,"éĬİ":123300,"ð¡IJ":123301,"ð¡IJĵ":123302,"å¢ļ":123303,"æĴĸ":123304,"ðª¤":123305,"ðª¤Ĺ":123306,"éĿ½":123307,"éŀģ":123308,"èĶĮ":123309,"èĶĪ":123310,"èĵ°":123311,"è͹":123312,"èĶĬ":123313,"åĺı":123314,"榰":123315,"æ¦ij":123316,"æ§ļ":123317,"ð£Ĺ":123318,"ð£Ĺĭ":123319,"æ§ľ":123320,"æ¦į":123321,"çĸIJ":123322,"ð¬¸ĺ":123323,"éħº":123324,"éħ¾":123325,"éħ²":123326,"éħ´":123327,"碶":123328,"äĥİ":123329,"ð¬ĴĹ":123330,"碨":123331,"ð¥Ķ":123332,"ð¥Ķ²":123333,"碹":123334,"碥":123335,"åĬĤ":123336,"ð«ļĸ":123337,"ä´Ĺ":123338,"夥":123339,"çŀį":123340,"é¹ĸ":123341,"ã¬İ":123342,"è·½":123343,"èľ¾":123344,"å¹ĸ":123345,"å¶į":123346,"åľĻ":123347,"ð¨±ı":123348,"éĶº":123349,"éͼ":123350,"éͽ":123351,"ð¬Ń¤":123352,"é;":123353,"éĶ¿":123354,"éķĥ":123355,"éķĦ":123356,"éķħ":123357,"é¦Ŀ":123358,"é¹Ļ":123359,"箨":123360,"ç®ĸ":123361,"åĬĦ":123362,"åĥ¬":123363,"åĥ¦":123364,"åĥĶ":123365,"åĥİ":123366,"æ§ĥ":123367,"ãϦ":123368,"é²Ĵ":123369,"é²ķ":123370,"ð«ļķ":123371,"é²ĸ":123372,"é²Ĺ":123373,"é²ĺ":123374,"é²Ļ":123375,"ð¬¶IJ":123376,"ð¬¶ı":123377,"ð©½":123378,"𩽾":123379,"å¤IJ":123380,"çįį":123381,"é£Ĺ":123382,"ð¬¸ļ":123383,"åĩĺ":123384,"å»ij":123385,"å»Ļ":123386,"çĺĹ":123387,"çĺ¥":123388,"çĺķ":123389,"é²Ŀ":123390,"éĦ«":123391,"çĨĩ":123392,"æ¼¹":123393,"æ¼ĸ":123394,"æ½Ĩ":123395,"漤":123396,"潩":123397,"æ¼¼":123398,"æ¼´":123399,"ã½":123400,"ã½ı":123401,"æ¼Ī":123402,"æ¼ĭ":123403,"æ¼»":123404,"æħ¬":123405,"窬":123406,"çªŃ":123407,"ã®":123408,"㮾":123409,"ð¬¤Ŀ":123410,"è¤ķ":123411,"禼":123412,"ç¦ļ":123413,"éļ©":123414,"å«ķ":123415,"å«Ń":123416,"å«ľ":123417,"嫪":123418,"ð¬ĻĤ":123419,"ã»":123420,"㻬":123421,"麹":123422,"çĴĨ":123423,"漦":123424,"åıĩ":123425,"墣":123426,"墦":123427,"墡":123428,"åĬIJ":123429,"èĸģ":123430,"èķ°":123431,"èĶĥ":123432,"é¼Ĵ":123433,"æ§±":123434,"é¹Ŀ":123435,"ç£ı":123436,"ç£ī":123437,"殣":123438,"æħŃ":123439,"éľħ":123440,"æļµ":123441,"æļ²":123442,"æļ¶":123443,"踦":123444,"踣":123445,"äĹĸ":123446,"èĿĺ":123447,"èĿ²":123448,"èĿ¤":123449,"åĻĩ":123450,"åĻĤ":123451,"åĻĢ":123452,"ç½¶":123453,"å¶²":123454,"å¶ĵ":123455,"ãłĩ":123456,"å¶Ł":123457,"å¶Ĵ":123458,"éķĨ":123459,"éķĪ":123460,"éķĭ":123461,"éķİ":123462,"ð¬Ń©":123463,"éķķ":123464,"稹":123465,"åĦĩ":123466,"çļŀ":123467,"çļĽ":123468,"ä´ĺ":123469,"èīİ":123470,"èīı":123471,"é¹Ł":123472,"ð©¾ĥ":123473,"鲦":123474,"鲪":123475,"鲬":123476,"æ©¥":123477,"è§Ń":123478,"é¹ł":123479,"鹡":123480,"ç³ĩ":123481,"ç³Ī":123482,"翦":123483,"é¹¢":123484,"é¹£":123485,"çĨĽ":123486,"æ½ĸ":123487,"æ½µ":123488,"ãµ":123489,"ãµIJ":123490,"æ¾Ĥ":123491,"æ¾Ľ":123492,"çij¬":123493,"æ½½":123494,"æ½¾":123495,"æ½ı":123496,"æĨŃ":123497,"æĨķ":123498,"𬸣":123499,"æĪŃ":123500,"褯":123501,"禤":123502,"ð«į½":123503,"嫽":123504,"éģ¹":123505,"ð¬´Ĭ":123506,"çĴ¥":123507,"çĴ²":123508,"çĴĴ":123509,"æĨĻ":123510,"æĵIJ":123511,"éĦ¹":123512,"èĸ³":123513,"éŀĶ":123514,"é»ĩ":123515,"ð¬ŀ":123516,"ð¬ŀŁ":123517,"èķĹ":123518,"èĸ¢":123519,"èķ¹":123520,"æ©ŀ":123521,"æ©ij":123522,"橦":123523,"éĨij":123524,"è§±":123525,"磡":123526,"ð¥ķ":123527,"ð¥ķ¢":123528,"ç£ľ":123529,"è±®":123530,"ð«Ł¦":123531,"ð¬ºĪ":123532,"ð«łľ":123533,"é¹¾":123534,"èϤ":123535,"æļ¿":123536,"æĽĮ":123537,"æĽĪ":123538,"ã¬ļ":123539,"è¹ħ":123540,"踶":123541,"äĹĽ":123542,"èŀĹ":123543,"çĸģ":123544,"ãłĵ":123545,"幪":123546,"ðª©":123547,"ðª©ĺ":123548,"嶦":123549,"ð¬Ń¬":123550,"ð¨±ij":123551,"ð¬Ń¯":123552,"é¦ŀ":123553,"ç©Ħ":123554,"ç¯ļ":123555,"篯":123556,"ç°ī":123557,"é¼½":123558,"è¡ł":123559,"缦":123560,"èŀ£":123561,"縢":123562,"é²Ń":123563,"鲯":123564,"é²°":123565,"鲺":123566,"é²¹":123567,"ð«Ĺ´":123568,"亸":123569,"çĻĢ":123570,"çĺŃ":123571,"𬸦":123572,"ç¾±":123573,"ç³Ĵ":123574,"çĩĭ":123575,"çĨ»":123576,"çĩĬ":123577,"çĩļ":123578,"çĩı":123579,"æ¿©":123580,"æ¿ĭ":123581,"澪":123582,"æ¾½":123583,"æ¾´":123584,"æ¾Ń":123585,"æ¾¼":123586,"æĨ·":123587,"æĨº":123588,"æĩĶ":123589,"é»ī":123590,"å¬Ľ":123591,"鹨":123592,"翯":123593,"ð«Ħ·":123594,"çĴ±":123595,"𤩽":123596,"çĴ¬":123597,"çĴ®":123598,"髽":123599,"æĵ¿":123600,"èĸ¿":123601,"èĸ¸":123602,"æªij":123603,"æ«Ĩ":123604,"æªŀ":123605,"éĨ¨":123606,"ç¹Ħ":123607,"磹":123608,"磻":123609,"çŀ«":123610,"çŀµ":123611,"è¹IJ":123612,"èŁı":123613,"ãĺ":123614,"ãĺİ":123615,"ð¬Ń³":123616,"éķ¤":123617,"ð¬Ń¶":123618,"ð«Ķį":123619,"éķ¥":123620,"éķ¨":123621,"ð¬Ń¸":123622,"ð¨±Ķ":123623,"ð¬Ń¼":123624,"ð«Ķİ":123625,"磰":123626,"ç©Ļ":123627,"穾":123628,"穣":123629,"ç°ķ":123630,"ç°ĥ":123631,"ç°ı":123632,"åĦ¦":123633,"éŃĭ":123634,"æĸ¶":123635,"èīļ":123636,"𬸪":123637,"è°¿":123638,"ä²ł":123639,"ð¬¶Ł":123640,"é²¾":123641,"ð¬¶ł":123642,"鲿":123643,"é³ģ":123644,"é³Ĥ":123645,"é³Ī":123646,"é³ī":123647,"çį¯":123648,"äĹª":123649,"é¦ĺ":123650,"è¥ķ":123651,"è¥ļ":123652,"𬶨":123653,"èŀ±":123654,"çĶĵ":123655,"嬬":123656,"嬥":123657,"ð¦Ī":123658,"ð¦Ī¡":123659,"ð«Ħ¸":123660,"çĵĢ":123661,"éĩIJ":123662,"鬶":123663,"çĪĩ":123664,"éŀ³":123665,"éŀ®":123666,"ð¬Łģ":123667,"èĹŁ":123668,"èŦ":123669,"èŨ":123670,"é¹²":123671,"檫":123672,"黡":123673,"ç¤ŀ":123674,"ç¤Į":123675,"ð¥ĸ":123676,"ð¥ĸ¨":123677,"è¹¢":123678,"è¹ľ":123679,"èŁ«":123680,"äĹ´":123681,"åļļ":123682,"é«ĥ":123683,"éķ®":123684,"éķ±":123685,"éħĤ":123686,"馧":123687,"ç°ł":123688,"ç°Ŀ":123689,"ç°°":123690,"鼫":123691,"鼩":123692,"çļ¦":123693,"èĩij":123694,"ä²¢":123695,"é³ij":123696,"é³Ĵ":123697,"é¹±":123698,"鹯":123699,"çĻĹ":123700,"ð¦Ĵ":123701,"ð¦Ĵį":123702,"æĹŀ":123703,"ç¿·":123704,"åĨģ":123705,"äİĸ":123706,"çĢĶ":123707,"çĢį":123708,"çĢĮ":123709,"è¥ľ":123710,"ä´Ļ":123711,"ð¬ĻĬ":123712,"åļŃ":123713,"ã°":123714,"ã°Ģ":123715,"鬷":123716,"éĨŃ":123717,"蹯":123718,"èłĭ":123719,"翾":123720,"é³ĺ":123721,"åĦ³":123722,"åĦ´":123723,"é¼Ĺ":123724,"ð¬¶Ń":123725,"ð©¾Į":123726,"é³ļ":123727,"é³Ľ":123728,"éºij":123729,"éºĸ":123730,"èłĥ":123731,"å½Ł":123732,"嬿":123733,"é¬Ĵ":123734,"èĺĺ":123735,"æ¬Ĥ":123736,"éĨµ":123737,"颥":123738,"çĶĹ":123739,"ð¨Ł":123740,"ð¨Łł":123741,"å·ĩ":123742,"éħħ":123743,"é«İ":123744,"çĬ¨":123745,"𬶮":123746,"ð¨Ń":123747,"ð¨Ńī":123748,"ã¸Į":123749,"çĪĶ":123750,"ç̱":123751,"ç̹":123752,"ç̼":123753,"ç̵":123754,"襫":123755,"åŃħ":123756,"骦":123757,"ð¬Ļĭ":123758,"ḛ̀":123759,"ð¤«":123760,"ð¤«ī":123761,"çĵĸ":123762,"é¬ĺ":123763,"趯":123764,"ð¬ºĵ":123765,"ç½į":123766,"é¼±":123767,"é³ł":123768,"鳡":123769,"é³£":123770,"çĪŁ":123771,"çĪļ":123772,"çģĪ":123773,"éŁĤ":123774,"ç³µ":123775,"èĺ¼":123776,"礵":123777,"é¹´":123778,"èºĶ":123779,"çļŃ":123780,"é¾¢":123781,"鳤":123782,"亹":123783,"ç±¥":123784,"é¼·":123785,"ð«ļŃ":123786,"çİĥ":123787,"éĨ¾":123788,"é½ĩ":123789,"è§¿":123790,"èł¼":123791,"×§":123792,"פ":123793,"׼":123794,"×ķת":123795,"ס":123796,"×Ļ×Ŀ":123797,"צ":123798,"×Ĵ":123799,"×ĺ":123800,"×ķר":123801,"×Ŀ":123802,"×ķ׾":123803,"×ĸ":123804,"à¹Ĥ":123805,"ïº":123806,"ðŁį":123807,"ðŁIJ":123808,"×Ļר":123809,"ï»":123810,"ðŁij":123811,"ðĿIJ":123812,"ðŁı":123813,"ðŁĶ":123814,"ðŁĮ":123815,"ðŁİ":123816,"ðŁĵ":123817,"ף":123818,"ðĿij":123819,"×ķ×ĵ":123820,"ï¦":123821,"Ġ×ķ":123822,"×ķ×ij":123823,"à¸Ńà¸ĩ":123824,"ðĿĺ":123825,"×Ļת":123826,"ðĿķ":123827,"à¸Ĺีà¹Ī":123828,"ائ":123829,"ð٤":123830,"×ķף":123831,"رÙĬ":123832,"×Ļ׾":123833,"ระ":123834,"าย":123835,"ï¯":123836,"ï®":123837,"าม":123838,"âĩ":123839,"ðŁ¥":123840,"ïŃ":123841,"ðĿĻ":123842,"×ķ׳":123843,"á½":123844,"Ġ׼":123845,"ðŁļ":123846,"âļ":123847,"ï§":123848,"×ijר":123849,"×Ļ׳":123850,"á´":123851,"Ġ×Ĺ":123852,"á¼":123853,"ðĿĹ":123854,"Ġ×¢":123855,"×Ļ×Ķ":123856,"ãģ£ãģŁ":123857,"ãģĵãģ¨":123858,"á¸":123859,"ÙĬÙĨ":123860,"ãģªãģĦ":123861,"اع":123862,"ศ":123863,"à¹Īà¸ĩ":123864,"×Ļ×ĵ":123865,"×ŀש":123866,"áĪ":123867,"׳×Ļ":123868,"×Ļ×ij":123869,"ï¥":123870,"ðĿĵ":123871,"Ġ×Ļ":123872,"×ļ":123873,"ัà¸ĩ":123874,"âĵ":123875,"ï¤":123876,"ĠاÙĦØ£":123877,"าà¸ģ":123878,"à¹īà¸Ļ":123879,"à¹Ģร":123880,"×ķ×Ŀ":123881,"á¹":123882,"ึ":123883,"×Ļ×§":123884,"à¸ĭ":123885,"à¸Ħร":123886,"à¸ĺ":123887,"ัà¸ģ":123888,"ðŁķ":123889,"ÙĪÙĨ":123890,"à¸Ńย":123891,"âĬ":123892,"ðĿĴ":123893,"ĠاÙĦع":123894,"าà¸Ļ":123895,"×Ļף":123896,"ÙĦÙĬ":123897,"×Ļש":123898,"à¸Ľà¸£à¸°":123899,"à¹Ģà¸Ľ":123900,"Ġ׳":123901,"×ķס":123902,"à¸ł":123903,"ÙħÙĨ":123904,"×ķ×¢":123905,"×ķ×ŀ":123906,"âĮ":123907,"ð٧":123908,"à¹ĩà¸Ļ":123909,"à¸į":123910,"ãİ":123911,"áµ":123912,"ĠاÙĦس":123913,"×ķ×§":123914,"หล":123915,"ðŁĩ":123916,"âı":123917,"ð٦":123918,"Ġ×Ķ×ŀ":123919,"ÙĪØ§":123920,"Ġת":123921,"ר×IJ":123922,"à¸Ńà¸Ļ":123923,"ษ":123924,"à¹Īว":123925,"×ķצ":123926,"íĹ":123927,"ãĦ":123928,"ï¨":123929,"ï¹":123930,"âİ":123931,"ï²":123932,"ðĿļ":123933,"ðIJ":123934,"à¸Ħว":123935,"หà¸Ļ":123936,"Ġר":123937,"بÙĬ":123938,"รà¹Į":123939,"را":123940,"شر":123941,"×ķ×Ĺ":123942,"×ķפ":123943,"×ķש":123944,"×ķ×Ĵ":123945,"íĿ":123946,"âĽ":123947,"à¸ķิ":123948,"à¹Ģà¸ģ":123949,"ï³":123950,"ï±":123951,"à¸Ķà¹ī":123952,"ë¹":123953,"ï¬":123954,"á¿":123955,"ðŁĽ":123956,"ðĿĸ":123957,"à¹Īาà¸ĩ":123958,"ูà¹ī":123959,"Ġ×Ķ×IJ":123960,"ĠاÙĦØŃ":123961,"פר":123962,"ÙĪÙħ":123963,"à¹Ģล":123964,"íĸ":123965,"×Ļ×¢":123966,"ìĪ":123967,"íĵ":123968,"ðŁħ":123969,"áł":123970,"à¸Ħวาม":123971,"à¸Īะ":123972,"׳×Ķ":123973,"Ġ×§":123974,"à¸Ł":123975,"à¹īà¸ĩ":123976,"หม":123977,"تÙħ":123978,"׾×Ļ":123979,"ÙĬد":123980,"à¹Īà¸Ļ":123981,"×Ĺר":123982,"שר":123983,"à¹Ģà¸Ĺ":123984,"×ŀר":123985,"ëĸ":123986,"عÙĦ":123987,"×ŀ×¢":123988,"â²":123989,"׾×Ķ":123990,"Ġפ":123991,"à¸Ńà¸ģ":123992,"سÙĦ":123993,"×Ļ×ŀ":123994,"ÙĤÙĬ":123995,"íİ":123996,"تØŃ":123997,"×Ļס":123998,"×Ļ×Ĺ":123999,"íĽ":124000,"ï°":124001,"â½":124002,"áī":124003,"áĬ":124004,"á¨":124005,"Ùĩا":124006,"Ġ׾×Ķ":124007,"×ķ×IJ":124008,"Ùħا":124009,"à¹īà¸Ńà¸ĩ":124010,"رب":124011,"ĠاÙĦج":124012,"×ŀ×ĵ":124013,"ÙħÙĦ":124014,"تر":124015,"à¹Ģà¸Ķ":124016,"קר":124017,"íħ":124018,"ì¼":124019,"ê¿":124020,"ãĪ":124021,"áIJ":124022,"ðŁĹ":124023,"ê¦":124024,"áĭ":124025,"ðĿĶ":124026,"à¹Ģà¸Ľà¹ĩà¸Ļ":124027,"à¹ĥห":124028,"มา":124029,"วà¹Īา":124030,"มี":124031,"ีà¹ī":124032,"à¹Ħมà¹Ī":124033,"ÙĨÙĬ":124034,"ؤ":124035,"รา":124036,"×ķ×Ļ":124037,"ãĤĪãģĨ":124038,"ิà¸Ķ":124039,"×Ļפ":124040,"×Ĺ׾":124041,"ÙĤد":124042,"à¹Ģส":124043,"×Ļ×ĺ":124044,"à¸ģล":124045,"ר׼":124046,"×ķ׼":124047,"×Ļ׼":124048,"ëĪ":124049,"ëĥ":124050,"ðŁĸ":124051,"áħ":124052,"â¼":124053,"ãī":124054,"à¹Ħà¸Ķà¹ī":124055,"ת×Ļ":124056,"×Ļ×IJ":124057,"ĠاÙĦØ¥":124058,"à¸łà¸²":124059,"ริ":124060,"ÙĤØ©":124061,"ØŃد":124062,"ê»":124063,"ì±":124064,"ת×Ĺ":124065,"ìº":124066,"âĭ":124067,"áĦ":124068,"á¾":124069,"âµ":124070,"â¾":124071,"ĠÙĪØ§ÙĦ":124072,"׳×ķ":124073,"ÙĢ":124074,"ÙĬا":124075,"à¸ģà¹ĩ":124076,"×ŀ×Ķ":124077,"ãģĦãĤĭ":124078,"عد":124079,"ĠاÙĦÙĨ":124080,"Ġ×Ķש":124081,"ئ":124082,"ัà¹īà¸ĩ":124083,"รัà¸ļ":124084,"ÙĪÙĤ":124085,"ãģ§ãģį":124086,"à¹Ģà¸ŀ":124087,"׼׾":124088,"×ĺר":124089,"ัà¸Ķ":124090,"à¸Ńา":124091,"ì¢":124092,"à¸Ńà¸ļ":124093,"à¸ķร":124094,"à¹Ģà¸Ĭ":124095,"ìĶ":124096,"ãģĹãģ¾":124097,"ëģ":124098,"ëķ":124099,"ðŁĻ":124100,"âĴ":124101,"á¶":124102,"à¹ģล":124103,"ÙĨا":124104,"à¹ĥหà¹ī":124105,"à¹Ħà¸Ľ":124106,"×£":124107,"ัว":124108,"าà¸ĩ":124109,"×ĵר":124110,"×ij׾":124111,"פ×Ļ":124112,"Ġ×ĵ":124113,"ĠاÙĦÙģ":124114,"à¹Ģà¸Ĥ":124115,"ש×Ķ":124116,"×IJר":124117,"ë¬":124118,"ãģ«ãģª":124119,"ÑĢо":124120,"วิ":124121,"Ùħر":124122,"×IJת":124123,"Ùĥر":124124,"سب":124125,"ÙĨت":124126,"ãģĹãģĦ":124127,"اج":124128,"à¸Ńรà¹Į":124129,"ÙĥÙĦ":124130,"سÙħ":124131,"สิ":124132,"×Ļצ":124133,"ëĿ":124134,"íľ":124135,"ìī":124136,"áĨ":124137,"ÙĩÙħ":124138,"à¸Ļีà¹ī":124139,"ãģĤãĤĭ":124140,"ãģĦãģ¦":124141,"سÙĬ":124142,"׾×IJ":124143,"در":124144,"ãģļ":124145,"ÙĪØ¬":124146,"ĠاÙĦØ®":124147,"صر":124148,"íı":124149,"à¹īาà¸ĩ":124150,"ุà¸Ķ":124151,"×ķ×ĺ":124152,"×ij×¢":124153,"íĨ":124154,"à¸Ĭา":124155,"รม":124156,"ש×ŀ":124157,"×ŀס":124158,"ê´":124159,"ì´":124160,"ëľ":124161,"ì¿":124162,"ì©":124163,"ë»":124164,"â¤":124165,"ðŁĨ":124166,"áĮ":124167,"áķ":124168,"ذا":124169,"à¸Ĺำ":124170,"à¸ķà¹Ī":124171,"ĠاÙĦÙĤ":124172,"ÙĦÙĥ":124173,"ูà¹Ī":124174,"à¸Ħุ":124175,"ÙĬÙħ":124176,"׳×Ļ×Ŀ":124177,"ืà¹Īà¸Ń":124178,"ÙĪØ¹":124179,"ãĤĩ":124180,"اÙĤ":124181,"Ġ×ij×¢":124182,"à¹Ģม":124183,"جÙħ":124184,"ừ":124185,"ãģĵãģ¨ãģĮ":124186,"بد":124187,"×ķ×Ķ":124188,"ש׾":124189,"Ùĩر":124190,"à¹Ģà¸Ļ":124191,"ãģ¹":124192,"íĭ":124193,"ì»":124194,"ì½":124195,"ëŃ":124196,"ìĮ":124197,"íĢ":124198,"ëĮ":124199,"ëº":124200,"ãĬ":124201,"à¹ĥà¸Ļ":124202,"Ġ×Ĵ":124203,"à¹Ĩ":124204,"à¸Īาà¸ģ":124205,"วย":124206,"à¹ĥà¸Ĭ":124207,"à¸ĩาà¸Ļ":124208,"ĠاÙĦØ´":124209,"اØŃ":124210,"à¹īาà¸Ļ":124211,"ืà¹Īà¸Ńà¸ĩ":124212,"×IJ×Ļ":124213,"بÙĦ":124214,"ã썿ĢĿ":124215,"×ł×¡":124216,"ãģ¾ãģĽ":124217,"ÙĥÙĨ":124218,"ער":124219,"ĠاÙĦد":124220,"שת":124221,"íŀ":124222,"Ùħس":124223,"صÙĦ":124224,"×ķ׳×Ķ":124225,"ارة":124226,"ÙĦÙħ":124227,"สม":124228,"Ø£ÙĨ":124229,"תר":124230,"×IJ×ŀ":124231,"عب":124232,"خت":124233,"ãĤĥ":124234,"ì¡":124235,"ì£":124236,"ива":124237,"สั":124238,"ึà¸ģ":124239,"ì¸":124240,"ëĨ":124241,"алÑĮн":124242,"ì³":124243,"ìį":124244,"ê¼":124245,"ê½":124246,"ìı":124247,"ãĮ":124248,"ãı":124249,"ï©":124250,"êª":124251,"áİ":124252,"Ġ×ĸ":124253,"à¸ģัà¸Ļ":124254,"×Ļ×ķ":124255,"à¸Ħà¸Ļ":124256,"׳×ķת":124257,"à¸ľà¸¹à¹ī":124258,"à¹ĥà¸Ī":124259,"ãģĦãģŁ":124260,"Ù쨱":124261,"×ĺ×Ļ":124262,"צ×Ļ":124263,"ãĤĤãģ®":124264,"ĠاÙĦص":124265,"ãģ¾ãģĽãĤĵ":124266,"دة":124267,"×ij×Ļ":124268,"ĠاÙĦر":124269,"Ġ×ŀ×IJ":124270,"สำ":124271,"à¹Ģห":124272,"عر":124273,"ãģªãģı":124274,"à¸ģระ":124275,"×ij×ĵ":124276,"à¹Ģà¸Ī":124277,"×Ļ×ļ":124278,"×Ĺ×Ļ":124279,"ÙĬع":124280,"ש×ij":124281,"ÙĨØ©":124282,"ÙĪØ¶":124283,"ÙĦÙģ":124284,"ÙĢÙĢ":124285,"פע":124286,"íĪ":124287,"×ŀ×§":124288,"à¸IJ":124289,"ØŃØ©":124290,"اص":124291,"Ñĭва":124292,"à¸Ħม":124293,"วั":124294,"à¸Ľà¸¥":124295,"ìŁ":124296,"íļ":124297,"ë´":124298,"ëij":124299,"ëī":124300,"ëĩ":124301,"ì¨":124302,"ë±":124303,"ëİ":124304,"â¬":124305,"á¥":124306,"áĹ":124307,"áĽ":124308,"áį":124309,"Å©":124310,"à¸Ķี":124311,"ôi":124312,"Ġס":124313,"׾×ķ":124314,"á»Ŀi":124315,"à¸Ħุà¸ĵ":124316,"ây":124317,"à¸Ļา":124318,"×Ĺ×ĵ":124319,"×ĵ×Ļ":124320,"หา":124321,"جÙĦ":124322,"à¹Ģว":124323,"ãĤĩãģĨ":124324,"ÙħØ©":124325,"ĠاÙĦÙĥ":124326,"Ġ×Ķ×¢":124327,"جر":124328,"×ĸר":124329,"اط":124330,"×Ľ×ª":124331,"×ķ׳×Ļ×Ŀ":124332,"ØŃÙħ":124333,"ê¶":124334,"رÙĥ":124335,"Ġ×ľ×¢":124336,"×ķ×ĸ":124337,"สร":124338,"צ׾":124339,"Ø¢":124340,"است":124341,"à¹Īม":124342,"خر":124343,"צע":124344,"×Ļר×ķת":124345,"ادة":124346,"شار":124347,"×ŀ×Ĺ":124348,"íĴ":124349,"à¹Ģรีย":124350,"×Ĺ×§":124351,"اث":124352,"รà¸ĩ":124353,"à¹Ģà¸ķ":124354,"à¸Īำ":124355,"à¸Ŀ":124356,"à¹Īาย":124357,"à¸Ħล":124358,"ÙĤÙĪ":124359,"иÑĩеÑģк":124360,"à¸ĵà¹Į":124361,"ัย":124362,"Ùħع":124363,"ë¨":124364,"ë¿":124365,"ë®":124366,"ï´":124367,"ì¥":124368,"ì«":124369,"ëµ":124370,"á¡":124371,"âį":124372,"ðĵ":124373,"â°":124374,"à¸Ĥà¸Ńà¸ĩ":124375,"Ùĭ":124376,"à¸ģัà¸ļ":124377,"ãģ®ãģ§":124378,"à¹īว":124379,"à¸Ńยà¹Īาà¸ĩ":124380,"ãģŃ":124381,"á»ĩt":124382,"à¸ķà¹īà¸Ńà¸ĩ":124383,"×ŀ×Ļ":124384,"à¹ģà¸ļ":124385,"×Ĵר":124386,"ÙĪÙģ":124387,"ÙĤÙĦ":124388,"à¸łà¸²à¸ŀ":124389,"ר×Ļ":124390,"ลา":124391,"ÙĬس":124392,"Ġצ":124393,"ÙĬÙģ":124394,"Ġ×ĺ":124395,"à¸ľà¸¥":124396,"áng":124397,"รว":124398,"Ġ×ŀש":124399,"×IJ×ķת":124400,"×ĸ×Ķ":124401,"ูà¸ģ":124402,"à¸Ļัà¸ģ":124403,"اÙĨÙĬ":124404,"دا":124405,"ãģ³":124406,"׼ף":124407,"ãĤīãĤĮ":124408,"ãĤĮãģ°":124409,"תק":124410,"úc":124411,"ÙĪØ²":124412,"×Ļר×Ķ":124413,"Ġngh":124414,"ánh":124415,"Ġ×ķ×IJ":124416,"á»ħ":124417,"สุà¸Ķ":124418,"ëį°":124419,"اض":124420,"اÙĦÙĬ":124421,"بار":124422,"عÙħ":124423,"à¸ļา":124424,"تج":124425,"à¸ŀร":124426,"×ķר×Ķ":124427,"ảng":124428,"Ø®ÙĦ":124429,"à¸ī":124430,"ắc":124431,"ש×Ļ×Ŀ":124432,"íĶ":124433,"Ù쨳":124434,"×Ļ×Ĵ":124435,"пÑĢ":124436,"ĠاÙĦØ«":124437,"سط":124438,"รูà¹ī":124439,"ีà¹Īย":124440,"à¸Ńà¸Ķ":124441,"ãģªãĤĬ":124442,"×Ĵ×ĵ":124443,"ãģĦãģ¾ãģĹãģŁ":124444,"סק":124445,"خص":124446,"laÅŁ":124447,"енно":124448,"بØŃ":124449,"สà¸Ļ":124450,"ฮ":124451,"ר×IJש":124452,"ÙħÙĪ":124453,"دÙĬد":124454,"ษา":124455,"×ķ×ļ":124456,"ãĥ§ãĥ³":124457,"à¸ķุ":124458,"Ġêµ":124459,"ĠÑģво":124460,"צ×ij":124461,"à¸Ńม":124462,"à¸Ľà¸£":124463,"تع":124464,"×Ķת":124465,"اÙħÙĦ":124466,"×ŀ׳":124467,"ç¶ļ":124468,"ฤ":124469,"íį":124470,"ëĺ":124471,"ë¤":124472,"ìij":124473,"â´":124474,"ãĭ":124475,"ĠباÙĦ":124476,"á»ģu":124477,"ĠاÙĦÙĦ":124478,"à¸ķัว":124479,"ذÙĩ":124480,"ึà¸ĩ":124481,"à¹ĥà¸Ĭà¹ī":124482,"á»ĵng":124483,"à¸Ļั":124484,"มาà¸ģ":124485,"ãĥŁ":124486,"×ŀ×ķ":124487,"à¸Ĺย":124488,"á»Ļi":124489,"ằ":124490,"ảo":124491,"à¹Ĥà¸Ķ":124492,"×IJ׾":124493,"สาม":124494,"ÙĪØ¨":124495,"à¸Ĺุ":124496,"ยัà¸ĩ":124497,"עת":124498,"×ķ׳×ķת":124499,"à¸Ĥึ":124500,"à¸Ĥึà¹īà¸Ļ":124501,"à¸ģà¹Ī":124502,"ẫ":124503,"á»ijc":124504,"ãģĹãĤĩãģĨ":124505,"á»ĭch":124506,"Ġ×IJ×ķת":124507,"Ġש×IJ":124508,"׼×ķ׾":124509,"á»Ļc":124510,"عة":124511,"à¸Ĺี":124512,"à¹Ģà¸Ń":124513,"Ùĥت":124514,"ãģ»":124515,"ẻ":124516,"ìĹħ":124517,"à¸Ńà¸Ńà¸ģ":124518,"اÙĨت":124519,"à¹Ħร":124520,"Ġ×IJ×Ĺר":124521,"طر":124522,"ÙĨد":124523,"ืà¹īà¸Ń":124524,"Ø·ÙĦ":124525,"×IJ×Ķ":124526,"uyên":124527,"íĸī":124528,"×ij×Ķ":124529,"à¸Ħà¹Ī":124530,"à¸Ĭà¹Īว":124531,"ãģĤãĤĬãģ¾ãģĻ":124532,"ÙĬب":124533,"ק׾":124534,"ãĥĻ":124535,"Ä©":124536,"سر":124537,"าว":124538,"ãĤ±":124539,"à¸ļริ":124540,"ר×Ĵ":124541,"á»ĥu":124542,"ØŃت":124543,"×ķ×ŀ×Ļ":124544,"بÙĨ":124545,"êµIJ":124546,"ÄŁu":124547,"ãģªãĤĵ":124548,"×ij×§":124549,"Ġפר":124550,"ắn":124551,"ØŃÙĦ":124552,"×ij×Ĺ":124553,"ấu":124554,"×ij×ķ×ĵ":124555,"ãĥ¯":124556,"Ġ׾ק":124557,"ัà¸į":124558,"à¸ŀิ":124559,"×Ĺ×Ķ":124560,"×ĸ׼":124561,"ãĥ¼ãĥł":124562,"ÑĤелÑĮ":124563,"×ŀ×Ļ×ĵ":124564,"ÙĬØ®":124565,"ẳ":124566,"تص":124567,"à¸ĺิ":124568,"è¾¼":124569,"ìĵ":124570,"ÙĥØ©":124571,"ÙĤب":124572,"à¸Ħà¹Į":124573,"à¹īาย":124574,"à¸ĵะ":124575,"าะ":124576,"ëĴ":124577,"ê¾":124578,"ë·":124579,"ìĩ":124580,"êº":124581,"ìģ":124582,"ëĢ":124583,"ì¾":124584,"ë½":124585,"ëļ":124586,"ìŃ":124587,"ìİ":124588,"áij":124589,"ëĹ":124590,"êĴ":124591,"à¡":124592,"à¬":124593,"ðIJĮ":124594,"ãĩ":124595,"ðĿĦ":124596,"Ġ׾×IJ":124597,"ãģ¨ãģĦãģĨ":124598,"Ġnhi":124599,"×Ļ×ķת":124600,"Ġש×Ķ":124601,"à¹ģลà¹īว":124602,"Æ°á»Ľc":124603,"à¸Ķà¹īวย":124604,"à¸Ĺาà¸ĩ":124605,"×ł×ª":124606,"פת":124607,"à¹ģà¸ķà¹Ī":124608,"ưng":124609,"à¸Ńยูà¹Ī":124610,"à¹īำ":124611,"Ġ×IJ׾":124612,"ÙĥÙħ":124613,"ấp":124614,"ลà¸ĩ":124615,"ãģŁãĤģ":124616,"×Ĵ׾":124617,"หร":124618,"ĠÑĢе":124619,"à¹Ģà¸Ĥà¹īา":124620,"ÙĤر":124621,"Ġ×Ķס":124622,"ÙĪÙĬ":124623,"สามาร":124624,"สามารà¸ĸ":124625,"Äĥn":124626,"à¸Ńี":124627,"פ×ķ":124628,"×Ļ׳×ķ":124629,"วัà¸Ļ":124630,"ặc":124631,"íķĻ":124632,"×ŀת":124633,"êu":124634,"ẹ":124635,"ÙģÙĬ":124636,"×ŀצ":124637,"à¸Ħา":124638,"ãģĿãģĨ":124639,"ãĢħ":124640,"از":124641,"اÙĩ":124642,"ר×Ļ×Ŀ":124643,"ấn":124644,"หาร":124645,"ạt":124646,"ÙĨÙĩ":124647,"à¹Ģà¸Ħร":124648,"جÙĩ":124649,"׼×Ļ":124650,"ắt":124651,"à¸Ħà¹īา":124652,"رة":124653,"ãĥı":124654,"ÙĥÙĪÙĨ":124655,"ứng":124656,"Ġìļ°":124657,"ยà¹Į":124658,"à¹Īวà¸Ļ":124659,"à¸ģำ":124660,"ثر":124661,"Ñģи":124662,"ĠاÙĦØ·":124663,"Ġ×Ķצ":124664,"ĠØ·":124665,"ĠاÙĦÙĪ":124666,"ê¹Į":124667,"ØŃÙĬ":124668,"ارات":124669,"à¹Ģà¸ĭ":124670,"با":124671,"гÑĢ":124672,"รี":124673,"ืà¸Ńà¸Ļ":124674,"عت":124675,"ÙĤاÙĦ":124676,"دÙħ":124677,"Ø¡":124678,"Ġ×ŀ×§":124679,"×ĵ×Ļ×Ŀ":124680,"×¢×ľ":124681,"ãģĴ":124682,"ëĭĺ":124683,"×¢×Ķ":124684,"Ġìĸ´":124685,"ÑģÑĮ":124686,"ÙĤØ·":124687,"ãĥĽ":124688,"èĢĥãģĪ":124689,"à¹ģà¸Ļ":124690,"ÙĪØ§Øª":124691,"âu":124692,"ĠìĤ¬ëŀ":124693,"หว":124694,"ĠاÙĦØ£Ùħ":124695,"Ġ×Ķ×ŀש":124696,"بÙĪ":124697,"à¸Ĭà¸Ļ":124698,"ãĤĵãģ§ãģĻ":124699,"วà¸Ļ":124700,"à¸ģรรม":124701,"×ŀ×ķ×ĵ":124702,"ÙĥاÙĨ":124703,"×ķ×£":124704,"олог":124705,"تÙĨ":124706,"à¸ķà¹Į":124707,"ê²ĥ":124708,"ר×ĺ":124709,"ừng":124710,"×ķ×ij×Ķ":124711,"ÙħØŃ":124712,"ĠЧ":124713,"פ×Ĵ":124714,"สà¸ĸ":124715,"ãģĭãĤĬ":124716,"ınız":124717,"à¹Ģย":124718,"ãĥ¼ãĥ³":124719,"ãģĬãĤĬ":124720,"פש":124721,"ิà¸ķ":124722,"Ø·ÙĨ":124723,"×Ļת×Ļ":124724,"×IJ׳":124725,"çek":124726,"ìª":124727,"×ŀ×ij":124728,"ศา":124729,"ãĤ¹ãĤ¿":124730,"à¸ļุ":124731,"×ĵ×ijר":124732,"ãģĦãģı":124733,"สะ":124734,"à¹Ģหล":124735,"ิà¸ĩ":124736,"à¸ŀัà¸Ļ":124737,"ãģĦãģŁãģł":124738,"ãĤĤãĤī":124739,"à¹īม":124740,"ãģĵãģ¨ãģĮãģ§ãģį":124741,"ารà¹Į":124742,"ุà¸ĩ":124743,"íij":124744,"ì¯":124745,"ë¼":124746,"íĤ":124747,"ì·":124748,"ê¡":124749,"áı":124750,"áĴ":124751,"ðĿľ":124752,"á©":124753,"ðŁĦ":124754,"ðIJ¤":124755,"Ġש׾":124756,"Ġ×ŀ×Ķ":124757,"à¹ģละ":124758,"Ġ׼׾":124759,"ẽ":124760,"á»Ļng":124761,"ذÙĬ":124762,"ле":124763,"×¥":124764,"ãģªãģ©":124765,"ĠÙĪØ£":124766,"หà¸Ļà¹īา":124767,"ãģ¾ãģ§":124768,"à¸ķà¹Īà¸Ń":124769,"à¸Ĺัà¹īà¸ĩ":124770,"ãģłãģij":124771,"à¹ģà¸ļà¸ļ":124772,"à¹Ģรา":124773,"פ׾":124774,"ãģŁãģĦ":124775,"à¹Ģลย":124776,"ãģ£ãģ¦ãģĦãĤĭ":124777,"ếp":124778,"ึà¹Īà¸ĩ":124779,"ê´Ģ":124780,"ê³Ħ":124781,"׼×ķ":124782,"à¹Ģรืà¹Īà¸Ńà¸ĩ":124783,"×§×Ļ":124784,"êµŃ":124785,"פס":124786,"تÙĬ":124787,"ãĥĦ":124788,"Ġ×Ķ×Ĺ":124789,"ги":124790,"ר×IJ׾":124791,"×ŀ׾":124792,"ĠØ£ÙĬ":124793,"ĠعÙĦÙĬ":124794,"ãģĭãģ£ãģŁ":124795,"ש×Ļ":124796,"дÑĥ":124797,"×ŀף":124798,"׳×ĺ":124799,"׳×Ļת":124800,"miÅŁ":124801,"׼×Ŀ":124802,"Ġ×ijר":124803,"Ġ׾×ij":124804,"ĠÐĽ":124805,"çe":124806,"×ķ׳×Ļ":124807,"ãĤĪãģĨãģ«":124808,"פ×ķר":124809,"ãĥį":124810,"ÙĥÙĬ":124811,"×Ĺת":124812,"ÙģÙĦ":124813,"Ġ×Ķ×§":124814,"Ġ×Ķ×ij":124815,"Ġ×ŀס":124816,"à¹Īาà¸Ļ":124817,"пеÑĢ":124818,"à¹Īาว":124819,"Ġ×ij×IJ":124820,"ĠÙĪÙĩ":124821,"à¸Ļำ":124822,"Ġ×ijש":124823,"׳ק":124824,"ãģ©ãģĨ":124825,"ש×ķת":124826,"×ĵ×Ķ":124827,"à¹Ģà¸ļ":124828,"ÙĨس":124829,"Ġìļ°ë¦¬":124830,"สà¹Īวà¸Ļ":124831,"ลัà¸ĩ":124832,"جز":124833,"Ġ×Ĺ×Ļ":124834,"Ùĥثر":124835,"ละ":124836,"Ùĩد":124837,"ĠÙĪØ¨":124838,"اÙĦÙħ":124839,"à¹ģม":124840,"Æ¡i":124841,"Ġ×ij×Ĺ":124842,"ữa":124843,"à¹Ģà¸Ĺศ":124844,"à¸ķัà¹īà¸ĩ":124845,"огда":124846,"׾ק":124847,"دد":124848,"สรà¹īาà¸ĩ":124849,"à¸Ĭี":124850,"Ù쨶":124851,"à¹ģห":124852,"uyá»ĩn":124853,"รัà¸ģ":124854,"á»ĩm":124855,"สา":124856,"פק":124857,"ียà¸ĩ":124858,"à¸ķà¹Īาà¸ĩ":124859,"à¸Ħรัà¹īà¸ĩ":124860,"ØŃÙĤ":124861,"à¹Ģà¸Ńà¸ĩ":124862,"ائÙĬ":124863,"×ĺ×¢":124864,"اÙĦØ©":124865,"ิà¹Īม":124866,"ãĤ½":124867,"دÙī":124868,"Ġר×IJ":124869,"ãģ£ãģ¨":124870,"ãĥĥãĥĹ":124871,"ÙĬرة":124872,"ê±´":124873,"×ŀ×IJ":124874,"×ķ×ķ":124875,"بع":124876,"ãģ²":124877,"ราย":124878,"×ĵ×Ŀ":124879,"تÙģ":124880,"à¸ķà¸ģ":124881,"ạng":124882,"ãĤĴè¦ĭ":124883,"à¸Ĭั":124884,"Æ°á»Ł":124885,"Æ°á»Łng":124886,"جب":124887,"×ķ×ŀר":124888,"ĠìĤ¬ëŀĮ":124889,"óng":124890,"รั":124891,"Ġ×Ķ×ĸ":124892,"רצ":124893,"Ġ×Ĺ×ĵ":124894,"ذÙĦÙĥ":124895,"×ķר×Ļ":124896,"ãģ¡ãĤĥ":124897,"Ù쨹":124898,"Ġ׾צ":124899,"ái":124900,"à¹ĩà¸ļ":124901,"ãģİ":124902,"à¸ģิ":124903,"ạc":124904,"ë©°":124905,"ãģªãĤĭ":124906,"×ķ׾×Ŀ":124907,"à¹ģà¸Ĺ":124908,"×ķ×¥":124909,"меÑĤ":124910,"Ã¼ÅŁ":124911,"ÑĢÑı":124912,"à¸Ĵ":124913,"ÑģÑĤоÑı":124914,"عÙĪØ¯":124915,"Ùħار":124916,"طة":124917,"à¸ŀื":124918,"кÑĢ":124919,"à¹ģà¸ģ":124920,"à¹Ĥรà¸ĩ":124921,"×ij×Ļ×ĺ":124922,"ê²ł":124923,"×ķ׾×Ķ":124924,"ØŃر":124925,"ืà¹Īà¸Ńà¸Ļ":124926,"×ķ×ijר":124927,"×Ĺש":124928,"ãĥķãĤ¡":124929,"×ŀ×ĺ":124930,"út":124931,"Ġdön":124932,"ắng":124933,"ëłĩ":124934,"ẳng":124935,"วà¸ģ":124936,"صد":124937,"خط":124938,"à¸Ńั":124939,"ãĤıãĤĮ":124940,"سÙĦاÙħ":124941,"à¹Ģรà¹ĩ":124942,"×Ļש×Ļ":124943,"جاÙĦ":124944,"ãģijãĤĭ":124945,"à¸Ĭาà¸ķิ":124946,"ÙĪØ§ÙĤ":124947,"à¹Ĥà¸Ļ":124948,"ãģ¦ãģĹãģ¾":124949,"اعة":124950,"ãĤŃãĥ£":124951,"à¸įา":124952,"ÙĦاÙĤ":124953,"ิà¸ģ":124954,"ĠÑģов":124955,"ÑĢак":124956,"×Ļ׳×Ļ":124957,"Ã¼ÄŁ":124958,"Ã¼ÄŁÃ¼":124959,"×§×ij":124960,"à¹Īà¸Ńà¸ĩ":124961,"Ġgerçek":124962,"à¸Ĺั":124963,"ованиÑı":124964,"×ŀ׼":124965,"سة":124966,"×Ļ×£":124967,"leÅŁ":124968,"Ùħؤ":124969,"ĠìĿĺ":124970,"à¸IJาà¸Ļ":124971,"ĠÑģоб":124972,"ĠêµŃ":124973,"עצ":124974,"зв":124975,"สà¸ĩ":124976,"زÙĦ":124977,"ãģıãĤĮ":124978,"иÑĢÑĥ":124979,"تأ":124980,"полн":124981,"ìĺĢ":124982,"ÙĨØ´":124983,"׼×IJ":124984,"ÙħØ´":124985,"à¸Ķà¹Į":124986,"ÙĪÙĬÙĦ":124987,"à¹ģà¸Ĥ":124988,"ãģ£ãģ¦ãģĹãģ¾":124989,"ноÑģÑĤ":124990,"вл":124991,"ÙħÙĤ":124992,"راج":124993,"å¤ī":124994,"ëĽ":124995,"â¸":124996,"ìIJ":124997,"à»":124998,"áļ":124999,"â»":125000,"êĻ":125001,"â§":125002,"ðĴ":125003,"ðĿĩ":125004,"Ġ×IJת":125005,"ĠÙĦÙĦ":125006,"ĠØ£ÙĨ":125007,"Ġ×ķ×Ķ":125008,"ãģ«ãģ¯":125009,"Ġ×Ļש":125010,"تÙĩ":125011,"ÃŃnh":125012,"ÙĬات":125013,"Ġ×ij×ŀ":125014,"à¸Ļัà¹īà¸Ļ":125015,"à¸Ļà¹īำ":125016,"Ãło":125017,"à¸ķาม":125018,"ãģ®ãģ¯":125019,"dır":125020,"Ġnghi":125021,"ặt":125022,"×ŀ×Ļ×Ŀ":125023,"ãģ¦ãģĦãĤĭ":125024,"Ġ×ijת":125025,"หรืà¸Ń":125026,"ĠسÙĬ":125027,"ãģªãĤī":125028,"à¹Ĥà¸Ķย":125029,"ıyor":125030,"à¸Ńีà¸ģ":125031,"á»ĩnh":125032,"Ñĭм":125033,"à¸Ĺุà¸ģ":125034,"Ġ׾×Ĺ":125035,"Ġ×Ķר":125036,"Ġ×Ķ×Ļ":125037,"à¸ŀระ":125038,"à¹Ģวลา":125039,"Ġغ":125040,"ẫn":125041,"mÄ±ÅŁ":125042,"׼×Ķ":125043,"á»ijn":125044,"ãģ§ãģĹãĤĩãģĨ":125045,"ãĥ¢":125046,"à¸Ľà¸µ":125047,"ס×Ļ":125048,"ãģĵãĤį":125049,"Ġ׾פ":125050,"รà¸ĸ":125051,"ê¸Ī":125052,"à¸ģวà¹Īา":125053,"무":125054,"á»įng":125055,"ãĤĵãģ§":125056,"ãĤĪãģĨãģª":125057,"á»ĵi":125058,"ãĤ¬":125059,"สà¹Īà¸ĩ":125060,"×Ļ׳×Ķ":125061,"à¸ĸูà¸ģ":125062,"à¸Īัà¸Ķ":125063,"Ġ×Ķ×Ĵ":125064,"ãĥľ":125065,"×ŀ×ķת":125066,"ÙĪÙĥ":125067,"ëĭ¨":125068,"ĠØ«":125069,"ãģ®ãģĮ":125070,"à¹Ģหà¹ĩà¸Ļ":125071,"عا":125072,"à¸Ļิ":125073,"Åŀ":125074,"à¸Ńะ":125075,"ãģĪãĤĭ":125076,"Ø«ÙĦ":125077,"ØŃÙħد":125078,"à¹Ģà¸ģิà¸Ķ":125079,"פשר":125080,"פ×Ķ":125081,"มิ":125082,"ئÙĬس":125083,"à¸Ĺำà¹ĥหà¹ī":125084,"×¢×ĵ":125085,"ìĭ¤":125086,"à¸Ĭà¹Īวย":125087,"ĠاÙĦÙħÙĨ":125088,"زÙĬ":125089,"عÙĬ":125090,"Ġ׼×IJ":125091,"ạnh":125092,"ỹ":125093,"ãĤĵãģª":125094,"สู":125095,"צר":125096,"Æ°á»Ľng":125097,"×ķ×ķ×Ķ":125098,"à¹Ĥล":125099,"ĠاÙĦÙĩ":125100,"วา":125101,"หลาย":125102,"Ñīе":125103,"à¸Ĥà¹īà¸Ń":125104,"à¹īà¸Ńย":125105,"بط":125106,"каÑı":125107,"ĠØ¢":125108,"ĠиÑģ":125109,"ĠاÙĦغ":125110,"à¸ģา":125111,"à¸Ļà¹Īา":125112,"ÙĬÙĪ":125113,"×ij×ķר":125114,"á»ħn":125115,"วà¸ĩ":125116,"×Ļ×ĸ":125117,"ì²Ń":125118,"ним":125119,"룰":125120,"×Ĵ×ķר":125121,"صØŃ":125122,"ÙĦÙĪ":125123,"×Ĺ×ķת":125124,"สุ":125125,"رÙĬÙĤ":125126,"ס×ĺ":125127,"Ġ×ŀ×¢":125128,"ãĥĨãĤ£":125129,"à¸Ħิà¸Ķ":125130,"ãĤįãģĨ":125131,"à¹Ħล":125132,"à¸Ļà¹Į":125133,"á»ıi":125134,"ÑģÑĤÑĢо":125135,"สà¸Ķ":125136,"สาร":125137,"ÙĪÙĦØ©":125138,"ầm":125139,"รà¹Īว":125140,"รà¹Īวม":125141,"รุ":125142,"ĠاÙĦسÙĬ":125143,"ìĺģ":125144,"Ġ×ŀ×ij":125145,"פ×ĺ":125146,"à¸ķิà¸Ķ":125147,"×ĺ×Ļ×Ŀ":125148,"Ġ무":125149,"ÙĤدÙħ":125150,"ĠdÃ¼ÅŁ":125151,"ائÙĦ":125152,"мÑĭ":125153,"ØŃس":125154,"ÙĪØµ":125155,"×Ļ×§×Ķ":125156,"ãģ§ãģ¯ãģªãģĦ":125157,"à¹Ģหม":125158,"оÑĢÑĤ":125159,"íĨµ":125160,"ãģIJ":125161,"кÑĢа":125162,"ียว":125163,"عار":125164,"ئة":125165,"íĥĢ":125166,"ãģ«ãģªãĤĬ":125167,"جة":125168,"ÙĪÙĤع":125169,"ÑĮÑı":125170,"×ķצ×Ķ":125171,"ש×Ŀ":125172,"بÙĤ":125173,"Ġ×Ļ×Ķ":125174,"ÙĬØ·":125175,"ımız":125176,"деÑĢж":125177,"×Ļשר×IJ׾":125178,"غÙĬر":125179,"รà¸Ńà¸ĩ":125180,"à¹Ģรียà¸Ļ":125181,"Ġ×Ķ×ĺ":125182,"หมาย":125183,"ÙħÙĩ":125184,"اÙ쨩":125185,"ĠоÑĢг":125186,"ÙĪÙī":125187,"ãĥ©ãĤ¤":125188,"×ŀ׳×Ķ":125189,"ĠÄijo":125190,"ĠгоÑĢ":125191,"اÙħØ©":125192,"楽":125193,"Ø«ÙĬر":125194,"à¸ģิà¸Ī":125195,"á»ĵn":125196,"ÙĨب":125197,"ÑĢÑĥд":125198,"ìĹĪ":125199,"Ġ×Ĺ×ijר":125200,"ÑĢаж":125201,"ạch":125202,"تÙĪ":125203,"à¹Ĥม":125204,"×ij×Ļ×ij":125205,"ĠíĨµ":125206,"acaģı":125207,"جÙĦس":125208,"à¹Ģà¸Ľà¸¥":125209,"วà¸Ķ":125210,"à¸Ńล":125211,"ãģŁãĤĬ":125212,"à¸Ľà¸±à¸į":125213,"ĠìķĮ":125214,"عرÙģ":125215,"à¹Ħà¸Ł":125216,"أخ":125217,"å¤ļãģĦ":125218,"à¸Ķัà¸ĩ":125219,"Ø´Ùģ":125220,"ãģ£ãģ¦ãģĦãģ¾ãģĻ":125221,"×Ľ×ł×¡":125222,"ÑĨе":125223,"еÑģп":125224,"ÙħاÙħ":125225,"à¸ŀืà¹īà¸Ļ":125226,"иÑĩеÑģки":125227,"خد":125228,"ÙĥÙĪÙħ":125229,"Ġ×Ķר×IJש":125230,"تاب":125231,"é£Łãģ¹":125232,"ืà¸Ļ":125233,"оÑĢо":125234,"Ġböl":125235,"×ķ×Ĺ×ĵ":125236,"دÙĬر":125237,"ắm":125238,"دع":125239,"ãģķãģĽ":125240,"à¸ĺร":125241,"à¸ĺรรม":125242,"ãģĭãĤĤ":125243,"å¤ļãģı":125244,"rä":125245,"سع":125246,"×Ļ׾×Ķ":125247,"ضر":125248,"ĠاÙĦشر":125249,"×ĸ×ķר":125250,"×¢×ijר":125251,"ạm":125252,"алÑĮно":125253,"رÙĨ":125254,"اÙħج":125255,"׼×ļ":125256,"dıģ":125257,"ден":125258,"ضا":125259,"ÙĦÙĬÙħ":125260,"Ġê·¸ëŁ¬":125261,"تÙħاع":125262,"ارÙĬØ®":125263,"à¹Ĥà¸ķ":125264,"ĠÑģÑĢед":125265,"Ġ׳×ķס":125266,"ÙĤبÙĦ":125267,"оÑĤов":125268,"leÅŁtir":125269,"ĠмеÑģÑĤ":125270,"سÙĦÙħ":125271,"Ġעצ":125272,"ĠاÙĦسÙĦ":125273,"еÑĤÑĮ":125274,"ابة":125275,"нак":125276,"สà¸ĸาà¸Ļ":125277,"Ġ×ij׳":125278,"à¸ļัà¸Ļ":125279,"׼׳":125280,"ĠÃ¶ÄŁ":125281,"ãģ¨è¨Ģ":125282,"uyến":125283,"diÄŁ":125284,"áºŃu":125285,"ÑĢаÑģ":125286,"ãĤ·ãĥ§ãĥ³":125287,"nız":125288,"×ķ×ĵ×Ķ":125289,"تس":125290,"ÙħاÙĦ":125291,"à¹Ģหà¸ķุ":125292,"ยว":125293,"à¸ŀัà¸ģ":125294,"ãģĦãģªãģĦ":125295,"ĠкаÑĩ":125296,"ลà¹Į":125297,"×¨×Ľ×ª":125298,"ÅŁtur":125299,"×ŀ×ķס":125300,"ãģ¥":125301,"бол":125302,"عÙħاÙĦ":125303,"×ķרת":125304,"ÑĨион":125305,"ศึà¸ģ":125306,"à¸ı":125307,"ÑĢен":125308,"اسÙĬ":125309,"ائر":125310,"à¹Ĥà¸Ľà¸£":125311,"Ġseç":125312,"غÙĬ":125313,"ÑįÑĤ":125314,"енн":125315,"ãģªãģ®":125316,"×Ļש×Ķ":125317,"×Ļפ×ķר":125318,"ãģŁãĤģãģ«":125319,"زة":125320,"Ġçoc":125321,"ãĤ¯ãĥª":125322,"ÑĪен":125323,"ãĤıãģij":125324,"رÙĬد":125325,"ĠÑĢаÑģÑģ":125326,"Ùĥات":125327,"สà¸Ńà¸ļ":125328,"ceÄŁi":125329,"ãĤ¿ãĤ¤":125330,"à¸ļร":125331,"ĠاÙĦبر":125332,"׳×ķ×¢":125333,"rün":125334,"راض":125335,"ศาส":125336,"à¸ķรà¹Į":125337,"ãģįãģŁ":125338,"×ķ׾×ĵ":125339,"еÑĢи":125340,"íĹĺ":125341,"ắp":125342,"تعÙĦ":125343,"Ùĥد":125344,"иÑĤелÑĮно":125345,"Ø·Ùģ":125346,"ĠавÑĤом":125347,"Ġ×ŀצ":125348,"ÑĪиÑħ":125349,"اتÙģ":125350,"ĠÑħоÑĤ":125351,"ÙİØ§":125352,"ãģıãĤĭ":125353,"×Ķפ":125354,"à¹Ĥà¸Ĺ":125355,"à¹ģà¸ŀ":125356,"à¹Īà¸Ńย":125357,"ĠاÙĦÙħØ´":125358,"à¸ģารà¸ĵà¹Į":125359,"аниз":125360,"×Ķ׾":125361,"ظÙħ":125362,"ยุ":125363,"liÄŁ":125364,"à¹Ħà¸Ĥ":125365,"à¸ĸืà¸Ń":125366,"öz":125367,"ãģijãģ¦":125368,"à¹Ģà¸ľ":125369,"ุม":125370,"ãĥĹãĥ¬":125371,"Ġ×Ķ×IJ×Ĺר":125372,"ختÙĦÙģ":125373,"à¸İ":125374,"ÙĦاØŃ":125375,"Ġdüzen":125376,"צ×Ķ":125377,"ساء":125378,"×ķר×ļ":125379,"×ķ×ĵ×Ļ":125380,"ÑĢаÑĦ":125381,"ÅŁtır":125382,"ãģ«åħ¥":125383,"ãģĪãģ°":125384,"صÙĪÙĦ":125385,"ĠÐľÐ¾Ñģ":125386,"اÙĩر":125387,"ãģ£ãģ":125388,"ĠлÑİб":125389,"×Ļ×¢×Ķ":125390,"Ġ×Ķ×ŀ×§":125391,"สิà¸Ĺ":125392,"สิà¸Ĺà¸ĺิ":125393,"×Ļ׳×Ŀ":125394,"ÙĦاÙģ":125395,"à¸ŀัà¸Ļà¸ĺ":125396,"×ķ×IJ×Ķ":125397,"มั":125398,"à¸Ĥà¸ĵะ":125399,"доÑĢ":125400,"ãģ¨ãģª":125401,"à¸ģระà¸Ĺ":125402,"acı":125403,"×ķ׾×ķ×Ĵ":125404,"ÑĥÑĪ":125405,"ãĥ¥ãĥ¼":125406,"ãĥ¦":125407,"Ùħست":125408,"ĠaÅŁ":125409,"שק":125410,"פת×Ĺ":125411,"ายà¸Ļ":125412,"íĩ":125413,"ë¢":125414,"ï·":125415,"íī":125416,"ìµ":125417,"ì¬":125418,"ðĿĽ":125419,"ìĴ":125420,"ëĻ":125421,"ê§":125422,"áĸ":125423,"â¨":125424,"â±":125425,"áĺ":125426,"ðĸ":125427,"àł":125428,"áĶ":125429,"ðIJŃ":125430,"ững":125431,"Å©ng":125432,"Ġ×Ķת":125433,"ĠاÙĦا":125434,"Ġ×ŀת":125435,"à¸ĸึà¸ĩ":125436,"òn":125437,"á»ĭnh":125438,"нÑĭм":125439,"Ġcả":125440,"à¸Ķู":125441,"Ġà¹ģà¸ķà¹Ī":125442,"Ġ×ij×Ķ":125443,"ói":125444,"ãģ¨ãģĹãģ¦":125445,"úng":125446,"Ġذ":125447,"Ġ×Ķ׳":125448,"ĠبÙĨ":125449,"ÙĦاÙĦ":125450,"à¹Ħà¸Ĺย":125451,"á»ĩp":125452,"tı":125453,"มัà¸Ļ":125454,"ằng":125455,"á»ijt":125456,"ком":125457,"à¸ĭึà¹Īà¸ĩ":125458,"à¸Ħรัà¸ļ":125459,"à¸ļà¹īาà¸Ļ":125460,"ĠاÙĦÙĬ":125461,"lü":125462,"ÙĪØ³":125463,"ãģłãģ£ãģŁ":125464,"à¹Ģà¸ĩ":125465,"Ġê³µ":125466,"нÑĥ":125467,"ãĤĪãĤĬ":125468,"мÑĥ":125469,"à¹Ģà¸Ĥา":125470,"ãĤĢ":125471,"ние":125472,"ãģ«ãģªãĤĭ":125473,"áºŃy":125474,"ĠÙĪØ§":125475,"볤":125476,"ש×ķ":125477,"áp":125478,"×ĵ×ķ":125479,"ãģ§ãģĹãģŁ":125480,"عض":125481,"Ñģкой":125482,"æĦŁãģĺ":125483,"ÑİÑĤÑģÑı":125484,"Ġ×Ļ׼×ķ׾":125485,"ãĤĵãģł":125486,"ви":125487,"à¹Ģลà¹Īà¸Ļ":125488,"ìĿ´ëĭ¤":125489,"ĠÙĦÙĩ":125490,"à¸Ħืà¸Ń":125491,"تÙĥ":125492,"ÙħÙĥÙĨ":125493,"aģı":125494,"׳×ĵ":125495,"민":125496,"à¹Ħว":125497,"สำห":125498,"สำหรัà¸ļ":125499,"Ñģлед":125500,"tır":125501,"ĠÙĦÙĬ":125502,"ĠاÙĦعÙħÙĦ":125503,"×ij×ķת":125504,"×ij×Ļ×Ŀ":125505,"à¸Ħำ":125506,"à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ":125507,"lıģı":125508,"ืà¸Ńà¸ĩ":125509,"جد":125510,"íŀĪ":125511,"ìĭ¬":125512,"×¢×ķת":125513,"สิà¸Ļ":125514,"Ñĩи":125515,"رض":125516,"à¹Ģà¸Ľà¸´à¸Ķ":125517,"à¸Ħà¹Īา":125518,"ìĦł":125519,"ÙĪØ±Ø©":125520,"×§×ĺ":125521,"ìľł":125522,"عÙħÙĦ":125523,"×IJ×Ļ×Ŀ":125524,"׾×Ļ×Ŀ":125525,"à¹ĥหà¸į":125526,"à¹ĥหà¸įà¹Ī":125527,"ừa":125528,"á»įi":125529,"ãģ¶":125530,"ÃŃch":125531,"ãĥĩãĤ£":125532,"×ķר×Ļ×Ŀ":125533,"Ñģо":125534,"ìķ½":125535,"ова":125536,"ÑĩаÑģÑĤ":125537,"à¹Ģà¸Īà¹īา":125538,"пÑĢо":125539,"Ġ×ŀ×Ĺ":125540,"ãĥİ":125541,"×ķ×Ļ×ķת":125542,"Ġде":125543,"ë§Ī":125544,"ì§ģ":125545,"×Ļפ×Ķ":125546,"ĠاÙĦعاÙĦÙħ":125547,"르":125548,"ר×IJ×Ķ":125549,"uyá»ĥn":125550,"×¢×Ļ":125551,"มืà¸Ń":125552,"Ø¥ÙĨ":125553,"รู":125554,"Ġز":125555,"×Ļ×ķ×Ŀ":125556,"à¸ķà¹īà¸Ļ":125557,"ãģ¦ãģĦãģ¾ãģĻ":125558,"ÙħاÙĨ":125559,"ĠÐ¥":125560,"à¸Ľà¸£à¸°à¹Ģà¸Ĺศ":125561,"ỳ":125562,"׾×ij":125563,"à¹Ģà¸Ķà¹ĩ":125564,"ãģŁãģ¡":125565,"à¸Ĺีม":125566,"à¸Ļะ":125567,"ìŰ":125568,"ĠìłĢ":125569,"ÙĦÙĩ":125570,"ợi":125571,"ĠاÙĦز":125572,"دار":125573,"ãĤ³ãĥ³":125574,"мин":125575,"à¹ģหà¹Īà¸ĩ":125576,"à¸Ķัà¸ļ":125577,"׼ר":125578,"жа":125579,"íĸĪ":125580,"×ŀ×ĸ":125581,"ợi":125582,"à¸Ķา":125583,"Ġعبد":125584,"à¹ģร":125585,"×IJתר":125586,"×¢×ł×Ļ":125587,"à¹Ģà¸Ħ":125588,"×ķצר":125589,"ì§Ģë§Į":125590,"ائÙħ":125591,"أس":125592,"uyá»ģn":125593,"Ġ×IJ׳":125594,"×Ĺ׳×ķ":125595,"×ĸ×Ļ":125596,"รà¹īาà¸Ļ":125597,"ĠÐłÐ¾Ñģ":125598,"ĠÐłÐ¾ÑģÑģ":125599,"ربÙĬØ©":125600,"tür":125601,"ãĤĭãģĵãģ¨":125602,"ظر":125603,"бÑĭ":125604,"à¸Ĺีà¹Īสุà¸Ķ":125605,"Ġצר":125606,"èĩªåĪĨ":125607,"лаÑģ":125608,"ĠÑıв":125609,"ĠÑıвлÑı":125610,"à¸ŀรà¹īà¸Ńม":125611,"à¸Ńาà¸Ī":125612,"à¸ļริà¸ģาร":125613,"Ġçı":125614,"ëįĺ":125615,"ĠاÙĦÙħست":125616,"تش":125617,"ש×ķ×ij":125618,"ãĤ´":125619,"Ġyapıl":125620,"ĠاÙĦذ":125621,"ุà¹Īม":125622,"à¸ĸà¹īา":125623,"ìĦ¤":125624,"ì°¨":125625,"ваÑĢ":125626,"à¹Ģà¸ŀิà¹Īม":125627,"Æ°á»Ľi":125628,"Ùĥس":125629,"à¸Ńยาà¸ģ":125630,"ãģ¦ãĤĤ":125631,"Ġгод":125632,"ÙĬار":125633,"à¸ķà¸Ńà¸Ļ":125634,"ĠигÑĢ":125635,"à¹Ħà¸Ķà¹īรัà¸ļ":125636,"ĠاÙĦÙħر":125637,"ÙĤت":125638,"Ġëĺ":125639,"ĠëĺIJ":125640,"ẩn":125641,"ãģĻãĤĭãģĵãģ¨":125642,"×Ĵ×Ŀ":125643,"Ġ×ij×ij":125644,"تد":125645,"ÙĪØ§Ø±":125646,"ãĤ®":125647,"пол":125648,"Ġмог":125649,"ترÙĥ":125650,"ÙĪØ«":125651,"Ġçık":125652,"اة":125653,"à¹Ģà¸Ķียว":125654,"มีà¸Ħวาม":125655,"Ġ×ŀ×Ĵ":125656,"صÙģ":125657,"ĠТак":125658,"Ġ×Ľ×ª":125659,"×Ļ×ĵ×Ļ":125660,"овоÑĢ":125661,"ầy":125662,"สิà¹Īà¸ĩ":125663,"بت":125664,"ürü":125665,"ÙĨج":125666,"หลัà¸ģ":125667,"×Ļ×Ķ×Ŀ":125668,"ÙĤص":125669,"зÑĭ":125670,"×Ľ×ª×ij":125671,"ưu":125672,"mız":125673,"ĠìĦ¸":125674,"лог":125675,"ÙħÙĬÙĦ":125676,"ÙĬج":125677,"íĴĪ":125678,"à¸ŀà¸ļ":125679,"หัว":125680,"зна":125681,"רק":125682,"à¹Ĥร":125683,"Ġ×ijס":125684,"ĠBaÅŁkan":125685,"ĠëͰ":125686,"à¸Ńัà¸Ļ":125687,"ีà¹Īยว":125688,"неÑģ":125689,"à¹Ģà¸Ķิà¸Ļ":125690,"ÙĬاÙĨ":125691,"×ķ׾×Ļ":125692,"اخت":125693,"צ×ķת":125694,"ãģĵãģĵ":125695,"ĠاÙĦاÙĨ":125696,"ĠпÑĢоÑĨ":125697,"ãģ¾ãģł":125698,"×Ľ×¡":125699,"ĠاÙĦØ¢":125700,"ÙĬز":125701,"ĠاÙĦدÙĪÙĦ":125702,"ĠíķĺëĤĺ":125703,"ضع":125704,"ê»ĺ":125705,"ÅĽwi":125706,"ยิ":125707,"ãģ¡ãĤĥãĤĵ":125708,"ĠÙħØ´":125709,"à¸ĺี":125710,"ãģ¨ãģį":125711,"׳×Ļ×ķת":125712,"Ġë¯":125713,"Ġ미":125714,"Ġsı":125715,"ëĭĪê¹Į":125716,"Ġпл":125717,"غÙĦ":125718,"à¹ģรà¸ĩ":125719,"بÙĬر":125720,"ãģĤãĤĬãģ¾ãģĽãĤĵ":125721,"ê·¼":125722,"Ġyüz":125723,"ĠdeÄŁer":125724,"åł´åIJĪ":125725,"ỡ":125726,"маÑĤ":125727,"ราà¸Ĭ":125728,"ÙĪØ±ÙĬ":125729,"жен":125730,"ãģ¾ãĤĬ":125731,"ãģ®ä¸Ń":125732,"×Ļ×ĵ×¢":125733,"à¸Ńุ":125734,"à¸ļà¸Ńล":125735,"à¸Ľà¸±à¸įหา":125736,"زÙħ":125737,"ÄŁa":125738,"à¸Ńืà¹Ī":125739,"à¸Ńืà¹Īà¸Ļ":125740,"пл":125741,"ĠнеобÑħодим":125742,"׼×ij":125743,"à¹Ģศ":125744,"קר×Ķ":125745,"ì²ĺ":125746,"볨":125747,"×ŀ×§×ķ×Ŀ":125748,"jÄħc":125749,"ÙĩÙĦ":125750,"Ġ×¢×ij×ķ×ĵ":125751,"à¹Ħมà¹ī":125752,"à¸ģลัà¸ļ":125753,"×ķ׼׾":125754,"×§×ĵ":125755,"اÙĦÙĬØ©":125756,"رÙĩ":125757,"ãģijãĤĮãģ°":125758,"ĠÙĨÙ쨳":125759,"ãĤ¢ãĥ«":125760,"ìĹĪëĭ¤":125761,"×§×ķר":125762,"неÑĢ":125763,"باب":125764,"ãĤ¶":125765,"سبب":125766,"ÙĦÙĬÙĦ":125767,"صÙĨ":125768,"صدر":125769,"ếm":125770,"à¸Ĭà¹Īวà¸ĩ":125771,"ØŃÙĨ":125772,"Ġ×ij×Ĵ":125773,"×ŀ×ķ×¢":125774,"׾×Ĺ":125775,"大ãģį":125776,"تب":125777,"неÑĤ":125778,"×Ļ×ij×Ķ":125779,"бл":125780,"ãĥĹãĥª":125781,"اصة":125782,"ãģ¤ãģij":125783,"×Ļ×ŀ×ķש":125784,"ãģĮãģĤ":125785,"ëĭ´":125786,"ãģĭãĤĤãģĹ":125787,"ãģĭãĤĤãģĹãĤĮ":125788,"ãģ¡ãĤī":125789,"×ij×ĺ":125790,"ĠbaÄŁ":125791,"×Ļ×Ĺס":125792,"×ij×ķ×¢":125793,"ลี":125794,"פע×Ļ׾":125795,"ими":125796,"gÅĤ":125797,"Ġиме":125798,"خداÙħ":125799,"×IJ×Ļר":125800,"Ġyapt":125801,"ãģ¨ãģĦ":125802,"à¸ĩà¹Īาย":125803,"׾×Ļ×ķ":125804,"ØŃدث":125805,"راÙĤ":125806,"ĠÄIJi":125807,"ادر":125808,"ãģĵãģ¨ãĤĤ":125809,"×ij×Ļר":125810,"Ġвз":125811,"ضاÙģ":125812,"ת×ķ׼":125813,"ÑĢом":125814,"رات":125815,"à¹Ģà¸Ĺà¹Īา":125816,"ãģĺãĤĥ":125817,"ãģĿãģĵ":125818,"اجتÙħاع":125819,"à¹īà¸Ńà¸Ļ":125820,"ÙĤÙħ":125821,"본":125822,"Äŀ":125823,"ש×Ļ×ķ":125824,"×ij׳×Ļ":125825,"ìľĦìĽIJ":125826,"à¹ģà¸Ī":125827,"×Ĺ×ķר":125828,"دÙĬÙĨØ©":125829,"تط":125830,"ằm":125831,"òa":125832,"ยà¸Ńà¸Ķ":125833,"Ġëĭ¹":125834,"สุà¸Ĥ":125835,"×ĵר×ļ":125836,"دÙĨ":125837,"سÙĬÙĨ":125838,"ÙĪÙĤÙģ":125839,"ÑĨÑĭ":125840,"гоÑĤов":125841,"еждÑĥ":125842,"à¸ŀวà¸ģ":125843,"اÙĤتص":125844,"اÙĤتصاد":125845,"czÄĻ":125846,"niÄĻ":125847,"ÑĢеб":125848,"ØŃÙĪ":125849,"à¸Ĺà¹Į":125850,"ãĤĪãģŃ":125851,"дж":125852,"à¸ģลà¹Īาว":125853,"دÙĬØ«":125854,"ãĤ³ãĥŁ":125855,"ÙĤÙĪÙħ":125856,"ĠتØŃ":125857,"à¹Ģà¸ķิ":125858,"اÙ쨏":125859,"à¸Īุ":125860,"رÙĬاض":125861,"×ŀש×ļ":125862,"à¹Ĥย":125863,"еÑĢе":125864,"ãģ¿ãģŁãģĦ":125865,"ìĿ´ëĿ¼":125866,"ĠاÙĦÙħÙĪ":125867,"ĠÑģÑĤо":125868,"à¹Ģรà¹ĩว":125869,"ĠдеÑĤ":125870,"ĠÑģдел":125871,"à¹Ģà¸Ĭืà¹Īà¸Ń":125872,"פ׳×Ļ":125873,"ÙĪØ¶ÙĪØ¹":125874,"×ijס":125875,"à¹ģà¸Ķ":125876,"óc":125877,"ริม":125878,"ÑĢад":125879,"ìĪł":125880,"ãĥ¼ãĤº":125881,"ãģ«ãģĬ":125882,"ино":125883,"פ×Ļ׾":125884,"à¸Ĭัà¹Īà¸Ļ":125885,"×Ĺ×ĵש":125886,"à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ":125887,"׳×Ļס":125888,"غرب":125889,"ãĤ¸ãĥ£":125890,"สัà¸ĩ":125891,"à¹Ģà¸Ĺีà¹Ī":125892,"à¹Ģà¸Ĺีà¹Īยว":125893,"ëŁ¼":125894,"à¹ģà¸Ł":125895,"ãĥ¼ãĤ·":125896,"ãĥ¼ãĤ·ãĥ§ãĥ³":125897,"Ġвозмож":125898,"جÙħÙĪØ¹":125899,"×ijר×Ļ×Ŀ":125900,"ãĥĪãĥ©":125901,"ĠкаÑĩеÑģÑĤв":125902,"Ø·ÙĬ":125903,"ÑĤÑı":125904,"צ×ķ×¢":125905,"ģını":125906,"عÙĦÙī":125907,"اذ":125908,"ÙĪØ§ÙĤع":125909,"ÙħÙĪØ§":125910,"ائÙĬÙĦ":125911,"кол":125912,"á»ģm":125913,"à¸ľà¸¥à¸´à¸ķ":125914,"×Ļ׳×ĺר":125915,"سÙĥ":125916,"ש×Ļר":125917,"ศึà¸ģษา":125918,"à¸ļั":125919,"ÑĩаÑģ":125920,"×ķפ×Ķ":125921,"×Ļפ×ķ׾":125922,"ĠاÙĦساب":125923,"رÙĬب":125924,"ĠاÙĦبÙĬ":125925,"ãĤ¹ãĥĨ":125926,"Ñĩен":125927,"à¹ģà¸ľ":125928,"Ġ׳ש":125929,"زÙĬد":125930,"ØŃاد":125931,"ëįĶ":125932,"رÙĪØ¹":125933,"à¸Ĺุà¸Ļ":125934,"สมา":125935,"czeÅĦ":125936,"×Ļ×ĵ×Ķ":125937,"ãģ§ãģĤ":125938,"Ġçocuk":125939,"خب":125940,"à¸ļาย":125941,"à¸Ľà¸£à¸°à¸Ĭา":125942,"×ŀש׾":125943,"ãģªãģĭ":125944,"à¸ģาย":125945,"ãĥģãĥ£":125946,"аÑĢи":125947,"ĠÑĩа":125948,"à¸Ķำ":125949,"à¸Ĺัà¹Īว":125950,"ÑĥÑħ":125951,"Ġöz":125952,"Ġì¢ĭ":125953,"جرÙĬ":125954,"ائÙĤ":125955,"à¸łà¸±à¸¢":125956,"طار":125957,"دارة":125958,"Ä©nh":125959,"Ø«ÙĨ":125960,"zellik":125961,"اÙĦت":125962,"Ġgeli":125963,"ãĥķãĤ©":125964,"олод":125965,"ربع":125966,"שת×ŀש":125967,"à¸ļรร":125968,"íĿ¬":125969,"Ġürün":125970,"Ġê·¸ëłĩ":125971,"ศาสà¸ķรà¹Į":125972,"ãģľ":125973,"×Ļ×ij׾":125974,"ĠпÑĢедÑģÑĤав":125975,"سطÙĬÙĨ":125976,"ãĤĴ使":125977,"ĠпомоÑī":125978,"×ķקר":125979,"ãĥ¯ãĥ¼":125980,"Ġyönet":125981,"×Ļקר":125982,"à¸Ĥา":125983,"еÑĢиал":125984,"ØŃÙģ":125985,"Ġ×Ļצ":125986,"à¸Ĺิ":125987,"売":125988,"à¸Ļà¸Ńà¸ģ":125989,"×ķ׼ר":125990,"íĻľ":125991,"á»§y":125992,"ĠاÙĦÙĤر":125993,"×Ļ×ij×ķת":125994,"ÅĽni":125995,"Ùħشار":125996,"ượt":125997,"ĠÙĦدÙĬ":125998,"ÑĤел":125999,"ĠØ¥ÙĦÙĬ":126000,"عÙĦÙĪÙħ":126001,"ìķĺ":126002,"виÑĤ":126003,"à¸Ħะ":126004,"yrı":126005,"ãģ¨ãģ£ãģ¦":126006,"à¹Ģà¸ī":126007,"à¸ĸาม":126008,"ÙĤار":126009,"عÙĦاÙħ":126010,"ặng":126011,"ÙħÙĴ":126012,"×Ļ×ŀת":126013,"سبة":126014,"ãĤ¯ãĥ©":126015,"×ķסף":126016,"ĠпÑĢин":126017,"ãģĦãĤį":126018,"ساس":126019,"عتبر":126020,"วิà¸Ĺย":126021,"วิà¸Ĺยา":126022,"سÙĥر":126023,"ãĤ·ãĥ§":126024,"ãģģ":126025,"ัà¸ģษ":126026,"×ij×ķ×Ķ":126027,"หย":126028,"ãģ¾ãĤĮ":126029,"ĠоÑĢганиз":126030,"казал":126031,"ĠÑģвÑıз":126032,"uyết":126033,"ĠпÑĢоиз":126034,"Ġ×§×ĺ":126035,"à¹ģà¸ģà¹ī":126036,"пÑĥÑģ":126037,"Ġê·¸ê²ĥ":126038,"ëĬIJ":126039,"лекÑģ":126040,"ãĥ¼ãĥĹ":126041,"à¸ķำ":126042,"ת×Ĺ×Ļ׾":126043,"à¸Ńà¸ĩà¸Ħà¹Į":126044,"ẵ":126045,"׳צ":126046,"أش":126047,"Ø´Ùĩ":126048,"ยะ":126049,"à¸ģà¸İ":126050,"ĠاÙĦإسÙĦاÙħ":126051,"едÑĮ":126052,"ãģ²ãģ¨":126053,"ëıĦë¡Ŀ":126054,"ãģ©ãģ®":126055,"Ñĥв":126056,"еÑĩение":126057,"ĠاÙĦتج":126058,"ãģ«è¡Į":126059,"Ġпозв":126060,"ãĤıãĤĬ":126061,"ÙĦاث":126062,"íķĺìĺĢ":126063,"ĠмаÑĢ":126064,"ĠkonuÅŁ":126065,"ãĥ¬ãĤ¹":126066,"ãĤĴæĮģ":126067,"ĠоÑģнов":126068,"×Ĺ×ij":126069,"ÙĪØ¬ÙĪØ¯":126070,"פ×ķף":126071,"воÑĢ":126072,"Ġник":126073,"ãģĭãĤĭ":126074,"ÅŁtırma":126075,"×Ļס×ĺ":126076,"Ø£ÙĦ":126077,"หà¹Į":126078,"иона":126079,"лÑĮн":126080,"ĠгоÑģ":126081,"ĠÐľÐ¾Ñģк":126082,"ÑĢоб":126083,"×ķ×IJ×Ļ":126084,"ãģĬãĤĬãģ¾ãģĻ":126085,"ãģ£ãģ±":126086,"кл":126087,"à¸Ļà¸Ķà¹Į":126088,"رÙĬÙģ":126089,"اسب":126090,"ĠÑĢеÑĪ":126091,"Ġдол":126092,"ãģ¹ãģį":126093,"×Ļ×ij×ķר":126094,"меÑī":126095,"ĠнаÑĪ":126096,"à¹ģà¸Ľà¸¥":126097,"ÑĢиÑĤ":126098,"кÑĥÑģ":126099,"иÑĢа":126100,"аÑĤÑĥÑĢ":126101,"ÙĪØ§ØµÙĦ":126102,"à¹Ģà¸ľà¸¢":126103,"à¸Ńำ":126104,"à¹Ģà¸ģิà¸Ļ":126105,"غÙħ":126106,"ãģĻãģİ":126107,"lıkl":126108,"ÅĦsk":126109,"견":126110,"×Ļ׼×Ķ":126111,"×Ĺש×ij":126112,"ÙĪØ±ÙĬØ©":126113,"ĠдейÑģÑĤв":126114,"×Ĺ׾×ĺ":126115,"Ġ׾×ŀ×¢":126116,"צ׾×Ļ×Ĺ":126117,"еÑĩа":126118,"ÙģØ§Ø¹":126119,"×Ĵ×Ļ×ĵ":126120,"áºŃm":126121,"ÄĻb":126122,"شع":126123,"ãģıãĤĬ":126124,"à¸ŀุ":126125,"едеÑĢ":126126,"à¸Ĥà¸Ļ":126127,"à¸Ħาร":126128,"ĠболÑĮÑĪ":126129,"ãģıãģªãĤĬ":126130,"à¸ĵา":126131,"×ĵ×ķ×Ĵ":126132,"Ġмн":126133,"ä¸ĬãģĮ":126134,"ç¶ļãģį":126135,"ฤษ":126136,"à¸Ĩ":126137,"Ø®ÙĬ":126138,"à¹Ģà¸Ĺà¸ŀ":126139,"สัม":126140,"à¹Ģสà¸Ļ":126141,"à¹Ģสà¸Ļà¸Ń":126142,"ãĥ´":126143,"ĠиÑģÑĤ":126144,"باشر":126145,"ĠÑĥÑĢов":126146,"×ŀ×ķ×ĸ":126147,"abı":126148,"waż":126149,"×ķצ×IJ×Ķ":126150,"ÑĤвеÑĢ":126151,"à¸ŀัà¸Ļà¸ĺà¹Į":126152,"׳×Ĵ×ĵ":126153,"ãĤĭãģĵãģ¨ãģĮãģ§ãģį":126154,"ĠÑĤÑĢеб":126155,"à¸ģรุà¸ĩ":126156,"ØŃتاج":126157,"à¹Ģà¸Ħล":126158,"ãĨ":126159,"ÄĻtr":126160,"Ġszczeg":126161,"Ġרש":126162,"à¸Ĺà¸ĺ":126163,"Ġнек":126164,"ĠнекоÑĤоÑĢ":126165,"вÑĪ":126166,"Ь":126167,"à¹Īวย":126168,"ลุ":126169,"бÑĢÑı":126170,"หมูà¹Ī":126171,"à¹ģà¸ķà¸ģ":126172,"ר׼×Ļ×Ŀ":126173,"Ġíĸī":126174,"ãi":126175,"Ùĥرة":126176,"âŃ":126177,"íIJ":126178,"ãį":126179,"áģ":126180,"â®":126181,"â¥":126182,"ì®":126183,"à¿":126184,"â¿":126185,"áĤ":126186,"á¤":126187,"âł":126188,"íŁ":126189,"ðIJį":126190,"ðIJ°":126191,"ðĿĨ":126192,"ðŁĪ":126193,"Ġ×¢×ľ":126194,"ĠعÙĨ":126195,"ĠÙħع":126196,"Ġ×ĸ×Ķ":126197,"ĠÙħا":126198,"ĠmÃł":126199,"Ġdụ":126200,"á»ĩc":126201,"аÑħ":126202,"sı":126203,"íķĺê³ł":126204,"Ġ×ķ×ij":126205,"ĠÐŁÐ¾":126206,"×ķתר":126207,"ĠÙĦÙħ":126208,"Ġ×ķ׾":126209,"ãģĹãģ¦ãģĦãĤĭ":126210,"Ġ×ŀ×Ļ":126211,"ĠبÙĬÙĨ":126212,"за":126213,"ĠÙĥاÙĨ":126214,"Ġ×Ķ×Ļ×Ķ":126215,"ëħĦ":126216,"×IJ×ķ":126217,"ди":126218,"ĠпеÑĢе":126219,"dı":126220,"Ġ׾ש":126221,"Ġש×ŀ":126222,"ãģĮãģĤãĤĭ":126223,"ãģĦãģĦ":126224,"ÑĢе":126225,"×§×ķ":126226,"или":126227,"ме":126228,"ÙĬت":126229,"ãģ§ãģĤãĤĭ":126230,"Ġво":126231,"à¹ĥหม":126232,"à¹ĥหมà¹Ī":126233,"Ġש×ij":126234,"Ġà¹Ĥà¸Ķย":126235,"ÙĬÙĩ":126236,"ãģ§ãģĻãģĮ":126237,"ãģ¨ãģ¯":126238,"ר×ķ":126239,"Ġà¸ĭึà¹Īà¸ĩ":126240,"ãģ§ãģįãĤĭ":126241,"мо":126242,"à¹Ģà¸ŀืà¹Īà¸Ń":126243,"צ×ķ":126244,"×ĺ×ķ":126245,"ìķĪ":126246,"Ġhá»į":126247,"à¹Ģà¸ĩิà¸Ļ":126248,"ĠاÙĦب":126249,"Ġมี":126250,"물":126251,"Ñģе":126252,"ëĵ¤ìĿ´":126253,"Ġë§IJ":126254,"ĠlỼ":126255,"aÅĤ":126256,"×Ĺ×ijר":126257,"Ġdá»±":126258,"ÙĬØ«":126259,"Ġthá»ĭ":126260,"à¸ģà¹Īà¸Ńà¸Ļ":126261,"Ġ×ij׼׾":126262,"ãģ¸":126263,"ã썿ĢĿãģĦãģ¾ãģĻ":126264,"ảnh":126265,"ยา":126266,"Ù쨧":126267,"สี":126268,"à¸ķา":126269,"ë²ķ":126270,"ãĥªãĥ¼":126271,"ราà¸Ħา":126272,"Ġ×ķ׾×IJ":126273,"ãģ¨ãģĵãĤį":126274,"à¹Ģลืà¸Ń":126275,"diÄŁi":126276,"ÙĪØ§ÙĨ":126277,"Ġ׾×Ķת":126278,"รวม":126279,"פ×Ļ×Ŀ":126280,"à¸ľà¸¡":126281,"жи":126282,"cı":126283,"ÑĢод":126284,"ĠkarÅŁÄ±":126285,"×Ĵ×ķ":126286,"ãģ«ãģ¤":126287,"ãģ«ãģ¤ãģĦãģ¦":126288,"rÃł":126289,"×Ļ×ķתר":126290,"ĠìĨĮ":126291,"×§×Ķ":126292,"ÑģÑĤво":126293,"ãģijãģ©":126294,"gé":126295,"à¸Ķà¹īาà¸Ļ":126296,"çļĦãģ«":126297,"ĠÙĬÙħÙĥÙĨ":126298,"ìĨį":126299,"ÙĬÙĥ":126300,"à¹Ħวà¹ī":126301,"Ñģкий":126302,"ìm":126303,"Ġ׾×IJ×Ĺר":126304,"à¸Ńาหาร":126305,"Ġà¹Ģà¸ŀ":126306,"ราะ":126307,"ลูà¸ģ":126308,"ÑģÑĤа":126309,"Ġìľł":126310,"ÙĤÙĪÙĦ":126311,"боÑĢ":126312,"Ñģкого":126313,"หลัà¸ĩ":126314,"à¸Ĥà¹Īาว":126315,"à¹Ģมืà¸Ńà¸ĩ":126316,"ê°ģ":126317,"tÃł":126318,"ÙĬÙĬÙĨ":126319,"عرض":126320,"ë°©":126321,"ĠëıĻ":126322,"Ġà¹Ģà¸Ľ":126323,"Ġà¹Ģà¸Ľà¹ĩà¸Ļ":126324,"çi":126325,"liÄŁi":126326,"ìĹIJê²Į":126327,"ãĤ¿ãĥ¼":126328,"Ġ×ľ×ª":126329,"פ×ķת":126330,"à¸Ĥà¸Ń":126331,"رس":126332,"ìłIJ":126333,"à¸ľà¹Īาà¸Ļ":126334,"ÑĦи":126335,"جÙĨ":126336,"ì¢ħ":126337,"Ġ×Ķפ":126338,"Ġngo":126339,"á»ĭa":126340,"Ġtá»ķ":126341,"Ġ그리":126342,"à¹Ģมืà¹Īà¸Ń":126343,"ذÙĥر":126344,"ìĸij":126345,"ìĹŃ":126346,"×ĺ׾":126347,"kı":126348,"ĠعÙħÙĦ":126349,"ĠعÙĨد":126350,"à¸ĭืà¹īà¸Ń":126351,"Ġê±°":126352,"ве":126353,"rü":126354,"à¹Ģà¸Ńา":126355,"สà¹Į":126356,"à¸Īà¸Ļ":126357,"סת":126358,"Ġgiả":126359,"ãĤĭãģ¨":126360,"à¸ģำลัà¸ĩ":126361,"ней":126362,"à¸Īริ":126363,"à¸Īริà¸ĩ":126364,"Ġëį":126365,"ĠëįĶ":126366,"à¸Ħà¹Īะ":126367,"ìn":126368,"Ġsüre":126369,"Ġquy":126370,"à¸ļาà¸ĩ":126371,"åıĸãĤĬ":126372,"ר×Ĺ":126373,"×ijת":126374,"ãģĮãģĤãĤĬãģ¾ãģĻ":126375,"רש":126376,"ìĹIJëĬĶ":126377,"Ġ×IJפשר":126378,"ayı":126379,"ãģĮãĤī":126380,"ØŃب":126381,"анÑģ":126382,"سÙĪ":126383,"ĠпÑĢе":126384,"دÙĪ":126385,"ãģ«ãĤĪ":126386,"à¹Ģà¸ģม":126387,"สูà¸ĩ":126388,"makt":126389,"maktad":126390,"maktadır":126391,"Ġönem":126392,"×Ļ×ŀ×Ļ×Ŀ":126393,"бо":126394,"ÙĪÙĬØ©":126395,"à¸£à¸¹à¸Ľ":126396,"à¹Ĥลà¸ģ":126397,"ÙħÙĬع":126398,"ÑģÑĤÑĥп":126399,"à¹Ĥà¸Ń":126400,"دÙĬÙĨ":126401,"ì¤ij":126402,"ãģĹãģı":126403,"à¹Ģสีย":126404,"вÑĭ":126405,"Ùħت":126406,"íĺĦ":126407,"ãĥIJãĥ¼":126408,"اش":126409,"קס":126410,"Ġtụ":126411,"ลà¸Ķ":126412,"Ù쨩":126413,"íijľ":126414,"رج":126415,"kÅĤad":126416,"ĠÅŁey":126417,"ĠØ£Ùħ":126418,"Ġà¹Ģม":126419,"ĠبÙĦ":126420,"ÑģкаÑı":126421,"ãģ¨ãģ®":126422,"Ġìĭ¤":126423,"ấm":126424,"หà¹īà¸Ńà¸ĩ":126425,"à¸Ĭม":126426,"dü":126427,"Ġçek":126428,"Ġê³ł":126429,"×Ĵ×ij":126430,"à¸Ĭีวิ":126431,"à¸Ĭีวิà¸ķ":126432,"Ù쨶ÙĦ":126433,"ฯ":126434,"çı":126435,"Ġبش":126436,"ĠÙĩÙĨا":126437,"ãģįãģ¾ãģĹãģŁ":126438,"tü":126439,"Ġìĺģ":126440,"ĠTürk":126441,"кÑĤ":126442,"פרס":126443,"ãģ¨ãģĦãģĨãģĵãģ¨":126444,"íĶĦ":126445,"à¹ģรà¸ģ":126446,"ר×ķף":126447,"Ġaras":126448,"×ŀצ×IJ":126449,"Ġtá»ī":126450,"سا":126451,"à¸ŀà¸Ń":126452,"ĠاÙĦÙħØŃ":126453,"ãĥ¤":126454,"ĠاÙĦاست":126455,"ÙģÙĨ":126456,"×Ļ×ŀ×Ķ":126457,"رت":126458,"ãģ¨ãĤĤ":126459,"ĠнаÑģ":126460,"пÑĢи":126461,"Ġ×Ĺ×ķ":126462,"ила":126463,"ÙĬØ´":126464,"Ġgöz":126465,"Ġ×ij׳×Ļ":126466,"ımı":126467,"ĠÑĤеÑħ":126468,"Ġhá»Ļ":126469,"غر":126470,"кон":126471,"اØŃت":126472,"Ġà¸ŀ":126473,"à¸Ńà¸Ńà¸Ļ":126474,"à¸Ńà¸Ńà¸Ļà¹Ħล":126475,"à¸Ńà¸Ńà¸Ļà¹Ħลà¸Ļà¹Į":126476,"Ñħо":126477,"Ñıв":126478,"à¹ģสà¸Ķ":126479,"à¹ģสà¸Ķà¸ĩ":126480,"à¹Ģà¸ŀียà¸ĩ":126481,"ÑĤов":126482,"اÙĬ":126483,"Ġ×Ķ×ĵ":126484,"Ġ×ķ׼":126485,"ãĤīãģĦ":126486,"×ķפף":126487,"Ġë¶Ī":126488,"ลà¸Ńà¸ĩ":126489,"طاÙĦ":126490,"Ġни":126491,"ĠÙħست":126492,"ếc":126493,"Ġש׼":126494,"ĠëķĮ문":126495,"วัà¸Ļà¸Ĺีà¹Ī":126496,"×Ļ׾×ĵ":126497,"ØŃا":126498,"еÑĨ":126499,"Ġcứ":126500,"×ĵ×ķר":126501,"ĠÙħØŃ":126502,"ר׼×ij":126503,"بÙĬع":126504,"нии":126505,"ĠاÙĦØ£ÙĪÙĦ":126506,"à¸Ħวร":126507,"ã썿ĢĿãģĨ":126508,"ĠСо":126509,"ائÙĬØ©":126510,"راء":126511,"оÑģоб":126512,"ĠبأÙĨ":126513,"×¢×ķ×ĵ":126514,"ĠÑĤе":126515,"ãģĵãģĨ":126516,"ÑģÑĤÑĢа":126517,"айн":126518,"Ġsöz":126519,"تÙĨا":126520,"à¸Ńิ":126521,"ặp":126522,"ĠìķĦëĭĪ":126523,"íķŃ":126524,"Ġר×IJש":126525,"Ġà¹Ħà¸Ķà¹ī":126526,"Ġ×Ĵ×ĵ":126527,"Ġספר":126528,"обÑīе":126529,"ĠÙĪØ¥":126530,"adaÅŁ":126531,"ãģ¡ãĤĩ":126532,"×§×ķ׾":126533,"ÑĢез":126534,"ĠdÃ¼ÅŁÃ¼n":126535,"Ġ×ij×IJ×ŀ":126536,"Ġìĸ´ëĸ":126537,"ער×ij":126538,"нее":126539,"ĠÑģÑĤÑĢан":126540,"ساÙĨ":126541,"ynı":126542,"ĠاÙĦرئÙĬس":126543,"ãģĹãģª":126544,"Ġ×ł×ª":126545,"ãģ«ãģªãģ£ãģŁ":126546,"gü":126547,"åıĹãģij":126548,"×ľ×ª":126549,"ìłĪ":126550,"ëĬĶëį°":126551,"Ø®ÙĬر":126552,"à¸ķà¹īà¸Ńà¸ĩà¸ģาร":126553,"ĠÙĦØ£ÙĨ":126554,"Ġchá»ĭ":126555,"ÙĪØ©":126556,"à¹ĥส":126557,"ë¶ĢíĦ°":126558,"íķĺë©´":126559,"ữu":126560,"à¹Ģหมืà¸Ńà¸Ļ":126561,"беÑĢ":126562,"ĠìĿ´ìļ©":126563,"ĠÑģеб":126564,"wiÄĻks":126565,"Ġ×ł×¢":126566,"ÑĤÑĥÑĢ":126567,"ĠnghÄ©":126568,"ש×ķ×ĺ":126569,"tiÄŁi":126570,"ĠdeÄŁi":126571,"×IJ×ij":126572,"Ġ×ŀ×ŀ":126573,"ãĥĹãĥŃ":126574,"waÅĤ":126575,"à¸Īึà¸ĩ":126576,"خدÙħ":126577,"×IJ×Ŀ":126578,"Ä±ÅŁÄ±":126579,"czÄħ":126580,"ר×ĵ":126581,"ĠÑĢÑĥб":126582,"خرÙī":126583,"ã쮿ĸ¹":126584,"ĠденÑĮ":126585,"×Ĺ×Ļ×Ŀ":126586,"еÑĤе":126587,"ëĤľ":126588,"×IJ×Ĵ":126589,"×¢×ķר":126590,"ë³Ħ":126591,"åIJĮãģĺ":126592,"ãĤ²":126593,"ר×ļ":126594,"×ķש×IJ":126595,"ìľ¡":126596,"اخ":126597,"צ×Ļ×Ķ":126598,"á»±a":126599,"ãģĪãģ¦":126600,"ש×Ķ×ķ":126601,"анÑĤ":126602,"ลาà¸Ķ":126603,"инг":126604,"ë¡ł":126605,"اعد":126606,"ÙĪØ³Ø·":126607,"Ġвоп":126608,"ĠвопÑĢоÑģ":126609,"ÙħÙĬÙĨ":126610,"à¸Ħà¸ĩ":126611,"×Ļר×Ļ×Ŀ":126612,"ców":126613,"격":126614,"Ġê·¸ëŁ°":126615,"Ġì§Ħ":126616,"Ġש׾×Ķ":126617,"à¹Ģริà¹Īม":126618,"à¸Ĭà¸Ńà¸ļ":126619,"деÑĤ":126620,"ÑİÑīиÑħ":126621,"à¸ļà¸Ńà¸ģ":126622,"æĢĿãģĦ":126623,"عÙĬد":126624,"ס×ŀ":126625,"×Ĵ×Ļ×¢":126626,"צ×ĵ":126627,"بات":126628,"ĠëͰëĿ¼":126629,"à¸Īัà¸ĩ":126630,"ãģłãģijãģ§":126631,"×¢×Ļר":126632,"ĠÑĩел":126633,"ĠÑĩелов":126634,"ĠÑĩеловек":126635,"ãĥĥãĥģ":126636,"à¹Ģà¸ģีà¹Īยว":126637,"à¸Ķิ":126638,"Ġפע":126639,"×Ļ×ŀ×Ļ":126640,"ë°ĺ":126641,"خار":126642,"×ij×Ļת":126643,"×¢×Ļ×Ŀ":126644,"üyor":126645,"ãĤģãģ¦":126646,"клад":126647,"Ġà¸Īาà¸ģ":126648,"à¹Ģà¸Ħย":126649,"สà¸Ńà¸ĩ":126650,"à¹ģà¸Ħà¹Ī":126651,"ẫu":126652,"หà¸Ļัà¸ĩ":126653,"ש׾×ķ×Ŀ":126654,"اÙĨÙĬØ©":126655,"åĩºä¼ļ":126656,"åĩºä¼ļãģĦ":126657,"à¸łà¸²à¸¢":126658,"à¸ļาà¸Ĺ":126659,"à¸Ĭาว":126660,"muÅŁ":126661,"Ġ׾ק×ij׾":126662,"ãĤ·ãĥ£":126663,"ĠÄ°ÅŁ":126664,"×Ĵ×ĵ×ķ׾":126665,"جعÙĦ":126666,"ë³Ģ":126667,"ยิà¹Īà¸ĩ":126668,"à¸Ļาย":126669,"à¸Ļีà¹Ī":126670,"วิà¸ĺี":126671,"ãĤīãģªãģĦ":126672,"ëłĪ":126673,"Ġë¬¸ìłľ":126674,"Ġà¸ģ":126675,"à¸Ĺำà¸ĩาà¸Ļ":126676,"à¹Ģวà¹ĩà¸ļ":126677,"ÑĦе":126678,"楽ãģĹ":126679,"สำà¸Ħ":126680,"สำà¸Ħัà¸į":126681,"رÙħ":126682,"ãģķãĤĮãģ¦":126683,"Ġобла":126684,"ר×IJ×Ļ":126685,"หมà¸Ķ":126686,"ÙĨÙĬØ©":126687,"лин":126688,"ĠeÄŁ":126689,"itim":126690,"ëł¹":126691,"صاÙĦ":126692,"ÅĽl":126693,"à¸ľà¸´à¸Ķ":126694,"ãĥŀãĥ³":126695,"åħ¥ãĤĮ":126696,"à¹Ģà¸ķà¸Ńรà¹Į":126697,"ارÙĬ":126698,"ĠЦ":126699,"dür":126700,"สวย":126701,"립":126702,"رÙĥØ©":126703,"Ġhã":126704,"×Ļת×Ķ":126705,"à¸Ĥà¸Ļา":126706,"à¸Ĥà¸Ļาà¸Ķ":126707,"à¸Īำà¸Ļ":126708,"à¸Īำà¸Ļวà¸Ļ":126709,"ש×ķ×§":126710,"Ġдом":126711,"ì±ħ":126712,"ãģĭãģij":126713,"פ×ķ׾":126714,"à¸Ĭาย":126715,"ÑģмоÑĤÑĢ":126716,"ÑģлÑĥж":126717,"ש×IJ׾":126718,"кÑĢÑĭÑĤ":126719,"Ġìŀĺ":126720,"é«ĺãģĦ":126721,"ĠÑĢÑĥк":126722,"ÙĨص":126723,"дав":126724,"ưỡ":126725,"ưỡng":126726,"راÙħ":126727,"×Ļ׳×Ļ×Ŀ":126728,"ãĥ©ãĥ¼":126729,"ëĦ¤":126730,"Ġتع":126731,"lke":126732,"好ãģį":126733,"æĮģãģ¡":126734,"Ġë§İ":126735,"Ġyük":126736,"ĠÑģоÑģÑĤав":126737,"енÑĤÑĢ":126738,"peÅĤ":126739,"à¹Ģà¸Ľà¸¥à¸µà¹Īย":126740,"à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļ":126741,"íıī":126742,"ãĤĦãģĻ":126743,"×Ĺ×ĸ":126744,"×ijר×Ķ":126745,"루":126746,"ìĶĢ":126747,"بØŃØ«":126748,"à¹Ģà¸ķà¹ĩ":126749,"ówi":126750,"بÙĩ":126751,"ãģįãģ¾ãģĻ":126752,"Ġ×¢×ŀ":126753,"×Ĵ×ķ׾":126754,"езд":126755,"ÙĬÙ쨩":126756,"สà¸Ļà¹ĥà¸Ī":126757,"Ġ×ª×ľ":126758,"ÑıÑī":126759,"ĠسÙĨ":126760,"ĠÙĪØ§ØŃد":126761,"ĠÑģм":126762,"ladı":126763,"ıld":126764,"×Ļרת":126765,"ียà¸Ļ":126766,"ת×Ĺת":126767,"Ġжиз":126768,"à¸ŀั":126769,"à¸ŀัà¸Ĵ":126770,"à¸ŀัà¸Ĵà¸Ļา":126771,"à¸Ĭิ":126772,"اخÙĦ":126773,"ãģ£ãģ¦ãģĦãģŁ":126774,"รัà¸IJ":126775,"ãĤģãĤĭ":126776,"à¹Ĥà¸ģ":126777,"ĠTá»ķ":126778,"Ġhakk":126779,"رÙģ":126780,"ìłĢ":126781,"Ñģоб":126782,"ãģªãģijãĤĮãģ°":126783,"ÙĩÙĪ":126784,"Ġë²ķ":126785,"ãĤĨ":126786,"ĠاÙĦسعÙĪØ¯":126787,"Ġ×IJתר":126788,"اغ":126789,"Ġ׾×ĵ":126790,"à¹ģà¸ķ":126791,"à¹ģà¸ķà¹Īà¸ĩ":126792,"íĮĮ":126793,"ÑĥпиÑĤÑĮ":126794,"à¸ŀืà¹īà¸Ļà¸Ĺีà¹Ī":126795,"×ijת×Ļ":126796,"à¹ĩà¸ģ":126797,"ÅĤat":126798,"Ġê°ľìĿ¸":126799,"ìłķë³´":126800,"ÑĤал":126801,"Ġgüven":126802,"Ġİl":126803,"Ġê°ģ":126804,"Ġبت":126805,"×ŀ×ķ׳×Ķ":126806,"ĠاÙĦØŃÙĥÙĪÙħ":126807,"ÙĤات":126808,"à¹ģà¸ģà¹Ī":126809,"หาà¸ģ":126810,"нÑĮ":126811,"à¸Ľà¸£à¸±à¸ļ":126812,"มาà¸ĵ":126813,"ĠнеÑģк":126814,"Ġض":126815,"สมั":126816,"สมัà¸Ħร":126817,"ãģĮãģĤãĤĬ":126818,"меÑģÑĤ":126819,"Ġ×IJצ׾":126820,"Ġкомпани":126821,"סר":126822,"ÙĬÙħØ©":126823,"ĠÑħоÑĢо":126824,"ĠÑħоÑĢоÑĪ":126825,"Ġ×Ļ×ķ×ĵ":126826,"üs":126827,"×Ĵ×Ļש":126828,"à¸ļà¸Ĺ":126829,"تÙĨظ":126830,"วาà¸ĩ":126831,"มหา":126832,"Ġ׼×ķ׾":126833,"à¸Ĥà¹īาà¸ĩ":126834,"ë°ľ":126835,"год":126836,"дан":126837,"ãģĭãĤĤãģĹãĤĮãģ¾ãģĽãĤĵ":126838,"ãģĵãģ¡ãĤī":126839,"ãĥIJãĤ¤":126840,"eceÄŁi":126841,"دÙĬدة":126842,"ÙĨÙī":126843,"Ġëĭ¤ìĿĮ":126844,"วี":126845,"غا":126846,"лиз":126847,"à¹Ģà¸Ķิ":126848,"à¹Ģà¸Ķิม":126849,"ĠÙĬست":126850,"Ġyılı":126851,"koÅĦ":126852,"ãģ§ãģĹãĤĩãģĨãģĭ":126853,"ãģĤãģª":126854,"ãģĤãģªãģŁ":126855,"ÑĨен":126856,"ĠÙĪØ²":126857,"×IJ×Ļש":126858,"à¹Īà¸Ń":126859,"رØŃ":126860,"ê´ij":126861,"ÑĢаÑģÑĤ":126862,"Ġ×Ķ׾":126863,"ãģĹãģ¦ãĤĤ":126864,"×ŀר׼":126865,"×ŀר׼×ĸ":126866,"éģķãģĦ":126867,"ãģŁãģı":126868,"ĠÑģÑĥд":126869,"веÑģÑĤи":126870,"ĠíķĦìļĶ":126871,"ãĥķãĤ§":126872,"ÑĤелÑĮно":126873,"à¹Ģà¸ŀืà¹Īà¸Ńà¸Ļ":126874,"ÅĤuż":126875,"à¹Ģà¸Ķิà¸Ļà¸Ĺาà¸ĩ":126876,"ש×ķר":126877,"Ġ×ŀ×ĵ":126878,"×ķ×¢×ľ":126879,"ÙĦاÙħ":126880,"à¹Ħà¸ĭ":126881,"лей":126882,"кÑĥÑĢ":126883,"Ả":126884,"à¸Ĺาà¸Ļ":126885,"ì§ij":126886,"ĠгоÑĢод":126887,"רס":126888,"׾×ķ×Ĵ":126889,"masını":126890,"ĠлÑĥÑĩ":126891,"ลà¹Īา":126892,"ìļ¸":126893,"ש×ĺ":126894,"ĠÐĺн":126895,"íĤ¤":126896,"ÙĪÙĦا":126897,"ìķł":126898,"ĠØ£ÙĬضا":126899,"Ùĥار":126900,"ĠاÙĦتع":126901,"สูà¹Ī":126902,"ãĤ¼":126903,"×ij×Ļ×IJ":126904,"ยà¸ģ":126905,"ĠØŃÙĤ":126906,"ربÙĬ":126907,"ãģĺãĤĥãģªãģĦ":126908,"รัà¸ģษา":126909,"ÑħодиÑĤ":126910,"à¸ķà¸Ńà¸ļ":126911,"׳×ĺ×Ļ":126912,"ĠاÙĦÙħج":126913,"تÙħع":126914,"оваÑĤÑĮ":126915,"ÙĦÙĬÙĨ":126916,"×Ļ×ŀ×ķת":126917,"Ġmù":126918,"nÄĻ":126919,"ĠدÙĬ":126920,"׼ש×Ļ×ķ":126921,"Ġhiç":126922,"ëijIJ":126923,"ÙĪØ§Ø¡":126924,"ÙĪØ·":126925,"ĠاÙĦبÙĦ":126926,"à¹ģมà¹ī":126927,"×§×ķת":126928,"ÙĪØ¬Ø¯":126929,"å§ĭãĤģ":126930,"ÙĬئة":126931,"Ġ매":126932,"صبØŃ":126933,"פ×IJ":126934,"гоÑĢ":126935,"ס×Ķ":126936,"بÙĬÙĤ":126937,"ยาà¸ģ":126938,"Ġнад":126939,"ÙĬÙij":126940,"ĠبÙĪ":126941,"ס×ķר":126942,"ÙħÙĥاÙĨ":126943,"ר×ij":126944,"×Ĵ×ĸ":126945,"צת":126946,"bilit":126947,"лаг":126948,"ĠNgo":126949,"×IJ×ķר":126950,"à¸ķà¸Ļ":126951,"íĬ¹":126952,"à¸Ĺีà¹Īà¸Ķี":126953,"à¸Ľà¸£à¸°à¸Īำ":126954,"ование":126955,"ãģĦãģ¤":126956,"ãĥĥãĤ¯ãĤ¹":126957,"åIJĪãĤı":126958,"åIJĪãĤıãģĽ":126959,"×Ļ׳×ķ×Ļ":126960,"ạy":126961,"Ø«ÙĤ":126962,"ĠпÑĢоб":126963,"ĠпÑĢоблем":126964,"ÅŁeh":126965,"ÅŁehir":126966,"عادة":126967,"اÙĨÙĪÙĨ":126968,"à¸ķัวà¹Ģà¸Ńà¸ĩ":126969,"ì¶ķ":126970,"ılan":126971,"бан":126972,"ãĥ³ãĥī":126973,"à¸Īี":126974,"Ġ×Ķש׳×Ļ":126975,"поÑĤ":126976,"×ķ׾×Ļ×Ŀ":126977,"ลัà¸ļ":126978,"ĠÑįÑĤи":126979,"×ijקש":126980,"ë¹ĦìĬ¤":126981,"à¸Ńยà¹Īาà¸ĩà¹Ħร":126982,"×Ļ׾×Ļ":126983,"à¹ĥà¸Ĭà¹Ī":126984,"ĠاÙĦÙĥÙĦ":126985,"ãĥļãĥ¼ãĤ¸":126986,"صة":126987,"ÑĤиÑĢ":126988,"ãĤĵãģ©":126989,"зÑĭк":126990,"wyż":126991,"ÙĩÙĬ":126992,"ĠÙħÙĦÙĬ":126993,"Ġвиде":126994,"ظاÙħ":126995,"داÙĪÙĦ":126996,"×ŀת×Ļ":126997,"Ġsık":126998,"à¹Ģà¸ķิม":126999,"ãĤ¢ãĤ¤":127000,"каÑħ":127001,"צ×Ļ׾":127002,"à¹Ģà¸Ĭà¹Īà¸Ļ":127003,"маг":127004,"магаз":127005,"магазин":127006,"à¸Ľà¸±":127007,"à¸Ľà¸±à¸Ī":127008,"Ġש×Ļר×ķת":127009,"ียม":127010,"ãĥĸãĥ«":127011,"ĠدÙĪÙĦ":127012,"קר×Ļ×Ŀ":127013,"ÙĩÙı":127014,"ово":127015,"Ġüret":127016,"دÙĪÙĨ":127017,"à¹ģà¸Ļว":127018,"à¹Ģà¸Ļืà¹īà¸Ń":127019,"ĠÑĦоÑĤ":127020,"ãĥĺ":127021,"ãģ¤ãģĭ":127022,"ÑıÑģ":127023,"ĠíķĺëĤĺëĭĺ":127024,"ائع":127025,"ĠплаÑĤ":127026,"ìĺĪ":127027,"ĠdostÄĻp":127028,"ÙĪØ¬Ùĩ":127029,"Ġ×Ķ×Ĺ×Ļ":127030,"׳×Ļ×§":127031,"дей":127032,"íĽĦ":127033,"ıy":127034,"بØŃر":127035,"à¹Ģสริม":127036,"Ġ׾×Ĵ":127037,"ذÙĩب":127038,"جÙĬÙĦ":127039,"رÙĥز":127040,"Ġëħ":127041,"Ġëħ¸":127042,"פ×Ļ׾×ķ":127043,"ãģ¾ãģļ":127044,"iriÅŁ":127045,"ĠÙĥÙĬÙģ":127046,"Ġ×ijצ":127047,"ĠêµIJ":127048,"ÑĢоÑģÑģ":127049,"ĠØ´ÙĬ":127050,"Ġiçer":127051,"×Ĵ×ķ×ij×Ķ":127052,"менно":127053,"×¢×ij×Ļר":127054,"×ķ×ŀ×Ķ":127055,"ãĤīãģĹãģĦ":127056,"ãģ¼":127057,"Ñīин":127058,"è²·ãģĦ":127059,"جÙħÙĪØ¹Ø©":127060,"Ġdönem":127061,"Ġ×ij×IJר":127062,"веÑģÑĤ":127063,"×ķר×ķת":127064,"سÙģ":127065,"à¹ģà¸Ĺà¸Ļ":127066,"ĠдокÑĥменÑĤ":127067,"ĠاÙĬ":127068,"جاÙĨ":127069,"צ×ķ×¢×Ļ":127070,"ĠоÑģоб":127071,"ĠاÙĦÙħس":127072,"ÑĢаб":127073,"à¸łà¸¹":127074,"à¸Ķาว":127075,"лекÑĤ":127076,"عÙĤ":127077,"×ķ×ĵ×ķת":127078,"Ġolu":127079,"ĠoluÅŁtur":127080,"ãģ¾ãģ¾":127081,"един":127082,"à¹Ģà¸Ńà¸ģ":127083,"ãĤµãĤ¤":127084,"ëĦĪ":127085,"Ø·ÙĨÙĬ":127086,"Ø·ÙĤØ©":127087,"ĠÐłÐ°Ð·":127088,"ÙĦÙij":127089,"Ñĩем":127090,"Ġ׾×ĺ":127091,"สัà¹Īà¸ĩ":127092,"سرائÙĬÙĦ":127093,"Ġפר×ĺ×Ļ":127094,"деÑģÑĮ":127095,"Ġ׳׼":127096,"اÙĨب":127097,"ÙĬاة":127098,"Ùħبر":127099,"Ġkı":127100,"à¸Ľà¸ı":127101,"à¸Ľà¸ıิ":127102,"à¸ļัà¸ķิ":127103,"×ł×ª×Ļ":127104,"ìĨ¡":127105,"راب":127106,"à¹ĥà¸ķ":127107,"à¹ĥà¸ķà¹ī":127108,"×Ļ×ł×ª":127109,"ÙĪÙĬر":127110,"Ġ×Ķ×ŀ×Ļ":127111,"ейÑĩаÑģ":127112,"×§×ķ×ij":127113,"دراس":127114,"ĠÙħÙĤ":127115,"رÙĬÙĨ":127116,"خاص":127117,"ãģĬéĩij":127118,"Ġجدا":127119,"ãģĨãģ¡":127120,"ëħ¸":127121,"ırım":127122,"æ§ĺ":127123,"ãģ«å¯":127124,"ãģ«å¯¾":127125,"ÑĨев":127126,"Ġvard":127127,"ĠÐIJн":127128,"eÄŁ":127129,"ÑģÑĤвенно":127130,"Ш":127131,"سد":127132,"à¸ģุ":127133,"à¹ģà¸ľà¸Ļ":127134,"รูà¹īส":127135,"รูà¹īสึà¸ģ":127136,"اتØŃاد":127137,"ÑijÑĤ":127138,"×Ĺ×ķ×§":127139,"ãģĻãģIJ":127140,"Ø·ÙĦاÙĤ":127141,"Ġ×§×ķ×ĵ":127142,"à¹ĥà¸Ĭà¹īà¸ĩ":127143,"à¹ĥà¸Ĭà¹īà¸ĩาà¸Ļ":127144,"ãĥ¼ãĤ¿":127145,"Ġsür":127146,"ÑĢок":127147,"ë³ij":127148,"สมาà¸Ĭ":127149,"สมาà¸Ĭิà¸ģ":127150,"ãĥķãĥ¬":127151,"è¾¼ãģ¿":127152,"ãĤ»ãĥ³":127153,"Ġê°Ģì§Ģ":127154,"à¸ľà¹īา":127155,"ÑįÑĤомÑĥ":127156,"иÑĤел":127157,"à¸łà¸±":127158,"à¸ij":127159,"ãĥĸãĥ©":127160,"×Ľ×ª×ķ×ij":127161,"׳×Ŀ":127162,"еннÑĭе":127163,"×¢×¨×Ľ×ª":127164,"ĠìĤ":127165,"ĠìĤ´":127166,"à¸Ĥà¹īา":127167,"׳×ķס":127168,"ãĥ¬ãĥĵ":127169,"ÑĢеÑģ":127170,"à¹Ģลà¸Ĥ":127171,"ثاÙĦ":127172,"ìĹĨ":127173,"ĠÑĩаÑģÑĤ":127174,"าศ":127175,"ãĥªãĤ¢":127176,"uç":127177,"×Ļ׼×ķת":127178,"ลà¹īาà¸Ļ":127179,"ië":127180,"ãĤ¸ãĤ§":127181,"à¸Īà¸Ń":127182,"ÙĪØŃØ¯":127183,"×Ļצ×ķ×ij":127184,"Ġ×ijש׾":127185,"око":127186,"ضة":127187,"ذر":127188,"ĠÑĥд":127189,"İL":127190,"×ķצ×Ļ×Ŀ":127191,"×ĸ×ŀף":127192,"à¸Ľà¸ģ":127193,"íķĻêµIJ":127194,"ساÙħ":127195,"à¹Ħà¸Ķ":127196,"ละà¹Ģà¸Ń":127197,"ละà¹Ģà¸Ńีย":127198,"ละà¹Ģà¸Ńียà¸Ķ":127199,"ảy":127200,"аÑĨион":127201,"ãĤ¹ãĤ¯":127202,"פ×ķס":127203,"รà¹Īาà¸ĩ":127204,"еннÑĭй":127205,"عÙĨ":127206,"عÙĦÙĨ":127207,"ائÙģ":127208,"dÄĻ":127209,"ؤÙĪÙĦ":127210,"׾×ķ×ķ":127211,"Ġ×ijש×ij":127212,"ä»ĬåĽŀ":127213,"ĠاÙĦجÙĨ":127214,"داد":127215,"waÄĩ":127216,"ãĥªãĥ³":127217,"ĠìŀIJìĭł":127218,"اÙĨÙĬا":127219,"ãĥ¡ãĥª":127220,"ÙĦÙĪÙĨ":127221,"à¸Ĺà¹Īà¸Ńà¸ĩ":127222,"à¸Ĺà¹Īà¸Ńà¸ĩà¹Ģà¸Ĺีà¹Īยว":127223,"اÙģÙĬ":127224,"ĠлиÑĪ":127225,"ÙħÙĬØ©":127226,"оÑĤвеÑĤ":127227,"Ñĩин":127228,"ÃĬ":127229,"ãĥ¡ãĥ³":127230,"å®Ł":127231,"éļĽãģ«":127232,"ĠÑĢай":127233,"ãĤ¦ãĥ³":127234,"×Ļר×ķש":127235,"×Ļר×ķש׾×Ļ×Ŀ":127236,"มะ":127237,"Ġara":127238,"казаÑĤÑĮ":127239,"à¸ķัà¸Ķ":127240,"ÑĥÑİÑĤ":127241,"Ġüst":127242,"×Ĵ×ķ×ij":127243,"×Ĵ×ķ×ij×ķת":127244,"malı":127245,"егод":127246,"егоднÑı":127247,"اÙģÙĤ":127248,"à¸Ĭà¹Īà¸Ńà¸ĩ":127249,"Ġözellik":127250,"×Ļצ×ķר":127251,"ĠmiÄĻd":127252,"ĠiliÅŁ":127253,"ĠнаÑħод":127254,"×¢×ĸר":127255,"×ľ×Ľ×ª":127256,"ÙĨتاج":127257,"ĠÑģем":127258,"à¸Īà¹Īาย":127259,"à¸ķรว":127260,"à¸ķรวà¸Ī":127261,"פר×ķ":127262,"à¸Ĥัà¸ļ":127263,"ãģŀ":127264,"Ġпло":127265,"колÑĮ":127266,"×ŀ×¢×ĺ":127267,"íķĺìĭľ":127268,"jÄħce":127269,"ÙĨاÙĨ":127270,"ลีà¸ģ":127271,"нÑĥÑĤ":127272,"ĠобÑĢаз":127273,"Ùĥبر":127274,"ĠاÙĦÙĪØ·ÙĨ":127275,"ãģķãģĽãģ¦":127276,"ÙĤاء":127277,"×ŀ×ĵ×Ļ׳":127278,"yü":127279,"פ×Ļת":127280,"׳×ķף":127281,"ÙħÙĨظ":127282,"หà¸Ļัà¸ģ":127283,"ìŀĪ":127284,"ãĤ«ãĥ¼ãĥī":127285,"عÙĨÙĬ":127286,"под":127287,"ضاء":127288,"à¸Ļà¸ķà¹Į":127289,"×ŀשפ":127290,"วà¹Į":127291,"ר×ķ×§":127292,"สืà¹Īà¸Ń":127293,"פק×Ļ×ĵ":127294,"ãģªãĤīãģªãģĦ":127295,"ĠìŬ룬":127296,"ÙĦج":127297,"ÑīиÑĤ":127298,"ãĥĥãĤ·":127299,"ÙĦÙĬس":127300,"ĠÙĦÙħا":127301,"ìłij":127302,"×ij×Ļף":127303,"ãĥģãĤ§":127304,"Ġgüç":127305,"Ġchứ":127306,"×ķצ×IJ":127307,"קר×ij":127308,"à¹Ĥà¸ŀ":127309,"оÑĩно":127310,"סק×Ļ":127311,"ש׾×Ŀ":127312,"صرÙģ":127313,"ĠLÃł":127314,"×¢×Ļת":127315,"á»·":127316,"à¹Ĥà¸Ńà¸ģ":127317,"à¹Ĥà¸Ńà¸ģา":127318,"à¹Ĥà¸Ńà¸ģาส":127319,"Ġ×Ķ×ĵ×ijר":127320,"à¸Ļัà¹Īà¸Ļ":127321,"زر":127322,"нако":127323,"íļį":127324,"ãĤĤãģ¡":127325,"ãĤĤãģ¡ãĤį":127326,"ãĤĤãģ¡ãĤįãĤĵ":127327,"اÙħت":127328,"عداد":127329,"инÑĭ":127330,"ÅĤyw":127331,"à¸Ħà¸ĵะ":127332,"à¸Ĺะ":127333,"ktör":127334,"×Ļ×Ĺ×Ķ":127335,"Ġме":127336,"ĠмеÑģÑı":127337,"׳×Ķ×Ĵ":127338,"ĠÑģÑĥÑīеÑģÑĤв":127339,"à¸Ļัà¸Ļ":127340,"ÑĦÑĦ":127341,"екÑĤив":127342,"عÙĦÙĪÙħات":127343,"бÑĥд":127344,"à¸Ļัà¸ģà¸ĩาà¸Ļ":127345,"หà¸Ļà¹īาà¸Ĺีà¹Ī":127346,"ÙĤÙĬÙĤ":127347,"ãĤ·ãĥ³":127348,"ãģ«éĸ¢":127349,"×IJר×Ĵ":127350,"ĠпÑĢоÑĤ":127351,"ĠпÑĢоÑĤив":127352,"ĠìŀĪìĸ´":127353,"ÙĤÙĬÙĤØ©":127354,"ìĹĩ":127355,"kür":127356,"ãģ«ãģªãĤĬãģ¾ãģĹãģŁ":127357,"ĠдеÑıÑĤ":127358,"ĠдеÑıÑĤелÑĮ":127359,"פ×ķר×ĺ":127360,"à¸Łà¹īา":127361,"à¹Ģà¸ł":127362,"ĠавÑĤомаÑĤ":127363,"×ĸ×Ļ×§":127364,"Ġolduk":127365,"عاÙħ":127366,"ĠÑĤоÑĢ":127367,"yrıca":127368,"êÌ":127369,"ãĤŃãĥ³ãĤ°":127370,"ãģ«ãģ¨ãģ£ãģ¦":127371,"à¹Ģà¸īà¸ŀ":127372,"à¹Ģà¸īà¸ŀาะ":127373,"ãģ¯ãģļ":127374,"×ŀ×IJ×Ļ":127375,"สะà¸Ķ":127376,"สะà¸Ķวà¸ģ":127377,"ìľ¼ë©°":127378,"à¸ģี":127379,"ฬ":127380,"Ġ×¢×ķש":127381,"à¸łà¸²à¸©à¸²":127382,"à¸Ĺัà¸Ļ":127383,"acakt":127384,"acaktır":127385,"اعدة":127386,"ĠÑĥÑģлÑĥг":127387,"סר×ĺ":127388,"×ķ×ŀ×ķת":127389,"×Ķ×ķר":127390,"×ŀ×ķ×ij":127391,"×ŀ×ķ×ijף":127392,"سÙĬاس":127393,"اتÙ쨧ÙĤ":127394,"×Ķצ׾":127395,"Ùħؤس":127396,"Ġpó":127397,"Ġкни":127398,"×Ļ׼×ķ׾":127399,"à¹Ģหลืà¸Ń":127400,"׼׾׼":127401,"׳×ĸ":127402,"ÑĪие":127403,"rès":127404,"ĠاÙĦØŃÙĤ":127405,"лÑıÑĢ":127406,"หà¸į":127407,"หà¸įิà¸ĩ":127408,"ר×Ĵ×Ļש":127409,"à¹Ģสà¹īà¸Ļ":127410,"ש×ij×ķף":127411,"ôtel":127412,"апÑĢ":127413,"апÑĢимеÑĢ":127414,"ابÙĦ":127415,"ĠÑĢазвиÑĤ":127416,"ĠполÑĮз":127417,"ĠСеÑĢ":127418,"×ķ×ij×Ļ":127419,"róż":127420,"ìĭŃ":127421,"ãĤ¯ãĥĪ":127422,"ãģĹãĤĪãģĨ":127423,"à¸ģรม":127424,"ØŃÙĥÙĪÙħ":127425,"à¹Ĥà¸ļ":127426,"à¸Ĺà¹īาย":127427,"ĠMá":127428,"ĠÑĤÑĭ":127429,"à¸Ħรัว":127430,"ÑĢÑĥб":127431,"ạp":127432,"ĠmÅĤ":127433,"ĠmÅĤod":127434,"ĠgörÃ¼ÅŁ":127435,"ĠgeliÅŁ":127436,"ươi":127437,"×ŀשק":127438,"ÙĢÙĢÙĢÙĢ":127439,"ราว":127440,"ãģĹãģ£":127441,"ãģĹãģ£ãģĭãĤĬ":127442,"ĠÐļон":127443,"Ġkê":127444,"à¹Ĥà¸Ĺร":127445,"èIJ½ãģ¡":127446,"åĩºãģ¦":127447,"ลัà¸ģษ":127448,"Ġ×Ĵ×ij×ķ×Ķ":127449,"ãĥĻãĥ«":127450,"ê±°ëĤĺ":127451,"ë§IJ":127452,"×Ļ׾×ĵ×Ļ×Ŀ":127453,"ĠëĦĪ":127454,"×ŀר×Ļ":127455,"รส":127456,"ãĥŃãĥ³":127457,"ило":127458,"ноÑģÑĤÑĮÑİ":127459,"×ĸר×Ĺ":127460,"пон":127461,"Ġ×Ķש׾":127462,"ê²łìĬµëĭĪëĭ¤":127463,"ĠkiÅŁ":127464,"ĠÐļи":127465,"วร":127466,"داع":127467,"ÅŁim":127468,"ÙĨÙij":127469,"ваÑĤ":127470,"راÙĥ":127471,"باÙĦ":127472,"иде":127473,"Ġ×Ķ×ŀ×Ĺ":127474,"ìĸµ":127475,"تÙģØ§Ø¹":127476,"أت":127477,"ëĬĺ":127478,"ש×Ļת":127479,"ستÙħر":127480,"ĠÑĦак":127481,"ĠاÙĦØ£ÙħرÙĬ":127482,"ëŀ¨":127483,"اسÙħ":127484,"ĠaÄŁ":127485,"Ġçev":127486,"ÙĥÙĪØ±":127487,"ãģķãģ¾":127488,"Ġçöz":127489,"Ġرس":127490,"Äħda":127491,"สà¸Ļุ":127492,"ãģĹãģ¦ãģıãĤĮ":127493,"нÑİ":127494,"leÅŁme":127495,"ãĤªãĥ³":127496,"ãģ¨ãģªãĤĬ":127497,"avaÅŁ":127498,"×ĺ×Ļ×ij":127499,"ØŃض":127500,"×ķצ×IJ×ķת":127501,"ÙĨÙħÙĪ":127502,"ıt":127503,"ĠÑħа":127504,"ĠÑħаÑĢак":127505,"ĠÑħаÑĢакÑĤеÑĢ":127506,"ĠdÅĤ":127507,"ãĥĹãĥ©":127508,"à¸Ĭุม":127509,"à¹Īà¸Ńà¸Ļ":127510,"×ķ×ij׾":127511,"Ñģол":127512,"×ĵ×Ĵ":127513,"аÑĢаÑĤ":127514,"nivers":127515,"ĠgerçekleÅŁtir":127516,"ĠاÙĦÙĦÙĬ":127517,"ระยะ":127518,"ĠÙħختÙĦÙģ":127519,"Ġgönder":127520,"ÙģØ§Ø±":127521,"doÄŁ":127522,"doÄŁan":127523,"صÙĦاØŃ":127524,"Ġyayın":127525,"ãĥĨãĥ³":127526,"รวà¸Ī":127527,"×Ļ×Ĺ×Ļ×ĵ":127528,"ünkü":127529,"ÑĨиалÑĮн":127530,"à¸ļู":127531,"มุ":127532,"hä":127533,"Ø®Ùģ":127534,"å¢Ĺ":127535,"å¢ĹãģĪ":127536,"еÑĩно":127537,"ĠاÙĦسÙĨ":127538,"à¸Ĥาว":127539,"imdi":127540,"Ы":127541,"à¸Ļà¸Ńà¸ģà¸Īาà¸ģ":127542,"à¸ļาล":127543,"תש":127544,"Ġdüzenle":127545,"мÑĭÑģл":127546,"ãģıãģª":127547,"żu":127548,"ĠwspóÅĤ":127549,"Ġназ":127550,"ındaki":127551,"ترة":127552,"ÅŁek":127553,"Ġöd":127554,"ĠÙĪÙĥ":127555,"ĠпозволÑı":127556,"Ġת×ķ׼":127557,"ÙħÙĨتج":127558,"ë§ī":127559,"ĠاÙĦØ«ÙĦاث":127560,"аÑĨиÑİ":127561,"ÙĪØ±ÙĪ":127562,"ÑĭваеÑĤ":127563,"خصص":127564,"ĠاÙĦÙģÙĦ":127565,"ĠاÙĦÙģÙĦسطÙĬÙĨ":127566,"إجر":127567,"إجراء":127568,"اÙĨتخ":127569,"اÙĨتخاب":127570,"ارÙĬØ©":127571,"×ķÖ":127572,"Ø¢ÙĨ":127573,"×ŀ×¢×ķת":127574,"Ġмал":127575,"Ġ×IJ×Ĺ":127576,"à¸Ĺà¹īà¸Ńà¸ĩ":127577,"zeÅĽ":127578,"Ġë§Įëĵ¤":127579,"رÙĬع":127580,"äºĭãĤĴ":127581,"à¸ļริหาร":127582,"׾×ŀ×Ļ×ĵ":127583,"ĠмÑĥж":127584,"ترÙĪ":127585,"ĠباÙĦØ¥":127586,"פ×Ļ×§":127587,"زÙħØ©":127588,"ĠÃ¶ÄŁrenc":127589,"ãĥ¶":127590,"اÙħعة":127591,"×§×ij×ķצ":127592,"×ŀ׳×ķת":127593,"رÙĬÙħ":127594,"Ġоказ":127595,"ãģłãģijãģ©":127596,"Ġhız":127597,"Ġש×IJת":127598,"ãĤ¢ãĥ¼":127599,"Ġmożliwo":127600,"ìĦ¼":127601,"ÙĪØ§Ø¨":127602,"огÑĢаÑĦ":127603,"ĠعبداÙĦ":127604,"ãĤĴè¡Į":127605,"بÙĬÙĦ":127606,"Ġİç":127607,"ยาย":127608,"ĠÑĥÑĩаÑģÑĤ":127609,"ÑĦеÑģÑģ":127610,"ÑĦеÑģÑģиона":127611,"Ấ":127612,"ÙĨÙĬÙĨ":127613,"عدÙĦ":127614,"สรร":127615,"دÙĬÙĦ":127616,"×ij×Ļ×§":127617,"czyÅĤ":127618,"ÑĢоме":127619,"Ġмед":127620,"ìĻĶ":127621,"ãĥ©ãĤ¤ãĥ³":127622,"ĠÑĤеп":127623,"еÑĢÑĮ":127624,"iÄŁi":127625,"вели":127626,"ÑĢиÑģÑĤ":127627,"ס×ķפ":127628,"×ŀ׾×Ĺ":127629,"ĠاÙĦØ¥ÙĨ":127630,"Ġ׾×Ķש":127631,"è¶ĬãģĹ":127632,"ĠÑĢÑĭ":127633,"×ķ×IJר":127634,"رÙĩاب":127635,"פ×ķ×IJ×Ļ":127636,"ĠгоÑģÑĥд":127637,"ĠгоÑģÑĥдаÑĢ":127638,"ĠгоÑģÑĥдаÑĢÑģÑĤв":127639,"ĠاÙĦØ£ÙħÙĬر":127640,"Ùħج":127641,"à¹Ģหมาะ":127642,"ÑĢев":127643,"à¸Ĭีà¸ŀ":127644,"ãĥķãĥĪ":127645,"иÑĩно":127646,"ĠاÙĦÙħؤ":127647,"Ġiht":127648,"íħľ":127649,"دÙĨÙĬ":127650,"رص":127651,"лаÑģÑĤ":127652,"à¹Ģหลà¹Īา":127653,"ılır":127654,"รà¸ĵà¹Į":127655,"×ŀש×Ļ×ļ":127656,"Ġdá»ĭ":127657,"Ø·Ù쨧ÙĦ":127658,"×ĺ×ķף":127659,"Ġ×ij×Ļ׳":127660,"ãģ¾ãģ£ãģŁ":127661,"ложениÑı":127662,"تØŃر":127663,"باØŃ":127664,"à¹Ģสืà¹īà¸Ń":127665,"ãģĻãģĶ":127666,"ltür":127667,"à¸ĩาม":127668,"Ġtü":127669,"ĠпÑĢим":127670,"ĠпÑĢимен":127671,"Ġhayat":127672,"ëĥIJ":127673,"ëĭĮ":127674,"׳×Ļ×ķ":127675,"веден":127676,"ìħ¨":127677,"à¸Īัย":127678,"à¸ģà¹Īà¸Ń":127679,"Ġвод":127680,"оÑģÑĤоÑı":127681,"наÑĤ":127682,"à¹ģหล":127683,"سÙħÙĬ":127684,"à¸Ķำà¹Ģà¸Ļ":127685,"à¸Ķำà¹Ģà¸Ļิà¸Ļ":127686,"wód":127687,"öyle":127688,"ãĥĢãĤ¤":127689,"ÑĪий":127690,"меÑīен":127691,"ãģĹãģ¾ãģĨ":127692,"ãĥīãĥ©":127693,"ÙĪØ¶ØŃ":127694,"à¸Ńà¸Ļุ":127695,"ĠاÙĦاجتÙħاع":127696,"laÅŁma":127697,"à¸Ħà¸Ńà¸Ļ":127698,"×ŀר×Ļ×Ŀ":127699,"ÙĨاÙħج":127700,"שר×ķת":127701,"اÙĦØ£":127702,"ĠksiÄħż":127703,"Ġан":127704,"ÑĢай":127705,"اÙĩرة":127706,"×ŀ×ĵ×Ķ":127707,"ä¸Ģç·":127708,"ä¸Ģç·Ĵ":127709,"ä¸Ģç·Ĵãģ«":127710,"ÑĢиÑĤоÑĢ":127711,"dıkl":127712,"à¹ģà¸ĸ":127713,"à¹ģà¸Ĥà¹Īà¸ĩ":127714,"екÑĤоÑĢ":127715,"×ŀסע":127716,"ÑĢакÑĤи":127717,"uÄŁu":127718,"×ķ×ijת":127719,"สูà¸ķร":127720,"ĠçalÄ±ÅŁm":127721,"ĠçalÄ±ÅŁmalar":127722,"Ġана":127723,"ãĥĽãĥ¼ãĥł":127724,"Ġbölüm":127725,"Ġبص":127726,"олоÑģ":127727,"ĠìķĬëĬĶ":127728,"à¹Īะ":127729,"ÙĪØªØ±":127730,"ä¹Ĺ":127731,"ستخداÙħ":127732,"פ×Ļ×Ļס":127733,"פ×Ļ×Ļס×ij":127734,"פ×Ļ×Ļס×ij×ķ×§":127735,"ĠкÑĢаÑģ":127736,"лик":127737,"رÙĬØŃ":127738,"×ŀש׾×Ķ":127739,"à¹Ģยีà¹Īย":127740,"à¹Ģยีà¹Īยม":127741,"виÑģ":127742,"омн":127743,"ÄŁun":127744,"ãĥŃãĥ¼ãĥ³":127745,"أتÙĬ":127746,"à¸ķรี":127747,"çͳãģĹ":127748,"تÙħر":127749,"ìĹĪìĬµëĭĪëĭ¤":127750,"ĠÙĪØºÙĬر":127751,"redni":127752,"ĠاÙĦصÙģ":127753,"ĠнаÑģÑĤоÑı":127754,"ĠнаÑģÑĤоÑıÑī":127755,"à¸ķรา":127756,"ĠÑĥÑģлов":127757,"ĠÑĥÑģловиÑı":127758,"ÑĨеп":127759,"×Ķ×Ĺ׾×ĺ":127760,"Ø·ÙĬع":127761,"ĠBakan":127762,"ĠاÙĦرÙĪ":127763,"илÑĮно":127764,"ĠмеÑĤ":127765,"à¸Ķà¸Ńà¸ģ":127766,"ãģĭãĤīãģªãģĦ":127767,"ĠпоÑģÑĤоÑı":127768,"ĠпоÑģÑĤоÑıн":127769,"ĠÑĩаÑģ":127770,"üc":127771,"wró":127772,"бÑĥÑĢ":127773,"ãĥIJãĥĥãĤ¯":127774,"ãĥ©ãĥ³ãĥī":127775,"ĠогÑĢ":127776,"สัà¸į":127777,"สัà¸įà¸įา":127778,"มัà¹Īà¸Ļ":127779,"à¸Ħà¸Ńม":127780,"alık":127781,"Ġнед":127782,"ümüz":127783,"ĠÅĽwie":127784,"ério":127785,"×Ļ×IJ×Ķ":127786,"دÙħات":127787,"ırl":127788,"ĠоÑĤз":127789,"ĠоÑĤзÑĭв":127790,"ä»ĺãģį":127791,"Ġkażde":127792,"миниÑģÑĤ":127793,"ãĤ°ãĥ«":127794,"ë°ĸ":127795,"езн":127796,"اÙĦÙģ":127797,"Ġשק׾":127798,"Ùħض":127799,"ãĥĿãĥ¼ãĥĪ":127800,"ÙħÙĨت":127801,"ÙĤÙĬاÙħ":127802,"Ø´ÙĨ":127803,"×Ļר×ķ×¢":127804,"ãĤŃãĥ£ãĥ³":127805,"доÑĢов":127806,"×ŀ×Ļת×Ļ":127807,"ÙĪÙĦÙĪØ¬":127808,"ÙĥاÙģ":127809,"ĠÑĢазлиÑĩ":127810,"иÑĤеÑĤ":127811,"нолог":127812,"ลà¸ĩà¸Ĺุà¸Ļ":127813,"ĠyaklaÅŁ":127814,"ãĥ¬ãĤ¤":127815,"ê²łëĭ¤":127816,"æ±ĤãĤģ":127817,"رÙĪÙģ":127818,"ĠíĬ":127819,"ĠíĬ¹":127820,"ãģ£ãģıãĤĬ":127821,"à¸Ħวามà¸Ħิà¸Ķ":127822,"×Ķ×Ļס×ĺ":127823,"Ø¥ÙĤ":127824,"ãģ¦ãģĦ":127825,"à¹Ĥà¸Ĭ":127826,"ĠBüyük":127827,"ĠФедеÑĢ":127828,"ÑĨин":127829,"ÑĢова":127830,"ĠاÙĦاÙĤتصاد":127831,"Ġchá":127832,"à¸ĺาà¸Ļ":127833,"ë¥ł":127834,"à¹Ħà¸ķ":127835,"ÃŃpio":127836,"Ùĭا":127837,"ĠобÑıз":127838,"Ùĩج":127839,"Ġì¤ijìļĶ":127840,"ãģ®ãģ§ãģ¯ãģªãģĦ":127841,"باراة":127842,"ãĤ¤ãĥ«":127843,"ĠноÑĢм":127844,"á»īnh":127845,"mö":127846,"möglich":127847,"ÑĨип":127848,"ãĤ¢ãĤ¯":127849,"×Ķ×Ļ":127850,"ÑĨиалÑĮно":127851,"ĠÅĽwi":127852,"تÙĤ":127853,"ĠÑģÑĤоим":127854,"بÙĬعÙĬ":127855,"Ġ׾ש×ŀ":127856,"глÑı":127857,"глÑıд":127858,"ãģ¦ãģıãĤĮ":127859,"ÄĻdzi":127860,"à¸Ĥั":127861,"à¸Ĥัà¹īà¸Ļ":127862,"Ø·ÙĤ":127863,"ĠìĹŃ":127864,"ãģ£ãģ¦ãģĹãģ¾ãģĨ":127865,"ĠdeÄŁerl":127866,"ĠdeÄŁerlendir":127867,"Ġülk":127868,"Ġмног":127869,"à¹ĭ":127870,"ë¿IJ":127871,"ĠУкÑĢа":127872,"ÄŁini":127873,"Ġбезоп":127874,"ĠбезопаÑģ":127875,"à¸Ńà¸Ńà¸ģà¹ģà¸ļà¸ļ":127876,"اظ":127877,"ØŃداث":127878,"леÑĢ":127879,"×Ļ×¥":127880,"×Ļ׳×ĺר׳×ĺ":127881,"larınız":127882,"ØŃÙĬØŃ":127883,"żeli":127884,"à¸Ńัà¸ĩ":127885,"à¸Ńัà¸ĩà¸ģ":127886,"à¸Ńัà¸ĩà¸ģฤษ":127887,"ĠоÑĤлиÑĩ":127888,"ัส":127889,"ëŀį":127890,"ожно":127891,"ãĤ¹ãĥĿ":127892,"ĠÑħоÑĩ":127893,"Ġкап":127894,"еÑĩен":127895,"ØŃÙĦØ©":127896,"ÙĬاÙĩ":127897,"нал":127898,"×ķצר×Ļ×Ŀ":127899,"Ġkald":127900,"åĥį":127901,"ĠاÙĦشخص":127902,"Ġзна":127903,"Ġwzgl":127904,"życz":127905,"ê°Ŀ":127906,"à¸ŀลัà¸ĩ":127907,"íģ¼":127908,"Ġöl":127909,"Ġbụ":127910,"Ø´Ùĩر":127911,"Ġзам":127912,"Ġдев":127913,"×Ļ×ĺת":127914,"تعÙĦÙĤ":127915,"ÙĪÙħØ©":127916,"ãĤĴä½ľ":127917,"ãģįãģ¦":127918,"íĥĿ":127919,"rasında":127920,"ãĤĴæİ¢":127921,"ĠÙħباشر":127922,"راجع":127923,"Ġвозд":127924,"ÙħØŃا":127925,"×ķשר":127926,"ĠиÑģÑĤоÑĢ":127927,"มัà¸ģ":127928,"tıģ":127929,"ثار":127930,"ترÙĨت":127931,"à¹ģà¸Ĥà¹ĩ":127932,"à¹ģà¸Ĥà¹ĩà¸ĩ":127933,"поÑĩ":127934,"Ġ×ij×IJ×ķת":127935,"ë¯Ģ":127936,"ëĿ¼ëıĦ":127937,"à¸Ĭัà¸Ķ":127938,"สà¸ķà¹Į":127939,"ãĥĭãĥĥãĤ¯":127940,"иденÑĤ":127941,"ĠгÑĢÑĥпп":127942,"تخ":127943,"áºł":127944,"ยืà¸Ļ":127945,"ยัà¸Ļ":127946,"óry":127947,"TÃľ":127948,"ãģĹãĤĥ":127949,"ĠпÑĢовед":127950,"лÑıеÑĤ":127951,"ÙħØ®":127952,"ยà¸Ńม":127953,"×Ľ×ł×¡×ª":127954,"ĠاÙĦÙħÙĨت":127955,"Ġolmad":127956,"ר׼×ĸ×Ļ":127957,"ĠвÑģÑĤÑĢ":127958,"ĠиÑģÑģлед":127959,"ÑĤвеÑĢж":127960,"بدÙĪ":127961,"еÑĢÑĤ":127962,"ï»·":127963,"±ħ":127964,"สัมà¸ŀัà¸Ļà¸ĺà¹Į":127965,"ิà¹Īà¸Ļ":127966,"צ×Ļ×ij":127967,"wiÄĻt":127968,"Ġì°¸":127969,"ĠzwiÄħz":127970,"سبÙĪØ¹":127971,"ãĥĥãĤ°":127972,"à¸Ľà¸¥à¸Ńà¸Ķ":127973,"à¸Ľà¸¥à¸Ńà¸Ķà¸łà¸±à¸¢":127974,"ãĤĤãĤĬ":127975,"ÙĤدس":127976,"Ġsprz":127977,"Ġsprzeda":127978,"Ġistedi":127979,"Ġkhu":127980,"Ġден":127981,"ĠkoÅĦ":127982,"Ġ×ij×Ĺ×Ļ":127983,"à¹Ģà¸Ĺà¹īา":127984,"×ķס×Ļ×£":127985,"ãĥĭãĥ¥ãĥ¼":127986,"ĠпÑĢедоÑģÑĤ":127987,"ĠпÑĢедоÑģÑĤав":127988,"à¹Ĥà¸Ł":127989,"év":127990,"ĠاÙĦصØŃ":127991,"صØŃاب":127992,"à¹Ģà¸Īà¹ĩà¸ļ":127993,"влек":127994,"วัà¸ķ":127995,"à¸ĸุ":127996,"ãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ":127997,"ÙĤÙĬÙĤÙĬ":127998,"×ķ×Ĺר":127999,"ÑĭÑĪ":128000,"ĠоÑĤно":128001,"ĠоÑĤноÑĪ":128002,"обилÑĮ":128003,"ÙģØŃ":128004,"ınt":128005,"ıntı":128006,"Ġ׾×ij×ĵ":128007,"íİĺìĿ´ì§Ģ":128008,"ãĥĬãĥ«":128009,"ĠÙħساء":128010,"×Ļ×ĺ×ij":128011,"ÑĮеÑĢ":128012,"ëĦ·":128013,"ÑĭÑĤа":128014,"ĠоÑĩеÑĢ":128015,"à¸Ķืà¹Ī":128016,"à¸Ķืà¹Īม":128017,"ĠNgh":128018,"تعب":128019,"ÙĦاÙĤات":128020,"×ķ׾×ķ×Ĵ×Ļ×Ķ":128021,"ĠìĿ´ê²ĥ":128022,"Ġ×Ķ×ijר":128023,"ìľµ":128024,"à¹Ģà¸Ħลืà¹Īà¸Ńà¸Ļ":128025,"ÙĩØ©":128026,"à¸Īำà¹Ģà¸Ľà¹ĩà¸Ļ":128027,"å¤īãģĪ":128028,"wiÅĽcie":128029,"chod":128030,"chodzÄħ":128031,"вÑĢо":128032,"×ŀ×Ĺ×Ļר":128033,"Ġyı":128034,"Ġyıll":128035,"ì¡Į":128036,"à¹Ħหว":128037,"ãģªãģıãģª":128038,"ĠзавиÑģ":128039,"ĠìĺĪìĪĺ":128040,"Ù쨰":128041,"á»§ng":128042,"à¸ŀุà¸Ĺà¸ĺ":128043,"зн":128044,"layan":128045,"ãĤ¡":128046,"à¸ģà¹ĩà¸ķาม":128047,"ĠsaÄŁlam":128048,"รà¸ĵ":128049,"ĠÑģиÑĤ":128050,"ĠÑģиÑĤÑĥ":128051,"ĠاÙĦتÙĨ":128052,"×Ķ×ĸ":128053,"ĠØ·ÙĪÙĬÙĦ":128054,"taÅĤ":128055,"Ġgörd":128056,"å¤īãĤı":128057,"ëĥ¥":128058,"à¸Ħà¹Īà¸Ńย":128059,"×IJ×ķ×ĺ":128060,"ëħIJ":128061,"ãĥ©ãĥ³ãĤ¹":128062,"วัà¸Ĵ":128063,"วัà¸Ĵà¸Ļ":128064,"ĠoluÅŁ":128065,"פע×ķ׾":128066,"ĠszczegóÅĤ":128067,"à¸Ħาสิ":128068,"à¸Ħาสิà¹Ĥà¸Ļ":128069,"powied":128070,"ĠÑĤеб":128071,"หà¸Ļà¹Īวย":128072,"Ġмил":128073,"ØŃÙĥ":128074,"à¸Ĺà¸Ķ":128075,"ĠмаÑĤеÑĢиал":128076,"ÅĤow":128077,"à¹Ģà¸ģีย":128078,"ĠÑģовеÑĢ":128079,"ãĤ©":128080,"à¸Ľà¸£à¸´":128081,"ĠиÑİ":128082,"наÑĩен":128083,"ÑĢенд":128084,"muÅŁtur":128085,"ĠпÑĢодÑĥк":128086,"зд":128087,"ÑıÑĤи":128088,"ÑıÑĤиÑı":128089,"à¹Ģมีย":128090,"راتÙĬج":128091,"Ġamacı":128092,"ש×ķ׾":128093,"ש×ķ׾×Ĺ":128094,"สะà¸Ńา":128095,"สะà¸Ńาà¸Ķ":128096,"פ×Ĵ×¢":128097,"عبة":128098,"dın":128099,"íħĶ":128100,"Ġ×ŀש×Ĺ×§":128101,"Ġfiyat":128102,"ĠзаÑı":128103,"ĠзаÑıв":128104,"à¹Ĥหล":128105,"à¹Ĥหลà¸Ķ":128106,"à¸ģรุà¸ĩà¹Ģà¸Ĺà¸ŀ":128107,"צ×Ļ×Ļף":128108,"ìļ±":128109,"Ùħب":128110,"Ùħباد":128111,"landır":128112,"ĠвеÑģÑĮ":128113,"Ġhük":128114,"ĠÐĴоз":128115,"ÑĩиÑĤÑĭва":128116,"วล":128117,"×ķצע":128118,"à¸Ĥà¸ĵะà¸Ĺีà¹Ī":128119,"ĠaÅŁaģı":128120,"׾×IJ×ķ×ŀ×Ļ":128121,"trzym":128122,"Ã¤ÃŁig":128123,"owoÅĽci":128124,"ãģĿãĤĤ":128125,"ĠrozwiÄħz":128126,"ĠgÅĤówn":128127,"монÑĤ":128128,"×ŀ×ķ×ŀ":128129,"ĠÑģÑĤан":128130,"ÙĦاÙĤØ©":128131,"prowad":128132,"prowadzi":128133,"ĠÑģоÑģÑĤоÑı":128134,"×Ļ×IJ×ķת":128135,"rı":128136,"gı":128137,"ãĥijãĥij":128138,"ĠналиÑĩ":128139,"×Ķצע":128140,"Ġ׳×Ķ":128141,"à¸Ħัà¸ļ":128142,"عراض":128143,"иж":128144,"ÙĩائÙĬ":128145,"ãĤīãģı":128146,"ожеÑĤ":128147,"ĠобоÑĢ":128148,"ĠобоÑĢÑĥд":128149,"أسÙĦ":128150,"à¹ĩà¸Ķ":128151,"ÑĢÑĥÑĤ":128152,"دÙĬÙħÙĤ":128153,"دÙĬÙħÙĤرا":128154,"Ġjeste":128155,"×ķ×ķ×Ļר":128156,"×ij×ĵ×Ļ×§":128157,"деÑĢжива":128158,"ãģĬãģı":128159,"ewnÄĻtr":128160,"ewnÄĻtrzn":128161,"à¸ŀฤ":128162,"Ġ×IJ×ķ×Ķ":128163,"ת×Ĺ×ķש":128164,"Ġzob":128165,"дÑĥм":128166,"ĠÑģÑĭ":128167,"ÙĬرا":128168,"ĠwiÄĻks":128169,"à¹ģà¸ķà¸ģà¸ķà¹Īาà¸ĩ":128170,"lararas":128171,"lararası":128172,"íĺĢ":128173,"ëī´":128174,"×ķ×Ĵ׾":128175,"ĠоÑĤмеÑĤ":128176,"ĠÑĢан":128177,"تÙĥÙĦ":128178,"иÑĤелÑĮн":128179,"à¸Ľà¸£à¸°à¸§à¸±":128180,"à¸Ľà¸£à¸°à¸§à¸±à¸ķิ":128181,"ìŀĸ":128182,"можно":128183,"pieczeÅĦ":128184,"pieczeÅĦst":128185,"못":128186,"ìĬ¨":128187,"×ŀס×ŀ":128188,"Ủ":128189,"ศิ":128190,"ศิล":128191,"à¸¨à¸´à¸¥à¸Ľ":128192,"ĠÅļw":128193,"ãĥĥãĤ·ãĥ§ãĥ³":128194,"unitÃł":128195,"Ġmieszka":128196,"ĠmieszkaÅĦ":128197,"przed":128198,"przedsi":128199,"przedsiÄĻb":128200,"przedsiÄĻbior":128201,"à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺิ":128202,"à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺà¸´à¸łà¸²à¸ŀ":128203,"ยà¹Ī":128204,"ìķĻ":128205,"รวà¸Ķ":128206,"รวà¸Ķà¹Ģรà¹ĩว":128207,"å½ĵãģŁãĤĬ":128208,"älle":128209,"ÑĥеÑĤÑģÑı":128210,"ãn":128211,"ëłµ":128212,"thè":128213,"ãĤĴåĪ©ç͍":128214,"ìµľ":128215,"íĵ¨":128216,"à¸Ĺัà¸ļ":128217,"าà¸Ħม":128218,"ãģĩ":128219,"ëĤĮ":128220,"à¹Ģà¸Ľà¸¥à¹Īา":128221,"â¦":128222,"ë¾":128223,"êĢ":128224,"êĩ":128225,"â¡":128226,"ðŁŁ":128227,"ãIJ":128228,"âº":128229,"áŃ":128230,"áĻ":128231,"áĵ":128232,"á²":128233,"ðĵı":128234,"á¬":128235,"â¯":128236,"ä¨":128237,"êĿ":128238,"ê«":128239,"ðij":128240,"ðĵĥ":128241,"ðĿħ":128242,"":128244,"":128245,"":128247,"ĠعÙĦÙī":128248,"Ġmá»Ļt":128249,"ĠvỼi":128250,"Ġngưá»Ŀi":128251,"ĠØ¥ÙĦÙī":128252,"Ġnhững":128253,"Ġthá»ĥ":128254,"Ġ×IJ×ķ":128255,"Ġ×¢×Ŀ":128256,"اÙĭ":128257,"Ġà¹ģละ":128258,"ĠÙĦا":128259,"Ġnhư":128260,"ĠاÙĦتÙĬ":128261,"Ġ×Ķ×ķ×IJ":128262,"ĠÄijến":128263,"ĠØ£ÙĪ":128264,"Ġvá»ģ":128265,"ĠlÃłm":128266,"Ġsẽ":128267,"ĠcÅ©ng":128268,"Ġợ":128269,"ĠÄijó":128270,"Ġnhiá»ģu":128271,"Ġtại":128272,"Ġtrên":128273,"Ġ×Ĵ×Ŀ":128274,"ĠnhÃł":128275,"Ġ׼×Ļ":128276,"Ġsá»±":128277,"ĠÄijầu":128278,"Ġbá»ĭ":128279,"ĠÙĩذا":128280,"Ġnhất":128281,"Ġphải":128282,"Ġhiá»ĩn":128283,"Ġdụng":128284,"ĠÄijá»Ļng":128285,"ĠاÙĦÙĦÙĩ":128286,"ĠØĮ":128287,"ĠÙĥÙĦ":128288,"Ġviá»ĩc":128289,"ĠnÄĥm":128290,"Ġthì":128291,"Ġhá»įc":128292,"ĠÙĪØª":128293,"té":128294,"ĠاÙĨ":128295,"Ġtôi":128296,"Ġ×IJ׳×Ļ":128297,"Ġ׾×Ļ":128298,"Ġ×ŀ×ķ":128299,"ĠngÃły":128300,"ĠnÆ°á»Ľc":128301,"Ġ×Ķ×Ļ×IJ":128302,"Ġ×IJ×Ļ":128303,"ĠhÆ¡n":128304,"ĠÙĩذÙĩ":128305,"ĠÙĪÙĬ":128306,"ĠاÙĦذÙĬ":128307,"Ġ×ķ×ŀ":128308,"Ġgiá":128309,"Ġnhân":128310,"ĠchÃŃnh":128311,"Ġmình":128312,"ĠÐĿа":128313,"Ġthế":128314,"Ġ×Ļ×ķתר":128315,"Ġ×IJ×Ŀ":128316,"Ġnên":128317,"Ġhợ":128318,"Ġhợp":128319,"Ġcòn":128320,"ĠÙĩÙĪ":128321,"ĠcÆ¡":128322,"Ġrất":128323,"ĠViá»ĩt":128324,"Ġبعد":128325,"Ġש×Ļ":128326,"Ġthá»Ŀi":128327,"Ġcách":128328,"ĠÄijá»ĵng":128329,"Ġно":128330,"Ġtrưá»Ŀng":128331,"ØŁ":128332,"ĠÄijá»ĭnh":128333,"ĠÄijiá»ģu":128334,"×Ļ×Ļ×Ŀ":128335,"Ġthá»±c":128336,"nın":128337,"Ġhình":128338,"Ġnói":128339,"Ġcùng":128340,"Ġ×Ķ×Ķ":128341,"ĠØ¥ÙĨ":128342,"Ġ×IJ×ij׾":128343,"Ġnhưng":128344,"Ġbiết":128345,"Ġже":128346,"Ġchúng":128347,"ĠÄijang":128348,"ĠذÙĦÙĥ":128349,"Ġlên":128350,"Ġkhách":128351,"ĠnÃło":128352,"Ġsá»Ń":128353,"Ġkhác":128354,"Ġë°ı":128355,"Ġlý":128356,"×Ļ×Ļ":128357,"ĠÄijây":128358,"Ġ׾×ŀ":128359,"Ġcần":128360,"Ġtrình":128361,"Ġphát":128362,"ãģ«ãĤĤ":128363,"по":128364,"ĠnÄĥng":128365,"Ġbá»Ļ":128366,"Ġvụ":128367,"ĠÄijá»Ļ":128368,"Ñĩе":128369,"ĠnháºŃn":128370,"ĠtrÆ°á»Ľc":128371,"Ġ×¢×ĵ":128372,"ĠhÃłnh":128373,"ĠØ®ÙĦاÙĦ":128374,"Ġlượng":128375,"Ġcấp":128376,"Ġtá»±":128377,"Ġvì":128378,"Ġtư":128379,"Ġchất":128380,"Ġ׼×ŀ×ķ":128381,"Ġgì":128382,"Ġש׳":128383,"Ġtế":128384,"ת×ķ":128385,"Ġnghiá»ĩp":128386,"Ġmặt":128387,"ĠÙĥÙħا":128388,"Ġ×ij×Ļף":128389,"Ġרק":128390,"Ġthấy":128391,"Ġmáy":128392,"ĠÙģÙī":128393,"Ġdân":128394,"Ġ×IJ×Ĺ×ĵ":128395,"Ġtâm":128396,"Ġ׼×ļ":128397,"Ġ׾×ķ":128398,"во":128399,"Ġtác":128400,"ĠtoÃłn":128401,"ĠÙĪÙħ":128402,"Ġkết":128403,"Ġหรืà¸Ń":128404,"ĠÙĪØ§ÙĦÙħ":128405,"ĠÄijiá»ĥm":128406,"Ġ×ĸ×ķ":128407,"Ġ×ij×ķ":128408,"׼×ķת":128409,"Ġhá»Ļi":128410,"Ġbằng":128411,"تÙĩا":128412,"Ġ׼×ĵ×Ļ":128413,"Ġ×Ķ×Ŀ":128414,"Ġxuất":128415,"ĠÙĤد":128416,"Ġbảo":128417,"Ġtá»ijt":128418,"Ġtình":128419,"ĠÙĩÙĬ":128420,"ĠÄijá»iji":128421,"Ġthiết":128422,"Ġhiá»ĩu":128423,"Ġtiếp":128424,"Ġtạo":128425,"ת×Ķ":128426,"Ġchá»§":128427,"oÅĽÄĩ":128428,"Ġgiú":128429,"Ġgiúp":128430,"Ġý":128431,"Ġquả":128432,"Ġloại":128433,"Ġcô":128434,"Ġô":128435,"Ġông":128436,"Ġ×Ķ×ķ":128437,"ĠاÙĦÙĬÙĪÙħ":128438,"ĠtÃŃnh":128439,"га":128440,"Ġphòng":128441,"ĠÄĥn":128442,"ĠعاÙħ":128443,"Ġvá»ĭ":128444,"larını":128445,"rÃŃa":128446,"ĠtỼi":128447,"ĠÄijưá»Ŀng":128448,"ĠgiỼi":128449,"Ġbản":128450,"Ġcầu":128451,"Ġnhiên":128452,"Ġbá»ĩnh":128453,"Ġthưá»Ŀng":128454,"Ġ×IJ×Ļף":128455,"ĠÄijá»ģ":128456,"Ġhá»ĩ":128457,"Ġ×Ļשר×IJ׾":128458,"Ġquá":128459,"ĠÐĹа":128460,"ãģ®ãģ§ãģĻãģĮ":128461,"ĠÐŁÑĢи":128462,"Ġphần":128463,"ĠÙĪÙĦا":128464,"ĠlỼn":128465,"Ġtrá»ĭ":128466,"Ġcảm":128467,"Ġмо":128468,"Ġdùng":128469,"ĠاÙĦÙī":128470,"ĠعÙĦÙĬÙĩ":128471,"ĠìŀĪìĬµëĭĪëĭ¤":128472,"ÙĬÙĤ":128473,"ĠÙĤبÙĦ":128474,"Ġhoặc":128475,"ĠØŃÙĬØ«":128476,"Ġà¸Ĺีà¹Ī":128477,"ĠغÙĬر":128478,"ĠÄijại":128479,"Ġsá»ijng":128480,"нÑĭми":128481,"Ġthức":128482,"Ġפ×Ļ":128483,"ĠÄijiá»ĩn":128484,"ãģªãģĭãģ£ãģŁ":128485,"Ġgiải":128486,"Ġvẫn":128487,"ĠиÑħ":128488,"Ġönce":128489,"ĠváºŃy":128490,"Ġmuá»ijn":128491,"Ġảnh":128492,"à¹ĥà¸Ļà¸ģาร":128493,"ĠQuá»ijc":128494,"Ġkế":128495,"׳×IJ":128496,"Ġס×Ļ":128497,"Ġyêu":128498,"ãģ®ãģĭ":128499,"ĠÄijẹ":128500,"ĠÄijẹp":128501,"Ġchức":128502,"Ġyıl":128503,"ĠTürkiye":128504,"dé":128505,"ĠÙĤاÙĦ":128506,"Ġdá»ĭch":128507,"ĠolduÄŁu":128508,"Ġchá»įn":128509,"ĠتÙħ":128510,"หà¸Ļึà¹Īà¸ĩ":128511,"ãģķãĤĮãģŁ":128512,"Ġpháp":128513,"ìĽĶ":128514,"Ġtiá»ģn":128515,"ãģĹãģ¾ãģĹãģŁ":128516,"Ġש׾×IJ":128517,"ÙĦØ©":128518,"Ġ׾פ׳×Ļ":128519,"Ġ×ij×Ļת":128520,"ĠHÃł":128521,"ĠØŃت":128522,"ĠØŃتÙī":128523,"Ġ×¢×ķ×ĵ":128524,"Ġnó":128525,"Ġtháng":128526,"à¹Ģลืà¸Ńà¸ģ":128527,"ר×Ķ":128528,"ĠtÄĥng":128529,"Ġcái":128530,"Ġtriá»ĥn":128531,"Ġ×IJ×ķת×ķ":128532,"ìłģìĿ¸":128533,"ĠCông":128534,"Ġ׾×Ķ×Ļ×ķת":128535,"Ġгода":128536,"иÑİ":128537,"Ġبعض":128538,"Ġà¸ģาร":128539,"èī¯ãģĦ":128540,"ÙĪØª":128541,"Ġliên":128542,"ĠÐĿо":128543,"ĠÐĿе":128544,"çļĦãģª":128545,"ĠÙħت":128546,"ĠÑĤакже":128547,"ĠкоÑĤоÑĢÑĭе":128548,"Ġ×Ļ×ĵ×Ļ":128549,"Ġtrá»įng":128550,"ãĤµãĤ¤ãĥĪ":128551,"ìłģìľ¼ë¡ľ":128552,"ĠtáºŃp":128553,"Ġש׾×Ļ":128554,"íķĺê²Į":128555,"ĠtÃłi":128556,"ĠЯ":128557,"Ġrá»ĵi":128558,"اÙĥ":128559,"Ġthương":128560,"Ġ×Ķ×ĸ×Ķ":128561,"ĠÙĪÙħÙĨ":128562,"à¸Ĺีà¹Īมี":128563,"Ġcuá»Ļc":128564,"Ġbüyük":128565,"ãģ¨ãģĭ":128566,"Ġ×ij×Ļ×ķתר":128567,"Ġlần":128568,"Ġgöre":128569,"Ġtrợ":128570,"Ġ×ĺ×ķ×ij":128571,"ÑĤÑĮÑģÑı":128572,"Ġthá»ijng":128573,"Ġ׼ש":128574,"Ġtiêu":128575,"Ġ×ŀ×IJ×ķ×ĵ":128576,"ØĽ":128577,"kÄħ":128578,"Ġà¹ĥà¸Ļ":128579,"Ġvấn":128580,"Ġש׾×ķ":128581,"ĠÄijá»ģu":128582,"ÙģØª":128583,"Ġê²ĥìĿ´":128584,"Ġhóa":128585,"ĠاÙĦعاÙħ":128586,"ĠÙĬÙĪÙħ":128587,"кой":128588,"Ġbiá»ĩt":128589,"ÑģÑĤо":128590,"Ġ×Ķ×Ļ×ķ":128591,"à¸Ĺีà¹Īà¸Īะ":128592,"Ġ×ĵ×Ļ":128593,"Ġ×IJ×ļ":128594,"Ġán":128595,"صÙĪØ±":128596,"ĠtrÃŃ":128597,"ĠÐŁÑĢо":128598,"Ġlá»±c":128599,"ãģĹãģ¦ãģĦãģ¾ãģĻ":128600,"ĠbÃłi":128601,"Ġ×ĸ×IJת":128602,"Ġbáo":128603,"à¸ļà¸Ļ":128604,"ĠëĮĢíķľ":128605,"Ġtiế":128606,"Ġtiếng":128607,"Ġbên":128608,"ãģķãĤĮãĤĭ":128609,"sión":128610,"Ġtìm":128611,"×¢×ķ":128612,"mé":128613,"ниÑı":128614,"ãģ»ãģ©":128615,"Ġà¹Ģà¸ŀราะ":128616,"بة":128617,"Ġë¶Ħ":128618,"Ġ×IJ×ĸ":128619,"à¸Ĺà¹Īาà¸Ļ":128620,"ת×Ŀ":128621,"Ġthêm":128622,"Ġhoạt":128623,"yı":128624,"×ĸ×ķ":128625,"Ġgiá»Ŀ":128626,"Ġbán":128627,"à¸Ĥาย":128628,"Ñĩа":128629,"Ġà¹Ĩ":128630,"ĠاÙĦÙħت":128631,"ĠоÑĩенÑĮ":128632,"Ġbất":128633,"Ġtrẻ":128634,"ÑĤÑĢ":128635,"ĠØ£ÙĨÙĩ":128636,"ĠØ«Ùħ":128637,"Ġ׼×ŀ×Ķ":128638,"Ġkhó":128639,"Ġrằng":128640,"ĠÙĪÙģÙĬ":128641,"ний":128642,"ĠhoÃłn":128643,"tó":128644,"Ġ×IJשר":128645,"ĠìĥĿê°ģ":128646,"Ñģа":128647,"Ġ׼×ijר":128648,"ĠÑįÑĤом":128649,"larının":128650,"Ġchưa":128651,"зи":128652,"Ġdẫn":128653,"ĠÐļак":128654,"جÙĪ":128655,"ĠбÑĭло":128656,"ĠÙĬت":128657,"nı":128658,"ÅĤam":128659,"ĠÙĪÙĩÙĪ":128660,"×ij×ķ":128661,"пи":128662,"רת":128663,"Ġquá»ijc":128664,"жд":128665,"ĠÄijÆ¡n":128666,"Ùĥتب":128667,"Ġmắt":128668,"ระà¸ļ":128669,"ระà¸ļà¸ļ":128670,"ĠÙĥاÙĨت":128671,"Ġthân":128672,"สิà¸Ļà¸Ħà¹īา":128673,"×Ĵ×Ļ":128674,"Ġphương":128675,"à¹Ħมà¹Īà¹Ħà¸Ķà¹ī":128676,"ĠìĦ±":128677,"ĠCác":128678,"Ġ×Ķ×ŀ×ķ":128679,"ĠÑĤем":128680,"Ġ×ĵ×ķ":128681,"à¸Ńะà¹Ħร":128682,"ĠvÄĥn":128683,"ãģªãģ®ãģ§":128684,"ĠNá»Ļi":128685,"Ġ×¢×ķ":128686,"ãĤīãĤĮãĤĭ":128687,"Ġsáng":128688,"Ġgöster":128689,"ãģĵãģ¨ãĤĴ":128690,"Ġtarafından":128691,"Ġма":128692,"ĠпоÑģле":128693,"Ġ׳×Ļת":128694,"Ġ׳×Ļ×ª×Ł":128695,"ĠлеÑĤ":128696,"Ġ׾׳×ķ":128697,"ÑģÑģ":128698,"Ġ×Ļ×ķ":128699,"пе":128700,"ĠÙĪÙĦÙĥ":128701,"ĠÙĪÙĦÙĥÙĨ":128702,"ĠngoÃłi":128703,"ĠÄijá»ĭa":128704,"rzÄħd":128705,"dziaÅĤ":128706,"ĠÙħر":128707,"иÑĤÑĮÑģÑı":128708,"Ġ×IJ×Ĺר×Ļ":128709,"Ġ׾׼׾":128710,"à¸Ĥà¹īà¸Ńม":128711,"à¸Ĥà¹īà¸Ńมูล":128712,"Ġбол":128713,"Ġболее":128714,"جÙħع":128715,"леÑĤ":128716,"Ġlá»ĭch":128717,"ĠÙħØ«ÙĦ":128718,"Ġê·¸ë¦¬ê³ł":128719,"Ġthứ":128720,"ĠdeÄŁil":128721,"ÙĪØŃ":128722,"Ġש׾×ļ":128723,"ĠÙħØŃÙħد":128724,"Ġnếu":128725,"ĠÄijá»ķi":128726,"Ġvừa":128727,"Ġmá»įi":128728,"Ġони":128729,"Ġlúc":128730,"ĠÙĬÙĥÙĪÙĨ":128731,"ì§Ī":128732,"Ġש׾׳×ķ":128733,"ĠÐĶо":128734,"Ġש׳×Ļ":128735,"ลิ":128736,"×IJפשר":128737,"Ġsức":128738,"ê¶Į":128739,"Ġứng":128740,"à¹Ħมà¹Īมี":128741,"Ø·ÙĦب":128742,"ĠÑĩем":128743,"Ġchuyên":128744,"ĠthÃŃch":128745,"Ġ×ķ×Ļ":128746,"íķ©":128747,"ĠÙħصر":128748,"до":128749,"ĠÄijất":128750,"Ġchế":128751,"à¸Ĭืà¹Īà¸Ń":128752,"Ġìĭł":128753,"Ġإذا":128754,"ĠرئÙĬس":128755,"Ġש×Ļש":128756,"Ġgiảm":128757,"Ñģка":128758,"larında":128759,"Ġsợ":128760,"ĠtÃŃch":128761,"ĠÙĦÙĥÙĨ":128762,"ĠبÙħ":128763,"×¢×ķ×ij":128764,"×¢×ķ×ij×ĵ":128765,"ÅĤÄħcz":128766,"larına":128767,"Ġש×Ŀ":128768,"ĠÙĦت":128769,"Ġש×Ķ×ķ×IJ":128770,"tów":128771,"Ġëĭ¤ë¥¸":128772,"ĠØ£Ùĥثر":128773,"ãģ®ãģ§ãģĻ":128774,"׼×Ļ×Ŀ":128775,"ĠolduÄŁunu":128776,"ãģĭãģª":128777,"ãĤĤãģĨ":128778,"ÙĬØŃ":128779,"Ġnhìn":128780,"Ġnghá»ĩ":128781,"ãģ«ãģªãģ£ãģ¦":128782,"па":128783,"Ġquyết":128784,"ÙĦÙĤ":128785,"tá":128786,"Ġluôn":128787,"ĠÄijặc":128788,"Ġ×IJר":128789,"Ġtuá»ķi":128790,"são":128791,"ìϏ":128792,"رد":128793,"ĠبÙĩا":128794,"Ġ×Ķ×Ļ×ķ×Ŀ":128795,"×ķ×ķ×Ļ":128796,"ãģ§ãģĻãģŃ":128797,"ĠÑĤого":128798,"Ġthá»§":128799,"ãģĹãģŁãģĦ":128800,"رÙĤ":128801,"Ġbắt":128802,"гÑĥ":128803,"Ġtá»Ń":128804,"ÑĪа":128805,"Ġà¸Ľà¸µ":128806,"Ġ×Ķ×IJ×Ŀ":128807,"íı¬":128808,"ża":128809,"Ġ×IJת×Ķ":128810,"Ġná»Ļi":128811,"ĠphÃŃ":128812,"ĠÅŁekilde":128813,"Ġlá»Ŀi":128814,"dıģı":128815,"Ġ׼×IJף":128816,"Ġtüm":128817,"Ġmạnh":128818,"ĠMỹ":128819,"ãģĿãĤĵãģª":128820,"Ġnhá»ı":128821,"ãģªãģĮãĤī":128822,"Ġbình":128823,"ıp":128824,"à¸ŀา":128825,"ĠÄijánh":128826,"ĠÙĪÙĦ":128827,"ר×ķת":128828,"Ġ×IJ×Ļ×ļ":128829,"Ġchuyá»ĥn":128830,"Ùĥا":128831,"ãĤĮãĤĭ":128832,"à¹ģมà¹Ī":128833,"ãĤĪãģı":128834,"ĠÙĪÙĤد":128835,"íĸĪëĭ¤":128836,"ĠnÆ¡i":128837,"ãģ«ãĤĪãģ£ãģ¦":128838,"Ġviết":128839,"Ġà¹Ģà¸ŀืà¹Īà¸Ń":128840,"ëIJĺëĬĶ":128841,"ادÙĬ":128842,"ĠÙ쨥ÙĨ":128843,"ì¦Ŀ":128844,"ĠÄijặt":128845,"ĠhÆ°á»Ľng":128846,"Ġxã":128847,"Ġönemli":128848,"ãģłãģ¨":128849,"Ġmẹ":128850,"Ġ×ij×Ļ":128851,"Ġ×ĵ×ijר":128852,"ĠváºŃt":128853,"ĠÄijạo":128854,"Ġdá»±ng":128855,"ĠÑĤом":128856,"ĠÙģÙĬÙĩا":128857,"ĠجÙħÙĬع":128858,"ĠthuáºŃt":128859,"stÄĻp":128860,"Ġtiết":128861,"Ø´ÙĬ":128862,"ĠеÑīе":128863,"ãģĻãĤĭãģ¨":128864,"ĠmÃłu":128865,"ĠÑįÑĤого":128866,"Ġvô":128867,"ĠÐŃÑĤо":128868,"ĠtháºŃt":128869,"Ġnữa":128870,"Ġbiến":128871,"Ġnữ":128872,"Ġ׾׼×Ŀ":128873,"×Ļ×Ļף":128874,"Ġست":128875,"ĠÐŀÑĤ":128876,"Ġphụ":128877,"ê¹Įì§Ģ":128878,"Ġ׾×ļ":128879,"Ġkỳ":128880,"à¹ĥà¸Ħร":128881,"Ġgây":128882,"ĠÙĦÙĦÙħ":128883,"Ġtục":128884,"تÙĬÙĨ":128885,"Ġtrợ":128886,"Ġ׾פ×Ļ":128887,"Ġbá»ij":128888,"ĠÐļа":128889,"ĠÄijình":128890,"owÄħ":128891,"sında":128892,"Ġkhiến":128893,"sız":128894,"Ġкогда":128895,"×¡×ľ":128896,"ĠбÑĭл":128897,"à¸Ļà¹īà¸Ńย":128898,"обÑĢаз":128899,"Ġê²ĥìĿ´ëĭ¤":128900,"ëĵ¤ìĿĢ":128901,"ãģ¸ãģ®":128902,"Ġà¹Ģมืà¹Īà¸Ń":128903,"Ġphục":128904,"Ġ×Ĺ׾ק":128905,"Ġhết":128906,"ĠÄija":128907,"à¹Ģà¸Ķà¹ĩà¸ģ":128908,"íĺķ":128909,"lÃŃ":128910,"ê¸ī":128911,"Ġعدد":128912,"ĠÄijá»ĵ":128913,"Ġgần":128914,"Ġ×Ļ×ķ×Ŀ":128915,"ĠsÄ©":128916,"ÑĢÑıд":128917,"Ġquyá»ģn":128918,"Ġ×IJ׾×IJ":128919,"ÙĩÙħا":128920,"׳×Ļ×Ķ":128921,"׾×ķת":128922,"Ġ×Ķר×ij×Ķ":128923,"Ġtiên":128924,"Ġalın":128925,"Ġdá»ħ":128926,"人ãģĮ":128927,"ноÑģ":128928,"лÑģÑı":128929,"ĠÄijưa":128930,"สาว":128931,"иÑĢован":128932,"Ġ×ŀספר":128933,"×Ĵף":128934,"Ġkiến":128935,"ĠШ":128936,"pé":128937,"бÑĥ":128938,"овой":128939,"ба":128940,"ĠØ¥ÙĦا":128941,"×IJ׾×Ļ":128942,"Ġxây":128943,"Ġbợi":128944,"Ġש×ķ":128945,"人ãģ®":128946,"×§×Ļ×Ŀ":128947,"à¹Ģà¸Ķืà¸Ńà¸Ļ":128948,"Ġkhá":128949,"Ġ×ķ׾×Ķ":128950,"×ĵ×ķת":128951,"Ġ×¢×ij×ķר":128952,"ĠبشÙĥÙĦ":128953,"ĠÙĩÙĨاÙĥ":128954,"ÑĤÑĢа":128955,"ĠíķĺëĬĶ":128956,"รà¸Ńà¸ļ":128957,"owaÅĤ":128958,"hé":128959,"Ġdiá»ħn":128960,"Ġ×Ķ׼׾":128961,"Ġأس":128962,"Ġchuyá»ĩn":128963,"ระà¸Ķัà¸ļ":128964,"ĠNhững":128965,"Ġ×IJ×Ĺת":128966,"ĠØŃÙĪÙĦ":128967,"лов":128968,"׳ר":128969,"Ġ×ķ׳":128970,"ĠchÆ¡i":128971,"Ġiçinde":128972,"ÑģÑĤвÑĥ":128973,"Ġphá»ij":128974,"ĠÑģÑĥ":128975,"ç§ģãģ¯":128976,"Ġchứng":128977,"Ġvá»±c":128978,"à¹ģà¸Ń":128979,"ĠláºŃp":128980,"Ġtừng":128981,"å°ijãģĹ":128982,"ĠNguy":128983,"ĠNguyá»ħn":128984,"ĠÙģÙĬÙĩ":128985,"Ġба":128986,"×Ļ×Ļת":128987,"Ġ×ľ×¢×©×ķת":128988,"Ġ×ŀ׼":128989,"Ġnghiá»ĩm":128990,"Ġмного":128991,"Ġее":128992,"ëIJĺìĸ´":128993,"Ġlợi":128994,"Ġ׾׾×IJ":128995,"Ġ׼ף":128996,"ĠchÃŃ":128997,"ãģ§ãģ®":128998,"×Ĺ×ķ":128999,"ש×ķ×Ŀ":129000,"Ġ×ŀר":129001,"ĠÐĶлÑı":129002,"Åģ":129003,"Ġ׼×IJשר":129004,"ĠMá»Ļt":129005,"ĠÙĪØ§ÙĦت":129006,"ĠìĿ´ëٰ":129007,"ÅŁa":129008,"Ġchiến":129009,"Ġarasında":129010,"Ġ×ij×IJתר":129011,"ãģķãĤĮãģ¦ãģĦãĤĭ":129012,"Ø´ÙĥÙĦ":129013,"Ġtượng":129014,"Ġتت":129015,"ĠCó":129016,"Ġbá»ı":129017,"Ġtá»īnh":129018,"ĠkhÃŃ":129019,"ĠпÑĢоÑģÑĤ":129020,"ĠпÑĢоÑģÑĤо":129021,"ĠÙĪÙĤاÙĦ":129022,"Ġgiáo":129023,"ĠNếu":129024,"×IJ×ŀר":129025,"×¢×ł×Ļ×Ļף":129026,"íݸ":129027,"ÙĩدÙģ":129028,"ĠBá»Ļ":129029,"ĠbÃłn":129030,"Ġnguyên":129031,"Ġgüzel":129032,"สาย":129033,"ì²ľ":129034,"×ŀ×ķר":129035,"Ġphân":129036,"ספק":129037,"×§×ij׾":129038,"ĠاÙĦÙħتØŃ":129039,"ĠاÙĦÙħتØŃدة":129040,"ائد":129041,"Ġ×IJ×ŀר":129042,"ĠkiÅŁi":129043,"ì¤Ģ":129044,"Ġtruyá»ģn":129045,"ĠÙĦÙĩا":129046,"ĠÐľÐ°":129047,"à¸ļริษ":129048,"à¸ļริษั":129049,"à¸ļริษัà¸Ĺ":129050,"Ġש׳×Ļ×Ŀ":129051,"ĠменÑı":129052,"ÅŁe":129053,"Ġdiá»ĩn":129054,"Ġ×IJ׳×Ĺ׳×ķ":129055,"kü":129056,"Ġcá»ķ":129057,"Ġmá»Ĺi":129058,"wä":129059,"ÙħÙĬ":129060,"Ġhiá»ĥu":129061,"ëĭ¬":129062,"Ġ×Ķ×Ĺ׾":129063,"Ġtên":129064,"Ġkiá»ĩn":129065,"ÙĨÙĤÙĦ":129066,"Ġvá»ĩ":129067,"×ĵת":129068,"ĠÐłÐ¾ÑģÑģии":129069,"лÑĥ":129070,"ĠاÙĦعربÙĬØ©":129071,"ĠطرÙĬÙĤ":129072,"Ġ×Ķ×ij×Ļת":129073,"ÑģеÑĢ":129074,"Ġмне":129075,"äu":129076,"Ġtriá»ĩu":129077,"ĠÄijá»§":129078,"Ġר×ij":129079,"تÙĩÙħ":129080,"à¸ĭี":129081,"Ġì§Ģê¸Ī":129082,"liÅĽmy":129083,"دعÙħ":129084,"ãģłãĤįãģĨ":129085,"Ñģкие":129086,"Ġhá»ıi":129087,"Ġ×§×ķ":129088,"ÑĢÑĥÑģ":129089,"ÙĨظر":129090,"ãģ®ãĤĤ":129091,"Ġ×Ķ׼×Ļ":129092,"ĠìĽIJ":129093,"ÙĪÙĩ":129094,"ĠÙĪÙİ":129095,"ĠBạn":129096,"плаÑĤ":129097,"Ġ×ŀ×ŀש":129098,"лÑİб":129099,"ĠнÑĥжно":129100,"Ġthư":129101,"ãģµ":129102,"ãģıãĤīãģĦ":129103,"رش":129104,"ר×ķ×Ĺ":129105,"ĠÙĬتÙħ":129106,"Ġצר×Ļ×ļ":129107,"Ġphá":129108,"มà¸Ńà¸ĩ":129109,"Ġ×ij×IJ×ķפף":129110,"Ġcảnh":129111,"Ġíķľëĭ¤":129112,"Ġ×Ķ×ŀת":129113,"à¸ķà¹Īาà¸ĩà¹Ĩ":129114,"มีà¸ģาร":129115,"ÑģкиÑħ":129116,"ĠÐĴÑģе":129117,"ĠاÙĪ":129118,"جÙĬ":129119,"ãģĵãģ¨ãģ¯":129120,"ĠdÃłi":129121,"Ġhá»ĵ":129122,"èĩªåĪĨãģ®":129123,"à¹Ħหà¸Ļ":129124,"ëĵ¤ìĿĦ":129125,"ĠVÄĥn":129126,"Ġдаж":129127,"Ġдаже":129128,"Ñĭми":129129,"лаÑģÑĮ":129130,"ÙĬÙĪÙĨ":129131,"ÙĨÙĪ":129132,"có":129133,"ãģĹãģ¦ãģĦãģŁ":129134,"ãģłãģĭãĤī":129135,"طاÙĦب":129136,"Ġcá»Ńa":129137,"пÑĢоÑģ":129138,"ãģªãģ©ãģ®":129139,"รุà¹Īà¸Ļ":129140,"Ġchiếc":129141,"лÑĭ":129142,"ĠÑıвлÑıеÑĤÑģÑı":129143,"Ġná»ķi":129144,"ãģ®ãģĬ":129145,"Ġ×IJת×Ŀ":129146,"ĠëķĮ문ìĹIJ":129147,"à¸ģลาà¸ĩ":129148,"ĠbaÅŁka":129149,"ìĦĿ":129150,"ĠÑĨел":129151,"ÙģÙĤ":129152,"ãģ«ãĤĪãĤĭ":129153,"ÙĤا":129154,"Ġçıkar":129155,"Ġcứu":129156,"طا":129157,"Ġשת":129158,"à¹Ĥà¸Ħ":129159,"Ġ×ŀ׾":129160,"Ġ×Ķפר":129161,"Ġгде":129162,"Ġخط":129163,"åīįãģ«":129164,"cjÄĻ":129165,"Ġ×Ĺש×ķ×ij":129166,"ר×Ĵ×¢":129167,"Ġkhoảng":129168,"ĠÄijá»Ŀi":129169,"ĠÐłÐµ":129170,"Ġона":129171,"Ġ×IJ׳×ķ":129172,"ãģ®ãģ«":129173,"ĠاÙĦذÙĬÙĨ":129174,"кÑĥп":129175,"ãĤµãĥ¼ãĥ":129176,"ãĤµãĥ¼ãĥĵ":129177,"ãĤµãĥ¼ãĥĵãĤ¹":129178,"вал":129179,"ге":129180,"Ġgiữa":129181,"ĠKhông":129182,"ĠâĹĭ":129183,"à¸ģลุà¹Īม":129184,"ĠÙħÙĨذ":129185,"à¸Ńà¹Īาà¸Ļ":129186,"ĠÑģпоÑģоб":129187,"ĠÄijá»Ļi":129188,"ĠdiÄŁer":129189,"Ġà¸ĸà¹īา":129190,"ÙħØ«ÙĦ":129191,"Ġ×Ķ×IJ×Ļ":129192,"ĠدÙĪÙĨ":129193,"ÙĬراÙĨ":129194,"Ñīи":129195,"بÙĨاء":129196,"Ġآخر":129197,"ظÙĩر":129198,"Ġ×ij׼":129199,"ĠاÙĦÙħع":129200,"ãĥĴ":129201,"Ġtất":129202,"Ġmục":129203,"ĠdoÄŁru":129204,"ãģŁãĤī":129205,"Ġס×ķ":129206,"Ġxác":129207,"รà¸Ń":129208,"ĠcÄĥn":129209,"Ġонл":129210,"Ġонлайн":129211,"Ġký":129212,"Ġchân":129213,"Ġà¹Ħมà¹Ī":129214,"اØŃØ©":129215,"rán":129216,"׳×Ļ×Ļ×Ŀ":129217,"Ġ×ijף":129218,"ĠÐĸ":129219,"à¸ķรà¸ĩ":129220,"дÑĭ":129221,"Ġsắc":129222,"ÙĦت":129223,"ãĥŃãĥ¼":129224,"ĠÙĦÙĨ":129225,"Ġר×ķ":129226,"ĠdÆ°á»Ľi":129227,"à¹Ģà¸ĺ":129228,"à¹Ģà¸ĺà¸Ń":129229,"eÄŁi":129230,"Ġ×ķש":129231,"ĠÙĦØ£":129232,"Ġgặp":129233,"Ġcá»ij":129234,"ãģ¨ãģ¦ãĤĤ":129235,"رÙĪØ³":129236,"Ġ׾×Ķ×Ļ":129237,"Ġ본":129238,"ä¸ĬãģĴ":129239,"Ġmức":129240,"Ñħа":129241,"Ġìŀ¬":129242,"à¸īัà¸Ļ":129243,"ÑĢÑĥж":129244,"Ġaçık":129245,"ÙĪØ§ÙĦ":129246,"Ġ×ĸ×ŀף":129247,"人ãģ¯":129248,"عÙĬÙĨ":129249,"ÑıÑħ":129250,"Ġ×Ĵ×ĵ×ķ׾":129251,"ר×ķ×ij":129252,"gó":129253,"ëĿ¼ê³ł":129254,"ĠarkadaÅŁ":129255,"ÙĨشر":129256,"ĠгодÑĥ":129257,"ĠболÑĮÑĪе":129258,"ãģ¡ãĤĩãģ£ãģ¨":129259,"Ġcâu":129260,"Ġsát":129261,"íͼ":129262,"Ġtiến":129263,"íķ´ìķ¼":129264,"ĠÙĪØ£ÙĨ":129265,"à¸Ļาà¸Ļ":129266,"Ġ×ij×IJ×ŀצע":129267,"Ġ×ij×IJ×ŀצע×ķת":129268,"Ġ׾ר":129269,"Ġquản":129270,"ĠÙĪØ§ÙĦØ£":129271,"Ġ×IJ×ķת×Ķ":129272,"Ġìĸ´ëĸ¤":129273,"Ġê²ĥìĿĢ":129274,"ØŃسÙĨ":129275,"Ġmất":129276,"à¸Ħูà¹Ī":129277,"ãĥ¬ãĥ¼":129278,"ĠÐĶа":129279,"Ġolması":129280,"Ġthuá»Ļc":129281,"׳×Ĺ":129282,"íĨł":129283,"Ġsöyle":129284,"ãģĿãģĨãģ§ãģĻ":129285,"ĠتÙĥÙĪÙĨ":129286,"лÑĥÑĩ":129287,"׾×Ļ×ļ":129288,"ĠØ£ØŃد":129289,"лиÑģÑĮ":129290,"ĠвÑģего":129291,"Ġ×Ķר×ij":129292,"Ġ못":129293,"oÄŁ":129294,"oÄŁlu":129295,"ĠìĦł":129296,"ĠкаÑĢ":129297,"à¸łà¸²à¸Ħ":129298,"eÅĦ":129299,"Ġà¸ģà¹ĩ":129300,"Ġaynı":129301,"ĠbÃł":129302,"ãģªãĤĵãģ¦":129303,"Ġ모ëĵł":129304,"ÙĤرار":129305,"ãģĹãģªãģĦ":129306,"ĠÐĴо":129307,"ĠÙĪÙĩÙĬ":129308,"ники":129309,"ãĤĮãģŁ":129310,"Ġchuẩn":129311,"רע":129312,"Ù쨱ÙĬÙĤ":129313,"ãĤĴåıĹãģij":129314,"ĠÄijúng":129315,"бе":129316,"׼×ķ×Ĺ":129317,"пÑĥ":129318,"Ġ×ķ×Ĵ×Ŀ":129319,"×ŀ׳×Ļ":129320,"íĸ¥":129321,"צ×Ļ×Ŀ":129322,"à¸ĭิ":129323,"ÙĩÙĨ":129324,"нем":129325,"Ġ×ij×ij×Ļת":129326,"رع":129327,"Ġส":129328,"ĠÄIJÃł":129329,"íķĺëĭ¤":129330,"Ġấy":129331,"×Ĺ×ķ×ĵ":129332,"×Ĺ×ķ×ĵש":129333,"ĠÑĩеÑĢез":129334,"Ñĥл":129335,"ĠBình":129336,"Ġê²ĥìĿĦ":129337,"Ġ×Ĵר":129338,"ä»ĺãģij":129339,"×Ĺ׾ק":129340,"ĠتÙĦÙĥ":129341,"à¹ĥสà¹Ī":129342,"szÄħ":129343,"ÙĤاÙħ":129344,"دÙĪØ±":129345,"ĠÙģÙĤØ·":129346,"Ġhữu":129347,"ĠмогÑĥÑĤ":129348,"Ġgá»įi":129349,"Ġקר":129350,"à¸Īะมี":129351,"تÙĤدÙħ":129352,"Ġعبر":129353,"Ġ׾×Ķ×Ŀ":129354,"ĠÑģамо":129355,"ס×ĵר":129356,"ĠcÃłng":129357,"rÃŃ":129358,"Ġìŀ¥":129359,"ëĵ¤ìĿĺ":129360,"ĠÙĦÙĥ":129361,"поÑĢÑĤ":129362,"Ġkhả":129363,"ĠÑģебÑı":129364,"׳ף":129365,"ĠدÙĪØ±":129366,"Ġmợ":129367,"Ġcây":129368,"Ġfark":129369,"Ġfarklı":129370,"аÑİÑĤ":129371,"Ġtrá»±c":129372,"wiÄĻksz":129373,"Ġthuá»ijc":129374,"ĠتØŃت":129375,"تÙĦ":129376,"овÑĭе":129377,"ëĤł":129378,"Ġвам":129379,"بÙĦغ":129380,"Ġê°ĻìĿĢ":129381,"íĮIJ":129382,"ÙĦب":129383,"Ġnasıl":129384,"Ġодин":129385,"ман":129386,"ĠعÙĦÙĬÙĩا":129387,"би":129388,"Ġפש×ķ×ĺ":129389,"×ijר×Ļ":129390,"Ġש׳×Ķ":129391,"ĠëıĦ":129392,"ĠÄIJại":129393,"Ġ×IJ×ķת×Ŀ":129394,"ĠاÙĦØŃر":129395,"Ġбо":129396,"à¸Īุà¸Ķ":129397,"Ġrõ":129398,"ĠdeÄŁiÅŁ":129399,"Ġëĭ¨":129400,"ĠÑģлÑĥÑĩа":129401,"ĠÑģлÑĥÑĩае":129402,"Ġ×IJ׳ש×Ļ×Ŀ":129403,"×ĵ×£":129404,"ש×ijת":129405,"Ġש׾׼×Ŀ":129406,"Ġchú":129407,"ników":129408,"Ġtanı":129409,"Ġcáo":129410,"ĠÄijá":129411,"Ġ×IJ×ĵ×Ŀ":129412,"Ġê°ķ":129413,"Ġnhiá»ĩm":129414,"Ġ×ľ×¡":129415,"Ġ×Ľ×ª×ij":129416,"Ġ×Ķספר":129417,"ĠÄijÄĥng":129418,"ĠëijIJ":129419,"à¸ľà¸´":129420,"à¸ľà¸´à¸§":129421,"جا":129422,"Ġê°IJ":129423,"رأ":129424,"ستخدÙħ":129425,"ãģ«ãģªãĤĬãģ¾ãģĻ":129426,"Ġtá»·":129427,"×ĺ×ķר":129428,"говоÑĢ":129429,"ĠвоÑģ":129430,"ĠÙħÙĨÙĩا":129431,"иÑĢоваÑĤÑĮ":129432,"ĠÄijầy":129433,"׳×Ĵ":129434,"ĠÙħÙĪ":129435,"ĠÙħÙĪÙĤع":129436,"ר׼×Ļ":129437,"تÙı":129438,"모":129439,"Ġת×ķ":129440,"ÙĬاÙĭ":129441,"à¹ĥà¸Ķ":129442,"ãĤĬãģ¾ãģĻ":129443,"à¸Ńยูà¹Īà¹ĥà¸Ļ":129444,"ĠØ£ÙĪÙĦ":129445,"ĠأخرÙī":129446,"Ġcư":129447,"صار":129448,"×ŀ×Ĺש×ij":129449,"бÑĢа":129450,"ÅĦski":129451,"бÑĢ":129452,"ĠÙĬÙı":129453,"à¸ģิà¸Ļ":129454,"Ġchá»ijng":129455,"ÙħÙı":129456,"Ġà¸Ħืà¸Ń":129457,"ĠتÙĨ":129458,"tÃŃ":129459,"yÄĩ":129460,"Ġmạng":129461,"ÙģÙĪ":129462,"Ġdünya":129463,"קר×IJ":129464,"Ġק׾":129465,"ĠØŃاÙĦ":129466,"cÃŃa":129467,"Ġà¹Ģรา":129468,"Ġר×ķצ×Ķ":129469,"Ġáp":129470,"ë°ķ":129471,"اÙĤØ©":129472,"ниÑİ":129473,"Ġ×IJ׾×ķ":129474,"Ġ×ŀס×ķ":129475,"ãģ§ãģ¯ãģªãģı":129476,"Ġtrả":129477,"Ġקשר":129478,"miÅŁtir":129479,"Ġlưu":129480,"Ġhá»Ĺ":129481,"ĠбÑĭли":129482,"Ġlấy":129483,"عÙĦÙħ":129484,"Ġözel":129485,"æ°ĹãģĮ":129486,"Ġ×ĵר×ļ":129487,"Ùħد":129488,"sını":129489,"׳×ķש×IJ":129490,"rów":129491,"ÑĩеÑĢ":129492,"êµIJìľ¡":129493,"ĠÐľÐ¾":129494,"лег":129495,"ĠVỼi":129496,"วัà¸Ļà¸Ļีà¹ī":129497,"ÑİÑīие":129498,"ãģĬãģĻ":129499,"ãģĬãģĻãģĻ":129500,"ãģĬãģĻãģĻãĤģ":129501,"ëıħ":129502,"Ġ×Ļ×Ķ×Ļ×Ķ":129503,"×ŀ×ĺר":129504,"Ñıми":129505,"Ġlá»±a":129506,"ĠÄijấu":129507,"à¹Ģสียà¸ĩ":129508,"Ġtương":129509,"ëĵ±":129510,"ĠÑģÑĤаÑĢ":129511,"à¹ĥà¸ļ":129512,"วัà¸Ķ":129513,"Ġİstanbul":129514,"Ġà¸Īะ":129515,"à¸ķลาà¸Ķ":129516,"ĠبÙĬ":129517,"à¹ģà¸Ļะ":129518,"à¹ģà¸Ļะà¸Ļำ":129519,"ساعد":129520,"Ġبأ":129521,"Ġkiá»ĥm":129522,"ØŃسب":129523,"à¸Ĭัà¹īà¸Ļ":129524,"Ġ×ķ×¢×ķ×ĵ":129525,"овÑĭÑħ":129526,"оÑģнов":129527,"ĠtrÆ°á»Łng":129528,"צ×ij×¢":129529,"ĠÃŃt":129530,"Ġkỹ":129531,"cré":129532,"Ñıм":129533,"êµ°":129534,"ãģĮãģªãģĦ":129535,"ÙĬÙĦØ©":129536,"ãĥķãĤ£":129537,"رÙī":129538,"ĠÙĬجب":129539,"Ġ×IJ×£":129540,"Ġcá»±c":129541,"ãĤīãĤĮãģŁ":129542,"Ġà¸ľà¸¹à¹ī":129543,"Ġà¸Ń":129544,"larımız":129545,"Ġkadın":129546,"Ġê·¸ëŀĺ":129547,"Ġê·¸ëŀĺìĦľ":129548,"ĠëĺIJëĬĶ":129549,"ĠÄijả":129550,"ĠÄijảm":129551,"Ġ×IJ×ķ×ŀר":129552,"Ġyếu":129553,"ciÄħ":129554,"ciÄħg":129555,"Ġtá»ij":129556,"Ġש×IJ׳×Ļ":129557,"ĠdziaÅĤa":129558,"Ñīа":129559,"ĠÄijÃłn":129560,"sına":129561,"ãģĵãĤĮãģ¯":129562,"Ġ×ij׾×Ļ":129563,"Ġ×ij×Ļשר×IJ׾":129564,"лоÑģÑĮ":129565,"Ġgiữ":129566,"ê°IJ":129567,"ÑĢон":129568,"تجار":129569,"глав":129570,"вин":129571,"Ġhạn":129572,"Ġyapılan":129573,"بس":129574,"Ġà¸ŀรà¹īà¸Ńม":129575,"ê´Ģ리":129576,"mÄ±ÅŁtır":129577,"bü":129578,"rück":129579,"ĠBaÅŁkanı":129580,"ĠÙĦÙĬس":129581,"ĠsÆ¡":129582,"à¸Īัà¸ĩหว":129583,"à¸Īัà¸ĩหวัà¸Ķ":129584,"داء":129585,"Ġ×Ķ׼":129586,"vÃŃ":129587,"ש×IJר":129588,"ĠhÆ°á»Łng":129589,"Ġbóng":129590,"ĠChÃŃnh":129591,"Äħc":129592,"à¹Ģà¸ģีà¹Īยวà¸ģัà¸ļ":129593,"Ġtứ":129594,"Ġtức":129595,"ĠÑĨвеÑĤ":129596,"Ġtá»iji":129597,"ĠnghÄ©a":129598,"ÙĦاعب":129599,"دÙĦ":129600,"Ġפע×Ŀ":129601,"hör":129602,"à¸Ĭุà¸Ķ":129603,"à¸ŀู":129604,"à¸ŀูà¸Ķ":129605,"паÑģ":129606,"ĠÅŁu":129607,"ĠtÆ°á»Łng":129608,"خارج":129609,"Ġâm":129610,"ĠинÑĤеÑĢеÑģ":129611,"еннÑĭÑħ":129612,"×IJ׳×Ļ":129613,"بدأ":129614,"ëĿ¼ëĬĶ":129615,"ì¹´":129616,"æĸ¹ãģĮ":129617,"лив":129618,"Ġà¸Ħà¸Ļ":129619,"ער×ļ":129620,"à¸Ĥà¸Ńà¸ĩà¸Ħุà¸ĵ":129621,"пад":129622,"Ġcạnh":129623,"ĠëĤ¨":129624,"ĠÄijâu":129625,"Ġbiá»ĥu":129626,"ãĤĤãģĤãĤĭ":129627,"׾×Ĵ":129628,"Ġสำหรัà¸ļ":129629,"Ġxuá»ijng":129630,"ס×ķ":129631,"Ġذات":129632,"ĠÐľÐµ":129633,"عاÙĦÙħ":129634,"×IJס":129635,"بÙĬØ©":129636,"شا":129637,"ием":129638,"ĠNgưá»Ŀi":129639,"íĺij":129640,"Ñģлов":129641,"Ġпа":129642,"Ġmẫu":129643,"ĠпÑĢоÑĨеÑģÑģ":129644,"ĠNhÃł":129645,"пÑĢоиз":129646,"пÑĢоизвод":129647,"à¸łà¸²à¸¢à¹ĥà¸Ļ":129648,"Ġà¸ļาà¸Ĺ":129649,"×ŀ׳×ķ":129650,"ĠоÑĢган":129651,"רצ×ķ":129652,"×ķ×ŀ×Ļ×Ŀ":129653,"Ġyazı":129654,"Ġdù":129655,"ãĥ¬ãĥ³":129656,"ÙĪÙĦÙĬ":129657,"ยู":129658,"Ġtrò":129659,"à¹Ģà¸ŀลà¸ĩ":129660,"Ġ×ŀ׾×IJ":129661,"à¸ķล":129662,"à¸ķลà¸Ńà¸Ķ":129663,"ĠÄijạt":129664,"Ġ×Ĺ×ĵש":129665,"póÅĤ":129666,"Ġ×ŀ×ĵ×Ļ":129667,"ujÄħc":129668,"×ŀ׳×Ķ׾":129669,"Ġש×ij×ķ":129670,"Ġ×Ķ×ŀשפ×ĺ":129671,"Ġ×IJ׾×Ķ":129672,"ĠÙĪØ°ÙĦÙĥ":129673,"à¹Ģà¸ŀราะ":129674,"ĠÄijoÃłn":129675,"Ġíķ¨ê»ĺ":129676,"Ġdục":129677,"شت":129678,"Ġula":129679,"ĠulaÅŁ":129680,"Ġquý":129681,"Ġ×Ķ×Ĵ×ĵ×ķ׾":129682,"à¸ķัà¹īà¸ĩà¹ģà¸ķà¹Ī":129683,"Ġשר":129684,"Ø´Ùĩد":129685,"׳ש×Ļ×Ŀ":129686,"à¸ŀล":129687,"رÙĪØ§":129688,"ãĤĮãģ¦":129689,"ĠниÑħ":129690,"Ġдела":129691,"ãģ§ãģįãģªãģĦ":129692,"ÅĤoż":129693,"×IJ×Ĺר":129694,"ì½Ķ":129695,"ãĤ¢ãĥĥãĥĹ":129696,"دÙ쨹":129697,"Ġtiá»ĩn":129698,"Ġkhá»ı":129699,"Ġkhá»ıe":129700,"ĠاÙĦعاÙħØ©":129701,"ãģ«ãģĤãĤĭ":129702,"ĠÄijá»Ļc":129703,"족":129704,"Ġcụ":129705,"йÑĤе":129706,"Ġзакон":129707,"ĠпÑĢоекÑĤ":129708,"ìĸ¸":129709,"ÙĦØŃ":129710,"ĠçalÄ±ÅŁma":129711,"ãĤĴãģĻãĤĭ":129712,"Ñħи":129713,"عاد":129714,"Ġ׳×ŀצ×IJ":129715,"Ġר×Ļ":129716,"à¸Ńà¸Ńà¸ģมา":129717,"ĠTôi":129718,"Ġthần":129719,"ĠÙĬا":129720,"ลาย":129721,"ĠавÑĤо":129722,"Ġsıra":129723,"ĠÙĥØ«ÙĬر":129724,"ÙħÙĬز":129725,"ĠاÙĦعÙĦÙħ":129726,"æĸ¹ãģ¯":129727,"×ķ×¢×ĵ":129728,"ĠоблаÑģÑĤи":129729,"×Ļ׾×Ļ×Ŀ":129730,"ãģĮåĩº":129731,"à¸ĺุ":129732,"à¸ĺุร":129733,"à¸ĺุรà¸ģิà¸Ī":129734,"ÙĤتÙĦ":129735,"ר×IJ×ķ":129736,"Ġngu":129737,"Ġnguá»ĵn":129738,"Ġมา":129739,"Ġплан":129740,"tório":129741,"Ġcuá»iji":129742,"Ñģком":129743,"ĠاÙĦÙħاض":129744,"ĠاÙĦÙħاضÙĬ":129745,"Ġ×ij×¢×ľ":129746,"Ġר×ij×Ļ×Ŀ":129747,"ĠluáºŃn":129748,"ÙĥÙĪ":129749,"à¸Ĺัà¹īà¸ĩหมà¸Ķ":129750,"ван":129751,"Ġthoại":129752,"à¹Ħà¸Ń":129753,"биÑĢ":129754,"ĠاÙĦض":129755,"تا":129756,"ĠÑĢод":129757,"ĠVÃł":129758,"×ŀ×Ļף":129759,"ĠбÑĭла":129760,"ками":129761,"ĠÐĶе":129762,"tık":129763,"קר×Ļ":129764,"ĠeÄŁitim":129765,"ĠÙĥبÙĬر":129766,"بÙĥ":129767,"ĠÙĦÙĪ":129768,"вой":129769,"Ġãģĵãģ®":129770,"ĠÑĤÑĢÑĥд":129771,"myÅĽl":129772,"Ġsư":129773,"à¸ŀีà¹Ī":129774,"Ġà¹ģลà¹īว":129775,"×¢×§":129776,"Ġ×Ĺ×ijרת":129777,"ระหว":129778,"ระหวà¹Īาà¸ĩ":129779,"×Ļ×Ļ×Ķ":129780,"ĠاÙĦÙĨاس":129781,"ünü":129782,"Ġ׾×ŀ×Ķ":129783,"Ġchương":129784,"ĠHá»ĵ":129785,"ارت":129786,"ãĤĪãģĨãģ§ãģĻ":129787,"lá":129788,"×§×Ļ×Ļ×Ŀ":129789,"æľ¬å½ĵ":129790,"æľ¬å½ĵãģ«":129791,"ãģĵãĤĵãģª":129792,"Ñģов":129793,"Ġ×ķ×Ĺ":129794,"à¹Ģà¸ģà¹ĩà¸ļ":129795,"ĠкÑĤо":129796,"à¹Ĥรà¸Ħ":129797,"ĠشرÙĥØ©":129798,"عزÙĬ":129799,"عزÙĬز":129800,"Ø·ÙĦÙĤ":129801,"пÑĥÑģÑĤ":129802,"ÙģØªØŃ":129803,"ëŀĢ":129804,"Ġhãy":129805,"ضÙħ":129806,"린":129807,"åł´åIJĪãģ¯":129808,"ãĤªãĥ¼":129809,"Ġhắn":129810,"Ġ×IJ×ij×Ļ×ij":129811,"Ġש׾×Ķ×Ŀ":129812,"Ġ×Ķ×Ļ×Ļת×Ķ":129813,"ĠاÙĦدÙĪÙĦØ©":129814,"ĠاÙĦÙĪÙĤ":129815,"ĠاÙĦÙĪÙĤت":129816,"ãģĤãģ¾ãĤĬ":129817,"ĠtaÅŁÄ±":129818,"İN":129819,"עסק":129820,"ãģ¦ãģĦãģŁ":129821,"Ġtá»ķng":129822,"ĠاÙĦØ¥ÙĨس":129823,"ĠاÙĦØ¥ÙĨساÙĨ":129824,"ÑĢеÑĪ":129825,"Ġgái":129826,"ĠÑĨен":129827,"ĠÙģÙĤد":129828,"Ùħات":129829,"ãģķãĤĵãģ®":129830,"Ġphù":129831,"×ĺ×Ķ":129832,"ĠÙĪØ§ÙĦتÙĬ":129833,"ĠبÙĥ":129834,"ìĿ´ëĤĺ":129835,"кÑģ":129836,"ÙħÙĬر":129837,"Ġvùng":129838,"ĠاÙĦشعب":129839,"ĠNhưng":129840,"ãĥĢãĥ¼":129841,"Ġ×Ĺ×Ļ×Ļ×Ŀ":129842,"Ġشخص":129843,"×§×ķ×ĵ":129844,"ê²Ģ":129845,"עש":129846,"×¢×ķ׾×Ŀ":129847,"צ×ķר":129848,"عÙĤد":129849,"ĠiÅŁlem":129850,"Ġ×Ķ×ij×IJ":129851,"Ġdưỡng":129852,"à¸Łà¸£à¸µ":129853,"ĠphÃŃa":129854,"ãģ®ä¸Ńãģ§":129855,"Ġпи":129856,"ĠngÃłnh":129857,"нима":129858,"ĠÙĩÙĦ":129859,"Ġ×ķ×IJת":129860,"ĠÄijáng":129861,"équipe":129862,"ĠÑįÑĤоÑĤ":129863,"Ġgörev":129864,"매":129865,"Ġquân":129866,"å¼ķãģį":129867,"æĻĤãģ«":129868,"ĠبÙħا":129869,"×ŀ×Ļת":129870,"Ġülke":129871,"Ġ×ŀ×§×ķ×Ŀ":129872,"×ijף":129873,"æ°ĹæĮģãģ¡":129874,"Ġë§İìĿĢ":129875,"Ġyüksek":129876,"ÑĨенÑĤÑĢ":129877,"ĠÙħجÙĦس":129878,"ç§ģãģ®":129879,"ÙĤدر":129880,"Ġë¶Ģë¶Ħ":129881,"Ġì°¨":129882,"خرج":129883,"ãģĭãģªãĤĬ":129884,"ë³´ëĭ¤":129885,"Ġ×ŀ×Ļ×ĵ×¢":129886,"peÅĤni":129887,"Ġxá»Ń":129888,"ìĹIJìĦľëĬĶ":129889,"ĠباÙĦÙħ":129890,"ĠÙĪÙħا":129891,"ĠÑįÑĤой":129892,"بÙĬÙĨ":129893,"nü":129894,"ØŃز":129895,"ØŃزب":129896,"ĠÑĢабоÑĤа":129897,"ĠNháºŃt":129898,"ÙĦاء":129899,"Ġëĵ¤":129900,"Ġëĵ¤ìĸ´":129901,"ãĤĦãģĻãģĦ":129902,"×Ĺ×ĸ×§":129903,"Ġ×Ķ×Ĺ×ijר×Ķ":129904,"пиÑĤ":129905,"ãģĭãĤīãģ®":129906,"Ġë§IJìĶĢ":129907,"Ġפ×ķ":129908,"ÙĦÙİ":129909,"à¹Ģà¸ķà¹ĩม":129910,"ĠÐļо":129911,"Ġmówi":129912,"ĠtÃŃn":129913,"ר×Ĵש":129914,"פרק":129915,"Ġtrạng":129916,"ĠÐŀн":129917,"×Ĺ×ķ×¥":129918,"ĠعÙĨدÙħا":129919,"Ġبر":129920,"使ãģĦ":129921,"Ġrá»Ļng":129922,"ëĮĢë¡ľ":129923,"íά":129924,"Ġktórych":129925,"вид":129926,"ลูà¸ģà¸Ħà¹īา":129927,"ĠmogÄħ":129928,"Ġש×Ĺ":129929,"×ij×Ĺר":129930,"ãĥĸãĥŃãĤ°":129931,"ĠThÃłnh":129932,"Ġ×Ķר×Ļ":129933,"ĠÑģÑĤаÑĤÑĮ":129934,"ĠHá»Ļi":129935,"à¸ļà¹īาà¸ĩ":129936,"çī¹ãģ«":129937,"ĠÄIJức":129938,"èĢħãģ®":129939,"×¢×ŀ×ķ×ĵ":129940,"×ĺר×Ķ":129941,"Ð¥":129942,"ĠÙħÙħا":129943,"ĠeÅŁ":129944,"ĠнеобÑħодимо":129945,"ников":129946,"Ġüzerinde":129947,"aÅĤa":129948,"Ġchá»ĭu":129949,"ĠاÙĦدÙĬÙĨ":129950,"أخبار":129951,"ĠÄijau":129952,"ãģĮå¤ļãģĦ":129953,"jÄħcych":129954,"دخÙĦ":129955,"larınd":129956,"larından":129957,"Ġsẻ":129958,"à¸ŀิà¹Ģศ":129959,"à¸ŀิà¹Ģศษ":129960,"×ª×Ł":129961,"tıģı":129962,"ĠluáºŃt":129963,"ĠÅŀe":129964,"ãĤ«ãĥ¼":129965,"ãģ®ãģĤãĤĭ":129966,"Ġ×Ķ×IJתר":129967,"ĠاÙĦØ¢ÙĨ":129968,"ıldı":129969,"Ġáo":129970,"ĠнаÑĩал":129971,"Ġviá»ĩn":129972,"Ġ×ij×¢×ķ׾×Ŀ":129973,"знаÑĩ":129974,"×Ļ×ĺ×Ķ":129975,"кам":129976,"ĠÐĺз":129977,"à¹Ģà¸Ĥียà¸Ļ":129978,"à¸Ļà¹īà¸Ńà¸ĩ":129979,"ÑĤÑĢо":129980,"à¹Ģà¸Ł":129981,"Ġжизни":129982,"Ġสà¹Īวà¸Ļ":129983,"ĠváºŃn":129984,"Ġê´Ģ볨":129985,"Ġlâu":129986,"ס×ĺר":129987,"קש":129988,"سÙĬر":129989,"Ġ×IJ×ķת×Ļ":129990,"Ġmôi":129991,"ائب":129992,"ĠоÑģÑĤа":129993,"Ġmón":129994,"Ġ×ij×ŀ×§×ķ×Ŀ":129995,"ĠداخÙĦ":129996,"Ġ×IJ×ķר":129997,"ĠваÑģ":129998,"ÙĥØ´Ùģ":129999,"ìĺ¨":130000,"à¸ĸà¹Īาย":130001,"Ġkullanıl":130002,"Ġtô":130003,"ãģ«ãĤĪãĤĬ":130004,"ĠëĺIJíķľ":130005,"Ġ×¢×ij×ķ×ĵ×Ķ":130006,"Ġriê":130007,"Ġriêng":130008,"Ġyakın":130009,"زا":130010,"Å»":130011,"×IJ×ķ׼׾":130012,"شارÙĥ":130013,"ĠбеÑģ":130014,"×´":130015,"ĠابÙĨ":130016,"ĠTá»ķng":130017,"ÙĨظ":130018,"ÅĽwiad":130019,"ãĤµãĥ¼":130020,"หาย":130021,"ĠGün":130022,"Ġhakkında":130023,"à¹Ģà¸Ĥà¹īามา":130024,"زÙĨ":130025,"ĠÐłÐ¾":130026,"Ġbiá»ĥn":130027,"ãģ©ãģĵ":130028,"Ù쨹ÙĦ":130029,"زع":130030,"פר×ĺ":130031,"Ġ×Ķף":130032,"Ø£ÙĩÙĦ":130033,"Ġthất":130034,"ØŃÙħÙĦ":130035,"ÑĩÑĥ":130036,"ĠìĤ¬ìĭ¤":130037,"ì°¸":130038,"ĠìľĦíķ´":130039,"ÙĪØ¸":130040,"ĠÐŁÐ¾Ð´":130041,"Ġkhoản":130042,"ÑĤен":130043,"ĠÙ쨧ÙĦ":130044,"Ñģад":130045,"à¸Ļà¸Ńà¸Ļ":130046,"ĠاÙĦسعÙĪØ¯ÙĬØ©":130047,"\"ØĮ":130048,"ĠاÙĦÙĴ":130049,"ãĤīãģļ":130050,"Ġtoán":130051,"Ġchắc":130052,"׼×Ļר":130053,"méd":130054,"média":130055,"زÙĪ":130056,"Ġyanı":130057,"פ׳×Ļ×Ŀ":130058,"ØŃظ":130059,"ĠбеÑģп":130060,"ĠбеÑģплаÑĤ":130061,"ĠбеÑģплаÑĤно":130062,"ĠØ£ÙħاÙħ":130063,"à¸Ńาย":130064,"à¸Ńายุ":130065,"רשת":130066,"Ġgá»ĵ":130067,"Ġgá»ĵm":130068,"Ġuá»ijng":130069,"صب":130070,"kır":130071,"ãĥijãĥ¼":130072,"Ġ׾×ĵעת":130073,"ĠкÑĥпиÑĤÑĮ":130074,"׾×ķ×Ĺ":130075,"ÙĪØ¶Ø¹":130076,"ÙĤÙĬÙħ":130077,"à¸Ľà¸²":130078,"жив":130079,"à¸Ķิà¸Ļ":130080,"×IJ×ķפ":130081,"à¹Ģลà¹ĩà¸ģ":130082,"ãĥĥãĥī":130083,"иÑĩеÑģкиÑħ":130084,"ĠChá»§":130085,"кÑĢаÑģ":130086,"ÙĪØµÙĦ":130087,"pÅĤat":130088,"моÑĢ":130089,"Ġ×Ķ×IJ×ķ":130090,"à¸Ńิà¸Ļ":130091,"ĠíķľêµŃ":130092,"гÑĢе":130093,"Ġìłľê³µ":130094,"ì°½":130095,"Ġê°ľìĿ¸ìłķë³´":130096,"Ġnghá»ĭ":130097,"à¸ĭา":130098,"ØŃساب":130099,"ĠbyÅĤa":130100,"ÙħÙĦÙĥ":130101,"иÑĩеÑģкие":130102,"Ġbác":130103,"ضØŃ":130104,"길":130105,"ש×ŀ×¢":130106,"Ġìĸ´ëĸ»":130107,"Ġìĸ´ëĸ»ê²Į":130108,"ìĽĮ":130109,"اتÙĩ":130110,"à¹Ĥรà¸ĩà¹ģ":130111,"à¹Ĥรà¸ĩà¹ģรม":130112,"خدÙħØ©":130113,"ĠÐłÐ°":130114,"׼×ķ׾×Ŀ":130115,"×ŀש×Ĺ×§":130116,"ĠÙĪÙĥاÙĨ":130117,"ס×ķ×£":130118,"ĠاÙĦØŃÙĥÙĪÙħØ©":130119,"Ġ×ij×ĺ":130120,"ĠtráºŃn":130121,"Ġ×Ķ×¢×ķ׾×Ŀ":130122,"ĠÃŃch":130123,"tÄħ":130124,"ש×ŀ×ķ":130125,"Ġ×Ķר×IJש×ķף":130126,"Ġíķĺê³ł":130127,"ãģķãĤī":130128,"ãģķãĤīãģ«":130129,"ãģ«ãģĹãģ¦":130130,"Ġà¸ľà¸¡":130131,"ãģ®ãĤĪãģĨãģª":130132,"ĠÙĪÙĤت":130133,"ãĥįãĥĥãĥĪ":130134,"ÙĦعب":130135,"ÙĪØ´":130136,"ìĺ¬":130137,"Ġหาà¸ģ":130138,"ĠmiaÅĤ":130139,"à¸Ĺà¸Ńà¸ĩ":130140,"иÑĤа":130141,"اصر":130142,"илÑģÑı":130143,"зе":130144,"à¸Ľà¸£à¸°à¸¡à¸²à¸ĵ":130145,"ãģĿãĤĮãģ¯":130146,"Ġbır":130147,"Ġbırak":130148,"صÙĨاع":130149,"Ю":130150,"شعر":130151,"Ġ׳×Ĵ×ĵ":130152,"Ġبسبب":130153,"ãĥĿãĤ¤":130154,"ãĥĿãĤ¤ãĥ³ãĥĪ":130155,"ĠاÙĦجÙĪ":130156,"ĠнеÑģколÑĮко":130157,"Ġkiếm":130158,"ÙģÙİ":130159,"Ġضد":130160,"×ij×Ļ×ĺ×ķ×Ĺ":130161,"تابع":130162,"ÙĨز":130163,"ĠBản":130164,"Ġaçıkl":130165,"Ġaçıklama":130166,"Ġà¸Ħุà¸ĵ":130167,"à¸Ĺา":130168,"ÅĤów":130169,"طب":130170,"ÙĨØŃÙĨ":130171,"Ġ×ŀ×§×ķר":130172,"Ġİs":130173,"Ġдома":130174,"Ġวัà¸Ļ":130175,"ĠdÃłnh":130176,"Ñıн":130177,"миÑĢ":130178,"Ġmô":130179,"ĠvÃłng":130180,"صاب":130181,"sının":130182,"à¸Ħืà¸Ļ":130183,"خبر":130184,"×ĸ׼×ķ":130185,"Ġ×ŀש×Ķ×ķ":130186,"mü":130187,"Ġкомпании":130188,"Ġ×Ķ×¢×Ļר":130189,"ĠÙĥÙĪ":130190,"ÙĤÙĦب":130191,"ĠlỼp":130192,"ики":130193,"׳×ij":130194,"à¹Ĥà¸Ħร":130195,"à¹Ĥà¸Ħรà¸ĩ":130196,"à¹Ĥà¸Ħรà¸ĩà¸ģาร":130197,"×ŀ×ķ×¢×ĵ":130198,"ÑıÑĤÑģÑı":130199,"หลัà¸ĩà¸Īาà¸ģ":130200,"ениÑİ":130201,"Ġשע":130202,"ĠbÆ°á»Ľc":130203,"ãĥ¡ãĥ¼ãĥ«":130204,"ãĤĦãĤĬ":130205,"Ġ×Ļ×ķ×ĵ×¢":130206,"Ġê´Ģíķľ":130207,"ĠاÙĦØ£Ùħر":130208,"Ġbölge":130209,"ĠÑģвой":130210,"ÙĦس":130211,"Ġ×ŀ×Ļ×ķ×Ĺ×ĵ":130212,"ĠëĤ´ìļ©":130213,"ĠأجÙĦ":130214,"ĠÄIJông":130215,"Ġ×ŀ×ł×ª":130216,"Ġìĭľê°Ħ":130217,"ÙĥÙİ":130218,"ãģ¨ãģĦãģĨãģ®ãģ¯":130219,"Ġnależy":130220,"تÙĨظÙĬÙħ":130221,"ĠÑģозда":130222,"Ġphé":130223,"Ġphép":130224,"ãģ§ãģįãģ¾ãģĻ":130225,"ĠعÙĦÙħ":130226,"大ãģįãģª":130227,"ãĤ²ãĥ¼ãĥł":130228,"íħĮ":130229,"Ġ׼×ķ׾׾":130230,"ĠинÑĤеÑĢнеÑĤ":130231,"ĠTừ":130232,"ãģ¨ãģªãĤĭ":130233,"زاÙĦ":130234,"Ġktórym":130235,"Ġnhé":130236,"ìĪľ":130237,"нев":130238,"деÑĢ":130239,"ãĤ¢ãĥĹãĥª":130240,"iá»ĩu":130241,"×ij×Ļ׾":130242,"Ġتس":130243,"ĠÄIJây":130244,"ĠاÙĦخاصة":130245,"Ġà¹Ģà¸Ĭ":130246,"Ġà¹Ģà¸Ĭà¹Īà¸Ļ":130247,"صاد":130248,"Ġdạng":130249,"سعر":130250,"Ġש×Ļ×ŀ×ķש":130251,"×Ĵ×Ļ×Ŀ":130252,"ãģĮãģĤãģ£ãģŁ":130253,"пÑĢов":130254,"пÑĢовод":130255,"Ġ×IJ×Ļ׳×ķ":130256,"Ġ׾ר×IJ":130257,"Ġ׾ר×IJ×ķת":130258,"ĠØ£Ù쨶ÙĦ":130259,"ĠØŃÙĦ":130260,"ĠأبÙĪ":130261,"ê°ķ":130262,"Ġì§ij":130263,"ãģ®ãĤĪãģĨãģ«":130264,"Ġפ׳×Ļ":130265,"ס×Ļ×Ŀ":130266,"ĠÙĪÙĩذا":130267,"Ġkaç":130268,"Ġéén":130269,"Ġê±´":130270,"ë°Ķ":130271,"Ñĥз":130272,"à¸Ĥà¸Ńà¸ĩà¹Ģรา":130273,"iÅĤ":130274,"ĠÐľÑĭ":130275,"Ġchết":130276,"ĠاÙĦثاÙĨÙĬ":130277,"×IJ×§":130278,"Ġ×ķ×¢×ľ":130279,"ĠاÙĦطب":130280,"×ij×ĺ×Ĺ":130281,"ĠجدÙĬدة":130282,"ĠعدÙħ":130283,"عز":130284,"สิà¹Īà¸ĩà¸Ĺีà¹Ī":130285,"ãģĻãĤĮãģ°":130286,"ĠÄijô":130287,"ì£ł":130288,"دÙĤ":130289,"номÑĥ":130290,"Ġká»ĥ":130291,"ãĤ¢ãĥ³":130292,"å¤ļãģıãģ®":130293,"à¸Ľà¸£à¸°à¸ģ":130294,"à¸Ľà¸£à¸°à¸ģà¸Ńà¸ļ":130295,"פע×Ļ׾×ķת":130296,"ĠÑģÑĤол":130297,"mayı":130298,"ãģ¤ãģĦ":130299,"Ġyılında":130300,"Ġà¸Īึà¸ĩ":130301,"koÅĦcz":130302,"ĠThông":130303,"ĠакÑĤив":130304,"нÑģÑĤ":130305,"нÑģÑĤÑĢÑĥ":130306,"ĠÃĸz":130307,"Ġת×ŀ×Ļ×ĵ":130308,"ĠÙĥÙĨت":130309,"ÑģиÑģÑĤем":130310,"prés":130311,"présent":130312,"Ġnâ":130313,"Ġnâng":130314,"gÅĤos":130315,"ĠÙĪØ²ÙĬر":130316,"ØŃصÙĦ":130317,"ĠимееÑĤ":130318,"ØŃرÙĥØ©":130319,"à¸ŀà¹Īà¸Ń":130320,"ãĤĴãģĬ":130321,"ĠاستخداÙħ":130322,"×IJ×Ļר×ķ×¢":130323,"ä»ĸãģ®":130324,"Ġש×Ķ×Ŀ":130325,"ãģĹãģŁãĤī":130326,"ש×ŀ×Ļ":130327,"Ñģла":130328,"mı":130329,"Ġbazı":130330,"Ġíķĺì§Ģë§Į":130331,"×ĵ׾":130332,"Ġyaptıģı":130333,"ãĥĬãĥ¼":130334,"׾×Ļ׾×Ķ":130335,"ãģ¨ãģĦãģ£ãģŁ":130336,"ändig":130337,"ĠÅŁa":130338,"ĠÙģÙĬÙħا":130339,"иÑĤелÑı":130340,"×ŀ×ķש":130341,"à¸Ĥà¸Ńà¸ļ":130342,"lük":130343,"Ġhá»ĵi":130344,"Ġëªħ":130345,"ĠاÙĦÙĥØ«ÙĬر":130346,"צ×IJ":130347,"Ġhazır":130348,"طرÙģ":130349,"اÙĬا":130350,"ĠÄijôi":130351,"енд":130352,"ÙĦغ":130353,"×Ĺ×ĸ×ķר":130354,"ĠвÑģег":130355,"ĠвÑģегда":130356,"ëIJĺê³ł":130357,"×ĵ×ķ×ĵ":130358,"ана":130359,"دÙĪÙĦØ©":130360,"Ġhoạch":130361,"عÙĦا":130362,"عÙĦاج":130363,"Ġ×ķ×¢×ĵ":130364,"×Ķ×Ŀ":130365,"кий":130366,"ÙĦÙIJ":130367,"Ġ×¢×ľ×Ļ×ķ":130368,"ÑİÑīий":130369,"Ġngá»§":130370,"صÙĨع":130371,"ĠاÙĦعراÙĤ":130372,"à¸ķà¹Īà¸Ńà¹Ħà¸Ľ":130373,"ãģŁãģıãģķãĤĵ":130374,"Ġphạm":130375,"ÙĦاÙĨ":130376,"اتÙĩا":130377,"Ġböyle":130378,"تÙĨÙģÙĬ":130379,"تÙĨÙģÙĬذ":130380,"Ġש×Ķ×Ļ×IJ":130381,"ÑģÑĥ":130382,"ยาว":130383,"Ġש×ķ׳×Ļ×Ŀ":130384,"Ġ×ŀ×ķ׾":130385,"ĠÑģил":130386,"Ġ×IJ×Ĺר×Ļ×Ŀ":130387,"Ġphá»§":130388,"ÙĤطع":130389,"ĠThá»§":130390,"à¸Ľà¸£à¸°à¹Ģà¸Ĺศà¹Ħà¸Ĺย":130391,"ÙĨÙĤ":130392,"ĠÄijoạn":130393,"Ġبإ":130394,"пÑĢедел":130395,"×ķת×ķ":130396,"Ġyarı":130397,"пÑĢе":130398,"ĠczÄĻÅĽci":130399,"ØŃÙĥÙħ":130400,"×ķ׳×Ļת":130401,"×¤×¢×ľ":130402,"ãĤĴãģĹãģ¦":130403,"Ġktórzy":130404,"׾×Ŀ":130405,"ĠÄIJiá»ģu":130406,"ĠкоÑĤоÑĢаÑı":130407,"ĠìĿ´ìĥģ":130408,"ãģĤãģ£ãģŁ":130409,"Ġ×ŀ×ĵ×ķ×ijר":130410,"פ×ķ×¢×ľ":130411,"dım":130412,"éĢļãĤĬ":130413,"ĠбÑĥдÑĥÑĤ":130414,"à¹Ģวà¹ĩà¸ļà¹Ħà¸ĭ":130415,"à¹Ģวà¹ĩà¸ļà¹Ħà¸ĭà¸ķà¹Į":130416,"اخر":130417,"×Ĺ×Ļ׾":130418,"Ġ×Ļ׾":130419,"Ġ×Ļ׾×ĵ×Ļ×Ŀ":130420,"×Ĺ×Ļפ":130421,"×Ĺ×Ļפ×ķש":130422,"Ġdòng":130423,"Ġש×ĸ×Ķ":130424,"ÑĮе":130425,"ãģĤãģ¨":130426,"ìŀIJê°Ģ":130427,"×IJ×ĵ":130428,"Ġüz":130429,"Ġüzere":130430,"ظÙĦ":130431,"Ġ×IJ×ķ׾×Ļ":130432,"Ġ×ij×Ļ×ķ×Ŀ":130433,"ÙĦات":130434,"Ġmê":130435,"침":130436,"تØŃد":130437,"تØŃدث":130438,"Ġخاصة":130439,"ĠبرÙĨ":130440,"ĠبرÙĨاÙħج":130441,"ĠHÃłn":130442,"×Ĺס":130443,"ĠÙĪÙĦÙħ":130444,"×¢×Ŀ":130445,"Ġmı":130446,"à¸Łà¸±à¸ĩ":130447,"שע×Ķ":130448,"ÙĪÙģÙĤ":130449,"ס×ij×Ļר":130450,"алÑĮнÑĭй":130451,"×Ĺש×ķ×ij":130452,"ĠnÃłng":130453,"ë³¼":130454,"ĠкоÑĤоÑĢÑĭÑħ":130455,"Ġ×Ĺ×ķ×§":130456,"tör":130457,"ĠлÑĥÑĩÑĪе":130458,"ãĥijãĥ³":130459,"ลà¹Īาสุà¸Ķ":130460,"ĠجدÙĬد":130461,"ÙĬدة":130462,"à¸Ĺรà¸ĩ":130463,"ãĤĪãĤĬãĤĤ":130464,"ÙĦÙĦ":130465,"ãĤĤãģ£ãģ¨":130466,"ש×ĺ×Ĺ":130467,"Ġ×ķ×IJ×Ļ":130468,"Ġgiá»ijng":130469,"إضاÙģ":130470,"קת":130471,"ë§Ŀ":130472,"ĠzostaÅĤ":130473,"ÑĢоз":130474,"×Ļפ×Ļ×Ŀ":130475,"Ġ׼׾׾":130476,"ת×ķ׼ף":130477,"dıģını":130478,"ÙĤسÙħ":130479,"ĠÑģÑĩиÑĤ":130480,"ĠÑģÑĩиÑĤа":130481,"×ĺ×ķת":130482,"Ġưu":130483,"ĠØ¢ÙĦ":130484,"Ġмом":130485,"ĠмоменÑĤ":130486,"ĠاÙĦتعÙĦÙĬÙħ":130487,"×¢×ľ×ķת":130488,"Ġchữa":130489,"Ġyön":130490,"ĠtrÃł":130491,"ĠØŃÙĬÙĨ":130492,"à¸ĭั":130493,"ĠCá":130494,"×¢×ĸ":130495,"ĠاÙĦØ£ÙħÙĨ":130496,"cÃŃ":130497,"Ġvá»ijn":130498,"Ġà¸Ļาย":130499,"обÑĢа":130500,"×§×IJ":130501,"Ġthiếu":130502,"ãĥŀãĥ¼":130503,"สวà¸Ļ":130504,"Ġgá»Ń":130505,"Ġgá»Ńi":130506,"Ġê¹":130507,"Ġê¹Ģ":130508,"Ġthiá»ĩn":130509,"ÙĤع":130510,"wÄĻ":130511,"Ġнам":130512,"ÑĤол":130513,"Ġsân":130514,"ס×ķ×Ĵ":130515,"Ġgeçir":130516,"ÑĤон":130517,"ева":130518,"ĠÙĪØ¶Ø¹":130519,"Ġعشر":130520,"Ñģло":130521,"à¸Īัà¸ļ":130522,"ãĤ·ãĥ¼":130523,"ãĤĤãģĤãĤĬãģ¾ãģĻ":130524,"Ġvẻ":130525,"ĠÄIJá»ĥ":130526,"رÙ쨹":130527,"ĠاÙĦØ£ÙĪÙĦÙī":130528,"ÑĤаÑĢ":130529,"ãģªãģıãģ¦":130530,"ÙħÙİ":130531,"quÃŃ":130532,"×¢×ł×Ļ×Ļ׳":130533,"ген":130534,"Ġhôm":130535,"à¸Īา":130536,"ĠnhỼ":130537,"ĠاÙĦعربÙĬ":130538,"×IJף":130539,"Ġlá»Ļ":130540,"ĠjeÅĽli":130541,"à¹Ģà¸Ĺà¹Īาà¸Ļัà¹īà¸Ļ":130542,"ĠØ£ÙĨÙĩا":130543,"Ġtuy":130544,"Ġtuyá»ĩt":130545,"Ġتص":130546,"ĠتصÙĨÙĬ":130547,"ĠتصÙĨÙĬÙģ":130548,"Ġê·¸ëŁ¬ëĤĺ":130549,"оÑĨен":130550,"à¸ģิà¸Īà¸ģรรม":130551,"ãĤĦãģ£ãģ¦":130552,"Ġkhá»ıi":130553,"Ġlá»ĩ":130554,"ĠاÙĦÙħجتÙħع":130555,"à¸Ńาà¸Īà¸Īะ":130556,"à¸Īะà¹Ģà¸Ľà¹ĩà¸Ļ":130557,"овÑĭй":130558,"ר×Ŀ":130559,"รà¹īà¸Ńà¸Ļ":130560,"ש×ŀש":130561,"人ãģ«":130562,"Ġüzerine":130563,"פר×Ļ":130564,"duÄŁu":130565,"Ñĩик":130566,"Ġmùa":130567,"Ġ×ŀת×ķ×ļ":130568,"ĠcáºŃp":130569,"ĠتارÙĬØ®":130570,"×ij×ľ×ª×Ļ":130571,"Ġì¢Ģ":130572,"ÙĦع":130573,"باÙĨ":130574,"Ġchút":130575,"Ġ×Ķ×ĸ×ŀף":130576,"née":130577,"ĠLiên":130578,"ĠÙĦÙĦØ£":130579,"ØŃدÙĪØ¯":130580,"Ġ×¢×Ľ×©×Ļ×ķ":130581,"воз":130582,"Ġyaptı":130583,"Ġобо":130584,"à¹ĥหà¹īà¸ģัà¸ļ":130585,"Ġ×ij×Ķ×Ŀ":130586,"ãģıãģ¦":130587,"رأس":130588,"ĠÑģÑĢедÑģÑĤв":130589,"ĠBÃłi":130590,"ãģĵãģ¨ãģ«":130591,"ĠìĤ¬íļĮ":130592,"Ġ모ëijIJ":130593,"×ij×IJ":130594,"Ġtrắng":130595,"ĠاÙĦبÙĦد":130596,"ĠHoÃłng":130597,"либо":130598,"ĠдÑĢÑĥгиÑħ":130599,"İR":130600,"Ñĥма":130601,"ĠJeÅĽli":130602,"ãĤĤãģĹ":130603,"Ġvòng":130604,"Ġ×IJתר×Ļ×Ŀ":130605,"ĠÄijá»įc":130606,"ĠвоÑĤ":130607,"ãģłãģĮ":130608,"ë°°":130609,"à¸Ķูà¹ģล":130610,"Ġ×ŀ׼׾":130611,"ìĹIJëıĦ":130612,"газ":130613,"Ġ׳×ķספ×Ļ×Ŀ":130614,"ãģĵãģ¨ãģ§":130615,"ĠتÙĪ":130616,"ãģ§ãģĤãĤĬ":130617,"à¸Ļัà¹Īà¸ĩ":130618,"ĠможеÑĤе":130619,"szÄĻ":130620,"ãģ®ãģł":130621,"ĠÙħÙĨÙĩ":130622,"Ġbá»ķ":130623,"Ġbüt":130624,"Ġbütün":130625,"ë³´ê³ł":130626,"Ġchá»ĵng":130627,"à¹ģà¸Īà¹īà¸ĩ":130628,"ĠVì":130629,"ĠØŃر":130630,"Ġgiản":130631,"ĠÙħدÙĬÙĨØ©":130632,"تطبÙĬÙĤ":130633,"à¸Īิ":130634,"æĹ¥ãģ®":130635,"бил":130636,"à¸ģà¸Ńà¸ĩ":130637,"ê³³":130638,"ĠØ£Ùħا":130639,"ìĨIJ":130640,"Ġtrái":130641,"ĠвÑģем":130642,"ĠسÙĨØ©":130643,"ĠÑģайÑĤ":130644,"ĠгоÑĤов":130645,"пÑĭ":130646,"ĠëIJł":130647,"ĠاÙĦخط":130648,"ĠاÙĦرئÙĬسÙĬØ©":130649,"Ġíķ©ëĭĪëĭ¤":130650,"ĠìķĦëĭĪëĿ¼":130651,"ĠìĿ´ëłĩ":130652,"ĠìĿ´ëłĩê²Į":130653,")ØĮ":130654,"hält":130655,"ĠØ£Ùħر":130656,"ĠعÙħر":130657,"à¸ģà¹ĩà¸Īะ":130658,"Ġà¸Ĺำà¹ĥหà¹ī":130659,"Ġcân":130660,"Ġ×ij׾":130661,"Ġ×ij׾×ij×ĵ":130662,"פסק":130663,"ĠÙĬÙĤÙĪÙĦ":130664,"нÑĥÑĤÑĮ":130665,"à¹ģà¸Ħ":130666,"Ġקצת":130667,"Ġnằm":130668,"Ġhòa":130669,"bilitÃł":130670,"ĠìĹĨëĭ¤":130671,"Ġ׼פ×Ļ":130672,"ÑĢож":130673,"лага":130674,"Ġ×Ķש×Ļ":130675,"ĠNgoÃłi":130676,"ĠÙĪØ¬":130677,"ĠÙĪØ¬ÙĪØ¯":130678,"ĠìľĦíķľ":130679,"ĠusÅĤug":130680,"Ġtuần":130681,"dź":130682,"×ŀ×ķף":130683,"ĠاÙĦعدÙĬد":130684,"Ġchẳng":130685,"สุà¸Ĥà¸łà¸²à¸ŀ":130686,"Ġ×ij×ĵר×ļ":130687,"ĠÑģебе":130688,"ĠìŀĪìĿĦ":130689,"ĠاÙĦØŃاÙĦ":130690,"Ġdá":130691,"Ġcưá»Ŀi":130692,"Ġnghiên":130693,"ieÅĦ":130694,"ĠDương":130695,"ï¼ħ":130696,"شد":130697,"ãģĦãģ¤ãĤĤ":130698,"ĠвÑĭбоÑĢ":130699,"Ġcá»Ļng":130700,"ש×Ļ׳×ķ×Ļ":130701,"Ġchạy":130702,"Ġ×ij×¢×ľ×Ļ":130703,"اخبار":130704,"íķĺë©°":130705,"żÄħ":130706,"جاز":130707,"Ġ׳ר×IJ×Ķ":130708,"ศู":130709,"ศูà¸Ļ":130710,"ศูà¸Ļยà¹Į":130711,"×Ĵ×¢":130712,"Ġ×¢×ĵ×Ļ":130713,"Ġ×¢×ĵ×Ļ×Ļף":130714,"برا":130715,"ÑĨий":130716,"ĠÄIJá»ĵng":130717,"ÙĤاÙĨÙĪÙĨ":130718,"ĠÄijứng":130719,"ãģĹãģŁãĤĬ":130720,"Ġ×Ĺ×Ļ×Ļ":130721,"ĠëIJľ":130722,"ĠëIJľëĭ¤":130723,"ĠмеждÑĥ":130724,"à¸ŀวà¸ģà¹Ģà¸Ĥา":130725,"ĠBắc":130726,"ลำ":130727,"ë°±":130728,"ĠíĻķ":130729,"มาà¸ģม":130730,"มาà¸ģมาย":130731,"банк":130732,"à¸Ńาà¸ģาร":130733,"ĠhÃł":130734,"Ġ׾׳":130735,"à¸Ńà¸Ń":130736,"Ġë°Ķë¡ľ":130737,"лом":130738,"mática":130739,"ĠØŃد":130740,"ابت":130741,"à¸Ĺีà¹Īà¸Ļีà¹Ī":130742,"ĠcoÅĽ":130743,"ÙģÙĬدÙĬ":130744,"ÙģÙĬدÙĬÙĪ":130745,"ĠмеÑģÑĤо":130746,"Ġphút":130747,"มาà¸ģà¸ģวà¹Īา":130748,"×IJפ":130749,"بÙIJ":130750,"ĠPhú":130751,"ì±Ħ":130752,"ĠÙĪØ³ÙĦÙħ":130753,"à¸Īีà¸Ļ":130754,"поÑĤÑĢеб":130755,"Ġ×Ĺ×ĵש×ķת":130756,"Ø´ÙĪ":130757,"Ġעצ×ŀ×ķ":130758,"ĠعÙħÙĦÙĬØ©":130759,"à¸Ħุà¸ĵà¸łà¸²à¸ŀ":130760,"ãģ¾ãģĻãģĮ":130761,"دعÙĪ":130762,"طرÙĤ":130763,"à¹Ħมà¹Īà¸ķà¹īà¸Ńà¸ĩ":130764,"ë²Ķ":130765,"ìĬ¹":130766,"ĠkÃŃch":130767,"ĠìĹĨëĬĶ":130768,"ĠÑĤам":130769,"ĠÙĨØŃÙĪ":130770,"ĠاÙĦÙĤاÙĨÙĪÙĨ":130771,"×Ĺ×ķ×Ŀ":130772,"Ġkız":130773,"Ġ×ĵ×Ļף":130774,"ĠвÑĢемени":130775,"ãģ£ãģŁãĤĬ":130776,"ĠØ´Ùĩر":130777,"ĠìĦľë¹ĦìĬ¤":130778,"עש×Ķ":130779,"Ġgiác":130780,"ĠاÙĦسÙĦاÙħ":130781,"Ġ×IJש":130782,"ĠполÑĥÑĩа":130783,"à¸Īัà¸Ķà¸ģาร":130784,"коÑĢ":130785,"Ġ×Ķ×ĺ×ķ×ij":130786,"รายà¸ģาร":130787,"주ìĿĺ":130788,"à¹ģà¸ķà¹Īละ":130789,"Ġê·¸ëŁ°ëį°":130790,"à¸Ĺีà¹Īà¹Ģà¸Ľà¹ĩà¸Ļ":130791,"Ġת×ķ×ļ":130792,"بÙĬاÙĨ":130793,"ÐĻ":130794,"oÅĽciÄħ":130795,"ÑĤок":130796,"ĠÃĶ":130797,"ĠÃĶng":130798,"à¹Ħมà¹Īà¹ĥà¸Ĭà¹Ī":130799,"ãģ¿ãģ¦":130800,"ÐŁÐ¾":130801,"ĠЧÑĤо":130802,"íĻ©":130803,"×ĺ×ij×¢":130804,"меÑĤÑĢ":130805,"Ġ×ij×ŀ×Ķ":130806,"Ġ×ij×ŀ×Ķ׾":130807,"Ġ×ij×ŀ×Ķ׾×ļ":130808,"ÑĩÑĮ":130809,"קש×Ķ":130810,"знак":130811,"знаком":130812,"ujÄĻ":130813,"×Ļצר":130814,"ĠاÙĦÙħÙĦÙĥ":130815,"ıyla":130816,"×IJ×ŀת":130817,"à¸Ľà¸´à¸Ķ":130818,"×IJ×Ĺ×ĵ":130819,"راد":130820,"ĠmáºŃt":130821,"ëĭ¤ëĬĶ":130822,"Ġlạnh":130823,"ש׾×ķש":130824,"ØŃدÙĬØ«":130825,"تز":130826,"å¹´ãģ®":130827,"ĠкваÑĢ":130828,"ĠкваÑĢÑĤиÑĢ":130829,"ä½ľãĤĬ":130830,"رÙĪØ¨":130831,"ован":130832,"ĠТе":130833,"à¸Īำà¸ģ":130834,"à¸Īำà¸ģัà¸Ķ":130835,"باط":130836,"×Ĵת":130837,"ĠмаÑĪ":130838,"ĠмаÑĪин":130839,"×Ļצ×Ķ":130840,"ãģ»ãģ¨":130841,"ãģ»ãģ¨ãĤĵãģ©":130842,"ÃŃdo":130843,"ĠÑıзÑĭк":130844,"à¸ļิà¸Ļ":130845,"สà¸ĸาà¸Ļà¸Ĺีà¹Ī":130846,"ĠìĹ´":130847,"ãĤ¦ãĤ§":130848,"ĠcÃł":130849,"пан":130850,"åı£ãĤ³ãĥŁ":130851,"Ġرد":130852,"اÙĤت":130853,"ĠÙĥب":130854,"ĠÙĥبÙĬرة":130855,"ÑģÑĤал":130856,"ש×ŀ×Ĺ":130857,"posición":130858,"ĠÙħÙĦÙĬÙĪÙĨ":130859,"ĠìĿ´ìķ¼":130860,"ĠìĿ´ìķ¼ê¸°":130861,"Ġhút":130862,"ĠÅĽwiat":130863,"Ġë°©ë²ķ":130864,"ĠÑģвеÑĤ":130865,"Ġвидео":130866,"ĠاÙĦÙĨظاÙħ":130867,"Ġtrá»Ŀi":130868,"ĠëĮĢíķ´ìĦľ":130869,"ר×ŀת":130870,"تداÙĪÙĦ":130871,"×ķר×ĵ":130872,"ת×ŀ":130873,"ת×ŀ×ķ׳×ķת":130874,"Ġ×ŀף":130875,"Ġдва":130876,"Ġ×Ķ×§×ķ":130877,"æĹ¥ãģ«":130878,"Ġ×Ķ×Ĵ×Ļ×¢":130879,"à¹Ģà¸ŀิà¹Īมà¹Ģà¸ķิม":130880,"Ùħارس":130881,"Ġê²ĥìŀħëĭĪëĭ¤":130882,"ãģªãģĦãģ¨":130883,"Ġnhiá»ĩt":130884,"ëIJ©ëĭĪëĭ¤":130885,"Ġ×ij׳×ķש×IJ":130886,"Ġê°Ģìŀ¥":130887,"Ġvợ":130888,"ĠÄijóng":130889,"צ×Ļ׾×ķ×Ŀ":130890,"ê´Ģê³Ħ":130891,"ваÑı":130892,"×IJ×Ļ×ĸ":130893,"×IJ×Ļ×ĸ×Ķ":130894,"ĠÙĨظاÙħ":130895,"ÙħØŃاÙ쨏":130896,"Ġtải":130897,"기ëıĦ":130898,"à¸Ľà¸±à¸Īà¸Īุ":130899,"à¸Ľà¸±à¸Īà¸Īุà¸ļัà¸Ļ":130900,"׼×ĵ×ķר":130901,"ĠìķĦìĿ´":130902,"׼׳×Ļס":130903,"à¹Ģà¸ķร":130904,"à¹Ģà¸ķรียม":130905,"Ġngoại":130906,"ĠدÙĪÙĦار":130907,"Ġrẻ":130908,"ĠkhÄĥn":130909,"عدد":130910,"شعب":130911,"czyÄĩ":130912,"ĠاÙĦÙĥر":130913,"ĠÑĩеловека":130914,"ĠÙĪØ¥ÙĨ":130915,"×IJ×ĺ":130916,"ĠthÆ¡":130917,"ĠاÙĦرÙĬاض":130918,"опÑĢедел":130919,"опÑĢеделен":130920,"×Ķ×ŀש×ļ":130921,"ĠÐĿово":130922,"зÑĭва":130923,"ĠاÙĦدÙĪÙĦÙĬ":130924,"ĠÄijáp":130925,"ĠкÑĢед":130926,"ĠкÑĢедиÑĤ":130927,"ового":130928,"Ġmôn":130929,"à¸Ľà¸£à¸°à¹Ĥย":130930,"à¸Ľà¸£à¸°à¹Ĥยà¸Ĭà¸Ļ":130931,"à¸Ľà¸£à¸°à¹Ĥยà¸Ĭà¸Ļà¹Į":130932,"ÑģÑĤе":130933,"ĠThá»ĭ":130934,"دÙĬØ©":130935,"×ŀצ×ķ":130936,"ÙģØ§Øª":130937,"×§×ĵ×Ŀ":130938,"ìĿ´ëĿ¼ê³ł":130939,"ÙĪØ®":130940,"Ġ×Ĺ×ĸ":130941,"ĠÑĦоÑĤо":130942,"׾×Ļת":130943,"تÙİ":130944,"ÙĪØ¨Ø±":130945,"йÑĤи":130946,"ĠÃ¶ÄŁren":130947,"Ġ×Ķ×ĸ×ķ":130948,"Ġvá»įng":130949,"ÙĤÙĪØ©":130950,"ĠTây":130951,"ĠÐĿи":130952,"Ġש×ķ×ij":130953,"ãģ¨è¨ĢãĤıãĤĮ":130954,"ãģ©ãĤĵãģª":130955,"×Ĺצ×Ļ":130956,"ï½ľ":130957,"Ġ×ķ×Ķ×ķ×IJ":130958,"ä¸Ģãģ¤":130959,"ĠÑģÑĤоиÑĤ":130960,"niÄħ":130961,"×ĺר×Ļ":130962,"ĠдеÑĤей":130963,"нÑıÑĤÑĮ":130964,"ĠÑģделаÑĤÑĮ":130965,"Ġë§İìĿ´":130966,"ä½ķãģĭ":130967,"ãģĽãĤĭ":130968,"à¹Ħหม":130969,"à¸ķิà¸Ķà¸ķà¹Īà¸Ń":130970,"Ġ×ijת×Ĺ":130971,"Ġ×ijת×Ĺ×ķ×Ŀ":130972,"ìĻĦ":130973,"ì§ĢëĬĶ":130974,"ÑģÑĤаÑĤ":130975,"ÑıÑģн":130976,"üb":130977,"Ġthả":130978,"Ġ×ij×IJ×ŀת":130979,"Ġtuyến":130980,"×ĵ×Ļר×Ķ":130981,"Ġ×IJ×Ļש×Ļ":130982,"×ĸ׼ר":130983,"ãģ°ãģĭãĤĬ":130984,"Ġxét":130985,"׼×Ļ×ķ":130986,"׼×Ļ×ķ×ķף":130987,"diÄŁini":130988,"ĠاÙĦÙħÙĪØ¶ÙĪØ¹":130989,"ĠháºŃu":130990,"à¸Īาà¸ģà¸ģาร":130991,"×ijס×Ļס":130992,"Ġ×ŀ×Ĵ×Ļ×¢":130993,"×ij×Ļ×¢":130994,"ĠÙĪØ¬Ùĩ":130995,"à¹ģà¸Ķà¸ĩ":130996,"à¸Ļาà¸ĩ":130997,"ĠÅŀa":130998,"ì¡´":130999,"ë¡Ģ":131000,"à¸ķะ":131001,"Ġ×Ķ×Ĺ×Ļ×Ļ×Ŀ":131002,"ÙģÙĬد":131003,"ãģ§ãģĻãģĭãĤī":131004,"ê·ľ":131005,"źni":131006,"ĠлÑİдей":131007,"Ġyüzde":131008,"ıyorum":131009,"ĠاÙĦبØŃر":131010,"eño":131011,"паÑĢ":131012,"ÙĬÙĤØ©":131013,"обÑĢ":131014,"ר×ķ×ļ":131015,"تÙĪÙĤع":131016,"ĠاÙĦØ´ÙĬØ®":131017,"åĪĿãĤģãģ¦":131018,"ĠÑĤелеÑĦ":131019,"ĠÑĤелеÑĦон":131020,"Ġthôi":131021,"Ġ×Ļ׼×ķ׾×Ļ×Ŀ":131022,"ĠÅŁirk":131023,"ĠÅŁirket":131024,"Ġìļ°ë¦¬ê°Ģ":131025,"ĠÄijông":131026,"Ġת×ķ×ĵ×Ķ":131027,"ÑģмоÑĤÑĢеÑĤÑĮ":131028,"ĠÙĦÙĩÙħ":131029,"Ġ׾׼":131030,"ĠNó":131031,"ĠØŃاÙĦØ©":131032,"ãģĦãģij":131033,"קר×ķ":131034,"azı":131035,"ãĤ³ãĥ¼":131036,"ĠÙĦÙĦت":131037,"sınız":131038,"ĠHải":131039,"기ìĪł":131040,"ยัà¸ĩà¹Ħมà¹Ī":131041,"ëĭ¤ê³ł":131042,"פ×Ĺ":131043,"Ġ׾×Ĵ×ij×Ļ":131044,"ĠعÙĨÙĩ":131045,"Ġказ":131046,"Ġказино":131047,"بÙĪØ±":131048,"ÑĦеÑĢ":131049,"Ġê°ĻìĿ´":131050,"تسجÙĬÙĦ":131051,"ĠاÙĦÙħرÙĥز":131052,"ĠThái":131053,"даÑĤÑĮ":131054,"×ŀ×Ļ×Ļ׾":131055,"ĠpaylaÅŁ":131056,"ãģ¤ãģ®":131057,"à¹Ģรืà¸Ń":131058,"nça":131059,"׳×ķ×Ĺ":131060,"Ġ×IJפ×Ļ׾×ķ":131061,"ãģ¨èĢĥãģĪ":131062,"ãģ¨ãģĹãģ¦ãģ¯":131063,"à¹Ģà¸Īà¸Ń":131064,"×ŀפ":131065,"ĠgiriÅŁ":131066,"лиÑĤ":131067,"ÑĤелÑı":131068,"Ñijн":131069,"æ°Ĺãģ«":131070,"Ġgó":131071,"Ġgóp":131072,"åĪĩãĤĬ":131073,"Ġ×Ķ×Ĺ×ĵש":131074,"жал":131075,"Ġ×ĵעת":131076,"éģķãģĨ":131077,"à¹Ģà¸Ĥà¹īาà¹Ħà¸Ľ":131078,"Ġסר×ĺ":131079,"eña":131080,"æĸ°ãģĹãģĦ":131081,"رÙİ":131082,"ĠÐIJÑĢ":131083,"Ġphản":131084,"à¸Īะà¹Ħà¸Ķà¹ī":131085,"Ġ×ijצ×ķר×Ķ":131086,"شاÙĩ":131087,"شاÙĩد":131088,"ÙĪØ±Ø¯":131089,"à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩà¸Īาà¸ģ":131090,"илиÑģÑĮ":131091,"à¹ģละà¸ģาร":131092,"Ġ×Ķ×ĸ׼":131093,"Ġ×Ķ×ĸ׼×ķ×Ļ×ķת":131094,"eiÃŁ":131095,"ãĥ¨":131096,"ìĥĪ":131097,"ĠÃĩa":131098,"Ư":131099,"ש×Ĵ":131100,"ÙĬÙĨØ©":131101,"รà¹īà¸Ńà¸ĩ":131102,"ãĤµãĥ³":131103,"ÑĢоÑģÑģий":131104,"ÑĢоÑģÑģийÑģк":131105,"aÄŁa":131106,"ĠнаÑĩина":131107,"ĠصÙĦÙī":131108,"à¸Ĺุà¸ģà¸Ħà¸Ļ":131109,"íļĮìĤ¬":131110,"ĠлиÑĨ":131111,"Ø´ÙĬر":131112,"ĠØ´ÙĬØ¡":131113,"ÙĬÙĨا":131114,"Ġפ×Ĺ×ķת":131115,"Ġiçeris":131116,"Ġiçerisinde":131117,"ĠØ£ØŃÙħد":131118,"Ġżeby":131119,"ì´Ŀ":131120,"Ġпоказ":131121,"Ġименно":131122,"หà¸Ļัà¸ĩส":131123,"หà¸Ļัà¸ĩสืà¸Ń":131124,"ĠÑĤÑĢе":131125,"สัà¸ĩà¸Ħม":131126,"Ø¥ÙIJ":131127,"ãģĮå¿ħè¦ģ":131128,"ÙĬÙijØ©":131129,"פצ":131130,"íĭ°":131131,"ĠÙħجاÙĦ":131132,"׳פש":131133,"кан":131134,"×Ĺ×ķפ":131135,"×Ĺ×ķפש":131136,"ì²ĺëŁ¼":131137,"оваÑı":131138,"зов":131139,"Ġhạ":131140,"ĠdziÄĻki":131141,"×Ļר×ķ":131142,"Ġ׾×ŀצ":131143,"Ġ׾×ŀצ×ķ×IJ":131144,"×Ļ×ĵ×ķ":131145,"Ġsợ":131146,"Ġ׾×Ķ×Ĵ×Ļ×¢":131147,"×§×ij×¢":131148,"Ġchiá»ģu":131149,"ãĥŀãĤ¤":131150,"ĠdÃłng":131151,"à¹ģà¸Łà¸Ļ":131152,"Ġüye":131153,"×Ļ׳×Ĵ":131154,"à¹Ģรียà¸ģ":131155,"ç§ģãģĮ":131156,"thé":131157,"ĠÑĦилÑĮ":131158,"ĠÑĦилÑĮм":131159,"ĠNgÃły":131160,"Ġжен":131161,"ĠженÑīин":131162,"جÙĬد":131163,"nç":131164,"à¸Ľà¸£à¸²":131165,"×Ļ×ŀ×ķ":131166,"Ġná»ģn":131167,"×IJ×ķ׾×Ŀ":131168,"ĠвозможноÑģÑĤÑĮ":131169,"Ġëĭ¤ìĭľ":131170,"è¦ĭãģŁ":131171,"à¸ĸà¸Ļ":131172,"à¸ĸà¸Ļà¸Ļ":131173,"mızı":131174,"ĠÙħجÙħÙĪØ¹Ø©":131175,"cjÄħ":131176,"ĠÐłÐ¤":131177,"à¸ģำหà¸Ļ":131178,"à¸ģำหà¸Ļà¸Ķ":131179,"ĠìĹ¬ê¸°":131180,"landı":131181,"ниÑĨ":131182,"ÑģÑĤве":131183,"Ġ×ĵ×ijר×Ļ×Ŀ":131184,"ĠskÅĤad":131185,"ãĤĬãģ¾ãģĹãģŁ":131186,"ĠоÑĤкÑĢÑĭÑĤ":131187,"нÑıÑĤ":131188,"ĠÑģвоей":131189,"à¸Īิà¸ķ":131190,"ĠкаÑĩеÑģÑĤве":131191,"ĠettiÄŁi":131192,"ìĤ¬íķŃ":131193,"ĠاÙĦÙĬÙħÙĨ":131194,"иÑĩеÑģкий":131195,"ë¸Į":131196,"Ġ×ij×IJרץ":131197,"ĠاسÙħ":131198,"ĠизвеÑģÑĤ":131199,"rão":131200,"ĠattivitÃł":131201,"à¹Ģà¸Ľà¹ĩà¸Ļà¸ģาร":131202,"ĠاÙĦدÙĥت":131203,"ĠاÙĦدÙĥتÙĪØ±":131204,"ĠÙĪØ§ØŃدة":131205,"ĠÑģÑĩеÑĤ":131206,"ĠпÑĢиÑĩ":131207,"ĠпÑĢиÑĩин":131208,"ĠÙĪØ²Ø§Ø±Ø©":131209,"Ġhuyá»ĩn":131210,"ĠÙĥتاب":131211,"à¹ģà¸Ļà¹Īà¸Ļ":131212,"à¹ģà¸Ļà¹Īà¸Ļà¸Ńà¸Ļ":131213,"Ġgünü":131214,"гÑĢÑĥз":131215,"ĠاÙĦخاص":131216,"Ġgörül":131217,"׾×ŀ×ĵ":131218,"ĠìłķëıĦ":131219,"×ķ×ij×Ļ׾":131220,"Ġ×ŀקצ×ķ×¢×Ļ":131221,"ĠоÑģобенно":131222,"à¸Ľà¸£à¸°à¸ģา":131223,"à¸Ľà¸£à¸°à¸ģาศ":131224,"acaģını":131225,"ë¶ģ":131226,"à¸łà¸¹à¸¡à¸´":131227,"ĠÑįлекÑĤ":131228,"ĠÑįлекÑĤÑĢо":131229,"Ġקש×Ķ":131230,"سÙĦØ·":131231,"à¸Ĭà¸Ļะ":131232,"×¢×Ļ׾":131233,"ĠЧе":131234,"à¹ģà¸Ļà¹Ī":131235,"lıģ":131236,"lıģın":131237,"Ġ×ŀ×¢×¨×Ľ×ª":131238,"好ãģįãģª":131239,"มาà¸ģà¸Ĥึà¹īà¸Ļ":131240,"×ŀ×¢×ijר":131241,"ĠاÙĦÙħغرب":131242,"ĠпеÑĢи":131243,"ĠпеÑĢиод":131244,"Ġnhạc":131245,"اÙĪÙĬ":131246,"ĠÙĪØ¹ÙĦÙī":131247,"أخذ":131248,"ĠCô":131249,"תר×ij×ķת":131250,"×Ĵ×Ķ":131251,"Ġktórej":131252,"×IJ×Ļת":131253,"×ij×ķ×IJ":131254,"делÑĮ":131255,"รีวิ":131256,"รีวิว":131257,"жÑĥ":131258,"Ġ×ij×Ĺ×ķ":131259,"еÑĪÑĮ":131260,"ĠØ£ÙĦÙģ":131261,"ĠاÙĦÙĪØ·ÙĨÙĬ":131262,"ĠاÙĦÙħÙĨØ·ÙĤØ©":131263,"nÄħÄĩ":131264,"Ġthiên":131265,"иÑĩеÑģкой":131266,"ĠاÙĦÙħÙĦ":131267,"ĠعÙħ":131268,"ספר":131269,"Ġnhóm":131270,"ÙĪØµÙģ":131271,"ĠChúng":131272,"ĠرÙĤÙħ":131273,"ãģ¾ãģĹãģŁãģĮ":131274,"alité":131275,"ลม":131276,"ĠëĤ´ê°Ģ":131277,"׾ק×ķ×Ĺ":131278,"ĠSÆ¡n":131279,"posição":131280,"miÄĻ":131281,"Ġtránh":131282,"ĠÄIJá»Ļ":131283,"׼×Ĺ":131284,"ãģĤãģ£ãģ¦":131285,"à¸Ńยà¹Īา":131286,"Ġ×ŀ×Ĺ×Ļר":131287,"Ġ×Ķ×Ļת×Ķ":131288,"à¸Ľà¹Īา":131289,"à¸Ńืà¹Īà¸Ļà¹Ĩ":131290,"Ø´ÙĤ":131291,"×ł×¡×Ļ":131292,"림":131293,"ãģ¦ãģĹãģ¾ãģĨ":131294,"Ġ×ŀצ×ij":131295,"ãģ«åĩº":131296,"ÙħÙĪØ§Ø·ÙĨ":131297,"ยัà¸ĩมี":131298,"алÑĮнÑĭе":131299,"sanız":131300,"إسرائÙĬÙĦ":131301,"ĠvÃłi":131302,"ì¤Ħ":131303,"ã썿ĢĿãģ£ãģ¦":131304,"×Ļ×ķ׳×Ļ":131305,"çĶŁãģį":131306,"Ġsâu":131307,"ÑĩиÑģÑĤ":131308,"Ġlá»ħ":131309,"ĠGiá":131310,"à¸Ńà¸¸à¸Ľ":131311,"à¸Ńà¸¸à¸Ľà¸ģร":131312,"à¸Ńà¸¸à¸Ľà¸ģรà¸ĵà¹Į":131313,"Ġnhẹ":131314,"rö":131315,"ס×ĺ×Ļ":131316,"ãģķãĤĵãģĮ":131317,"Ġdầu":131318,"عÙİ":131319,"ترا":131320,"×Ĵ×ĵ׾":131321,"Ġtécnica":131322,"׼׳×Ļ×Ŀ":131323,"תקש":131324,"תקש×ķרת":131325,"Ġнего":131326,"était":131327,"Ġmá»ģm":131328,"ÑģеÑĤ":131329,"ĠnháºŃt":131330,"Ġ×ŀ×¢×ľ":131331,"Ġ×Ķ×¢×ij×ķ×ĵ":131332,"Ġ×Ķ×¢×ij×ķ×ĵ×Ķ":131333,"Ġ×Ĵ×Ļ׾":131334,"ãģ¯ãģªãģĦ":131335,"ائØŃ":131336,"ĠздеÑģÑĮ":131337,"×IJ×Ļ׳×ĺר":131338,"ÙħÙIJ":131339,"Ġ×Ļ×Ĺ×ĵ":131340,"راÙģ":131341,"ì²ĺ리":131342,"×ĵ×¢×ķת":131343,"ì¹ľ":131344,"ĠТо":131345,"ĠThế":131346,"ì¶©":131347,"Ġ׳׼×ķף":131348,"عÙĬØ´":131349,"низ":131350,"ĠجاÙĨب":131351,"×ŀקצ×ķ×¢":131352,"à¹Ĥà¸ĭ":131353,"ÑģÑĥÑĤ":131354,"ìĸ´ìļĶ":131355,"ãĤĴè¦ĭãģ¦":131356,"ارد":131357,"Ġaçıl":131358,"ĠاÙĦØŃÙĬاة":131359,"à¸ģà¹ĩà¹Ħà¸Ķà¹ī":131360,"ãģĿãĤĮãĤĴ":131361,"عضÙĪ":131362,"ĠгÑĢаж":131363,"ĠгÑĢаждан":131364,"à¸Īะà¸ķà¹īà¸Ńà¸ĩ":131365,"ĠìĿ´ë٬":131366,"ĠìĿ´ë٬íķľ":131367,"Ġtrách":131368,"ÙĨÙİ":131369,"Ġkısa":131370,"ÃĶ":131371,"ÑĪка":131372,"ãģ®äºº":131373,"ĠÐŁÐ¾Ñģ":131374,"ĠÐŁÐ¾Ñģле":131375,"ÑĥлÑĮ":131376,"ÙĪØ§Ø¬Ùĩ":131377,"ÙĤرب":131378,"à¸Ľà¸ıิà¸ļัà¸ķิ":131379,"ê°Ļ":131380,"Ġ×ŀ׳":131381,"ĠÑģвои":131382,"براÙħج":131383,"ĠرÙĪ":131384,"пÑĢод":131385,"пÑĢодаж":131386,"ĠbyÅĤy":131387,"วัย":131388,"Ġgörün":131389,"ĠÃĪ":131390,"ÑİÑīим":131391,"ĠÑĤакой":131392,"ÙģÙĪØ±":131393,"ĠÙ쨹ÙĦ":131394,"Ġбел":131395,"ëIJł":131396,"erÃŃa":131397,"ĠÑģвоÑİ":131398,"Ġlã":131399,"Ġlãnh":131400,"à¹Ģà¸ŀืà¹Īà¸Ńà¹ĥหà¹ī":131401,"ÙĤÙĨ":131402,"تطÙĪÙĬر":131403,"Ġsayı":131404,"ĠÑģейÑĩаÑģ":131405,"Ġ×IJ×Ĺרת":131406,"×§×ķפ×Ķ":131407,"×§×ķרס":131408,"ĠسÙħ":131409,"Ġ×ĺ×Ļפ×ķ׾":131410,"ìĿ´ëĿ¼ëĬĶ":131411,"دراسة":131412,"èµ·ãģĵ":131413,"×Ĺ×Ļ׳":131414,"×Ĺ×Ļ׳×ķ×ļ":131415,"×ĵ×§":131416,"Ġë§ŀ":131417,"Ġкоманд":131418,"ĠÐijо":131419,"ĠигÑĢÑĭ":131420,"à¸ļี":131421,"ĠØ£Ùİ":131422,"вен":131423,"ĠاÙĦجدÙĬد":131424,"ĠÙĦØ¥":131425,"Ġ×ķ×IJ׳×Ļ":131426,"Ġ×Ķס×Ļ":131427,"иÑĩеÑģкого":131428,"رÙĪØŃ":131429,"à¸ģารศึà¸ģษา":131430,"ĠTrưá»Ŀng":131431,"игÑĢа":131432,"ılması":131433,"ĠмаÑģÑģ":131434,"ãģ¨ãģįãģ«":131435,"à¸Ĺีà¹Īà¸ľà¹Īาà¸Ļ":131436,"à¸Ĺีà¹Īà¸ľà¹Īาà¸Ļมา":131437,"ĠاÙĦسابÙĤ":131438,"Ġ×ŀ×¢×ĺ":131439,"ваÑĤÑĮ":131440,"mÃ¼ÅŁ":131441,"Ġ׾׼×ļ":131442,"Ġtá»ĭch":131443,"ÙģÙĩÙħ":131444,"تدرÙĬب":131445,"Ø´Ùĥ":131446,"Ġ×ij×ŀ×Ļ":131447,"Ġ×ij×ŀ×Ļ×ķ×Ĺ×ĵ":131448,"ÙĤطاع":131449,"ãģªãģĹ":131450,"×ķצ×Ļ×IJ":131451,"ĠÙĪØ³ÙĬ":131452,"зÑĥ":131453,"Ġyat":131454,"Ġyatırım":131455,"ë§İ":131456,"Ġthắng":131457,"ãģĬ客":131458,"ãģĬ客æ§ĺ":131459,"ĠThiên":131460,"ãģ«å¯¾ãģĹãģ¦":131461,"ÑĢиÑģ":131462,"ÙĨتائ":131463,"ÙĨتائج":131464,"Ġ×ŀשר":131465,"Ġ×ŀשר×ĵ":131466,"ĠتعاÙĦ":131467,"ĠتعاÙĦÙī":131468,"ש׳×Ļ":131469,"ÙĩاÙħ":131470,"×IJ׳ש×Ļ×Ŀ":131471,"Ġżycia":131472,"ĠÑĢÑĥблей":131473,"ÙĬض":131474,"Ġkatıl":131475,"ĠÙħÙĪØ¶ÙĪØ¹":131476,"Ġvardır":131477,"ĠÙħÙĨØ·ÙĤØ©":131478,"ĠTrần":131479,"ĠвеÑģ":131480,"üp":131481,"ÙħÙĪÙĨ":131482,"ÑĪли":131483,"Ġnóng":131484,"Ø®ÙĦÙģ":131485,"ĠСÑĤа":131486,"ĠдоÑĢ":131487,"ĠдоÑĢог":131488,"ĠwÅĤaÅĽnie":131489,"eÄŁin":131490,"Ġhiá»ĥm":131491,"ĠСам":131492,"ê»ĺìĦľ":131493,"ĠÑĦа":131494,"ãģ»ãģĨ":131495,"ãģ»ãģĨãģĮ":131496,"×ķפ×Ļ×¢":131497,"ê°Ī":131498,"دÙĪÙĦ":131499,"Ġthuê":131500,"Ġchá»Ĺ":131501,"Ġëĭ¹ìĭł":131502,"ãģijãĤĮ":131503,"ãģijãĤĮãģ©":131504,"ë³´íĺ¸":131505,"ãģķãĤĮãģ¦ãģĦãģ¾ãģĻ":131506,"Ġнадо":131507,"ĠìĤ¬ëŀĮëĵ¤":131508,"à¹Ģà¸Ĥà¸ķ":131509,"สมัย":131510,"zÅĤ":131511,"تÙĪØ±":131512,"Ġשת×Ļ":131513,"vê":131514,"Ġ×ijת×ķ×ļ":131515,"à¸Ĭัย":131516,"ãģĦãģ£ãģŁ":131517,"ìĿij":131518,"Ġtầ":131519,"Ġtầng":131520,"ש׼ר":131521,"Ġê¸Ģ":131522,"Ġ×Ķש׳×Ķ":131523,"ĠاÙĨÙĩ":131524,"ç«ĭãģ¡":131525,"rés":131526,"führen":131527,"رØŃÙħ":131528,"ê·¹":131529,"ĠâĢ«":131530,"Ġsuất":131531,"à¸Łà¸´":131532,"ÙĬÙĩا":131533,"ĠاÙĦاتØŃاد":131534,"Ġtuyá»ĥn":131535,"ãģ¾ãĤĭ":131536,"Ġmại":131537,"Ġngân":131538,"ãĤ°ãĥ©":131539,"欲ãģĹãģĦ":131540,"سار":131541,"ãĤĤãģ®ãģ§ãģĻ":131542,"кие":131543,"Ġseçim":131544,"åħ¥ãĤĬ":131545,"ãģªãģ©ãĤĴ":131546,"ÑĤÑĢи":131547,"ĠÑģпеÑĨ":131548,"Ġأد":131549,"Ġодно":131550,"ÑĪел":131551,"ãĥĩãĥ¼ãĤ¿":131552,"ãĤ·ãĤ¹ãĥĨ":131553,"ãĤ·ãĤ¹ãĥĨãĥł":131554,"è¡Įãģį":131555,"ã썿ĢĿãģ£ãģŁ":131556,"à¹Ģà¸ģิà¸Ķà¸Ĥึà¹īà¸Ļ":131557,"ĠÑĤож":131558,"ĠÑĤоже":131559,"Ġsạch":131560,"ĠÑģÑĢок":131561,"ĠклиенÑĤ":131562,"ĠÙħشرÙĪØ¹":131563,"Ġaltında":131564,"Ġì·¨":131565,"ä¸Ńãģ®":131566,"ãģķãģĽãĤĭ":131567,"ãģĻãģ¹":131568,"ãģĻãģ¹ãģ¦":131569,"ê°ľë°ľ":131570,"ĠÄijêm":131571,"ãģªãģĦãģ®ãģ§":131572,"ì²ł":131573,"×¢×ij×ĵ":131574,"Ġdấu":131575,"à¸Ħà¸Ļà¸Ĺีà¹Ī":131576,"ĠCách":131577,"تعÙĦÙĬÙħ":131578,"Ġhại":131579,"ãĤ»ãĥķãĥ¬":131580,"ĠÙĨÙ쨳Ùĩ":131581,"ĠíĨµíķ´":131582,"ÑĪло":131583,"ĠнапÑĢав":131584,"ĠнапÑĢавлен":131585,"ÑĢÑĥÑĩ":131586,"íĶĮ":131587,"Ġ×ijר×Ļ×IJ":131588,"ãģ®ãģ¿":131589,"ãģ«ãģĬãģĦãģ¦":131590,"×ij׳ק":131591,"ãĤ¨ãĥ³":131592,"Ø«ÙĦاث":131593,"Ġmỹ":131594,"ĠÑģайÑĤе":131595,"ĠемÑĥ":131596,"تغÙĬ":131597,"تغÙĬÙĬر":131598,"خصÙĪØµ":131599,"ÑĤели":131600,"Ġ×ķ׾׼ף":131601,"פע×Ŀ":131602,"ĠпоÑįÑĤомÑĥ":131603,"راÙĨ":131604,"иÑĤелей":131605,"пиÑģан":131606,"×¢×¥":131607,"ĠìĤ¬ìĹħ":131608,"Ùħز":131609,"جÙħÙĬع":131610,"ë©´ìĦľ":131611,"à¸ľà¸¥à¸´à¸ķà¸łà¸±":131612,"à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵ":131613,"à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵà¸ij":131614,"à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵà¸ijà¹Į":131615,"ĠпÑĢимеÑĢ":131616,"ãĤŃãĥ¼":131617,"lâ":131618,"ĠchÄĥm":131619,"缮ãģ®":131620,"ãģĦãģĭ":131621,"ãģ¨è¨ĢãģĨ":131622,"×ĸ×ķ×Ĵ":131623,"Ġ×ij×ĵ×Ļ":131624,"Ġ×ij×ĵ×Ļ×ķ×§":131625,"ãģĬåºĹ":131626,"à¸ķà¸Ńà¸Ļà¸Ļีà¹ī":131627,"Ġphá»iji":131628,"пÑĤ":131629,"สà¸Ļาม":131630,"Ø·ÙĪ":131631,"صاØŃ":131632,"صاØŃب":131633,"ĠDü":131634,"ĠDünya":131635,"Ġпока":131636,"пал":131637,"ĠÄijảo":131638,"ĠاÙĦÙģÙĪØ±":131639,"ĠاÙĦÙģÙĪØ±Ùĥس":131640,"Ġmáu":131641,"кÑĢеп":131642,"ĠاÙĦساعة":131643,"ĠгоÑĢода":131644,"Ù쨵ÙĦ":131645,"айÑĤе":131646,"Ġдог":131647,"ĠдоговоÑĢ":131648,"Ġإذ":131649,"Ġ×ij׼׾׾":131650,"ÙĬتÙĩ":131651,"×Ĵ×ijר":131652,"Ġbirç":131653,"Ġbirçok":131654,"문íĻĶ":131655,"ãģĿãģĨãģª":131656,"راØŃ":131657,"ĠÙħرة":131658,"ĠденÑĮги":131659,"fä":131660,"à¸Ĥà¹īาว":131661,"ĠÑģовÑĢем":131662,"ĠÑģовÑĢеменн":131663,"׾×Ĺ×¥":131664,"èī¯ãģı":131665,"ĠÙ쨣":131666,"Ġ×ķ×ĸ×Ķ":131667,"Ġзани":131668,"Ġзанима":131669,"Ġê°Ģì§Ģê³ł":131670,"ĠhÆ¡i":131671,"ãģªãģ®ãģĭ":131672,"ãĥĨãĥ¬ãĥĵ":131673,"Ġר×ij×ķת":131674,"à¸ķี":131675,"Ġ×ij×©×ł×ª":131676,"ĠTại":131677,"ĠthuáºŃn":131678,"Ñģел":131679,"Ñijм":131680,"dziÄĩ":131681,"ĠÑģка":131682,"ĠÑģкаÑĩ":131683,"ĠÑģкаÑĩаÑĤÑĮ":131684,"×ķ×ŀ×ķ":131685,"гла":131686,"ĠминÑĥÑĤ":131687,"åĩºãģĻ":131688,"Ġ×Ĺ×Ļ×Ļ×ij":131689,"Ġת×Ĵ×ķ×ij×Ķ":131690,"à¸£à¸¹à¸Ľà¹ģà¸ļà¸ļ":131691,"ниÑĨа":131692,"Ġİn":131693,"Ġأع":131694,"ĠضÙħÙĨ":131695,"ÙħثاÙĦ":131696,"ĠyaÅŁan":131697,"ĠìĹ°êµ¬":131698,"ĠLê":131699,"ש׾×Ĺ":131700,"ãģıãģªãĤĭ":131701,"ìĹĨìĿ´":131702,"ĠÑĤÑĢи":131703,"ĠÑĩаÑģÑĤо":131704,"ĠобÑĢаÑĤ":131705,"пло":131706,"دخ":131707,"دخÙĪÙĦ":131708,"سÙĩ":131709,"à¸Ńาà¸ģ":131710,"à¸Ńาà¸ģาศ":131711,"Ġ׼×ĸ×Ķ":131712,"Ġ×Ķעסק":131713,"ĠاÙĦØ£ÙĨ":131714,"å¹´ãģ«":131715,"עש×ķ":131716,"Ġשע×ķת":131717,"ĠmÃłn":131718,"×IJר×Ļ":131719,"sıyla":131720,"Ù쨱ÙĤ":131721,"ниÑħ":131722,"Ġتست":131723,"è¦ĭãģ¦":131724,"ØŃاÙĪÙĦ":131725,"×IJ×Ļ׼×ķת":131726,"ĠbaÅŁladı":131727,"stÄħ":131728,"stÄħpi":131729,"à¸Ĺีà¹Īà¹Ģรา":131730,"ÙĤرر":131731,"جاب":131732,"Ġ×ijר×ķר":131733,"à¹Ģà¸Ĥà¹īาà¹ĥà¸Ī":131734,"×ŀ×Ĺקר":131735,"alım":131736,"Ġס×Ļפ×ķר":131737,"ãģ§ãģĤãĤĮãģ°":131738,"Ġש×ŀ×ķר×ķת":131739,"Ġ×ķ×ŀ×Ķ":131740,"ãģĵãģĿ":131741,"idée":131742,"ä¸ĭãģķãģĦ":131743,"تÙĨاÙĪÙĦ":131744,"Ġลà¹īาà¸Ļ":131745,"Ġìļ°ë¦¬ëĬĶ":131746,"اÙĨا":131747,"ÑģÑĤой":131748,"боÑĤ":131749,"ĠyaÅŁam":131750,"köy":131751,"Ø¥ÙĦ":131752,"ÑĢÑĭв":131753,"기ìĹħ":131754,"Ġ×Ķ×ŀ×ĵ":131755,"Ġ×Ķ×ŀ×ĵ×Ļ׳×Ķ":131756,"دب":131757,"×¢×Ļ׳×Ļ":131758,"×ŀת×Ĺ":131759,"Ġפר×Ļ":131760,"ãĥĭãĥ¼":131761,"اÙħÙĬ":131762,"Ġnhằm":131763,"ãĤĮãģªãģĦ":131764,"تعرÙģ":131765,"Ġë§ĪìĿĮ":131766,"ìĵ°":131767,"Ġhấp":131768,"ר×Ĵ×Ļ׾":131769,"بÙİ":131770,"ĠrÄĥng":131771,"glÄħd":131772,"ĠÑģиÑģÑĤемÑĭ":131773,"Ġkhóa":131774,"ãģ§ãģĻãĤĪãģŃ":131775,"大ãģįãģı":131776,"기를":131777,"Ġkéo":131778,"ÙĪØ¡":131779,"جاÙħ":131780,"جاÙħع":131781,"Ġ×¢×Ļצ×ķ×ij":131782,"téri":131783,"Ġתש":131784,"Ġ×IJ×ij×Ļ":131785,"ĠChương":131786,"à¸ļริà¹Ģว":131787,"à¸ļริà¹Ģวà¸ĵ":131788,"ãģ¤ãģı":131789,"Ġ×Ĺ×ķ׾":131790,"עת×Ļ×ĵ":131791,"ש×Ļ×ŀ×Ķ":131792,"ëĤ¨":131793,"Ġש×IJ×Ļף":131794,"ĠÙĪØ§ÙĦØ¥":131795,"ÑĦа":131796,"Ġkhám":131797,"Ġ×ĺ×ķ×ij×Ķ":131798,"ĠвÑĭÑģ":131799,"ĠвÑĭÑģоко":131800,"ĠاÙĦØŃدÙĬØ«":131801,"人ãĤĤ":131802,"dÃ¼ÄŁÃ¼":131803,"×Ļ×Ĺ×ķ×ĵ":131804,"تعÙĦÙĬ":131805,"تعÙĦÙĬÙĤ":131806,"lö":131807,"تØŃدÙĬد":131808,"него":131809,"ĠÑĥдоб":131810,"Ġ׾×ŀ×Ļ":131811,"Ġר×ķצ×Ļ×Ŀ":131812,"Ġجاء":131813,"Ġ×ij×ĸ×ŀף":131814,"à¸Ľà¸ģà¸ķิ":131815,"é«ĺãģı":131816,"à¸Ľà¸¥à¸²":131817,"Ġartık":131818,"Ġbugün":131819,"ק׳×Ļ":131820,"Ġkhoá":131821,"ĠÙħرÙĥز":131822,"ĠìŀIJ기":131823,"درجة":131824,"×ŀשר×ĵ":131825,"Ġgiấy":131826,"Ġchóng":131827,"קפ":131828,"ÙĬبة":131829,"ĠczÄĻsto":131830,"вали":131831,"Ùĥب":131832,"ìŁģ":131833,"สà¸ļาย":131834,"à¸Ľà¸£à¸°à¸Ĭาà¸Ĭà¸Ļ":131835,"×Ĵ×ķ×£":131836,"ëŁī":131837,"ãģ®ãģĵãģ¨":131838,"ลà¸Ń":131839,"Ġnghá»ī":131840,"åŃIJãģ©":131841,"åŃIJãģ©ãĤĤ":131842,"à¹Ħà¸Ķà¹īà¸Ńย":131843,"à¹Ħà¸Ķà¹īà¸Ńยà¹Īาà¸ĩ":131844,"×ĵ×¢":131845,"ĠاÙĦتÙī":131846,"ĠÑģовеÑĤ":131847,"ĠqualitÃł":131848,"åĩºãģĹ":131849,"ĠÑĢÑĥков":131850,"ĠÑĢÑĥковод":131851,"รายละà¹Ģà¸Ńียà¸Ķ":131852,"ãģªãģĭãģªãģĭ":131853,"기ê´Ģ":131854,"Ġ×Ĺ×ķש":131855,"Ġ×Ĺ×ķש×ij":131856,"лоÑĤ":131857,"à¸Ļะà¸Ħรัà¸ļ":131858,"×§×ij×ķצ×Ķ":131859,"Ġthái":131860,"Ġש×ij×Ķ":131861,"ĠÑĪкол":131862,"ĠÙĦÙĥÙĦ":131863,"à¹ĥà¸Ļà¸Ĭà¹Īวà¸ĩ":131864,"ĠÙħÙĥاÙĨ":131865,"ëķĮ":131866,"Ġcải":131867,"ĠChÃŃ":131868,"ÑĥÑĩа":131869,"ìĿµ":131870,"Ġxảy":131871,"à¸Ĭà¸Ļิà¸Ķ":131872,"ĠcáºŃu":131873,"кÑĢов":131874,"ssé":131875,"ĠÙĨÙĪØ¹":131876,"ĠТа":131877,"Ø®Ùħس":131878,"פ×ķס×ĺ":131879,"Ġmắc":131880,"ĠÄijem":131881,"à¸ģารà¹ĥà¸Ĭà¹ī":131882,"ר×ķס":131883,"ĠÐĽÐµ":131884,"Ġthá»Ń":131885,"รà¹Īาà¸ĩà¸ģาย":131886,"üzü":131887,"æĹ¥æľ¬ãģ®":131888,"ê³¼ìłķ":131889,"ש×Ļ×IJ":131890,"ĠìŀĪê³ł":131891,"×ij×ķ׾":131892,"ìķħ":131893,"ĠÙĪØ§ÙĦا":131894,"ĠÐĽÐ¸":131895,"ĠвÑģÑij":131896,"Ġużytkow":131897,"×Ĺ×ķ׾":131898,"رÙ쨶":131899,"Ġsonuç":131900,"ãģĦãģ¾ãģĽãĤĵ":131901,"ìĤ¬ìĹħ":131902,"ëĪĦ":131903,"ÑĤек":131904,"ĠudziaÅĤ":131905,"лез":131906,"Ġ×Ķ×Ļ×Ļת×Ļ":131907,"ãĤīãĤĮãģ¦":131908,"ÙħسؤÙĪÙĦ":131909,"رار":131910,"ÑĤан":131911,"ĠÄijÃło":131912,"Ġר×ķ×ij":131913,"Ġ×ijש×ij×Ļ׾":131914,"ä»ĬåĽŀãģ¯":131915,"ãĤ¸ãĥ¥":131916,"Ġ×¢×ijר":131917,"ãģĽãģ¦":131918,"полÑĮ":131919,"aklı":131920,"ĠkÃŃnh":131921,"دت":131922,"ложение":131923,"ĠاÙĦÙħص":131924,"ĠاÙĦÙħصرÙĬ":131925,"à¸Īริà¸ĩà¹Ĩ":131926,"ĠاÙĦشرÙĥØ©":131927,"ĠÄijá»ı":131928,"ãĥĽãĥĨ":131929,"ãĥĽãĥĨãĥ«":131930,"Ñįкон":131931,"Ñįконом":131932,"ĠÙĪØ¹ÙĨ":131933,"Ġ×ª×ł":131934,"Ġ×ª×ł×IJ×Ļ":131935,"ĠاÙĦدÙĪÙĦÙĬØ©":131936,"Ġì§ĢìĹŃ":131937,"ãģ§ãģĻãģĭ":131938,"ĠваÑĢи":131939,"ĠваÑĢианÑĤ":131940,"ĠاÙĦعرب":131941,"ела":131942,"ĠtÆ°á»Ľng":131943,"skÄħ":131944,"Ġmặc":131945,"สัà¸ģ":131946,"ãĥĵãĥ¼":131947,"Ġ×ij×Ĵ׾":131948,"Ġ×ij×Ĵ׾׾":131949,"ãĥķãĤ¡ãĥ³":131950,"×ij×Ļצ":131951,"×ij×Ļצ×ķ×¢":131952,"лиÑģÑĤ":131953,"à¸Łà¸¸":131954,"à¸Łà¸¸à¸ķ":131955,"à¸Łà¸¸à¸ķà¸ļà¸Ńล":131956,"à¸Ŀà¹Īาย":131957,"ìŀIJìĿĺ":131958,"ĠسÙĪÙģ":131959,"Ġש×Ķת":131960,"Ġ걸":131961,"×¢×ij×ķ×ĵ":131962,"ãģĻãĤĭãģĵãģ¨ãģĮ":131963,"ĠÑĩаÑģÑĤÑĮ":131964,"ãĤ¢ãĥ¡ãĥª":131965,"ãĤ¢ãĥ¡ãĥªãĤ«":131966,"Ġtakım":131967,"ĠsỼ":131968,"ĠsỼm":131969,"שר×Ķ":131970,"è¨ĢãģĨ":131971,"лан":131972,"커":131973,"׼׳×Ķ":131974,"ÙĪÙģÙĬ":131975,"íĹĪ":131976,"luÄŁu":131977,"ĠëĮĢíķ´":131978,"Ġ׾×ij×Ļת":131979,"Ġ×Ķר×IJש×ķ׳×Ķ":131980,"صÙħ":131981,"Ġsöyled":131982,"Ġsöyledi":131983,"à¸Ľà¸²à¸ģ":131984,"Ġardından":131985,"ãģĪãģŁ":131986,"à¸Ĺัà¹Īวà¹Ħà¸Ľ":131987,"Ġ׳×ķסף":131988,"болÑĮ":131989,"ãĤĵãģ§ãģĻãģijãģ©":131990,"ĠлиÑĪÑĮ":131991,"Ġ×ij×IJ×Ļ":131992,"ĠбÑĭÑģÑĤÑĢо":131993,"สัà¸Ļ":131994,"Ġ×ijפ׳×Ļ":131995,"леÑĩ":131996,"ĠاÙĦخبر":131997,"Ġsóc":131998,"Ġthú":131999,"ĠпÑıÑĤ":132000,"ãģĬé¡ĺ":132001,"ãģĬé¡ĺãģĦ":132002,"ÑĤин":132003,"ãģ«ãģ¤ãģĦãģ¦ãģ¯":132004,"פף":132005,"ĠдвÑĥÑħ":132006,"à¸įีà¹Ī":132007,"à¸įีà¹Īà¸Ľ":132008,"à¸įีà¹Īà¸Ľà¸¸":132009,"à¸įีà¹Īà¸Ľà¸¸à¹Īà¸Ļ":132010,"опеÑĢ":132011,"ĠاÙĦبشر":132012,"ĠاÙĦÙħاÙĦ":132013,"ıyoruz":132014,"تØŃÙħÙĬÙĦ":132015,"à¸ģะ":132016,"éĸĵãģ«":132017,"×Ĺ×ķש":132018,"ĠNguyên":132019,"ãģĦãģ¦ãģĦãĤĭ":132020,"дÑĥÑĪ":132021,"שפע":132022,"ÑĪÑĥ":132023,"å®ŁéļĽãģ«":132024,"ĠÑĢайон":132025,"ĠChá»ī":132026,"ÙĨصر":132027,"Ġìļ´":132028,"Ġìļ´ìĺģ":132029,"Ġ×Ķ×ĵ×Ļף":132030,"ØŃدد":132031,"رز":132032,"ĠاÙĦدÙħ":132033,"ĠPháp":132034,"ÑĤÑģÑı":132035,"è¦ĭãģĪ":132036,"Ġtiá»ĥu":132037,"Ġsá»Ńa":132038,"аÑİÑĤÑģÑı":132039,"ĠBá":132040,"Ġ×ķ׼׾":132041,"Ðĸ":132042,"ÑĪим":132043,"ìĿ´ëĬĶ":132044,"лев":132045,"dık":132046,"Ġprésente":132047,"Ġaraç":132048,"صدÙĤ":132049,"Ġпомог":132050,"ĠاÙĦشرÙĤ":132051,"ĠÙĪØ§ÙĦذÙĬ":132052,"رÙĬا":132053,"×ij׳×ķת":132054,"Ġngá»ĵi":132055,"ר×ķפ":132056,"ר×ķפ×IJ":132057,"Ġthấp":132058,"ãĤĦãģ¯":132059,"ãĤĦãģ¯ãĤĬ":132060,"ĠاÙĦجدÙĬدة":132061,"éĿŀ常ãģ«":132062,"ÙĬÙĦÙĬ":132063,"쪽":132064,"تعاÙħÙĦ":132065,"ãģłã썿ĢĿãģĦãģ¾ãģĻ":132066,"ÙħÙħ":132067,"иÑĤели":132068,"ãĤµãĤ¤ãĤº":132069,"ادات":132070,"ĠاÙĦÙħاÙĦÙĬØ©":132071,"Ùĥاتب":132072,"кли":132073,"веÑĢÑħ":132074,"ниÑĩ":132075,"Ġ×ľ×¢×ij×ķ×ĵ":132076,"׾×Ļ×Ķ":132077,"ØŃÙİ":132078,"ãĤ¤ãĥĻ":132079,"ãĤ¤ãĥĻãĥ³ãĥĪ":132080,"Ġת×Ĵ×ķ×ij×ķת":132081,"ÑĦон":132082,"ĠдÑĢÑĥгие":132083,"×IJ×ĸ×ķר":132084,"Ġperò":132085,"ìķŀ":132086,"åĢŁãĤĬ":132087,"רצ×Ļ":132088,"×IJ×ĸ":132089,"алÑĮнÑĭÑħ":132090,"Ġê²ĥìľ¼ë¡ľ":132091,"ĠпÑĢаво":132092,"ĠاÙĦأرض":132093,"à¹Ģà¸Ĺà¸Ħ":132094,"à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļ":132095,"à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥล":132096,"à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥลย":132097,"à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥลยี":132098,"צר×Ļ":132099,"ĠÐļÑĥ":132100,"ılma":132101,"決ãĤģ":132102,"اÙĪ":132103,"Ġ×ĵ×§×ķת":132104,"à¸Ħรู":132105,"ĠÙħستÙĪÙī":132106,"à¸Ľà¹īà¸Ńà¸ĩ":132107,"à¸Ľà¹īà¸Ńà¸ĩà¸ģัà¸Ļ":132108,"×ĵ×ķ×ŀ×Ķ":132109,"ĠÑģегоднÑı":132110,"سÙĪÙĤ":132111,"ר×Ĺ×ķ×ij":132112,"Ġإدارة":132113,"Ñħож":132114,"éģİãģİ":132115,"à¸Ħà¸Ń":132116,"нÑĥл":132117,"×ķ׼×Ķ":132118,"ÙĪØ§ÙģÙĤ":132119,"׼׾׾":132120,"Ġ×Ķ×ĵ×ķ":132121,"ĠlÄ©nh":132122,"Ġkhảo":132123,"×IJ×ŀצע":132124,"머":132125,"Ġ׼×Ļצ":132126,"Ġ׼×Ļצ×ĵ":132127,"ĠдолжнÑĭ":132128,"หวัà¸ĩ":132129,"ãĥĩãĤ¶":132130,"ãĥĩãĤ¶ãĤ¤ãĥ³":132131,"Ġngá»Ŀ":132132,"ä¸Ńãģ«":132133,"à¸ģลัà¸ļมา":132134,"جÙħاÙĦ":132135,"à¸Ķัà¸ĩà¸ģลà¹Īาว":132136,"سÙĥÙĨ":132137,"سÙĨ":132138,"Ġözellikle":132139,"зеÑĢ":132140,"rzÄĻ":132141,"×ŀ×ķר×Ķ":132142,"Ġlạ":132143,"×ŀ×Ļ׳×Ļ":132144,"ר×Ļת":132145,"ãģĿãĤĮãģĮ":132146,"ãģĭãĤĮ":132147,"ĠÙĬÙħÙĥÙĨÙĥ":132148,"öffentlich":132149,"ган":132150,"ĠاÙĦØŃÙĦ":132151,"ĠmiÄĻdzy":132152,"ĠÑĩаÑģÑĤи":132153,"ujÄħcy":132154,"ĠbaÄŁlı":132155,"ĠiliÅŁki":132156,"ÙģØ§Ø¡":132157,"ãĥªãĥ³ãĤ°":132158,"Ġhãng":132159,"ĠконÑĤÑĢ":132160,"ĠконÑĤÑĢол":132161,"коп":132162,"ש×Ļ×¢":132163,"ש×Ļ×¢×ķר":132164,"ĠÐĴаÑĪ":132165,"Ġ×Ķתק":132166,"ÙħÙĨع":132167,"ĠpolÃŃtico":132168,"Ġголов":132169,"ĠØ¥ÙĬ":132170,"Ø¥ÙĨتاج":132171,"à¸ļิ":132172,"ĠговоÑĢ":132173,"ĠговоÑĢиÑĤ":132174,"Ġphá»ķ":132175,"ĠÑģемÑĮ":132176,"ãģ¯ãģĤãĤĬãģ¾ãģĽãĤĵ":132177,"ĠÙĪØ§Ø³Øª":132178,"×ŀשפ×ĺ":132179,"зем":132180,"×ŀ×ĵ×ijר":132181,"Ġíģ°":132182,"ĠìĿ´ë²Ī":132183,"ê°ĢëĬĶ":132184,"Ġì§ĢìĽIJ":132185,"ĠcaÅĤy":132186,"ĠgeliÅŁtir":132187,"Ñģкое":132188,"posé":132189,"Ġkhô":132190,"à¸ķิà¸Ķà¸ķาม":132191,"missão":132192,"Ġ׾×ŀר":132193,"Ġ׾×ŀר×ķת":132194,"Ġbó":132195,"à¸ķรวà¸Īสà¸Ńà¸ļ":132196,"Ġnghá»ģ":132197,"Ġбиз":132198,"ĠбизнеÑģ":132199,"ÑģÑĤеÑĢ":132200,"ÙĪÙİ":132201,"楽ãģĹãģ":132202,"楽ãģĹãģ¿":132203,"ãģĵãĤĮãģĭãĤī":132204,"wiÄħzan":132205,"สà¸Ńà¸Ļ":132206,"ÙħÙĪØ±":132207,"׳×ĵ׾":132208,"Ġ×Ķ×IJ×ĵ×Ŀ":132209,"Ġмолод":132210,"ØŃÙħا":132211,"ØŃÙħاÙĬØ©":132212,"ÑģÑĤÑĢан":132213,"Ġbuá»ķi":132214,"ת×Ļ×Ļ×Ŀ":132215,"abileceÄŁi":132216,"Lİ":132217,"à¹Ģยà¸Ńะ":132218,"à¸Īร":132219,"سÙĥاÙĨ":132220,"à¸Ļัà¸Ķ":132221,"Ġmấy":132222,"ĠÐijа":132223,"sÅĤaw":132224,"ĠÙģÙĦا":132225,"ĠкоÑĤоÑĢой":132226,"ĠплоÑī":132227,"ĠплоÑīад":132228,"ãĤĤãģĤãĤĬ":132229,"szczÄĻ":132230,"×Ļפ×ķ":132231,"ש×ŀת":132232,"owaÅĤa":132233,"Ġnông":132234,"צ×ij×IJ":132235,"ĠìŀĪìĹĪ":132236,"ãģ¾ãģ¨":132237,"ãģ¾ãģ¨ãĤģ":132238,"ÙĤÙĪØ§Øª":132239,"ãģ¿ãĤĵãģª":132240,"Ġ׼×ŀ×¢×ĺ":132241,"Ġxúc":132242,"ï¼Ĩ":132243,"rÄĻ":132244,"rÄĻcz":132245,"×ĵ×ŀ×Ļ":132246,"ĠtáºŃn":132247,"à¸Ķวà¸ĩ":132248,"ê²½ìłľ":132249,"пÑĥÑĤ":132250,"أربع":132251,"Ġ×ŀשת×ŀש":132252,"ãĤ¿ãĤ¤ãĥĹ":132253,"Ġìłľê°Ģ":132254,"Ġ׾׼ף":132255,"ĠобÑĢазом":132256,"ÙĬÙĥا":132257,"wÅĤ":132258,"wÅĤasn":132259,"ĠاÙĦÙĪØ·ÙĨÙĬØ©":132260,"بÙĬب":132261,"×ŀ׾×Ļ":132262,"кÑĢаÑĤ":132263,"기ìĹIJ":132264,"ÙĤاد":132265,"ĠÙĦدÙī":132266,"à¸Ħวามรูà¹ī":132267,"×ŀ×ĵ×Ļ׳×Ļ×ķת":132268,"겨":132269,"ĠíĺĦìŀ¬":132270,"שת×Ļ":132271,"мол":132272,"Ġmái":132273,"à¸ŀิม":132274,"à¸ŀิมà¸ŀ":132275,"à¸ŀิมà¸ŀà¹Į":132276,"หลวà¸ĩ":132277,"Ġxuyên":132278,"×Ĺסר":132279,"رÙĪÙĨ":132280,"ãģĿãģĨãģĦãģĨ":132281,"ãģĿãĤĮãģŀ":132282,"ãģĿãĤĮãģŀãĤĮ":132283,"Ġ׼ש×Ķ":132284,"ÐŁÑĢав":132285,"×ŀ×ijצע":132286,"عرب":132287,"Ġbüyü":132288,"פ×Ļת×ķ×Ĺ":132289,"à¸Īà¸ļ":132290,"ĠØ£Ùĥبر":132291,"שרת":132292,"×ŀ׼ש×Ļר":132293,"ĠÙĪÙħع":132294,"ãģ®ãģŁãĤģãģ«":132295,"à¸Ļัà¸ļ":132296,"ì°°":132297,"ãĥªãĥķãĤ©":132298,"ãĥªãĥķãĤ©ãĥ¼ãĥł":132299,"Ġcưá»Ŀng":132300,"ĠìłĢíĿ¬":132301,"ÙħÙĨظÙħØ©":132302,"Ġhiçbir":132303,"ãģ§ãģ¯ãģĤãĤĬãģ¾ãģĽãĤĵ":132304,"รà¸Ńย":132305,"ëIJľëĭ¤":132306,"ãģĻãģIJãģ«":132307,"кла":132308,"Ġürünler":132309,"Ġkiá»ĥu":132310,"ĠëĤĺëĬĶ":132311,"ÑĤки":132312,"Ñģим":132313,"Ġchá»īnh":132314,"ãĤĤãģªãģĦ":132315,"ศรี":132316,"æĽ¿ãģĪ":132317,"taÅŁ":132318,"ĠبÙĥÙĦ":132319,"Ġ×ķ×Ļש":132320,"visão":132321,"ä¼Ŀ":132322,"ä¼ĿãģĪ":132323,"ÙĦد":132324,"׾×Ļ×ŀ":132325,"׾×Ļ×ŀ×ķ×ĵ":132326,"tória":132327,"دÙij":132328,"اÙħر":132329,"Ġê·¸ëłĩê²Į":132330,"ĠmateriaÅĤ":132331,"à¸Ĺรา":132332,"à¸Ĺราà¸ļ":132333,"ã쮿ĸ¹ãģĮ":132334,"ãģ¦ãģįãģŁ":132335,"ضغ":132336,"ضغط":132337,"ĠÙĬعÙĨÙĬ":132338,"ело":132339,"×IJ×Ķ×ij×Ķ":132340,"×¢×ŀ":132341,"ÅŁÄ±k":132342,"ìŀIJëĬĶ":132343,"ãĤ¿ãĥ³":132344,"ĠbáºŃt":132345,"×ŀשפ×Ĺ×Ķ":132346,"кÑĢи":132347,"бли":132348,"สัà¸ķ":132349,"สัà¸ķวà¹Į":132350,"ĠسÙĨÙĪØ§Øª":132351,"ĠPhương":132352,"ãģ¦ãģĹãģ¾ãģ£ãģŁ":132353,"ãģªãģľ":132354,"Ġ×ij×IJ×ķ":132355,"Ġcán":132356,"سجÙĦ":132357,"Ġlẽ":132358,"ãĤ±ãĥ¼ãĤ¹":132359,"Ġ×§×Ļ×ij׾":132360,"à¸ļà¸Ĺà¸Ħวาม":132361,"Ġ×ķ׼ף":132362,"ĠпÑĢедÑģÑĤавлен":132363,"Ġná»iji":132364,"Ġcomentário":132365,"ением":132366,"Ġtá»ı":132367,"lÃł":132368,"Ġש×Ķ×Ļ×Ķ":132369,"Ñģлав":132370,"ĠاÙĦÙĪÙĦا":132371,"ĠاÙĦÙĪÙĦاÙĬات":132372,"ÙĦجÙĨØ©":132373,"×§×ķר×IJ":132374,"бÑĭÑĤ":132375,"Ġì¦":132376,"Ġì¦ī":132377,"ãģ§ãģĻãģĹ":132378,"หรืà¸Ńà¹Ħมà¹Ī":132379,"заÑīиÑĤ":132380,"ÙģÙĦسطÙĬÙĨ":132381,"Ġmiá»ħn":132382,"à¹Ģยà¹ĩà¸Ļ":132383,"ĠçalÄ±ÅŁan":132384,"×Ļ×Ĵ×Ķ":132385,"ĠEÄŁ":132386,"ĠEÄŁitim":132387,"ãĥĥãĤ·ãĥ¥":132388,"ĠопÑĭ":132389,"ĠопÑĭÑĤ":132390,"رغ":132391,"رغب":132392,"ĠÑģвоиÑħ":132393,"à¸Ľà¸£à¸°à¸ķ":132394,"à¸Ľà¸£à¸°à¸ķู":132395,"Ġ×ŀ×IJ×ĵ":132396,"׼×ķ׳×Ļ×Ŀ":132397,"à¸Ļี":132398,"ĠвÑĭÑħод":132399,"ãģ®ä¸Ńãģ«":132400,"פ׾×IJ":132401,"ĠÙĪÙĦÙĬس":132402,"פ×ķרס":132403,"פ×ķרס×Ŀ":132404,"ÙħسÙĦÙħ":132405,"Ġngôi":132406,"×ĵ×ŀ×ķת":132407,"ãĤĴ使ãģ£ãģ¦":132408,"ĠпомоÑīÑĮÑİ":132409,"أسر":132410,"блок":132411,"ÙĤÙĩ":132412,"ãģĹãģ¾ãģĦ":132413,"ãģ¨ãģĹãģŁ":132414,"ĠпеÑģ":132415,"ãĥīãĥ«":132416,"×Ĺ×Ŀ":132417,"ãģĹãģªãģĮãĤī":132418,"ĠÐŁÑĢед":132419,"ãĥģãĤ§ãĥĥãĤ¯":132420,"å¼·ãģĦ":132421,"ש×Ļר×ķת":132422,"даеÑĤ":132423,"×Ļ×ij×ķ":132424,"Ġgenç":132425,"илаÑģ":132426,"илаÑģÑĮ":132427,"ĠبÙĦد":132428,"æĤª":132429,"æĤªãģĦ":132430,"Ġ×ŀשת":132431,"æ§ĺãĢħ":132432,"æ§ĺãĢħãģª":132433,"à¸ĺรรมà¸Ĭาà¸ķิ":132434,"ĠÙĥاÙħÙĦ":132435,"ĠاÙĦسÙħ":132436,"×ij×ĺ×Ļ×Ĺ":132437,"cá":132438,"gência":132439,"ãĤ¹ãĤ¿ãĥ¼":132440,"à¸Ĺำà¸ģาร":132441,"×Ļ×ľ×ª":132442,"Ġ×Ļ×ķצ×IJ":132443,"wój":132444,"à¸ļุà¸Ħ":132445,"à¸ļุà¸Ħà¸Ħล":132446,"عتÙħ":132447,"عتÙħد":132448,"ãģĿãĤĮãģ«":132449,"ĠاÙĦتارÙĬØ®":132450,"ÙĤراء":132451,"Ġyönetim":132452,"קשר":132453,"ĠÑģпоÑĢÑĤ":132454,"Ġר×IJש×ķף":132455,"Ġseñal":132456,"Ġchắn":132457,"çĦ¡ãģĦ":132458,"ĠдоÑģÑĤаÑĤ":132459,"ĠдоÑģÑĤаÑĤоÑĩно":132460,"Ġágua":132461,"à¸ģรà¸ĵ":132462,"à¸ģรà¸ĵี":132463,"Ġ×ŀש×ķ":132464,"Ġtrải":132465,"ë²Į":132466,"ujÄħcych":132467,"ÙģØ±Ø¯":132468,"à¹ĥà¸ģล":132469,"à¹ĥà¸ģลà¹ī":132470,"ãĤĭãģ®ãģ¯":132471,"ר×ķ×ķ×Ĺ":132472,"ÙĨÙĥ":132473,"ĠاÙĦÙĨÙĤ":132474,"ãģ®ãģ§ãģĹãĤĩãģĨ":132475,"ãģ®ãģ§ãģĹãĤĩãģĨãģĭ":132476,"ÙħعرÙģ":132477,"ÙħعرÙ쨩":132478,"ÑĥÑīе":132479,"Ġ×ij×¢×Ļקר":132480,"تصÙĦ":132481,"Ġ×Ķ×IJר":132482,"Ġ×Ķ×IJרץ":132483,"ĠÅŀi":132484,"à¸Ĥาà¸Ķ":132485,"íŀĺ":132486,"ãģªãĤĵãģ¨":132487,"ĠìĤ¬ëŀij":132488,"lÃ¼ÄŁÃ¼":132489,"باء":132490,"ĠاÙĦآخر":132491,"ĠfamÃŃlia":132492,"ĠTháng":132493,"ÑīениÑı":132494,"ãĤ¯ãĥŃ":132495,"ĠThứ":132496,"æĽ¸ãģį":132497,"енной":132498,"ìŀ¡":132499,"благ":132500,"благо":132501,"пов":132502,"à¹ģว":132503,"à¸ĩà¸Ħà¹Į":132504,"à¸Ńัà¸Ļà¸Ķัà¸ļ":132505,"ãģĤãģĴ":132506,"รà¹īาย":132507,"ünün":132508,"Ġ×Ļ׼×ķ׾×Ķ":132509,"зон":132510,"ĠÐľÐ¸":132511,"маÑĤеÑĢиал":132512,"Ġë³´ë©´":132513,"ØŃÙ쨏":132514,"êÌģ":132515,"ãģ«ãģĻãĤĭ":132516,"Ġת×IJ":132517,"Ġ×Ķס×ķ":132518,"ĠÑģÑĤоÑĢ":132519,"ĠÑģÑĤоÑĢон":132520,"ãĥĪãĥĥãĥĹ":132521,"ÅĤoÅĽÄĩ":132522,"ëħ¼":132523,"ëĵĿ":132524,"ĠÙĪØ§ÙĦع":132525,"ì¶Ķ":132526,"Ġ×Ļצ×IJ":132527,"ĠÑĢаздел":132528,"алÑĮнаÑı":132529,"×IJ׳ש×Ļ":132530,"spoÅĤ":132531,"spoÅĤec":132532,"spoÅĤeczn":132533,"إعÙĦ":132534,"إعÙĦاÙĨ":132535,"ÙĤÙĪÙī":132536,"íķĺë©´ìĦľ":132537,"تطÙĪØ±":132538,"Ġsiêu":132539,"Ỽt":132540,"дви":132541,"движ":132542,"Ġquần":132543,"kıl":132544,"ĠпÑĢизна":132545,"ĠHã":132546,"ĠHãy":132547,"ĠباÙĦت":132548,"manın":132549,"ãĤ«ãĥ«":132550,"Ġká»·":132551,"ק׾×Ļ":132552,"ëIJĺì§Ģ":132553,"تعÙĦÙħ":132554,"ìĭľìĦ¤":132555,"ìĭ¶":132556,"íĺ¼":132557,"ÙĥÙĬÙģ":132558,"売ãĤĬ":132559,"วิà¸Ĭา":132560,"бал":132561,"ĠØ£ØŃ":132562,"Ġдолжен":132563,"ราà¸ĩ":132564,"ราà¸ĩวั":132565,"ราà¸ĩวัล":132566,"Ùħاء":132567,"جار":132568,"Åļ":132569,"Ġ×ŀ×IJ×ĸ":132570,"ר×ŀ×Ķ":132571,"ãģĭãĤĤãģĹãĤĮãģªãģĦ":132572,"étude":132573,"czÄħc":132574,"Ġgór":132575,"×ł×¡×Ķ":132576,"ÙħÙĬد":132577,"ĠÐŁÐµÑĢе":132578,"أخر":132579,"ãģĿãģ®å¾Į":132580,"à¹Ģà¸Ķียวà¸ģัà¸Ļ":132581,"×ŀ×Ĵ×ķ":132582,"×ŀ×Ĵ×ķ×ķף":132583,"дов":132584,"masına":132585,"×¢×ł×Ķ":132586,"ãĤ±ãĥĥãĥĪ":132587,"סע":132588,"סע×Ļ×£":132589,"ĠTư":132590,"Ġtóc":132591,"íĻľëıĻ":132592,"ĠÐŀд":132593,"ĠÐŀднако":132594,"Ġdolayı":132595,"ؤÙĥد":132596,"ê³Ħíļį":132597,"׾ר":132598,"веÑĩ":132599,"Ġkhợi":132600,"Ġthá»§y":132601,"×ĵף":132602,"รà¸ģ":132603,"à¸ļัà¸ķร":132604,"à¹Ģà¸ģà¹Īา":132605,"ĠاÙĦثاÙĦ":132606,"ĠاÙĦثاÙĦØ«":132607,"Ġpodrá":132608,"ער×Ļ":132609,"ÙĨجاØŃ":132610,"Ġkhắc":132611,"측":132612,"İM":132613,"ãĤ»ãĥĥãĥĪ":132614,"żenia":132615,"Ġ׾×Ĺ×ijר":132616,"erÃł":132617,"ì´Ī":132618,"Ġküç":132619,"Ġküçük":132620,"اتÙĩÙħ":132621,"à¸ĭà¹Į":132622,"ÙħشارÙĥØ©":132623,"ĠاÙĦبط":132624,"Ġdây":132625,"еннÑĭм":132626,"à¸Ĺีà¹Īà¹Ħมà¹Ī":132627,"ÙĤÙİ":132628,"Ġvượt":132629,"Ġtrì":132630,"ĠwpÅĤyw":132631,"AÅŀ":132632,"зо":132633,"ĠاÙĦسÙĬد":132634,"à¸Ĺะà¹Ģล":132635,"ĠÑģодеÑĢжа":132636,"عطÙĬ":132637,"ĠاÙĦعÙĨ":132638,"èĢħãģĮ":132639,"à¹Ģหà¸Ļ":132640,"à¹Ģหà¸Ļืà¸Ń":132641,"ĠbÃŃ":132642,"Ġüzerinden":132643,"ĠVÅ©":132644,"Ġnuôi":132645,"ÙĨÙħ":132646,"алÑĮного":132647,"×¢×Ļף":132648,"ØŃضر":132649,"ĠоÑĤдел":132650,"ëªĩ":132651,"ìķ¡":132652,"ĠÙĦدÙĬÙĩ":132653,"ìĻľ":132654,"Ġsektör":132655,"Ġвозможно":132656,"ĠÐĶж":132657,"Ġhô":132658,"äºĭãģĮ":132659,"иÑĢование":132660,"алÑĮной":132661,"Ġ미êµŃ":132662,"رØŃÙĦ":132663,"ĠÑįкÑģ":132664,"пÑĢавлÑı":132665,"Ġnhá»Ŀ":132666,"ĠÄijẩ":132667,"ĠÄijẩy":132668,"ÙģÙĥر":132669,"ĠÙĪØ£Ø¶Ø§Ùģ":132670,"ãĥIJãĤ¹":132671,"ת×ķ׼׳×Ļת":132672,"ÑĤелей":132673,"ĠØ¥ÙĦÙĬÙĩ":132674,"ãģ¨è¨Ģãģ£ãģ¦":132675,"Ġдве":132676,"Ġchấp":132677,"ĠLö":132678,"à¸Ħลิ":132679,"à¸Ħà¸¥à¸´à¸Ľ":132680,"ĠسÙĪØ±":132681,"ĠسÙĪØ±ÙĬا":132682,"×ŀ×Ĺ×ķ":132683,"stä":132684,"доб":132685,"Ġniá»ĩm":132686,"ãģ®å¤§":132687,"פר×ķ×Ļ×§":132688,"פר×ķ×Ļ×§×ĺ":132689,"ĠChâu":132690,"Ġ×ŀ×Ķ×Ŀ":132691,"Ñģким":132692,"ĠполÑĥÑĩиÑĤÑĮ":132693,"ÙĬÙĪÙħ":132694,"Ø«ÙĪØ±":132695,"פ×ķ׾×Ļ×ĺ":132696,"פ×ķ׾×Ļ×ĺ×Ļ":132697,"ĠмеÑģÑıÑĨ":132698,"åħ¨ãģ¦":132699,"ĠاÙĦÙħجÙĦس":132700,"ĠاÙĦتاÙĦÙĬ":132701,"Ġ×Ĺר":132702,"åIJijãģij":132703,"׼×ŀ×Ķ":132704,"бед":132705,"أعض":132706,"أعضاء":132707,"ÙĪÙĦد":132708,"วà¹Īาà¸Īะ":132709,"Ġbánh":132710,"à¸Ļิย":132711,"à¸Ļิยม":132712,"à¸Ľà¸£à¸°à¸ģัà¸Ļ":132713,"ÑģÑĤавиÑĤÑĮ":132714,"à¸ŀà¸Ļัà¸Ļ":132715,"ĠÑįÑĦÑĦ":132716,"ĠÑįÑĦÑĦекÑĤив":132717,"ĠавÑĤоÑĢ":132718,"ĠÄIJÄĥng":132719,"ĠthÆ°á»Łng":132720,"ãĤĴæĦŁãģĺ":132721,"à¸ģัà¸ļà¸ģาร":132722,"å¾Įãģ«":132723,"ĠyaÄŁ":132724,"ستاÙĨ":132725,"Ġliá»ģn":132726,"ãģĦãģ¾":132727,"iêu":132728,"à¹Ĥà¸Ķà¸Ļ":132729,"ĠÙĦذÙĦÙĥ":132730,"à¹Ĥรà¸ĩà¹Ģรียà¸Ļ":132731,"צ×Ļ×Ĵ":132732,"ĠاÙĦÙħعÙĦÙĪÙħات":132733,"ç§ģãģŁãģ¡":132734,"à¸Ĺีà¹Īà¸Ħุà¸ĵ":132735,"ãģ«ãģªãģ£ãģ¦ãģĦãĤĭ":132736,"×ŀ×ĵ×Ļ׳×Ķ":132737,"×¡×Ľ×Ŀ":132738,"Ġвне":132739,"à¸ŀà¸Ļัà¸ģà¸ĩาà¸Ļ":132740,"ÑĢей":132741,"à¹Ģà¸Īà¹īาหà¸Ļà¹īาà¸Ĺีà¹Ī":132742,"ĠHiá»ĩn":132743,"Ġmédico":132744,"ĠتØŃÙĤÙĬÙĤ":132745,"ÑĮÑĤе":132746,"miÅŁti":132747,"ÙĤÙĬادة":132748,"ãĤıãģĭãĤĬ":132749,"มาà¸Īาà¸ģ":132750,"ëħĢ":132751,"ãģ«éĸ¢ãģĻãĤĭ":132752,"×IJר×Ĵ×ķף":132753,"mètre":132754,"Ġעצ×ŀ×Ļ":132755,"ĠChúa":132756,"รูà¹īà¸Ī":132757,"รูà¹īà¸Īัà¸ģ":132758,"ì£Ħ":132759,"ëĭµ":132760,"à¹ģà¸Ĺà¹ī":132761,"Ġgeçen":132762,"Ġlança":132763,"ĠاÙĦبØŃØ«":132764,"×ĵ×ŀ×ķ":132765,"ãģ¯ãģĺ":132766,"ãģ¯ãģĺãĤģ":132767,"ĠdönÃ¼ÅŁ":132768,"è¿ijãģı":132769,"à¹Ģสม":132770,"à¹Ģสมà¸Ń":132771,"ëĿ½":132772,"Ġüç":132773,"á»ŀ":132774,"ÑĪаÑı":132775,"à¸Ĺร":132776,"ØŃÙĤÙĬÙĤØ©":132777,"à¸Ĥà¸Ńà¸ĩà¸ģาร":132778,"Ġ무ìĹĩ":132779,"Ġ×Ķ׼ר":132780,"ĠاÙĦصÙĬÙĨ":132781,"ĠлÑİди":132782,"à¸ķาย":132783,"بÙĪÙĦ":132784,"Ġviêm":132785,"Ġthiá»ĩu":132786,"à¸ģà¸Ķ":132787,"Ġ׾×ĵ×ijר":132788,"פ׳×Ķ":132789,"×IJר×ij×¢":132790,"سÙī":132791,"ĠاÙĦسÙĬاس":132792,"ĠاÙĦسÙĬاسÙĬØ©":132793,"ydı":132794,"ÙĪØŃØ¯Ø©":132795,"ĠдеÑıÑĤелÑĮноÑģÑĤи":132796,"Ġ×ķ×Ķ×ŀ":132797,"пеÑĩ":132798,"пеÑĩаÑĤ":132799,"иÑĢованиÑı":132800,"ĠÑģог":132801,"ĠÑģоглаÑģ":132802,"Ġ׼×ĵ":132803,"Ġ׼×ĵ×IJ×Ļ":132804,"ĠиÑģполÑĮзоваÑĤÑĮ":132805,"ספ×ķר×ĺ":132806,"Ġilçe":132807,"expérience":132808,"ĠThá»Ŀi":132809,"İK":132810,"à¹Ħà¸Łà¸Łà¹īา":132811,"ëĵ¤ìĹIJê²Į":132812,"à¸Ľà¸£à¸°à¹Ģà¸ł":132813,"à¸Ľà¸£à¸°à¹Ģà¸łà¸Ĺ":132814,"Ġmümk":132815,"Ġmümkün":132816,"Ġ×IJ×ķ×ª×ł×ķ":132817,"ìĦ±ìĿĦ":132818,"ĠìĿ´ìľł":132819,"زÙĬارة":132820,"Ġoldukça":132821,"rób":132822,"ĠØ£ÙĨا":132823,"Ġ×Ķ×ij×Ļ":132824,"Ñģен":132825,"×¢×Ļקר":132826,"×Ļ×ĵ×ķ×¢":132827,"dzÄħ":132828,"ÙħعÙĦÙĪÙħات":132829,"شاب":132830,"Ġparça":132831,"à¸Ļะà¸Ħะ":132832,"باس":132833,"ĠÑĤоÑĢг":132834,"ĠÑĤоÑĢгов":132835,"Ġ×Ĺ×ĵר":132836,"׼ר×ĺ":132837,"׼ר×ĺ×Ļס":132838,"ĠAyrıca":132839,"ệ":132840,"ìľ¨":132841,"ĠÑĤакие":132842,"Ġ×ŀצ×ķ×Ļ":132843,"ãĥ©ãĥ³ãĤŃãĥ³ãĤ°":132844,"ש×Ļ×ķ×ķ×§":132845,"åīįãģ®":132846,"ĠBảo":132847,"ÑīÑĥ":132848,"æĹ©ãģı":132849,"ĠPhòng":132850,"à¸ŀระราà¸Ĭ":132851,"פ×Ĺ×ķת":132852,"Ġгл":132853,"Ġглаз":132854,"à¸Ĺà¹Īา":132855,"Ġdạy":132856,"ÑĢоÑģÑĤ":132857,"à¹Ĥà¸Ķยà¹Ģà¸īà¸ŀาะ":132858,"ĠquáºŃn":132859,"Ġ×Ĺ×ijר×ķת":132860,"même":132861,"mÄ±ÅŁtı":132862,"ĠاÙĦتداÙĪÙĦ":132863,"Ġnạn":132864,"Ġ×Ķ×ĵ×Ļ":132865,"ĠاÙĦطرÙĬÙĤ":132866,"×Ĵ×ķת":132867,"Ġ×Ķ×ĵר×ļ":132868,"ujÄħce":132869,"Ġchữ":132870,"ãĤĤãģ®ãģ®":132871,"ë°Ľ":132872,"ãģķãĤĵãģ¯":132873,"Ġyardım":132874,"ĠاÙĦعÙħ":132875,"Ġì§Ħíĸī":132876,"Ġ×Ļ×Ĺ":132877,"Ġ×Ļ×Ĺס×Ļ":132878,"ĠاÙĦÙħدÙĬÙĨØ©":132879,"Ġcú":132880,"à¸ģีฬ":132881,"à¸ģีฬา":132882,"Ġniên":132883,"misión":132884,"׳×Ļס×Ļ":132885,"׳×Ļס×Ļ×ķף":132886,"ĠвозÑĢаÑģÑĤ":132887,"Ġ×¢×ķש×Ķ":132888,"ĠÙħدÙĬر":132889,"ÑıÑģÑĮ":132890,"ØŃجÙħ":132891,"íĻĺê²½":132892,"ĠاÙĦأخرÙī":132893,"uÃŁer":132894,"ĠاÙĦعاÙĦÙħÙĬØ©":132895,"ĠNgá»įc":132896,"êµIJíļĮ":132897,"ä¸Ĭãģ§":132898,"×Ļ×Ķ×ķ×ĵ":132899,"×Ļ×Ķ×ķ×ĵ×Ļ×Ŀ":132900,"Ùħساعدة":132901,"ĠжизнÑĮ":132902,"ĠпоÑĤомÑĥ":132903,"ĠاÙĦÙħÙħÙĦ":132904,"ĠاÙĦÙħÙħÙĦÙĥØ©":132905,"ĠGör":132906,"رÙIJ":132907,"×ŀ×§×ķ×ŀ×ķת":132908,"åĩºæĿ¥ãĤĭ":132909,"ÑĦÑĤ":132910,"ĠìĿ´ìłľ":132911,"ĠÑĢем":132912,"ĠÑĢемонÑĤ":132913,"ת×ķ×ļ":132914,"æĻĤãģ¯":132915,"ãĤīãĤĮãģªãģĦ":132916,"altı":132917,"å®¶ãģ®":132918,"ĠاÙĦإعÙĦاÙħ":132919,"리ëĬĶ":132920,"ãģĭãĤīãģ¯":132921,"ĠHạ":132922,"ãģĤãģ®":132923,"×ĵ×Ļ×ķף":132924,"رÙĬس":132925,"ĠsocietÃł":132926,"ĠاÙĦÙĥبÙĬر":132927,"Ġ×ij×ŀס":132928,"Ġ×ij×ŀס×Ĵר":132929,"Ġ×ij×ŀס×Ĵרת":132930,"ĠìŀĪìľ¼ë©°":132931,"Ġnặng":132932,"ÙĩÙī":132933,"ĠBÃł":132934,"×ŀר×ķ":132935,"ĠjÄĻ":132936,"ĠjÄĻzy":132937,"ĠjÄĻzyk":132938,"Ġ׼×ŀ×ķ×ijף":132939,"×¢×ľ×Ķ":132940,"à¸Ĺีà¹Īà¹Ħà¸Ķà¹ī":132941,"ãģ¾ãģĹãĤĩãģĨ":132942,"×ŀספר":132943,"ТÐŀ":132944,"سÙĬاسة":132945,"ĠкаждÑĭй":132946,"ë²ł":132947,"tım":132948,"yá»ĩn":132949,"รีà¹Ī":132950,"ĠдеÑĤÑģк":132951,"วิà¸ĺีà¸ģาร":132952,"mówi":132953,"×ĺ×¢×Ŀ":132954,"×Ķצ׾×Ĺ×Ķ":132955,"ضÙĬÙģ":132956,"ĠÑħоÑĤÑı":132957,"ãĤĵãģ§ãģĦãĤĭ":132958,"à¸Ħาà¸Ķ":132959,"à¸Ħรà¸ļ":132960,"ĠкÑĥÑĢÑģ":132961,"ĠbaÅŁarı":132962,"×ijר×ķ":132963,"ÙĬعة":132964,"ĠÐĿÑĥ":132965,"à¸Ħวามà¹Ģà¸Ľà¹ĩà¸Ļ":132966,"Ġ׾×ŀש׾":132967,"Ġì¢ĭìĿĢ":132968,"Ùħؤسس":132969,"Ùħؤسسات":132970,"Ġprécis":132971,"Ġthảo":132972,"à¸ģà¹ĩà¸Ħืà¸Ń":132973,"Ġש׼׾":132974,"führung":132975,"ãģĦãģ§":132976,"à¹ģละมี":132977,"à¸ģà¹ĩมี":132978,"Ġשש":132979,"мел":132980,"Ġкниг":132981,"ĠباÙĦÙĨ":132982,"ĠباÙĦÙĨسبة":132983,"Ġaldı":132984,"ÑĤай":132985,"Ġ×Ĺ×ĵש×Ļ×Ŀ":132986,"å®Łãģ¯":132987,"عÙĪØ§":132988,"ĠìĿĺ미":132989,"изм":132990,"ÑĢабоÑĤаÑĤÑĮ":132991,"Ù쨵":132992,"Ġ×ij׳×ķסף":132993,"ãģ¨ãģĹãģ¦ãĤĤ":132994,"à¹Ģà¸Ľà¹ĩà¸Ļà¸Ĺีà¹Ī":132995,"ĠÑģледÑĥеÑĤ":132996,"èĢĥãģĪãģ¦":132997,"Ġ׼×Ļ×ķ×Ŀ":132998,"ÑģÑĤÑĭ":132999,"׼׾׼׾×Ļ":133000,"æµģãĤĮ":133001,"ãĤĴãģ¤ãģij":133002,"ÑĩаÑĤ":133003,"×Ļ׼×ķף":133004,"×Ļר×Ļ":133005,"larıyla":133006,"ãĤ¤ãĥ¡":133007,"ãĤ¤ãĥ¡ãĥ¼ãĤ¸":133008,"׳×ĸ×§":133009,"Ġciò":133010,"Ġsın":133011,"Ġsınır":133012,"à¸Ļà¸Ħร":133013,"каÑĤ":133014,"Ġlá»Ĺi":133015,"ëŀĮ":133016,"تÙģØ§Øµ":133017,"تÙģØ§ØµÙĬÙĦ":133018,"ëĨĵ":133019,"ĠÙħض":133020,"ilmiÅŁ":133021,"بارÙĥ":133022,"ÐĿÐĺ":133023,"Ġthẩm":133024,"Ġ×IJ×ķת×ļ":133025,"ĠпÑĢиним":133026,"ĠпÑĢинима":133027,"Ġyönt":133028,"Ġyöntem":133029,"Ġ×ŀ×§×ij׾":133030,"Ġktórego":133031,"ê·Ģ":133032,"شرÙģ":133033,"داÙħ":133034,"ãģĦãĤįãģĦãĤį":133035,"ĠAlém":133036,"Ġgörü":133037,"Ġgörünt":133038,"Ġgörüntü":133039,"دس":133040,"ÑĪки":133041,"гÑĢад":133042,"Ġlạc":133043,"Ġsữa":133044,"ãĤīãĤĮãģ¾ãģĻ":133045,"oÃłi":133046,"Ñīен":133047,"ãģĭãģªãģĦ":133048,"Ġпоп":133049,"ĠпопÑĥ":133050,"ĠпопÑĥлÑıÑĢ":133051,"ĠاÙĦÙħÙĪÙĤع":133052,"räg":133053,"A":133054,"íķĦ":133055,"ãĤĴè¦ĭãĤĭ":133056,"اÙħا":133057,"ĠاÙĦØŃرب":133058,"ĠÐŁÐ°":133059,"Ġ׾×IJתר":133060,"Ġtá»ijc":133061,"×ij׾×Ķ":133062,"رئÙĬس":133063,"вÑĥ":133064,"ÙĬدÙĬ":133065,"казан":133066,"Ġ×Ĺש×ij×ķף":133067,"hôtel":133068,"×¢×ķ׳×Ķ":133069,"بÙĨÙĬ":133070,"×ŀ×ķ׾":133071,"ĠднÑı":133072,"éĽ£ãģĹãģĦ":133073,"ведениÑı":133074,"Ġ×ķ×ŀת":133075,"напÑĢимеÑĢ":133076,"ÙĤابÙĦ":133077,"Ġrésultat":133078,"ĠÑĢазвиÑĤиÑı":133079,"رÙij":133080,"ìłĦ문":133081,"ĠاÙĦÙħزÙĬد":133082,"ĠìľĦíķ´ìĦľ":133083,"ëĨį":133084,"íĻķ":133085,"ĠThiết":133086,"íĮ¨":133087,"malıdır":133088,"ĠczÅĤ":133089,"ĠczÅĤowie":133090,"ĠczÅĤowiek":133091,"ĠÙĦبÙĨ":133092,"ĠÙĦبÙĨاÙĨ":133093,"üsü":133094,"ãģªãĤĵãģł":133095,"Ġżycie":133096,"ĠÑħоÑĢоÑĪо":133097,"æĸ¹ãģ«":133098,"ëĭ¤ë©´":133099,"иÑĩеÑģкаÑı":133100,"ער×Ļ׼":133101,"ער×Ļ×Ľ×ª":133102,"ãģ¾ãģĽãĤĵãģ§ãģĹãģŁ":133103,"ĠÑģобой":133104,"Ġgá»Ĺ":133105,"ĠделаÑĤÑĮ":133106,"daÄĩ":133107,"аÑĢа":133108,"różni":133109,"à¹Ģลีà¹ī":133110,"à¹Ģลีà¹īย":133111,"à¹Ģลีà¹īยà¸ĩ":133112,"à¸Ŀาà¸ģ":133113,"ĠتÙĤ":133114,"ĠتÙĤدÙĬ":133115,"ĠتÙĤدÙĬÙħ":133116,"หà¸Ļุà¹Īม":133117,"Ġmücade":133118,"Ġmücadele":133119,"ì§Ģ를":133120,"ãĤ¤ãĤ¹":133121,"Ġأساس":133122,"jÄħcego":133123,"ĠÅŁeh":133124,"нÑĤеÑĢ":133125,"ÑĨиÑİ":133126,"ï»»":133127,"ÑİÑīего":133128,"à¹Ĥà¸Ľà¸£à¹ģ":133129,"à¹Ĥà¸Ľà¸£à¹ģà¸ģรม":133130,"ĠmieÄĩ":133131,"ØŃÙĥÙĪÙħØ©":133132,"ãģ§ãģĹãģŁãģĮ":133133,"×Ļס×Ķ":133134,"ãĤĤãģ®ãĤĴ":133135,"Ġ×ŀ×IJת":133136,"สุà¸Ķà¸Ĺà¹īาย":133137,"ĠcÅ©":133138,"ÙĨسب":133139,"ĠпÑĢоÑĩ":133140,"Ġдней":133141,"ĠÑįÑĤиÑħ":133142,"׾×ŀת":133143,"нÑıÑı":133144,"Ñįк":133145,"Ġì§ĢëĤľ":133146,"มหาวิà¸Ĺยา":133147,"มหาวิà¸Ĺยาล":133148,"มหาวิà¸Ĺยาลัย":133149,"dão":133150,"ĠMáy":133151,"ĠêµŃê°Ģ":133152,"à¸ļุรี":133153,"×Ĵ×Ļ׾":133154,"ĠÑĤÑĭÑģÑı":133155,"ĠÑĤÑĭÑģÑıÑĩ":133156,"ÙģÙĥ":133157,"ĠÐĺÑģ":133158,"è¡ĮãĤıãĤĮ":133159,"פר×ĵ":133160,"ãģ¤ãģį":133161,"à¸Ħรà¸Ńà¸ļ":133162,"à¸Ħรà¸Ńà¸ļà¸Ħรัว":133163,"à¸Ĥึà¹īà¸Ļมา":133164,"ä»ĬæĹ¥ãģ¯":133165,"ĠìĤ¬ëŀĮìĿ´":133166,"עצ×ŀ×Ķ":133167,"поÑĢ":133168,"ĠKỳ":133169,"ĠÆ¡n":133170,"ĠthÄĥm":133171,"Ù쨧ÙĤ":133172,"ãģļãģ«":133173,"Ġ׾קר":133174,"Ġ׾קר×ķ×IJ":133175,"اÙģÙĬØ©":133176,"ÙħÙİØ§":133177,"гаÑĢ":133178,"صÙĦا":133179,"صÙĦاة":133180,"Ġ×ŀ×ĸ×Ķ":133181,"lıģını":133182,"Ġ×IJ×Ļ׳×Ķ":133183,"кÑĢо":133184,"Ġngươi":133185,"Ġвним":133186,"Ġвнимание":133187,"jÄħcy":133188,"ÙĢÙĢÙĢÙĢÙĢ":133189,"ÑģÑħод":133190,"ãģªãĤĵãģĭ":133191,"×ŀ×Ļ׾":133192,"Ġ×Ķ×IJ×Ĺ":133193,"ãĤıãģªãģĦ":133194,"عسÙĥر":133195,"ĠìĦ¸ê³Ħ":133196,"ĠÑĩего":133197,"ĠÑģÑĢедÑģÑĤва":133198,"ĠÐłÐ°Ñģ":133199,"ãģªãģģ":133200,"ÙĨÙ쨳":133201,"ר×Ļ×ķף":133202,"ÑģÑĥд":133203,"ĠìĿ¸ê°Ħ":133204,"ĠاÙĦÙħÙĤبÙĦ":133205,"ÙĨعÙħ":133206,"تÙĪÙ쨱":133207,"ש×ij×¢":133208,"ılm":133209,"ılmÄ±ÅŁ":133210,"Ġ×ľ×ª×ª":133211,"تصÙģ":133212,"×Ķפ×ķ×ļ":133213,"à¹ĥà¸Ļà¸Ľà¸µ":133214,"ìĿ´ê³ł":133215,"ÙģÙĪØ²":133216,"à¸ľà¸¥à¸ĩาà¸Ļ":133217,"ĠGiáo":133218,"à¸ļà¸Ńà¸ģวà¹Īา":133219,"ĠdÄ±ÅŁ":133220,"ĠdÄ±ÅŁÄ±nda":133221,"죽":133222,"ĠdzieÅĦ":133223,"кÑĨии":133224,"иÑĨе":133225,"ãģ®ä¸Ģ":133226,"عش":133227,"пÑĢеÑģÑģ":133228,"หà¸Ļà¹Īà¸Ńย":133229,"ลัà¸ģษà¸ĵะ":133230,"ĠpossibilitÃł":133231,"à¹Ħà¸Ķà¹īรัà¸ļà¸ģาร":133232,"หยุà¸Ķ":133233,"Ġphiên":133234,"çĶŁãģ¾ãĤĮ":133235,"Ø·ÙĪÙĦ":133236,"ÑĦин":133237,"für":133238,"ØŃÙĬاة":133239,"íĸĪìĬµëĭĪëĭ¤":133240,"׼׳×ķת":133241,"à¸Ľà¸£à¸°à¸ª":133242,"à¸Ľà¸£à¸°à¸ªà¸ļ":133243,"à¸Ľà¸£à¸°à¸ªà¸ļà¸ģารà¸ĵà¹Į":133244,"ëIJĺìĹĪ":133245,"Ġkażdy":133246,"Ġluyá»ĩn":133247,"ĠоÑĢганизаÑĨии":133248,"å°ijãģªãģı":133249,"ÑģÑĤÑĢоен":133250,"Ġtécnico":133251,"×§×Ķ׾":133252,"Ġ×ķ×IJ×Ĺ":133253,"ĠعÙĦÙĬÙĥ":133254,"Ñīение":133255,"Ġ×Ķ×Ļ׾×ĵ×Ļ×Ŀ":133256,"ÙĪØ³Ø§Ø¦ÙĦ":133257,"Ġ×ķ×Ķת":133258,"تÙħÙĬز":133259,"ĠÑģказал":133260,"Ġполи":133261,"Ġ×Ķ×ŀס":133262,"ÙĦÙijÙİ":133263,"Ùħؤسسة":133264,"Ġ×ŀ×Ļ×ĵ":133265,"ãģ£ãģ¡":133266,"ĠëĦĪ무":133267,"à¸ŀี":133268,"Ġtặng":133269,"Ġtấn":133270,"רש×Ŀ":133271,"Ġmédica":133272,"Ġ×¢×ķ×ŀ":133273,"Ġ×¢×ķ×ŀ×ĵ":133274,"ÑĦоÑĢ":133275,"Ùħرة":133276,"Ġvatanda":133277,"ĠvatandaÅŁ":133278,"Ġдело":133279,"à¸Ļม":133280,"ãģ¨åIJĮãģĺ":133281,"ÙģÙī":133282,"ÑģоÑĢ":133283,"Ġ×Ķסר×ĺ":133284,"Ġépoca":133285,"ìłķì±ħ":133286,"ĠÑģвÑıзан":133287,"ضرب":133288,"ĠÙĦÙĨا":133289,"Ġużywa":133290,"ĠاÙĦجÙĬØ´":133291,"ÑİÑĢ":133292,"×ijס×ķ×£":133293,"ĠмÑĥ":133294,"ĠмÑĥзÑĭк":133295,"bilité":133296,"Ġmaç":133297,"سÙİ":133298,"تÙĦÙĥ":133299,"ãģ¬":133300,"ÙĬÙĦا":133301,"ÑĪла":133302,"ÙĢÙĢÙĢ":133303,"Ġодной":133304,"зван":133305,"ĠÑģÑĢаз":133306,"ĠÑģÑĢазÑĥ":133307,"ÙĨظÙħ":133308,"راÙĩ":133309,"ĠÙĦÙĩذا":133310,"׼×ķר":133311,"Ġ×Ķש×ij×ķ×¢":133312,"Ġ×Ķשת":133313,"ĠQuảng":133314,"ãĥ«ãĥ¼":133315,"ãģĪãģªãģĦ":133316,"×ĺ×IJ":133317,"Ġmiá»ģn":133318,"ĠPháºŃt":133319,"ĠاÙĦسÙĪÙĤ":133320,"ÄĤ":133321,"ĠاÙĦجÙħع":133322,"ĠاÙĦجÙħعة":133323,"ÑİÑīей":133324,"aÅĤem":133325,"عتÙĤد":133326,"Ø£ÙĦÙħ":133327,"Ñģке":133328,"ĠìĿ´íķ´":133329,"ÙĨسخ":133330,"è¨ĢãģĦ":133331,"добав":133332,"سبÙĤ":133333,"×¢×ķרר":133334,"ÑĤип":133335,"ãģĿãģĵãģ§":133336,"visión":133337,"عÙĪØ¯Ø©":133338,"먹":133339,"×ŀ×ĸר×Ĺ":133340,"ĠØ¥ØŃ":133341,"Ġ׾×ij×Ļף":133342,"Ġ׾צ×IJת":133343,"Ġyardı":133344,"Ġyardımc":133345,"Ġyardımcı":133346,"İZ":133347,"קפ×Ķ":133348,"tré":133349,"liÄŁini":133350,"клÑİÑĩа":133351,"Ġüretim":133352,"Ġayrı":133353,"ĠkiÅŁiler":133354,"à¸Ħà¹īà¸Ļ":133355,"à¸Ħà¹īà¸Ļหา":133356,"ĠSá»±":133357,"Ġ×Ľ×¡":133358,"Ġ×Ľ×¡×£":133359,"ĠÑĤакиÑħ":133360,"ĠXuân":133361,"Ġлег":133362,"Ġлегко":133363,"Ø«ÙĤاÙ쨩":133364,"ÐĿÐŀ":133365,"ãĤ¹ãĤ¿ãĥĥ":133366,"ãĤ¹ãĤ¿ãĥĥãĥķ":133367,"åIJĪãģĦ":133368,"Ġ×Ķש×Ļ×ŀ×ķש":133369,"manız":133370,"ĠÐĴаÑģ":133371,"gün":133372,"ìľĦìĽIJíļĮ":133373,"Ġwspóln":133374,"ĠÑģвое":133375,"íĥģ":133376,"à¹Ģà¸Ļีย":133377,"ÙĪØ¨Ø©":133378,"вÑıз":133379,"ıdır":133380,"ëIJĺìĹĪëĭ¤":133381,"ĠdeÄŁiÅŁtir":133382,"ãĤĭãģĵãģ¨ãģĮ":133383,"Ġ×Ĺ×ĵש×Ķ":133384,"ãĤīãĤĮãģ¦ãģĦãĤĭ":133385,"×Ĺ×Ļ×Ļ×ij":133386,"ĠÐļаÑĢ":133387,"׳×Ļת×ķ×Ĺ":133388,"Ġ×§×ĺף":133389,"ר×ĸ":133390,"ÙĪØº":133391,"èªŃãģ¿":133392,"ĠتÙĤÙĪÙħ":133393,"ĠÙĥاÙĦ":133394,"à¸Ŀึà¸ģ":133395,"Ġë°ľìĥĿ":133396,"ológico":133397,"راع":133398,"à¹ģà¸ģà¹īà¹Ħà¸Ĥ":133399,"ĠÑĢабоÑĤÑĥ":133400,"ÙĨÙijÙİ":133401,"à¸Ńยูà¹Īà¸Ĺีà¹Ī":133402,"ĠاÙĦثاÙĨÙĬØ©":133403,"ĠNhân":133404,"ÑħваÑĤ":133405,"öne":133406,"Ġعدة":133407,"à¹ģสà¸ĩ":133408,"ÑĤоп":133409,"пÑĥÑģка":133410,"شراء":133411,"ĠÐļом":133412,"Ġפע×ķ׾×Ķ":133413,"ìĤ¬ìĿ´":133414,"ìĤ¬ìĿ´íĬ¸":133415,"è¡Įãģ£ãģ¦":133416,"Ġ×Ķ×Ķת":133417,"ĠÑģÑĤоÑĢо":133418,"ĠÑģÑĤоÑĢонÑĭ":133419,"درس":133420,"à¸ĭู":133421,"à¸ķà¹Īำ":133422,"ĠأبÙĬ":133423,"подоб":133424,"ãģ«ãģ¦":133425,"ارتÙģØ§Ø¹":133426,"ĠÙħؤ":133427,"иков":133428,"geführt":133429,"มืà¸Ńà¸ĸืà¸Ń":133430,"ĠÙĦÙĤد":133431,"ĠØ£ÙĨÙij":133432,"سÙĬطر":133433,"ãģ¾ãģļãģ¯":133434,"ס×ĵ":133435,"ÑģколÑĮко":133436,"ãģ¿ãģŁãģĦãģª":133437,"×ĵר×Ĵ":133438,"×¢×Ļ×ĵ":133439,"à¹ĥหà¹īà¸ļริà¸ģาร":133440,"ĠÐĶи":133441,"×ij×¢×Ļ×ķת":133442,"Ġ×Ķ×Ĺ×ķ":133443,"пиÑģÑĮ":133444,"ĠاÙĦØ®ÙĦ":133445,"бав":133446,"Ġİlk":133447,"ĠاÙĦØ®Ùħ":133448,"ĠاÙĦØ®ÙħÙĬس":133449,"ĠÙĬÙĤÙĪÙħ":133450,"æĻĤãģ®":133451,"ĠsÅĤow":133452,"ĠØ£ÙĩÙħ":133453,"Ø®ÙĦÙĤ":133454,"ĠأصبØŃ":133455,"Ġchứa":133456,"Ġthác":133457,"Ù쨧ÙĦ":133458,"Ġchá»Ŀ":133459,"ĠاÙĦخار":133460,"ĠاÙĦخارج":133461,"ĠاÙĦخارجÙĬØ©":133462,"طائر":133463,"ĠtÃł":133464,"ĠtÃłu":133465,"à¸ģลà¹īà¸Ńà¸ĩ":133466,"ĠاÙĦÙħرأ":133467,"ĠاÙĦÙħرأة":133468,"åħ¨ãģı":133469,"ĠÃĸn":133470,"çļĦãģ«ãģ¯":133471,"Ġpièce":133472,"×Ĵ×Ļ×ij":133473,"ĠاÙĦÙĪØ§ÙĤع":133474,"ä»Ĭãģ®":133475,"ĠاÙĦÙħÙĤ":133476,"cznÄħ":133477,"ÙģØ¹Ø§ÙĦ":133478,"енного":133479,"ĠÑĦакÑĤ":133480,"ìĭłì²Ń":133481,"ĠÐŀни":133482,"ĠاÙĦبÙĦاد":133483,"овиÑĩ":133484,"ëıĮ":133485,"ÑĦÑĥнкÑĨи":133486,"Ġìĸ´ëĬIJ":133487,"ãĥķãĤ©ãĥ¼":133488,"dÃŃ":133489,"илоÑģÑĮ":133490,"ÙħÙī":133491,"ĠاÙĦØ£ÙħرÙĬÙĥ":133492,"ĠاÙĦØ£ÙħرÙĬÙĥÙĬØ©":133493,"×ĺ×Ļפ×ķ׾":133494,"íĶĦë¡ľê·¸":133495,"íĶĦë¡ľê·¸ëŀ¨":133496,"Ġש×ķ׳×ķת":133497,"Ø´ÙħÙĦ":133498,"ĠпаÑĢа":133499,"Ġ×Ķ×Ĺ×ķ×§":133500,"ÙĪØ²Ø§Ø±Ø©":133501,"ãģ¨ãģĻãĤĭ":133502,"Ġquảng":133503,"Ġaģır":133504,"ĠاÙĦÙĦج":133505,"ĠاÙĦÙĦجÙĨØ©":133506,"긴":133507,"ĠTân":133508,"جÙħÙĦ":133509,"дол":133510,"à¹ģà¸ŀà¸Ĺย":133511,"à¹ģà¸ŀà¸Ĺยà¹Į":133512,"Ġר×IJש×Ļ":133513,"Ñīей":133514,"Ġçevre":133515,"ĠкомплекÑģ":133516,"Ġ×ij×ŀש×ļ":133517,"Ġaltın":133518,"ĠأعÙħاÙĦ":133519,"ĠÑģвоего":133520,"ãĤĪãģĦ":133521,"×Ĺ׾×Ļ×ĺ":133522,"×ŀ×ł×¢":133523,"Ġר×ij×Ķ":133524,"ĠØ£ÙĬضاÙĭ":133525,"×ĸ׾":133526,"ĠاÙĦسÙĬاسÙĬ":133527,"æĢĿãģĨ":133528,"קרק":133529,"קרקע":133530,"ĠاÙĦÙ쨱ÙĬÙĤ":133531,"биÑĤ":133532,"ק׳×Ķ":133533,"ĠØ¥ÙĨÙĩ":133534,"ĠÐĴам":133535,"ÐłÐŀ":133536,"ãĥĪãĥª":133537,"å¿ħè¦ģãģª":133538,"Ġchâu":133539,"ç¶ļãģij":133540,"Ġçözüm":133541,"gÅĤow":133542,"عÙĤÙĦ":133543,"売ãĤĭ":133544,"iết":133545,"à¸Ĭิà¹īà¸Ļ":133546,"ĠØŃÙĤÙĪÙĤ":133547,"Ø·ÙĦع":133548,"ĠÄijen":133549,"ĠÙĥاÙ쨩":133550,"ãģ®ãģĶ":133551,"Ġë¬":133552,"Ġ물":133553,"Ġë¬¼ë¡ł":133554,"ĠرسÙĪÙĦ":133555,"зам":133556,"замен":133557,"Ġkullanıcı":133558,"×¢×ķ׾":133559,"èī²ãĢħ":133560,"ÑĪиÑĢ":133561,"Ġ×Ĺש":133562,"Ġwygl":133563,"ĠwyglÄħda":133564,"ש×Ļ×ŀ×ķש":133565,"å¿ĺãĤĮ":133566,"×¢×Ļצ×ķ×ij":133567,"ĠاÙĦسÙĪØ±ÙĬ":133568,"å°ijãģªãģĦ":133569,"ĠпоиÑģк":133570,"สำà¸Ļัà¸ģà¸ĩาà¸Ļ":133571,"Ġ×ŀצ×ĵ":133572,"ĠmÃ¼ÅŁ":133573,"ĠmÃ¼ÅŁter":133574,"ĠmÃ¼ÅŁteri":133575,"ĠÙħÙĨÙĩÙħ":133576,"à¸ķำà¹ģ":133577,"à¸ķำà¹ģหà¸Ļ":133578,"à¸ķำà¹ģหà¸Ļà¹Īà¸ĩ":133579,"ÅĽmie":133580,"Ġ×©×ł×ª":133581,"Ġ×Ķפ×Ļ":133582,"פרש":133583,"×¢×ijר×Ļת":133584,"สà¸Ļัà¸ļ":133585,"สà¸Ļัà¸ļสà¸Ļุ":133586,"สà¸Ļัà¸ļสà¸Ļุà¸Ļ":133587,"è¨Ģãģ£ãģ¦":133588,"à¸ģารà¸Īัà¸Ķ":133589,"ĠMoże":133590,"изаÑĨии":133591,"ứt":133592,"ĠÙĪØ¨Ø¹Ø¯":133593,"ĠdeÄŁild":133594,"ĠdeÄŁildir":133595,"Ġת×ŀ":133596,"Ġ×ŀ×ŀ׳×ķ":133597,"話ãĤĴ":133598,"ĠÑĨена":133599,"Ġthúc":133600,"×Ļ×ŀ×ķף":133601,"ĠBáo":133602,"ãĤĴåıĸãĤĬ":133603,"å®īãģĦ":133604,"Ġ×¢×ķש×Ļ×Ŀ":133605,"èĩªåĪĨãģĮ":133606,"lée":133607,"ãĤĭãģ®ãģ§":133608,"иÑĢÑĥеÑĤ":133609,"ãģ¦ãĤĭ":133610,"ستر":133611,"ĠاÙĦØŃÙĬ":133612,"×Ļ׾×ķת":133613,"Ġ×Ĺ×ij":133614,"ÙĤرأ":133615,"تÙħÙĥÙĨ":133616,"سائÙĦ":133617,"prüf":133618,"ãģĭãģijãģ¦":133619,"ĠÑģобÑģÑĤвенно":133620,"ĠìľĦíķĺìŬ":133621,"׾×Ļ×ĺ":133622,"ãģĮå¤ļãģı":133623,"ÙĬتÙĩا":133624,"ç«ĭãģ¦":133625,"มà¸Ńà¸ļ":133626,"ìĭľìŀ¥":133627,"оÑĢа":133628,"ĠsavaÅŁ":133629,"×ĺ×Ļ×ij×Ļ":133630,"×ij׳×ķ":133631,"Ùħاذا":133632,"기ê°Ħ":133633,"ãģªãģ©ãģ§":133634,"Ġ×ŀת×Ĺ×Ļ׾":133635,"Ġnhiá»ħ":133636,"Ġnhiá»ħm":133637,"каÑĢ":133638,"каÑĢÑĤ":133639,"Ġ׾×Ķשת×ŀש":133640,"׳×Ļ×Ĺ":133641,"ادÙĬØ©":133642,"รายà¸ĩาà¸Ļ":133643,"ĠprzykÅĤad":133644,"Ñīий":133645,"ØŃضÙĪØ±":133646,"Ġhôn":133647,"ÃĿ":133648,"ת×ķצ×IJ×ķת":133649,"رابط":133650,"Ġbếp":133651,"ĠполÑĥÑĩи":133652,"åĩºä¼ļãģĦç³»":133653,"à¸Ľà¸¥à¹Īà¸Ńย":133654,"ĠاÙĦشباب":133655,"اÙĩÙĦ":133656,"ä»Ĭãģ¾ãģ§":133657,"رجع":133658,"ãĤ¶ãĥ¼":133659,"ÙĤÙģ":133660,"ĠGroÃŁ":133661,"ĠíļĮìĽIJ":133662,"اجر":133663,"Ġ×ij×ŀקר×Ķ":133664,"Ġsegurança":133665,"fühl":133666,"ãģ¦ãģĦãģı":133667,"หมà¸Ń":133668,"ĠкоÑĤоÑĢом":133669,"ĠNÄĥm":133670,"ĠdÅĤugo":133671,"ÙħÙĨØŃ":133672,"ש×ķ×ķ×Ļ":133673,"ĠØ£ÙĬاÙħ":133674,"à¸ªà¸łà¸²à¸ŀ":133675,"rzÄħ":133676,"شرÙĥات":133677,"ãĤĴèĢĥãģĪ":133678,"даÑĢ":133679,"à¸Ľà¸£à¸°à¸Ĭุม":133680,"Ġ×ķ×IJ×ĸ":133681,"iá»ĩn":133682,"Ġtươi":133683,"ש×Ļ×Ĺ":133684,"à¸Ńà¹Īà¸Ńà¸Ļ":133685,"æĽ¸ãģĦãģ¦":133686,"Ġngữ":133687,"×ij×Ļ×ĺ×Ĺ":133688,"×ij×Ļ×ĺ×Ĺ×ķף":133689,"Ġsẵ":133690,"Ġsẵn":133691,"ì§ĢëıĦ":133692,"ĠпÑĢеп":133693,"ĠпÑĢепаÑĢаÑĤ":133694,"ĠнаÑĥÑĩ":133695,"ĠÃľnivers":133696,"ĠÃľniversites":133697,"ĠÃľniversitesi":133698,"Ġ×Ĵ×ĵ×ķ׾×Ķ":133699,"Ġ×Ķ×ł×ª":133700,"Ġ×Ķ×ł×ª×ij×¢":133701,"ãģ§ãģĤãģ£ãģŁ":133702,"ĠmiesiÄħ":133703,"ĠmiesiÄħc":133704,"гÑĢам":133705,"гÑĢамм":133706,"ĠبشأÙĨ":133707,"ĠÑħÑĢ":133708,"×§×Ļ×ĵ":133709,"×§×Ļ×ĵ×ķ×Ŀ":133710,"Ø´Ùĥر":133711,"Ġá»ķ":133712,"Ġá»ķn":133713,"ãģĮãģĤãģ£ãģ¦":133714,"ãģķãĤĮãģ¾ãģĻ":133715,"Ġ×Ĺ×ķ×ĵ":133716,"Ġ×Ĺ×ķ×ĵש×Ļ×Ŀ":133717,"ÙħÙĪØ§Ø¬Ùĩ":133718,"ÙħÙĪØ§Ø¬ÙĩØ©":133719,"أشخاص":133720,"بغ":133721,"à¹Ģรียà¸Ļรูà¹ī":133722,"ãģĹãģ¦ãģĦãģı":133723,"Ġsạn":133724,"å¿ħãģļ":133725,"׳×Ļ×Ĵ":133726,"׳×Ļ×Ĵ×ķ×ĵ":133727,"باÙĦغ":133728,"×Ĺש×ŀ":133729,"×Ĺש×ŀ׾":133730,"Ġnapraw":133731,"ĠnaprawdÄĻ":133732,"Ø´Ùĩاد":133733,"×IJ×ķ×Ķ":133734,"×IJ×ķ×Ķ×ij":133735,"иÑĨÑĭ":133736,"Ġ×Ķר׼×ij":133737,"ëŀij":133738,"Ġתע":133739,"Ġ×Ķ×Ļש":133740,"Ġ×Ķ×Ļשר×IJ":133741,"Ġ×Ķ×Ļשר×IJ׾×Ļ":133742,"Ø£ÙħÙĨ":133743,"ÑİÑīаÑı":133744,"skór":133745,"LERİ":133746,"Ġ×Ķ×IJ×Ĺר×ķף":133747,"×¢×ł×§":133748,"ĠÙĪÙĥÙĦ":133749,"ãģĵãģĵãģ§":133750,"Ġquán":133751,"liÄŁin":133752,"à¸ģà¸İหมาย":133753,"Ø·Ùħ":133754,"أجÙĩ":133755,"أجÙĩزة":133756,"ĠErdoÄŁan":133757,"ãģ§ãģĬ":133758,"ĠвÑĢа":133759,"ĠвÑĢаÑĩ":133760,"ĠPhó":133761,"à¸Ĭัà¹Īว":133762,"à¸Ĭัà¹Īวà¹Ĥม":133763,"à¸Ĭัà¹Īวà¹Ĥมà¸ĩ":133764,"Ġphúc":133765,"×Ļפ×ķת":133766,"×¢×Ļ×ķף":133767,"Ġdużo":133768,"ãĥģãĥ¼ãĥł":133769,"ĠÙĬÙİ":133770,"ĠзадаÑĩ":133771,"Ġ×Ĵ×ij×ķ×Ķ×Ķ":133772,"Ġ׼׼׾":133773,"ложен":133774,"état":133775,"ĠngÄĥn":133776,"èµ·ãģį":133777,"ĠTiến":133778,"صعب":133779,"Ġexperiência":133780,"Ø®Ùħ":133781,"à¸ģารà¸Ĺำà¸ĩาà¸Ļ":133782,"سÙĬد":133783,"ĠDá»±":133784,"ĠкоÑĤоÑĢого":133785,"ladıģı":133786,"Ġkhá»ķ":133787,"Ġê³ĦìĨį":133788,"Ñīик":133789,"สà¹Īวà¸Ļà¸ķัว":133790,"зоÑĢ":133791,"ÙĨÙı":133792,"Ġà¸Ķัà¸ĩ":133793,"Ġà¸Ķัà¸ĩà¸Ļัà¹īà¸Ļ":133794,"Ġcấu":133795,"ĠÄijá»ijc":133796,"оÑĦ":133797,"ĠاÙĦأعÙħاÙĦ":133798,"ãģªãģıãģ¦ãĤĤ":133799,"×ķ׼×Ļ×Ŀ":133800,"à¹ģà¸Ľ":133801,"ĠBên":133802,"ãĥ¯ãĥ³":133803,"Ġgiám":133804,"ĠÅŀu":133805,"Ġdáng":133806,"عÙĦÙĬ":133807,"à¹Ģà¸ģษ":133808,"à¹Ģà¸ģษà¸ķร":133809,"ÙĪØ¬Ø¨":133810,"ннÑĭе":133811,"ÙĤضاء":133812,"à¸Ħวà¸ļ":133813,"à¸Ħวà¸ļà¸Ħุ":133814,"à¸Ħวà¸ļà¸Ħุม":133815,"ãģ¤ãģ¤":133816,"ĠViá»ĩc":133817,"×ŀ×ij×ĺ":133818,"ש×Ļת×ķ×£":133819,"ĠведÑĮ":133820,"kaza":133821,"kazaÅĤ":133822,"à¸ķำรวà¸Ī":133823,"ãĤ¿ãĥ«":133824,"ĠповÑĭ":133825,"ĠповÑĭÑĪен":133826,"ĠSợ":133827,"ĠìĦ¤ëªħ":133828,"ĠÃĩünkü":133829,"ìĥĿíĻľ":133830,"Ö¾":133831,"ãĤĮãģ¦ãģĦãĤĭ":133832,"Ġ×ijר×IJש":133833,"ר×ķ×Ĵ":133834,"ĠоÑĦи":133835,"ĠоÑĦиÑĨиалÑĮн":133836,"ĠÑĥÑģÑĤанов":133837,"ĠÑĥÑģÑĤановлен":133838,"ĠاÙĦÙħصر":133839,"ĠاÙĦÙħصرÙĬØ©":133840,"ĠÐŁÐ¾ÑįÑĤомÑĥ":133841,"ÙĨصÙģ":133842,"ĠÙĪØ§ÙĦÙĨ":133843,"ĠhÃłi":133844,"à¸Ħิ":133845,"ĠAprès":133846,"ì³IJ":133847,"à¹Ģà¸ĭีย":133848,"×ĵ×ŀ×Ķ":133849,"activité":133850,"à¸Ħิà¸Ķวà¹Īา":133851,"ÑĤÑĢен":133852,"à¹Ģฮ":133853,"ãĥıãĤ¤":133854,"ãģĮå¢ĹãģĪ":133855,"еннаÑı":133856,"Ġìĺ¤ëĬĺ":133857,"ãĥ¢ãĥ³":133858,"ĠконеÑĩно":133859,"ĠÙħÙĤابÙĦ":133860,"clé":133861,"Ġhü":133862,"Ġthẳng":133863,"ìłģìĿ´":133864,"ĠÐIJлекÑģ":133865,"ĠÐIJлекÑģан":133866,"ĠÐIJлекÑģандÑĢ":133867,"ãĥŀãĥ³ãĤ·ãĥ§ãĥ³":133868,"ãģ²ãģ¨ãģ¤":133869,"ãģªãģĬ":133870,"à¹Ģà¸Īà¹īาà¸Ĥà¸Ńà¸ĩ":133871,"ëĵľë¦¬":133872,"شاء":133873,"ĠsaÄŁlık":133874,"ĠÅŁimdi":133875,"×Ļ×IJ׾":133876,"تأثÙĬر":133877,"أسب":133878,"أسباب":133879,"ĠвÑĭполнен":133880,"лок":133881,"ש×Ļ×ij×Ķ":133882,"Ġlắm":133883,"ĠTrÆ°á»Ľc":133884,"Ġ×Ķ×¢×ľ":133885,"리를":133886,"ĠÑĢеж":133887,"ĠÑĢежим":133888,"inté":133889,"intégr":133890,"×Ĵ׳×Ļ":133891,"ĠاÙĦشعر":133892,"Ġmilhões":133893,"Ġpequeño":133894,"ãĤ³ãĥ¼ãĤ¹":133895,"×ķ׼×Ĺ":133896,"à¹Ģà¸Ĭà¹īา":133897,"شرÙĤ":133898,"Ġhương":133899,"รัà¸IJà¸ļาล":133900,"à¸ģลาย":133901,"à¸ģลายà¹Ģà¸Ľà¹ĩà¸Ļ":133902,"ĠподÑħод":133903,"תש×ķ×ij×Ķ":133904,"ãģıãģªãģ£ãģ¦":133905,"ĠاÙĦØ£ÙħÙħ":133906,"ĠHá»įc":133907,"ĠwspóÅĤpr":133908,"ĠwspóÅĤprac":133909,"ÑĩÑĥв":133910,"ÑĩÑĥвÑģÑĤв":133911,"ÃŃstico":133912,"à¹Ģà¸ģาะ":133913,"ìĽĢ":133914,"Ġназад":133915,"ãĤĭãĤĪãģĨãģ«":133916,"ĠСШ":133917,"ĠСШÐIJ":133918,"мон":133919,"ĠAsÃŃ":133920,"×ķר×Ĵ":133921,"полнен":133922,"×ŀ×¡×ľ":133923,"×ŀ×¡×ľ×ķ׾":133924,"à¹Ģลืà¸Ńà¸Ķ":133925,"à¹Ģริà¹Īมà¸ķà¹īà¸Ļ":133926,"ĠاÙĦØ¥Ùħ":133927,"ĠاÙĦØ¥Ùħارات":133928,"צ×Ķר":133929,"ãĥ¡ãĥªãĥĥãĥĪ":133930,"ĠпоÑĤом":133931,"виз":133932,"ĠÙģØªØ±Ø©":133933,"å¾Įãģ®":133934,"ÐĿÐIJ":133935,"×ŀסר":133936,"ÙĬرÙĬ":133937,"pré":133938,"ĠteÅŁek":133939,"ĠteÅŁekkür":133940,"Ġödeme":133941,"داÙĨ":133942,"ãģ¾ãģĹãģ¦":133943,"缮ãģ«":133944,"ĠÑĤеÑĩение":133945,"lard":133946,"lardır":133947,"à¹Ģราà¸Īะ":133948,"ספ×Ļ":133949,"ĠÙĪÙĥذÙĦÙĥ":133950,"Ġhát":133951,"Ġtá»Ļc":133952,"à¸Ħุย":133953,"Ġbức":133954,"ØŃÙĬÙĨ":133955,"èģŀãģĦãģ¦":133956,"Ùħؤشر":133957,"ĠNhư":133958,"Ġменее":133959,"ละà¸Ħร":133960,"Ñģин":133961,"ĠÑĢек":133962,"ĠÑĢекл":133963,"ĠÑĢеклам":133964,"ĠÙģÙĩÙĪ":133965,"Ġ׾×ĸ":133966,"×Ļ׳×ķת":133967,"ĠÅŁart":133968,"ÑģÑĤавка":133969,"Ġíı¬íķ¨":133970,"ãģ«è¡Įãģı":133971,"ï¼Ŀ":133972,"ĠпозволÑıеÑĤ":133973,"Ġת×ķ׼׾×ķ":133974,"овал":133975,"صÙĦØ©":133976,"Ġ׾ש׳×ķת":133977,"ĠÐĺгÑĢ":133978,"ÙħÙĨتجات":133979,"ĠsatÄ±ÅŁ":133980,"Ñģко":133981,"ĠاÙĦØ«ÙĦاثاء":133982,"Ġ×Ķ×ĵ×ijר×Ļ×Ŀ":133983,"ãģĹãģ¾ãģĹãĤĩãģĨ":133984,"بÙĤÙī":133985,"åĬĽãĤĴ":133986,"ĠÃĩok":133987,"ãĥģãĥ¥":133988,"à¹Ģà¸Ĭืà¹īà¸Ń":133989,"ยุà¸Ħ":133990,"ศาล":133991,"Ġ×§×ķ×ĵ×Ŀ":133992,"×ĸר×Ļ×Ŀ":133993,"ãģ®åł´åIJĪ":133994,"ĠìķĬìķĺ":133995,"ãģĤãĤĬãģ¾ãģĻãģĮ":133996,"×IJשר":133997,"è¡Įãģı":133998,"ãģ»ãģĭ":133999,"æ°Ĺãģ«ãģªãĤĭ":134000,"йдеÑĤ":134001,"íķĺìĺĢëĭ¤":134002,"ستÙħرار":134003,"ĠÐŁÑĢе":134004,"ĠÑģбоÑĢ":134005,"ĠìķĦ무":134006,"ç§ģãĤĤ":134007,"عص":134008,"ĠниÑĩ":134009,"ĠниÑĩего":134010,"ĠпÑĢием":134011,"×§×ķ×ŀ":134012,"ĠìĪĺëıĦ":134013,"Ġì¡´":134014,"Ġì¡´ìŀ¬":134015,"ĠأثÙĨ":134016,"ĠأثÙĨاء":134017,"ĠÙĪØ§ÙĦØŃ":134018,"ãģĮãģ§ãģįãĤĭ":134019,"Ġת×Ķ":134020,"Ġת×Ķ×Ļ×Ķ":134021,"רף":134022,"ĠÑģвÑıзи":134023,"×Ĵשת":134024,"ÑģпекÑĤ":134025,"ס×ij×Ļ×ij":134026,"ס×ij×Ļ×ij×Ķ":134027,"ĠíķĦìļĶíķľ":134028,"تخصص":134029,"Ġжив":134030,"ĠживоÑĤ":134031,"ĠMayıs":134032,"تعا":134033,"تعاÙĪÙĨ":134034,"ĠعÙĨÙĩا":134035,"ówki":134036,"ĠاÙĦÙģÙĦسطÙĬÙĨÙĬ":134037,"ãģłãģijãģ§ãģªãģı":134038,"ìĿ¸ì§Ģ":134039,"ĠاÙĦسÙĪØ¯":134040,"ĠاÙĦسÙĪØ¯Ø§ÙĨ":134041,"إجراءات":134042,"Ġkötü":134043,"Ġ×Ļתר":134044,"×Ĵ×Ļש×Ķ":134045,"Ġצ×ķר×ļ":134046,"รà¸ĸย":134047,"รà¸ĸยà¸Ļà¸ķà¹Į":134048,"ÑħоÑĤ":134049,"ÐłÐIJ":134050,"ÙĪØ·ÙĨ":134051,"Ġsayısı":134052,"ס×Ĺר":134053,"ÙħÙĪÙĦ":134054,"ãĤĴæĮģãģ£ãģ¦":134055,"عاÙĨ":134056,"Ġtá»Ļi":134057,"ĠвÑĭÑĪе":134058,"Ġtầm":134059,"ãĥĪãĥ¬":134060,"×Ļצ×ķ":134061,"มุม":134062,"سÙĪØ¯":134063,"ìłĦìŀIJ":134064,"ãĤµãĥŃãĥ³":134065,"ìĤ°ìĹħ":134066,"ĠоÑģнован":134067,"Ø®Ù쨶":134068,"רצ×Ķ":134069,"بÙĬض":134070,"×ķÖ¹":134071,"ס×Ļ×Ļ×¢":134072,"Ġש×IJ×Ļ":134073,"ĠاÙĦÙĤرآÙĨ":134074,"ĠТакже":134075,"×ŀש×ŀ×¢×ķת":134076,"سÙĩÙĦ":134077,"Ġ×Ķ׳×Ķ":134078,"ãĤĴãģĹãģ¦ãģĦãĤĭ":134079,"×Ļ×Ļס":134080,"×Ķ×ķ×IJ":134081,"ĠBÃŃ":134082,"Ġмало":134083,"ĠëͰëĿ¼ìĦľ":134084,"Ġר×Ĺ×ij":134085,"ãģĮé«ĺãģĦ":134086,"ÙĪØ§Ø³":134087,"ìĤ¼":134088,"×ł×¢":134089,"ãģ£ãģ¡ãĤĥ":134090,"ĠTüm":134091,"à¸Ńีà¸ģà¸Ķà¹īวย":134092,"ãģĹãģ¦ãģıãģłãģķãģĦ":134093,"ÙĨشاط":134094,"ãĥĹãĥ©ãĥ³":134095,"алиÑģÑĮ":134096,"×ĵ×ľ×ª":134097,"ĠwczeÅĽ":134098,"ĠwczeÅĽniej":134099,"ĠÑįÑĤим":134100,"Ġthá»ĭt":134101,"à¸ļัà¸į":134102,"à¸ļัà¸įà¸Ĭี":134103,"ãģļãģ£ãģ¨":134104,"ÑĢин":134105,"ĠswojÄħ":134106,"íķĺëĬĶëį°":134107,"Ġë§Įëĵ¤ìĸ´":134108,"تشÙĥ":134109,"تشÙĥÙĬÙĦ":134110,"ائÙĩ":134111,"Ġ׾פ×Ĺ×ķת":134112,"ãĥĭãĥ¥":134113,"ãĥĭãĥ¥ãĥ¼ãĤ¹":134114,"׼×IJף":134115,"ãģ§ãģįãģŁ":134116,"звон":134117,"ĠstaÅĤ":134118,"×Ĺ×ijרת×Ļ":134119,"ĠأعÙĦÙĨ":134120,"à¹ģà¸ļà¸ļà¸Ļีà¹ī":134121,"بدء":134122,"ãĤģãģŁ":134123,"Ġ×ŀש×ŀ×¢×ķת":134124,"Ġ×ŀש×ŀ×¢×ķת×Ļ":134125,"örü":134126,"Ġhạnh":134127,"zähl":134128,"ĠLý":134129,"Ġ×ij×Ķת":134130,"Ġ×ij×Ķת×IJ×Ŀ":134131,"баÑĢ":134132,"ì¦Ī":134133,"ä»ĬåĽŀãģ®":134134,"Ġyü":134135,"Ġyüks":134136,"Ġyüksel":134137,"ãĤ½ãĥ¼":134138,"ãģĤãĤĮ":134139,"×ª×ľ×ŀ×Ļ×ĵ":134140,"ãģ¤ãģª":134141,"×ij׳×Ļ×Ŀ":134142,"Ġxếp":134143,"ĠмÑĥжÑĩин":134144,"ĠاÙĦÙĥتاب":134145,"׼×ŀ×ķת":134146,"Ġçe":134147,"ĠçeÅŁ":134148,"ĠçeÅŁit":134149,"ĠçeÅŁitli":134150,"×ĵ×Ļר×ķת":134151,"à¸ļุà¸į":134152,"ĠاÙĦØ¥ÙĦÙĥ":134153,"ĠاÙĦØ¥ÙĦÙĥترÙĪ":134154,"ĠاÙĦØ¥ÙĦÙĥترÙĪÙĨÙĬ":134155,"ĠباÙĦإض":134156,"ĠباÙĦإضاÙ쨩":134157,"Ġyönel":134158,"Ġyönelik":134159,"mysÅĤ":134160,"à¸Ķà¹īวยà¸ģาร":134161,"à¸ģารà¸Ĺำ":134162,"овÑĭм":134163,"أزÙħØ©":134164,"æİ¢ãģĹ":134165,"íļ¨":134166,"Ġ×ķ×IJ×Ŀ":134167,"Ġnghiêm":134168,"ÑĪин":134169,"кал":134170,"Ġcrianças":134171,"èĩªåĪĨãģ§":134172,"Ġнай":134173,"ĠнайÑĤи":134174,"ĠSá»ij":134175,"ĠÃ¶ÄŁrenciler":134176,"ãĥ¶æľĪ":134177,"Ñģан":134178,"ĠJá":134179,"ĠkonuÅŁma":134180,"شرط":134181,"ëĪĪ":134182,"arrière":134183,"ضرÙĪØ±Ø©":134184,"ãĥĶãĥ³":134185,"עשר":134186,"аÑĢÑĮ":134187,"جÙħاع":134188,"Ġdéco":134189,"Ġ×Ļ×Ķ×ķ×ĵ×Ļ":134190,"à¸ŀลาà¸Ķ":134191,"ĠÙĬÙĥÙĨ":134192,"ĠجاÙħعة":134193,"طبÙĤ":134194,"ĠboÅŁ":134195,"×ķ×ķ×IJ":134196,"×ŀ×ĵ×¢":134197,"×§×ij×ķצת":134198,"פ×Ļר":134199,"jÄħcym":134200,"Ùħشا":134201,"ÙħشاÙĥÙĦ":134202,"צפ×ķף":134203,"إست":134204,"×ŀ׼ר":134205,"سÙħع":134206,"Ġкакой":134207,"ÑĤвоÑĢ":134208,"ØŃج":134209,"ÙģØ±Ø¶":134210,"пÑĢавлен":134211,"Ġникак":134212,"Ġmiá»ĩ":134213,"Ġmiá»ĩng":134214,"Ã¼ÃŁ":134215,"иÑĢовал":134216,"׾×ŀ×ķת":134217,"次ãģ®":134218,"ÙĦØ·":134219,"à¸ķัà¸Ļ":134220,"×Ķת×Ĺ×Ļ׾":134221,"ĠfotoÄŁ":134222,"ĠfotoÄŁraf":134223,"طرØŃ":134224,"à¸Ńà¸Ńà¸ģà¹Ħà¸Ľ":134225,"Ġyên":134226,"Ġпок":134227,"ĠпокÑĥп":134228,"ĠпокÑĥпа":134229,"ÑĨÑĥ":134230,"ĠкомпÑĮÑİ":134231,"ĠкомпÑĮÑİÑĤеÑĢ":134232,"ĠاÙĦÙĥرÙĬÙħ":134233,"تصÙħ":134234,"تصÙħÙĬÙħ":134235,"Ġоказа":134236,"Ġzarówn":134237,"Ġzarówno":134238,"ëĮĢì¶ľ":134239,"ãĤ»ãĥ³ãĤ¿ãĥ¼":134240,"ĠjakoÅĽci":134241,"æĤ©":134242,"æĤ©ãģ¿":134243,"Ø£ÙĨÙĪ":134244,"Ø£ÙĨÙĪØ§Ø¹":134245,"ë¹ł":134246,"Ġìłķë§IJ":134247,"Ġkẻ":134248,"ĠÑģайÑĤа":134249,"Ġ×Ķער×ij":134250,"Ùĩز":134251,"presión":134252,"ĠÑģÑĤен":134253,"ãģ£ãģ¦ãĤĭ":134254,"Ġhızlı":134255,"ÐļÐIJ":134256,"×ŀשפ×Ĺת":134257,"ĠÙĨÙĩا":134258,"ĠÙĨÙĩاÙĬØ©":134259,"ãģ¾ãģĦ":134260,"оÑħÑĢан":134261,"รà¹īà¸Ńย":134262,"ลึà¸ģ":134263,"ĠÙĪØ¨Ø§ÙĦ":134264,"ãĤĤãģ®ãģĮ":134265,"ר׼×Ļ×ij":134266,"ãĤ¤ãĥ¤":134267,"سؤ":134268,"سؤاÙĦ":134269,"ĠÙĦØ£ÙĨÙĩ":134270,"ĠkonuÅŁtu":134271,"ÐļÑĥпиÑĤÑĮ":134272,"Ġש×IJת×Ķ":134273,"ĠÙĪØ§ÙĦس":134274,"ĠmożliwoÅĽci":134275,"Ġprób":134276,"ëͰ":134277,"ãģ©ãĤĮ":134278,"ĠÐľÐ¸Ð½":134279,"ĠоÑĢганизм":134280,"ãģ«å¯¾ãģĻãĤĭ":134281,"ĠPré":134282,"Ġprivé":134283,"chè":134284,"ãģĦãģŁãģłãģį":134285,"สà¸Ļุà¸ģ":134286,"ajÄħce":134287,"ĠDzi":134288,"ĠDziÄĻki":134289,"ÅĤatw":134290,"rän":134291,"ränk":134292,"æĿ¥ãģŁ":134293,"Ġ×Ķ×Ļ×Ķ×ķ×ĵ×Ļ":134294,"ãĤ¬ãĥ¼":134295,"ĠÑĢад":134296,"ĠÑĢади":134297,"кÑĤив":134298,"Ø£Ùĩد":134299,"Ø£ÙĩداÙģ":134300,"ש×IJ×Ļר":134301,"ãģ¦ãģĦãģªãģĦ":134302,"Ġfrüh":134303,"Ġокол":134304,"Ġоколо":134305,"Ġregião":134306,"ĠÑĩиÑģле":134307,"Ġponiew":134308,"Ġponieważ":134309,"ìĦ¼íĦ°":134310,"Ġbầu":134311,"Ġê·":134312,"Ġê·ľ":134313,"Ġê·ľìłķ":134314,"ĠHòa":134315,"ĠÑĤоÑĤ":134316,"ãĤĤå¤ļãģĦ":134317,"ĠاÙĦإسÙĦاÙħÙĬØ©":134318,"ãģĭãģĦ":134319,"Ñįн":134320,"ĠÑĥказан":134321,"ĠÑĤакое":134322,"ï¼³":134323,"ëĮĢíķĻ":134324,"ĠgeniÅŁ":134325,"ĠاÙĦØ®ÙĬ":134326,"ĠاÙĦØ®ÙĬارات":134327,"ãĤĴè¡ĮãģĨ":134328,"ש×ŀ×Ķ":134329,"ĠLÃłm":134330,"ÙĪÙĨÙĬ":134331,"Ġ×IJ׾×Ļ×ķ":134332,"Äĺ":134333,"à¹Ħมà¹Īสามารà¸ĸ":134334,"人ãģ¨":134335,"برز":134336,"×Ļס×ķ×ĵ":134337,"×Ĵ׾×Ļ":134338,"ĠÙĬÙĨا":134339,"ĠÙĬÙĨاÙĬر":134340,"ĠкаÑĢÑĤин":134341,"Ġtôn":134342,"à¹Ģà¸ģร":134343,"à¸Ħà¸Ķี":134344,"Ġ׾×IJ×ķר×ļ":134345,"ãĤĤãĤīãģĨ":134346,"ãģĭãģĭãĤĭ":134347,"ании":134348,"ĠaraÅŁtırma":134349,"ÙĦاØŃظ":134350,"ãģĦãĤĦ":134351,"ĠTÃłi":134352,"Ġà¸Ļà¸Ńà¸ģà¸Īาà¸ģ":134353,"Ġà¸Ļà¸Ńà¸ģà¸Īาà¸ģà¸Ļีà¹ī":134354,"ĠÄIJảng":134355,"ãģ£ãģ¦ãģįãģŁ":134356,"Ġà¸ĭึà¹Īà¸ĩà¹Ģà¸Ľà¹ĩà¸Ļ":134357,"Ġtả":134358,"ĠmożliwoÅĽÄĩ":134359,"ĠSản":134360,"Ġİki":134361,"Ġcắt":134362,"سأÙĦ":134363,"Ġbakım":134364,"شب":134365,"à¸ķีà¹ī":134366,"à¸ŀยาย":134367,"à¸ŀยายาม":134368,"à¸ªà¸±à¸Ľ":134369,"à¸ªà¸±à¸Ľà¸Ķา":134370,"à¸ªà¸±à¸Ľà¸Ķาหà¹Į":134371,"ë°Ģ":134372,"еÑĢÑĭ":134373,"Ġcánh":134374,"Ġthuế":134375,"تبع":134376,"ãģ«åħ¥ãĤĮ":134377,"ÑİÑģÑĮ":134378,"íļĮìĿĺ":134379,"ç°¡åį":134380,"ç°¡åįĺ":134381,"ç°¡åįĺãģ«":134382,"Ġtrúc":134383,"ĠاÙĦÙĥÙĪÙĬ":134384,"ĠاÙĦÙĥÙĪÙĬت":134385,"ãĤıãģijãģ§ãģĻ":134386,"ĠÑģвоб":134387,"ĠÑģвобод":134388,"ĠÑĥÑĩаÑģÑĤник":134389,"สิà¹īà¸Ļ":134390,"ĠпÑĢоÑĦеÑģÑģиона":134391,"ĠпÑĢоÑĦеÑģÑģионалÑĮн":134392,"ÑģпоÑĢ":134393,"×Ĺ×ķ×ij×Ķ":134394,"ÙħعÙĨÙī":134395,"ĠاÙĦÙģØªØ±Ø©":134396,"สูà¸ĩสุà¸Ķ":134397,"ãĤıãģļ":134398,"ĠÄijè":134399,"ĠÄijèn":134400,"æ¯Ķãģ¹":134401,"าà¸ĺิ":134402,"Ġmożemy":134403,"à¹ģà¸ĭ":134404,"à¸Īะà¹Ħมà¹Ī":134405,"Ġsắp":134406,"ÐļÐŀ":134407,"Ġpráctica":134408,"ÙĪÙĥاÙĦØ©":134409,"è¾¼ãĤĵãģ§":134410,"ológica":134411,"ĠеÑī":134412,"ĠеÑīÑij":134413,"تعدÙĬÙĦ":134414,"ĠØ£Ùĥد":134415,"Ġצר×Ļ׼":134416,"Ġצר×Ļ׼×Ļ×Ŀ":134417,"Ø«Ùħ":134418,"ĠкÑĢÑĥ":134419,"ĠкÑĢÑĥп":134420,"×ij×Ļ×§×ķרת":134421,"Ġì¡°ê¸Ī":134422,"ãģ¨ãģįãģ¯":134423,"Ġbạc":134424,"ĠÑĢаÑģпол":134425,"ĠÑĢаÑģполож":134426,"ĠÑĢаÑģположен":134427,"زÙĬÙĨ":134428,"ĠÐļÑĢоме":134429,"ĠاÙĦÙĨظر":134430,"×Ķ×ķ×ĵ":134431,"ĠاÙĦسبت":134432,"ã썿ĢĿãģĦ":134433,"ĠpaÅĦst":134434,"ĠpaÅĦstw":134435,"ĠÙĦÙĬست":134436,"ĠбÑĥдÑĥ":134437,"à¸Ĺัà¸Ļà¸Ĺี":134438,"ราม":134439,"ØŃصÙĪÙĦ":134440,"ãģĹãģ¦ãģıãĤĮãĤĭ":134441,"ĠاÙĦإسرائÙĬÙĦ":134442,"ĠاÙĦإسرائÙĬÙĦÙĬ":134443,"ãģĵãĤĮãģ¾ãģ§":134444,"ìĤ¬ë¥¼":134445,"Ġsürü":134446,"à¹Ģวà¸Ńรà¹Į":134447,"à¹Ģà¸ĭà¸Ńรà¹Į":134448,"Ġutilisé":134449,"ĠÑģиÑģÑĤема":134450,"Ġdwó":134451,"Ġdwóch":134452,"Ġpróprio":134453,"Ġëĵ±ìĿĦ":134454,"arrêt":134455,"ĠЧа":134456,"×IJ×ŀ׳×ķת":134457,"عارض":134458,"à¹Ģà¸ģมสà¹Į":134459,"Ġ׾×Ķ×ij×Ļף":134460,"Ġ׾×ij×Ĺ":134461,"Ġ׾×ij×Ĺ×ķר":134462,"สาà¸Ĥา":134463,"ĠÐľÐ¾Ñģкве":134464,"بعد":134465,"ĠاÙĦÙĤرار":134466,"ĠÄIJá»ĭa":134467,"Ġ×Ĺ×Ĵ":134468,"ÙģØªØ±":134469,"ÙĪÙĨØ©":134470,"Ġ×Ķ×ĸ×IJת":134471,"å¸Ĥãģ®":134472,"ãģ»ãģĹãģĦ":134473,"Ġ×ij×¢×Ļר":134474,"ĠÑĤепеÑĢÑĮ":134475,"ìĬµëĭĪê¹Į":134476,"à¹Ħมà¹Īว":134477,"à¹Ħมà¹Īวà¹Īา":134478,"à¹Ħมà¹Īวà¹Īาà¸Īะ":134479,"×ŀ×IJ×Ķ":134480,"æĥħåł±":134481,"æĥħåł±ãĤĴ":134482,"غÙĨ":134483,"ĠпоÑı":134484,"ĠпоÑıви":134485,"éģİãģĶ":134486,"تشغ":134487,"تشغÙĬÙĦ":134488,"вел":134489,"Ġ×Ĺ×ŀ":134490,"ãģ¨ãģªãĤĬãģ¾ãģĻ":134491,"ĠraÄŁ":134492,"ĠraÄŁmen":134493,"ãģĭãģ©ãģĨ":134494,"ãģĭãģ©ãģĨãģĭ":134495,"енко":134496,"ì§Ģê³ł":134497,"Ġ×IJ׾×Ļ×Ķ":134498,"ĠØ£ÙĦ":134499,"à¸Īำหà¸Ļ":134500,"à¸Īำหà¸Ļà¹Īาย":134501,"nızı":134502,"Ġ׾ק×Ĺת":134503,"Ø£ÙĩÙħ":134504,"Ø£ÙĩÙħÙĬØ©":134505,"تغÙĬر":134506,"ש×Ĺר":134507,"ס×ķפר":134508,"×ĵ×Ļר":134509,"èī¯ãģĭãģ£ãģŁ":134510,"×ŀ׾×Ĺ×ŀ×Ķ":134511,"ÑģÑĤвие":134512,"ÑĤÑĢаÑĤ":134513,"ĠاÙĦأخ":134514,"ĠاÙĦأخÙĬرة":134515,"ĠاÙĦØŃصÙĪÙĦ":134516,"Ġcrédito":134517,"צ×Ļ×¢":134518,"ãĥ¬ãĥĻãĥ«":134519,"برÙĬ":134520,"ëIJIJ":134521,"ãģłãģ£ãģ¦":134522,"ĠrealtÃł":134523,"سÙ쨱":134524,"×ķ׳×ķ":134525,"×Ĵ×ķ×ĵ":134526,"×Ĵ×ķ×ĵ׾":134527,"ฮา":134528,"ãģĹãģ¦ãģĬãĤĬãģ¾ãģĻ":134529,"ĠgÃł":134530,"Ġ׾×ijצע":134531,"å¼ķè¶ĬãģĹ":134532,"Ġ×ŀ×Ļ׾×Ļ":134533,"Ġ×ŀ×Ļ׾×Ļ×ķף":134534,"Ùħدر":134535,"Ùħدرسة":134536,"פ×ķ×ĺ":134537,"à¸Ļà¹īำมัà¸Ļ":134538,"ëģĿ":134539,"عÙĥس":134540,"ĠÙĤض":134541,"ĠÑĢÑĭб":134542,"خطط":134543,"×ŀ×ķס×ĵ":134544,"Ġ׼׾׾×Ļ":134545,"ĠкоÑĤоÑĢое":134546,"צ×Ļ×ķף":134547,"ĠмеÑģÑĤа":134548,"ãģĭãģ¤":134549,"гÑĢÑĥпп":134550,"׾×Ļ׾":134551,"ת×ķ×IJר":134552,"ë³µì§Ģ":134553,"à¹ģà¸ľà¹Īà¸Ļ":134554,"Ġ×ijעת":134555,"æĻĤéĸĵãĤĴ":134556,"ï¼£":134557,"ãģ¨ãģĦãģĨãģĵãģ¨ãģ§":134558,"Ġ׾×Ķ×§":134559,"Ġ׾×ĸ×Ķ":134560,"ĠìłĢëĬĶ":134561,"ĠاÙĦإرÙĩاب":134562,"ĠìŀĪëĬĶëį°":134563,"ĠÑĤогда":134564,"Ġ×Ķצ×Ļ":134565,"×ķ׾×ĺ":134566,"Ġרפ×ķ×IJ×Ļ":134567,"ãģĵãģ¨ãģ§ãģĻ":134568,"ĠÄijÃŃch":134569,"ØŃÙĬا":134570,"Ġ×Ķ×ŀש×Ĺ×§":134571,"ãģľãģ²":134572,"Ġ×ŀ×IJפשר":134573,"ãģ¿ãģ¾ãģĹãģŁ":134574,"ĠاÙĦØ£ÙħÙĬرÙĥÙĬ":134575,"ÙħجتÙħع":134576,"Ġساب":134577,"ĠسابÙĤ":134578,"׼×Ļ׾":134579,"Ế":134580,"ãĥªãĤ¹ãĥĪ":134581,"Ġìĥ":134582,"ĠìĥĪ":134583,"ĠìĥĪë¡ľ":134584,"ĠìĥĪë¡ľìļ´":134585,"ĠDá»ĭch":134586,"à¹Ģหมาะสม":134587,"ĠاÙĦÙĨبÙĬ":134588,"׾׾":134589,"ÙĨع":134590,"Ðĵлав":134591,"ÐĵлавнаÑı":134592,"Ùħرض":134593,"Ġ×ķ×ĵ":134594,"تÙĤÙĬ":134595,"تÙĤÙĬÙĬÙħ":134596,"Ġbảng":134597,"ĠÙģÙĤاÙĦ":134598,"×¢×ŀ×Ļ":134599,"дÑĢа":134600,"Ġsuá»ijt":134601,"سرعة":134602,"Ġcá»Ń":134603,"Ġ×Ķ×Ļ×Ĺ×Ļ×ĵ":134604,"سعÙĬد":134605,"à¸Ńาà¸Ĭีà¸ŀ":134606,"ĠسÙĪØ§Ø¡":134607,"ãĤ½ãĥķãĥĪ":134608,"ĠлиÑĩно":134609,"ĠÐļоÑĢ":134610,"اÙĩتÙħ":134611,"اÙĩتÙħاÙħ":134612,"à¸Ńà¸Ķี":134613,"à¸Ńà¸Ķีà¸ķ":134614,"ãģIJãĤīãģĦ":134615,"Ġihtiya":134616,"Ġihtiyaç":134617,"ãģ¾ãģ§ãģ®":134618,"ìĭľìĬ¤":134619,"ìĭľìĬ¤íħľ":134620,"ÑĢÑĥÑĪ":134621,"ãĤĦãģ£ãģ±":134622,"ãĤĦãģ£ãģ±ãĤĬ":134623,"кеÑĢ":134624,"Ġży":134625,"Ġżyw":134626,"клон":134627,"Ġlượt":134628,"þ":134629,"даÑĩи":134630,"türk":134631,"غÙĪ":134632,"ĠигÑĢок":134633,"Ġphê":134634,"Ġ×©×¢×ľ":134635,"ĠاÙĦÙħدÙĨÙĬ":134636,"ĠìŬ룬ë¶Ħ":134637,"ער×Ļ×Ŀ":134638,"ÑħодÑıÑĤ":134639,"Ġxứ":134640,"ÐĹа":134641,"ĠÙģØ±Øµ":134642,"à¸Īะà¸Ĺำà¹ĥหà¹ī":134643,"íģ´":134644,"×¢×ij×ķר":134645,"à¹Ģหลà¹Īาà¸Ļีà¹ī":134646,"èĢĥãģĪãĤĭ":134647,"ÑĢеÑģÑĤ":134648,"ннÑĭй":134649,"Ġcầm":134650,"داخÙĦ":134651,"ĠÙħÙĦÙĬار":134652,"ĠÐIJл":134653,"ĠвÑĢемен":134654,"à¸Ĭà¹Īวยà¹ĥหà¹ī":134655,"ר×Ļ×ķת":134656,"ëĵ¯":134657,"飲ãģ¿":134658,"׳׾":134659,"שתף":134660,"ĠاÙĦسعÙĪØ¯ÙĬ":134661,"uÃŁ":134662,"ìĿ¸ëį°":134663,"ĠìĿ¼ë°ĺ":134664,"ÅĤÄĻ":134665,"Ġmá»iji":134666,"×ŀ×Ļ׳":134667,"ĠاÙĦأطÙ쨧ÙĦ":134668,"Ġçıkan":134669,"école":134670,"×§×Ļש":134671,"×§×Ļש×ķר":134672,"ĠоÑģÑĥÑīеÑģÑĤв":134673,"ĠоÑģÑĥÑīеÑģÑĤвлÑı":134674,"×ij×IJר":134675,"à¹Ħà¸Ľà¸Ķà¹īวย":134676,"Ġ×¢×ķ׾×Ķ":134677,"à¸ģà¹ĩà¹Ħมà¹Ī":134678,"ãĥ¢ãĥĩ":134679,"ãĥ¢ãĥĩãĥ«":134680,"تØŃÙĪÙĦ":134681,"Ġодного":134682,"ת×Ĺ×Ļ×ľ×ª":134683,"Ġتخ":134684,"Ġchcia":134685,"ĠchciaÅĤ":134686,"ãĥIJãĥ³":134687,"èĢħãģ¯":134688,"ĠÙħØŃÙĦ":134689,"Ñģлож":134690,"Ñģложн":134691,"ĠtÄĻ":134692,"Ġçıkt":134693,"Ġçıktı":134694,"ĠCÆ¡":134695,"à¹Ħà¸Ķà¹īà¹Ģลย":134696,"ırken":134697,"à¹Ģà¸Ĥà¹īาสูà¹Ī":134698,"ÙħØŃÙĥ":134699,"ÙħØŃÙĥÙħØ©":134700,"à¸Ħุà¹īม":134701,"à¸Ļà¹Īาà¸Īะ":134702,"лÑİд":134703,"деÑģÑı":134704,"деÑģÑıÑĤ":134705,"ĠлÑİбой":134706,"تØŃرÙĬر":134707,"צע×ĵ":134708,"ĠеÑij":134709,"ĠاÙĦØŃÙĥÙħ":134710,"ĠصباØŃ":134711,"à¹Ģà¸ļà¸Ńรà¹Į":134712,"Ġróżnych":134713,"гиб":134714,"ĠÑģоÑĤ":134715,"ĠÑģоÑĤÑĢÑĥд":134716,"ĠÑģоÑĤÑĢÑĥдник":134717,"ĠобÑĬем":134718,"פ×ĺר":134719,"ãģĻãģĶãģı":134720,"ãģ«éĸ¢ãģĹãģ¦":134721,"вол":134722,"Ø«ÙħاÙĨ":134723,"Ġdần":134724,"æĬľ":134725,"æĬľãģij":134726,"Ġעש":134727,"Ġעש×ķ×Ļ":134728,"ס×ķף":134729,"ãģªãģ®ãģ§ãģĻ":134730,"ãģ¯ãģ©ãģĨ":134731,"×ŀער×ij":134732,"ï¼°":134733,"Ùħصر":134734,"ÙħÙĨاسب":134735,"ÙħÙĨاسبة":134736,"ä¸Ĭãģ®":134737,"×IJ×Ļש×ķר":134738,"ĠìĦ¤ì¹ĺ":134739,"×ŀ×ĵ×Ļ׳×ķת":134740,"×ŀרת":134741,"ãĤĭãģ®ãģĮ":134742,"دÙİ":134743,"ĠاÙĦشرÙĥات":134744,"ìĭľê°Ħ":134745,"ĠÑĢеÑĪение":134746,"ãģĻãĤĭãģ®ãģ¯":134747,"ĠìŀIJìĭłìĿĺ":134748,"׾×ŀ×ķ":134749,"ãģ¨ãģĵãĤįãģ§":134750,"Ġקצר":134751,"Ġmãi":134752,"Ġkültür":134753,"ãĥ©ãĤ¤ãĥĸ":134754,"à¸ľà¸¹à¹īหà¸įิà¸ĩ":134755,"æĻĤéĸĵãģĮ":134756,"клÑİÑĩи":134757,"diÄŁiniz":134758,"มาà¸ģà¹Ĩ":134759,"تØŃÙħÙĦ":134760,"Ġhạt":134761,"ãĤ¦ãĤ£":134762,"пле":134763,"×ŀ׾×IJ":134764,"ÅĤó":134765,"Ġgá»ijc":134766,"Ġ×IJ×ķ×ĵ×ķת":134767,"หวาà¸Ļ":134768,"ĠاÙĦÙĪØ²":134769,"ĠاÙĦÙĪØ²Ø±Ø§Ø¡":134770,"ëĵ¤ê³¼":134771,"ĠصØŃ":134772,"ĠصØŃÙĬÙ쨩":134773,"Ġмм":134774,"تدخÙĦ":134775,"Ġpersönlich":134776,"ĠزÙĬ":134777,"ĠزÙĬادة":134778,"ãĤ·ãĤ¢":134779,"Ġngắn":134780,"à¸Ħลิà¸ģ":134781,"Ġsông":134782,"Ġtüket":134783,"ÑįÑĦÑĦ":134784,"ÑįÑĦÑĦекÑĤ":134785,"ש×Ļ×ij":134786,"Ġاعت":134787,"تض":134788,"تضÙħÙĨ":134789,"ĠاÙĦÙħشرÙĪØ¹":134790,"Ġprodução":134791,"ĠпÑĢименÑı":134792,"ниÑĨÑĭ":134793,"주ëĬĶ":134794,"رÙı":134795,"ĠmÆ¡":134796,"Ġhayatı":134797,"ëŁ½":134798,"Ġücret":134799,"Ġyanında":134800,"Ġprática":134801,"×ij×Ļ×§×ķר":134802,"ÃľN":134803,"ÑģоÑĤ":134804,"ãĤıãģijãģ§":134805,"Ġдолго":134806,"×ª×Ľ×ķ":134807,"ĠìķĦëĭĮ":134808,"ëį°ìĿ´":134809,"Ġçiz":134810,"ĠchoÄĩ":134811,"Ġ×Ķ×Ļת":134812,"Ġ×Ķ×Ļתר":134813,"Ġsoát":134814,"׼×ij×ĵ":134815,"à¹Ģลà¹Īา":134816,"ĠдеÑĢ":134817,"ĠдеÑĢев":134818,"ãĤĴåħ¥ãĤĮ":134819,"×Ĺ×ķס":134820,"×Ĺ×ķסר":134821,"جÙĬÙĨ":134822,"tón":134823,"onné":134824,"ĠполноÑģÑĤÑĮÑİ":134825,"人ãģŁãģ¡":134826,"Ġprêt":134827,"본":134828,"Ġdécembre":134829,"cılar":134830,"Ġתת":134831,"Ġê²½ìļ°ìĹIJëĬĶ":134832,"ÙĪØ¹Ø¯":134833,"è¦ĭãĤĭ":134834,"วิà¸Īัย":134835,"ë¶Ī":134836,"زÙĪØ§":134837,"زÙĪØ§Ø¬":134838,"dì":134839,"ãģ§ãģĻãĤĪ":134840,"Ġводо":134841,"ĠÙĬÙĪØ¬Ø¯":134842,"ÑģоÑģÑĤоÑı":134843,"ÐŀС":134844,"ĠÄIJó":134845,"×Ĺפש":134846,"Ġצ×Ļ×ij×ķר":134847,"ĠاÙĦÙĤØ·":134848,"ĠاÙĦÙĤطاع":134849,"ĠимеÑİÑĤ":134850,"ĠpháºŃn":134851,"×Ľ×¡×¤×Ļ":134852,"полниÑĤелÑĮ":134853,"éĻIJãĤĬ":134854,"ĠÑģÑĢав":134855,"ĠÑģÑĢавн":134856,"ÙħاÙĦÙĥ":134857,"×ĵר×ķ×Ŀ":134858,"çļĨãģķãĤĵ":134859,"ØŃÙĤÙĤ":134860,"à¹ģหลà¹Īà¸ĩ":134861,"ĠاÙĦرسÙħÙĬ":134862,"оÑĩки":134863,"×ĺ×ij×Ĺ":134864,"Ġcanlı":134865,"Ġ׾׾":134866,"Ġ׾׾×ŀ×ķ×ĵ":134867,"×ŀ×ij×ķ":134868,"×ª×Ľ":134869,"×ª×Ľ×ł×Ļת":134870,"ĠاÙĦÙħشار":134871,"ĠاÙĦÙħشارÙĥØ©":134872,"İÅŀ":134873,"ĠسÙĬاسÙĬ":134874,"волÑĮ":134875,"ĠÑģпÑĢав":134876,"æĿ¥ãģ¦":134877,"פ×ķר×ķ×Ŀ":134878,"สำà¹Ģรà¹ĩ":134879,"สำà¹Ģรà¹ĩà¸Ī":134880,"ĠÅŁÃ¶yle":134881,"ĠzostaÅĤa":134882,"ĠHü":134883,"ר×ķש":134884,"دÙĦÙĬÙĦ":134885,"ÑĢид":134886,"שף":134887,"×ŀ×§×ķר":134888,"ĠÑĥÑĩ":134889,"ĠÑĥÑĩеб":134890,"ĠÑįÑĤа":134891,"кова":134892,"à¸ķà¸Ļà¹Ģà¸Ńà¸ĩ":134893,"ÙĨÙIJ":134894,"à¸Ńีà¸ģà¸Ħรัà¹īà¸ĩ":134895,"ระà¸ļุ":134896,"Ġdữ":134897,"ĠاÙĦØŃاÙĦÙĬ":134898,"׼×ķ׼":134899,"׼×ķ׼×ij":134900,"Ġ×ŀ×IJשר":134901,"Ġtrụ":134902,"ÑĤелем":134903,"Ġвли":134904,"ĠвлиÑı":134905,"Ġש×IJת×Ŀ":134906,"Ġuwag":134907,"ĠuwagÄĻ":134908,"×ĺ×Ļת":134909,"×IJ×ĵ×Ŀ":134910,"à¸Ķุ":134911,"Ġ×Ķ×IJ׾×Ķ":134912,"ĠkarÄ±ÅŁ":134913,"ĠÄIJá»iji":134914,"даÑİÑĤ":134915,"ãģªãģ®ãģ«":134916,"Äħcych":134917,"à¹Ģà¸Ļà¹īà¸Ļ":134918,"ãģĹãģ¦ãģĹãģ¾ãģĨ":134919,"intérieur":134920,"ĠfÃŃsica":134921,"ĠÐŁÐ¾Ð»":134922,"ãģĹãģķ":134923,"à¸Ĺำà¹Ħม":134924,"ĠLâm":134925,"ĠاÙĦÙħسÙĦÙħ":134926,"ĠاÙĦÙħسÙĦÙħÙĬÙĨ":134927,"صØŃØ©":134928,"ìĹĦ":134929,"à¹Ģà¸Ķà¹ĩà¸Ķ":134930,"ĠÑĥÑĩеÑĤ":134931,"âÌģ":134932,"ĠبÙĦا":134933,"ĠاÙĦاجتÙħاعÙĬ":134934,"פרס×Ŀ":134935,"ãĥķãĥ©":134936,"ĠÐļогда":134937,"mieÅĽci":134938,"ĠبÙĬÙĨÙħا":134939,"Ġ×ŀ×IJ×ŀר×Ļ×Ŀ":134940,"Ġ×ij×IJ×ĸ×ķר":134941,"×ķש×Ļ×Ŀ":134942,"ĠÑģдела":134943,"entrée":134944,"à¹Ģà¸Ħà¹īา":134945,"Ñĥгл":134946,"ĠاÙĦÙģÙĨÙĬ":134947,"ĠÐĴоÑĤ":134948,"à¸Ĺีà¹Īมา":134949,"×ķצ×Ĵ":134950,"ÙĤدرة":134951,"Ġ목":134952,"Ġ목ìłģ":134953,"íıīê°Ģ":134954,"ĠاÙĦأربع":134955,"ĠاÙĦأربعاء":134956,"פס×Ļ×§":134957,"ĠÑıвлÑıÑİÑĤÑģÑı":134958,"بÙĪÙĨ":134959,"ì°¾":134960,"×ŀ×¢×¨×Ľ":134961,"×ŀ×¢×¨×Ľ×ķת":134962,"ãĤ·ãĤ§":134963,"ĠباÙĦØ£":134964,"íĸĪëįĺ":134965,"ĠاÙĦبرÙĨاÙħج":134966,"ĠاÙĦØ£ØŃد":134967,"ĠmÅ©":134968,"ĠmÅ©i":134969,"паÑĤ":134970,"بث":134971,"ĠÑĨенÑĭ":134972,"Ġ×ij×ª×ľ":134973,"è¨ĢãĤıãĤĮ":134974,"ĠاÙĦÙħجاÙĦ":134975,"ĠìĦ¸ìĥģ":134976,"Ġ×Ĵ×ķפ":134977,"ĠнаÑĪей":134978,"ĠкомпаниÑı":134979,"бин":134980,"ölü":134981,"×Ļ×Ļ×ĺ":134982,"Ġ×ŀספ×Ļ×§":134983,"ยัà¸ĩà¸Ħà¸ĩ":134984,"ĠЧи":134985,"ĠанÑĤи":134986,"ĠÑģÑĢеди":134987,"สà¹Īวà¸Ļà¹ĥหà¸įà¹Ī":134988,"оÑĩка":134989,"íĬ¹ë³Ħ":134990,"วà¹Īาà¸ĩ":134991,"гоÑĢод":134992,"باÙĥ":134993,"à¹Ģสีà¹Īย":134994,"à¹Ģสีà¹Īยà¸ĩ":134995,"ãĤĤãĤīãģĦ":134996,"×§×ķ×Ŀ":134997,"ãģĽãģļ":134998,"ĠاÙĦÙĤاÙĩرة":134999,"Ġ×ij׼×ļ":135000,"ÙħشارÙĬع":135001,"باØŃØ«":135002,"ĠпоÑĩ":135003,"ĠпоÑĩÑĤи":135004,"ĠÑĦоÑĢма":135005,"Sİ":135006,"Ġ×ŀצ×Ļ×¢":135007,"ลื":135008,"ลืม":135009,"ĠÑĤеÑĢ":135010,"ĠÑĤеÑĢÑĢиÑĤоÑĢ":135011,"ĠÑĤеÑĢÑĢиÑĤоÑĢии":135012,"ĠвмеÑģÑĤ":135013,"ĠвмеÑģÑĤе":135014,"dıkları":135015,"opération":135016,"à¹Ĥห":135017,"صدÙĬ":135018,"صدÙĬÙĤ":135019,"íĸīìłķ":135020,"تجا":135021,"تجاÙĪØ²":135022,"Ġsuç":135023,"Ġarty":135024,"Ġartyku":135025,"ĠartykuÅĤ":135026,"ãĤ·ãĥ§ãĥĥãĥĹ":135027,"שפ":135028,"שפ×Ļ×¢":135029,"Ġ×Ķש×Ļר×ķת":135030,"à¹ģà¸ĸม":135031,"ë¸Ķ":135032,"ĠukÅĤad":135033,"Ġ×ķ׼×Ļ":135034,"หลาà¸ģ":135035,"หลาà¸ģหลาย":135036,"æĸ¹ãĤĤ":135037,"Ġpodróż":135038,"ĠEÄŁer":135039,"ĠкомнаÑĤ":135040,"ĠÑģамÑĭÑħ":135041,"ĠвкÑĥÑģ":135042,"беж":135043,"Ġ×ij×§×ķ":135044,"æİĽãģij":135045,"ãģ¿ãĤĭãģ¨":135046,"ĠiliÅŁkin":135047,"ĠÙĬعÙħÙĦ":135048,"ĠподаÑĢ":135049,"Ġyazılı":135050,"ãĤĴå¾Ĺ":135051,"ĠwystÄĻp":135052,"à¸Ĺีà¹Īà¹ĥà¸Ĭà¹ī":135053,"ØŃادث":135054,"ÙĪÙĬد":135055,"кÑĥлÑĮÑĤ":135056,"кÑĥлÑĮÑĤÑĥÑĢ":135057,"à¸ģารà¹ģà¸Ĥà¹Īà¸ĩ":135058,"à¸ģารà¹ģà¸Ĥà¹Īà¸ĩà¸Ĥ":135059,"à¸ģารà¹ģà¸Ĥà¹Īà¸ĩà¸Ĥัà¸Ļ":135060,"ÙħÙĪØ¸":135061,"ÙħÙĪØ¸Ùģ":135062,"ÙĬÙħÙĬ":135063,"ãĤĵãģ§ãģĻãģĮ":135064,"diÄŁim":135065,"diÄŁimiz":135066,"ĠÐŁÐµÑĢ":135067,"ĠÐŁÐµÑĢв":135068,"Ġmão":135069,"ĠÑģез":135070,"ĠÑģезон":135071,"Ġ×Ķ×ŀ×¢":135072,"ÙħجÙħÙĪØ¹Ø©":135073,"ĠинÑĦоÑĢмаÑĨии":135074,"iếc":135075,"ãng":135076,"ĠÄijấy":135077,"ãģĶç´":135078,"ãģĶç´¹":135079,"ãģĶç´¹ä»ĭ":135080,"Ġadım":135081,"à¹Ħหล":135082,"ĠпÑĢакÑĤи":135083,"ĠпÑĢакÑĤиÑĩ":135084,"ĠпÑĢакÑĤиÑĩеÑģ":135085,"ĠпÑĢакÑĤиÑĩеÑģки":135086,"ĠاÙĦÙĨÙ쨳":135087,"ĠÑĢабоÑĤе":135088,"ÙĦÙĬÙģ":135089,"ĠاÙĦجÙĨÙĪØ¨":135090,"ĠводÑĭ":135091,"ì¹Ļ":135092,"ĠмиÑĢа":135093,"ĠÄijừng":135094,"ĠпÑĢоÑĤиво":135095,"ĠÑģÑĤÑĢанÑĭ":135096,"ลู":135097,"ìĤ¶":135098,"kreÅĽl":135099,"Ġbulund":135100,"ĠbulunduÄŁu":135101,"à¹ģสà¸Ļ":135102,"ãĤ±ãĤ¢":135103,"ת×Ĺ×ķ×ŀ×Ļ":135104,"ר׼×Ķ":135105,"Ġ׾ק×ķ×Ĺ":135106,"Ġ׾ק×ķ×Ĺ×ķת":135107,"Ġ×Ľ×ª×ķ×ijת":135108,"ĠÙĦÙĥÙħ":135109,"بشر":135110,"ĠrÃłng":135111,"Ġ×ŀ×Ķ×ŀ":135112,"Ġ×IJ×Ĺר×ķת":135113,"Ġбон":135114,"ĠбонÑĥÑģ":135115,"ï½Ĺ":135116,"à¹ģยà¸ģ":135117,"ãģĤãģªãģŁãģ®":135118,"ĠÑĥÑĩаÑģÑĤие":135119,"ĠEyl":135120,"ĠEylül":135121,"ĠçalÄ±ÅŁmaları":135122,"خطر":135123,"ìĿ½":135124,"à¸ģารà¹ĥà¸Ĭà¹īà¸ĩาà¸Ļ":135125,"Ġанализ":135126,"תק×ij׾":135127,"нием":135128,"Ġİns":135129,"Ġİnsan":135130,"ĠبÙĪØ§Ø³":135131,"ĠبÙĪØ§Ø³Ø·Ø©":135132,"Ġ×ł×Ľ×ł×¡":135133,"Ġ×Ķ×ŀ×Ļ×ĵ×¢":135134,"Ġço":135135,"ĠçoÄŁu":135136,"á»ĺ":135137,"ĠêµŃ민":135138,"ãĤĤãģĦãģĦ":135139,"Ġ׼׾×Ļ":135140,"ĠÑģÑĢедне":135141,"gÅĤo":135142,"gÅĤoÅĽ":135143,"Ġnegó":135144,"Ġnegócio":135145,"ĠÑĢегиÑģÑĤ":135146,"ĠÑĢегиÑģÑĤÑĢа":135147,"ĠÑĢегиÑģÑĤÑĢаÑĨии":135148,"Ġtrá»ĵng":135149,"ĠпÑĢÑı":135150,"ĠпÑĢÑıмо":135151,"ëłĪìĿ´":135152,"Ġkém":135153,"кле":135154,"à¸Ļำมา":135155,"ĠÑĦин":135156,"ĠÑĦинанÑģ":135157,"ĠÑĦинанÑģов":135158,"Ġkiá»ĩm":135159,"ยัà¸ĩà¹Ħ":135160,"ยัà¸ĩà¹Ħà¸ĩ":135161,"ยิà¸ĩ":135162,"à¹Ĥà¸Ľ":135163,"ĠполÑĥÑĩил":135164,"×Ļ×ĸ×Ŀ":135165,"à¹ģละà¸Ħวาม":135166,"ĠвообÑīе":135167,"صÙĬر":135168,"ãĥıãĥ³":135169,"ĠاÙĦÙĤاد":135170,"ĠاÙĦÙĤادÙħ":135171,"ĠبدÙĪÙĨ":135172,"عظÙħ":135173,"×ª×ł×ķ×¢":135174,"×ª×ł×ķ×¢×Ķ":135175,"Ø£ÙħÙĦ":135176,"ãģķãģĪ":135177,"ÑĤем":135178,"ÑĤемпеÑĢ":135179,"ÑĤемпеÑĢаÑĤÑĥÑĢ":135180,"Ġ׾×Ļצ×ķר":135181,"ĠrÄĻk":135182,"رسÙĦ":135183,"ìŀIJ를":135184,"Ġ×Ļצ×Ļרת":135185,"ÙĨبÙĬ":135186,"ÑĩнаÑı":135187,"تØŃÙĦÙĬÙĦ":135188,"Ġмик":135189,"ĠмикÑĢо":135190,"ĠSöz":135191,"Ġforça":135192,"Ñģон":135193,"ĠاÙĦعرا":135194,"ĠاÙĦعراÙĤÙĬ":135195,"ĠHá»ĵng":135196,"ãģĻãĤĭãģŁãĤģãģ«":135197,"à¸Ĺีà¹Īà¸Ńยูà¹Ī":135198,"Ġ×ķ×IJ×£":135199,"صÙĬد":135200,"ĠìķĬê³ł":135201,"รัà¸ĩ":135202,"ĠاÙĦتÙĪØ§ØµÙĦ":135203,"à¹Ģมà¸ķร":135204,"ÑĥÑģÑĤÑĢой":135205,"ÑĥÑģÑĤÑĢойÑģÑĤв":135206,"mıyor":135207,"ĠباسÙħ":135208,"Ġ×ķ׼×ķ":135209,"ĠGül":135210,"á»IJ":135211,"Ãītat":135212,"غاÙĦ":135213,"Ø¥ÙĨØ´":135214,"Ø¥ÙĨشاء":135215,"Tİ":135216,"à¸Ĥà¹īาม":135217,"Ġtroch":135218,"ĠtrochÄĻ":135219,"إص":135220,"إصابة":135221,"ĠثاÙĨÙĬ":135222,"ĠاÙĦصØŃØ©":135223,"Ġ×ĸ×Ķ×ķ":135224,"jÄħcej":135225,"ãĥĢãĥ³":135226,"ìĿ¸ìĿ´":135227,"ĠволоÑģ":135228,"ëIJĺë©´":135229,"ĠzakÅĤad":135230,"ãģĻãģĵãģ¨":135231,"以ä¸Ĭãģ®":135232,"Ġ×Ķ×ŀ×§×ķ×Ŀ":135233,"ÙħشاÙĩ":135234,"ÙħشاÙĩدة":135235,"Ñĩив":135236,"بش":135237,"ยà¹īาย":135238,"Ġsürdür":135239,"ĠNẵ":135240,"ĠNẵng":135241,"ĠигÑĢаÑĤÑĮ":135242,"Ġê·¸ëŁ¬ë©´":135243,"ãĥķãĥ«":135244,"ลà¹Īะ":135245,"Ġtendrá":135246,"ĠbÃły":135247,"à¹Ģà¸Ľà¹ĩà¸Ļà¸ľà¸¹à¹ī":135248,"Ġoko":135249,"ĠokoÅĤo":135250,"wÅĤa":135251,"wÅĤaÅĽci":135252,"wÅĤaÅĽciw":135253,"æĢĿãĤı":135254,"ĠYaÅŁ":135255,"ĠBá»ĩnh":135256,"íıŃ":135257,"بÙĬد":135258,"קרף":135259,"à¹Ģศร":135260,"à¹Ģศรษ":135261,"à¹Ģศรษà¸IJ":135262,"à¹Ģศรษà¸IJà¸ģิà¸Ī":135263,"ĠاÙĦØ£ÙĪØ±ÙĪ":135264,"ĠاÙĦØ£ÙĪØ±ÙĪØ¨ÙĬ":135265,"fläche":135266,"ä¹ĹãĤĬ":135267,"Ġbá»ģn":135268,"Ùĩب":135269,"æľĢãĤĤ":135270,"Ġsaç":135271,"à¸Ńำà¹Ģà¸ł":135272,"à¸Ńำà¹Ģà¸łà¸Ń":135273,"Ġأج":135274,"ĠاÙĦداخÙĦ":135275,"ĠاÙĦداخÙĦÙĬØ©":135276,"×ĺ×ķ×ij":135277,"ãĤĤãģªãģı":135278,"ĠлиÑĨа":135279,"à¹ģลà¹īวà¸ģà¹ĩ":135280,"×ĸ׼×Ļר":135281,"ĠquÃł":135282,"ĠÙĥذÙĦÙĥ":135283,"صØŃÙģ":135284,"ĠÃĤu":135285,"ÙĪØ¨Ø§":135286,"à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļà¹ģà¸Ľà¸¥":135287,"à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļà¹ģà¸Ľà¸¥à¸ĩ":135288,"à¸ķัวà¸Ńยà¹Īาà¸ĩ":135289,"Ġrápida":135290,"Ġtasar":135291,"Ġtasarım":135292,"ĠعÙĦÙĬÙĩÙħ":135293,"ס×ķ׾":135294,"cılı":135295,"cılık":135296,"ĠرغÙħ":135297,"ìĭľíĤ¤":135298,"Ġ×IJ׾ק":135299,"Ġ×IJ׾ק×ĺר":135300,"Ġ×IJ׾ק×ĺר×ķ׳×Ļ":135301,"à¹ģà¸ļà¹Īà¸ĩ":135302,"Ġhạng":135303,"ãģ£ãģ¦ãģıãĤĮ":135304,"ĠÙĨتÙĬ":135305,"ĠÙĨتÙĬجة":135306,"ıklı":135307,"غاÙĨ":135308,"à¸Ĥà¹īà¸Ńà¸Ħวาม":135309,"à¸Ľà¸¥à¸²à¸¢":135310,"ĠØ£Ùħس":135311,"à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยว":135312,"à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยวà¸Ĥ":135313,"à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยวà¸Ĥà¹īà¸Ńà¸ĩ":135314,"Ġdéfin":135315,"Ġdéfini":135316,"ÙģÙĨاد":135317,"ÙģÙĨادÙĤ":135318,"à¹Ħà¸Ķà¹īวà¹Īา":135319,"ãģªãģĦãĤĪãģĨãģ«":135320,"Ġprópria":135321,"ĠPhát":135322,"ãĤĦãģĻãģı":135323,"สวยà¸ĩาม":135324,"ê³łìļĶ":135325,"ÑıеÑĤ":135326,"ãģĭãĤĤãģĹãĤĮãģ¾ãģĽãĤĵãģĮ":135327,"ترجÙħ":135328,"ĠкÑĢаÑģив":135329,"Ġ×ŀר×IJש":135330,"деж":135331,"ĠÙĬÙĪÙĨ":135332,"ĠÙĬÙĪÙĨÙĬÙĪ":135333,"ÑģкоÑĢ":135334,"ĠKasım":135335,"ê³Ħìķ½":135336,"коÑģ":135337,"ĠнаÑĢÑĥ":135338,"ĠнаÑĢÑĥÑĪен":135339,"Ġduże":135340,"accès":135341,"Ġhá»ĵng":135342,"ĠvÅ©":135343,"ãģĦãģŁãģĹãģ¾ãģĻ":135344,"Ġ×ĺ×Ļ":135345,"Ġ×ĺ×Ļ×ķ׾":135346,"lıkları":135347,"Ġquê":135348,"ëħ¸ëıĻ":135349,"ìķĶ":135350,"CIÃĵN":135351,"Ġtắc":135352,"pressão":135353,"ĠìŀĪìľ¼":135354,"สิà¸Ĺà¸ĺิà¹Į":135355,"íĥĦ":135356,"Ġ×Ķ×ŀ×ŀש׾×Ķ":135357,"å¬īãģĹãģĦ":135358,"ĠÄIJặc":135359,"ÙĨزÙĦ":135360,"ĠдÑĢÑĥгой":135361,"дÑĥÑĤ":135362,"ìĪĻ":135363,"Ġthụ":135364,"à¹Ģสร":135365,"à¹Ģสรà¹ĩ":135366,"à¹Ģสรà¹ĩà¸Ī":135367,"Ġtoplant":135368,"Ġtoplantı":135369,"×IJ×ŀף":135370,"×ķ×ľ×ª":135371,"помн":135372,"ĠyoÄŁun":135373,"ÅĦskiego":135374,"ì°©":135375,"ĠØ«ÙĦاث":135376,"ĠØ«ÙĦاثة":135377,"Ġlắng":135378,"릴":135379,"ราà¸Ĭà¸ģาร":135380,"ĠÑģлова":135381,"á»Ĩ":135382,"à¸Ķีà¸ģวà¹Īา":135383,"ãģĶãģĸãģĦãģ¾ãģĻ":135384,"Ġдиз":135385,"Ġдизайн":135386,"férence":135387,"lıklar":135388,"ãģªãĤĵãģ§ãģĻ":135389,"ajÄħcy":135390,"Ġëĭ¤ìĸij":135391,"Ġëĭ¤ìĸijíķľ":135392,"×§×Ļר":135393,"ØŃار":135394,"สูà¹ī":135395,"Ġzro":135396,"Ġzrobi":135397,"ĠzrobiÄĩ":135398,"×ŀ×Ļ׼×Ķ":135399,"à¸Ĭà¹Īวยà¹Ģหลืà¸Ń":135400,"ĠÑįÑĤÑĥ":135401,"ë´ī":135402,"楽ãģĹãģĦ":135403,"سÙĪØ±":135404,"íķĺê±°ëĤĺ":135405,"ÙħؤتÙħر":135406,"ĠpoczÄħ":135407,"ĠpoczÄħtk":135408,"ĠpoczÄħtku":135409,"ĠعربÙĬ":135410,"اÙĦأر":135411,"اÙĦأردÙĨ":135412,"à¸Ķร":135413,"Åĵuvre":135414,"ĠÙĪÙĥاÙĨت":135415,"ĠÅĽredni":135416,"خضر":135417,"Ġchuyến":135418,"нÑĤ":135419,"ĠìķĮê³ł":135420,"Ġvá»Ŀi":135421,"Ġ×ij×Ļ×ĵ×Ļ":135422,"×ŀ×ĵ×ķ×ijר":135423,"ÙĪÙ쨱":135424,"ÙĬØ¡":135425,"×ł×Ľ×¡":135426,"ĠÐĽÐ°":135427,"лон":135428,"Ġxấu":135429,"ÙģÙĬÙĨ":135430,"Ġfévrier":135431,"ĠÐŀна":135432,"ĠVá»ģ":135433,"ĠÅŁeyler":135434,"ĠполÑĥÑĩен":135435,"зад":135436,"Ġnét":135437,"à¹Ħà¸Ľà¸¢à¸±à¸ĩ":135438,"×Ĺש×ij×ķ":135439,"à¸ļัà¸Ļà¸Ĺ":135440,"à¸ļัà¸Ļà¸Ĺึà¸ģ":135441,"ĠgerçekleÅŁ":135442,"иÑĩеÑģкое":135443,"ìĪĺê°Ģ":135444,"ثبت":135445,"ãģ¤ãģ¾ãĤĬ":135446,"ĠÑĥÑģловиÑıÑħ":135447,"ëĭ¤ê°Ģ":135448,"รายà¹Ħà¸Ķà¹ī":135449,"׼×IJ×ij":135450,"à¹Ĥà¸Ľà¸£à¹Ĥม":135451,"à¹Ĥà¸Ľà¸£à¹Ĥมà¸Ĭัà¹Īà¸Ļ":135452,"jähr":135453,"jährige":135454,"ק׳×Ļ×Ŀ":135455,"×ŀ×ķ×§":135456,"×ŀ×ķ×§×ĵ":135457,"ãģ«è¡Įãģ£ãģ¦":135458,"Ø¢ÙĦ":135459,"ведение":135460,"Ġ×ľ×Ľ×ª×ķ×ij":135461,"جÙħÙĩ":135462,"جÙħÙĩÙĪØ±ÙĬØ©":135463,"à¸īà¸ļ":135464,"à¸īà¸ļัà¸ļ":135465,"ĠCòn":135466,"à¸ľà¸ªà¸¡":135467,"ãģªãģ©ãģĮ":135468,"×IJ×Ķ×ij":135469,"ĠдейÑģÑĤвиÑı":135470,"yız":135471,"à¹Ħมà¹Īà¹Ģà¸Ħย":135472,"جÙĪØ²":135473,"×Ķ×Ĺ׾×ĺ×Ķ":135474,"fällt":135475,"ãĥĵãĤ¸":135476,"ãĥĵãĤ¸ãĥį":135477,"ãĥĵãĤ¸ãĥįãĤ¹":135478,"Ġ×IJ×Ļ׳×Ŀ":135479,"ĠнаÑħодиÑĤÑģÑı":135480,"ĠdziÅĽ":135481,"ستطÙĬع":135482,"׾×Ļף":135483,"Ø®ÙĦاÙģ":135484,"ÙĩÙIJ":135485,"Ġatrás":135486,"íĺģ":135487,"ãĤĴãģĶ":135488,"Ġ×Ķ×ŀ×ķצר":135489,"ĠBakanlıģı":135490,"ÑİÑīее":135491,"ÙħÙĨاط":135492,"ÙħÙĨاطÙĤ":135493,"Ù쨝":135494,"à¸Ļำà¹Ħà¸Ľ":135495,"Ġваж":135496,"Ġважно":135497,"Ġmạch":135498,"׼׳×ķ":135499,"بعث":135500,"lanması":135501,"Ġayr":135502,"Ġayrıl":135503,"ìĤ¬íļĮ":135504,"dÃŃa":135505,"pÅĤyw":135506,"اÙħÙĬØ©":135507,"íĺľ":135508,"×IJ׳×Ĵ׾":135509,"×IJ׳×Ĵ׾×Ļת":135510,"ĠìŀĪëĭ¤ëĬĶ":135511,"Ġساعة":135512,"ĠëĤĺíĥĢ":135513,"bö":135514,"à¸Ħัà¸Ļ":135515,"ĠdziaÅĤania":135516,"Ø©Ùĭ":135517,"ĠngÅ©":135518,"׳צ×Ĺ":135519,"ãģ¯ãģĤãĤĭ":135520,"ĠyaÅŁÄ±nda":135521,"stück":135522,"caracter":135523,"caracterÃŃsticas":135524,"Ġrá»Ńa":135525,"ĠÙħختÙĦÙ쨩":135526,"ãģ«ãģĬãģijãĤĭ":135527,"à¹ģà¸ŀà¸ĩ":135528,"วิà¹Īà¸ĩ":135529,"תפ×ķ":135530,"ساÙĩÙħ":135531,"使ãģĨ":135532,"ÙĥرÙĬ":135533,"×IJפ×Ļ":135534,"...............":135535,"ĠÑĤаким":135536,"×Ļ׼×ķ×Ļ":135537,"شبÙĩ":135538,"جÙĬر":135539,"ãģĿãģ®ãģ¾ãģ¾":135540,"acjÄĻ":135541,"ĠاÙĦترÙĥ":135542,"ĠاÙĦترÙĥÙĬ":135543,"ĠпÑĢавилÑĮно":135544,"ĠتعÙħÙĦ":135545,"à¸ģลà¹īา":135546,"Ġbiên":135547,"Ġ×ij׳×Ļ×Ļת":135548,"ĠклÑĥб":135549,"Ġ×ŀש×Ķ":135550,"вÑĪий":135551,"ãģĵãģ¨ãģĮãģ§ãģįãĤĭ":135552,"à¸ŀัà¸Ļà¸ĺุ":135553,"à¸ŀัà¸Ļà¸ĺุà¹Į":135554,"ר×ķ×Ŀ":135555,"ĠاÙĦÙ쨱ÙĨ":135556,"ĠاÙĦÙ쨱ÙĨسÙĬ":135557,"à¹Ģà¸Ľà¹ĩà¸Ļà¸Ħà¸Ļ":135558,"ãģĹãģ¦ãģĬãĤĬ":135559,"Ġthầy":135560,"ãĤĵãģłãģijãģ©":135561,"ì͍":135562,"ÙħدÙĨ":135563,"تÙĪÙĨ":135564,"ĠмеÑĤал":135565,"ĠмеÑĤалл":135566,"ĠinÃŃcio":135567,"à¸Ńà¸Ńà¸ģà¸Īาà¸ģ":135568,"ëĴ¤":135569,"Ġcuá»ijn":135570,"Ġbuá»Ļc":135571,"ÙĨسÙĬ":135572,"ächt":135573,"×ŀ×Ļ׳×Ļ×Ŀ":135574,"ãģķãģ¦":135575,"ãģĮãģ§ãģį":135576,"ÑĬем":135577,"Ġtái":135578,"ĠЧÑĤ":135579,"ĠЧÑĤобÑĭ":135580,"à¸Ľà¸¥à¸¹à¸ģ":135581,"à¸Ĭุมà¸Ĭà¸Ļ":135582,"нÑģкий":135583,"Ġvững":135584,"Ġ×Ķ׾×ij":135585,"ële":135586,"Ġשע×ijר":135587,"ваÑĤÑĮÑģÑı":135588,"бой":135589,"عÙĪÙĨ":135590,"à¹ģà¸Ķà¸Ļ":135591,"Ġספר×Ļ×Ŀ":135592,"Ġtuyên":135593,"Ġnhiêu":135594,"ĠQuý":135595,"Ġhuyết":135596,"ãĤıãģĭãĤīãģªãģĦ":135597,"Ġ×ŀ׼ף":135598,"Ġ×Ķק׾":135599,"Ġ׾×IJ×ķר":135600,"ĠÄIJiá»ĩn":135601,"شؤ":135602,"شؤÙĪÙĨ":135603,"Ġ×ŀ×Ĺפש":135604,"ĠпоÑģÑĤоÑıнно":135605,"×ŀ×Ļר":135606,"ìħĶ":135607,"ÐŀÑģ":135608,"ÐŀÑģнов":135609,"×ĸ×Ļת":135610,"ĠHá":135611,"ĠÑĩаÑģов":135612,"×IJ×ķ׾×Ļ":135613,"Ġmát":135614,"خرÙĪ":135615,"خرÙĪØ¬":135616,"ÙĤضا":135617,"ÙĤضاÙĬا":135618,"à¹Ģà¸Ľà¸Ńรà¹Į":135619,"ĠÙĬÙĪÙĦ":135620,"ĠÙĬÙĪÙĦÙĬÙĪ":135621,"à¹Ĥà¸Ĺษ":135622,"׳פ׾":135623,"ת×ķש":135624,"ת×ķש×ij×Ļ":135625,"Ġvários":135626,"×ŀר×IJ×Ķ":135627,"ëĿ¼ìĿ´":135628,"ÙĨغ":135629,"×ijצע":135630,"гон":135631,"ĠÄIJược":135632,"عÙı":135633,"пÑĥÑģк":135634,"ĠÙĪØ§ÙĦÙģ":135635,"ücü":135636,"×Ļ×§×Ļ×Ŀ":135637,"ĠسبÙĬÙĦ":135638,"׾×ijף":135639,"ĠاÙĦÙĤرÙĨ":135640,"ס×ķת":135641,"ĠQuáºŃn":135642,"ãģĵãĤĮãģĮ":135643,"ãĥĸãĥ©ãĥ³ãĥī":135644,"×Ĵ×ŀר":135645,"ĠwartoÅĽci":135646,"ĠÙĪØ¨ÙĬÙĨ":135647,"Ġdạ":135648,"ÐIJв":135649,"ÐIJвÑĤо":135650,"Ġolacaktır":135651,"à¸Ļà¸Ĺà¹Į":135652,"Ùħطار":135653,"Ġ×¢×§×ij":135654,"Ġתפ":135655,"ãģĹãģ¦ãģĦãģ¦":135656,"צ×ŀ×Ĺ":135657,"à¸Īà¸Ńà¸ĩ":135658,"Ġöde":135659,"ìį¨":135660,"ÙĨاس":135661,"調ãģ¹":135662,"ĠогÑĢомн":135663,"ë³´íĹĺ":135664,"×ĺ×§":135665,"×ĺקס×ĺ":135666,"ĠbaÅŁv":135667,"ĠbaÅŁvuru":135668,"Ġpomys":135669,"ĠpomysÅĤ":135670,"ãģ«ä¹Ĺ":135671,"Ġש׼ף":135672,"ĠاÙĦÙħسؤÙĪÙĦ":135673,"Ġзан":135674,"ĠзанÑıÑĤ":135675,"Ġdương":135676,"ãĥĹãĥ¬ãĤ¤":135677,"ลà¸ļ":135678,"ÑĤика":135679,"ĠAralık":135680,"Ġнедо":135681,"Ġmá»Ļ":135682,"Ġoran":135683,"Ġoranı":135684,"Ġktór":135685,"ĠktórÄħ":135686,"Ġ×Ķ×IJ×Ĺר×ķ׳×ķת":135687,"ائÙĨ":135688,"ÅĦs":135689,"ÅĦska":135690,"åĽ½ãģ®":135691,"×ŀ×ĺ×Ļ":135692,"ĠвопÑĢоÑģÑĭ":135693,"à¸Ńà¸ĩà¸Ħà¹Įà¸ģร":135694,"×ŀ×ķצ×IJ":135695,"Ġpóź":135696,"Ġpóźniej":135697,"ש×ŀ×IJ׾":135698,"Ġkaps":135699,"Ġkapsam":135700,"Ġkapsamında":135701,"Ġmáquina":135702,"ĠÅĽwiecie":135703,"ĠhoÃłng":135704,"Ġözgü":135705,"×Ĵ×ķר×Ŀ":135706,"ãģĤãģŁãĤĬ":135707,"à¸ķัà¸Ķสิà¸Ļ":135708,"à¸ķัà¸Ķสิà¸Ļà¹ĥà¸Ī":135709,"бÑĢи":135710,"ãģ«ãģªãĤĭãģ¨":135711,"تÙĥÙĪÙĨ":135712,"Ġ×ķ×Ķ×Ļ×IJ":135713,"Ġchiếu":135714,"ÑģÑĤанав":135715,"ÑģÑĤанавли":135716,"ÑģÑĤанавлива":135717,"×ŀ×ķ×Ĵ":135718,"cité":135719,"ĠKörper":135720,"Ġש×Ĵ×Ŀ":135721,"عظ":135722,"عظÙĬÙħ":135723,"Ġ×Ķ×IJ×Ļש×Ļ":135724,"Ġmatière":135725,"ĠÙģÙĪÙĤ":135726,"Ġkto":135727,"ĠktoÅĽ":135728,"à¸Ļà¹Ĥย":135729,"à¸Ļà¹Ĥยà¸ļาย":135730,"å¾ħãģ¡":135731,"à¹Ģมà¸Ļ":135732,"à¹Ģมà¸Ļู":135733,"AÃĩÃĥO":135734,"Ġtù":135735,"Ġtùy":135736,"ãĥĪãĥ³":135737,"ĠоÑĤказ":135738,"Ġ×ŀ×ķצר":135739,"ülü":135740,"ãģķãĤĵãģ«":135741,"Ġ×Ĺ×ķ×ij":135742,"קר×Ļ×IJ×Ķ":135743,"ĠاÙĦخدÙħات":135744,"ĠÙĦÙħدة":135745,"رؤ":135746,"رؤÙĬØ©":135747,"ãĤĴè¦ĭãģ¤ãģij":135748,"à¸Łà¸²":135749,"Ġréussi":135750,"à¸Ļัà¸ģà¹Ģรียà¸Ļ":135751,"ĠÑĩиÑģл":135752,"à¸ģารà¹Ģลà¹Īà¸Ļ":135753,"Ġhazırl":135754,"Ġhazırlan":135755,"ĠпеÑĢвÑĭй":135756,"лим":135757,"ĠоÑĤзÑĭвÑĭ":135758,"ĠwyjÄħ":135759,"ĠwyjÄħtk":135760,"ĠØ£ÙĤÙĦ":135761,"ס×ļ":135762,"Ġê²°ìłķ":135763,"Ġ׾×ŀעש×Ķ":135764,"Ġlắp":135765,"à¹ģà¸ļร":135766,"à¹ģà¸ļรà¸Ļà¸Ķà¹Į":135767,"วà¹Īาà¹Ģà¸Ľà¹ĩà¸Ļ":135768,"Ġبدا":135769,"ĠبداÙĬØ©":135770,"ãģ¨ãģĦãģĨãģ®ãģĮ":135771,"иÑĩеÑģким":135772,"à¸ģารà¸ŀัà¸Ĵà¸Ļา":135773,"ĠbÃło":135774,"ĠmiaÅĤa":135775,"ywaÄĩ":135776,"ĠMärz":135777,"ĠÙĨسبة":135778,"Ġéconomique":135779,"×ĸ×ŀ":135780,"×ĸ×ŀ׳×Ļ×Ŀ":135781,"æŃ¢ãĤģ":135782,"Ġtá»§":135783,"íķĺìĭł":135784,"Ġkażdego":135785,"straÃŁe":135786,"à¸Ĭีà¹ī":135787,"à¹Ģà¸ļา":135788,"ÑĢеÑģÑĥÑĢÑģ":135789,"евой":135790,"شباب":135791,"à¸ķà¹Īาà¸ĩà¸Ľà¸£à¸°à¹Ģà¸Ĺศ":135792,"Ġ×IJ×Ļש":135793,"Ġ×IJ×Ļש×Ļת":135794,"×Ļ×ķפ":135795,"×Ļ×ķפ×Ļ":135796,"ĠìļĶ구":135797,"ì¡°ìĤ¬":135798,"ãģ£ãģŁãĤī":135799,"׾×Ļ×§":135800,"миниÑģÑĤÑĢ":135801,"ãĤĤãģ®ãģ¯":135802,"Ġlương":135803,"Ġнаи":135804,"Ġнаибол":135805,"Ġнаиболее":135806,"íİĺ":135807,"à¹ģà¸ŀà¹ī":135808,"ãĤŃãĥ¥":135809,"ĠкоÑĤоÑĢÑĭм":135810,"à¹ģà¸Ĺà¸ĩ":135811,"à¹ģà¸Ĺà¸ĩà¸ļà¸Ńล":135812,"Ġ׳×Ļ×Ķ":135813,"Ġ׳×Ļ×Ķ×ķ׾":135814,"âĤª":135815,"ĠGiải":135816,"ĠиÑģполÑĮзова":135817,"ëł¥ìĿĦ":135818,"ãģĹãģĭãĤĤ":135819,"à¸ģà¹ĩà¸ķà¹īà¸Ńà¸ĩ":135820,"ĠÑĢеб":135821,"ĠÑĢебен":135822,"ĠÑĢебенка":135823,"تÙĪØ§ØµÙĦ":135824,"ãĤ°ãĥ«ãĥ¼ãĥĹ":135825,"ãĤĦãĤī":135826,"à¹Ģà¸Ľà¸´à¸Ķà¸ķัว":135827,"бÑĢо":135828,"ë°ĸìĹIJ":135829,"ÙĨÙİØ§":135830,"×Ķ×Ĵ":135831,"×Ķ×Ĵ׳×Ķ":135832,"à¸Ĺรั":135833,"à¸Ĺรัà¸ŀ":135834,"à¸Ĺรัà¸ŀยà¹Į":135835,"Ġkhá»iji":135836,"עצ×ŀ×ķ":135837,"болезн":135838,"Ġë°ĽìķĦ":135839,"มà¸Ļ":135840,"มà¸Ļุ":135841,"มà¸Ļุษ":135842,"มà¸Ļุษยà¹Į":135843,"âĹĨ":135844,"×ŀצ׾×Ļ×Ĺ":135845,"Ñıвление":135846,"ÙħØ·ÙĦ":135847,"ÙħØ·ÙĦÙĪØ¨":135848,"خاÙĦÙģ":135849,"تÙĪÙĤÙģ":135850,"ãģ§ãģįãģ¾ãģĽãĤĵ":135851,"оÑģÑĤей":135852,"меÑĩа":135853,"기ëĬĶ":135854,"תשע":135855,"صÙĬب":135856,"Ġ×ij×¢×ķ×ĵ":135857,"à¸Ĥà¸Ńà¸ĩà¹Ģà¸Ĥา":135858,"ÑĤÑıж":135859,"ĠÑĥпÑĢав":135860,"ĠÑĥпÑĢавлениÑı":135861,"Ġgénér":135862,"ĠthÃŃ":135863,"פ×ļ":135864,"ĠرÙħض":135865,"ĠرÙħضاÙĨ":135866,"Ġtruyá»ĩn":135867,"إعداد":135868,"ãĤµãĥĿãĥ¼ãĥĪ":135869,"Ġполно":135870,"خاÙħ":135871,"ÐŁÐµÑĤ":135872,"ÐŁÐµÑĤеÑĢ":135873,"ÐŁÐµÑĤеÑĢбÑĥÑĢ":135874,"ÐŁÐµÑĤеÑĢбÑĥÑĢг":135875,"ÙħÙĨتدÙī":135876,"ãģķãĤĮãģ¾ãģĹãģŁ":135877,"ĠëĮĢíķĺìŬ":135878,"à¸ľà¸¹à¹īà¸Ĺีà¹Ī":135879,"Ġ×ŀ×IJ×ķ":135880,"׾׳×ĵ":135881,"оÑĩнÑĭе":135882,"ĠнаÑĩала":135883,"Ġ׾×Ļ׾×ĵ×Ļ×Ŀ":135884,"овое":135885,"ãģĻãĤĭãģĵãģ¨ãģ§":135886,"ĠاÙĦÙĨÙģ":135887,"ĠاÙĦÙĨÙ쨷":135888,"ìŀĪëĬĶ":135889,"غÙĨÙĬ":135890,"פ×ĵ":135891,"ãĤ¾":135892,"ĠCré":135893,"ãģ©ãģ¡ãĤī":135894,"ثاÙĨ":135895,"ÑĢабаÑĤ":135896,"ÑĢабаÑĤÑĭва":135897,"Ġê°Ļëĭ¤":135898,"à¸Īั":135899,"à¸Īัà¸ģร":135900,"Ġchụ":135901,"Ġchụp":135902,"ĠмаÑģÑĤ":135903,"ĠмаÑģÑĤеÑĢ":135904,"Ġnắm":135905,"ĠÑģÑĤали":135906,"Ġ×Ķ×IJ×Ļר×ķ×¢":135907,"ãĤ½ãĥ³":135908,"åĪĨãģĭãĤĬ":135909,"طبع":135910,"بدا":135911,"gráfico":135912,"геÑĢ":135913,"à¸Ķำà¹Ģà¸Ļิà¸Ļà¸ģาร":135914,"Ġsaldır":135915,"Ġsaldırı":135916,"вÑĪиÑħ":135917,"ãģĭãģ£ãģŁãģ§ãģĻ":135918,"Ġyapıyor":135919,"ĠاÙĦÙģØª":135920,"צרפת":135921,"здоÑĢов":135922,"×ij×¢×ľ":135923,"Ġ×IJ×ŀ×Ļת×Ļ":135924,"ĠобÑĭ":135925,"ĠобÑĭÑĩ":135926,"ĠобÑĭÑĩно":135927,"Ġ׾×ķ×ŀר":135928,"تÙĥÙĨ":135929,"تÙĥÙĨÙĪÙĦÙĪØ¬":135930,"تÙĥÙĨÙĪÙĦÙĪØ¬ÙĬا":135931,"Ġhakkı":135932,"ĠÑĢав":135933,"ĠÑĢавно":135934,"رÙĬÙĥ":135935,"Ġ×ij×ŀ×Ļ×ĵ":135936,"Ġ×ij×ŀ×Ļ×ĵ×Ķ":135937,"à¹ģà¸ģà¹īว":135938,"Ġìĸĺ":135939,"Ġìĸĺ기":135940,"ãģĹãģ¦ãģĦãģ¾ãģĹãģŁ":135941,"Ġkısm":135942,"Ġkısmı":135943,"걸":135944,"åĨħãģ®":135945,"ì§ķ":135946,"à¹Ģหมืà¸Ńà¸Ļà¸ģัà¸Ļ":135947,"ĠÙģÙIJ":135948,"ĠÙģÙIJÙĬ":135949,"ÙĤاعدة":135950,"Ġmożesz":135951,"ÙħصاÙĦ":135952,"ÙħصاÙĦØŃ":135953,"ãģ¾ãģŁãģ¯":135954,"бег":135955,"Ġsıc":135956,"Ġsıcak":135957,"ÑĩиÑģ":135958,"ÑĩиÑģлен":135959,"Ġног":135960,"ãĥģãĥ£ãĥ³":135961,"ãĥ«ãĥī":135962,"Ġgió":135963,"Ġsını":135964,"Ġsınıf":135965,"иваÑĤÑĮ":135966,"Ġquên":135967,"Ġìłģ":135968,"Ġìłģìļ©":135969,"ĠJoão":135970,"ÙģØ§Ø¯":135971,"ĠGlück":135972,"à¸Ĺà¸Ńà¸Ķ":135973,"Ġgói":135974,"ï¼Ĭ":135975,"Ġdétail":135976,"ĠدÙĬسÙħ":135977,"ĠدÙĬسÙħبر":135978,"ë¡ľìĦľ":135979,"×ŀ×ķ×Ĺ":135980,"à¹Ħฮ":135981,"ĠоÑĤд":135982,"ĠоÑĤдÑĭÑħ":135983,"Ġkhuyến":135984,"à¸Ħà¸Ńย":135985,"ĠجÙĨÙĬ":135986,"ĠجÙĨÙĬÙĩ":135987,"ĠاÙĦدÙģØ§Ø¹":135988,"à¸Ļà¹īำหà¸Ļัà¸ģ":135989,"ĠìĤ¬ëŀĮëĵ¤ìĿ´":135990,"Ġthừa":135991,"ĠÃ¶ÄŁrenci":135992,"ĠпомоÑīи":135993,"ĠczÄĻÅĽÄĩ":135994,"ש×ĺר":135995,"ĠNhi":135996,"ĠNhiá»ģu":135997,"׳צ×Ļ":135998,"ĠнаÑĪем":135999,"ĠkarÅŁÄ±laÅŁ":136000,"Ġ×Ķש׳×Ļ×Ŀ":136001,"ĠÄIJưá»Ŀng":136002,"Ġtrú":136003,"ĠÑĢазлиÑĩнÑĭÑħ":136004,"ĠاÙĦØ´Ùĩر":136005,"Ġ×ľ×¢×ķ׾×Ŀ":136006,"ØŃجر":136007,"ĠÄijá»ķ":136008,"ĠìĿĺíķ´":136009,"à¸ļà¹Īà¸Ńย":136010,"Ġ×Ķ×Ļ׾×ĵ":136011,"ãģ¨ãģªãģ£ãģŁ":136012,"Ġ×Ĺ×ķ×ķת":136013,"Ġש×Ļר×ķת×Ļ":136014,"Äħcy":136015,"سرÙĬ":136016,"Kİ":136017,"פ׳×ķ":136018,"ÑģÑĤÑĢÑĥкÑĤÑĥÑĢ":136019,"ÑĤÑĢÑĥд":136020,"Ġ×Ķקר":136021,"Ġ×Ķקר×ķ×ij":136022,"ĠtháºŃm":136023,"èģŀãģį":136024,"ÙĤÙĪÙĬ":136025,"клÑİÑĩен":136026,"ÑĤеÑħ":136027,"ÑĤеÑħнолог":136028,"è¡Įãģ£ãģŁ":136029,"Ġ×ķ×IJ×Ļף":136030,"ĠÅŁeklin":136031,"ĠÅŁeklinde":136032,"rô":136033,"ÑĢог":136034,"ĠновÑĭе":136035,"Ġס×ij×Ļ×ij":136036,"ĠtecnologÃŃa":136037,"×¡×Ľ":136038,"×¡×Ľ×ķ×Ŀ":136039,"ĠÅŀub":136040,"ĠÅŀubat":136041,"Ġ×Ķ×ŀ׾×IJ":136042,"Ġwypos":136043,"Ġwyposaż":136044,"ãģ¯ä½ķ":136045,"ãĤ¬ãĥ³":136046,"ê°ĸ":136047,"Ġкакие":136048,"Ġçocuklar":136049,"Ġ׾צ×ĵ":136050,"Ġkayıt":136051,"ĠмеÑģÑĤе":136052,"ÙħدÙĬÙĨØ©":136053,"Ġ׼×Ĵ":136054,"Ġ׼×Ĵ×ķף":136055,"ãģĹãģ¦ãĤĭ":136056,"ĠÙħاÙĬÙĪ":136057,"ãģ£ãģ¦ãģĹãģ¾ãģ£ãģŁ":136058,"ĠпÑĢогÑĢаммÑĭ":136059,"à¹ģลà¸Ļà¸Ķà¹Į":136060,"ãĥ¯ãĤ¤":136061,"ער×ķ×¥":136062,"Ñģид":136063,"ĠBöyle":136064,"Ġì²ĺìĿĮ":136065,"Ġתפק×Ļ×ĵ":136066,"ĠTrên":136067,"íĥĪ":136068,"ĠÐłÐ¾ÑģÑģий":136069,"ĠÐłÐ¾ÑģÑģийÑģкой":136070,"ĠsÃłn":136071,"Ġrègle":136072,"ĠyaklaÅŁÄ±k":136073,"à¹Ģลิà¸ģ":136074,"ĠدائÙħ":136075,"Ġ×ķ×Ĵ":136076,"ابر":136077,"Ġbè":136078,"ĠاÙĦÙĤدÙħ":136079,"ĠÑĢеÑĪениÑı":136080,"hiên":136081,"ÑĤик":136082,"ÄĦ":136083,"à¸ļรรยาà¸ģ":136084,"à¸ļรรยาà¸ģาศ":136085,"רצ×ķף":136086,"åĭķãģį":136087,"ĠGäste":136088,"Ġ기본":136089,"ĠÙĬعرÙģ":136090,"ĠSá»Ń":136091,"gÅĤÄĻb":136092,"à¹Ģà¸Ńส":136093,"×IJ×ŀ×Ļף":136094,"ĠпÑĥнк":136095,"ĠпÑĥнкÑĤ":136096,"Ġ×Ļ×ķ×ĵ×¢×Ļ×Ŀ":136097,"ãĤ«ãĥ©ãĥ¼":136098,"Ġ×ijס×ĵר":136099,"Ġbuá»ĵn":136100,"йÑĤ":136101,"йÑĤеÑģÑĮ":136102,"ãĤĴæ±ĤãĤģ":136103,"Ġ×IJ×ª×Ľ×Ŀ":136104,"Ġ모르":136105,"ظرÙĪÙģ":136106,"ÑĩеÑģÑĤво":136107,"ìĸ´ìĦľ":136108,"Ġодна":136109,"Ġkapı":136110,"Ġëħ¸ëł¥":136111,"ĠKüche":136112,"ĠاÙĦتش":136113,"Ø·ÙĬب":136114,"ĠíĬ¹íŀĪ":136115,"ĠвÑĭпÑĥÑģ":136116,"ĠвÑĭпÑĥÑģк":136117,"×ĵת×Ļ":136118,"ĠuÄŁ":136119,"ĠuÄŁra":136120,"ائÙĩا":136121,"Ġthoát":136122,"ãģªãĤĤãģ®":136123,"ÑijÑĢ":136124,"기ê°Ģ":136125,"ĠgeliÅŁme":136126,"تØŃÙĤ":136127,"تØŃÙĤÙĤ":136128,"ĠопаÑģ":136129,"бÑĢоÑģ":136130,"หุ":136131,"หุà¹īà¸Ļ":136132,"ì¼Ģ":136133,"ãĤ¹ãĥŀ":136134,"ãĤ¹ãĥŀãĥĽ":136135,"Ø£Ù쨱":136136,"Ø£ÙģØ±Ø§Ø¯":136137,"ĠThá»±c":136138,"Ġthắ":136139,"ãĥªãĥ³ãĤ¯":136140,"Ġniá»ģm":136141,"ĠHöhe":136142,"عÙħار":136143,"ÙĥÙĪØ±ÙĪÙĨ":136144,"ÙĥÙĪØ±ÙĪÙĨا":136145,"ĠÄIJến":136146,"ĠÑģамом":136147,"ĠÑĤеле":136148,"ĠÄijoán":136149,"à¸Ħวามà¸Ħิà¸Ķà¹Ģหà¹ĩà¸Ļ":136150,"ĠдиÑģк":136151,"أطÙ쨧ÙĦ":136152,"มารà¹Į":136153,"à¸Ĺหาร":136154,"à¸Ĺà¸Ļ":136155,"ĠبعÙĬد":136156,"ĠاÙĦÙĩÙĨد":136157,"åĩºãģĹãģ¦":136158,"Ġkarde":136159,"ĠkardeÅŁ":136160,"×Ķ×Ļס×ĺ×ķר":136161,"×Ķ×Ļס×ĺ×ķר×Ļ×Ķ":136162,"éģ¸ãģ³":136163,"عاÙħÙĦ":136164,"à¸Ĥยาย":136165,"Ġtürl":136166,"Ġtürlü":136167,"ĠìĿ¼ìĿ´":136168,"Ġmatéria":136169,"Ġ׼׾×ķ×ŀר":136170,"ãĥģãĥ£ãĥ¼":136171,"جÙħاعة":136172,"ĠÑģвоим":136173,"Ø¥ÙĤاÙħØ©":136174,"ä¾ĭãģĪãģ°":136175,"ساب":136176,"آخر":136177,"ÙĤدÙĬر":136178,"×IJ×ŀ×Ļ":136179,"ìĸ»":136180,"Ġ׳×ķספת":136181,"ĠÐĴлад":136182,"ĠÐĴладим":136183,"ĠÐĴладимиÑĢ":136184,"Ġestará":136185,"ãģĵãģĨãģĦãģĨ":136186,"ãĤĴ使ç͍":136187,"มาà¸ķร":136188,"มาà¸ķรà¸IJาà¸Ļ":136189,"ãģ£ãģ½":136190,"Ġnú":136191,"Ġnúi":136192,"ยาà¸ĩ":136193,"ĠاÙĦجÙĨس":136194,"Ġüstün":136195,"ëľ»":136196,"ãĤ»ãĥ«":136197,"ãģ¦ãģĦãģįãģ¾ãģĻ":136198,"Ġ×Ĺ×ķ×ĸ":136199,"Ġ×Ĺ×ķ×ĸר":136200,"ĠÐĵлав":136201,"à¹Ĥà¸Ĭà¸Ħ":136202,"íıIJ":136203,"ÙĨتظر":136204,"Ġ×Ĵ×ij×Ļ":136205,"عÙĤب":136206,"intér":136207,"intérêt":136208,"×ŀפ×Ĵ":136209,"×ŀפ×Ĵש":136210,"Ġthù":136211,"اÙģØª":136212,"Ġ×ŀשפ":136213,"Ġ×ŀשפ×ĺ×Ļ":136214,"ĠÙħÙĪØ§ÙĤع":136215,"è¦ļ":136216,"è¦ļãģĪ":136217,"×ĵ×Ļף":136218,"à¹Ģรืà¹Īà¸Ńà¸ĩราว":136219,"ãģ¾ãģĤ":136220,"Ġghế":136221,"иÑĢÑĥÑİÑĤ":136222,"à¸ģว":136223,"à¸ģวà¹īาà¸ĩ":136224,"ĠповеÑĢ":136225,"ĠповеÑĢÑħ":136226,"ĠповеÑĢÑħноÑģÑĤ":136227,"׳×ĵר":136228,"ĠконÑĨе":136229,"Ġдолжна":136230,"Ġ×Ļש×Ļר":136231,"acaģız":136232,"ìĹĶ":136233,"ĠnÃŃvel":136234,"Ġör":136235,"Ġörnek":136236,"ÙĥÙģ":136237,"ĠФедеÑĢаÑĨии":136238,"Ġ구ìĦ±":136239,"หัวà¹ĥà¸Ī":136240,"ĠVáºŃy":136241,"мед":136242,"меди":136243,"медиÑĨин":136244,"медиÑĨинÑģк":136245,"ازÙĬ":136246,"×Ĵ×ij×ķ׾":136247,"ÑĦÑĢ":136248,"Ġzusätzlich":136249,"à¸ģà¸ģ":136250,"ĠاÙĦاÙĤتصادÙĬØ©":136251,"Ġhè":136252,"luÄŁun":136253,"جÙİ":136254,"à¹Ħà¸Łà¸¥à¹Į":136255,"ÄIJT":136256,"ãģĿãģ®ä»ĸ":136257,"à¸Ĺิà¹īà¸ĩ":136258,"ĠاÙĦØ£ÙĪ":136259,"رسÙħ":136260,"æ°Ĺãģ¥":136261,"ìĿ´ë©°":136262,"ÑĮев":136263,"صط":136264,"ĠاÙĦاستث":136265,"ĠاÙĦاستثÙħار":136266,"à¸Ńาà¸Ħาร":136267,"ĠÑĤоÑĩно":136268,"ĠVân":136269,"à¸Ńร":136270,"à¸Ńรà¹Īà¸Ńย":136271,"ĠاÙĦسÙĨØ©":136272,"ĠcÆ°á»Ľi":136273,"×Ļ×Ķף":136274,"íį¼":136275,"話ãģĹ":136276,"âĹĭ":136277,"ĠìķĬìĿĢ":136278,"ãĥ¡ãĥ¼ãĤ":136279,"ãĥ¡ãĥ¼ãĤ«":136280,"ãĥ¡ãĥ¼ãĤ«ãĥ¼":136281,"ĠÑĤепло":136282,"å½¼ãĤī":136283,"Ġİz":136284,"Ġİzmir":136285,"íĻį":136286,"Ġrượ":136287,"Ġrượu":136288,"æĢĿãģĦåĩº":136289,"ĠPhạm":136290,"Ġcháu":136291,"צ×Ļ×ķת":136292,"ĠìĿ¼ë³¸":136293,"ìĤ¬ëĬĶ":136294,"ĠÑģоздан":136295,"Ġaracı":136296,"Ġער":136297,"Ġער×Ļ׼×Ķ":136298,"ĠíķĺëĤĺëĭĺìĿĺ":136299,"dziÅĤ":136300,"à¸Ľà¸£à¸°à¸ĺาà¸Ļ":136301,"ĠserÃŃa":136302,"ĠìŀĪëıĦë¡Ŀ":136303,"درج":136304,"íķľëĭ¤ëĬĶ":136305,"à¸Ńาà¸Ĺ":136306,"à¸Ńาà¸Ĺิà¸ķ":136307,"à¸Ńาà¸Ĺิà¸ķยà¹Į":136308,"ÑĤелÑĮнÑĭй":136309,"ĠخدÙħات":136310,"×ŀ׳×ĺ":136311,"Ġlược":136312,"ĠSÃłi":136313,"ĠÙĪØ§Ø¶":136314,"ĠÙĪØ§Ø¶ØŃ":136315,"غاز":136316,"ĠdoÄŁal":136317,"Ġ×ijש×Ŀ":136318,"Ġдлин":136319,"Ġإطار":136320,"Ġ×ijספר":136321,"ãĤĴä¸İ":136322,"ãĤĴä¸İãģĪ":136323,"Ġë²ķë¥ł":136324,"ĠÑĥвели":136325,"ĠÑĥвелиÑĩи":136326,"สà¹Ħà¸ķ":136327,"สà¹Ħà¸ķลà¹Į":136328,"à¹Ħà¸ģล":136329,"×ij×Ĺף":136330,"ĠìĿ´íĽĦ":136331,"Ġmunic":136332,"ĠmunicÃŃpio":136333,"تÙħØ«ÙĦ":136334,"ĠÄijáo":136335,"Hôtel":136336,"Ġlá»Ńa":136337,"ĠÄijẳng":136338,"Ñĩки":136339,"شرÙĪ":136340,"شرÙĪØ·":136341,"ĠìĿ´ë¥¼":136342,"ÙĬÙĭا":136343,"×ŀ׾×ļ":136344,"×ŀ×Ķ×Ļר×ķת":136345,"ĠобÑıзаÑĤелÑĮ":136346,"ĠобÑıзаÑĤелÑĮно":136347,"énergie":136348,"Ġmudança":136349,"Ġmụ":136350,"Ġmụn":136351,"Ġnº":136352,"ĠاÙĦتعا":136353,"ĠاÙĦتعاÙĪÙĨ":136354,"ĠاÙĦاجتÙħاعÙĬØ©":136355,"ĠплаÑģÑĤ":136356,"Ġëĵ±ìĿĺ":136357,"ãĥIJãĤ¤ãĤ¯":136358,"ÙĩجÙĪÙħ":136359,"ĠSaúde":136360,"Ġì¤ijìļĶíķľ":136361,"Ġ×Ķצ×Ļ×ij×ķר":136362,"×ª×§×Ł":136363,"ĠاÙĦعاÙĦÙħÙĬ":136364,"ĠболÑĮÑĪой":136365,"ĠÙĥÙĦÙħ":136366,"ĠÙĥÙĦÙħØ©":136367,"ãģ®ãģ§ãģ¯ãģªãģĦãģ§ãģĹãĤĩãģĨãģĭ":136368,"ĠÙħباراة":136369,"Ġש×IJ׳":136370,"Ġש×IJ׳×Ĺ׳×ķ":136371,"ãĤ¹ãĤ¿ãĤ¤ãĥ«":136372,"ĠSaÄŁ":136373,"ĠSaÄŁlık":136374,"Ġhư":136375,"׳×Ĺ×Ķ":136376,"Ġ×ijקר×ij":136377,"طعÙħ":136378,"หิà¸Ļ":136379,"à¸Ĺุà¸ģวัà¸Ļ":136380,"à¸Ħรัà¹īà¸ĩà¸Ĺีà¹Ī":136381,"ĠlÃłnh":136382,"Ġdonné":136383,"ãģĽãģĦ":136384,"جزÙĬرة":136385,"доÑĢож":136386,"ì¼ľ":136387,"تÙĨظÙĬÙģ":136388,"ãĥģãĥ§":136389,"Ġaldıģı":136390,"جاج":136391,"ĠÑĤомÑĥ":136392,"à¸Ľà¸´":136393,"Ġ×ijרשת":136394,"ãģıãģªãĤĬãģ¾ãģĻ":136395,"ĠпÑĢинÑĨип":136396,"Ġ×Ĺ׾×ķ":136397,"ëı¼":136398,"×ķ×Ĵש":136399,"سس":136400,"à¸Ľà¸¹":136401,"Ġhầu":136402,"æĦŁãģĺãĤĭ":136403,"ï¼´":136404,"دÙĪØ§":136405,"ĠÑģмог":136406,"scrição":136407,"ĠtháºŃn":136408,"Ġר×ķ×IJ×Ķ":136409,"обÑĢажен":136410,"ĠاÙĦتجارÙĬØ©":136411,"طبÙĬع":136412,"jÄħcÄħ":136413,"íĸīìľĦ":136414,"ĠновÑĭй":136415,"Ġ×ŀ×Ĺ×ĵש":136416,"æĮ¯ãĤĬ":136417,"gué":136418,"Ġ×IJ×Ļר×ķ×¢":136419,"Ġ×IJ×Ļר×ķ×¢×Ļ×Ŀ":136420,"ĠاÙĦذÙĩب":136421,"×ĵ×IJ":136422,"تاÙĨ":136423,"ãģłãģĹ":136424,"à¸Ńัà¸ķรา":136425,"à¹Ĥà¸Ī":136426,"بÙĦاد":136427,"×Ķ×Ļ×Ļ׳×ķ":136428,"ĠÑģпе":136429,"ĠÑģпеÑĨиалÑĮно":136430,"ĠÅĽwiata":136431,"ãĤĵãģ§ãģĻãĤĪ":136432,"شرÙĥØ©":136433,"ĠpÅĤyt":136434,"Ġsitué":136435,"Ġ׼×IJ׾×Ķ":136436,"ס×ijר":136437,"Ġkażd":136438,"Ġkażdym":136439,"ãĤĴæĮģãģ¤":136440,"׾×Ķ׾":136441,"׾×Ķ׾ף":136442,"ĠwÅĤas":136443,"ĠwÅĤasne":136444,"ĠsaÄŁlan":136445,"×ŀ×¢×ľ×Ķ":136446,"ĠاÙĦاÙĪÙĦ":136447,"ìĹIJìĦľëıĦ":136448,"×IJ×Ļר×ķפ×Ķ":136449,"تÙĤÙĨÙĬØ©":136450,"Ùħائ":136451,"Ùħائة":136452,"ĠcompañÃŃa":136453,"Ġsürek":136454,"Ġsürekli":136455,"ĠиÑģкÑĥÑģ":136456,"ĠиÑģкÑĥÑģÑģÑĤв":136457,"ĠBürger":136458,"ת×Ĺר":136459,"ת×Ĺר×ķת":136460,"à¸ŀรà¹īà¸Ńมà¸ģัà¸ļ":136461,"Ø´Ùħ":136462,"à¸ĸืà¸Ńวà¹Īา":136463,"è¾¼ãĤĢ":136464,"ä¼ijãģ¿":136465,"ĠاÙĦأب":136466,"ĠÑģÑĤоимоÑģÑĤÑĮ":136467,"ĠпÑĢава":136468,"mayın":136469,"หวย":136470,"ĠاÙĦطبÙĬعÙĬ":136471,"à¸Ĺีà¹Īà¸ŀัà¸ģ":136472,"ĠEstá":136473,"ÑĭваÑİÑĤ":136474,"بسÙĬ":136475,"بسÙĬØ·":136476,"Ġ×ij×¢×ijר":136477,"åı¯èĥ½ãģ§ãģĻ":136478,"Ġ×ĵ×ķ׾":136479,"Ġ×ĵ×ķ׾ר":136480,"ÙĩÙİØ§":136481,"воÑĢоÑĤ":136482,"ãģ¦ãģĦãģ¾ãģĹãģŁ":136483,"à¹Ĥà¸Ĺรศ":136484,"à¹Ĥà¸Ĺรศั":136485,"à¹Ĥà¸Ĺรศัà¸ŀ":136486,"à¹Ĥà¸Ĺรศัà¸ŀà¸Ĺà¹Į":136487,"Ġק׳":136488,"ĠاÙĦØ«ÙĨ":136489,"ĠاÙĦØ«ÙĨائÙĬØ©":136490,"Ġcoût":136491,"à¸ķิà¸Ķà¸ķัà¹īà¸ĩ":136492,"Ġörg":136493,"Ġörgüt":136494,"ĠاÙĦØ®ÙĦÙĬ":136495,"ĠاÙĦØ®ÙĦÙĬج":136496,"Ġbá»įn":136497,"×ķ׾×ķ×Ĵ×Ļ":136498,"ëŀľ":136499,"ĠÐijолÑĮ":136500,"ĠÐijолÑĮÑĪ":136501,"×Ĵ×ijר×Ļ×Ŀ":136502,"ÙĤÙĬد":136503,"×ij×Ļ×ĺ×ķ×Ļ":136504,"æīĵãģ¡":136505,"ĠolmuÅŁ":136506,"fäh":136507,"fähig":136508,"ลาà¸Ļ":136509,"ĠÙĤطر":136510,"שפ×Ķ":136511,"èªŃãĤĵãģ§":136512,"à¸Ĥวา":136513,"Ġchiếm":136514,"ãĤ¤ãĥ³ãĤ¿":136515,"ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥ":136516,"ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥį":136517,"ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥįãĥĥãĥĪ":136518,"Ġ׾ש×ŀ×ķר":136519,"ĠترÙĥ":136520,"ĠترÙĥÙĬا":136521,"ר×ķ×ĺ":136522,"ã썿ĢĿãģĦãģ¾ãģĹãģŁ":136523,"ĠاÙĦتÙĤ":136524,"Ġdư":136525,"ãģ¦ãģıãĤĮãĤĭ":136526,"ãģĹãģŁãģĵãģ¨":136527,"Ġróżne":136528,"ĠاÙĦØ·ÙģÙĦ":136529,"ĠPosté":136530,"Ġ×ŀש×ķ×Ŀ":136531,"ÑįÑĢ":136532,"ĠÑĢабоÑĤаеÑĤ":136533,"ãĤ·ãĥª":136534,"ãĤ·ãĥªãĥ¼ãĤº":136535,"Ġ×ij×Ķ×Ĺ׾×ĺ":136536,"×§×Ķ×Ļ׾×Ķ":136537,"ãĤ«ãĥ¡":136538,"ãĤ«ãĥ¡ãĥ©":136539,"O":136540,"ĠìĤ¬ìĿ´":136541,"Ġkì":136542,"ĠthÆ°á»Ľc":136543,"ضبط":136544,"ÙĤبÙĪÙĦ":136545,"åĪ¥ãģ®":136546,"Ġparticulière":136547,"ĠÑģвоем":136548,"Ġעסק":136549,"Ġעסק×Ļ×Ŀ":136550,"×ij×Ĺ×Ļר×ķת":136551,"×ij×Ļ׳×ķ":136552,"à¸ĭà¸Ń":136553,"Ġ×¢×ķ×ijר":136554,"ãģłãģ£ãģŁãģ®ãģ§":136555,"ıldıģı":136556,"Ùħدار":136557,"Ùħدارس":136558,"주ìĭľ":136559,"à¸Ńาศ":136560,"à¸Ńาศัย":136561,"Ġtấm":136562,"à¸ŀิà¸Ī":136563,"à¸ŀิà¸Īาร":136564,"à¸ŀิà¸Īารà¸ĵา":136565,"ÑĤелÑĮнÑĭе":136566,"ÑģкÑĥÑİ":136567,"ÐľÐĺ":136568,"à¹Ģà¸ģา":136569,"à¹Ģà¸ģาหล":136570,"à¹Ģà¸ģาหลี":136571,"×ĵ×Ĺ":136572,"à¹Ģà¸Ĭิà¸ĩ":136573,"ĠدÙĤÙĬÙĤØ©":136574,"íķĻìĥĿ":136575,"Ġש×IJ׾×Ķ":136576,"Ġcontrôle":136577,"Ġsituação":136578,"à¸Ĥà¸Ńà¸ĩà¸ľà¸¹à¹ī":136579,"ÙĨØ·ÙĤ":136580,"ê³¼íķĻ":136581,"หลายà¸Ħà¸Ļ":136582,"Ġnắng":136583,"ÙĤÙı":136584,"ì¡°ê±´":136585,"Ñķ":136586,"ãĥĥãģ¨":136587,"×ŀ×Ļ׾×Ķ":136588,"Grün":136589,"×Ļ×Ļ×¢":136590,"×Ļ×Ļ×¢×ķ×¥":136591,"×ŀ׳׼":136592,"ëŃIJ":136593,"×ŀ×¢×ŀ×ĵ":136594,"สำà¸Ļัà¸ģ":136595,"جدد":136596,"à¸Ħัà¸Ķ":136597,"Ġ×Ķ×ŀשפ":136598,"Ġ×Ķ×ŀשפ×Ĺ×Ķ":136599,"×ŀשק׾":136600,"ÙĦÙı":136601,"Ġtytu":136602,"ĠtytuÅĤ":136603,"ÑĪей":136604,"ĠìĿ¼ë¶Ģ":136605,"ÑĪение":136606,"Ġphóng":136607,"ĠìĹŃìĤ¬":136608,"ãĤ«ãĥ³":136609,"Ġtúi":136610,"ĠÙĨÙĪÙģ":136611,"ĠÙĨÙĪÙģÙħبر":136612,"grün":136613,"ĠاÙĦØ´ÙħاÙĦ":136614,"ÅĽwiadc":136615,"ÅĽwiadczenie":136616,"ער×Ķ":136617,"Ġ×¢×ķ×ij":136618,"Ġ×¢×ķ×ij×ĵ×Ļ×Ŀ":136619,"×ĵ×ķ×Ĵ×ŀ×IJ":136620,"ä»Ĭãģ¯":136621,"Ġvão":136622,"ĠТем":136623,"ÑģилÑĮ":136624,"Ġchợ":136625,"Ùħرا":136626,"ÙħراÙĤب":136627,"à¹Ħมà¹Īรูà¹ī":136628,"Ġرائع":136629,"×IJ׳×Ĺ׳×ķ":136630,"สà¹Īà¸ĩà¹Ģสริม":136631,"צ×Ĺ":136632,"ĠìŀĪìĸ´ìĦľ":136633,"Ġkurulu":136634,"ĠkuruluÅŁ":136635,"ĠÃĸzellik":136636,"ĠÃĸzellikle":136637,"Ġת×Ļ×§":136638,"Ġghé":136639,"ĠsprzÄĻ":136640,"ĠsprzÄĻt":136641,"ער×ķת":136642,"راØŃØ©":136643,"ãģ£ãģį":136644,"ãģ£ãģįãĤĬ":136645,"ĠìķĦëŀĺ":136646,"stituição":136647,"Ġдолжно":136648,"×Ķרש":136649,"×Ķרש×ŀ×Ķ":136650,"×Ķ׾×ļ":136651,"ãģ¡ãģª":136652,"ãģ¡ãģªãģ¿":136653,"ãģ¡ãģªãģ¿ãģ«":136654,"פ×Ĺ×ĵ":136655,"ĠاÙĦجÙħÙĬع":136656,"×ij×¢×ľ×Ļ":136657,"Ġtrùng":136658,"Ġפת×Ĺ":136659,"×ŀ׾×Ĺ×ŀת":136660,"ãĥĨãĥ¼ãĥ":136661,"ãĥĨãĥ¼ãĥŀ":136662,"Ùħتاب":136663,"Ùħتابعة":136664,"Ġ모ìĬµ":136665,"ÙĬص":136666,"åIJĪãģĨ":136667,"ĠYap":136668,"ĠYapı":136669,"ĠÑģказаÑĤÑĮ":136670,"몰":136671,"à¸Ĺีà¹Īสำà¸Ħัà¸į":136672,"ĠìĹĨìĬµëĭĪëĭ¤":136673,"Ġnhắc":136674,"Ġülkeler":136675,"Ġмногие":136676,"íķĺìħ¨":136677,"มาà¸ģà¸Ĺีà¹Īสุà¸Ķ":136678,"à¸ģà¹īา":136679,"à¸ģà¹īาว":136680,"Ġİyi":136681,"леж":136682,"лежа":136683,"ãĤ¸ãĥ§":136684,"à¸Ĺัà¸ŀ":136685,"اÙĪØ±":136686,"Ġ×Ĺ×ijר×Ļ":136687,"Ġ׾ש×Ŀ":136688,"첫":136689,"ĠTá»Ń":136690,"×ŀ×ķ׳×Ļ":136691,"ÙĤÙĪØ¯":136692,"à¸ģระà¹Ģà¸Ľ":136693,"à¸ģระà¹Ģà¸Ľà¹ĭ":136694,"à¸ģระà¹Ģà¸Ľà¹ĭา":136695,"ĠпÑĢоблемÑĭ":136696,"Ġaçıs":136697,"Ġaçısından":136698,"Ġ×Ķ×ŀ׼":136699,"ĠÙħعظÙħ":136700,"ÙĤÙĬاس":136701,"ĠпÑĢодолж":136702,"ĠпÑĢодолжа":136703,"ĠverdiÄŁi":136704,"ĠпÑĢедмеÑĤ":136705,"ãģĦãģ¾ãģĻãģĮ":136706,"ĠëĶ°ë¥¸":136707,"ĠاÙĦÙĤÙĬاÙħ":136708,"ĠØ¥ÙĦÙĬÙĩا":136709,"ТÐIJ":136710,"поз":136711,"ãĤ·ãĥ¥":136712,"ä¸ĬãģĮãĤĬ":136713,"à¹Ģà¸Ķิมà¸ŀัà¸Ļ":136714,"à¸ģุล":136715,"ØŃرÙĬØ©":136716,"×§×ij×ķצ×ķת":136717,"믿":136718,"ĠاÙĦÙħÙĨا":136719,"ĠاÙĦÙħÙĨاطÙĤ":136720,"ĠвÑĭпол":136721,"ĠвÑĭполнÑı":136722,"ãĥĭãĤ¢":136723,"Ġê²°êµŃ":136724,"×Ĺ×ķ×ŀ":136725,"×Ĺ×ķ×ŀר×Ļ×Ŀ":136726,"ĠУкÑĢаинÑĭ":136727,"หà¸Ńม":136728,"ר×Ļס":136729,"ĠÑħоÑĤел":136730,"ĠобÑĢазованиÑı":136731,"Ġkhẳng":136732,"Ġmưa":136733,"Ġgörme":136734,"Ġgüçlü":136735,"سعÙī":136736,"มัà¹Īà¸Ļà¹ĥà¸Ī":136737,"íķĺê²łìĬµëĭĪëĭ¤":136738,"ĠполÑĥ":136739,"Ġfünf":136740,"ã썿ĢĿãģ£ãģ¦ãģĦãģ¾ãģĻ":136741,"Ġê·¸ê²ĥìĿĢ":136742,"ĠdÃ¼ÅŁÃ¼nce":136743,"ìŀł":136744,"ĠHÆ°á»Ľng":136745,"ĠTiá»ĥu":136746,"Ġçift":136747,"ãģijãģ°":136748,"à¸Īà¸Ļà¸ĸึà¸ĩ":136749,"à¸Ĺำà¹Ħà¸Ķà¹ī":136750,"ĠìŀIJì²´":136751,"Ġdõ":136752,"Ġdõi":136753,"à¸Īัà¸Ļ":136754,"à¸Īัà¸Ļà¸Ĺ":136755,"à¸Īัà¸Ļà¸Ĺรà¹Į":136756,"eceÄŁini":136757,"׳×ķער":136758,"غار":136759,"ĠاÙĦØ£ÙħرÙĬÙĥÙĬ":136760,"داعش":136761,"ĠбезопаÑģноÑģÑĤи":136762,"ĠбÑİ":136763,"ĠбÑİдж":136764,"ĠбÑİджеÑĤ":136765,"ãĥĬãĤ¤":136766,"à¸ŀà¸ļวà¹Īา":136767,"daÄŁ":136768,"×IJ×ķפף":136769,"íĹĮ":136770,"ãĥĢãĤ¤ãĤ¨":136771,"ãĥĢãĤ¤ãĤ¨ãĥĥãĥĪ":136772,"ĠëĮĢíĨµ":136773,"ĠëĮĢíĨµëł¹":136774,"Dİ":136775,"Ø£ØŃداث":136776,"ĠAÄŁ":136777,"ĠAÄŁust":136778,"ĠAÄŁustos":136779,"ØŃÙĦÙĪÙĦ":136780,"ĠwÅĽ":136781,"ĠwÅĽród":136782,"ĠÑģооÑĤвеÑĤ":136783,"ĠÑģооÑĤвеÑĤÑģÑĤв":136784,"ĠÑģооÑĤвеÑĤÑģÑĤвии":136785,"ĠLuáºŃt":136786,"Ġ׼׾פ×Ļ":136787,"ĠвеÑī":136788,"ĠвеÑīеÑģÑĤв":136789,"×§×Ļ×¥":136790,"ĠبÙĩذا":136791,"عاش":136792,"à¹Ģà¸Ľà¹ĩà¸Ļà¹Ģรืà¹Īà¸Ńà¸ĩ":136793,"ТÐķ":136794,"Ġ×ij×IJ×Ļ׳×ĺר׳×ĺ":136795,"سعد":136796,"Ġ×Ķ×ĺ×Ļפ×ķ׾":136797,"פ×Ļס":136798,"à¸ĩà¹Īายà¹Ĩ":136799,"ĠGerät":136800,"׾×Ļ×ĵ×Ķ":136801,"ĠÑĢиÑģк":136802,"׾ק×Ĺ":136803,"ннаÑı":136804,"ר×Ļ×ĵ":136805,"пÑĢакÑĤи":136806,"пÑĢакÑĤик":136807,"à¸Ĥัà¹īà¸Ļà¸ķà¸Ńà¸Ļ":136808,"à¸Ļà¹Īารัà¸ģ":136809,"larınızı":136810,"à¸Ńà¸Ļุà¸įา":136811,"à¸Ńà¸Ļุà¸įาà¸ķ":136812,"ĠzdjÄĻcia":136813,"Ġbây":136814,"ÑģÑĢ":136815,"ÑģÑĢоÑĩ":136816,"ãĥĭãĥ³ãĤ°":136817,"Ġöner":136818,"Ġöneri":136819,"ĠновÑĭÑħ":136820,"دعÙĪØ©":136821,"Ġgắn":136822,"ĠاÙĦÙĦبÙĨ":136823,"ĠاÙĦÙĦبÙĨاÙĨÙĬ":136824,"ãĥĨãĤ£ãĥ¼":136825,"ĠصØŃÙĬØŃ":136826,"емÑĭÑħ":136827,"çĸ²ãĤĮ":136828,"ĠпÑĢоиÑģ":136829,"ĠпÑĢоиÑģÑħодиÑĤ":136830,"สà¸ķิ":136831,"ĠTết":136832,"Ġ×Ķ׾׾×ķ":136833,"à¹Ģรืà¹Īà¸Ńà¸ĩà¸Ļีà¹ī":136834,"×ŀ×ij׳×Ķ":136835,"Ġconteúdo":136836,"Ġاخت":136837,"ĠاختÙĬار":136838,"ÙħسÙĦ":136839,"ÙħسÙĦسÙĦ":136840,"ëıĪ":136841,"Ġ׾×Ļ×ĵ":136842,"à¸ŀิà¸ĺี":136843,"ĠÑģовÑģ":136844,"ĠÑģовÑģем":136845,"ãģĮãģĤãĤĬãģ¾ãģĹãģŁ":136846,"Ġsóng":136847,"إصÙĦاØŃ":136848,"ë§ģ":136849,"ÙģÙĬر":136850,"ĠJeżeli":136851,"ìłľëıĦ":136852,"dÅĤug":136853,"ìĥģìĿĦ":136854,"ĠcáºŃn":136855,"Ġhá»įp":136856,"أست":136857,"أستاذ":136858,"Ġ×ŀ×Ļש×Ķ":136859,"Ġ×ŀ×Ļש×Ķ×ķ":136860,"ĠdÃły":136861,"ĠchÃłng":136862,"ãģ¡ãĤĥãĤĵãģ¨":136863,"ĠÄijám":136864,"Ġswój":136865,"Ġpoderá":136866,"ĠоÑĤлиÑĩа":136867,"Ġpériode":136868,"ündig":136869,"×ĺ×¢×Ł":136870,"ÑģÑĤÑĢоиÑĤелÑĮ":136871,"רת×Ļ":136872,"Ġ×Ļ×Ķ×Ļ×ķ":136873,"×ľ×¡":136874,"ĠاÙĦÙħÙĨزÙĦ":136875,"à¸Ļิà¹īว":136876,"иÑĦика":136877,"иÑĦикаÑĨи":136878,"ðŁĺī":136879,"Ġadına":136880,"ãĢĤãĢĤãĢĤ":136881,"×IJ×Ļף":136882,"ס×Ļר":136883,"ĠÙĬعد":136884,"çŃĶãģĪ":136885,"اÙĦجز":136886,"اÙĦجزائر":136887,"енÑĮк":136888,"รห":136889,"รหัส":136890,"ĠTürkçe":136891,"꾸":136892,"Ġ×Ļ×ķ׼׾":136893,"Ġש×ķ׳×Ķ":136894,"Ġ×ij×ŀצ×ij":136895,"ĠдейÑģÑĤвиÑĤелÑĮно":136896,"ĠبأÙĨÙĩ":136897,"×ŀ×§×ĵ":136898,"Ġ×Ķשק":136899,"Ø®ÙĬارات":136900,"Ġfı":136901,"Ġfırs":136902,"Ġfırsat":136903,"ëijĺ":136904,"ĠìĦľìļ¸":136905,"Ġ×Ķ×Ĵ×ķ×£":136906,"رعا":136907,"رعاÙĬØ©":136908,"ĠKết":136909,"кÑģи":136910,"ĠÑĥÑģлÑĥги":136911,"ноÑģÑĤей":136912,"ìļ´ëıĻ":136913,"ĠобÑĬÑı":136914,"ĠобÑĬÑıвл":136915,"неж":136916,"×Ķפ×ļ":136917,"Ġ×ij×¢×Ļ׳×Ļ":136918,"ëĨĴ":136919,"ĠпÑĢоÑĨед":136920,"ĠпÑĢоÑĨедÑĥÑĢ":136921,"Ġihtiy":136922,"Ġihtiyacı":136923,"Ġë°Ķëŀį":136924,"Ġë°ĶëŀįëĭĪëĭ¤":136925,"à¸ģลัว":136926,"ĠÑģложно":136927,"×§×Ļ×Ļ×ŀת":136928,"ĠÄIJình":136929,"ĠÙħÙĦÙģ":136930,"Ġà¹Ĥà¸Ķยมี":136931,"Ġkatkı":136932,"تØŃÙĪÙĬÙĦ":136933,"à¹Ħà¸ŀ":136934,"ĠHá»į":136935,"ñe":136936,"ĠдоÑħод":136937,"Ġthoải":136938,"íķĺìŬìķ¼":136939,"ãĤ¹ãĥĿãĥ¼ãĥ":136940,"ãĤ¹ãĥĿãĥ¼ãĥĦ":136941,"ĠGòn":136942,"Ġkè":136943,"Ġkèm":136944,"é̲ãĤģ":136945,"ãĤ¹ãĥ¼ãĥ":136946,"ãĤ¹ãĥ¼ãĥij":136947,"ãĤ¹ãĥ¼ãĥijãĥ¼":136948,"ĠgiÃłu":136949,"Ġإعادة":136950,"Ġ׾×ķ×§":136951,"Ġ׾×ķ×§×Ĺ":136952,"ĠÑħоÑĩеÑĤ":136953,"×ĺ׾×ķ×ķ":136954,"×ĺ׾×ķ×ķ×Ļ×ĸ":136955,"×ĺ׾×ķ×ķ×Ļ×ĸ×Ļ×Ķ":136956,"Ġthuyết":136957,"ãģĿãĤĮãģ§":136958,"Ġvardı":136959,"à¹Ħรà¹ī":136960,"عبد":136961,"ĠRepública":136962,"ãĥ¼ãĤ¿ãĥ¼":136963,"Ġ×ŀ×IJ×ķת":136964,"à¹Ħà¸Ľà¹ģลà¹īว":136965,"Ġyapılacak":136966,"ãĤ¹ãĤ¿ãĥ¼ãĥĪ":136967,"ãģ»ãģ¼":136968,"ĠkoÅŁ":136969,"ĠмаÑĤеÑĢи":136970,"Ġsiècle":136971,"ĠاÙĦÙħختÙĦÙģ":136972,"ĠاÙĦÙħختÙĦÙ쨩":136973,"Ġ׾קר×IJ":136974,"Ġ׾קר×IJת":136975,"Ġ×Ķפ×ķ×¢×ľ":136976,"Ġtòa":136977,"ĠrÆ¡i":136978,"åij¨ãĤĬ":136979,"à¸Ŀà¸Ļ":136980,"jÅĽÄĩ":136981,"ĠìķĬìĿĦ":136982,"اÙĨتÙĤاÙĦ":136983,"ëĸł":136984,"иваеÑĤ":136985,"ãĥĪãĥ«":136986,"ĠاÙĦÙģÙĦسطÙĬÙĨÙĬØ©":136987,"à¸ģลà¹Īาววà¹Īา":136988,"اÙĥت":136989,"ĠÃĸl":136990,"ĠÑĢеÑĪи":136991,"ĠÑĢеÑĪил":136992,"Ġ׳×ķספ×ķת":136993,"Ġìłķì¹ĺ":136994,"влеÑĩен":136995,"ÙħرØŃÙĦØ©":136996,"Ġcomeça":136997,"Ġyık":136998,"ìĤ´":136999,"à¸ĺà¸Ļา":137000,"à¸ĺà¸Ļาà¸Ħาร":137001,"à¸Ńà¸Ļา":137002,"à¸Ńà¸Ļาà¸Ħ":137003,"à¸Ńà¸Ļาà¸Ħà¸ķ":137004,"Ġpequeña":137005,"ä»ķäºĭãĤĴ":137006,"ĠبذÙĦÙĥ":137007,"Ġнового":137008,"ãģĹãģ¦ãģĦãģªãģĦ":137009,"ĠاÙĦÙħÙĬاÙĩ":137010,"à¸ģà¹ĩà¹Ģà¸Ľà¹ĩà¸Ļ":137011,"ĠжÑĥÑĢ":137012,"ĠжÑĥÑĢнал":137013,"веÑģ":137014,"ختار":137015,"Ġ매ìļ°":137016,"ĠMã":137017,"ĠавÑĤомаÑĤÑĭ":137018,"ضعÙģ":137019,"ĠاÙĦÙģÙĥر":137020,"ãģ§ãģĻãģ®ãģ§":137021,"ãĥ¡ãĥ³ãĥIJãĥ¼":137022,"ĠкÑĢÑĥг":137023,"ĠاÙĦسÙĦطة":137024,"à¸Ħรัà¹īà¸ĩà¹ģรà¸ģ":137025,"à¸ģระà¸Ĺรว":137026,"à¸ģระà¸Ĺรวà¸ĩ":137027,"ÑĨов":137028,"éķ·ãģĦ":137029,"大ãģįãģĦ":137030,"ĠgeçmiÅŁ":137031,"ìĦ±ìĿ´":137032,"Ġצר×Ļ׼×Ķ":137033,"ĠмоÑī":137034,"ĠмоÑīн":137035,"Ġ×§×Ļש":137036,"Ġ×§×Ļש×ķר×Ļ×Ŀ":137037,"ĠNasıl":137038,"гÑĢан":137039,"Ġ×ŀ×ķצר×Ļ×Ŀ":137040,"Ġ×ŀס×ķ×Ĵ":137041,"Ġyür":137042,"Ġyürüt":137043,"Ġ׾×Ĺצ×ķ":137044,"×ķÖ¼":137045,"ĠìŀĪìĹĪëĭ¤":137046,"Ġterör":137047,"ĠThương":137048,"ĠÙĪÙĬÙħ":137049,"ĠÙĪÙĬÙħÙĥÙĨ":137050,"جÙĪÙĨ":137051,"ĠÙĪØºÙĬرÙĩا":137052,"×ŀפ×ķ":137053,"×Ĵ×ķר×ŀ×Ļ×Ŀ":137054,"׼×ij×Ļש":137055,"ĠاÙĦÙĦغ":137056,"ĠاÙĦÙĦغة":137057,"شرÙĥ":137058,"ĠاÙĦراب":137059,"ĠاÙĦرابع":137060,"ĠпÑĢек":137061,"ĠпÑĢекÑĢаÑģ":137062,"ĠпÑĢекÑĢаÑģн":137063,"ĠenergÃŃa":137064,"×§×ĵ×ŀ×Ļ":137065,"ãģıãģªãģ£ãģŁ":137066,"ĠÄijứ":137067,"ĠÄijứa":137068,"Servi":137069,"Serviço":137070,"Ġkaldır":137071,"åĥįãģį":137072,"Ġодеж":137073,"Ġодежд":137074,"물ìĿĦ":137075,"ãģĿãģĨãģ§":137076,"ãģĮãģĤãĤĮãģ°":137077,"ìĻķ":137078,"צ×ĵ×§":137079,"Ġartır":137080,"Ġileti":137081,"ĠiletiÅŁim":137082,"ãĤĪãģĨãģ§":137083,"ãĥĪãĥ¼":137084,"ãĤ¢ãĥĭ":137085,"ãĤ¢ãĥĭãĥ¡":137086,"×ĺ×Ļ×Ļ׾":137087,"ãĥķãĥªãĥ¼":137088,"ãĥĿãĥ³":137089,"ÐŁÑĢо":137090,"ĠعاÙĦÙĬØ©":137091,"ĠÃ¶ÄŁret":137092,"ĠÃ¶ÄŁretmen":137093,"ĠкаÑĩеÑģÑĤва":137094,"Ġ×Ķ×ĺ×ij×¢":137095,"ĠзнаÑİ":137096,"ãģ¦ãģıãĤĭ":137097,"Ġmừng":137098,"ÙħÙĪØª":137099,"ש×ķ×ŀר":137100,"×Ĺ׾×ij":137101,"ĠwzglÄĻ":137102,"ĠwzglÄĻdu":137103,"ë²Ī째":137104,"Ġtá»ĵ":137105,"Ġtá»ĵn":137106,"ãĥ¯ãĥ¼ãĤ¯":137107,"Ġpożycz":137108,"Ġpożyczk":137109,"×Ļ×ķצר×Ļ×Ŀ":137110,"ÙĥرÙħ":137111,"ĠгаÑĢ":137112,"ĠгаÑĢан":137113,"ĠгаÑĢанÑĤи":137114,"ลà¹īาà¸ĩ":137115,"ĠìĺģíĻĶ":137116,"×ĺ×Ļס":137117,"Ġthẻ":137118,"ĠìŀĪëĭ¤ê³ł":137119,"اÙĦتز":137120,"اÙĦتزاÙħ":137121,"ĠнаÑĪи":137122,"isée":137123,"ãģĵãĤĮãĤĴ":137124,"Ġmẽ":137125,"ضÙĦ":137126,"بÙĪØª":137127,"Ġ׼׼×Ķ":137128,"hợ":137129,"ĠاÙĦسÙĪØ±ÙĬØ©":137130,"Ġ×ľ×¢×ķ×ŀ":137131,"Ġ×ľ×¢×ķ×ŀת":137132,"ĠbaÅŁar":137133,"ĠbaÅŁarılı":137134,"еÑģÑĤÑĮ":137135,"à¸Ħรี":137136,"à¸Ħรีม":137137,"ĠìłĦì²´":137138,"ĠسÙĬÙĥÙĪÙĨ":137139,"Ġ×ŀ×ĵ×ķ×¢":137140,"ĠëķĮ문ìĿ´ëĭ¤":137141,"Ġcứng":137142,"gerät":137143,"ĠмиÑĢ":137144,"ĠмиÑĢе":137145,"ĠÙĥÙĬÙģÙĬØ©":137146,"Ġפר×ĺ×Ļ×Ŀ":137147,"ĠgoÅĽci":137148,"иÑĤеÑģÑĮ":137149,"ÑĥÑĪки":137150,"ؤÙħÙĨ":137151,"Ġ×IJ׼ף":137152,"ĠاÙĦرجÙĦ":137153,"Ġlá»įc":137154,"à¹Ģรียà¸ģวà¹Īา":137155,"ãģĵãģ®ãĤĪãģĨãģª":137156,"ë§Įíģ¼":137157,"ĠпеÑĩ":137158,"ÙĪÙĦات":137159,"ĠÃľye":137160,"liÄŁinde":137161,"à¸Ħะà¹ģà¸Ļ":137162,"à¸Ħะà¹ģà¸Ļà¸Ļ":137163,"ãĤĭãģĵãģ¨ãģ¯":137164,"วิà¹Ģà¸Ħร":137165,"วิà¹Ģà¸Ħราะ":137166,"วิà¹Ģà¸Ħราะหà¹Į":137167,"ĠвозможноÑģÑĤи":137168,"ĠاÙĦÙĨساء":137169,"ãĥīãĥ©ãĥŀ":137170,"Ġgüc":137171,"Ġgücü":137172,"Ġtưá»Ŀng":137173,"Ġacompaña":137174,"ãĤ¤ãĥ©":137175,"קצ×ij":137176,"ĠYö":137177,"ĠYönet":137178,"ĠYönetim":137179,"à¸ªà¸±à¸¡à¸ľ":137180,"à¸ªà¸±à¸¡à¸ľà¸±à¸ª":137181,"à¸Ļาม":137182,"ĠÄijợi":137183,"à¹ģหà¹Īà¸ĩà¸Ĭาà¸ķิ":137184,"ãģĿãĤĮãģ§ãĤĤ":137185,"ätig":137186,"ת×ķ×Ŀ":137187,"ĠbaÅŁlat":137188,"ĠвÑģей":137189,"ת×Ļ×§":137190,"ת×Ļ×§×ķף":137191,"ĠNgô":137192,"ĠGeschä":137193,"ĠGeschäfts":137194,"Ø£Ùħ":137195,"Ø£Ùħراض":137196,"à¹Ģà¸Ĺà¸Ħà¸Ļ":137197,"à¹Ģà¸Ĺà¸Ħà¸Ļิ":137198,"à¹Ģà¸Ĺà¸Ħà¸Ļิà¸Ħ":137199,"ĠменÑĮ":137200,"ĠменÑĮÑĪе":137201,"Ġölç":137202,"Ġölçü":137203,"ĠÙĬجعÙĦ":137204,"ĠÄijỡ":137205,"ש×Ļ׾":137206,"ש×Ļ׾×ķ×ij":137207,"ĠGrÃ¶ÃŁe":137208,"ĠÙĩاتÙģ":137209,"รà¹īาà¸Ļà¸Ńาหาร":137210,"×Ķ׾×Ļ׼":137211,"×Ķ׾×Ļ׼×Ļ":137212,"иÑĢÑĥÑİÑī":137213,"èĭ¥ãģĦ":137214,"ĠÃĸzel":137215,"ãģĦãģŁãĤī":137216,"à¸Ħำà¸ĸาม":137217,"ĠzostaÅĤy":137218,"Ġ×Ķס×Ļפ×ķר":137219,"×Ķ×ķ׾":137220,"×Ķ×ķ׾×ļ":137221,"à¹Ģà¸Ĭà¹Īà¸Ļà¸ģัà¸Ļ":137222,"à¹Ĥà¸Ĩ":137223,"à¹Ĥà¸Ĩษ":137224,"à¹Ĥà¸Ĩษà¸ĵา":137225,"×IJרצ×ķת":137226,"×Ĵרפ×Ļ":137227,"Ġaoût":137228,"ĠÙĬرÙĬد":137229,"تÙĪØ¬":137230,"تÙĪØ¬ÙĬÙĩ":137231,"ĠÑįÑĤап":137232,"ãĤ¹ãĤ¿ãĥ³":137233,"Ġkró":137234,"Ġkrótk":137235,"ãĤĴ使ãģĨ":137236,"ì·¨":137237,"éĸ¢ãĤı":137238,"à¸Ķà¹īวยà¸Ħวาม":137239,"à¸Ļำà¹Ģสà¸Ļà¸Ń":137240,"Ġayrıca":137241,"à¸Īà¹īาà¸ĩ":137242,"ĠÑĦоÑĤогÑĢаÑĦ":137243,"ĠвеÑĩ":137244,"ĠвеÑĩеÑĢ":137245,"åĩºãģĹãģŁ":137246,"ĠХо":137247,"Ġ×ŀר×Ĵ×Ļש":137248,"à¹ĥหà¹īà¹Ģà¸Ľà¹ĩà¸Ļ":137249,"ãĤĴ缮":137250,"ãĤĴ缮æĮĩ":137251,"׾×ŀ×Ļ×Ŀ":137252,"nÄħÅĤ":137253,"ĠÑģÑĤанд":137254,"ĠÑģÑĤандаÑĢÑĤ":137255,"ĠSüd":137256,"ĠTâm":137257,"اختبار":137258,"à¹Ģà¸ģà¸Ńรà¹Į":137259,"ÙħسرØŃ":137260,"Ġbiá»ĩn":137261,"بÙı":137262,"ĠصاÙĦ":137263,"ĠصاÙĦØŃ":137264,"ĠPhụ":137265,"íľ´":137266,"ãĥ¬ãĥĵãĥ¥ãĥ¼":137267,"Ġbụng":137268,"Ġrégime":137269,"ĠأشÙĩر":137270,"ĠÑĢабоÑĤник":137271,"à¸Ŀัà¸Ļ":137272,"اعتÙħ":137273,"اعتÙħاد":137274,"ĠзамеÑĤ":137275,"ãģ¾ãģ£ãģ¦":137276,"Ġchặt":137277,"æĿ¥ãĤĭ":137278,"ĠاÙĦÙĤÙĪØ§Øª":137279,"ãģ«åħ¥ãģ£ãģ¦":137280,"تØŃاÙĦÙģ":137281,"ÙħزÙĬد":137282,"ĠÙĬصÙĦ":137283,"ìĹ¼":137284,"à¹Ģà¸Ĭà¹ĩ":137285,"à¹Ģà¸Ĭà¹ĩà¸Ħ":137286,"Ġká»ĭ":137287,"Ġká»ĭp":137288,"ĠìķĦì§ģ":137289,"×IJ׳×Ĵ":137290,"ĠоблаÑģÑĤÑĮ":137291,"ĠpomocÄħ":137292,"Ġ×ķש׾":137293,"ëĵłì§Ģ":137294,"ĠGiám":137295,"ĠStück":137296,"Ġcháy":137297,"ĠëĤĺìĺ¤":137298,"ש×Ļ×ĺת":137299,"×ŀ×ĵר":137300,"×ŀ×ĵר×Ļ×ļ":137301,"Ġsüreç":137302,"ква":137303,"×ij׾×Ļ×Ŀ":137304,"×Ķת×Ļ":137305,"×Ķת×Ļ×Ļ×Ĺס":137306,"ÙĤباÙĦ":137307,"Ġס×ķ×Ĵ":137308,"Ġס×ķ×Ĵ×Ļ":137309,"ÑģÑĤолÑĮ":137310,"ä½ķãĤĤ":137311,"×ĸ׼×ķר":137312,"è²·ãģĨ":137313,"å®īãģı":137314,"à¸Ħรัà¹īà¸ĩà¸Ļีà¹ī":137315,"köp":137316,"ĠÑģеÑĢвиÑģ":137317,"оÑĩнÑĭÑħ":137318,"ê±°ëŀĺ":137319,"تأÙĥ":137320,"تأÙĥÙĬد":137321,"×ĵ׾ק":137322,"ĠпоÑĩем":137323,"ĠпоÑĩемÑĥ":137324,"пиÑģаÑĤÑĮ":137325,"×ijשר":137326,"ĠHÃłng":137327,"ĠTìm":137328,"Ġtrừ":137329,"ãĤ»ãĥĥãĤ¯ãĤ¹":137330,"×ķ׳×Ĵ":137331,"mızda":137332,"пÑģи":137333,"ĠìŀĪ기":137334,"Ġrút":137335,"زاÙĨ":137336,"تÙĨÙĪØ¹":137337,"ÙħÙĤا":137338,"ÙħÙĤاÙĪÙħØ©":137339,"Ġ׾צ×ķר×ļ":137340,"Ġ×ij×Ļר×ķש׾×Ļ×Ŀ":137341,"ãĥ´ãĤ£":137342,"ebile":137343,"ebileceÄŁi":137344,"ãĥ¦ãĥ¼ãĤ":137345,"ãĥ¦ãĥ¼ãĤ¶":137346,"ãĥ¦ãĥ¼ãĤ¶ãĥ¼":137347,"ãĤĴä½ľãĤĭ":137348,"ÑģмеÑĢ":137349,"ÑģмеÑĢÑĤ":137350,"Ġì§ģ":137351,"Ġì§ģìłij":137352,"ĠÐŁÐ°ÑĢ":137353,"ØŃاض":137354,"ØŃاضر":137355,"ÙħÙĥاÙģ":137356,"ÙħÙĥاÙģØŃØ©":137357,"ลิà¸Ļ":137358,"ãģ¦ãģįãģ¦":137359,"ÑĢоÑģл":137360,"ĠÄ°ÅŁte":137361,"ÙĤصÙĬر":137362,"Ġ×ij×Ĵ×Ļ׾":137363,"Ġ×ŀת×IJ×Ļ×Ŀ":137364,"Ġ×Ķ×Ĺ×ĵ":137365,"Ġ×Ķ×Ĺ×ĵש×Ķ":137366,"ר×ķ×¢":137367,"Ġproduktów":137368,"ĠÙħصدر":137369,"неÑĨ":137370,"ĠاÙĦعÙħÙĦات":137371,"Ġçıkma":137372,"ĠدبÙĬ":137373,"×§×Ļף":137374,"ת×IJר":137375,"ת×IJר×Ļ×ļ":137376,"׳×Ļ×Ļ×ĵ":137377,"صراع":137378,"lève":137379,"צ×Ļר":137380,"à¸Ķัà¸Ļ":137381,"à¹ĥหà¹īà¹Ħà¸Ķà¹ī":137382,"ãĤ¿ãĤ¤ãĥł":137383,"Ġgiảng":137384,"Ð¡ÐŁ":137385,"ĠاÙĦÙħØŃÙĦ":137386,"ĠاÙĦÙħØŃÙĦÙĬØ©":137387,"ĠTất":137388,"׾×ķ×ĺ":137389,"há»ķ":137390,"Ġaméric":137391,"Ġaméricain":137392,"Ġ×ijש׾×ij":137393,"Ġ׾×IJ×ķ×ŀ×Ļ":137394,"Ġpeça":137395,"ĠÑĢазнÑĭÑħ":137396,"ãģĦãĤĭãģ¨":137397,"ãĥĩãĥ³":137398,"סקר":137399,"Ġ×Ķ×ŀ×Ĺ×Ļר":137400,"ãģ¨ãģĦãģĨãĤĤãģ®":137401,"رتبط":137402,"ĠиÑģÑĤоÑĩ":137403,"ĠиÑģÑĤоÑĩник":137404,"สมัà¸Ħรสมาà¸Ĭิà¸ģ":137405,"Ġà¸Ĺัà¹īà¸ĩ":137406,"Ġà¸Ĺัà¹īà¸ĩà¸Ļีà¹ī":137407,"ĠTáºŃp":137408,"ãģ£ãģ¦ãģĦãģĨ":137409,"ĠاÙĦÙĪØµÙĪÙĦ":137410,"Ġdécada":137411,"ĠоÑĦоÑĢм":137412,"ĠоÑĦоÑĢмлен":137413,"สำหรัà¸ļà¸ģาร":137414,"Ġogóln":137415,"ãģĨãģ¡ãģ«":137416,"Ġvárias":137417,"ãģĻãģİãĤĭ":137418,"ÙĪÙĩا":137419,"à¹Ĥà¸Ľà¸£à¸Ķ":137420,"ĠÐłÐ¾ÑģÑģиÑı":137421,"人ãĢħ":137422,"ãģĹãģ¦ãģįãģŁ":137423,"Ġsırasında":137424,"Ġngôn":137425,"سÙĨØ©":137426,"تÙħتع":137427,"×ŀ׼×ij×Ļ":137428,"Ġnhấn":137429,"×¢×ŀ×Ļ×ĵ":137430,"Ứ":137431,"жиÑĤÑĮ":137432,"ãĤīãģĽ":137433,"gráf":137434,"gráfica":137435,"ĠÙĤÙĪÙĦ":137436,"ĠÙĤÙĪÙĦÙĩ":137437,"ëĭ¨ì²´":137438,"หà¹īา":137439,"หà¹īาม":137440,"使ãģ£ãģ¦":137441,"ת×Ļ×ij":137442,"ת×Ļ×ijת":137443,"iá»ĥu":137444,"à¹ģà¸Ĭม":137445,"à¹ģà¸Ĭà¸¡à¸Ľ":137446,"à¹ģà¸Ĭà¸¡à¸Ľà¹Į":137447,"Ậ":137448,"ĠëĤĺëĿ¼":137449,"ĠÙħباشرة":137450,"ĠtrÄĥm":137451,"سÙĥÙĪ":137452,"ĠاÙĦذÙī":137453,"Ġbiç":137454,"Ġbiçim":137455,"تراجع":137456,"ĠобеÑģп":137457,"ĠобеÑģпеÑĩ":137458,"ĠобеÑģпеÑĩива":137459,"ĠвоздÑĥÑħ":137460,"ÑĭваÑĤÑĮ":137461,"ÙĦØŃÙĤ":137462,"ĠMüdü":137463,"ĠMüdürl":137464,"ĠMüdürlÃ¼ÄŁÃ¼":137465,"Ġyaptır":137466,"Ġפרס":137467,"Ġפרס×ķ×Ŀ":137468,"Ø·ÙĪØ±":137469,"ÑģÑĤвоваÑĤÑĮ":137470,"ìŀ¥ìĿĦ":137471,"à¸Ĺีà¹Īà¸Ķีà¸Ĺีà¹Īสุà¸Ķ":137472,"à¸Ńัล":137473,"ÑĢÑİ":137474,"ÙħستÙĤبÙĦ":137475,"ÑģлÑĥÑĪ":137476,"ÑģлÑĥÑĪа":137477,"èªįãĤģ":137478,"Ġ׾×Ļ×ŀ":137479,"Ġ׾×Ļ×ŀ×ķ×ĵ×Ļ":137480,"תש×ķ×ij":137481,"תש×ķ×ij×ķת":137482,"ĠgerçekleÅŁtiril":137483,"ĠاÙĦاتÙ쨧ÙĤ":137484,"ĠÑĥÑĢовне":137485,"ĠÑĤÑĢав":137486,"Ġ×Ķ×ŀ×ķף":137487,"ØŃÙģØ§Ø¸":137488,"ĠÙħÙIJ":137489,"ĠÙħÙIJÙĨ":137490,"ĠÙħÙIJÙĨÙĴ":137491,"Ġdemás":137492,"×ŀ×ķ×ĸ×Ļ×§×Ķ":137493,"ש×Ļ×Ĺ×Ķ":137494,"Ġbú":137495,"алÑĮнÑĭм":137496,"ãĤıãģŁ":137497,"ãĤıãģŁãģĹ":137498,"ĠاÙĦÙħÙĪØ§Ø¯":137499,"×ª×Ľ×ł":137500,"×ª×Ľ×ł×ķף":137501,"ãĥŃãĥĥãĤ¯":137502,"hiếu":137503,"ĠÑĥме":137504,"ÙħØŃاÙĪÙĦØ©":137505,"×IJ×ķשר":137506,"ĠконкÑĥÑĢ":137507,"ĠконкÑĥÑĢÑģ":137508,"Ġ×ŀ×ij×Ĺ":137509,"Ġ×ŀ×ij×Ĺ×Ļ×ł×ª":137510,"Ġanlam":137511,"Ġanlamı":137512,"Ġliá»ĩt":137513,"ĠвÑħод":137514,"ĠHình":137515,"ĠÙĨÙĬ":137516,"ĠÙĨÙĬÙĪØ²":137517,"ãĤ¸ãĥ£ãĥ¼":137518,"×ij×Ļ×¥":137519,"ÑĤелÑĮнÑĭÑħ":137520,"à¸Ĺุà¸ģà¸Ńยà¹Īาà¸ĩ":137521,"ĠkiÅŁinin":137522,"Ø£Ùĥثر":137523,"ĠиÑģÑĤоÑĢии":137524,"Ġë³ĢíĻĶ":137525,"×¤×ľ×¡×ĺ":137526,"×¤×ľ×¡×ĺ×Ļ׳×Ļ":137527,"ĠÑģеÑĤ":137528,"ĠÑģеÑĤи":137529,"dıģımız":137530,"íķĺëıĦë¡Ŀ":137531,"×Ķר":137532,"×Ķר×ij×Ķ":137533,"ãģĻãĤĭãģĵãģ¨ãģ¯":137534,"Ġphiếu":137535,"تØŃسÙĬÙĨ":137536,"ĠÅĽrod":137537,"ĠÅĽrodow":137538,"ĠÅĽrodowisk":137539,"ĠÑĢаÑģÑħод":137540,"برÙĬد":137541,"ĠرÙĬ":137542,"ĠرÙĬاÙĦ":137543,"Ġ×ķ׼×ļ":137544,"ì§ĢìļĶ":137545,"׼×ŀ×ķ":137546,"Ġ×¢×ľ×Ļ×Ķ×Ŀ":137547,"fÃŃcio":137548,"Ġkararı":137549,"tıģını":137550,"ĠСов":137551,"ĠСовеÑĤ":137552,"ãģĬéĩijãĤĴ":137553,"междÑĥ":137554,"междÑĥна":137555,"междÑĥнаÑĢод":137556,"междÑĥнаÑĢодн":137557,"Ġmá»Ŀi":137558,"ĠاÙĦØ¥ÙĬر":137559,"ĠاÙĦØ¥ÙĬراÙĨÙĬ":137560,"ĠاÙĦرÙĪØ³ÙĬ":137561,"صÙĨد":137562,"صÙĨدÙĪÙĤ":137563,"ĠاÙĦØ¥ÙĨترÙĨت":137564,"Ġtắm":137565,"ĠÑĤакого":137566,"Ġ×ij׾×ķ×Ĵ":137567,"Ġücrets":137568,"Ġücretsiz":137569,"×Ĺ×ĸ×Ļר":137570,"ìĸ´ìķ¼":137571,"ĠPhần":137572,"ï¼ľ":137573,"Ġ×ĺ×ij×¢":137574,"Ġ×ĺ×ij×¢×Ļ":137575,"×IJ×ŀ×IJ":137576,"اÙĤÙĦ":137577,"Ġcondições":137578,"ÙĤاتÙĦ":137579,"ĠÑĢезÑĥлÑĮÑĤаÑĤе":137580,"ĠÑģвоими":137581,"צ×ij×Ļ×¢":137582,"géni":137583,"Ġzes":137584,"Ġzespo":137585,"ĠzespoÅĤ":137586,"ÑĪив":137587,"Ġפר×ĺ×Ļ×ķת":137588,"ÙħستشÙģ":137589,"ÙħستشÙģÙī":137590,"شرع":137591,"ĠkoÅĽci":137592,"Ġ×Ķ×IJ×Ļ׳×ĺר׳×ĺ":137593,"ĠЧеÑĢ":137594,"поÑĩÑĤ":137595,"Ġactivités":137596,"çŁ¥ãģ£ãģ¦":137597,"Ġ×ij×ĸ×Ķ":137598,"Ġyüzden":137599,"ãģªãĤĬãģ¾ãģĽãĤĵ":137600,"Ġíĺ¹":137601,"Ġíĺ¹ìĿĢ":137602,"Ġ×ŀש׳×Ķ":137603,"ĠÐĴеÑĢ":137604,"Ġ×ij×IJ×ķת×ķ":137605,"éĿ¢çϽ":137606,"éĿ¢çϽãģĦ":137607,"شرØŃ":137608,"gründe":137609,"Ù쨴":137610,"Ù쨴ÙĦ":137611,"Ġséjour":137612,"ë´IJ":137613,"Ġrôle":137614,"شعار":137615,"емÑĭе":137616,"ĠاÙĦجسÙħ":137617,"алÑĮное":137618,"Ġìĥģíĥľ":137619,"D":137620,"ë¯Ģë¡ľ":137621,"ĠÙĨÙĤØ·":137622,"ĠÙĨÙĤطة":137623,"ãģĿãģĨãģł":137624,"ãģĻãĤĭãģ®ãģĮ":137625,"หู":137626,"Ġnhá»ĭ":137627,"Ġeconómica":137628,"ס×ĺ×ķ×ĵ":137629,"ס×ĺ×ķ×ĵ׳×ĺ":137630,"มีà¹Ĥà¸Ńà¸ģาส":137631,"Ġgestão":137632,"รูà¹īวà¹Īา":137633,"Ġloạt":137634,"ĠاÙĦÙħÙı":137635,"ĠاÙĦØŃÙħÙĦ":137636,"ĠاÙĦعÙħÙĦÙĬØ©":137637,"Ġê²ĥëıĦ":137638,"ĠÐľÐ¾Ñģква":137639,"×§×ĺ×ķר":137640,"ĠподÑĢоб":137641,"ĠподÑĢобн":137642,"Ġlưng":137643,"تÙ쨳":137644,"تÙ쨳ÙĬر":137645,"ĠاÙĦبع":137646,"ĠاÙĦبعض":137647,"ئت":137648,"ÐķÐĿ":137649,"ìĹ°êµ¬":137650,"à¹ĥหà¹īà¸Ħุà¸ĵ":137651,"ãģĤãĤĬãģ¾ãģĹãģŁ":137652,"Ġbirka":137653,"Ġbirkaç":137654,"Ġİsl":137655,"Ġİslam":137656,"çĹĽãģ¿":137657,"Ġhảo":137658,"ĠмаÑı":137659,"ĠiÅŁÃ§i":137660,"ש×":137661,"ש×ģ":137662,"à¸ģารà¹Ģมืà¸Ńà¸ĩ":137663,"×ķ×Ķר":137664,"Ġchó":137665,"ëĨĢ":137666,"Ġyanlı":137667,"ĠyanlÄ±ÅŁ":137668,"幸ãģĽ":137669,"×IJר×Ĵ×ķ׳×Ļ":137670,"à¸Ńาà¸Īาร":137671,"à¸Ńาà¸Īารยà¹Į":137672,"ĠинÑĦоÑĢмаÑĨиÑİ":137673,"ÐĵÐŀ":137674,"׳×Ĺש":137675,"ĠìķĮìķĦ":137676,"ĠÑħаÑĢакÑĤеÑĢиÑģÑĤ":137677,"ĠÑħаÑĢакÑĤеÑĢиÑģÑĤик":137678,"à¸Ħุà¸ĵสามารà¸ĸ":137679,"è¦ĭãģĪãĤĭ":137680,"à¸Ĭัà¸Ķà¹Ģà¸Ī":137681,"à¸Ĭัà¸Ķà¹Ģà¸Īà¸Ļ":137682,"ĠdziaÅĤal":137683,"ĠdziaÅĤalnoÅĽci":137684,"à¹Ĥà¸ŀสà¸ķà¹Į":137685,"ĠÐļол":137686,"ĠÙģÙĩÙĬ":137687,"Ġ×ŀפ׳×Ļ":137688,"Ġ×Ķקשר":137689,"ÙħرÙĥ":137690,"ÙħرÙĥز":137691,"Ġhoá":137692,"Ġапп":137693,"ĠаппаÑĢаÑĤ":137694,"Ġpami":137695,"ĠpamiÄĻ":137696,"ĠpamiÄĻta":137697,"Ġçünkü":137698,"×ĵ×ķף":137699,"ãģ¯ãģĵãģ¡ãĤī":137700,"ĠMÃł":137701,"ĠÙĬÙĤدÙħ":137702,"ĠпÑĢез":137703,"ĠпÑĢезиденÑĤ":137704,"à¸Ńุà¸ķ":137705,"à¸Ńุà¸ķสา":137706,"à¸Ńุà¸ķสาห":137707,"à¸Ńุà¸ķสาหà¸ģรรม":137708,"ì§ĢìĽIJ":137709,"Ġ×IJפשר×ķת":137710,"schüt":137711,"schütz":137712,"ĠTiên":137713,"Ġsayılı":137714,"ĠгÑĢÑĥппÑĭ":137715,"оÑĩнÑĭй":137716,"Ġ×ľ×¢×ŀ×ķ×ĵ":137717,"ĠwrzeÅĽ":137718,"ĠwrzeÅĽnia":137719,"ĠÄIJầu":137720,"à¹Ģà¸Ĥà¹īารà¹Īวม":137721,"nızda":137722,"Ø®ÙĬص":137723,"Ġgünc":137724,"Ġgüncel":137725,"ĠÙĦÙĩذÙĩ":137726,"ĠÙĬعتبر":137727,"légi":137728,"ãĤıãģĭãĤĭ":137729,"Ġrừng":137730,"ظÙĩ":137731,"ظÙĩÙĪØ±":137732,"Ġ×ŀ×ij×Ļף":137733,"Ġ기íĥĢ":137734,"åĪĩãĤĮ":137735,"lanmÄ±ÅŁ":137736,"à¸Ĺีà¹Īมีà¸Ħวาม":137737,"Ġhá»ģ":137738,"تÙĪØ¬Ùĩ":137739,"ĠاÙĦإدارة":137740,"Ġútil":137741,"ספ×ķ":137742,"à¸Ħวามรัà¸ģ":137743,"à¹Ĥฮ":137744,"ĠполиÑĤ":137745,"ĠполиÑĤик":137746,"Ġsatın":137747,"ĠÅŀimdi":137748,"×ŀ×ķר×Ļ×Ŀ":137749,"ìķĺëĭ¤":137750,"×Ĺ×ķ×ķ":137751,"×Ĺ×ķ×ķ×Ļ×Ķ":137752,"à¸Ħà¸Ńมà¸ŀิ":137753,"à¸Ħà¸Ńมà¸ŀิว":137754,"à¸Ħà¸Ńมà¸ŀิวà¹Ģà¸ķà¸Ńรà¹Į":137755,"Ġاذا":137756,"تخاذ":137757,"ãĤ¨ãĥ«":137758,"Ġpossibilité":137759,"ยืà¸Ļยัà¸Ļ":137760,"Ġünivers":137761,"Ġüniversite":137762,"ĠاÙĦدÙĪØ±ÙĬ":137763,"ĠìķĬëĬĶëĭ¤":137764,"ĠìĦľë¡ľ":137765,"ØŃاÙĦ":137766,"Ġë¨":137767,"Ġ먼":137768,"Ġ먼ìłĢ":137769,"à¸Ĺีà¹Īà¸ĸูà¸ģ":137770,"ì§ľ":137771,"Ġskóry":137772,"лÑĮÑĨ":137773,"à¹ĥà¸Ĭà¹īà¹Ģวลา":137774,"×ijקשת":137775,"ĠذÙĪ":137776,"æĹ¥ãĢħ":137777,"ĠкоÑĤоÑĢÑĥÑİ":137778,"ĠÑĥÑĢовенÑĮ":137779,"깨":137780,"à¹Ħà¸Ĺ":137781,"ãĤµãĥĹãĥª":137782,"ãĤ¸ãĥ§ãĥ³":137783,"ãģĻãģ¹ãģį":137784,"ĠGór":137785,"ãĥĪãĤ¤":137786,"ãĥĪãĤ¤ãĥ¬":137787,"ĠyaÅŁama":137788,"Ġdá»ĭp":137789,"Ġbữa":137790,"à¸ĭุ":137791,"Ġölüm":137792,"ãģ£ãģ¦ãģıãĤĭ":137793,"à¸ģารà¸Ħà¹īา":137794,"שער":137795,"ĠÑĤипа":137796,"ĠгеÑĢ":137797,"ĠгеÑĢо":137798,"רקע":137799,"Ġuważ":137800,"Ġuważa":137801,"ש×ŀף":137802,"Ġhastalık":137803,"ãĤıãĤĮãĤĭ":137804,"baÅŁÄ±":137805,"ÑĩÑĤо":137806,"Ġ×ij×ŀר׼×ĸ":137807,"Ġìļ°ë¦¬ìĿĺ":137808,"ĠÙĥاÙĨÙĪØ§":137809,"Ġأبر":137810,"ĠأبرÙĬÙĦ":137811,"층":137812,"à¹Ħà¸Ĥà¹Ī":137813,"ĠÙĪÙĦÙĪ":137814,"à¸Ĺัว":137815,"à¸Ĺัวรà¹Į":137816,"ĠÙĪØ£Ùĥد":137817,"à¸Ĭวà¸Ļ":137818,"׾×ķ×§":137819,"æį¨":137820,"æį¨ãģ¦":137821,"Ġİçin":137822,"péri":137823,"Ġyal":137824,"Ġyalnız":137825,"ÑĮÑıн":137826,"Ġgắng":137827,"à¸ģà¹ĩยัà¸ĩ":137828,"ĠУкÑĢаин":137829,"ĠÑģами":137830,"ĠпÑĢоведен":137831,"à¸ķà¸ģà¹ģà¸ķà¹Īà¸ĩ":137832,"ĠQuân":137833,"éparation":137834,"ĠbaÅŁÄ±nda":137835,"Ġznale":137836,"Ġznaleź":137837,"ĠznaleźÄĩ":137838,"ãĤ±ãĥ¼":137839,"ãĥİãĥ¼":137840,"à¸ĸูà¸ģà¸ķà¹īà¸Ńà¸ĩ":137841,"몸":137842,"ĠëıĮ":137843,"ĠëıĮìķĦ":137844,"ĠSchüler":137845,"ĠподгоÑĤов":137846,"ĠподгоÑĤовк":137847,"عرÙĪ":137848,"عرÙĪØ¶":137849,"laÅŁtır":137850,"ĠÑģоÑģÑĤавлÑıеÑĤ":137851,"ĠпÑĢоизвод":137852,"ĠпÑĢоизводÑģÑĤва":137853,"ĠоÑģнове":137854,"ĠØ´ÙħاÙĦ":137855,"à¸ģรี":137856,"ĠgörÃ¼ÅŁme":137857,"оÑĩек":137858,"Ġ×Ĺ×ijר×Ļ×Ŀ":137859,"Ùħخاط":137860,"Ùħخاطر":137861,"ï¼Ń":137862,"רפ×IJ":137863,"ĠMẹ":137864,"ยà¸Ńมรัà¸ļ":137865,"Ġvết":137866,"خذ":137867,"ĠاÙĦتط":137868,"ĠاÙĦتطبÙĬÙĤ":137869,"à¸Ļึà¸ģ":137870,"Ġ×Ķ×Ľ×ł×¡×ª":137871,"ĠогÑĢани":137872,"ĠогÑĢаниÑĩен":137873,"ĠÃĩalÄ±ÅŁ":137874,"ĠاÙĦÙħÙĨتدÙī":137875,"à¸Īำà¸Ļวà¸Ļมาà¸ģ":137876,"ĠÑĤоÑĢÑĢ":137877,"ĠÑĤоÑĢÑĢенÑĤ":137878,"ĠìĤ´ìķĦ":137879,"à¸ŀลัà¸ĩà¸ĩาà¸Ļ":137880,"à¸Ĭัà¸Ļ":137881,"ĠÐIJндÑĢ":137882,"Ġréalisé":137883,"×ŀש×IJ":137884,"à¹ģà¸Ĭ":137885,"à¹ģà¸Ĭรà¹Į":137886,"Ġбог":137887,"มาà¹ģลà¹īว":137888,"ĠاÙĦÙĨار":137889,"Ġolmadıģı":137890,"×ĵ×¢×Ķ":137891,"ĠÑĥвеÑĢ":137892,"ĠÑĥвеÑĢен":137893,"ãĤĭãĤĤãģ®":137894,"أد":137895,"أدÙĪØ§Øª":137896,"Ġ×Ķ×ĸ×ķ×Ĵ":137897,"إعÙĦاÙħ":137898,"há»ı":137899,"ĠNähe":137900,"ĠÑĤеÑģÑĤ":137901,"Ġ×ŀ×ķ׼ר":137902,"Ġë¬¸ìłľê°Ģ":137903,"ת×ķצ×IJ×Ķ":137904,"mó":137905,"móvel":137906,"ĠاÙĦتجارة":137907,"ĠмногиÑħ":137908,"обÑīа":137909,"Ġעסק×Ļ":137910,"ĠEducação":137911,"קש×Ļ×Ŀ":137912,"établ":137913,"établissement":137914,"Ġделе":137915,"иÑĢÑĥеÑĤÑģÑı":137916,"آثار":137917,"Ġ×Ķ×ŀר׼×ĸ×Ļ":137918,"ãĥIJãĥ«":137919,"ĠвÑģÑĤÑĢеÑĩ":137920,"ãģĴãĤĭ":137921,"ĠciÄħ":137922,"ĠciÄħgu":137923,"ÙĬست":137924,"à¸łà¸²à¸§":137925,"à¸łà¸²à¸§à¸°":137926,"Ø£Ùħر":137927,"Ġожи":137928,"Ġожида":137929,"Ġá»§y":137930,"ãĥŀãĥ«":137931,"راس":137932,"оÑĩной":137933,"ת×Ĵ×ķ×ij×ķת":137934,"تعرÙĬÙģ":137935,"ĠÑģоÑĨиалÑĮно":137936,"ãĤĴéĸĭ":137937,"ĠиÑģÑģледова":137938,"Ġdú":137939,"Ġdúvida":137940,"ĠskÅĤ":137941,"ĠskÅĤada":137942,"Ġhäufig":137943,"ĠвÑĭбÑĢ":137944,"ĠвÑĭбÑĢаÑĤÑĮ":137945,"ãģ®ãģ§ãģ¯ãģªãģĦãģĭ":137946,"ĠÑģилÑĮно":137947,"ÑĤвеÑĢжден":137948,"רפ":137949,"רפ×ķ×IJ×Ķ":137950,"æĢĿãģĦãģ¾ãģĻ":137951,"ØŃرص":137952,"ש×ķתף":137953,"Ùħسجد":137954,"à¹Ĥà¸Ĭวà¹Į":137955,"емÑģÑı":137956,"вÑĪие":137957,"Ġмл":137958,"Ġмлн":137959,"Ġ׾×Ķ×ij×Ļ×IJ":137960,"ĠÙĬتعÙĦÙĤ":137961,"à¸ķูà¹ī":137962,"ĠпÑĢаз":137963,"ĠпÑĢазд":137964,"ĠпÑĢаздник":137965,"Ġнем":137966,"Ġнемного":137967,"ĠsÃłng":137968,"تÙĨسÙĬ":137969,"تÙĨسÙĬÙĤ":137970,"Ġtá»Ŀ":137971,"Ġмеди":137972,"ã쫿Ī":137973,"ã쫿λ":137974,"à¸Ħวà¹īา":137975,"ãģĭãģijãĤĭ":137976,"×ij׾×ķת":137977,"ĠÑįкÑģп":137978,"ĠÑįкÑģпеÑĢÑĤ":137979,"ĠдевÑĥÑĪ":137980,"ĠдевÑĥÑĪк":137981,"ĠØŃص":137982,"ÙĨشأ":137983,"ãģĮãģĤãĤĭãģ®ãģ§":137984,"ĠتراÙħ":137985,"ĠتراÙħب":137986,"أسÙĪØ§ÙĤ":137987,"Ġ׾פ׳×ķת":137988,"Ġاﻷ":137989,"ãģ«ãģı":137990,"ãģ«ãģıãģĦ":137991,"ĠأعÙĦÙī":137992,"Ġ׾×Ķ×ŀש×Ļ×ļ":137993,"räu":137994,"ש×ŀ×Ļ×Ŀ":137995,"åĪĨãģij":137996,"ãģĻãģ§":137997,"ãģĻãģ§ãģ«":137998,"×Ķ׾׼×Ķ":137999,"×Ĺ׾×Ļ×£":138000,"Ġì±ħ":138001,"Ġì±ħìŀĦ":138002,"à¹Ģà¸Īริ":138003,"à¹Ģà¸Īริà¸į":138004,"éģĬãģ³":138005,"جسد":138006,"สาà¸ĺ":138007,"สาà¸ĺาร":138008,"สาà¸ĺารà¸ĵ":138009,"Ġbasın":138010,"ÑĢаг":138011,"гад":138012,"ĠhoÅŁ":138013,"íķµ":138014,"×ij×Ĺ×Ļר×Ķ":138015,"×ŀס×ļ":138016,"ĠìłľíĴĪ":138017,"تÙħÙĪÙĬÙĦ":138018,"ĠLưu":138019,"ë¡ľë¶ĢíĦ°":138020,"Ġпоб":138021,"Ġпобед":138022,"ÙħÙĨذ":138023,"常ãģ«":138024,"ÙĤس":138025,"ĠاÙĦÙħصدر":138026,"ĠÙĪØ§ÙĦاست":138027,"Ġkhắp":138028,"ĠاÙĦجاÙĨب":138029,"Ġnguyá»ĩn":138030,"éĸĵéģķãģĦ":138031,"ĠÑģÑĤÑĢа":138032,"ĠÑģÑĤÑĢаÑħ":138033,"ĠÑģÑĤÑĢаÑħов":138034,"รีà¸ļ":138035,"Ġxương":138036,"Ġì°¾":138037,"Ġì°¾ìķĦ":138038,"Ġngại":138039,"гал":138040,"à¸ĭีà¹Ī":138041,"Ġ×ijפ×Ļ×Ļס×ij×ķ×§":138042,"ЦенÑĤÑĢ":138043,"Ġavaliação":138044,"Ġeconómico":138045,"×ĸף":138046,"ĠÐľÐ°Ðº":138047,"Ġinterés":138048,"à¸ģลิà¹Īà¸Ļ":138049,"ÑģÑĤÑĮÑİ":138050,"ĠÄijương":138051,"å¼·ãģı":138052,"ĠKhách":138053,"à¹Ģà¸Ļืà¹īà¸Ńหา":138054,"ĠYazı":138055,"è²·ãģ£ãģ¦":138056,"ÐłÐķ":138057,"à¹Ģà¸ŀิà¹Īมà¸Ĥึà¹īà¸Ļ":138058,"สมà¸ļู":138059,"สมà¸ļูรà¸ĵà¹Į":138060,"ĠмиÑĢов":138061,"×Ĵ׳×Ļ×Ŀ":138062,"ĠÄijức":138063,"à¸Ńารà¹Į":138064,"صاص":138065,"ãģĬãĤĪ":138066,"ãģĬãĤĪãģ³":138067,"êÌī":138068,"ĠاÙĦÙħؤتÙħر":138069,"ĠاÙĦÙħرØŃÙĦØ©":138070,"สà¸Ńà¸ļà¸ĸาม":138071,"Ġà¸Īาà¸ģà¸Ļัà¹īà¸Ļ":138072,"Ġتعد":138073,"ãģĿãģ®ãģŁãĤģ":138074,"Ġkháng":138075,"à¸Ļิà¸Ķ":138076,"ãĥĬãĥ³":138077,"ëĦ¤ìļĶ":138078,"ĠاÙĦاØŃت":138079,"ĠاÙĦاØŃتÙĦاÙĦ":138080,"ìļķ":138081,"Ġмодели":138082,"ĠпÑĢоÑĨенÑĤ":138083,"à¸ŀวà¸ģà¹Ģรา":138084,"Ġ×Ķצ×ĵ":138085,"Ġ×Ķצ×ĵ×ĵ×Ļ×Ŀ":138086,"stände":138087,"׳×Ĵר":138088,"Ġdotyc":138089,"ĠdotyczÄħ":138090,"ĠdotyczÄħce":138091,"ĠÅĽwiÄĻt":138092,"×ŀר×Ķ":138093,"ãģĻãģĶãģĦ":138094,"ãĥĩãĤ£ãĥ³ãĤ°":138095,"à¸ģารสรà¹īาà¸ĩ":138096,"ëĤ¬":138097,"Ġì°¸ìŬ":138098,"ÑģÑħ":138099,"ÑģÑħем":138100,"ÙħÙĪØ³":138101,"Ġnấu":138102,"Ġ׾×ŀ×¢×ľ×Ķ":138103,"à¹Ģà¸Ľà¹īา":138104,"à¹Ģà¸Ľà¹īาหมาย":138105,"Ġmùi":138106,"ائز":138107,"íĽĪ":138108,"×Ĺ×ij×ķר×Ķ":138109,"à¸ľà¸¹à¹īà¹ĥà¸Ĭà¹ī":138110,"Ġpaź":138111,"Ġpaździ":138112,"Ġpaździern":138113,"Ġpaździernika":138114,"ลà¸ĩà¹Ħà¸Ľ":138115,"ÙĤاع":138116,"ĠcháºŃm":138117,"Ġözellikleri":138118,"ĠÄIJo":138119,"ĠÄIJoÃłn":138120,"жение":138121,"Ġhẳ":138122,"Ġhẳn":138123,"ĠaÅŁk":138124,"ï½į":138125,"ãĥijãĤ¹":138126,"×Ķ×ķר×IJ×ķת":138127,"ĠÅ»":138128,"ĠÅ»y":138129,"×ŀ×ĸ׾":138130,"ĠÑĥкÑĢа":138131,"ĠÑĥкÑĢаин":138132,"à¹Ģà¸Ĭิ":138133,"à¹Ģà¸Ĭิà¸į":138134,"ÐłÐĺ":138135,"ĠzwiÄħzku":138136,"×Ķ×Ĺ׾×ĺת":138137,"ãĤĵãģ§ãģĻãĤĪãģŃ":138138,"ãģ¦ãģĬãĤĬ":138139,"ложиÑĤÑĮ":138140,"×ŀ×ķ׳×Ļ×Ŀ":138141,"ฮิ":138142,"ì°¬":138143,"ĠاÙĦÙħشترÙĥ":138144,"ĠdÃ¼ÅŁÃ¼k":138145,"агенÑĤ":138146,"ĠاÙĦأسبÙĪØ¹":138147,"ĠÙĤرÙĬب":138148,"инд":138149,"индив":138150,"индивид":138151,"индивидÑĥ":138152,"индивидÑĥалÑĮн":138153,"förder":138154,"Ġseçen":138155,"Ġseçenek":138156,"Ġétant":138157,"ĠлÑİбим":138158,"казÑĭваеÑĤ":138159,"วิà¸Ļ":138160,"Ġ×Ķ×ij×IJ×Ļ×Ŀ":138161,"Ġдов":138162,"ĠдоволÑĮ":138163,"ĠдоволÑĮно":138164,"×¢×ĵ×Ļ×£":138165,"Ġokre":138166,"ĠokreÅĽ":138167,"ĠokreÅĽlon":138168,"ĠترÙĬد":138169,"à¹Ģมืà¹Īà¸Ńวัà¸Ļà¸Ĺีà¹Ī":138170,"ãĤĪãģĭãģ£ãģŁ":138171,"Cumh":138172,"Cumhur":138173,"Cumhurba":138174,"CumhurbaÅŁ":138175,"CumhurbaÅŁkan":138176,"CumhurbaÅŁkanı":138177,"Ġnợ":138178,"à¸ľà¸¹à¹īà¹Ģลà¹Īà¸Ļ":138179,"Ġcomplète":138180,"à¹Ģà¸ŀศ":138181,"دÙIJ":138182,"Ġdüz":138183,"Ġdüzey":138184,"ãģ§ãģĤãĤĭãģĵãģ¨":138185,"extérieur":138186,"׳":138187,"Ġinformação":138188,"ãĤ¯ãĥªãĥĭãĥĥãĤ¯":138189,"ĠPubli":138190,"ĠPublié":138191,"ר×ķ×ĵ":138192,"à¸Ħà¸§à¸²à¸¡à¸Ľà¸¥à¸Ńà¸Ķà¸łà¸±à¸¢":138193,"ĠØ£ÙĬض":138194,"ĠØ£ÙĬضÙĭا":138195,"تسبب":138196,"ãģ¤ãĤĤãĤĬ":138197,"изма":138198,"à¸Ĥึà¹īà¸Ļà¹Ħà¸Ľ":138199,"ÙĥÙIJ":138200,"ÙĦÙĪÙħ":138201,"Ġשצר":138202,"Ġשצר×Ļ×ļ":138203,"ãģ¯ãĤĤãģ¡ãĤįãĤĵ":138204,"Ġкан":138205,"Ġканал":138206,"ãģ«ãģªãģ£ãģ¦ãģĦãģ¾ãģĻ":138207,"ĠاÙĦØ£Ùĥثر":138208,"تاØŃ":138209,"ÙĨتÙĩ":138210,"ÙĨتÙĩاء":138211,"اÙĪÙĬØ©":138212,"ĠBugün":138213,"нÑģкого":138214,"à¸Ķà¹Īวà¸Ļ":138215,"évolution":138216,"ãģ£ãģ¦ãģĦãģ¾ãģĹãģŁ":138217,"ãĤħ":138218,"ĠVương":138219,"à¸łà¸²à¸ŀย":138220,"à¸łà¸²à¸ŀยà¸Ļ":138221,"à¸łà¸²à¸ŀยà¸Ļà¸ķรà¹Į":138222,"Ġ×Ķצ׾×Ļ×Ĺ":138223,"ĠاÙĦإسÙĦاÙħÙĬ":138224,"ÙĦÙĬب":138225,"Ġedição":138226,"ÑģÑĤÑĢел":138227,"Ġkhúc":138228,"ÙĨÙħÙĪØ°":138229,"ÙĨÙħÙĪØ°Ø¬":138230,"׾צ×Ķ":138231,"ÑģÑĤавил":138232,"à¸ĸา":138233,"สรà¹īาà¸ĩà¸Ħวาม":138234,"ãģĦãģ£ãģ±":138235,"ãģĦãģ£ãģ±ãģĦ":138236,"ÑģÑĤавлен":138237,"ĠاÙĦÙĤدس":138238,"Ġngược":138239,"بخ":138240,"สหร":138241,"สหรั":138242,"สหรัà¸IJ":138243,"Ġأغ":138244,"Ġأغسط":138245,"Ġأغسطس":138246,"ãģĨãģ¾":138247,"ãģĨãģ¾ãģı":138248,"ĠêµŃìłľ":138249,"ØŃضار":138250,"Ġdừng":138251,"æĬ¼ãģĹ":138252,"تÙĪØ§":138253,"تÙĪØ§Ø¬Ø¯":138254,"ש×ŀ×Ĺ×Ķ":138255,"ãģıãĤĵ":138256,"Ġ×ijעצ":138257,"Ġ×ijעצ×Ŀ":138258,"×ŀ׳×Ļ×ķת":138259,"×ķ×Ļ×ĵ":138260,"×ķ×Ļ×ĵ×IJ×ķ":138261,"à¸Ĭิà¸ĩ":138262,"ĠpracÄĻ":138263,"ĠзаÑĤ":138264,"ĠзаÑĤем":138265,"ĠìŀIJìľł":138266,"Ġì¤Ģ":138267,"Ġì¤Ģë¹Ħ":138268,"ĠbáºŃ":138269,"ĠbáºŃc":138270,"Ġ×Ķ×ŀצ×ij":138271,"ĠÙĤÙĬÙħØ©":138272,"à¹Ģà¸Ńà¹Ģà¸Ĭ":138273,"à¹Ģà¸Ńà¹Ģà¸Ĭีย":138274,"Ġperchè":138275,"ĠاÙĦعسÙĥر":138276,"ĠاÙĦعسÙĥرÙĬØ©":138277,"جÙĬب":138278,"ëŀµ":138279,"ÙħÙĩر":138280,"ÙħÙĩرجاÙĨ":138281,"ÙħراÙĥ":138282,"ÙħراÙĥز":138283,"Ġоднако":138284,"à¸Ķีà¹Ĩ":138285,"Ġצפ×ķ":138286,"Ġkullanılan":138287,"Ġкино":138288,"ãĥĨãĤ£ãĥ³ãĤ°":138289,"ĠGiỼi":138290,"تÙĪØ²":138291,"تÙĪØ²ÙĬع":138292,"ยิà¸Ļ":138293,"ยิà¸Ļà¸Ķี":138294,"ĠcÅĵur":138295,"ĠiÅŁaret":138296,"Ġ×ij×¢×ĸר":138297,"Ġ×ij×¢×ĸרת":138298,"ĠпаÑĨи":138299,"ĠпаÑĨиенÑĤ":138300,"ãģ¿ãģŁãģĦãģ§ãģĻ":138301,"вез":138302,"лина":138303,"оде":138304,"Ġ×IJ×ķ×ª×Ł":138305,"dıģınız":138306,"ĠÐIJв":138307,"ĠÐIJвÑĤоÑĢ":138308,"ï¼®":138309,"ĠCần":138310,"ĠاÙĦاخ":138311,"ĠاÙĦاخبار":138312,"Ġê±°ìĿĺ":138313,"Ġatenção":138314,"ĠgeldiÄŁi":138315,"ãĤªãĤ¹":138316,"ãĤªãĤ¹ãĤ¹":138317,"ãĤªãĤ¹ãĤ¹ãĥ¡":138318,"евÑĭе":138319,"кÑĢÑĭл":138320,"à¹Ģà¸Ĭียà¸ĩ":138321,"à¹Ģà¸Ĭียà¸ĩà¹ĥหมà¹Ī":138322,"Ġmarço":138323,"ĠاÙĦÙħادة":138324,"Ġгол":138325,"Ġsprzedaży":138326,"Ġíķ´ê²°":138327,"ĠÐķго":138328,"ê¹Ģ":138329,"Ġ׾ק×ij×ľ×ª":138330,"ĠاÙĦÙģÙĨاÙĨ":138331,"Ġcomunicación":138332,"à¹Ģสà¹īà¸Ļà¸Ĺาà¸ĩ":138333,"íĺ¹":138334,"à¸Ĭำ":138335,"à¸Ĭำระ":138336,"Ġ׼×IJ×ŀ":138337,"Ġ׼×IJ×ŀ×ķר":138338,"à¸Ĭà¹Īาà¸ĩ":138339,"زÙĩر":138340,"Ġklientów":138341,"иваÑİÑĤ":138342,"анг":138343,"׳×ļ":138344,"Ġgá»įn":138345,"ÃľR":138346,"ìĺģìĥģ":138347,"Ġغزة":138348,"ìĿĮìĿĦ":138349,"Ġbezpo":138350,"ĠbezpoÅĽ":138351,"ĠbezpoÅĽredni":138352,"ĠاÙĦÙħÙĪØ§":138353,"ĠاÙĦÙħÙĪØ§Ø·ÙĨ":138354,"ĠاÙĦÙħÙĪØ§Ø·ÙĨÙĬÙĨ":138355,"ãĤĮãģ¾ãģĻ":138356,"ĠмаÑĤÑĩ":138357,"×IJ×ķף":138358,"ĠرسÙħÙĬ":138359,"ĠÑįкон":138360,"ĠÑįконом":138361,"ĠÑįкономиÑĩеÑģк":138362,"ãĥľãĥ¼":138363,"ĠдиÑĢ":138364,"ĠдиÑĢекÑĤоÑĢ":138365,"ĠÑģкоÑĢо":138366,"à¸ļำ":138367,"à¸ļำร":138368,"à¸ļำรุà¸ĩ":138369,"ĠÑĦÑĥÑĤ":138370,"ĠÑĦÑĥÑĤбол":138371,"Ġ×IJ×Ļ׾":138372,"Ġì¤ijêµŃ":138373,"ìľ¤":138374,"eÄŁe":138375,"à¹Ħà¸ģà¹Ī":138376,"traî":138377,"traîn":138378,"ĠÑĤÑĢÑĥб":138379,"à¹Ģà¸ļื":138380,"à¹Ģà¸ļืà¹īà¸Ńà¸ĩ":138381,"à¹ģมà¸Ļ":138382,"ĠتØŃدÙĬØ«":138383,"Ġ×Ľ×¢×ª":138384,"ØŃاسب":138385,"lıģa":138386,"×§×Ļ×Ļ×ŀ×Ļ×Ŀ":138387,"оÑģÑĤÑĮÑİ":138388,"à¸Ŀั":138389,"à¸Ŀัà¹Īà¸ĩ":138390,"شغÙĦ":138391,"ìĽ¹":138392,"Ġкаждого":138393,"Ġbölümü":138394,"หà¸Ļี":138395,"ĠistediÄŁi":138396,"Ġtrưng":138397,"ãĥĮ":138398,"ฮà¸Ń":138399,"Ø£ÙĨØ´":138400,"Ø£ÙĨشطة":138401,"ĠاÙĦÙħسÙĬ":138402,"ĠاÙĦÙħسÙĬØŃ":138403,"ลัà¸ģษà¸ĵà¹Į":138404,"Ġná»Ńa":138405,"à¸Ĺีà¹Īà¸ķà¹īà¸Ńà¸ĩà¸ģาร":138406,"ÑĪек":138407,"лÑij":138408,"Ġש×Ļ×Ķ":138409,"Ġש×Ļ×Ķ×Ļ×Ķ":138410,"Ġkhuôn":138411,"ĠÑĤÑĢебованиÑı":138412,"Ġ×ľ×¢×ĸ×ķר":138413,"ĠاÙĦعÙħر":138414,"ราà¸Ħาà¸ĸูà¸ģ":138415,"ÙĩÙıÙħÙĴ":138416,"üst":138417,"üstü":138418,"Ġденег":138419,"Ġnạ":138420,"à¸Ĥà¸Ļม":138421,"Ġблаг":138422,"Ġблагод":138423,"ĠблагодаÑĢ":138424,"ĠблагодаÑĢÑı":138425,"إسÙĦاÙħ":138426,"à¸Ļิว":138427,"çŁ¥ãĤīãģªãģĦ":138428,"Ø«ÙĤØ©":138429,"ĠголоÑģ":138430,"×IJ×ķר×Ĺ":138431,"Ġtrứng":138432,"Ġодном":138433,"ĠkoÅĦcu":138434,"Ġ×ķרק":138435,"WiÄĻ":138436,"WiÄĻcej":138437,"Ġ×IJ×Ļ׼×ķת":138438,"Ġ×IJ×Ļ׼×ķת×Ļ":138439,"ÑģоÑģ":138440,"Ġjeżeli":138441,"以ä¸ĭãģ®":138442,"å°ıãģķ":138443,"å°ıãģķãģª":138444,"ологии":138445,"ĠобÑģлÑĥж":138446,"ĠобÑģлÑĥжива":138447,"Ùĥتابة":138448,"Ġê´Ģìĭ¬":138449,"עש×Ļר":138450,"Ġarasındaki":138451,"ĠÑĢайона":138452,"ÙĪØ§Ø¬Ø¨":138453,"Ġ×ij×Ĺ×Ļ×Ļ":138454,"íķ´ì£¼":138455,"Ġgóc":138456,"айл":138457,"ĠTình":138458,"æļ®ãĤī":138459,"æļ®ãĤīãģĹ":138460,"æĻĤãģ«ãģ¯":138461,"ĠгоÑĢоде":138462,"Ġ׼×IJ×Ļ׾":138463,"Ġ׼×IJ×Ļ׾×ķ":138464,"ĠCá»Ļng":138465,"ãģ©ãģĨãģĹãģ¦ãĤĤ":138466,"×Ĺ×ķ×£":138467,"تØŃرÙĥ":138468,"ĠÑģловам":138469,"à¸Īะà¸Ĭà¹Īวย":138470,"ĠاÙĦÙħستÙĤبÙĦ":138471,"ÙĤض":138472,"ÙĤضÙĬ":138473,"×ijס×ķפ":138474,"×ijס×ķפ×ķ":138475,"iÄĻÄĩ":138476,"ĠYıl":138477,"Ø´ÙĬØ®":138478,"à¸Ħุà¸ĵà¸Īะ":138479,"ש×ŀ×ķת":138480,"Ġتعرض":138481,"Ġanálise":138482,"ĠÑģобиÑĢа":138483,"à¹Ģà¸ŀà¸Ĭ":138484,"à¹Ģà¸ŀà¸Ĭร":138485,"Ġвели":138486,"Ġвелик":138487,"สัà¹īà¸Ļ":138488,"Ġpopulação":138489,"รà¹Īวมà¸ģัà¸Ļ":138490,"×Ĺ×ŀ":138491,"×Ĺ×ŀ×Ļש×Ļ":138492,"ס×Ļס":138493,"åĨħãģ§":138494,"ĠsobÄħ":138495,"ĠYay":138496,"ĠYayın":138497,"ãĥ¡ãĥĭãĥ¥ãĥ¼":138498,"ĠпÑĢедоÑģÑĤавлÑı":138499,"ãģłã썿ĢĿãģĨ":138500,"Ġê³łê°Ŀ":138501,"Ġодним":138502,"à¹ĥà¸Ļà¹Ģรืà¹Īà¸Ńà¸ĩ":138503,"Ġsá»ķ":138504,"ĠÐĹдеÑģÑĮ":138505,"ĠизменениÑı":138506,"ĠìĿ¼ìĿĦ":138507,"ãģªãģ®ãģł":138508,"кладÑĭва":138509,"ÑĢма":138510,"Ġ×ķ×ij׼׾":138511,"تأÙħÙĬÙĨ":138512,"ĠпÑĢиÑıÑĤ":138513,"ĠпÑĢиÑıÑĤн":138514,"ÙħÙħار":138515,"ÙħÙħارسة":138516,"ãģ¨ãģªãģ£ãģ¦":138517,"ĠجÙħÙĬÙĦ":138518,"Ġì§Ī":138519,"Ġì§Ī문":138520,"Ġquestão":138521,"ié":138522,"iéndo":138523,"หà¹īà¸Ńà¸ĩà¸ŀัà¸ģ":138524,"ãĥijãĥ¼ãĥĪ":138525,"ÑĤвеÑĢжда":138526,"нÑģкой":138527,"зал":138528,"มุà¹Īà¸ĩ":138529,"á»Ĭ":138530,"Ġ×Ķ×IJ×Ĺר×ķ׳×Ķ":138531,"ĠThư":138532,"주민":138533,"ĠاÙĦعب":138534,"évén":138535,"événement":138536,"ÙĤÙĪØ§Ø¹Ø¯":138537,"دÙı":138538,"ĠìķĬìĬµëĭĪëĭ¤":138539,"Ġ보기":138540,"Ġyapılması":138541,"à¹Ģราà¸ģ":138542,"à¹Ģราà¸ģà¹ĩ":138543,"ØŃذر":138544,"ÙĤصر":138545,"ãģ¦ãģĹãģ¾ãģĦãģ¾ãģĹãģŁ":138546,"Ġà¹Ģà¸Ľà¹ĩà¸Ļà¸ķà¹īà¸Ļ":138547,"ãģ¨ãģ«":138548,"ãģ¨ãģ«ãģĭ":138549,"ãģ¨ãģ«ãģĭãģı":138550,"нÑĨе":138551,"звÑĥк":138552,"ãģĹãĤĪãģĨãģ¨":138553,"ĠاÙĦصØŃÙĬØ©":138554,"Ġש×Ķ×Ļ×ķ":138555,"ĠDiÄŁer":138556,"ÙĤÙĦÙĤ":138557,"ãĤ¸ãĥ£ãĥ³":138558,"Ġrá»Ŀi":138559,"ĠлеÑĩ":138560,"ĠлеÑĩениÑı":138561,"تباد":138562,"تبادÙĦ":138563,"צפ×Ķ":138564,"à¸Ħวามà¹Ģหà¹ĩà¸Ļ":138565,"Ġشب":138566,"ĠشبÙĥØ©":138567,"ר×Ļ×§":138568,"Ùħعد":138569,"Ùħعدات":138570,"dıģında":138571,"Ġ×ijש׳×Ļ×Ŀ":138572,"Ġ×Ķ×Ļשר×IJ׾":138573,"Ġ×Ķ×Ļשר×IJ׾×Ļת":138574,"Ġsınav":138575,"׳צ×Ļ×Ĵ":138576,"วัà¸ķà¸ĸุ":138577,"ĠاÙĦبرÙĦÙħ":138578,"ĠاÙĦبرÙĦÙħاÙĨ":138579,"tivitÃł":138580,"ãĤĵãģłãĤįãģĨ":138581,"×§×Ļ×Ļ×ŀ":138582,"ÙĦÙĬÙĥ":138583,"ĠÄijò":138584,"ĠÄijòi":138585,"ĠÐĺнÑĤеÑĢ":138586,"ĠÐĺнÑĤеÑĢнеÑĤ":138587,"ãģ«ãģ¨ãģ£ãģ¦ãģ¯":138588,"ãģ£ãģĵ":138589,"×§×ķס":138590,"ستØŃÙĤ":138591,"æķĻãģĪãģ¦":138592,"ãĥĢãĥ¡":138593,"ĠÙħÙĨزÙĦ":138594,"à¹Ģà¸ĭà¹ĩà¸Ļ":138595,"使ãģĪãĤĭ":138596,"è¦ĭç©į":138597,"è¦ĭç©įãĤĤãĤĬ":138598,"Ø£Ùģ":138599,"Ø£ÙģÙĥار":138600,"ĠигÑĢов":138601,"ĠигÑĢовÑĭе":138602,"ĠmÄĻż":138603,"ĠmÄĻżczy":138604,"ĠmÄĻżczyzn":138605,"ĠاÙĦØŃÙĤÙĬÙĤÙĬ":138606,"عبر":138607,"׼×ķ׾׳×ķ":138608,"íĿ¥":138609,"×ŀ×IJ×ķ×Ĺר":138610,"ختص":138611,"ãĥŀãĥŀ":138612,"Ġ×IJ×Ĺ×ķ×ĸ":138613,"íĮĢ":138614,"Ġrá»iji":138615,"ĠвÑĤоÑĢ":138616,"ĠвÑĤоÑĢой":138617,"Ġlẫn":138618,"пÑĢом":138619,"пÑĢомÑĭÑĪ":138620,"пÑĢомÑĭÑĪлен":138621,"пÑĢомÑĭÑĪленн":138622,"ĠоÑĤноÑĪениÑı":138623,"Ġsứ":138624,"ĠмобилÑĮ":138625,"ĠмобилÑĮн":138626,"ĠÑįÑĤомÑĥ":138627,"Ġtạp":138628,"ĠìĤ¬ê±´":138629,"ĠìķĮ볤":138630,"ÙĥÙı":138631,"ÙĥÙıÙħÙĴ":138632,"Ġ×§×ķר×Ķ":138633,"ĠÑĦиÑĢ":138634,"ĠÑĦиÑĢм":138635,"Ġsıkıntı":138636,"׳׼":138637,"׳׼×ķף":138638,"ÙĪÙĦÙĪØ¬ÙĬ":138639,"ØŃاÙĨ":138640,"Ġloạn":138641,"Ġ×IJ×ľ×£":138642,"Ġmắn":138643,"abhäng":138644,"abhängig":138645,"ĠÑĥÑĢовнÑı":138646,"Ġ׾×ij×ĵ×ķ×§":138647,"ÙĬÙħÙĨ":138648,"layın":138649,"Ġhải":138650,"Ġзавод":138651,"ĠìķĦ주":138652,"สà¸ĸา":138653,"สà¸ĸาà¸ļัà¸Ļ":138654,"Ġgüvenlik":138655,"à¹Ģà¸Ķà¹Īà¸Ļ":138656,"×ij×ĵ×§":138657,"ĠëĪ":138658,"ĠëĪĦ":138659,"ĠëĪĦ구":138660,"éĩįè¦ģãģª":138661,"รà¸Ńà¸ĩรัà¸ļ":138662,"schlie":138663,"schlieÃŁen":138664,"Ġìĸ¼":138665,"Ġìĸ¼ë§Ī":138666,"Ġìĸ¼ë§ĪëĤĺ":138667,"ÑĤики":138668,"íķľëĭ¤ê³ł":138669,"ãģłãģ£ãģŁãĤī":138670,"Ġ×Ķ×Ļ×ĺ×ij":138671,"ãģªãģijãĤĮãģ°ãģªãĤīãģªãģĦ":138672,"âÌ":138673,"ậ":138674,"Ġphạt":138675,"akÄ±ÅŁ":138676,"ãģ¦ãģĹãģ¾ãģĦãģ¾ãģĻ":138677,"à¹Ģà¸ĭà¹ĩ":138678,"ĠСегоднÑı":138679,"Ġinsanların":138680,"Ġdéveloppe":138681,"תפר":138682,"תפר×Ļ×ĺ":138683,"اÙĨتشار":138684,"ê°ij":138685,"François":138686,"Ø£ÙĦع":138687,"Ø£ÙĦعاب":138688,"ãĤĴè¶ħ":138689,"ãĤĴè¶ħãģĪ":138690,"Ġê°ĻìĬµëĭĪëĭ¤":138691,"ãĤ³ãĥ¬":138692,"ĠмеÑģÑıÑĨев":138693,"íĮħ":138694,"ĠاÙĦجاÙħعة":138695,"ìĿ¸íĦ°":138696,"ìĿ¸íĦ°ëĦ·":138697,"×ĵר×ķש":138698,"ĠÙĪØ£Ø´Ø§Ø±":138699,"ĠпÑĢавила":138700,"ãģĿãģĵãģ«":138701,"×Ĺ×ŀ×ĵ":138702,"à¹Ģหà¸ķุà¸ģารà¸ĵà¹Į":138703,"Ġê²½íĹĺ":138704,"ãģ¶ãĤĬ":138705,"׾ש":138706,"׾ש×ķף":138707,"à¹Ģà¸ĸ":138708,"ĠDoÄŁu":138709,"ĠиÑģполÑĮзование":138710,"ĠçocuÄŁu":138711,"магазине":138712,"ĠÄijiá»ĥn":138713,"Ġaslı":138714,"Ġaslında":138715,"Ġdoença":138716,"Ġساع":138717,"Ġساعات":138718,"ĠиÑģполÑĮзованиÑı":138719,"ר×ķצ×Ļ×Ŀ":138720,"ĠзнаÑĩиÑĤ":138721,"ĠÑĢам":138722,"ĠÑĢамкаÑħ":138723,"거리":138724,"ĠпÑĭÑĤа":138725,"ãĥģãĥ³":138726,"ĠпоÑģк":138727,"ĠпоÑģколÑĮ":138728,"ĠпоÑģколÑĮкÑĥ":138729,"إبر":138730,"إبراÙĩ":138731,"إبراÙĩÙĬÙħ":138732,"ĠÑĤÑĢеÑħ":138733,"ĠGenç":138734,"سÙĪÙģ":138735,"ĠveÃŃculo":138736,"ĠNgân":138737,"ĠоÑĩеÑĢедÑĮ":138738,"à¸Ħรึà¹Īà¸ĩ":138739,"×IJ×ij×Ļ":138740,"à¸ķà¹īม":138741,"ãĤĴè¡ĮãģĦ":138742,"ĠاÙĦسابÙĤØ©":138743,"наÑĨи":138744,"наÑĨиона":138745,"наÑĨионалÑĮн":138746,"Ġgestión":138747,"تÙĤد":138748,"ĠاÙĦبÙĬاÙĨ":138749,"ĠاÙĦبÙĬاÙĨات":138750,"ĠاÙĦاÙĨتخاب":138751,"ĠاÙĦاÙĨتخابات":138752,"à¹Ģà¸Ĭà¹Īา":138753,"×ĵ×IJ×Ĵ":138754,"Ġ׾×Ĵ×ŀר×Ļ":138755,"ĠتØŃتاج":138756,"Ġthôn":138757,"à¸ķà¹īà¸Ńà¸Ļ":138758,"à¸ķà¹īà¸Ńà¸Ļรัà¸ļ":138759,"女ãģ®":138760,"女ãģ®åŃIJ":138761,"Ġthợ":138762,"Ø·ØŃÙĨ":138763,"ารà¹Įà¸Ķ":138764,"ת×ŀ×Ļ×ĵ":138765,"ĠÑģамÑĭм":138766,"Ġìĭľíĸī":138767,"إصد":138768,"إصدار":138769,"ĠNghá»ĩ":138770,"ìķķ":138771,"سئ":138772,"سئÙĦ":138773,"à¸Ńาร":138774,"à¸Ńารม":138775,"à¸Ńารมà¸ĵà¹Į":138776,"à¹ģฮ":138777,"׳×ĺ׾":138778,"Ġì¢ĭìķĦ":138779,"×ķ׾׾":138780,"Ġ×ij×Ľ×ª×ij":138781,"ãĤ«ãĥ©":138782,"צע×Ļר×Ļ×Ŀ":138783,"تعبÙĬر":138784,"Ġ×ŀקר×Ķ":138785,"ĠÑĦакÑĤоÑĢ":138786,"ĠتÙħاÙħ":138787,"ĠتÙħاÙħا":138788,"ëįķ":138789,"Ġvưá»Ŀ":138790,"Ġvưá»Ŀn":138791,"ĠdÄ±ÅŁÄ±":138792,"ãģĦãģ¡":138793,"Ġ׾ק׳×ķת":138794,"ĠاÙĦعÙĦاÙĤات":138795,"пÑĥб":138796,"пÑĥбли":138797,"Ø¥ÙĬÙħ":138798,"Ø¥ÙĬÙħاÙĨ":138799,"à¸Ńำà¸Ļา":138800,"à¸Ńำà¸Ļาà¸Ī":138801,"åIJ«ãģ¾ãĤĮ":138802,"ãĤĭãģŁãĤģãģ«":138803,"ס×Ĵ":138804,"ס×Ĵ׳×ķף":138805,"تØŃدÙĬ":138806,"Ġauprès":138807,"ĠاÙĦجÙĩا":138808,"ĠاÙĦجÙĩاز":138809,"Ġ×ŀת×Ĺת":138810,"еннÑĥÑİ":138811,"Ġзим":138812,"à¸ģาà¹ģà¸Ł":138813,"Ġ×ijת×ķר":138814,"Ġnghè":138815,"Ġnghèo":138816,"ĠÐĽÑİ":138817,"ĠÐĽÑİб":138818,"תקצ×Ļ×ij":138819,"×ŀעש×Ķ":138820,"ĠاÙĦبÙĬت":138821,"צ×Ļפ":138822,"ĠобÑıзан":138823,"ĠMá»Ĺi":138824,"ĠТÑĥÑĢ":138825,"ĠÙĪØ¨Ø§ÙĦت":138826,"ĠÙĪØ¨Ø§ÙĦتاÙĦÙĬ":138827,"Ġdécision":138828,"Ġبد":138829,"Ġبدأت":138830,"Ġcục":138831,"Ġbask":138832,"Ġbaskı":138833,"Ġhatırl":138834,"Ġhatırla":138835,"å°ıãģķãģĦ":138836,"Ġgerçekten":138837,"à¸ľà¸±à¸ģ":138838,"åı¯èĥ½ãģª":138839,"×ŀ×IJס":138840,"ĠcrÃŃtica":138841,"ĠìĿĺìĽIJ":138842,"عÙĤÙĪØ¯":138843,"×ĺ׼׳":138844,"×ĺ׼׳×ķ׾×ķ×Ĵ×Ļ×Ķ":138845,"è¨ĢãģĪãģ°":138846,"ĠÙĤÙĨا":138847,"ĠÙĤÙĨاة":138848,"ĠìĿ´ê²ĥìĿĢ":138849,"تصر":138850,"à¸Łà¸±à¸Ļ":138851,"ĠÑĢеÑĨеп":138852,"ĠÑĢеÑĨепÑĤ":138853,"ĠبÙĨÙ쨳":138854,"ÑĢоÑĪ":138855,"ĠмаÑĢÑĤа":138856,"Ġsonras":138857,"Ġsonrası":138858,"×ķ×ijש":138859,"ãĥªãĤ¹ãĤ¯":138860,"ĠFrançais":138861,"á»ļ":138862,"ê°Ķ":138863,"Ġ×Ķ×ijר×Ļת":138864,"פ×Ļצ":138865,"פ×Ļצ×ķ×Ļ":138866,"ĠÙĦÙħاذا":138867,"ĠÐļиев":138868,"ĠÑģмÑĭÑģл":138869,"ê¸Īìľµ":138870,"ãĤ·ãĥ£ãĥ«":138871,"ãĥ©ãĤ¤ãĥĪ":138872,"ìĽĥ":138873,"×ŀ×Ĺר":138874,"ãĨį":138875,"Ġkullanım":138876,"Ġ×IJצ׾׳×ķ":138877,"ĠtÃłn":138878,"ãĥıãĥ¼":138879,"ãģ¨ãģ¨ãĤĤ":138880,"ãģ¨ãģ¨ãĤĤãģ«":138881,"ÑĢег":138882,"ÑĢеги":138883,"ÑĢегион":138884,"ãģªãģıãģªãĤĭ":138885,"Ġchảy":138886,"ĠجÙĩØ©":138887,"ÅĦskiej":138888,"à¸Ńีà¹Ģม":138889,"à¸Ńีà¹Ģมล":138890,"ãģįãģ£ãģ¨":138891,"ĠìĺĪìĤ°":138892,"Ġkitabı":138893,"Ġeducação":138894,"ĠbuluÅŁ":138895,"ологиÑı":138896,"ĠконкÑĢ":138897,"ĠконкÑĢеÑĤ":138898,"×Ĵ×Ļר":138899,"ĠпÑĢедлаг":138900,"ĠпÑĢедлагаеÑĤ":138901,"ĠYên":138902,"Ġíķľë²Ī":138903,"Ġ×ŀר׼×ĸ×Ļ":138904,"à¹Ģà¸Ľà¸´à¸Ķà¹Ģà¸ľà¸¢":138905,"ÑĤвеÑĢд":138906,"ĠHá»ĩ":138907,"ĠÐĵÑĢ":138908,"à¸Ŀà¹īา":138909,"×Ķשק":138910,"×Ķשקע×Ķ":138911,"ĠнаÑĥк":138912,"ìłIJìĿĦ":138913,"ĠнелÑĮ":138914,"ĠнелÑĮз":138915,"ĠнелÑĮзÑı":138916,"гин":138917,"ĠBöl":138918,"ĠBölge":138919,"Ġвла":138920,"ĠвлаÑģÑĤи":138921,"à¹Ģà¸Ļà¹ĩ":138922,"à¹Ģà¸Ļà¹ĩà¸ķ":138923,"골":138924,"Ġöld":138925,"Ġöldür":138926,"×Ľ×ł×¢":138927,"ĠاÙĦÙĩÙĬئة":138928,"تارÙĬØ®":138929,"ĠÐijÑĢ":138930,"ĠÑģмож":138931,"ĠÑģможеÑĤе":138932,"ĠLúc":138933,"à¹Ħà¸Ľà¸ĸึà¸ĩ":138934,"ĠBakanı":138935,"Ġerklärt":138936,"ĠÐIJна":138937,"Ġscène":138938,"åķıãģĦ":138939,"åķıãģĦåIJĪãĤıãģĽ":138940,"ÙħÙĩÙĨد":138941,"ÙħÙĩÙĨدس":138942,"Ġназвание":138943,"иваниÑı":138944,"ãĤĴå¤īãģĪ":138945,"ä»ĺãģįåIJĪ":138946,"ãĥijãĤ½":138947,"ãĥijãĤ½ãĤ³ãĥ³":138948,"æĺİãĤī":138949,"æĺİãĤīãģĭ":138950,"à¹Ģà¸Ńà¸ģสาร":138951,"à¹Ģà¸ģิà¸Ļà¹Ħà¸Ľ":138952,"леп":138953,"ãģĹãģŁãĤĤãģ®":138954,"ĠCâm":138955,"ĠCâmara":138956,"×§×ķ׾׳×ķ×¢":138957,"Ġ×ij×Ĵ×Ļף":138958,"Ġoczy":138959,"ĠoczywiÅĽcie":138960,"attivitÃł":138961,"ãĥĵãĥ¥ãĥ¼":138962,"Ġeducación":138963,"İYE":138964,"ê¹ĮìļĶ":138965,"ãĤ¨ãĥªãĤ¢":138966,"неÑģÑĤи":138967,"Ġmóg":138968,"ĠmógÅĤ":138969,"Ġ×§×ĺ׳×Ļ×Ŀ":138970,"ĠPrä":138971,"Ġ×ľ×¢×ij×ķר":138972,"بÙĨÙī":138973,"зол":138974,"золоÑĤ":138975,"ĠwnÄĻtr":138976,"ĠwnÄĻtrz":138977,"Ġconstrução":138978,"รัà¸ļรà¸Ńà¸ĩ":138979,"سجÙĨ":138980,"Ġ×§×ķ׳":138981,"ס×Ļפ×ķר":138982,"ĠÙħدÙī":138983,"رضÙī":138984,"плав":138985,"ï¼¥":138986,"Ġila":138987,"Ġilaç":138988,"ãĤĭãģ¹ãģį":138989,"ĠÙħÙĪÙĤÙģ":138990,"à¸ģรุ":138991,"à¸ģรุà¸ĵา":138992,"chodzÄħc":138993,"ĠÑĤÑĭÑģ":138994,"ÐķвÑĢо":138995,"ĠÙĬØŃدث":138996,"ãĥ¡ãĤ¤ãĥ³":138997,"ĠاÙĦصØŃÙĬ":138998,"ĠÐĶан":138999,"دعاء":139000,"ãĤ´ãĥ¼ãĥ«":139001,"×©×ł×ª×Ļ":139002,"×©×ł×ª×Ļ×Ļ×Ŀ":139003,"à¸Ķà¹īวยà¸ģัà¸Ļ":139004,"Ġolacaģı":139005,"Ġ×ij×ŀ×Ĺ×Ļר":139006,"×Ķ×§":139007,"×Ķ×§×ŀת":139008,"ãĥ¢ãĥİ":139009,"ĠçalÄ±ÅŁtı":139010,"Ġjóvenes":139011,"ãģĦãģıãĤī":139012,"ĠÙħعدÙĦ":139013,"ĠCÅ©ng":139014,"ĠSegún":139015,"Ġdönemde":139016,"Ġ׾×Ļ×ĵ×Ļ":139017,"ãģįãģ¡":139018,"ãģįãģ¡ãĤĵ":139019,"ãģįãģ¡ãĤĵãģ¨":139020,"Ù쨱ÙĨس":139021,"Ù쨱ÙĨسا":139022,"åIJijãģį":139023,"Ġcampaña":139024,"ĠÑģамоÑģÑĤоÑı":139025,"ĠÑģамоÑģÑĤоÑıÑĤелÑĮно":139026,"á»Ģ":139027,"ÙĤÙĪØ§":139028,"سÙĦاØŃ":139029,"à¸ģระà¹ģ":139030,"à¸ģระà¹ģส":139031,"ĠполÑĮзÑĥ":139032,"nqu":139033,"nquête":139034,"รà¹Īวมà¸ģัà¸ļ":139035,"ëĬIJëĥIJ":139036,"à¸Ĺีมà¸Ĭาà¸ķิ":139037,"Ġyıllık":139038,"ìĬ¬":139039,"ĠأصØŃاب":139040,"illé":139041,"Ġdóla":139042,"Ġdólares":139043,"Ġкож":139044,"Ġкожи":139045,"ลà¹īà¸Ń":139046,"à¹Ģรียà¸ļร":139047,"à¹Ģรียà¸ļรà¹īà¸Ńย":139048,"à¹Ģà¸ŀิ":139049,"à¹Ģà¸ŀิà¹Īà¸ĩ":139050,"ÑĢиÑĤоÑĢи":139051,"Ġíijľ":139052,"ĠíijľíĺĦ":139053,"ĠпеÑĢев":139054,"ĠпеÑĢевод":139055,"פ×Ĵ×Ļ×¢×Ķ":139056,"ĠdeÄŁerlendirme":139057,"ÙģØ§Ø¦":139058,"ĠвÑĭгод":139059,"ınızı":139060,"×ķ׼×Ļ×Ĺ":139061,"ĠдоÑģÑĤиг":139062,"ĠngÃłn":139063,"æĢĿãģ£ãģŁ":139064,"ĠÐķÑģÑĤÑĮ":139065,"ĠاÙĦرغÙħ":139066,"ĠzwiÄħzane":139067,"ربط":139068,"à¸Ļึà¸ĩ":139069,"Ġ׾×Ĺ×ķ×§":139070,"Ġszczególn":139071,"Ġszczególnie":139072,"ĠباستخداÙħ":139073,"ĠfÃŃsico":139074,"עס":139075,"עס×ķ×§":139076,"سÙĦÙĪÙĥ":139077,"ĠاØŃد":139078,"ÑĩÑijÑĤ":139079,"×ĸ׼×Ķ":139080,"Ġlá»ĩnh":139081,"ĠÙĪØŃØª":139082,"ĠÙĪØŃØªÙī":139083,"à¸Ħวามสามารà¸ĸ":139084,"à¸Ńยูà¹Īà¹ģลà¹īว":139085,"à¸ģารà¹Ģà¸Ķิà¸Ļà¸Ĺาà¸ĩ":139086,"تخذ":139087,"צ×Ļ×ķ×ĵ":139088,"ĠاÙĦأس":139089,"ĠاÙĦأسÙĩÙħ":139090,"Ġtá»ĩ":139091,"ãģ£ãģ¦ãģĦãģ¦":139092,"สรุ":139093,"à¸ªà¸£à¸¸à¸Ľ":139094,"ĠкомÑĦ":139095,"ĠкомÑĦоÑĢÑĤ":139096,"ìĺ¤ëĬĶ":139097,"ĠÑĢазв":139098,"ĠÑĢазвива":139099,"ланд":139100,"hänge":139101,"ĠبÙĨسبة":139102,"à¹Ģà¸Ĥียว":139103,"עצ×Ŀ":139104,"Ġ×ľ×ľ×Ľ×ª":139105,"ÑģоÑĨиалÑĮн":139106,"Ġëĭ¤ìĿĮê³¼":139107,"Ġרש×ķ×ŀ":139108,"×ŀר×Ĺ×ij":139109,"سÙĤØ·":139110,"Ġalanı":139111,"ĠÄijá»ĩ":139112,"é£Łãģ¹ãĤĭ":139113,"à¸Ķึà¸ĩ":139114,"Ġgegenüber":139115,"ĠبÙĩذÙĩ":139116,"à¸ĸืà¸Ńà¹Ģà¸Ľà¹ĩà¸Ļ":139117,"ëķħ":139118,"à¸Ħà¸Ļà¹Ħà¸Ĺย":139119,"ãĤ¢ãĤ¦":139120,"ãĤ¢ãĤ¦ãĥĪ":139121,"ศัà¸ģ":139122,"ศัà¸ģà¸Ķิ":139123,"ศัà¸ģà¸Ķิà¹Į":139124,"ÙĤÙĪØ§ÙĨ":139125,"ÙĤÙĪØ§ÙĨÙĬÙĨ":139126,"Ġhá»Ļp":139127,"ãģªãģıãģªãģ£ãģ¦":139128,"Ġ×IJ×ŀ׳":139129,"Ġ×IJ×ŀ׳×Ŀ":139130,"à¹Ģà¸ķืà¸Ńà¸Ļ":139131,"ĠзавиÑģим":139132,"ĠзавиÑģимоÑģÑĤи":139133,"ת×Ļ×IJ":139134,"ת×Ļ×IJ×ķר":139135,"å§ĭãĤģãģŁ":139136,"Ġngá»į":139137,"Ġngá»įt":139138,"íĴį":139139,"ê³¼ìŀ¥":139140,"Ġbại":139141,"ãģ§ãģįãģ¦":139142,"Ġcomeçar":139143,"à¸Ľà¸£à¸²à¸ģ":139144,"à¸Ľà¸£à¸²à¸ģà¸ı":139145,"ĠгодÑĭ":139146,"меÑģ":139147,"ĠاÙĦÙħستÙĪÙī":139148,"ĠÑģамÑĭе":139149,"ллеÑĢ":139150,"ãģ£ãģ¦ãģĹãģ¾ãģĦãģ¾ãģĻ":139151,"ãģ¨ãģ®ãģĵãģ¨":139152,"bió":139153,"à¸ģลà¹Īà¸Ńà¸ĩ":139154,"ĠاÙĦزÙĪØ¬":139155,"ãģ«è¡Įãģ£ãģŁ":139156,"à¸Ħà¹Īà¸Ńà¸Ļ":139157,"à¸Ħà¹Īà¸Ńà¸Ļà¸Ĥà¹īาà¸ĩ":139158,"ĠbaÄŁl":139159,"ĠbaÄŁlant":139160,"ĠbaÄŁlantı":139161,"確ãģĭ":139162,"確ãģĭãģ«":139163,"ãĥľãĥ¼ãĥ«":139164,"çµĤãĤıãĤĬ":139165,"ש×ŀר":139166,"à¸Ĺีà¹Īสามารà¸ĸ":139167,"ÙĦزÙħ":139168,"даеÑĤÑģÑı":139169,"รัà¸ļà¸Ľà¸£à¸°":139170,"รัà¸ļà¸Ľà¸£à¸°à¸Ĺาà¸Ļ":139171,"å¤īãĤıãĤĬ":139172,"ï¼¢":139173,"ĠìĺĪìĪĺëĭĺ":139174,"ãĤĪãģĨãģ¨":139175,"มัà¸ģà¸Īะ":139176,"ĠHương":139177,"ÙĨÙ쨰":139178,"×ŀ×ĵ×ĵ":139179,"ĠìĿ¸ìłķ":139180,"ÑħодиÑĤÑĮ":139181,"ĠзавиÑģиÑĤ":139182,"×ķ×ĵ×Ļ×¢":139183,"ãģĵãģ¨ãģĮãģĤãĤĬãģ¾ãģĻ":139184,"عراÙĤ":139185,"سطØŃ":139186,"à¸ģำà¹Ħร":139187,"ëĵ¤ëıĦ":139188,"×Ļצ×Ļר×Ķ":139189,"ãģĨãģĵãģ¨":139190,"ÙĦاØŃÙĤ":139191,"ãģĦãĤĮãģ°":139192,"ĠиÑģполÑĮзÑĥÑİÑĤ":139193,"ĠBợi":139194,"Ġשק׾×Ļ×Ŀ":139195,"ÑĨикл":139196,"ÐIJÐŀ":139197,"Ġ×ijש׳×Ķ":139198,"ÙĨشط":139199,"Ġש×Ļ׳×ķ×Ļ":139200,"Ġש×Ļ׳×ķ×Ļ×Ļ×Ŀ":139201,"Ġpoblación":139202,"ĠHưng":139203,"ระว":139204,"ระวัà¸ĩ":139205,"رÙĬاضة":139206,"رصد":139207,"تÙĤÙĦÙĬ":139208,"تÙĤÙĦÙĬد":139209,"Ġülkem":139210,"Ġülkemiz":139211,"à¸Ĭะ":139212,"ãĤ¯ãĥªãĥ¼ãĥł":139213,"èģŀãģĦãģŁ":139214,"Ġważ":139215,"Ġważne":139216,"ê±°ëĵł":139217,"ê±°ëĵłìļĶ":139218,"×ŀ×IJ×ij×§":139219,"×Ĺ×ĵש×ķת":139220,"ĠWroc":139221,"ĠWrocÅĤaw":139222,"ĠKültür":139223,"sist":139224,"sistência":139225,"×¢×ĸר×Ķ":139226,"Ġgương":139227,"รà¹īาà¸Ļà¸Ħà¹īา":139228,"ĠÙĪØ£ÙĪØ¶ØŃ":139229,"ándose":139230,"ãĤ·ãĥ¼ãĥ³":139231,"×IJ׳ר×Ĵ":139232,"×IJ׳ר×Ĵ×Ļ×Ķ":139233,"ãģªãģĦãģ§ãģĻ":139234,"Ġkhá»§ng":139235,"Ġ문ìĦľ":139236,"Ġ×ij×ĵ×ijר":139237,"×ĵ×Ļ×ķ":139238,"×ĵ×Ļ×ķ×ķ×Ĺ":139239,"Ġrégl":139240,"ÙħÙĪØ§Ø¯":139241,"обоÑĢ":139242,"обоÑĢоÑĤ":139243,"Ġ×Ķ×ij׾":139244,"Ġ×Ķ×ij׾×ķ×Ĵ":139245,"ØŃاÙħ":139246,"ĠاÙĦعاص":139247,"ĠاÙĦعاصÙħØ©":139248,"пеÑĢаÑĤоÑĢ":139249,"تخÙĦ":139250,"تخÙĦص":139251,"ãģŁãģłãģĹ":139252,"تسÙħ":139253,"à¹Ĥรà¸ĩà¸ŀ":139254,"à¹Ĥรà¸ĩà¸ŀยา":139255,"à¹Ĥรà¸ĩà¸ŀยาà¸ļาล":139256,"ĠYük":139257,"ĠYüksek":139258,"Ġש׳×Ļת":139259,"Ġש׳×Ļ×ª×Ł":139260,"liÄŁe":139261,"Ġפת":139262,"Ġפת×ķ×Ĺ":139263,"ĠbeÄŁ":139264,"ĠbeÄŁen":139265,"Ġ×ŀ×ķר":139266,"Ġ×ŀ×ķר׼×ij":139267,"ĠرساÙĦØ©":139268,"íĨµìĭł":139269,"Ġavalia":139270,"Ġavaliações":139271,"Ġmanh":139272,"Ġmanhã":139273,"Ġìķŀ":139274,"Ġìķŀìľ¼ë¡ľ":139275,"ÙĤتر":139276,"ÙĤترØŃ":139277,"à¹Ģà¸ģืà¸Ń":139278,"à¹Ģà¸ģืà¸Ńà¸ļ":139279,"Ġproposé":139280,"Ø£Ùħا":139281,"Ø£ÙħاÙĥÙĨ":139282,"ĠÐŀÐŀ":139283,"ĠÐŀÐŀÐŀ":139284,"ÙħÙĤار":139285,"ÙħÙĤارÙĨØ©":139286,"ëĦIJ":139287,"ãģĦãģŁãģłãģı":139288,"ÙĤÙĬÙĦ":139289,"ĠнаÑĪиÑħ":139290,"ãĤ«ãĥĥãĥĹ":139291,"×Ĺ×ľ×ª":139292,"Ġëĭ¤ë§Į":139293,"à¸Ĺัà¹Īวà¹Ĥลà¸ģ":139294,"ãĥįãĤ¿":139295,"ØŃساس":139296,"ãģ«ãģªãĤĮ":139297,"جائ":139298,"جائزة":139299,"échange":139300,"économ":139301,"économie":139302,"ТÐĺ":139303,"×¡×ª×Ľ×ľ":139304,"à¸Ĺัà¹īà¸ĩสà¸Ńà¸ĩ":139305,"ĠاÙĦخاÙħ":139306,"ĠاÙĦخاÙħس":139307,"×§×ĺ×¢":139308,"auważ":139309,"à¸ľà¸¹à¹īà¸Ĭาย":139310,"à¹ģà¸Ľà¸¥à¸ģ":139311,"åIJĮæĻĤãģ«":139312,"знаниÑı":139313,"ãģĦãģŁãģłãģįãģ¾ãģĹãģŁ":139314,"Ġ×ŀ×ij׾×Ļ":139315,"à¸Ĥà¸Ńà¹ĥหà¹ī":139316,"ĠاÙĦتربÙĬØ©":139317,"Ġdécouvert":139318,"Ġżyciu":139319,"après":139320,"Ġyab":139321,"Ġyabanc":139322,"Ġyabancı":139323,"ĠbaÅŁlayan":139324,"ìĹĪëįĺ":139325,"Ġhesabı":139326,"Ġë§Įìķ½":139327,"ë§Īëĭ¤":139328,"ĠThánh":139329,"ãĥ´ãĤ¡":139330,"à¸Ľà¸£à¸±à¸ļà¸Ľà¸£":139331,"à¸Ľà¸£à¸±à¸ļà¸Ľà¸£à¸¸à¸ĩ":139332,"ĠMặc":139333,"à¹Ģหà¸ķà¸¸à¸ľà¸¥":139334,"ĠÐijез":139335,"ĠcapacitÃł":139336,"ÅĤeÅĽ":139337,"ĠпÑĢеим":139338,"ĠпÑĢеимÑĥÑīеÑģÑĤв":139339,"ĠÅļwiÄĻt":139340,"Ġpublié":139341,"×ŀעצ×ij":139342,"ÙħشارÙĥات":139343,"à¸łà¸²à¸©":139344,"à¸łà¸²à¸©à¸µ":139345,"Ġdeuxième":139346,"ĠÙħØŃاÙ쨏":139347,"ĠÙħØŃاÙģØ¸Ø©":139348,"ĠSchön":139349,"、":139350,"Ġ×Ķ×ij×¢":139351,"Ġ×Ķ×ij×¢×Ļ×Ķ":139352,"ĠÙĪØ§ÙĦÙĦÙĩ":139353,"è¨Ģãģ£ãģŁ":139354,"à¸ķà¹īาà¸Ļ":139355,"วรรà¸ĵ":139356,"à¸Ĺิศ":139357,"ĠbaÅŁÄ±na":139358,"ĠmogÄĻ":139359,"ש×Ļפ×ķר":139360,"ĠÙĪØ¹Ø¯":139361,"ĠÙĪØ¹Ø¯Ùħ":139362,"Ġhistórico":139363,"Ġkısı":139364,"ĠìĿ´ê²Į":139365,"ĠPolÃŃtica":139366,"ĠÑģиÑĤÑĥаÑĨии":139367,"ĠkoÅĦca":139368,"×ij×ĵ×Ļ×§×Ķ":139369,"ĠاÙĦسÙĬارات":139370,"ãģªãĤīãģ°":139371,"ãĤµãĥ©":139372,"ãĤĭãģĵãģ¨ãģĮãģ§ãģįãĤĭ":139373,"Ġdecisão":139374,"×ķ×ķ×ĵ":139375,"läss":139376,"lässig":139377,"Ġ׾×Ļשר×IJ׾":139378,"ĠÙĬأتÙĬ":139379,"ר×ķ×ĸ":139380,"Ã¶ÄŁ":139381,"Ã¶ÄŁret":139382,"Ã¶ÄŁretim":139383,"Ġдек":139384,"Ġдекаб":139385,"ĠдекабÑĢÑı":139386,"Ġש×Ĺ×ķר":139387,"ãģ¦ãģıãĤĮãģŁ":139388,"عبارة":139389,"Ġélectrique":139390,"ĠاÙĦتÙĨÙħÙĬØ©":139391,"جرÙī":139392,"ĠìĪĺíĸī":139393,"à¸Ĺู":139394,"ĠÑĢеалÑĮно":139395,"ÑģпоÑģоб":139396,"à¸Ħลà¹īาย":139397,"ĠسعÙĪØ¯":139398,"önü":139399,"ĠÙģÙħÙĨ":139400,"تÙĥÙĪ":139401,"تÙĥÙĪÙĬÙĨ":139402,"ĠкаÑĩеÑģÑĤво":139403,"ĠконÑĤак":139404,"ĠконÑĤакÑĤ":139405,"ĠsözleÅŁme":139406,"à¸Ńà¹īาà¸ĩ":139407,"ĠتÙĪÙģ":139408,"ĠتÙĪÙģÙĬر":139409,"×Ķ×ĸ×ĵ":139410,"×Ķ×ĸ×ĵ×ŀ׳×ķת":139411,"ĠØ·ÙĪÙĬÙĦØ©":139412,"Ġtérmino":139413,"Ġ×IJ×Ļפ×Ķ":139414,"ãĥĵãĥ«":139415,"สà¹Ĥม":139416,"สà¹Ĥมสร":139417,"ĠاÙĦاث":139418,"ĠاÙĦاثÙĨÙĬÙĨ":139419,"евиÑĩ":139420,"Ġopinión":139421,"à¸Ľà¸§à¸Ķ":139422,"åı¤ãģĦ":139423,"รà¹Īา":139424,"ĠBiaÅĤ":139425,"ĠÑģÑĤал":139426,"ĠÑģÑĤало":139427,"ólogo":139428,"ĠìķĦëĭĪëĭ¤":139429,"Ġ×IJ×Ļת":139430,"Ġ×IJ×Ļת×ķ":139431,"à¹Ģหà¹ĩà¸Ļวà¹Īา":139432,"à¸ļารà¹Į":139433,"çĦ¼":139434,"çĦ¼ãģį":139435,"ĠìĿ´ìļ©ìŀIJ":139436,"ĠнекоÑĤоÑĢÑĭе":139437,"ksz":139438,"ksztaÅĤ":139439,"ksztaÅĤc":139440,"ãĤŃãĥ£ãĥĥãĤ·":139441,"ãĤŃãĥ£ãĥĥãĤ·ãĥ³ãĤ°":139442,"ĠroÅĽ":139443,"ĠroÅĽlin":139444,"ÑĢажа":139445,"×ij׳×Ļ×Ļ×Ķ":139446,"à¸Ľà¸£à¸ªà¸´":139447,"à¸Ľà¸£à¸ªà¸´à¸ķ":139448,"Ġgördü":139449,"×ŀ׳×Ķ×Ļ×Ĵ":139450,"å¤īãĤıãģ£ãģ¦":139451,"Ġ×IJ×Ķ":139452,"Ġ×IJ×Ķ×ijת×Ļ":139453,"à¹Ģรà¹Īà¸ĩ":139454,"Ġönünde":139455,"Ġê·¸ëĥ¥":139456,"полиÑĤ":139457,"полиÑĤиÑĩеÑģк":139458,"ãĥ¡ãĥĩãĤ£":139459,"ãĥ¡ãĥĩãĤ£ãĤ¢":139460,"ĠDetay":139461,"ĠDetaylı":139462,"ĠاÙĦصÙģØŃØ©":139463,"à¸ģารà¹Ģà¸ĩิà¸Ļ":139464,"Ġìµľê·¼":139465,"׼ש׾":139466,"I":139467,"вÑĪего":139468,"íķĺìĭ¤":139469,"ĠÐŃÑĤ":139470,"ĠÐŃÑĤоÑĤ":139471,"สื":139472,"สืà¸ļ":139473,"Ġngừng":139474,"ĠдокÑĥменÑĤов":139475,"даваÑĤÑĮ":139476,"ĠاÙĦشخصÙĬØ©":139477,"Ġצע×Ļר":139478,"درÙĥ":139479,"سØŃب":139480,"à¹Ħมà¹Īà¸Ħà¹Īà¸Ńย":139481,"Ġ×Ķ×ŀ×§×ķ×ŀ×Ļ":139482,"สัà¹Īà¸ĩà¸ĭืà¹īà¸Ń":139483,"Ġê·¸ê²ĥìĿĦ":139484,"ãģĤãĤĭãģĦ":139485,"ãģĤãĤĭãģĦãģ¯":139486,"×IJ×ķ×ĺ×ķ×ij":139487,"×IJ×ķ×ĺ×ķ×ij×ķס":139488,"кÑĨион":139489,"ĠÐľÐ¾Ð¶Ð½Ð¾":139490,"ãģıãģł":139491,"ãģıãģłãģķ":139492,"ĠинÑĦоÑĢмаÑĨиÑı":139493,"ï»Ł":139494,"ĠìŀijìĹħ":139495,"Ġ×Ļ×ķסף":139496,"إدارة":139497,"ĠاÙĦØŃاج":139498,"×ł×¡×Ļ×¢×Ķ":139499,"изаÑĨиÑı":139500,"×IJ׾×ij":139501,"×IJ׾×ij×ķ×Ŀ":139502,"пед":139503,"Ġ×§×ĺ׳×Ķ":139504,"ĠÙĨÙ쨳Ùĩا":139505,"ĠMinistério":139506,"Ġпен":139507,"ĠпенÑģи":139508,"ãĥIJãĥ©ãĥ³ãĤ¹":139509,"Ġ×Ķת×ķר×Ķ":139510,"Ġtạm":139511,"ĠìĹŃìĭľ":139512,"。":139513,"Ġthá»±":139514,"Ġısı":139515,"컨":139516,"ãģĹãģ£ãģĭãĤĬãģ¨":139517,"Ġxưa":139518,"Ġcặp":139519,"×Ĺ×Ļ×ij×ķר":139520,"วัà¸Ĵà¸Ļà¸ĺรรม":139521,"stär":139522,"stärke":139523,"ĠÑģамÑĭй":139524,"pisa":139525,"pisaÄĩ":139526,"ĠoluÅŁan":139527,"ĠاÙĦØ¥ÙħاÙħ":139528,"ĠcÄĥng":139529,"Ġgünl":139530,"Ġgünlük":139531,"Ġ׳ש×IJר":139532,"Ġkhiá»ĥn":139533,"ç¶ļãģijãĤĭ":139534,"stitución":139535,"Ġcapacité":139536,"Ġjaki":139537,"ĠjakiÅĽ":139538,"вÑĪиÑģ":139539,"вÑĪиÑģÑĮ":139540,"פע×ķ׾×ķת":139541,"ĠØŃÙĬات":139542,"ĠØŃÙĬاتÙĩ":139543,"Ġникогда":139544,"ÐĽÐ¬":139545,"Ġ×Ķ×¢×ķ×ij":139546,"Ġ×Ķ×¢×ķ×ij×ĵ×Ķ":139547,"ĠchÃło":139548,"หลายà¹Ĩ":139549,"ĠÑıн":139550,"ĠÑıнваÑĢ":139551,"ĠÑıнваÑĢÑı":139552,"à¸Īำà¹Ģà¸Ľà¹ĩà¸Ļà¸ķà¹īà¸Ńà¸ĩ":139553,"Ġhöher":139554,"ãģķãĤĮãģ¦ãģĦãģŁ":139555,"สà¸ĩสั":139556,"สà¸ĩสัย":139557,"ĠاÙĦاس":139558,"ĠاÙĦاسÙĦاÙħ":139559,"ĠاÙĦØ´Ùħس":139560,"สà¸ĸาà¸Ļี":139561,"ãĤ¯ãĥ©ãĤ¹":139562,"à¸ŀรร":139563,"à¸ŀรรà¸Ħ":139564,"põ":139565,"põe":139566,"Ġporém":139567,"à¸Ľà¸£à¸°à¸ªà¸ĩ":139568,"à¸Ľà¸£à¸°à¸ªà¸ĩà¸Ħà¹Į":139569,"powiedzie":139570,"powiedzieÄĩ":139571,"ĠмогÑĥ":139572,"Ġжел":139573,"Ġжелез":139574,"ĠاÙĦØ«ÙĤ":139575,"ĠاÙĦØ«ÙĤاÙģÙĬ":139576,"ĠпÑĢавило":139577,"Ġgdyż":139578,"פש×ķ×ĺ":139579,"ÑĢабоÑĤка":139580,"ĠÙĥرة":139581,"شدد":139582,"ÙħارÙĥ":139583,"ÙħÙĥØ©":139584,"ĠподпиÑģ":139585,"×ĺ×ķ×ķ×Ĺ":139586,"ĠÅĽc":139587,"ĠÅĽcian":139588,"ĠرجاÙĦ":139589,"Ġ×ª×ľ×ķ×Ļ":139590,"иÑĪ":139591,"иÑĪÑĮ":139592,"Ġmédec":139593,"Ġmédecin":139594,"ëįĶëĿ¼ëıĦ":139595,"ĠÑĤебÑı":139596,"Ġ׾×Ķ×ķס×Ļ×£":139597,"ãģĬ話":139598,"Ġà¹ģà¸ķà¹Īà¸ģà¹ĩ":139599,"داÙģ":139600,"داÙ쨹":139601,"ĠCùng":139602,"ãĥ»ãĥ»ãĥ»ãĥ»":139603,"ê¶ģ":139604,"ĠdeberÃŃa":139605,"หà¸Ļà¹Īวยà¸ĩาà¸Ļ":139606,"ĠvaÌĢ":139607,"Ġעצ×ŀ":139608,"Ġעצ×ŀ×Ŀ":139609,"à¹Ģà¸Ĭืà¹Īà¸Ńวà¹Īา":139610,"שקע":139611,"Ġ×Ķ׼×ķ׾":139612,"Ġ×Ķ׼×ķ׾׾":139613,"нибÑĥд":139614,"нибÑĥдÑĮ":139615,"ĠëĦĪíĿ¬":139616,"ĠобÑĢаÑī":139617,"ĠобÑĢаÑīа":139618,"Ġ×¢×ij×ķ×ĵת":139619,"ĠاÙĦÙħÙĨتخب":139620,"ıyord":139621,"ıyordu":139622,"ÙĪØ°":139623,"×Ĺש×Ļ×ij×ķת":139624,"Ġ×Ķ×¢×Ļ×§":139625,"Ġ×Ķ×¢×Ļקר×Ļ":139626,"ì¢Į":139627,"ยุà¹Ĥร":139628,"ยุà¹Ĥà¸£à¸Ľ":139629,"ĠапÑĢ":139630,"ĠапÑĢелÑı":139631,"szed":139632,"szedÅĤ":139633,"дон":139634,"à¹Ģà¸ķิà¸ļ":139635,"à¹Ģà¸ķิà¸ļà¹Ĥà¸ķ":139636,"коло":139637,"Ġkażdej":139638,"帰":139639,"帰ãĤĬ":139640,"Ġмилли":139641,"Ġмиллион":139642,"ç¾İåij³ãģĹãģĦ":139643,"تÙĤار":139644,"تÙĤارÙĬر":139645,"ĠìĿ´ë£¨":139646,"ĠìĿ´ë£¨ìĸ´":139647,"Ġsprzedaż":139648,"×Ķ×ķצ×IJ×ķת":139649,"ãĤ¢ãĤ¯ãĤ»":139650,"ãĤ¢ãĤ¯ãĤ»ãĤ¹":139651,"ר×ķ×¥":139652,"ĠгоÑģÑĥдаÑĢÑģÑĤвенн":139653,"Ø£ØŃÙĥ":139654,"Ø£ØŃÙĥاÙħ":139655,"ĠoluÅŁu":139656,"ĠAç":139657,"ĠAçık":139658,"ãĤ¸ãĥ¼":139659,"ç´łæĻ´":139660,"ç´łæĻ´ãĤīãģĹãģĦ":139661,"Ġ×ijש×ij×ķ×¢":139662,"بذ":139663,"بذÙĦ":139664,"สาà¹Ģหà¸ķุ":139665,"Ġpozosta":139666,"ĠpozostaÅĤ":139667,"ØŃرÙħ":139668,"Ġimportância":139669,"leÅŁtirme":139670,"ĠдÑĢев":139671,"Ġmóvil":139672,"ĠAynı":139673,"Ġналог":139674,"Ġналогов":139675,"Ġ×Ĺ×Ļפ×Ķ":139676,"ĠÑĦоÑĢмÑĥ":139677,"à¸Ĺà¸Ķสà¸Ńà¸ļ":139678,"ĠksiÄħżki":139679,"ĠmaÅĤe":139680,"ÙħسأÙĦ":139681,"ÙħسأÙĦØ©":139682,"^^":139683,"çãeste":139684,"éviter":139685,"ĠконÑģÑĤÑĢÑĥк":139686,"ĠконÑģÑĤÑĢÑĥкÑĨи":139687,"ï¾ŀ":139688,"Ġת×ķ׼׳":139689,"ãĤ¹ãĥĪãĥ¬ãĤ¹":139690,"ĠاÙĦاÙĤتصادÙĬ":139691,"×ŀ×ĵ×Ļ":139692,"ĠwÅĤad":139693,"ĠwÅĤadz":139694,"Ø®ÙĪÙģ":139695,"ĠмаÑĤеÑĢиалов":139696,"ãģ¨ãģ£ãģ¦ãĤĤ":139697,"Ġznajdu":139698,"ĠznajdujÄħ":139699,"ÙģØ¦Ø©":139700,"ãģ©ãģ®ãĤĪãģĨãģª":139701,"æĬijãģĪ":139702,"׳×Ĺ׾":139703,"Ġdüny":139704,"Ġdünyan":139705,"Ġdünyanın":139706,"гÑĢани":139707,"гÑĢаниÑĩ":139708,"Ġ×Ķש׾×Ļש×Ļ":139709,"Ġ×Ķ×IJש":139710,"åıĬãģ³":139711,"ìĭŃìĭľ":139712,"ìĭŃìĭľìĺ¤":139713,"Ġдолл":139714,"ĠдоллаÑĢ":139715,"ĠповÑĤоÑĢ":139716,"Ġ×Ĺ×Ļ׳×Ŀ":139717,"תפת×Ĺ":139718,"Ñĥвели":139719,"ÑĥвелиÑĩен":139720,"ãĤ«ãĥª":139721,"rawid":139722,"rawidÅĤow":139723,"×ķ×ķ׾":139724,"ãĥŁãĥ¥":139725,"ì½ĺ":139726,"ĠByÅĤ":139727,"ÐľÐIJ":139728,"عÙIJ":139729,"ĠÑģовеÑĢÑĪ":139730,"ĠÑģовеÑĢÑĪенно":139731,"Ġмой":139732,"Ġ×ķ׾×IJ×Ĺר":139733,"æħ£":139734,"æħ£ãĤĮ":139735,"ØŃاÙ쨏":139736,"Ġ무ë£Į":139737,"à¸Ħà¸ĵะà¸ģรรม":139738,"à¸Ħà¸ĵะà¸ģรรมà¸ģาร":139739,"Ġìĸ´ëĶĶ":139740,"Ġdiferen":139741,"Ġdiferença":139742,"ĠاÙĦأساس":139743,"ĠاÙĦأساسÙĬØ©":139744,"Ġ׾×IJ×Ĺר×ķ׳×Ķ":139745,"ê·ł":139746,"Ġ×Ķש׳×Ļ×Ļ×Ķ":139747,"ìľĦìĽIJìŀ¥":139748,"ลุà¸ģ":139749,"çiler":139750,"Ġ×Ķ×IJ׾×ķ":139751,"èģŀãģı":139752,"Ġ×ķ×IJפ×Ļ׾×ķ":139753,"ĠÑĢеализ":139754,"ĠÑĢеализаÑĨи":139755,"ระยะà¹Ģวลา":139756,"ĠجداÙĭ":139757,"تباع":139758,"ĠvehÃŃculo":139759,"Ġдолг":139760,"à¸Ľà¸£à¸´à¸¡à¸²à¸ĵ":139761,"ì¦IJ":139762,"Ġ׾×ŀ×§×ķ×Ŀ":139763,"ĠìĤ¬ì§Ħ":139764,"à¸Ĭà¹īา":139765,"Ġ×ŀ×¢×ķ׾×Ķ":139766,"Ġgörm":139767,"Ġgörmek":139768,"ĠÙĪÙĩذÙĩ":139769,"пеÑĢв":139770,"пеÑĢвÑĭÑħ":139771,"ê·¸ëŀĺ":139772,"ĠاÙĦبرÙĬØ·":139773,"ĠاÙĦبرÙĬطاÙĨÙĬ":139774,"ĠиÑİнÑı":139775,"ĠÐĵоÑĢ":139776,"Ġ׾ש׾×Ŀ":139777,"ÐIJÐĿ":139778,"ĠназнаÑĩен":139779,"ооÑĢ":139780,"ооÑĢÑĥж":139781,"Ġözelli":139782,"ĠözelliÄŁi":139783,"Ġниже":139784,"ç¶ļãģijãģ¦":139785,"ĠаÑĢенд":139786,"Ġkatılı":139787,"Ġkatılım":139788,"ĠإطÙĦاÙĤ":139789,"ĠÙĪØ¥Ø°Ø§":139790,"ĠокÑĤÑı":139791,"ĠокÑĤÑıбÑĢÑı":139792,"à¹Ĥà¸ķà¹":139793,"à¹Ĥà¸ķà¹Ĭ":139794,"à¹Ĥà¸ķà¹Ĭะ":139795,"Ġoldukları":139796,"ÙħÙĪÙĤع":139797,"ëĤ©":139798,"ã썿ĢĿãģ£ãģ¦ãģĦãĤĭ":139799,"Ġש×Ļ׼×ķ׾":139800,"วาà¸Ķ":139801,"سÙĬÙĦ":139802,"à¸Ĥวั":139803,"à¸Ĥวัà¸į":139804,"تØŃÙĥÙħ":139805,"ìĤŃ":139806,"Ġconnaît":139807,"×ł×¤×ª×Ĺ":139808,"Ġchặ":139809,"Ġchặn":139810,"ĠÙħØŃÙħ":139811,"ĠÙħØŃÙħÙĪØ¯":139812,"ãģ´":139813,"ĠпÑĢодÑĥкÑĨии":139814,"здÑĢав":139815,"ãģĶè¦":139816,"ãģĶ覧":139817,"×IJ×ij×IJ":139818,"Ġvéritable":139819,"ĠØ·ÙģÙĦ":139820,"ãĥĪãĥ©ãĥĸãĥ«":139821,"곡":139822,"Ġת×ŀ×ķ׳×Ķ":139823,"Ġkiên":139824,"ĠÙĤادر":139825,"Ø¥ÙĤÙĦÙĬÙħ":139826,"ĠпÑĢедпÑĢи":139827,"ĠпÑĢедпÑĢиÑıÑĤиÑı":139828,"ĠbÄĥng":139829,"Ġayında":139830,"Ġgấp":139831,"еÑħал":139832,"ĠgiÃłnh":139833,"Ġдав":139834,"Ġдавно":139835,"ìĺĢëĭ¤":139836,"à¸Ļัà¸ģà¹Ģà¸ķ":139837,"à¸Ļัà¸ģà¹Ģà¸ķะ":139838,"Ùħستشار":139839,"ستراتÙĬج":139840,"ستراتÙĬجÙĬ":139841,"رÙħز":139842,"ĠtÄ©nh":139843,"ë¡Ń":139844,"ĠÑĩеÑĤ":139845,"ĠÑĩеÑĤÑĭ":139846,"ĠÑĩеÑĤÑĭÑĢе":139847,"ĠEntão":139848,"Ġصغ":139849,"ĠصغÙĬرة":139850,"×ij×Ļ×ĺ×ķ׾":139851,"خطÙĪØ·":139852,"ĠÑĢазвиÑĤие":139853,"Ġamacıyla":139854,"à¸Ĺีวี":139855,"ĠоÑģÑĤ":139856,"ĠоÑģÑĤалÑĮн":139857,"ש×ķ׾×Ĺף":139858,"Ġ׼׳×Ļס":139859,"Ġ׼׳×Ļס×Ķ":139860,"ĠdáºŃy":139861,"ĠyaÅŁayan":139862,"Ġ×ŀ×Ķ×ķ×ķ×Ķ":139863,"ĠÑĥÑģи":139864,"ĠÑĥÑģили":139865,"×ŀפ×Ļ":139866,"ĠпÑĢоведениÑı":139867,"Ġرب":139868,"ĠربÙħا":139869,"ĠاÙĦØ£ÙĪØ³Ø·":139870,"Ġìľłì§Ģ":139871,"Ġpracownik":139872,"Ġpracowników":139873,"×ŀס×ķרת":139874,"ÙĤارب":139875,"à¸Ħวามรูà¹īสึà¸ģ":139876,"à¹ģหละ":139877,"ĠاÙĦÙĨÙĤد":139878,"Ġ×IJ׾פ×Ļ":139879,"Ùħسئ":139880,"ÙħسئÙĪÙĦ":139881,"евÑĭÑħ":139882,"клÑİÑĩениÑı":139883,"×ij×Ļ׳":139884,"×ij×Ļ׳×Ļ×Ķ×Ŀ":139885,"ש×ķ×IJ×Ķ":139886,"ĠÅŁark":139887,"ĠÅŁarkı":139888,"Ġsürec":139889,"Ġsürecin":139890,"à¹Ģà¸Ħรà¸Ķ":139891,"à¹Ģà¸Ħรà¸Ķิà¸ķ":139892,"ãĥIJãĥ¬":139893,"ĠشأÙĨ":139894,"à¹Ģà¸Ńาà¹Ħวà¹ī":139895,"niÄĻcie":139896,"רצ×Ĺ":139897,"ĠaÅŁama":139898,"׳פ×Ĵ×¢":139899,"Ġthá»Ŀ":139900,"Ġkhuẩn":139901,"diÄŁinde":139902,"ÑıÑīиÑħ":139903,"ãĥĺãĥ«":139904,"Ġüberh":139905,"Ġüberhaupt":139906,"ĠÑĤÑĢебова":139907,"ĠdÅĤugi":139908,"×ĺ×Ļף":139909,"à¸Ĥà¸Ļาà¸Ķà¹ĥหà¸įà¹Ī":139910,"ĠاÙĦØ£Ùĩ":139911,"ĠاÙĦØ£ÙĩÙĦÙĬ":139912,"ĠMüd":139913,"ĠMüdürü":139914,"Ġ×Ļ×Ķ×ķ×ĵ×Ķ":139915,"ÑĭваеÑĤÑģÑı":139916,"ساط":139917,"×Ķ×ª×ł×Ķ×Ĵ":139918,"×Ķ×ª×ł×Ķ×Ĵ×ķת":139919,"à¸ģà¸²à¸£à¸ľà¸¥à¸´à¸ķ":139920,"íĴĢ":139921,"สà¸ĸาà¸Ļà¸ģารà¸ĵà¹Į":139922,"ĠоÑĦ":139923,"ĠоÑĦиÑģ":139924,"ĠÙĦعبة":139925,"ĠstronÄĻ":139926,"Ġר×IJ×ķ×Ļ":139927,"×Ĺ×ij׾":139928,"ĠÑĢÑĭн":139929,"ĠÑĢÑĭнке":139930,"Ġ׾×ŀ×¢×Ł":139931,"اسÙĦ":139932,"หัà¸Ļ":139933,"Ġ×IJ×Ĺ×Ļ":139934,"ĠпÑĢодол":139935,"ê°Ģìŀħ":139936,"Ġ×ijר×Ĺ":139937,"Ġ×ijר×Ĺ×ij×Ļ":139938,"джеÑĢ":139939,"Ġ׾×Ĺ׾":139940,"Ġ׾×Ĺ׾×ķ×ĺ":139941,"Ġ׾×Ĺ׾×ķ×ĺ×Ļף":139942,"ศาสà¸Ļา":139943,"ãĤ¢ãĤ¤ãĥĨ":139944,"ãĤ¢ãĤ¤ãĥĨãĥł":139945,"Ġפר×ķפ":139946,"جزاء":139947,"ลà¸Ńย":139948,"ĠciaÅĤa":139949,"Ġgiết":139950,"ĠзнаÑĩиÑĤелÑĮно":139951,"Ġolmadıģ":139952,"Ġolmadıģını":139953,"нд":139954,"ндекÑģ":139955,"تأÙĥد":139956,"Ġìĸ¸":139957,"Ġìĸ¸ìłľ":139958,"aydın":139959,"ãĥīãĥ¬ãĤ¹":139960,"Ġsắt":139961,"Ġíĺ¸íħĶ":139962,"Ġë¶ģ":139963,"Ġë¶ģíķľ":139964,"ãĥijãĤ¤":139965,"Ġ×ŀש×Ĺ×§×Ļ":139966,"à¸Ħà¸Ļà¸Ńืà¹Īà¸Ļ":139967,"ĠизгоÑĤов":139968,"ĠизгоÑĤовлен":139969,"à¹Ģà¸ģียร":139970,"à¹Ģà¸ģียรà¸ķิ":139971,"תקשר":139972,"ĠÑĢаÑģÑĩеÑĤ":139973,"สà¹Ģà¸ķ":139974,"Ġlänger":139975,"ĠiÅŁlet":139976,"ĠiÅŁletme":139977,"ĠعÙĦÙĬÙĨ":139978,"ĠعÙĦÙĬÙĨا":139979,"élection":139980,"ĠاÙĦغربÙĬØ©":139981,"íĭĢ":139982,"ãĤĤãĤīãģĪ":139983,"Ġкниги":139984,"أسÙħ":139985,"أسÙħاء":139986,"Ġthá»ı":139987,"Ġthá»ıa":139988,"หà¸Ļู":139989,"Ġ×ł×¢×©×Ķ":139990,"à¸łà¸²à¸¢à¹ĥà¸ķà¹ī":139991,"à¸ŀืà¸Ĭ":139992,"رÙĬØ·":139993,"ÙģÙĪØ¶":139994,"ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸãģĦãģ¾ãģĹãģŁ":139995,"ש×ĵ×Ķ":139996,"Ġngá»±c":139997,"ĠÑģеÑĢÑĮ":139998,"ĠÑģеÑĢÑĮезн":139999,"Tôi":140000,"Ġfiyatları":140001,"ĠвÑģÑİ":140002,"ĠCódigo":140003,"Ġ×Ķש×IJ":140004,"Ġ×Ķש×IJ׾×Ķ":140005,"ĠPública":140006,"إخ":140007,"إخÙĪØ§ÙĨ":140008,"ĠзаÑıвил":140009,"ãĥ¦ãĥ¼":140010,"ר×IJ×Ļת":140011,"volución":140012,"Ġszko":140013,"ĠszkoÅĤy":140014,"جرÙĬدة":140015,"Ġpensé":140016,"ìī¬":140017,"ĠBüyükÅŁehir":140018,"ĠØ£ÙħرÙĬ":140019,"ĠØ£ÙħرÙĬÙĥÙĬ":140020,"à¸Ļัà¸ģศึà¸ģษา":140021,"Ġtodav":140022,"ĠtodavÃŃa":140023,"ĠСан":140024,"ĠСанкÑĤ":140025,"íķĺìŀIJ":140026,"ØŃÙĪØ§ÙĦ":140027,"׼×ķשר":140028,"à¹Ģลยà¸Ħรัà¸ļ":140029,"Ġalgu":140030,"Ġalguém":140031,"Ù쨲":140032,"Ġçekil":140033,"Ġ×ĵר׼×Ļ×Ŀ":140034,"ãĥIJãĥ©":140035,"à¸ģà¹ĩสามารà¸ĸ":140036,"สà¹Īวà¸Ļลà¸Ķ":140037,"íı°":140038,"ĠPúb":140039,"ĠPúblico":140040,"à¹ģà¸Ļวà¸Ĺาà¸ĩ":140041,"×IJת×Ĵר":140042,"شاش":140043,"شاشة":140044,"ciÅĽni":140045,"ĠÃľrün":140046,"ÙĦÙĪØŃ":140047,"ĠاÙĦبÙĨ":140048,"ĠاÙĦبÙĨÙĥ":140049,"ì¡°ì¹ĺ":140050,"Ġorganización":140051,"ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸãģĦãģ¾ãģĻ":140052,"sätze":140053,"ĠÑģемей":140054,"ÙĤصد":140055,"ÑģÑĤвеннÑĭе":140056,"Ġprécéd":140057,"Ġprécédent":140058,"à¸ģรุà¸ĩà¹Ģà¸Ĺà¸ŀฯ":140059,"ãģ¨è¨ĢãģĦ":140060,"×ij׳×Ļ×Ļף":140061,"ĠØŃÙĪ":140062,"ĠØŃÙĪØ§ÙĦÙĬ":140063,"סקס":140064,"ĠsaÄŁlamak":140065,"Ġ׾צ×Ļ×Ļף":140066,"×§×ĵש":140067,"Ġ×Ķ×ŀ×¢×¨×Ľ×ª":140068,"Ġ׾×Ķ×¢×ij×Ļר":140069,"Ġgünd":140070,"Ġgündem":140071,"ĠнаÑĪего":140072,"à¹ĥà¸Ļà¸ŀืà¹īà¸Ļà¸Ĺีà¹Ī":140073,"à¹Ģà¸Ħรืà¸Ń":140074,"à¹Ģà¸Ħรืà¸Ńà¸Ĥ":140075,"à¹Ģà¸Ħรืà¸Ńà¸Ĥà¹Īาย":140076,"ظاÙĩرة":140077,"ÙħÙĨظÙħ":140078,"ÙħÙĨظÙħات":140079,"Ùħتاز":140080,"追ãģĦ":140081,"dıkt":140082,"dıktan":140083,"ĠëįĶìļ±":140084,"ĠÐĿапÑĢимеÑĢ":140085,"twór":140086,"×ŀ×ķעצ×Ķ":140087,"ÙĥÙĪÙĥ":140088,"Щ":140089,"×ŀ×ĺפ׾":140090,"ólica":140091,"訪ãĤĮ":140092,"ĠëĮĢë¶Ģ":140093,"ĠëĮĢë¶Ģë¶Ħ":140094,"ãĤ¯ãĥªãĥĥãĤ¯":140095,"ãĤĴéģ¸":140096,"ãĤĴéģ¸ãģ¶":140097,"Ġpowsta":140098,"ĠpowstaÅĤ":140099,"Ġrazón":140100,"×ij×ķ×Ĺר":140101,"ĠÑģообÑīил":140102,"Ġ×§×ij×ķ×¢":140103,"rêt":140104,"à¸Ķีà¸Ĥึà¹īà¸Ļ":140105,"×ŀסע×ĵ":140106,"×ŀסע×ĵ×ķת":140107,"ĠÃĸsterreich":140108,"Ġ׳×Ĺש×ij":140109,"Ùħبادرة":140110,"ì´ī":140111,"×Ĵ׳×ĺ×Ļ":140112,"ä¿¡ãģĺ":140113,"duÄŁ":140114,"duÄŁunu":140115,"Ġphú":140116,"ĠاÙĦأخÙĬر":140117,"Ġتعتبر":140118,"landırıl":140119,"ãģ¨ãģ¯ãģĦ":140120,"ãģ¨ãģ¯ãģĦãģĪ":140121,"ĠاÙĦØ·ÙĦ":140122,"ĠاÙĦØ·ÙĦاب":140123,"ĠNº":140124,"éģ¿ãģij":140125,"اÙĦÙħع":140126,"اÙĦÙħعرÙĪÙģ":140127,"à¸ªà¸łà¸²":140128,"éĽ¢ãĤĮ":140129,"ĠпомоÑīÑĮ":140130,"ĠзнаеÑĤ":140131,"ãĥĹãĥ¬ãĤ¼":140132,"ãĥĹãĥ¬ãĤ¼ãĥ³ãĥĪ":140133,"Ġsupérieur":140134,"Ġש׾×Ļש×Ļ":140135,"ĠاÙĦÙĨÙĪØ¹":140136,"ãĤĵãģ§ãģĻãģŃ":140137,"à¸Ńà¸ļรม":140138,"Ġgiá»įng":140139,"ĠwzglÄĻd":140140,"ĠاÙĦÙģÙĤر":140141,"èrent":140142,"Ġ×ŀ×IJ×Ĺ":140143,"Ġ×ŀ×IJ×Ĺ×ķר×Ļ":140144,"×Ĵ×Ĵ":140145,"×Ļ×Ļ×ij":140146,"ÙħÙĦاب":140147,"ÙħÙĦابس":140148,"Ġhükü":140149,"Ġhükümet":140150,"Ġ×ŀ×Ĵ×Ļ×ij":140151,"ĠÐŀÑĩ":140152,"ĠÐŀÑĩенÑĮ":140153,"æĹ©ãģĦ":140154,"Ġconstrucción":140155,"Ġthượng":140156,"ï¼ĭ":140157,"Ġcoração":140158,"à¹Ģหลà¹ĩà¸ģ":140159,"ĠBaÅŁb":140160,"ĠBaÅŁbakan":140161,"éĢ£ãĤĮ":140162,"ãģĻãĤĭãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ":140163,"ĠÙĤاÙħت":140164,"ĠاÙĥثر":140165,"ÙģØ§Ø¹ÙĦ":140166,"ĠÑĦоÑĢ":140167,"ĠÑĦоÑĢÑĥм":140168,"غذÙĬ":140169,"ĠiÅŁle":140170,"ĠiÅŁleml":140171,"ĠiÅŁlemleri":140172,"ĠìĤ¬ëŀĮìĿĢ":140173,"ĠìŀijìĦ±":140174,"Ġë§Ī볨":140175,"ÙħجÙĦس":140176,"หมู":140177,"дв":140178,"двиг":140179,"двига":140180,"à¹Ģสียà¸Ĭีวิà¸ķ":140181,"×Ķתפת×Ĺ":140182,"×Ķתפת×Ĺ×ķת":140183,"ĠмеÑĤÑĢо":140184,"ĠÑģенÑĤ":140185,"ĠÑģенÑĤÑı":140186,"ĠÑģенÑĤÑıбÑĢÑı":140187,"ê³§":140188,"Ġ×ľ×¤×¢":140189,"Ġ×ľ×¤×¢×ŀ×Ļ×Ŀ":140190,"à¹Ģà¸ļีย":140191,"詳ãģĹãģı":140192,"çķ°ãģªãĤĭ":140193,"Ġİlçe":140194,"ĠAtat":140195,"ĠAtatür":140196,"ĠAtatürk":140197,"รุà¹Īà¸ĩ":140198,"Ġkaldı":140199,"Ġ주ìŀ¥":140200,"Ġprésence":140201,"Ġнаб":140202,"ĠнаблÑİ":140203,"ĠнаблÑİда":140204,"ĠÑģамого":140205,"×Ĵ×ķש":140206,"×ŀ×ĺ×ķפ":140207,"×ŀ×ĺ×ķפ׾":140208,"ĠвÑĭбиÑĢа":140209,"ĠìŀIJ리":140210,"åĪĨãģĭãĤīãģªãģĦ":140211,"ĠзÑĥб":140212,"Ġש׼×ijר":140213,"Ġدائ":140214,"ĠدائÙħا":140215,"ĠпаÑĢÑĤи":140216,"ï¼²":140217,"ĠاÙĬضا":140218,"ĠÑħоз":140219,"ĠÑħозÑı":140220,"ĠÑħозÑıй":140221,"ĠÑħозÑıйÑģÑĤв":140222,"ĠاÙĦأج":140223,"ĠاÙĦأجÙĨب":140224,"ĠاÙĦأجÙĨبÙĬØ©":140225,"ĠÐĹна":140226,"ĠApós":140227,"ĠÑįнеÑĢ":140228,"ĠÑįнеÑĢги":140229,"Ġyans":140230,"Ġyansı":140231,"ĠJusti":140232,"ĠJustiça":140233,"Ġprévu":140234,"มวล":140235,"ìŀ¥ëĭĺ":140236,"à¸ģระà¸ļ":140237,"à¸ģระà¸ļวà¸Ļ":140238,"à¸ģระà¸ļวà¸Ļà¸ģาร":140239,"×ŀ×ŀ":140240,"×ŀ×ŀ×ķצע":140241,"Ġhẹ":140242,"Ġhẹn":140243,"здание":140244,"ĠakÅŁ":140245,"ĠakÅŁam":140246,"×ĺ×ķפ":140247,"Ġgerekt":140248,"Ġgerekti":140249,"ĠgerektiÄŁini":140250,"Ġnarz":140251,"ĠnarzÄĻdzi":140252,"épo":140253,"époque":140254,"ĠThần":140255,"Ġwysoko":140256,"ĠwysokoÅĽci":140257,"à¸ľà¸¹à¹īà¸Ľ":140258,"à¸ľà¸¹à¹īà¸Ľà¹Īวย":140259,"ĠÙĬبدÙĪ":140260,"ÑĤелÑĮного":140261,"ĠвзглÑıд":140262,"ĠjednÄħ":140263,"ĠìĿĺ견":140264,"Ġà¸Ĥà¸ĵะà¸Ĺีà¹Ī":140265,"פ×Ļ×ĵ":140266,"ìĥģëĭ´":140267,"Ġmỡ":140268,"×Ķ×ŀ׾":140269,"×Ķ×ŀ׾צ×ķת":140270,"ĠÑģоÑģÑĤо":140271,"ĠÑģоÑģÑĤоиÑĤ":140272,"Ġави":140273,"Ġавиа":140274,"ĠLänder":140275,"تصÙĪÙĬر":140276,"×ŀ×ĵ×Ļ×Ķ":140277,"ìłĪì°¨":140278,"ãģ¨ãĤĬ":140279,"ãģ¨ãĤĬãģĤ":140280,"ãģ¨ãĤĬãģĤãģĪ":140281,"ãģ¨ãĤĬãģĤãģĪãģļ":140282,"ĠÑĢÑıд":140283,"ĠÑĢÑıдом":140284,"ĠNhất":140285,"ĠاÙĦÙĥاÙħÙĦ":140286,"×Ĺ׾׾":140287,"ĠGiấy":140288,"צ×ĺר":140289,"צ×ĺרף":140290,"Ġ׾×ij×ĺ׾":140291,"ĠимеÑĤÑĮ":140292,"ס×ŀ×ķ×ļ":140293,"Ġparticipação":140294,"íķľëĭ¤ë©´":140295,"ÙħÙĨتدÙĬ":140296,"ÙħÙĨتدÙĬات":140297,"ĠeÄŁlen":140298,"gänge":140299,"ربØŃ":140300,"ãĤ®ãĥ£":140301,"ĠاÙĦرÙĤÙħ":140302,"à¸ĭà¹īำ":140303,"ĠHóa":140304,"×ŀר×Ĺ×§":140305,"ØŃÙħاÙħ":140306,"بÙĪÙĥ":140307,"ĠArtÃŃculo":140308,"ãĥĦãĤ¢ãĥ¼":140309,"×Ķפ׼×Ķ":140310,"×Ĺ׾×ķף":140311,"ĠпеÑĢеÑħод":140312,"lenmiÅŁ":140313,"زراعة":140314,"Ġseñor":140315,"ãģ£ãģ¦ãģįãģ¦":140316,"إش":140317,"إشارة":140318,"ĠpodÃŃa":140319,"ĠÃľlke":140320,"нÑģкаÑı":140321,"Ġadapté":140322,"Ġdüzenlen":140323,"Ġdüzenlenen":140324,"ĠÑģÑĤала":140325,"ĠÙĬØŃتاج":140326,"Ġnier":140327,"Ġnieruch":140328,"Ġnieruchomo":140329,"ĠnieruchomoÅĽci":140330,"ãģĵãģ¨ãģĮãģĤãĤĭ":140331,"ยà¸Ńà¸Ķà¹Ģยีà¹Īยม":140332,"ĠÙħج":140333,"ĠÙħجاÙĨÙĬ":140334,"Ġзаб":140335,"Ġзабол":140336,"Ġзаболев":140337,"ĠзаболеваниÑı":140338,"ĠÅĽro":140339,"ĠÅĽrodk":140340,"ĠÅĽrodków":140341,"Ġ×Ķ׾×IJ×ķ×ŀ×Ļ":140342,"ĠdokÅĤad":140343,"ĠdokÅĤadnie":140344,"ãģŁãģıãģªãģĦ":140345,"ãģ¯ãģļãģ§ãģĻ":140346,"ã썿ĢĿãģ£ãģ¦ãģĦãģŁ":140347,"écran":140348,"ìĹħì²´":140349,"trzymaÅĤ":140350,"ÑģÑĤвеннÑĭй":140351,"ĠNotÃŃc":140352,"ĠNotÃŃcias":140353,"ÙħرÙĬ":140354,"ÙħرÙĬض":140355,"æ°Ĺè»":140356,"æ°Ĺ軽":140357,"æ°Ĺ軽ãģ«":140358,"ëĵ£":140359,"Ġ×ĵ×ķ×IJר":140360,"Ġ׾×ŀ׳":140361,"Ġ׾×ŀ׳×ķ×¢":140362,"ĠçalÄ±ÅŁÄ±yor":140363,"ĠÅŁidd":140364,"ĠÅŁiddet":140365,"ĠMặt":140366,"ĠateÅŁ":140367,"ĠполÑĥÑĩениÑı":140368,"à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩมืà¸Ń":140369,"ĠgrÃ¶ÃŁer":140370,"دائ":140371,"دائرة":140372,"Ġbulun":140373,"Ġbulunmaktadır":140374,"à¹Ģหร":140375,"à¹Ģหรีย":140376,"à¹Ģหรียà¸į":140377,"à¸Ļัà¸ģà¸Ĺà¹Īà¸Ńà¸ĩà¹Ģà¸Ĺีà¹Īยว":140378,"Ġalanında":140379,"ĠÑĥзна":140380,"ĠлеÑĩение":140381,"売ãĤĮ":140382,"Ġçevir":140383,"ĠdesteÄŁi":140384,"ĠheiÃŁt":140385,"âĸ²":140386,"ØŃØ·":140387,"à¸Ħำà¸ķà¸Ńà¸ļ":140388,"ãĤªãĥ³ãĥ©ãĤ¤ãĥ³":140389,"Ġ×ij×Ĺ×Ļ×Ļ×Ŀ":140390,"ãĥ¦ãĥĭ":140391,"Ġdüzenleme":140392,"ĠmodalitÃł":140393,"سرط":140394,"سرطاÙĨ":140395,"×ŀ׼×ķף":140396,"ĠданнÑĭй":140397,"ترت":140398,"ترتÙĬب":140399,"à¸ļาà¸ĩà¸Ħà¸Ļ":140400,"ĠÄIJá»ĭnh":140401,"มูล":140402,"มูลà¸Ħà¹Īา":140403,"ÙĨÙĤص":140404,"à¸ģารรัà¸ģษา":140405,"ĠÑĦон":140406,"ĠÑĦонд":140407,"ãĤĪãģĨãģ«ãģªãģ£ãģŁ":140408,"ÙħعاÙĦ":140409,"ÙħعاÙĦجة":140410,"ĠOsman":140411,"ĠOsmanlı":140412,"иÑĩеÑģком":140413,"à¸Ńยาà¸ģà¸Īะ":140414,"ãģķãģ¾ãģĸ":140415,"ãģķãģ¾ãģĸãģ¾":140416,"ãģķãģ¾ãģĸãģ¾ãģª":140417,"Ġת×ķ׼׾":140418,"עצ×ij":140419,"ĠاÙĦعسÙĥ":140420,"ĠاÙĦعسÙĥرÙĬ":140421,"Ġvéhic":140422,"Ġvéhicule":140423,"Ġ×Ļצ×Ĺ×§":140424,"ĠاÙĦÙĪØŃ":140425,"ĠاÙĦÙĪØŃÙĬد":140426,"ĠاÙĦعدÙĪ":140427,"ĠQuản":140428,"Ġê³µëıĻ":140429,"بدÙĦ":140430,"ĠÄijảng":140431,"Ġmá»ĩnh":140432,"Ġniezb":140433,"ĠniezbÄĻ":140434,"ĠniezbÄĻdn":140435,"Ġyayınlan":140436,"обÑīи":140437,"Ġgötür":140438,"צפ":140439,"צפ×ķ×Ļ":140440,"ĠÙĦÙĬبÙĬ":140441,"ĠÙĦÙĬبÙĬا":140442,"ØŃÙĪØ§":140443,"Ġдоб":140444,"ĠдобÑĢо":140445,"иÑĢÑĥем":140446,"ĠاÙĦØŃÙĥÙĪÙħÙĬØ©":140447,"mÃ¤ÃŁig":140448,"Ġedición":140449,"влекаÑĤелÑĮ":140450,"влекаÑĤелÑĮн":140451,"Ġ×ª×©×ľ×ķ×Ŀ":140452,"Ġ×Ķש×ķ׳×Ļ×Ŀ":140453,"มิà¸ĸุ":140454,"มิà¸ĸุà¸Ļ":140455,"มิà¸ĸุà¸Ļายà¸Ļ":140456,"é£Łãģ¹ãģ¦":140457,"ĠìĪĺì§ij":140458,"ס×ij×Ļ":140459,"ĠиÑİлÑı":140460,"Ġà¹Ħà¸Ķà¹īà¹ģà¸ģà¹Ī":140461,"׾×Ĺ×Ŀ":140462,"trä":140463,"trägt":140464,"ãģĿãĤĤãģĿãĤĤ":140465,"ÐĿÐķ":140466,"ĠвнÑĥÑĤ":140467,"ĠвнÑĥÑĤÑĢи":140468,"ãģ¨ä¸Ģç·Ĵãģ«":140469,"ãĤ«ãĥķãĤ§":140470,"Ġ×ij×Ĺ×ĵר":140471,"×Ĺ×ŀש":140472,"ãĤ¨ãĥį":140473,"ãĤ¨ãĥįãĥ«":140474,"ãĤ¨ãĥįãĥ«ãĤ®":140475,"ãĤ¨ãĥįãĥ«ãĤ®ãĥ¼":140476,"à¸Ĥà¸Ńà¸ĩà¸ķัวà¹Ģà¸Ńà¸ĩ":140477,"بÙĤاء":140478,"פס×Ļ׼":140479,"פס×Ļ׼×ķ׾×ķ×Ĵ":140480,"ãĥ¡ãĥĥ":140481,"ãĥ¡ãĥĥãĤ»":140482,"ãĥ¡ãĥĥãĤ»ãĥ¼ãĤ¸":140483,"ÙĦÙĤب":140484,"AÄŀ":140485,"שק×Ļ×¢":140486,"ÙĤساÙħ":140487,"×ĵ×ķ×Ĵ×ŀ×Ķ":140488,"æ·±ãģĦ":140489,"íĸĪëĬĶëį°":140490,"ĠrozwiÄħzanie":140491,"à¸Ļัà¹Īà¸Ļà¹Ģà¸Ńà¸ĩ":140492,"×Ļצ×ij":140493,"Ġtrông":140494,"à¹ĥà¸Ĭà¹īà¸ļริà¸ģาร":140495,"ĠاÙĦÙħÙĪØ³Ùħ":140496,"ĠдеÑĤи":140497,"ãģĹãģĭãģªãģĦ":140498,"ס×Ļף":140499,"Ġréférence":140500,"à¹ģหà¹īà¸ĩ":140501,"ãĤĤãĤīãģ£ãģŁ":140502,"Ġ׾ר׼":140503,"Ġ׾ר׼×ķש":140504,"شعÙĪØ±":140505,"ĠÐijог":140506,"Ġlazım":140507,"Ġ×Ļש׳×Ŀ":140508,"ĠпаÑĢÑĤ":140509,"ĠпаÑĢÑĤнеÑĢ":140510,"ĠÑĥника":140511,"ĠÑĥникалÑĮн":140512,"Ġmatériel":140513,"×ŀרק":140514,"Ġphưá»Ŀng":140515,"Ġзай":140516,"Ġзайм":140517,"ÙģÙĤد":140518,"UniversitÃł":140519,"×¢×¨×Ľ×Ļ×Ŀ":140520,"Ġbaño":140521,"ĠноÑı":140522,"ĠноÑıбÑĢÑı":140523,"à¸Ľà¹īาย":140524,"Ġtats":140525,"Ġtatsäch":140526,"Ġtatsächlich":140527,"ĠÑĤÑĢеÑĤÑĮ":140528,"Ñįм":140529,"ãĥĻãĥ¼ãĤ¹":140530,"Ġnhá»±a":140531,"ìĬ¤íģ¬":140532,"ĠعبداÙĦÙĦÙĩ":140533,"Ġת×ķר×Ķ":140534,"أشÙĬ":140535,"أشÙĬاء":140536,"ĠÙĦÙĦغا":140537,"ĠÙĦÙĦغاÙĬØ©":140538,"ÙħÙĪØ§ÙĤ":140539,"ÙħÙĪØ§ÙĤÙģ":140540,"ĠgÅĤówna":140541,"ĠartÄ±ÅŁ":140542,"Ġ×ŀ×§×ķ×ŀ×Ļ":140543,"ãĤ¯ãĥ©ãĥĸ":140544,"ĠسÙĪÙī":140545,"ĠìŬìĦ±":140546,"اسر":140547,"اسرائÙĬÙĦ":140548,"Ġ×ł×Ľ×ª×ij":140549,"ยà¹īà¸Ńà¸Ļ":140550,"Ġdeberá":140551,"Ġphẫu":140552,"ÑİÑīем":140553,"ĠÙĦدÙĬÙĨا":140554,"×ŀ×ĺ×Ķ":140555,"Ġ׳×ķ׾×ĵ":140556,"ĠвÑģÑĤÑĢеÑĩа":140557,"ãĤīãĤĮãģ¦ãģĦãģ¾ãģĻ":140558,"ĠcaÅĤej":140559,"ยึ":140560,"ยึà¸Ķ":140561,"поÑĤен":140562,"поÑĤенÑĨи":140563,"ĠлиÑĤ":140564,"ĠлиÑĤеÑĢ":140565,"ĠлиÑĤеÑĢаÑĤÑĥÑĢ":140566,"Ġкаждом":140567,"ĠíĮIJ":140568,"ĠíĮIJëĭ¨":140569,"à¸Īู":140570,"Ġpresença":140571,"ãģªãĤĵãģ§":140572,"ÙħÙĬاÙĩ":140573,"инÑĦоÑĢм":140574,"инÑĦоÑĢмаÑĨион":140575,"инÑĦоÑĢмаÑĨионн":140576,"ĠìŀIJìŰ":140577,"ר׼ש":140578,"Ġödül":140579,"ç¶ļãģı":140580,"ĠпÑģ":140581,"ĠпÑģиÑħ":140582,"ĠпÑģиÑħолог":140583,"تذÙĥر":140584,"Ġìŀħìŀ¥":140585,"ลà¸Ķà¹Į":140586,"ìĦłê±°":140587,"ãģ£ãģ¦ãģĬãĤĬãģ¾ãģĻ":140588,"Ġ×Ļ×¢":140589,"Ġ×Ļ×¢×§×ij":140590,"ĠاÙĦطعاÙħ":140591,"ãĥĨãĤ¹ãĥĪ":140592,"ĠTuấn":140593,"Ġparticipación":140594,"×ŀ×ķ×ŀ×Ĺ×Ķ":140595,"×Ĵרס×Ķ":140596,"ĠاÙĦتÙĨÙģÙĬ":140597,"ĠاÙĦتÙĨÙģÙĬذÙĬ":140598,"ĠбезопаÑģн":140599,"gef":140600,"gefähr":140601,"Ø´ÙĪØ±":140602,"ĠmyÅĽli":140603,"ÙĪØ§Ø´ÙĨ":140604,"ÙĪØ§Ø´ÙĨØ·ÙĨ":140605,"׳×ķסע":140606,"ÙĥÙĩ":140607,"ÙĥÙĩرب":140608,"ÙĥÙĩرباء":140609,"ĠmusiaÅĤ":140610,"ìĭ¸":140611,"ãĥĸãĥ©ãĥĥãĤ¯":140612,"Ġcréé":140613,"ÙĨÙĩار":140614,"owoÅĽÄĩ":140615,"ÙħØŃاÙĥÙħ":140616,"ĠwÅĤaÅĽ":140617,"ĠwÅĤaÅĽc":140618,"ĠwÅĤaÅĽciciel":140619,"ĠÙĬؤ":140620,"ĠÙĬؤدÙĬ":140621,"×ŀ×¢×ķ׳":140622,"×IJ×ij׾":140623,"خطأ":140624,"ĠÑħолод":140625,"×ĸ×ķ׾":140626,"ãģĵãĤĮãĤī":140627,"ãģĵãĤĮãĤīãģ®":140628,"Ġbásica":140629,"ฤà¸Ķ":140630,"ฤà¸Ķูà¸ģ":140631,"ฤà¸Ķูà¸ģา":140632,"ฤà¸Ķูà¸ģาล":140633,"èIJ½ãģ¡çĿĢ":140634,"ãģªãģĦãģĵãģ¨":140635,"صÙĪÙħ":140636,"ÙĨجØŃ":140637,"׳ק×ķ×ĵ":140638,"׳ק×ķ×ĵת":140639,"клаÑģÑģ":140640,"íķĺìĭľëĬĶ":140641,"ëĦĺ":140642,"Ġש×IJ×Ļ׳×ķ":140643,"ĠСейÑĩаÑģ":140644,"mayacaģı":140645,"Ġyapılır":140646,"ĠcategorÃŃa":140647,"عباد":140648,"ĠТеп":140649,"ĠТепеÑĢÑĮ":140650,"×Ķ×Ļס×ĺ×ķר×Ļ":140651,"hế":140652,"ãĤ³ãĥ¼ãĥī":140653,"Ġcabeça":140654,"جÙħا":140655,"جÙħاÙĩ":140656,"جÙħاÙĩÙĬر":140657,"ä½İãģĦ":140658,"ĠÑĤоваÑĢов":140659,"à¸Ĭาวà¸ļà¹īาà¸Ļ":140660,"ĠÑģÑĤанов":140661,"ĠÑģÑĤановиÑĤÑģÑı":140662,"ĠавÑĤомобилÑĮ":140663,"ĠÑģлÑĥÑĩай":140664,"à¸Ńัà¸ŀ":140665,"ĠGiriÅŁ":140666,"ĠìĿ¼ëĭ¨":140667,"ĠпÑĢоÑģ":140668,"ĠпÑĢоÑģмоÑĤÑĢ":140669,"ãģªãģıãģªãģ£ãģŁ":140670,"à¸¡à¸µà¸Ľà¸±à¸įหา":140671,"ïºİ":140672,"écoute":140673,"ĠÙħÙĪØ¬ÙĪØ¯":140674,"ĠسرÙĬع":140675,"ĠÙĪÙĩÙĨا":140676,"ĠÙĪÙĩÙĨاÙĥ":140677,"à¸Ħุà¸ĵสม":140678,"à¸Ħุà¸ĵสมà¸ļัà¸ķิ":140679,"Ġìļ°ìĦł":140680,"à¸ŀระà¸ŀุà¸Ĺà¸ĺ":140681,"好ãģ¿":140682,"ظÙĦÙħ":140683,"ĠмакÑģ":140684,"ĠмакÑģималÑĮ":140685,"ĠмакÑģималÑĮно":140686,"ãĥªãĤ¢ãĥ«":140687,"à¹ģมà¹īวà¹Īา":140688,"ĠاÙĦØŃÙĪØ§Ø±":140689,"ãĥĹãĥ©ãĤ¹":140690,"ĠعÙĦاÙĤØ©":140691,"ĠíĸīëıĻ":140692,"Ġgönderil":140693,"Ġlãi":140694,"ĠsaÄŁlıkl":140695,"ĠsaÄŁlıklı":140696,"ĠÑĪаг":140697,"Ġ×ij×IJר×Ķ":140698,"prowadziÄĩ":140699,"ãģĦãģıãģ¤ãģĭ":140700,"ĠبتارÙĬØ®":140701,"Ġ×ij×IJ×ķת×Ķ":140702,"Ġmóc":140703,"ĠÐľÐ½Ðµ":140704,"ãĥĹãĥ¬ãĥ¼":140705,"×IJ×ĸר×Ĺ":140706,"åł´åIJĪãģ«ãģ¯":140707,"使ãģĪ":140708,"à¹Ģรืà¸Ńà¸Ļ":140709,"ĠÐŁÐµÑĤ":140710,"ĠÐŁÐµÑĤÑĢ":140711,"ãģ«åħ¥ãĤĭ":140712,"Ùħادة":140713,"à¹Ģà¸ĩืà¹Īà¸Ńà¸Ļ":140714,"à¹Ģà¸ĩืà¹Īà¸Ńà¸Ļà¹Ħà¸Ĥ":140715,"ĠÑģоÑģÑĤоÑıние":140716,"ônica":140717,"ĠÑĦев":140718,"ĠÑĦевÑĢа":140719,"ĠÑĦевÑĢалÑı":140720,"Ġ×ķ×ĸ":140721,"Ġ×ķ×ĸ×IJת":140722,"à¸Ħริ":140723,"à¸Ħริส":140724,"ĠÐķÑīе":140725,"ãģ£ãģ¦ãģĹãģ¾ãģĦãģ¾ãģĹãģŁ":140726,"ĠпÑĢавиÑĤелÑĮ":140727,"ĠпÑĢавиÑĤелÑĮÑģÑĤв":140728,"Ġtäglich":140729,"Ġëĭ¹ìĭľ":140730,"×ŀ×ķ×¢×ŀ×ĵ":140731,"ĠдвоÑĢ":140732,"æīķ":140733,"æīķãģĦ":140734,"ĠÑģÑĤанеÑĤ":140735,"ĠвоздейÑģÑĤв":140736,"ĠвоздейÑģÑĤви":140737,"Ġfête":140738,"à¹Ģสา":140739,"תק×ķ×ķ×Ķ":140740,"Ġuyar":140741,"Ġuyarı":140742,"à¸ģลัà¸ļà¹Ħà¸Ľ":140743,"Ġgiưá»Ŀng":140744,"Ġва":140745,"ĠваÑĪи":140746,"ĠÄijáºŃu":140747,"ĠSpaÃŁ":140748,"ĠìķĦë§Ī":140749,"à¹Ħà¸Ķà¹īà¸ĩà¹Īาย":140750,"Ġ×Ķ×ŀ×ijקש":140751,"æĸ°ãģŁ":140752,"æĸ°ãģŁãģª":140753,"ılıyor":140754,"план":140755,"Ġ×Ķ×ijר×Ļ×IJ×ķת":140756,"ĠaÄŁrı":140757,"Ġsaygı":140758,"建ãģ¦":140759,"Ġnajwyż":140760,"Ġnajwyższ":140761,"سÙĬاسات":140762,"ãģĬå¾Ĺ":140763,"ĠاÙĦعÙĦÙĬ":140764,"ĠاÙĦعÙĦÙĬا":140765,"Ġcorazón":140766,"ì¹ĺë£Į":140767,"หัวà¸Ĥà¹īà¸Ń":140768,"ĠبØŃÙĬ":140769,"ĠبØŃÙĬØ«":140770,"звезд":140771,"بÙĪØ§Ø¨Ø©":140772,"ÐĽÐĺ":140773,"ÙĦازÙħ":140774,"Ġrozp":140775,"Ġrozpoc":140776,"ĠrozpoczÄĻ":140777,"触ãĤĮ":140778,"ĠاÙĦجÙħÙĩ":140779,"ĠاÙĦجÙħÙĩÙĪØ±":140780,"ĠspÄĻd":140781,"ĠspÄĻdz":140782,"วิà¸Ĺยาศาสà¸ķรà¹Į":140783,"иваеÑĤÑģÑı":140784,"Ġданной":140785,"Ġreprésente":140786,"ĠÄijá»ĭch":140787,"Ġ×¢×ŀ×ķ×§":140788,"à¸Ńัà¸Ļà¸ķร":140789,"à¸Ńัà¸Ļà¸ķราย":140790,"Ġestratég":140791,"Ġestratégia":140792,"padÅĤ":140793,"Ġвполн":140794,"Ġвполне":140795,"ĠпÑĢедоÑģÑĤавлен":140796,"×Ĺ׾×ķ×§":140797,"×Ĺ׾×ķקת":140798,"ãĤ¢ãĥĬ":140799,"ĠاÙĦغذ":140800,"ĠاÙĦغذائÙĬ":140801,"ĠÑĥзн":140802,"ĠÑĥзнаÑĤÑĮ":140803,"à¸ĭà¹īาย":140804,"å½ĵãģ¦":140805,"ØŃÙĬاء":140806,"Ġbásico":140807,"×§×ķ×ij×¢":140808,"ĠاÙĦÙħباراة":140809,"ĠاÙĦÙĩاتÙģ":140810,"Ġ׼׳×Ĵ×ĵ":140811,"à¸Ľà¸£à¸°à¸«à¸¢":140812,"à¸Ľà¸£à¸°à¸«à¸¢à¸±à¸Ķ":140813,"Ðļак":140814,"à¸Ĺีà¹Īà¸Ļà¹Īา":140815,"à¸Ĺีà¹Īà¸Ļà¹Īาสà¸Ļà¹ĥà¸Ī":140816,"ãģ¾ãģģ":140817,"ï½¢":140818,"Ñģкоп":140819,"Ġsonrasında":140820,"ĠurzÄħd":140821,"ĠurzÄħdzenia":140822,"׼×ķ×ķ׳":140823,"׼×ķ×ķ×ł×ª":140824,"Ġ׾×Ķת×ŀ×ķ×ĵ":140825,"Ġ׾×Ķת×ŀ×ķ×ĵ×ĵ":140826,"ĠÑģли":140827,"ĠÑģлиÑĪ":140828,"ĠÑģлиÑĪком":140829,"ĠÑģÑĤÑĥд":140830,"ĠÑģÑĤÑĥденÑĤ":140831,"Ġ×Ķ×ķ×ĵ":140832,"Ġ×Ķ×ķ×ĵ×¢×Ķ":140833,"ë¹Ħìļ©":140834,"à¸Ńยาà¸ģà¹ĥหà¹ī":140835,"Ġbá»ģ":140836,"ยุà¸Ĺà¸ĺ":140837,"ÐĺÐĿ":140838,"سائر":140839,"أصÙĪÙĦ":140840,"ĠاÙĦغرÙģ":140841,"ãģĵãģ¨ãĤĤãģĤãĤĬãģ¾ãģĻ":140842,"è¾¼ãģ¾ãĤĮ":140843,"ĠاÙĦسابع":140844,"Ġcá»§":140845,"ãģĦãģŁãģłãģĦãģŁ":140846,"ì§ĵ":140847,"ìĤ¬ë¬´":140848,"powiedź":140849,"تÙģÙĥ":140850,"تÙģÙĥÙĬر":140851,"иÑĢовки":140852,"ĠíĨµíķ´ìĦľ":140853,"ãĤ¨ãĤ¹ãĥĨ":140854,"ĠдеÑıÑĤелÑĮноÑģÑĤÑĮ":140855,"ĠданнÑĭм":140856,"Ġ×¢×ķר":140857,"Ġ×¢×ķר׼×Ļ":140858,"×ķ×ĵעת":140859,"Ġhayatını":140860,"ĠbÄħd":140861,"ĠbÄħdź":140862,"obsÅĤug":140863,"à¹Ģà¸ŀียà¸ĩà¹ģà¸Ħà¹Ī":140864,"à¸ĭà¹Īา":140865,"è²łãģij":140866,"ĠÑģÑĤÑĢем":140867,"ĠÄijá»īnh":140868,"ĠÐłÑĥÑģ":140869,"ĠNữ":140870,"Ġ׾×Ķש×Ļ×Ĵ":140871,"Ġjednoc":140872,"Ġjednocze":140873,"ĠjednoczeÅĽnie":140874,"Ġ×Ķ×Ĵ×ij×ķ×Ķ":140875,"أخÙĦاÙĤ":140876,"ĠнаÑģел":140877,"ĠнаÑģелениÑı":140878,"ĠÙĬÙĨب":140879,"ĠÙĬÙĨبغÙĬ":140880,"ãģĮãģĭ":140881,"ãģĮãģĭãģĭ":140882,"×Ĵעת":140883,"ÐŀÐł":140884,"ĠналиÑĩии":140885,"Ġë§Īì§Ģ":140886,"Ġë§Īì§Ģë§ī":140887,"ĠíĸīìĤ¬":140888,"ĠtreÅĽci":140889,"Ġê°Ģì¹ĺ":140890,"ì¦ĺ":140891,"Ġаналог":140892,"×Ķצעת":140893,"влад":140894,"владе":140895,"ĠÑģделал":140896,"Ġ׳×Ĵ×Ļש":140897,"Ġ׳×Ĵ×Ļש×ķת":140898,"полнение":140899,"à¸Ĩà¹Īา":140900,"ĠDön":140901,"׼׾׼׾×Ķ":140902,"×ŀ×ĸ×Ĵ":140903,"ÙħÙģ":140904,"ÙħÙģÙĩ":140905,"ÙħÙģÙĩÙĪÙħ":140906,"×Ķ×ĵ":140907,"×Ķ×ĵפס":140908,"×Ķ×ĵפס×Ķ":140909,"ãģĻãģİãģ¦":140910,"ĠгÑĢ":140911,"ĠгÑĢн":140912,"×ŀ×ĺ×ķס":140913,"Ġ기ìĸµ":140914,"ï¾Ł":140915,"ĠpÅĤyn":140916,"ĠGründe":140917,"ĠBücher":140918,"ĠwedÅĤug":140919,"ãģ¾ãģłãģ¾ãģł":140920,"Ġ׳×Ķ×ĵר":140921,"ĠÙĬستطÙĬع":140922,"ĠHiá»ĩp":140923,"ãĤŃãĥ£ãĥ³ãĥļ":140924,"ãĤŃãĥ£ãĥ³ãĥļãĥ¼ãĥ³":140925,"Ġthá»ķ":140926,"Ġeuropéenne":140927,"à¸ļัà¸ĩ":140928,"à¸ļัà¸ĩà¸Ħัà¸ļ":140929,"ĠszczegóÅĤowo":140930,"׳שק":140931,"ãĥķãĥ©ãĥ³ãĤ¹":140932,"×ŀ×ķ×ŀ×Ĺ×Ļ":140933,"Ġcomún":140934,"Ġçarp":140935,"ØŃتÙĬا":140936,"ØŃتÙĬاج":140937,"ØŃتÙĬاجات":140938,"ëĭ´ëĭ¹":140939,"ä½ķ度":140940,"ä½ķ度ãĤĤ":140941,"×ĵ×ij×§":140942,"ãģįãĤĮ":140943,"ãģįãĤĮãģĦ":140944,"Ġкам":140945,"ĠкамеÑĢ":140946,"ĠespecÃŃfico":140947,"Ġteléfono":140948,"à¸ķัà¹īà¸ĩà¸Ńยูà¹Ī":140949,"IÅŀ":140950,"ãģ©ãĤĵãģ©":140951,"ãģ©ãĤĵãģ©ãĤĵ":140952,"עצ×ŀ×IJ×Ļ":140953,"à¸Ķัà¸ĩà¸Ļีà¹ī":140954,"ĠÑĦоÑĢмиÑĢов":140955,"ĠÑĦоÑĢмиÑĢова":140956,"×ķ×ŀ×ij":140957,"Ġkullanımı":140958,"ÐľÐŀ":140959,"עש×Ļ":140960,"עש×Ļ×Ļ×Ķ":140961,"Ġönlem":140962,"à¹Ģà¸Ńà¹ĩ":140963,"à¹Ģà¸Ńà¹ĩม":140964,"×ŀשק×Ļ×¢":140965,"ר×Ļ×Ĺ":140966,"à¸Ĥัà¸Ķ":140967,"ĠíĻľ":140968,"ĠíĻľìļ©":140969,"à¸ĭะ":140970,"ãĤĪãģĨãģ«ãģªãĤĬãģ¾ãģĹãģŁ":140971,"ĠÑĢаÑģпÑĢ":140972,"ĠÑĢаÑģпÑĢоÑģÑĤ":140973,"ĠÑĢаÑģпÑĢоÑģÑĤÑĢан":140974,"ĠÑĢаÑģпÑĢоÑģÑĤÑĢанен":140975,"׼×Ļ×ķף":140976,"ÙĤبض":140977,"تصرÙĬØŃ":140978,"تصرÙĬØŃات":140979,"ĠоÑĢи":140980,"ĠоÑĢиг":140981,"ĠоÑĢигина":140982,"ĠоÑĢигинал":140983,"ĠاÙĦعاÙĦÙĬ":140984,"à¹ģหà¹Īà¸ĩà¸Ļีà¹ī":140985,"ãĥķãĤ¡ãĥ¼":140986,"ãģ¦ãģĦãģį":140987,"ãģ¦ãģĦãģįãģŁãģĦ":140988,"פתר":140989,"פתר×ķ׳×ķת":140990,"Ġ×ij×Ļ×Ĺ":140991,"Ġ×ij×Ļ×Ĺ×ĵ":140992,"Ġodby":140993,"ĠodbyÅĤ":140994,"ĠоÑĩеÑĢед":140995,"Ġtrương":140996,"ãĤŃãĥ³":140997,"×ŀ×ķפ":140998,"×ŀ×ķפע":140999,"ëĵľë¦½":141000,"ëĵľë¦½ëĭĪëĭ¤":141001,"à¸ŀืà¹īà¸Ļà¸IJาà¸Ļ":141002,"ìŀIJ격":141003,"ĠViá»ĩn":141004,"ĠDespués":141005,"Ġ×IJ׾×Ļ׳×ķ":141006,"Ġdurée":141007,"íĩ´":141008,"Ġmüzik":141009,"iếu":141010,"ĠÑĢазмеÑīен":141011,"ĠкÑĥд":141012,"ĠкÑĥда":141013,"غض":141014,"غضب":141015,"ĠTambém":141016,"à¸Īัà¸Ķสà¹Īà¸ĩ":141017,"à¸ģารà¹ģสà¸Ķà¸ĩ":141018,"onomÃŃa":141019,"Ġанг":141020,"Ġангли":141021,"Ġанглий":141022,"ĠанглийÑģк":141023,"Ġznal":141024,"Ġznalaz":141025,"ĠznalazÅĤ":141026,"תר×Ĵ":141027,"תר×Ĵ×ķ×Ŀ":141028,"ĠÑģнов":141029,"ĠÑģнова":141030,"ĠÑĩаÑģа":141031,"Ġcommunauté":141032,"ĠespecÃŃfica":141033,"ĠLá»ĭch":141034,"Ġlié":141035,"ÙģØ¬Ø±":141036,"à¹Ģà¸ģà¹Īà¸ĩ":141037,"عاÙĦ":141038,"عاÙĦج":141039,"Ø£ÙĨظ":141040,"Ø£ÙĨظÙħØ©":141041,"ESİ":141042,"ĠاÙĦØŃدÙĬد":141043,"à¸ŀระà¸Ńà¸ĩà¸Ħà¹Į":141044,"Ġפרשת":141045,"Ġдвиж":141046,"ĠдвижениÑı":141047,"ĠاÙĦجارÙĬ":141048,"à¸ĺาà¸Ļี":141049,"неÑģен":141050,"ĠاÙĦÙĨÙĩائÙĬ":141051,"ĠбеÑĢ":141052,"ĠбеÑĢем":141053,"ĠбеÑĢеменн":141054,"Ġdépartement":141055,"à¹Ģà¸Ĺีย":141056,"à¹Ģà¸Ĺียà¸ļ":141057,"ĠÐľÐ°ÑĢи":141058,"ĠнекоÑĤоÑĢÑĭÑħ":141059,"обеÑģп":141060,"обеÑģпеÑĩен":141061,"×Ĺ×ķ×ĸ":141062,"×Ĺ×ķ×ĸ×Ķ":141063,"ÙĨتج":141064,"à¸Īะà¹Ħà¸Ķà¹īรัà¸ļ":141065,"á»°":141066,"Ġéléments":141067,"عط":141068,"عطاء":141069,"Ġtắt":141070,"iá»ĩm":141071,"ÑİÑīиÑħÑģÑı":141072,"ãģĹãģ°":141073,"ãģĹãģ°ãĤīãģı":141074,"ĠпоможеÑĤ":141075,"à¸Ĥà¸ĵะà¸Ļีà¹ī":141076,"Ġעשר×ķת":141077,"éģķãģ£ãģ¦":141078,"ĠпÑĢог":141079,"ĠпÑĢогн":141080,"ĠпÑĢогноз":141081,"ĠtÅĤ":141082,"ĠtÅĤum":141083,"ĠtÅĤumacz":141084,"Tür":141085,"Türkiye":141086,"ãģįãģ£":141087,"ãģįãģ£ãģĭãģij":141088,"Ġ×Ķ׳×ķ׼":141089,"Ġ×Ķ׳×ķ׼×Ĺ×Ļ":141090,"ĠìĥĿìĤ°":141091,"ĠÑĦоÑĢмÑĭ":141092,"ç¾İãģĹãģĦ":141093,"à¸Ľà¸£à¸¶à¸ģ":141094,"à¸Ľà¸£à¸¶à¸ģษา":141095,"Ġlumière":141096,"ãĤªãĥ¼ãĥĹ":141097,"ãĤªãĥ¼ãĥĹãĥ³":141098,"à¸Ľà¸·à¸Ļ":141099,"วัสà¸Ķ":141100,"วัสà¸Ķุ":141101,"еÑĢÑĤв":141102,"ÙĥÙĦÙģ":141103,"ï½£":141104,"à¸ĺรรมà¸Ķา":141105,"׳×ĺר":141106,"ĠпÑĢедÑģÑĤавлÑıеÑĤ":141107,"Ġanálisis":141108,"Ġbãi":141109,"باÙĤÙĬ":141110,"à¸Ľà¸£à¸°à¹Ģà¸Ķ":141111,"à¸Ľà¸£à¸°à¹Ģà¸Ķà¹ĩà¸Ļ":141112,"ĠÑģлÑĥÑĩаÑı":141113,"ĠÑģлÑĥÑĩаÑıÑħ":141114,"ÐĽÐIJ":141115,"สัà¸ĩà¹Ģà¸ģ":141116,"สัà¸ĩà¹Ģà¸ģà¸ķ":141117,"Ġprzec":141118,"Ġprzecież":141119,"ÙħصÙĦ":141120,"ÙħصÙĦØŃØ©":141121,"ש×ķ×§×ķ׾×ĵ":141122,"ĠобоÑĢÑĥдованиÑı":141123,"ĠtrwaÅĤ":141124,"رÙĪÙħ":141125,"ìķĪëĤ´":141126,"ĠNghá»ĭ":141127,"خش":141128,"à¸ļาà¸Ħาร":141129,"à¸ļาà¸Ħารà¹Īา":141130,"ĠопÑĨион":141131,"ĠÑģозданиÑı":141132,"ãĤ³ãĤ¹ãĥĪ":141133,"Ġ×Ķ×¢×ľ×Ļ":141134,"Ġ×Ķ×¢×ľ×Ļ×ķף":141135,"läuft":141136,"ãĥĻãĤ¹ãĥĪ":141137,"Ġrê":141138,"Ġrêve":141139,"×IJ×ij×Ļ×ij":141140,"×Ļ×Ļ×ļ":141141,"ë¶Ļ":141142,"ãĤ¤ãĥ³ãĥī":141143,"ÅĤoży":141144,"ÅĤożyÄĩ":141145,"عائÙĦ":141146,"عائÙĦØ©":141147,"Ø£ÙĪØ±":141148,"Ø£ÙĪØ±Ø§ÙĤ":141149,"à¸Ĺà¹īà¸Ńà¸ĩà¸ĸ":141150,"à¸Ĺà¹īà¸Ńà¸ĩà¸ĸิà¹Īà¸Ļ":141151,"Ġähn":141152,"Ġähnlich":141153,"ãĥŁãĥĭ":141154,"à¸ľà¸¹":141155,"à¸ľà¸¹à¹īà¸Ļ":141156,"à¸ľà¸¹à¹īà¸Ļำ":141157,"ĠмаÑĤеÑĢиалÑĭ":141158,"ĠкапиÑĤ":141159,"ĠкапиÑĤал":141160,"F":141161,"Ġseçil":141162,"Ġhứng":141163,"Ġintéressant":141164,"ãģ£ãģ¦ãģĦãģı":141165,"ĠeÄŁer":141166,"ëIJĺìĹĪìĬµëĭĪëĭ¤":141167,"ĠanlaÅŁma":141168,"ãģĶåĪ©ç͍":141169,"Ġ×ij×ĸ׼":141170,"Ġ×ij×ĸ׼×ķת":141171,"ëĿ¼ë©´":141172,"ĠÙĬÙĪØ³":141173,"ĠÙĬÙĪØ³Ùģ":141174,"أسÙĦØŃØ©":141175,"ĠGefühl":141176,"ĠноÑĢмалÑĮн":141177,"ãĥĻãĥ³":141178,"ãģķãĤĮãĤĭãģĵãģ¨":141179,"ĠÐijеÑģ":141180,"ãģ¨ãģĦãģĪãģ°":141181,"ĠÙħÙĩÙħ":141182,"ĠÙħÙĩÙħØ©":141183,"ãģ§ãģĹãĤĩãģĨãģŃ":141184,"ĠêµŃëĤ´":141185,"à¹Ģมà¹ĩà¸Ķ":141186,"×ŀ×ijקר":141187,"ĠاÙĦدÙĨÙĬ":141188,"ĠاÙĦدÙĨÙĬا":141189,"à¸Ĭู":141190,"кÑĢÑĥÑĤ":141191,"Ġthoáng":141192,"Ġ׳×ĵר":141193,"Ġ׳×ĵרש":141194,"ĠÑĢаÑģÑģказал":141195,"ĠAuÃŁerdem":141196,"פ×IJר":141197,"פ×IJרק":141198,"Ġ×ŀש×Ĺ×§×Ļ×Ŀ":141199,"צר׼×Ļ×Ŀ":141200,"×ŀ×ĵ×ķ":141201,"×ŀ×ĵ×ķ×Ļ×§":141202,"èĭ¦ãģĹ":141203,"ĠÑģиг":141204,"ĠÑģигнал":141205,"ĠMá»įi":141206,"Ġtrữ":141207,"ĠnastÄĻp":141208,"ĠnastÄĻpnie":141209,"Ġì¶Ķì§Ħ":141210,"ĠاÙĦÙģÙĨد":141211,"ĠاÙĦÙģÙĨدÙĤ":141212,"koÅĦczyÅĤ":141213,"สีà¹Ī":141214,"×§×Ļ×ij":141215,"×§×Ļ×ij×ķ×¥":141216,"ĠнÑĥжнÑĭ":141217,"大åĪĩ":141218,"大åĪĩãģª":141219,"æıĽãģĪ":141220,"ת×ķס":141221,"ת×ķספת":141222,"ãģ£ãģ¦ãģĦãģªãģĦ":141223,"ĠмÑı":141224,"ĠмÑıг":141225,"ĠмÑıгк":141226,"Ġjakie":141227,"ĠjakieÅĽ":141228,"à¸ķำà¸ļ":141229,"à¸ķำà¸ļล":141230,"ĠìŀĪì§Ģ":141231,"×ij×ĺ×IJ":141232,"ĠоÑĤлиÑĩно":141233,"ÙĤÙIJ":141234,"ĠавÑĤомоб":141235,"ĠавÑĤомоби":141236,"ĠавÑĤомобилÑı":141237,"دÙĬÙħÙĤراطÙĬ":141238,"ĠاÙĦÙĪØ§":141239,"ĠاÙĦÙĪØ§ØŃد":141240,"ĠسÙĪØ±ÙĬØ©":141241,"أغÙĦ":141242,"أغÙĦب":141243,"ĠÑįкÑĢан":141244,"ãĥĹãĥ©ãĤ¤":141245,"ĠjesteÅĽ":141246,"ãĥIJãĥª":141247,"Ġ×Ķ×IJ×ķ×ķ×Ļר":141248,"ائÙĥ":141249,"à¸Ńยà¹Īาà¸ĩยิà¹Īà¸ĩ":141250,"ÑĢекÑĤ":141251,"Ġumo":141252,"Ġumoż":141253,"Ġumożli":141254,"Ġumożliw":141255,"Ġumożliwia":141256,"Ġnächste":141257,"ĠìŀĪì§Ģë§Į":141258,"ĠпÑĢедн":141259,"ĠпÑĢедназ":141260,"ĠпÑĢедназнаÑĩен":141261,"Ġmaçı":141262,"Ġpomi":141263,"ĠpomiÄĻd":141264,"ĠpomiÄĻdzy":141265,"ĠاÙĦÙĦÙĤاء":141266,"à¹Ģà¸Ķà¸Ńะ":141267,"ĠновоÑģÑĤи":141268,"×ŀ×Ĺ׾×Ķ":141269,"رÙĬاضÙĬ":141270,"à¸Ķà¸Ļ":141271,"à¸Ķà¸Ļà¸ķรี":141272,"بصر":141273,"ìĬ¤íĥĢ":141274,"scripción":141275,"Ġnapisa":141276,"ĠnapisaÅĤ":141277,"Ġ׳ש×ŀ×¢":141278,"ĠاÙĦÙħØŃÙĦÙĬ":141279,"Ġhiá»ĥn":141280,"×IJ×Ĺ":141281,"×IJ×Ĺר×IJ×Ļ":141282,"ĠгÑĢаниÑĨ":141283,"æīĭç¶ļãģį":141284,"Ùĥسب":141285,"Ġà¹ģà¸ķà¹Īà¸ĸà¹īา":141286,"à¸Ķาวà¸Ļà¹Į":141287,"à¸Ķาวà¸Ļà¹Įà¹Ĥหลà¸Ķ":141288,"ãĤĭãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ":141289,"åŁºæľ¬çļĦãģ«":141290,"ÙĪÙĦاد":141291,"räume":141292,"دÙģØ§Ø¹":141293,"×Ļצע":141294,"ĠOczy":141295,"ĠOczywiÅĽcie":141296,"ĠÅģ":141297,"ĠÅģa":141298,"اÙĦÙĬاب":141299,"اÙĦÙĬاباÙĨ":141300,"áºłI":141301,"ĠBirliÄŁi":141302,"×Ķ×ķצ":141303,"×Ķ×ķצ×IJת":141304,"ĠÄijua":141305,"Ġê·¸ëŁ¬ëĭĪê¹Į":141306,"Ġréalité":141307,"عÙĦاÙĤات":141308,"Jeste":141309,"JesteÅĽ":141310,"Ġмнож":141311,"ĠмножеÑģÑĤво":141312,"K":141313,"ãĥĹãĥŃãĤ¸ãĤ§":141314,"ãĥĹãĥŃãĤ¸ãĤ§ãĤ¯ãĥĪ":141315,"ĠÑĦл":141316,"ظÙĨ":141317,"×Ĵ׾×Ĵ׾":141318,"ĠmÅĤodzie":141319,"ĠmÅĤodzież":141320,"à¸Ļà¹īำà¸ķา":141321,"à¸Ļà¹īำà¸ķาล":141322,"ÐĽÐķ":141323,"×ij×ķ×ĺ":141324,"Ġ׾×Ķ×Ĵ×Ļ×ĵ":141325,"ãģĵãģ¨ãĤĤãģĤãĤĭ":141326,"زاد":141327,"×ŀ×Ļ×ĵ×¢":141328,"ĠgÅĤównie":141329,"ãĥıãĤ¦":141330,"ãĥıãĤ¦ãĤ¹":141331,"бел":141332,"Ġétape":141333,"ðŁĺĢ":141334,"ĠмоделÑĮ":141335,"aģını":141336,"ש×Ĺ×§":141337,"ש×Ĺקף":141338,"Ġniño":141339,"à¸Ĭà¹īาà¸ĩ":141340,"à¹Ģลีย":141341,"ĠÑĦоÑĢме":141342,"ĠاÙĦشرÙĬÙģ":141343,"ĠÑĥдаÑĢ":141344,"arriv":141345,"arrivée":141346,"ĠmiesiÄĻ":141347,"ĠmiesiÄĻcy":141348,"ØŃرÙĥ":141349,"ØŃرÙĥات":141350,"ĠDiá»ħn":141351,"ÐĿЫ":141352,"ãģ¾ãģ£ãģŁãģı":141353,"Ġ×Ļר×ķ×§":141354,"еÑģÑĤеÑģÑĤв":141355,"еÑģÑĤеÑģÑĤвенн":141356,"Ġê·¸ëŁ¼":141357,"ĠاÙĦÙħتÙĪ":141358,"ĠاÙĦÙħتÙĪØ³Ø·":141359,"Ġbénéfic":141360,"Ġbénéficie":141361,"Ġwybra":141362,"ĠwybraÄĩ":141363,"ĠاÙĦزÙħÙĨ":141364,"ĠпÑĢинÑı":141365,"ĠпÑĢинÑıл":141366,"Ù쨱ØŃ":141367,"Ġksz":141368,"ĠksztaÅĤ":141369,"ĠksztaÅĤt":141370,"ק׾×ĺ":141371,"×ij×ĵ×Ļקת":141372,"Ġgiấ":141373,"Ġgiấc":141374,"ĠproprietÃł":141375,"деÑĢжан":141376,"ĠKöln":141377,"ĠGüzel":141378,"×Ļפ×ķ×Ļ":141379,"ĠCuá»Ļc":141380,"ÑįÑĤаж":141381,"ترÙĥÙĬ":141382,"ترÙĥÙĬز":141383,"ложений":141384,"ĠпÑĥ":141385,"ĠпÑĥÑĤи":141386,"اختÙĦاÙģ":141387,"åĩºãģ¦ãģıãĤĭ":141388,"à¸ļุà¸ģ":141389,"âĿ¤":141390,"ÑĦан":141391,"פש×ĺ":141392,"à¸ļัà¸Ļà¹Ģà¸Ĺ":141393,"à¸ļัà¸Ļà¹Ģà¸Ĺิà¸ĩ":141394,"ĠاÙĦساد":141395,"ĠاÙĦسادس":141396,"ĠاÙĦÙĤÙĪÙħ":141397,"ĠاÙĦÙĤÙĪÙħÙĬ":141398,"Ġyönetici":141399,"ÙĩÙĪØ§Øª":141400,"ÙĩÙĪØ§ØªÙģ":141401,"Ġresponsável":141402,"ĠподдеÑĢжива":141403,"ĠاÙĦسÙĦØ·":141404,"ĠاÙĦسÙĦطات":141405,"ãģĹãģ¦ãģĬãģı":141406,"ãĥļãĥĥãĥĪ":141407,"à¸Ľà¸¸à¹Īม":141408,"ĠoglÄħda":141409,"ÙĨاÙĤ":141410,"ÙĨاÙĤØ´":141411,"à¸Ħà¸Ńà¸Ļà¹Ĥà¸Ķ":141412,"ĠMüsl":141413,"ĠMüslü":141414,"ĠMüslüman":141415,"ĠMoż":141416,"ĠMożna":141417,"Ġnumérique":141418,"Ġvá»ı":141419,"ĠسÙĬتÙħ":141420,"ĠyerleÅŁ":141421,"монÑĤаж":141422,"Ġgoût":141423,"ãģ¦ãģĬãĤĬãģ¾ãģĻ":141424,"ĠKhánh":141425,"Ġедин":141426,"ĠединÑģÑĤв":141427,"اÙĨØ®Ùģ":141428,"اÙĨØ®ÙģØ§Ø¶":141429,"ìĭľíĹĺ":141430,"Ġlặng":141431,"ĠÑĢолÑĮ":141432,"à¸ķัวà¹ģà¸Ĺà¸Ļ":141433,"à¸Ħà¹Īาà¹ĥà¸Ĭà¹ī":141434,"à¸Ħà¹Īาà¹ĥà¸Ĭà¹īà¸Īà¹Īาย":141435,"Ġverfüg":141436,"Ġverfügbar":141437,"ìĻĶëĭ¤":141438,"ãģĦãģļ":141439,"ãģĦãģļãĤĮ":141440,"ĠиÑģÑģледованиÑı":141441,"меÑīа":141442,"×Ķ×Ĺ":141443,"×Ķ×Ĺ×ĸר":141444,"à¹ģà¸Łà¸Ĭัà¹Īà¸Ļ":141445,"تصرÙģ":141446,"إرÙĩاب":141447,"ĠexercÃŃcio":141448,"Ġélev":141449,"Ġélevé":141450,"สัà¸įà¸įาà¸ĵ":141451,"ÃĸZ":141452,"ãĥĹãĥŃãĤ°":141453,"ãĥĹãĥŃãĤ°ãĥ©":141454,"ãĥĹãĥŃãĤ°ãĥ©ãĥł":141455,"ĠwewnÄĻtrzn":141456,"Ġhenüz":141457,"é£Ľãģ³":141458,"à¹Ģà¸Ķà¸Ńรà¹Į":141459,"ÑģÑĥж":141460,"ÑģÑĥжден":141461,"شعÙĪØ¨":141462,"ãģ²ãģ¨ãĤĬ":141463,"ĠwyÅĤÄħ":141464,"ĠwyÅĤÄħcznie":141465,"ĠплоÑħо":141466,"ÐĶÐķ":141467,"Ầ":141468,"ÙģØ¹Ø§ÙĦÙĬ":141469,"ÙģØ¹Ø§ÙĦÙĬات":141470,"ĠاÙĦعشر":141471,"ÑģÑĤÑĥпил":141472,"Ġyarg":141473,"Ġyargı":141474,"нÑİÑİ":141475,"×ķ×IJ×ij":141476,"Ġuç":141477,"Ġuçak":141478,"ë²½":141479,"تÙĪÙĤÙĬ":141480,"تÙĪÙĤÙĬع":141481,"Ġì¤ijìĭ¬":141482,"׳×Ļ×ķ×ķ×ĺ":141483,"Ø£ÙĥÙĦ":141484,"ç½®ãģĦãģ¦":141485,"éłĤãģį":141486,"Ġ×Ķת×ij":141487,"Ġ×Ķת×ij×Ļ×¢×Ķ":141488,"Ġdürfen":141489,"ÙħÙĤاÙĦ":141490,"ÙħÙĤاÙĦات":141491,"ĠزÙħÙĨ":141492,"à¸ŀฤศ":141493,"à¸ŀฤศà¸Ī":141494,"à¸ŀฤศà¸Īิà¸ģ":141495,"à¸ŀฤศà¸Īิà¸ģายà¸Ļ":141496,"ĠнеÑģколÑĮ":141497,"ĠнеÑģколÑĮки":141498,"ĠнеÑģколÑĮкиÑħ":141499,"Ġcriança":141500,"มิà¸ķร":141501,"×ŀ׼×Ļר×ķת":141502,"à¸ģารà¸ļริหาร":141503,"Ġtélécharg":141504,"Ġ×IJ×ķ×Ķ×ijת":141505,"ĠBüro":141506,"ä½ľãģ£ãģŁ":141507,"ĠKiÅŁi":141508,"ç¾İåij³ãģĹ":141509,"à¹Ģลยà¸Ħà¹Īะ":141510,"à¸ŀà¸ļà¸ģัà¸ļ":141511,"à¸Īà¹īา":141512,"Ġçer":141513,"Ġçerç":141514,"Ġçerçeve":141515,"ãĤĴä½ľãģ£ãģ¦":141516,"ĠпеÑĢвÑĥÑİ":141517,"×ŀצר×Ļ×Ŀ":141518,"×IJ׾×ķ×Ķ":141519,"×IJ׾×ķ×Ķ×Ļ×Ŀ":141520,"Ġagré":141521,"Ġagréable":141522,"Ġayır":141523,"İLİ":141524,"ãĤ¥":141525,"ĠíĺĦ":141526,"ĠíĺĦìĭ¤":141527,"ثاÙĦØ«":141528,"ת×ĸ":141529,"ת×ĸ×ķ׳×Ķ":141530,"ãģ¨ãģĦãģ£ãģ¦":141531,"ãģ¨ãģĦãģ£ãģ¦ãĤĤ":141532,"ĠابÙĪ":141533,"ĠÑģобак":141534,"é£Łãģ¹ãģŁ":141535,"Ġданном":141536,"à¹Ģลิ":141537,"à¹Ģลิศ":141538,"Ġíļ":141539,"Ġíļ¨":141540,"Ġíļ¨ê³¼":141541,"ãĤĤãĤīãģĪãĤĭ":141542,"׳צ׾":141543,"ÑĦик":141544,"ÑĦикÑģ":141545,"ĠjesteÅĽmy":141546,"ת×Ĺ×ķש×Ķ":141547,"à¹Ħมà¹Īà¸Ħวร":141548,"ĠØŃسÙĬÙĨ":141549,"à¸ģารลà¸ĩà¸Ĺุà¸Ļ":141550,"ë´¤":141551,"ĠÐĺменно":141552,"à¸ļà¸Ńรà¹Į":141553,"à¸ļà¸Ńรà¹Įà¸Ķ":141554,"ĠCảnh":141555,"ìĦľë¹ĦìĬ¤":141556,"Ġполов":141557,"Ġполовин":141558,"ĠзамеÑĩа":141559,"ãģĦãĤįãĤĵãģª":141560,"Ġ×ij×Ļ×§":141561,"Ġ×ij×Ļקש":141562,"лÑĥÑĪ":141563,"ãĤĴè¿İ":141564,"ãĤĴè¿İãģĪ":141565,"جرÙĬÙħØ©":141566,"Ġtây":141567,"ĠاÙĦÙĨÙĪ":141568,"ĠاÙĦÙĨÙĪÙĪÙĬ":141569,"ÃĤN":141570,"ì¿ł":141571,"หà¸Ļาว":141572,"Ġ×ij×Ĺש×ij×ķף":141573,"زار":141574,"à¸Ķาร":141575,"à¸Ķารา":141576,"ĠÅĽl":141577,"ĠÅĽlub":141578,"มีà¸Ħวามสุà¸Ĥ":141579,"Ġnhu":141580,"ĠnhuáºŃn":141581,"ÙħØŃطة":141582,"à¹Ģสืà¹īà¸Ńà¸ľà¹īา":141583,"ĠТолÑĮко":141584,"ĠÙĥس":141585,"ĠÙĥسارة":141586,"ÙħشرÙĪØ¹":141587,"niÄĻcia":141588,"×¢×Ľ×©×Ļ×ķ":141589,"تÙĦÙģ":141590,"تÙĦÙ쨲ÙĬ":141591,"تÙĦÙ쨲ÙĬÙĪÙĨ":141592,"ĠlÆ°á»Ľi":141593,"ĠÐľÐ¾ÑģквÑĭ":141594,"Ġréserve":141595,"ĠanlaÅŁ":141596,"ĠanlaÅŁÄ±l":141597,"ĠedeceÄŁi":141598,"รà¸Ńà¸ĩà¹Ģà¸Ĺà¹īา":141599,"Ġبط":141600,"ĠبطرÙĬ":141601,"ĠبطرÙĬÙĤØ©":141602,"ãģ¦ãģĹãģ¾ãģ£ãģ¦":141603,"ãĤĤãĤīãģ£ãģ¦":141604,"برج":141605,"æ±ļ":141606,"æ±ļãĤĮ":141607,"Ġchoc":141608,"Ġchocia":141609,"Ġchociaż":141610,"Ġzobac":141611,"ĠzobaczyÄĩ":141612,"пÑĢÑı":141613,"пÑĢÑıжен":141614,"ĠÑĨиÑĦ":141615,"ĠÑĨиÑĦÑĢ":141616,"Ġмам":141617,"ĠвзÑıÑĤÑĮ":141618,"Ġchạm":141619,"جسÙħ":141620,"ØŃÙħاس":141621,"à¹Ģลà¹Īม":141622,"à¸ŀิษ":141623,"×Ķפ׼×ķ":141624,"à¸Ĭà¹Īà¸Ńà¸ĩà¸Ĺาà¸ĩ":141625,"Ġвек":141626,"Ġвека":141627,"Æ¡Ìģ":141628,"Æ¡Ìģi":141629,"ĠTiá»ģn":141630,"Ġtrầm":141631,"мÑĭÑĪ":141632,"мÑĭÑĪл":141633,"ĠÑĤÑĥ":141634,"ĠÑĤÑĥÑĢиÑģÑĤ":141635,"Ġchc":141636,"ĠchcÄħ":141637,"Ġавг":141638,"ĠавгÑĥÑģÑĤ":141639,"ĠавгÑĥÑģÑĤа":141640,"ס×IJ×ķת":141641,"Ġר×Ĵ׾":141642,"à¸ľà¸¥à¸ģระà¸Ĺ":141643,"à¸ľà¸¥à¸ģระà¸Ĺà¸ļ":141644,"å¤īãĤıãĤĭ":141645,"Ġ×Ķ×IJ×Ĺר×ķ׳×Ļ×Ŀ":141646,"سÙģÙĬر":141647,"ĠÑĩаÑīе":141648,"ãģĦãĤī":141649,"ãģĦãĤīãģ£":141650,"ãģĦãĤīãģ£ãģĹãĤĥ":141651,"×ķ×ŀ׳×Ļ×Ŀ":141652,"Ġarttır":141653,"ĠChá»ĭ":141654,"Ġì¡°ì§ģ":141655,"ĠÑĥÑģпеÑħ":141656,"Ġ×¢×ķס":141657,"Ġ×¢×ķסק":141658,"ĠìĥĿëªħ":141659,"ÑĨиÑĤ":141660,"Ġregión":141661,"ÐŀÐĿ":141662,"ĠdoÄŁum":141663,"ĠyaÅŁad":141664,"ĠyaÅŁadıģı":141665,"à¸Ĺà¸Ķลà¸Ńà¸ĩ":141666,"Ġgözü":141667,"ש×Ļר×Ķ":141668,"дÑĥмал":141669,"Ġdaģı":141670,"Ġdaģıt":141671,"à¸Ĺีมà¸ĩาà¸Ļ":141672,"Ġtiá»ģm":141673,"ĠاÙĦÙĥبر":141674,"ĠاÙĦÙĥبرÙī":141675,"ì¹Ń":141676,"ĠGünc":141677,"ĠGüncelle":141678,"ĠGüncelleme":141679,"ê¹Ĭ":141680,"ĠобоÑĢÑĥдование":141681,"ĠÑĢеÑĪа":141682,"Ụ":141683,"ĠпиÑĤ":141684,"ĠпиÑĤаниÑı":141685,"à¹Ģรียà¸ļ":141686,"×Ľ×ª×Ļ×ij×Ķ":141687,"Ġпон":141688,"ĠпонÑĢав":141689,"ĠпонÑĢави":141690,"Ġ×Ķ×ķ׾×ĵ":141691,"Ġ×Ķ×ķ׾×ĵת":141692,"Ġê²ģ":141693,"Ġê²ģëĭĪëĭ¤":141694,"ĠпеÑĢвой":141695,"ãĥ©ãĤ¤ãĥķ":141696,"ĠÅŁiir":141697,"krÄĻ":141698,"krÄĻc":141699,"Ġthiá»ĥu":141700,"à¹Ģลยà¸Ĺี":141701,"à¹Ģลยà¸Ĺีà¹Ģà¸Ķียว":141702,"×ĺ×¢×ł×ķת":141703,"ائÙĩÙħ":141704,"Ġ×IJס×ķר":141705,"ĠплаÑĤеж":141706,"تردد":141707,"Ġmożliwe":141708,"ĠkhỼ":141709,"ĠkhỼp":141710,"تÙģØ§Ø¹ÙĦ":141711,"ĠÑĪколÑĮ":141712,"ĠÑĪколÑĮн":141713,"ĠÙĤصة":141714,"Ġmétier":141715,"nÄĻÅĤa":141716,"หลà¹Īà¸Ń":141717,"Ġá»§ng":141718,"Ġprzegl":141719,"ĠprzeglÄħd":141720,"ĠاÙĦÙħتعÙĦ":141721,"ĠاÙĦÙħتعÙĦÙĤØ©":141722,"ĠÑģÑĭн":141723,"Ġволн":141724,"ãĥĩãĥ¼ãĥĪ":141725,"ĠÐŃÑĤи":141726,"ĠкÑĢоме":141727,"à¸Ħารà¹Į":141728,"׳ק×ķ×ĵ×Ķ":141729,"Ġ׾ש×ŀ×ķ×¢":141730,"Ġ×ĸ×ķ׼ר":141731,"ï¼§":141732,"ÙĬÙİØ§":141733,"Ġgiá»ıi":141734,"åĥįãģı":141735,"ĠÑģни":141736,"ĠÑģнижен":141737,"à¹ģà¸Ķà¸Ķ":141738,"รุà¸Ļ":141739,"รุà¸Ļà¹ģรà¸ĩ":141740,"Ġhiá»ĩp":141741,"ografÃŃa":141742,"à¹Ģà¸Īà¸Ńรà¹Į":141743,"Ġдвиг":141744,"ĠдвигаÑĤ":141745,"ĠдвигаÑĤел":141746,"Ġüy":141747,"Ġüyeler":141748,"Ġüyeleri":141749,"ĠбÑĥк":141750,"ĠбÑĥкв":141751,"ãĤĤå¤ļãģı":141752,"Ġthiá»ĩt":141753,"ĠPaÃŃs":141754,"ĠطبÙĬعÙĬ":141755,"à¹ģà¸Īà¸ģ":141756,"ĠاÙĦصØŃÙĬØŃ":141757,"Ġappré":141758,"Ġappréci":141759,"Ġdecisión":141760,"Ġë°ĺëĵľ":141761,"Ġë°ĺëĵľìĭľ":141762,"ĠÑĤебе":141763,"ãĤ·ãĥ¼ãĤº":141764,"ãĤ·ãĥ¼ãĤºãĥ³":141765,"ĠдалÑĮн":141766,"ĠìĬ¤":141767,"ĠìĬ¤ìĬ¤":141768,"ĠìĬ¤ìĬ¤ë¡ľ":141769,"ĠThá»ĥ":141770,"ĠkarÅŁ":141771,"ĠkarÅŁÄ±s":141772,"ĠkarÅŁÄ±sında":141773,"ĠKön":141774,"ĠKönig":141775,"ивание":141776,"×ij×ķצע":141777,"глаÑģ":141778,"Ġtwó":141779,"Ġtwórc":141780,"à¸Ľà¸ģà¸Ħร":141781,"à¸Ľà¸ģà¸Ħรà¸Ńà¸ĩ":141782,"ĠGÅĤ":141783,"ĠGÅĤówn":141784,"ĠUnterstüt":141785,"ĠUnterstützung":141786,"ĠдÑĥÑħ":141787,"ĠдÑĥÑħов":141788,"Ø£ÙħاÙĨ":141789,"×Ĺשש":141790,"تظ":141791,"تظاÙĩر":141792,"ĠлÑİбом":141793,"à¸ķาร":141794,"à¸ķาราà¸ĩ":141795,"Ġkról":141796,"Ø£ØŃدث":141797,"ì¡Įëĭ¤":141798,"ÐļÑĥÑĢÑģ":141799,"ãĥĥãĥĦ":141800,"×ŀ×§×ķ×ij׾":141801,"ĠÑģимвол":141802,"Ġdésorm":141803,"Ġdésormais":141804,"wüns":141805,"wünsche":141806,"Ñĥни":141807,"ÑĥниÑĨип":141808,"ÑĥниÑĨипалÑĮн":141809,"หลัà¸ģสูà¸ķร":141810,"ÙĨتشر":141811,"Ġал":141812,"Ġалк":141813,"Ġалког":141814,"Ġалкогол":141815,"ĠÑĥÑĩиÑĤÑĭва":141816,"à¸ģำà¸ģัà¸ļ":141817,"Ġ×ľ×¤×¢×ķ׾":141818,"ĠìĹ°ê²°":141819,"sÄħd":141820,"ĠاÙĦØ£ÙĬ":141821,"ĠاÙĦØ£ÙĬاÙħ":141822,"غÙĬاب":141823,"ĠнаÑĢ":141824,"ĠнаÑĢко":141825,"×ŀ×ķ×ĵ×¢":141826,"ĠÑģеÑĢии":141827,"пиÑģÑĭва":141828,"สิว":141829,"ç¶ļãģĦãģ¦":141830,"çͳãģĹè¾¼ãģ¿":141831,"Ġ׾×Ĵר":141832,"Ġ׾×Ĵר×ķ×Ŀ":141833,"Ġдем":141834,"Ġдемо":141835,"Ġë³´ëĤ´":141836,"تÙĩدÙĬد":141837,"ĠÙħØ´ÙĬرا":141838,"Ġduy":141839,"Ġduyá»ĩt":141840,"ĠwiÄĻksze":141841,"ÙħعاÙĬ":141842,"ÙħعاÙĬÙĬر":141843,"ĠGda":141844,"ĠGdaÅĦsk":141845,"Ġrah":141846,"Ġrahats":141847,"Ġrahatsız":141848,"ר×ķצ×Ķ":141849,"lös":141850,"lösung":141851,"ĠТаким":141852,"ÑĪед":141853,"ÑĪедÑĪ":141854,"عزÙĦ":141855,"Ġרש×Ļ×ŀת":141856,"Ġ׾×Ķ×Ļ׼":141857,"Ġ׾×Ķ×Ļ×Ľ×ł×¡":141858,"ĠпÑĥÑĤ":141859,"ĠпÑĥÑĤеÑĪ":141860,"ĠпÑĥÑĤеÑĪеÑģÑĤв":141861,"ĠnotÃŃcia":141862,"ĠalÄ±ÅŁ":141863,"ĠalÄ±ÅŁver":141864,"ĠalÄ±ÅŁveriÅŁ":141865,"ĠwÅĤos":141866,"ĠwÅĤosów":141867,"Ġبغ":141868,"Ġبغداد":141869,"Ġveröffent":141870,"Ġveröffentlicht":141871,"ĠKhá":141872,"Ġtán":141873,"ëIJĺ기":141874,"Ġ방문":141875,"ÙģÙĬÙĦ":141876,"à¹Ģà¸ģิà¸Ķà¸Īาà¸ģ":141877,"åı¯æĦĽ":141878,"åı¯æĦĽãģĦ":141879,"à¸ĸุà¸ĩ":141880,"ĠzewnÄĻtrzn":141881,"à¸łà¸²à¸©à¸²à¸Ńัà¸ĩà¸ģฤษ":141882,"Ġmáxima":141883,"Ġulus":141884,"Ġuluslararası":141885,"Ġ׳×Ķ׳":141886,"à¸Ĥà¹Īาวสาร":141887,"ĠìĿĺìĤ¬":141888,"à¹Ģหลืà¸Ńà¸ĩ":141889,"ĠدÙĤ":141890,"ĠدÙĤائÙĤ":141891,"สืà¹Īà¸Ńสาร":141892,"먼":141893,"ĠÑģоÑģÑĤоÑıнии":141894,"สมาà¸Ħม":141895,"á»Ĥ":141896,"ĠÐľÐ¾Ñģков":141897,"ĠÐľÐ¾ÑģковÑģк":141898,"×ŀס×ķ×Ĵ׾":141899,"ãģĭãģĭãĤĬ":141900,"ĠTruyá»ģn":141901,"à¹ģà¸Ĥà¹ĩà¸ĩà¹ģรà¸ĩ":141902,"×ŀ×Ĺ×ĸ×Ļ×§":141903,"à¹Ĥà¸ģà¹ī":141904,"ÙĬسر":141905,"ìĶ©":141906,"×IJ×ķ×§":141907,"×IJ×ķ×§×ĺ":141908,"×IJ×ķ×§×ĺ×ķ×ijר":141909,"Ġproximité":141910,"ÙħÙĨÙĩج":141911,"ĠاÙĦجز":141912,"ĠاÙĦجزائ":141913,"ĠاÙĦجزائرÙĬ":141914,"ĠÄIJiá»ĥm":141915,"Ġденеж":141916,"Ġденежн":141917,"ÙģØŃص":141918,"Ù쨦":141919,"ĠÐijÑĥд":141920,"×Ĵ×Ļ×ĵ×ķ׾":141921,"ĠÐĴедÑĮ":141922,"عÙĦاÙħØ©":141923,"Ġ×IJ×Ĺר×ķ׳×ķת":141924,"ãģĦãģŁãģłãģĦãģ¦":141925,"سÙĦØŃ":141926,"ØŃÙĦÙħ":141927,"زÙĪØ§Ø±":141928,"Ùĥسر":141929,"×ĺקס":141930,"Ġбан":141931,"Ġбанков":141932,"ĠпÑĢож":141933,"ĠпÑĢожива":141934,"liwo":141935,"liwoÅĽci":141936,"ĠTiếp":141937,"ĠاÙĦÙħÙĨاسب":141938,"ĠاÙĦØ®ÙĬار":141939,"ãģĬãģĭ":141940,"ãģĬãģĭãģĴ":141941,"à¸Ķà¸Ńà¸ģà¹Ħมà¹ī":141942,"ämp":141943,"ämpfe":141944,"à¸ķัà¹īà¸ĩà¹ĥà¸Ī":141945,"ĠзаÑīиÑĤ":141946,"ĠзаÑīиÑĤÑĭ":141947,"ĠThưá»Ŀng":141948,"ĠصÙģ":141949,"ĠصÙģØŃØ©":141950,"×Ĺ×ķרף":141951,"ãĥIJãĥĥãĤ°":141952,"Ġ×ĵ×Ļ×Ĵ":141953,"Ġ×ĵ×Ļ×Ĵ×Ļ×ĺ":141954,"Ġ×ĵ×Ļ×Ĵ×Ļ×ĺ׾×Ļ":141955,"Ġ×Ķ×Ĺ×ķ׾×Ļ×Ŀ":141956,"веÑī":141957,"веÑīа":141958,"ĠкÑĥлÑĮÑĤ":141959,"ĠкÑĥлÑĮÑĤÑĥ":141960,"ĠкÑĥлÑĮÑĤÑĥÑĢÑĭ":141961,"ĠاÙĦاÙĨترÙĨت":141962,"Ġhöch":141963,"Ġhöchst":141964,"Ġíĺķ":141965,"Ġíĺķíĥľ":141966,"Ġвой":141967,"ĠвойнÑĭ":141968,"ÐĽÐŀ":141969,"ìĭłìļ©":141970,"Ġ×ŀ×ij×ķס":141971,"Ġ×ŀ×ij×ķסס":141972,"×ŀ׳×Ļ×¢":141973,"Ġfiyatı":141974,"ĠÑģлÑĥж":141975,"ĠÑģлÑĥжбÑĭ":141976,"à¸Ĺัศ":141977,"à¸Ĺัศà¸Ļ":141978,"ãģĵãģ¨ãģĮå¤ļãģĦ":141979,"Ġ×Ķ×ŀשת":141980,"Ġ×Ķ×ŀשת×ŀש":141981,"å¯ĦãģĽ":141982,"×ŀש׾×ķ×Ĺ":141983,"æĻĤçĤ¹":141984,"æĻĤçĤ¹ãģ§":141985,"à¸ŀรี":141986,"à¸ŀรีà¹Ģมีย":141987,"à¸ŀรีà¹Ģมียรà¹Į":141988,"à¸ŀรีà¹Ģมียรà¹Įลีà¸ģ":141989,"Ġdifficolt":141990,"ĠdifficoltÃł":141991,"ãĥ¬ãĤ¹ãĥĪ":141992,"ãĥ¬ãĤ¹ãĥĪãĥ©ãĥ³":141993,"สมà¹Ģà¸Ķà¹ĩ":141994,"สมà¹Ģà¸Ķà¹ĩà¸Ī":141995,"Ġжид":141996,"Ġжидк":141997,"ĠzupeÅĤ":141998,"ĠzupeÅĤnie":141999,"ĠÙħجر":142000,"ĠÙħجرد":142001,"ãģĮå§ĭ":142002,"ãģĮå§ĭãģ¾":142003,"ãĤŃãĥ£ãĥ©":142004,"Ġ×IJ×ķ×ķ×Ļר":142005,"ãģĬäºĴ":142006,"ãģĬäºĴãģĦ":142007,"ĠpotrÃł":142008,"ĠPaÅĦst":142009,"ĠPaÅĦstwo":142010,"ĠبÙĬاÙĨ":142011,"ĠبÙĬاÙĨات":142012,"Ġиногда":142013,"ĠÑĢа":142014,"ĠÑĢаÑģÑĤв":142015,"ĠÑĢаÑģÑĤвоÑĢ":142016,"Ġ×ĸ×ŀ׳":142017,"ยิà¹īม":142018,"ÄĨ":142019,"ãģ¾ãģķ":142020,"ãģ¾ãģķãģ«":142021,"ãĥķãĤ¡ãĤ¤ãĥ«":142022,"ĠgördÃ¼ÄŁÃ¼":142023,"สà¸ĩà¸Ħร":142024,"สà¸ĩà¸Ħราม":142025,"ĠArkadaÅŁ":142026,"ĠrozwiÄħzania":142027,"×ŀ×ķ×ĺ":142028,"piÄĻ":142029,"piÄĻt":142030,"صغر":142031,"สย":142032,"สยาม":142033,"ãĤĨãģ£ãģıãĤĬ":142034,"Ġtrần":142035,"ĠeconomÃŃa":142036,"Ġgehören":142037,"ãĤ·ãĥ§ãĥ¼":142038,"ĠsÅĤucha":142039,"à¸ŀà¸Ńà¹ĥà¸Ī":142040,"ĠоÑĤмеÑĤил":142041,"ÙĨتÙĤÙĦ":142042,"Ġpropósito":142043,"ĠваÑĪего":142044,"Ġnhắn":142045,"à¹ģà¸ĸว":142046,"ĠкомиÑģ":142047,"ĠкомиÑģÑģи":142048,"ważnie":142049,"ĠyavaÅŁ":142050,"×ŀ×Ļ×§":142051,"×ŀ×Ļ×§×ķ×Ŀ":142052,"ש×IJ×ľ×ª":142053,"Ġyıllarda":142054,"ĠЮ":142055,"ĠЮÑĢ":142056,"×ł×¡×Ļ×ij×ķת":142057,"תצ":142058,"תצ×ķ×Ĵ":142059,"ĠоднÑĥ":142060,"Ġà¸Ńยà¹Īาà¸ĩà¹Ħร":142061,"Ġà¸Ńยà¹Īาà¸ĩà¹Ħรà¸ģà¹ĩà¸ķาม":142062,"ëģ¼":142063,"à¹Ħลà¹Ī":142064,"تسÙĦÙĬÙħ":142065,"بÙĦاغ":142066,"Ġìī":142067,"Ġìī½":142068,"Ġìī½ê²Į":142069,"ãĥļãĥ³":142070,"звÑĥÑĩ":142071,"ĠWäh":142072,"ĠWährend":142073,"Ġ×Ļ×Ļת":142074,"Ġ×Ļ×Ļ×ª×Ľ×Ł":142075,"Ġkhuyên":142076,"Ġvẽ":142077,"ĠамеÑĢ":142078,"ĠамеÑĢик":142079,"ĠамеÑĢикан":142080,"ĠамеÑĢиканÑģк":142081,"عجب":142082,"ãĥĽãĥ¼ãĥłãĥļãĥ¼ãĤ¸":142083,"ĠникÑĤо":142084,"ĠÙĤÙİ":142085,"ĠÙĤÙİØ§ÙĦ":142086,"ĠÙĤÙİØ§ÙĦÙİ":142087,"ÐIJÐĹ":142088,"ÙħجÙħÙĪØ¹":142089,"ÙħجÙħÙĪØ¹Ø§Øª":142090,"ĠnecessitÃł":142091,"Ġpobli":142092,"Ġpobliżu":142093,"Ġphấn":142094,"ĠСообÑī":142095,"ÙħÙĤاط":142096,"ÙħÙĤاطع":142097,"Ġ×Ķצ×ķר×ļ":142098,"laÅŁtırma":142099,"วิà¸Ķ":142100,"วิà¸Ķี":142101,"วิà¸Ķีà¹Ĥà¸Ń":142102,"Ġ그리ìĬ¤":142103,"Ġ그리ìĬ¤ëıĦ":142104,"ãĤ¿ãĤ¤ãĥŁ":142105,"ãĤ¿ãĤ¤ãĥŁãĥ³ãĤ°":142106,"×§×ĺ×Ĵ×ķר":142107,"×§×ĺ×Ĵ×ķר×Ļ×Ķ":142108,"Ġ×Ĺ×ķפ":142109,"Ġ×Ĺ×ķפש×Ļ":142110,"أجر":142111,"Ġимени":142112,"ĠÑĢанее":142113,"à¹Ģà¸ŀืà¹Īà¸Ńà¸Ļà¹Ĩ":142114,"ĠJesús":142115,"Ñģоедин":142116,"Ñģоединен":142117,"Ġר×Ĺ×ķ×§":142118,"à¹Ĥà¸ļรา":142119,"à¹Ĥà¸ļราà¸ĵ":142120,"ĠHÆ¡n":142121,"ĠtháºŃp":142122,"تعÙĬÙĬÙĨ":142123,"ĠtartÄ±ÅŁ":142124,"ĠtartÄ±ÅŁma":142125,"ĠGespr":142126,"ĠGespräch":142127,"תר×ķפ":142128,"תר×ķפ×ķת":142129,"Ġcatégorie":142130,"ĠоказÑĭва":142131,"ĠналиÑĩие":142132,"Ġprésenté":142133,"Ġkull":142134,"Ġkulland":142135,"Ġkullandı":142136,"Ġünl":142137,"Ġünlü":142138,"ĠÙģÙĥرة":142139,"изаÑĤоÑĢ":142140,"×IJ×ķ׳":142141,"×IJ×ķ׳×Ļ×ij":142142,"×IJ×ķ׳×Ļ×ijרס":142143,"×IJ×ķ׳×Ļ×ijרס×Ļ×ĺת":142144,"ĠÑĢаÑģÑģмаÑĤ":142145,"ĠÑĢаÑģÑģмаÑĤÑĢ":142146,"ĠÑĢаÑģÑģмаÑĤÑĢива":142147,"تÙĥÙĦÙħ":142148,"ÙĥترÙĪ":142149,"ÙĥترÙĪÙĨÙĬ":142150,"ĠÑģоÑĩеÑĤ":142151,"ĠÑģоÑĩеÑĤа":142152,"ãĤĴè¦ĭãģĽ":142153,"Ġngừa":142154,"ĠÐłÐµÑģп":142155,"ĠÐłÐµÑģпÑĥб":142156,"ĠÐłÐµÑģпÑĥблик":142157,"ãĤ¦ãĤ©":142158,"ãĤ¦ãĤ©ãĥ¼":142159,"ĠÐľÐµÐ¶Ð´Ñĥ":142160,"ĠìŀĪê²Į":142161,"Ġmâ":142162,"ĠìļĶì²Ń":142163,"ضار":142164,"ลุà¹īà¸Ļ":142165,"ëĮĢíķĻêµIJ":142166,"×ĸ×Ļ׼":142167,"×ĸ×Ļ׼ר×ķף":142168,"ãĤ¹ãĥļ":142169,"ãĤ¹ãĥļãĥ¼ãĤ¹":142170,"ĠкÑĢаÑģоÑĤ":142171,"H":142172,"ê¼Ń":142173,"ãĤĴéĽĨ":142174,"ãĤĴéĽĨãĤģ":142175,"ë°Ŀ":142176,"Ġ×Ķ׳×IJ":142177,"Ġ×Ķ׳×IJש×Ŀ":142178,"Ġê°Ģìļ´":142179,"Ġê°Ģìļ´ëį°":142180,"تÙĥÙĦÙ쨩":142181,"ĠØŃÙĤÙĬÙĤÙĬ":142182,"Ġhalk":142183,"Ġhalkın":142184,"ÑİÑīÑĥÑİ":142185,"ĠÑģпин":142186,"סר×ĺף":142187,"ĠпеÑĢвого":142188,"Ġполож":142189,"ĠположиÑĤелÑĮн":142190,"Ġдл":142191,"ĠдлиÑĤелÑĮн":142192,"ĠVÄ©nh":142193,"ê´´":142194,"ĠÑģÑĭÑĢ":142195,"ĠíĨµíķĺìŬ":142196,"ë³ijìĽIJ":142197,"à¹Ĥรà¸ĩà¸ĩาà¸Ļ":142198,"รัà¸ļà¸ľà¸´à¸Ķ":142199,"รัà¸ļà¸ľà¸´à¸Ķà¸Ĭà¸Ńà¸ļ":142200,"تجÙĨب":142201,"sÅĤ":142202,"sÅĤuch":142203,"ãĤ¢ãĥ«ãĥIJ":142204,"ãĤ¢ãĥ«ãĥIJãĥł":142205,"ëī´ìĬ¤":142206,"Ġpatië":142207,"Ġpatiënt":142208,"Ġìĺ¤í":142209,"Ġìĺ¤íŀ":142210,"Ġìĺ¤íŀĪ":142211,"Ġìĺ¤íŀĪ볤":142212,"ĠDerne":142213,"ĠDerneÄŁi":142214,"wróci":142215,"wróciÄĩ":142216,"ĠобÑī":142217,"ĠобÑīеÑģÑĤв":142218,"ĠобÑīеÑģÑĤвенно":142219,"ĠêµIJìĪĺ":142220,"tıģımız":142221,"Ġ×Ķ×ŀש×Ļ×ij":142222,"körper":142223,"Ġпозвол":142224,"ĠпозволиÑĤ":142225,"ĠChiến":142226,"أخÙĪ":142227,"ĠAydın":142228,"à¸Ķà¹īาà¸Ļล":142229,"à¸Ķà¹īาà¸Ļลà¹Īาà¸ĩ":142230,"Ġdru":142231,"Ġdruż":142232,"Ġdrużyn":142233,"Ġë°ľíijľ":142234,"ĠThảo":142235,"جÙĩاد":142236,"à¸ģระà¸Ĺูà¹ī":142237,"ĠкÑĢов":142238,"ĠкÑĢови":142239,"Ġiçerik":142240,"Ġnadzie":142241,"ĠnadziejÄĻ":142242,"ĠСмоÑĤÑĢ":142243,"Ġphức":142244,"جتÙħاع":142245,"جتÙħاعÙĬØ©":142246,"компон":142247,"компоненÑĤ":142248,"Ġбил":142249,"ĠбилеÑĤ":142250,"ãĥIJãĥ³ãĥī":142251,"ĠPolÃŃcia":142252,"اÙĦتÙĩ":142253,"اÙĦتÙĩاب":142254,"ØŃرÙģ":142255,"تخط":142256,"تخطÙĬØ·":142257,"ãĤ³ãĥ¼ãĥ":142258,"ãĤ³ãĥ¼ãĥĴ":142259,"ãĤ³ãĥ¼ãĥĴãĥ¼":142260,"・・・":142261,"à¸ĭà¸Ńย":142262,"Ġcrédit":142263,"è²·ãģ£ãģŁ":142264,"ĠпоÑĢÑıд":142265,"ĠпоÑĢÑıдке":142266,"Ġphó":142267,"Ġwida":142268,"ĠwidaÄĩ":142269,"جرائÙħ":142270,"à¸ľà¸µ":142271,"ĠbÄĻdÄĻ":142272,"Ġ×ŀפת×Ĺ":142273,"ãĥijãĥ¼ãĥ":142274,"ãĥijãĥ¼ãĥĨ":142275,"ãĥijãĥ¼ãĥĨãĤ£":142276,"ãĥijãĥ¼ãĥĨãĤ£ãĥ¼":142277,"ĠKaż":142278,"ĠKażdy":142279,"ĠнеобÑħодимоÑģÑĤи":142280,"à¸Łà¸Ńรà¹Į":142281,"à¸Łà¸Ńรà¹Įม":142282,"ĠмалÑĭÑĪ":142283,"ĠплоÑĤ":142284,"ĠÑĥÑģÑĤÑĢой":142285,"ĠÑĥÑģÑĤÑĢойÑģÑĤва":142286,"à¸ĸà¸Ńà¸Ļ":142287,"ĠoluÅŁturul":142288,"ĠÅĽwiad":142289,"ĠÅĽwiadom":142290,"ÙħعÙĩد":142291,"ĠпÑĢоизведен":142292,"Æł":142293,"ר×Ļש":142294,"Ùħستث":142295,"ÙħستثÙħر":142296,"׳×Ļ×Ļר":142297,"pañ":142298,"Ġ;-)":142299,"Ġë°ľê²¬":142300,"Ġgörüyor":142301,"ÙħؤÙĦÙģ":142302,"ĠÄIJá»ģ":142303,"ĠاÙĦÙĨÙĪØ§Ø¨":142304,"×Ĺ×§×Ļר×Ķ":142305,"Ġmá»ıi":142306,"è¿°ãģ¹":142307,"ÐĿик":142308,"ìŀĸìķĦ":142309,"ìŀĸìķĦìļĶ":142310,"prowadziÅĤ":142311,"lóg":142312,"lógica":142313,"פס×ĺ":142314,"פס×ĺ×Ļ×ij׾":142315,"Ġ×ŀ×ĵ×Ķ":142316,"Ġ×ŀ×ĵ×Ķ×Ļ×Ŀ":142317,"ãģĵãģĵãģ¾ãģ§":142318,"×Ķת×Ĺ":142319,"×Ķת×Ĺ׾×Ķ":142320,"Ġפ×ķס":142321,"Ġפ×ķס×ĺ×Ļ×Ŀ":142322,"Ġнев":142323,"Ġневоз":142324,"Ġневозможно":142325,"ĠdostÄĻpny":142326,"ĠغاÙĦ":142327,"ĠغاÙĦب":142328,"ĠbezpieczeÅĦst":142329,"ĠbezpieczeÅĦstwa":142330,"åĪĨãģĭãĤĭ":142331,"ĠFührung":142332,"à¸ģีà¹ī":142333,"gemÃ¤ÃŁ":142334,"à¸Ĭà¹Īวà¸ĩà¹Ģวลา":142335,"Ġìļ°ë¦¬ëĤĺ":142336,"Ġìļ°ë¦¬ëĤĺëĿ¼":142337,"ãģ¥ãģıãĤĬ":142338,"ĠاÙĦÙħسÙĦ":142339,"ĠاÙĦÙħسÙĦØŃØ©":142340,"Ġliberté":142341,"клÑİÑĩение":142342,"Ġzamów":142343,"Ġzamówienia":142344,"รà¸ĸà¹Ħà¸Ł":142345,"Ø£ÙģÙĦ":142346,"Ø£ÙģÙĦاÙħ":142347,"Ùħراج":142348,"Ùħراجعة":142349,"Ġë¹ĦêµIJ":142350,"ĠاÙĦتاب":142351,"ĠاÙĦتابعة":142352,"Ġë§ĮëĤĺ":142353,"ĠбÑĥм":142354,"ĠбÑĥмаг":142355,"Ġgénero":142356,"Ġìŀĺ못":142357,"×ŀפ×ķר×ĺ":142358,"è²·ãģĦçī©":142359,"ĠÙĦدÙĬÙĥ":142360,"Ġ×ľ×¢×Ļת":142361,"Ġ×ľ×¢×Ļת×Ļ×Ŀ":142362,"ĠsÅĤab":142363,"ĠпÑĢедÑģÑĤавлÑı":142364,"ãĤ¿ãĤ¤ãĥĪ":142365,"ãĤ¿ãĤ¤ãĥĪãĥ«":142366,"Ùħص":142367,"ÙħصطÙģ":142368,"ÙħصطÙģÙī":142369,"Ġdifficulté":142370,"ãĥĨãĤ£ãĥĸ":142371,"ĠpewnoÅĽci":142372,"ĠpewnoÅĽciÄħ":142373,"Ġ무ìĬ¨":142374,"إرس":142375,"إرساÙĦ":142376,"ĠдалÑĮ":142377,"ĠдалÑĮÑĪе":142378,"Ġ×ľ×ł×¡":142379,"Ġ×ľ×ł×¡×ķת":142380,"หมูà¹Īà¸ļà¹īาà¸Ļ":142381,"×ŀס×ŀ׼×Ļ":142382,"أسÙĦÙĪØ¨":142383,"ĠzwÅĤ":142384,"ĠzwÅĤas":142385,"ĠzwÅĤaszc":142386,"ĠzwÅĤaszcza":142387,"ĠпÑĢеж":142388,"ĠпÑĢежде":142389,"ĠоÑĢганизаÑĨиÑı":142390,"Ġdönemin":142391,"Ġdöneminde":142392,"ĠỦ":142393,"ĠỦy":142394,"ä¸ĭãģĴ":142395,"ĠпоÑģледние":142396,"Ġgüne":142397,"ĠgüneÅŁ":142398,"Ġ×IJ×ĸר":142399,"Ġ×IJ×ĸר×Ĺ×Ļ":142400,"ãģ§ãģĤãĤįãģĨ":142401,"ĠÙĨÙĤ":142402,"ĠÙĨÙĤاط":142403,"æŃ£ãģĹãģĦ":142404,"ĠÑĢег":142405,"ĠÑĢегиона":142406,"ĠFörder":142407,"ê²½ìĺģ":142408,"dıklar":142409,"dıklarını":142410,"trzymaÄĩ":142411,"أشÙĥ":142412,"أشÙĥاÙĦ":142413,"×Ķת×IJ":142414,"×Ķת×IJ×ŀ×Ķ":142415,"à¸Ĺำà¹ĥหà¹īà¹Ģà¸ģิà¸Ķ":142416,"ĠGebä":142417,"ĠGebäude":142418,"ĠСеÑĢг":142419,"ĠСеÑĢгей":142420,"ĠздоÑĢов":142421,"ĠздоÑĢовÑĮÑı":142422,"Ġrãi":142423,"ĠпÑĢедÑĥÑģ":142424,"ĠпÑĢедÑĥÑģмоÑĤÑĢ":142425,"ĠпÑĢедÑĥÑģмоÑĤÑĢен":142426,"Ġ×Ķצ×Ļ×ij":142427,"Ġ×Ķצ×Ļ×ij×ķר×Ļ":142428,"Ġdésir":142429,"ĠноÑĩ":142430,"ĠноÑĩÑĮ":142431,"möglichkeiten":142432,"Ġ×IJ×Ĺר×ķ׳×Ļ×Ŀ":142433,"Ġsoirée":142434,"ĠNháºŃn":142435,"Ùª":142436,"à¸Ľà¸£à¸°à¸§à¸±à¸ķิศาสà¸ķรà¹Į":142437,"êµIJíĨµ":142438,"ĠأخÙĬ":142439,"Ġdécid":142440,"Ġdécidé":142441,"Ġwyja":142442,"ĠwyjaÅĽni":142443,"Ġสิ":142444,"Ġสิà¸ĩ":142445,"Ġสิà¸ĩหา":142446,"Ġสิà¸ĩหาà¸Ħม":142447,"à¹ģà¸Ńรà¹Į":142448,"หà¸Ļà¹īาà¸Īà¸Ń":142449,"סתר":142450,"Ġê¶":142451,"Ġê¶Į":142452,"Ġê¶Į리":142453,"plätze":142454,"بطÙĦ":142455,"ê±´ìĦ¤":142456,"Ġ×IJ×Ļ×ŀ×Ļ":142457,"Ġ×IJ×Ļ×ŀ×Ļ×Ļ׾":142458,"ãģ½":142459,"تراث":142460,"×IJ׾×Ļ×ŀ×ķת":142461,"ĠdisponÃŃveis":142462,"Ġzale":142463,"Ġzależy":142464,"à¸Ľà¸£à¸°à¸Ĭาสัมà¸ŀัà¸Ļà¸ĺà¹Į":142465,"ĠÅļwiat":142466,"Ġporówn":142467,"Ġporówna":142468,"Ġ׾×ĺ×ķ×ijת":142469,"×Ķ×ĸ×ŀ׳×Ķ":142470,"Ġ×Ľ×ª×ķצ×IJ×Ķ":142471,"Ġ×ijק׾":142472,"Ġ×ijק׾×ķת":142473,"ĠоÑĤкÑĢ":142474,"ĠоÑĤкÑĢÑĭва":142475,"ãĥijãĥ¯ãĥ¼":142476,"ë¿IJë§Į":142477,"ĠвÑģÑı":142478,"ĠвÑģÑıк":142479,"ãģ¨ãģªãģ£ãģ¦ãģĦãĤĭ":142480,"ĠgiáºŃn":142481,"ĠокÑĢÑĥ":142482,"ĠокÑĢÑĥжа":142483,"ĠокÑĢÑĥжаÑİÑī":142484,"ĠUniversität":142485,"ĠÑĢож":142486,"ĠÑĢожд":142487,"ĠÑĢождениÑı":142488,"Ø®ÙĬÙĦ":142489,"Ġкомпаний":142490,"ĠÑĢазлиÑĩнÑĭе":142491,"ĠЦена":142492,"׳×Ļ×ķ×ĸ":142493,"׳×Ļ×ķ×ĸ׾":142494,"׳×Ļ×ķ×ĸ׾×ĺר":142495,"Ġê³µê°Ħ":142496,"Ġê°ľëħIJ":142497,"landırma":142498,"ĠÑĥдален":142499,"à¸ŀัà¸ģà¸ľ":142500,"à¸ŀัà¸ģà¸ľà¹Īà¸Ńà¸Ļ":142501,"Ġprotección":142502,"ĠbÅĤ":142503,"ĠbÅĤÄĻd":142504,"ÃĪ":142505,"Ġíĸīë³µ":142506,"ĠÅŁÃ¼":142507,"ĠÅŁÃ¼phe":142508,"ĠíĶ":142509,"Ġíͼ":142510,"Ġíͼíķ´":142511,"Ġëĭ¤ë¥´":142512,"à¹Ħมà¹Īà¹Ģà¸ģิà¸Ļ":142513,"ãģ¿ãģª":142514,"ãģ¿ãģªãģķãĤĵ":142515,"ĠпоÑĤÑĢеб":142516,"ĠпоÑĤÑĢебиÑĤел":142517,"ĠاÙĦÙĥÙĦاÙħ":142518,"ìķĦë²Ħ":142519,"ìķĦë²Ħì§Ģ":142520,"ãĤĴ使ãģ£ãģŁ":142521,"Ġbụi":142522,"ĠпоÑĤеÑĢ":142523,"ĠпоÑĤеÑĢÑı":142524,"ĠØ¢ÙĦاÙģ":142525,"ĠнаÑģÑĤоÑıÑīее":142526,"ãģıãģªãĤĬãģ¾ãģĹãģŁ":142527,"clusão":142528,"ãĤ³ãĥĶãĥ¼":142529,"צפ×Ļ":142530,"צפ×Ļ×Ļ×Ķ":142531,"Ø®ÙĦا":142532,"Ø®ÙĦاص":142533,"ลà¹īำ":142534,"ãĥ¯ãĤ¤ãĥ³":142535,"Ġมีà¸Ļา":142536,"Ġมีà¸Ļาà¸Ħม":142537,"شخص":142538,"شخصÙĬات":142539,"Ġ×ĸ×§":142540,"Ġ×ĸ×§×ķ×§":142541,"×Ļ×Ļצ":142542,"×Ļ×Ļצ×Ĵ":142543,"èĢĥãģĪæĸ¹":142544,"Ġürünü":142545,"ĠиÑģпол":142546,"ĠиÑģполни":142547,"Ġcompañero":142548,"קצ×Ķ":142549,"×ŀ×¢×ł×Ļ×§":142550,"ÙħØŃÙħد":142551,"Ġcámara":142552,"Ġпед":142553,"Ġпедаг":142554,"Ġпедагог":142555,"маÑĢ":142556,"маÑĢк":142557,"×Ķ×ª×ł×Ĵ×ĵ":142558,"ĠìĨĮê°ľ":142559,"ĠcomunitÃł":142560,"곤":142561,"ĠNgÃłi":142562,"สà¸ĩà¸ļ":142563,"ĠmieszkaÅĦców":142564,"ĠÙĨÙĩائÙĬ":142565,"ivité":142566,"Ġиде":142567,"ĠидеалÑĮн":142568,"ĠأسبÙĪØ¹":142569,"Ġ×Ļ×¢×ľ":142570,"Ġ׾ר×IJש":142571,"Ġ׾ר×IJש×ķ׳×Ķ":142572,"ĠзапиÑģи":142573,"ĠкоÑĢпÑĥÑģ":142574,"วà¸ĩศ":142575,"วà¸ĩศà¹Į":142576,"ĠÐĶм":142577,"ĠÐĶмиÑĤ":142578,"ĠÐĶмиÑĤÑĢ":142579,"Ġkönnt":142580,"Ġbölges":142581,"Ġbölgesinde":142582,"׼×Ļ׼":142583,"׼×Ļ׼ר":142584,"ĠاÙĦإثÙĨ":142585,"ĠاÙĦإثÙĨÙĬÙĨ":142586,"Ġngá»Ļ":142587,"ì¹ł":142588,"دراج":142589,"Ġuda":142590,"ĠudaÅĤo":142591,"ìºIJ":142592,"برÙĨاÙħج":142593,"ĠÑģÑĥдеб":142594,"ĠÑģÑĥдебн":142595,"Ġzunächst":142596,"ĠEducación":142597,"ãģ¨ãģªãģ£ãģ¦ãģĦãģ¾ãģĻ":142598,"Ġ×Ķ×IJ×ŀ×Ļת×Ļ":142599,"Ġİnt":142600,"Ġİnternet":142601,"ĠcaÅĤego":142602,"ãĥĹãĥªãĥ³":142603,"إبد":142604,"إبداع":142605,"ĠпоÑĢÑĤал":142606,"à¹Ĥà¸ķà¹ī":142607,"Ġ×Ķקש×ķר":142608,"плод":142609,"ĠÙħد":142610,"ĠÙħدرÙĬد":142611,"×ŀסע×ĵ×Ķ":142612,"ĠØ´ÙĬئ":142613,"ĠØ´ÙĬئا":142614,"à¸ģà¹Īà¸Ńสรà¹īาà¸ĩ":142615,"Ġì°¸ê³ł":142616,"à¹Ģà¸Ĺร":142617,"à¹Ģà¸Ĺรà¸Ķ":142618,"Ġ×ij×ŀקר×Ļ×Ŀ":142619,"Ġbât":142620,"Ġbâtiment":142621,"åij¼ãģ³":142622,"ç´łæķµ":142623,"ç´łæķµãģª":142624,"przedsiÄĻbiorst":142625,"przedsiÄĻbiorstw":142626,"Ġ×ł×ª×ķ׳×Ļ×Ŀ":142627,"×Ĺ׾×ķ×Ŀ":142628,"รวย":142629,"ÙħÙĪØ¶ÙĪØ¹":142630,"ĠÑģобÑĢан":142631,"ведÑĥÑī":142632,"ĠÑĤеаÑĤ":142633,"ĠÑĤеаÑĤÑĢ":142634,"meye":142635,"meyeceÄŁi":142636,"ĠpieniÄħ":142637,"ĠpieniÄħd":142638,"ĠpieniÄħdze":142639,"ÑĢезиденÑĤ":142640,"ØŃصر":142641,"ìĺ¥":142642,"à¹Ģยืà¸Ńà¸Ļ":142643,"ĠÑĥни":142644,"ĠÑĥнивеÑĢ":142645,"ĠÑĥнивеÑĢÑģ":142646,"ĠÑĥнивеÑĢÑģиÑĤеÑĤ":142647,"ĠاÙĦرØŃ":142648,"ĠاÙĦرØŃÙħÙĨ":142649,"ĠÑĤеÑħнолог":142650,"ĠÑĤеÑħнологии":142651,"ìĹIJëĦĪ":142652,"ìĹIJëĦĪì§Ģ":142653,"ĠíķŃ":142654,"ĠíķŃìĥģ":142655,"à¸ĺา":142656,"à¸ĺาà¸ķุ":142657,"ĠEspañol":142658,"×ĵ×Ĵש":142659,"Ġêµī":142660,"Ġêµīìŀ¥":142661,"Ġêµīìŀ¥íŀĪ":142662,"ĠÅĤat":142663,"ĠÅĤatwo":142664,"Ġká»ĭch":142665,"إز":142666,"إزاÙĦØ©":142667,"ĠдейÑģÑĤвие":142668,"ĠsaÄŁlayan":142669,"สุà¸Ķยà¸Ńà¸Ķ":142670,"ĠzostaÄĩ":142671,"ĠdisponÃŃvel":142672,"ïºį":142673,"verständ":142674,"verständlich":142675,"twor":142676,"tworzyÄĩ":142677,"عجز":142678,"à¹Ģà¸Ĥà¹īม":142679,"ยà¹Īà¸Ńม":142680,"Ġstratég":142681,"Ġstratégie":142682,"à¸ľà¸¥à¹Ħมà¹ī":142683,"Ġê°ģì¢ħ":142684,"ĠÙħÙĪØ§":142685,"ĠÙħÙĪØ§Ø¶":142686,"ĠÙħÙĪØ§Ø¶ÙĬع":142687,"اØŃتج":142688,"اØŃتجاج":142689,"ĠẤ":142690,"ĠẤn":142691,"×ŀ×ŀש׾×Ķ":142692,"ĠÅŁekil":142693,"×ŀ×Ĺ׾":142694,"×ŀ×Ĺ׾×ķת":142695,"Ġà¸ĺ":142696,"Ġà¸ĺัà¸Ļ":142697,"Ġà¸ĺัà¸Ļวา":142698,"Ġà¸ĺัà¸Ļวาà¸Ħม":142699,"Ġìĭ¤ìłľ":142700,"Ġìĭ¤ìłľë¡ľ":142701,"ì¤ijìķĻ":142702,"ëįĶëĿ¼":142703,"ĠÑĪиÑĢ":142704,"ĠÑĪиÑĢоко":142705,"Ġsolución":142706,"วาà¸ĩà¹ģà¸ľà¸Ļ":142707,"×IJ×ķ×ĺ×ķ×ŀ":142708,"×IJ×ķ×ĺ×ķ×ŀ×ĺ×Ļ":142709,"ĠÑĢеÑģÑĤ":142710,"ĠÑĢеÑģÑĤоÑĢ":142711,"ĠÑĢеÑģÑĤоÑĢан":142712,"ëį¸":142713,"ÑĤÑĢад":142714,"ÑĤÑĢади":142715,"ÑĤÑĢадиÑĨион":142716,"ÑĤÑĢадиÑĨионн":142717,"มะà¹Ģรà¹ĩ":142718,"มะà¹Ģรà¹ĩà¸ĩ":142719,"à¹Ĥส":142720,"Ġolmasını":142721,"×ŀ×ķסר":142722,"ĠоÑĤноÑĪении":142723,"Ġê°ĢëĬ¥ìĦ±":142724,"Ġyuk":142725,"Ġyukarı":142726,"ìĨĶ":142727,"ĠÑģÑĦ":142728,"ĠÑģÑĦеÑĢе":142729,"Ġ×§×ķפ":142730,"ãĤ±ãĥ¼ãĤ":142731,"ãĤ±ãĥ¼ãĤŃ":142732,"âĢķâĢķ":142733,"ĠاÙĦØ£ÙĦÙħ":142734,"ĠاÙĦØ£ÙĦÙħاÙĨÙĬ":142735,"ẢN":142736,"ת×ķ׼׳×Ļ×ķת":142737,"ĠÑģÑĥÑīеÑģÑĤвÑĥеÑĤ":142738,"æĪijãĢħ":142739,"ĠاÙĦصادر":142740,"ĠTrá»įng":142741,"Ġад":142742,"ĠадминиÑģÑĤ":142743,"ĠадминиÑģÑĤÑĢа":142744,"ĠадминиÑģÑĤÑĢаÑĨи":142745,"ĠдÑĢÑĥгими":142746,"ÑģпеÑĪ":142747,"عÙĦاÙħات":142748,"Ġаб":142749,"ĠабÑģол":142750,"ĠабÑģолÑİÑĤ":142751,"ĠабÑģолÑİÑĤно":142752,"ฤà¸Ķู":142753,"étr":142754,"étranger":142755,"нÑıÑĤи":142756,"нÑıÑĤие":142757,"×¢×ķ׳":142758,"×¢×ķ׳ש":142759,"ĠÙĤائ":142760,"ĠÙĤائÙĦا":142761,"ĠмаÑģ":142762,"ĠмаÑģло":142763,"ãĥīãĤ¤":142764,"ãĥīãĤ¤ãĥĦ":142765,"å¿ħè¦ģãģĮãģĤãĤĬãģ¾ãģĻ":142766,"×ŀ×ķ×ĸ×Ļ×IJ":142767,"×ŀ×ķ×ĸ×Ļ×IJ×ķף":142768,"ĠNgoại":142769,"Ġkênh":142770,"à¸ģารà¸Ńà¸Ńà¸ģà¹ģà¸ļà¸ļ":142771,"×ŀפק":142772,"×ŀפק×ĵ":142773,"ÙħÙĨاز":142774,"ÙħÙĨازÙĦ":142775,"ë·°":142776,"íŤ":142777,"ÙħÙĩارات":142778,"Ġpropriété":142779,"פ×Ĵ×Ļש×Ķ":142780,"ÑĩÑĢ":142781,"ÑĩÑĢеж":142782,"ÑĩÑĢежден":142783,"×Ķ×ķצ×IJ×Ķ":142784,"ØŃÙĥÙĬÙħ":142785,"ĠíĻĪ":142786,"ĠíĻĪíİĺìĿ´ì§Ģ":142787,"åݳ":142788,"åݳãģĹãģĦ":142789,"×¢×ŀ×ĵ×Ķ":142790,"ĠAuÃŁen":142791,"سÙĪØ¡":142792,"ë¹Ī":142793,"ĠÙĪØ®":142794,"ĠÙĪØ®Ø§ØµØ©":142795,"инÑĤеÑĢ":142796,"инÑĤеÑĢеÑģ":142797,"èĩ´ãģĹãģ¾ãģĻ":142798,"Ġhüküm":142799,"à¹Ħà¸Ĥมัà¸Ļ":142800,"Ġdavran":142801,"ĠdavranÄ±ÅŁ":142802,"à¹Ģà¸ķียà¸ĩ":142803,"вÑĢем":142804,"вÑĢеменно":142805,"à¹Ģà¸Ĺศà¸ģา":142806,"à¹Ģà¸Ĺศà¸ģาล":142807,"å¼ķãģ£":142808,"å¼ķãģ£è¶ĬãģĹ":142809,"×IJר×ķ×Ĺ":142810,"×IJר×ķ×Ĺת":142811,"à¹Ģวิ":142812,"à¹Ģวิรà¹Į":142813,"à¸Ńยà¹Īาà¸ĩรวà¸Ķà¹Ģรà¹ĩว":142814,"ĠìŬíĸī":142815,"ĠÑĢанÑĮ":142816,"ĠÑĢанÑĮÑĪе":142817,"Ġzobow":142818,"ĠzobowiÄħ":142819,"ĠzobowiÄħz":142820,"Ġ×ķ׼×ŀ×ķ×ijף":142821,"ĠاÙĦÙħÙĩ":142822,"ĠاÙĦÙħÙĩÙĨÙĬ":142823,"ãĤ¢ãĤ¸":142824,"ãĤ¢ãĤ¸ãĤ¢":142825,"ë°©ìĨ¡":142826,"à¸Ńà¸Ńà¸ģà¸ģำลัà¸ĩ":142827,"à¸Ńà¸Ńà¸ģà¸ģำลัà¸ĩà¸ģาย":142828,"améli":142829,"améliorer":142830,"å½ĵãģŁãĤĬåīį":142831,"Ġregelm":142832,"ĠregelmÃ¤ÃŁig":142833,"ãģĬåĭ":142834,"ãģĬåĭ§":142835,"ãģĬåĭ§ãĤģ":142836,"Ġmưá»Ŀi":142837,"برÙħج":142838,"ĠNatürlich":142839,"ĠDÅ©ng":142840,"ĠاÙĦرجاÙĦ":142841,"Ġthép":142842,"ĠolmuÅŁtur":142843,"×ŀ×ķס×Ļ×§×Ķ":142844,"fälle":142845,"주íĥĿ":142846,"ĠاÙĦÙģØ±Øµ":142847,"ĠnajwiÄĻks":142848,"ĠnajwiÄĻkszy":142849,"ĠçaÄŁ":142850,"ĠçaÄŁrı":142851,"ì¸ł":142852,"ĠvÃŃct":142853,"ĠvÃŃctima":142854,"ĠÑģовеÑĢÑĪен":142855,"×Ķ×Ļ×Ļת×Ļ":142856,"à¹Ģà¸Ķี":142857,"à¹Ģà¸Ķีà¹ĭ":142858,"à¹Ģà¸Ķีà¹ĭยว":142859,"üyü":142860,"Ġдоп":142861,"Ġдополн":142862,"ĠдополниÑĤелÑĮно":142863,"à¹ģà¸ķà¸ģà¸ķà¹Īาà¸ĩà¸ģัà¸Ļ":142864,"Ġál":142865,"Ġálbum":142866,"à¸Ľà¸£à¸°à¸Īà¸³à¸Ľà¸µ":142867,"ĠÑĦедеÑĢ":142868,"ĠÑĦедеÑĢалÑĮн":142869,"ĠobsÅĤ":142870,"ĠobsÅĤugi":142871,"à¹Ģรืà¹Ī":142872,"à¹Ģรืà¹Īà¸Ńย":142873,"à¹Ģรืà¹Īà¸Ńยà¹Ĩ":142874,"ëģĮ":142875,"Ġnghìn":142876,"ĠBaÅŁkanlıģı":142877,"تأسÙĬ":142878,"تأسÙĬس":142879,"Ġ×ij×ij×ķקר":142880,"Ġ×¢×ij×ķ×ĵ×ķת":142881,"ĠبصÙĪØ±Ø©":142882,"ãĤıãģijãģ§ãģ¯ãģªãģĦ":142883,"führer":142884,"ãĤ¹ãĤŃ":142885,"ãĤ¹ãĤŃãĥ«":142886,"ĠاÙĦÙĤض":142887,"ĠاÙĦÙĤضÙĬØ©":142888,"ĠдолжноÑģÑĤ":142889,"ÙģØ§Ø±ÙĤ":142890,"Ġcomeçou":142891,"Ġorganisé":142892,"Ġxuân":142893,"ĠÑģообÑīаеÑĤ":142894,"ĠпÑĢид":142895,"ĠпÑĢидеÑĤÑģÑı":142896,"TÃľRK":142897,"ãĥ¬ãĥ¼ãĤ·ãĥ§ãĥ³":142898,"Không":142899,"استÙģ":142900,"استÙģØ§Ø¯Ø©":142901,"ä¸ĬãģĮãģ£ãģ¦":142902,"Ġumie":142903,"ĠumiejÄĻ":142904,"ĠumiejÄĻtn":142905,"ĠumiejÄĻtnoÅĽci":142906,"ëĤ¸":142907,"à¹Ģà¸Ļà¸Ńรà¹Į":142908,"×ĵ×ķ×ķ×Ĺ":142909,"ÃŃsimo":142910,"IÃĬ":142911,"IÃĬN":142912,"Ġalcanç":142913,"Ġà¸ķุ":142914,"Ġà¸ķุลา":142915,"Ġà¸ķุลาà¸Ħม":142916,"ש׾×ĺ×ķף":142917,"Ġélè":142918,"Ġélèves":142919,"ĠÄiju":142920,"ĠÄijuá»ķi":142921,"ĠØ£Ùģ":142922,"ĠØ£Ù쨱ÙĬ":142923,"ĠØ£Ù쨱ÙĬÙĤÙĬ":142924,"ĠØ£Ù쨱ÙĬÙĤÙĬا":142925,"ãĤĴæİ¢ãģĻ":142926,"ĠпÑĢедложениÑı":142927,"جاد":142928,"ĠÑħоÑĤÑĮ":142929,"Ñģал":142930,"Ñģалон":142931,"à¸Ľà¸£à¸°à¹Ģม":142932,"à¸Ľà¸£à¸°à¹Ģมิà¸Ļ":142933,"ãĤŃãĥĥãĥģ":142934,"ãĤŃãĥĥãĥģãĥ³":142935,"×ij×ĵ×Ļ×§×ķת":142936,"Ġchù":142937,"Ġchùa":142938,"ÐĴиде":142939,"ÐĴидео":142940,"иÑĢовка":142941,"ĠÑħоÑĤиÑĤе":142942,"Ġspécifique":142943,"รสà¸Ĭาà¸ķิ":142944,"è¾¼ãĤĵãģł":142945,"伸ãģ³":142946,"×Ķצ׾×Ĺת":142947,"ãģ©ãģ®ãĤĪãģĨãģ«":142948,"سعادة":142949,"Ġлид":142950,"ĠлидеÑĢ":142951,"มà¸ĩ":142952,"มà¸ĩà¸Ħล":142953,"ØŃاÙħÙĦ":142954,"หลุà¸Ķ":142955,"à¸Ńยà¹Īาà¸ĩà¸ķà¹Īà¸Ń":142956,"à¸Ńยà¹Īาà¸ĩà¸ķà¹Īà¸Ńà¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ":142957,"ãģķãģĽãģ¦éłĤ":142958,"تسÙĪÙĬ":142959,"تسÙĪÙĬÙĤ":142960,"ĠaÅŁaģıd":142961,"ĠaÅŁaģıdaki":142962,"ĠÑĨелÑĮ":142963,"ĠÑĨелÑĮÑİ":142964,"ĠAraÅŁtırma":142965,"à¸Ĥัà¸ļรà¸ĸ":142966,"ÙĩذÙĩ":142967,"ลà¸ĩà¸Ĺะ":142968,"ลà¸ĩà¸Ĺะà¹Ģà¸ļ":142969,"ลà¸ĩà¸Ĺะà¹Ģà¸ļียà¸Ļ":142970,"تÙĥاÙħÙĦ":142971,"Ġcio":142972,"Ġcioè":142973,"ãģ¦ãģĬãģı":142974,"ĠاÙĦصØŃÙģÙĬ":142975,"ĠíĬ¹ìłķ":142976,"полниÑĤÑĮ":142977,"ãĤĵãģĺãĤĥãģªãģĦ":142978,"ãĤĵãģĺãĤĥãģªãģĦãģĭ":142979,"ĠاÙĦجÙĩ":142980,"ĠاÙĦجÙĩات":142981,"ĠÑĥÑģпеÑĪно":142982,"Ġвок":142983,"ĠвокÑĢÑĥг":142984,"ĠÑģиÑĤÑĥаÑĨиÑı":142985,"Ġ×Ķ×IJ×ŀר":142986,"Ġ×Ķ×IJ×ŀר×Ļ×§":142987,"Ġ×Ķ×IJ×ŀר×Ļ×§×IJ×Ļ":142988,"×ŀ×Ĵ×ĸ":142989,"×ŀ×Ĵ×ĸ×Ļף":142990,"ĠакÑĤÑĥ":142991,"ĠакÑĤÑĥалÑĮн":142992,"éta":142993,"étais":142994,"ĠmogÅĤa":142995,"ĠÑĤоÑĩки":142996,"Ġ×ŀ×Ķ×ŀ×¢":142997,"Ġ×ŀ×Ķ×ŀ×¢×¨×Ľ×ª":142998,"à¸¡à¸µà¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺà¸´à¸łà¸²à¸ŀ":142999,"×Ļר×Ļ×ĵ×Ķ":143000,"×Ĵר×ŀ׳":143001,"×Ĵר×ŀ׳×Ļ×Ķ":143002,"Ġглав":143003,"Ġглавное":143004,"Ġ미ëŀĺ":143005,"Ġ׳׼×ķ׳×Ķ":143006,"ĠÙĪØ·ÙĨÙĬ":143007,"opport":143008,"opportunitÃł":143009,"Ġhá»§y":143010,"ĠÙĦتØŃ":143011,"ĠÙĦتØŃÙĤÙĬÙĤ":143012,"Ġórg":143013,"Ġórgão":143014,"ãĤ¹ãĥĶ":143015,"ãĤ¹ãĥĶãĥ¼ãĥī":143016,"Ġönü":143017,"Ġönüne":143018,"ÙħعاÙħÙĦ":143019,"ש×ŀ×Ļר×Ķ":143020,"ĠвеÑģÑĮма":143021,"ĠwiÄĻkszo":143022,"ĠwiÄĻkszoÅĽÄĩ":143023,"ĠاستراتÙĬج":143024,"ĠاستراتÙĬجÙĬØ©":143025,"ĠÙ쨥":143026,"ĠÙģØ¥Ø°Ø§":143027,"à¹Ģà¸Ĭืà¹Īà¸Ńม":143028,"à¹Ģà¸Ĭืà¹Īà¸Ńมà¸ķà¹Īà¸Ń":143029,"Ġ׾פר":143030,"Ġ׾פר×ĺ×Ļ×Ŀ":143031,"ÙħضÙĬ":143032,"ĠGerçek":143033,"Ġçocukların":143034,"ÙĪØ«Ø§Ø¦ÙĤ":143035,"ĠÙħساءÙĭ":143036,"Ġunterstützt":143037,"Ġprést":143038,"Ġpréstamo":143039,"ĠÐłÐ°Ð·Ð¼ÐµÑĢ":143040,"ĠÅŁeker":143041,"Ġséculo":143042,"×ij×Ķ×Ļר":143043,"Ø´ÙĩÙĪØ±":143044,"Ġà¸Ńีà¸ģ":143045,"Ġà¸Ńีà¸ģà¸Ĺัà¹īà¸ĩ":143046,"Ġllegó":143047,"à¸¨à¸´à¸¥à¸Ľà¸°":143048,"æĪijãģĮ":143049,"æĪijãģĮå®¶":143050,"عÙĤÙĪ":143051,"عÙĤÙĪØ¨Ø§Øª":143052,"ĠFälle":143053,"ĠsÅĤuż":143054,"ĠsÅĤużb":143055,"ĠاÙĦØŃÙĤÙĪÙĤ":143056,"ĠплиÑĤ":143057,"ĠиноÑģÑĤ":143058,"ĠиноÑģÑĤÑĢан":143059,"ĠиноÑģÑĤÑĢанн":143060,"à¹ĥà¸Ļà¸Ĥà¸ĵะà¸Ĺีà¹Ī":143061,"ãĤ«ãĥĨ":143062,"ãĤ«ãĥĨãĤ´":143063,"ãĤ«ãĥĨãĤ´ãĥª":143064,"à¸Ńิส":143065,"à¸Ńิสระ":143066,"à¹Ģà¸ľà¸¢à¹ģ":143067,"à¹Ģà¸ľà¸¢à¹ģà¸ŀร":143068,"à¹Ģà¸ľà¸¢à¹ģà¸ŀรà¹Ī":143069,"ãģĬãģĦ":143070,"ãģĬãģĦãģĹãģĦ":143071,"استÙĤÙĦ":143072,"استÙĤÙĦاÙĦ":143073,"تØŃض":143074,"تØŃضÙĬر":143075,"åĬ©ãģij":143076,"ÙħراÙģÙĤ":143077,"Ġ×ĵ×ķר":143078,"Ġ×ĵ×ķרש":143079,"×ŀת×Ļ×Ļ×Ĺס":143080,"ס×Ļ׼":143081,"ס×Ļ׼×ķ×Ŀ":143082,"íĮĮíĬ¸":143083,"ĠwyÅĽ":143084,"ĠwyÅĽw":143085,"ĠwyÅĽwiet":143086,"ĠwyÅĽwietl":143087,"ĠاÙĦاÙĨساÙĨ":143088,"ĠStraÃŁen":143089,"L":143090,"ãģ«åŁº":143091,"ãģ«åŁºãģ¥":143092,"ĠcapÃŃtulo":143093,"ลุย":143094,"Ġ×Ķ×ŀקצ×ķ×¢×Ļ":143095,"ãģĤãĤĭç¨ĭ度":143096,"Ợ":143097,"ĠاÙĦÙĦا":143098,"ĠاÙĦÙĦازÙħØ©":143099,"æķĻãģĪ":143100,"Ġרש×IJ×Ļ":143101,"зав":143102,"завиÑģ":143103,"завиÑģим":143104,"à¸Ľà¸±à¸Īà¸Īัย":143105,"à¹Ģà¸ĭล":143106,"à¹Ģà¸ĭลลà¹Į":143107,"Ġdifférence":143108,"ĠAltın":143109,"ĠкÑĢай":143110,"ĠкÑĢайне":143111,"Ġзло":143112,"Ġgünümüz":143113,"ĠнаÑĤÑĥÑĢ":143114,"ĠнаÑĤÑĥÑĢалÑĮн":143115,"×Ĵ×ķ׾ש×Ļ×Ŀ":143116,"ĠкаÑĤегоÑĢ":143117,"ĠкаÑĤегоÑĢии":143118,"Ġзнак":143119,"à¸ģà¹Īà¸Ńà¸Ļหà¸Ļà¹īา":143120,"à¸ģà¹Īà¸Ńà¸Ļหà¸Ļà¹īาà¸Ļีà¹ī":143121,"ĠÙħÙĨت":143122,"ĠÙħÙĨتخب":143123,"ãĥĽãĥ¼ãĥ«":143124,"ĠевÑĢо":143125,"สว":143126,"สวม":143127,"ĠìľĦìĽIJ":143128,"ĠìľĦìĽIJëĭĺ":143129,"ĠاÙĦØŃÙĪØ«":143130,"ĠاÙĦØŃÙĪØ«ÙĬ":143131,"ĠÑģодеÑĢжиÑĤ":143132,"ãĥķãĤ¡ãĥĥãĤ·ãĥ§ãĥ³":143133,"Ġà¸ģัà¸Ļ":143134,"Ġà¸ģัà¸Ļย":143135,"Ġà¸ģัà¸Ļยายà¸Ļ":143136,"ãĤªãĥª":143137,"ãĤªãĥªãĤ¸":143138,"ãĤªãĥªãĤ¸ãĥĬãĥ«":143139,"ĠбÑĢенд":143140,"ãĤĴæĮģãģ£ãģ¦ãģĦãĤĭ":143141,"Ġinversión":143142,"Ġê°ĸ":143143,"Ġê°ĸê³ł":143144,"ĠnovitÃł":143145,"ê´Ģê´ij":143146,"Ġà¸ŀฤษ":143147,"Ġà¸ŀà¸¤à¸©à¸łà¸²":143148,"Ġà¸ŀà¸¤à¸©à¸łà¸²à¸Ħม":143149,"×ķר×Ĺ×Ļ×Ŀ":143150,"׼׾×ķ׾":143151,"Ġngạc":143152,"×Ļ×Ļש":143153,"×Ļ×Ļש×ķ×ij":143154,"fäll":143155,"fällig":143156,"ĠÑĤÑĢебÑĥеÑĤÑģÑı":143157,"Ġcará":143158,"Ġcarácter":143159,"ĠprincÃŃpio":143160,"ĠÅĤaz":143161,"ĠÅĤazien":143162,"ĠÅĤazienk":143163,"Ġgiãn":143164,"ÑģÑĤÑĢаива":143165,"Ùħساب":143166,"ÙħسابÙĤØ©":143167,"à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩà¸Ķืà¹Īม":143168,"ترÙĥÙĬب":143169,"volução":143170,"ĠÐŁÐ¾Ñĩ":143171,"ĠÐŁÐ¾Ñĩем":143172,"ĠÐŁÐ¾ÑĩемÑĥ":143173,"казалоÑģÑĮ":143174,"ĠпÑĢименениÑı":143175,"à¹Ģà¸Ĺียม":143176,"íĮĶ":143177,"à¸Ĥà¹īà¸Ńà¹Ģสà¸Ļà¸Ń":143178,"à¸Ľà¸±à¸įà¸įา":143179,"ĠобÑĥÑĩ":143180,"ĠобÑĥÑĩениÑı":143181,"ĠÑģеÑĢи":143182,"ĠÑģеÑĢиал":143183,"Ġinglés":143184,"ĠÙĦÙĥرة":143185,"Ġ×ĺ׾":143186,"Ġ×ĺ׾פ×ķף":143187,"Ġìłij":143188,"Ġìłijê·¼":143189,"×IJ×ķ×Ĵ":143190,"×IJ×ķ×Ĵ×ķס":143191,"×IJ×ķ×Ĵ×ķס×ĺ":143192,"ĠболÑĮÑĪое":143193,"ĠÐļонеÑĩно":143194,"×¢×Ļת×ķ׳":143195,"×¢×Ļת×ķ׳×IJ×Ļ":143196,"Ġкнопк":143197,"Ġзн":143198,"ĠзнаÑĤÑĮ":143199,"ĠÄijá»±":143200,"ĠÄijá»±ng":143201,"влаж":143202,"влажн":143203,"×ŀ×Ļ×ĺ×ij":143204,"ãĤ¬ãĤ¤":143205,"ãĤ¬ãĤ¤ãĥī":143206,"..........":143207,"Ġà¸ģุม":143208,"Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀ":143209,"Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļ":143210,"Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļà¸ĺ":143211,"Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļà¸ĺà¹Į":143212,"bez":143213,"bezpieczeÅĦst":143214,"bezpieczeÅĦstw":143215,"ãĥijãĥijæ´»":143216,"عاط":143217,"عاطÙģ":143218,"ĠÄijáºŃm":143219,"ĠзÑĢ":143220,"ĠзÑĢениÑı":143221,"Ġborç":143222,"Ġнедел":143223,"ĠнеделÑİ":143224,"Ġhá»ı":143225,"Ġhá»ıng":143226,"ìŀ¥ìķł":143227,"ìŀ¥ìķłìĿ¸":143228,"ĠاÙĦعÙĦاÙĤØ©":143229,"Ġíģ¬":143230,"Ġíģ¬ê²Į":143231,"à¹Ħรà¹Ī":143232,"à¸ļาà¸Ķ":143233,"à¸ļาà¸Ķà¹Ģà¸Īà¹ĩà¸ļ":143234,"à¸Ŀรั":143235,"à¸Ŀรัà¹Īà¸ĩ":143236,"à¸Ŀรัà¹Īà¸ĩà¹Ģศ":143237,"à¸Ŀรัà¹Īà¸ĩà¹Ģศส":143238,"רע×Ļ":143239,"רע×Ļ×ķ׳×ķת":143240,"ĠëĮ":143241,"ĠëĮĵ":143242,"ĠëĮĵê¸Ģ":143243,"Ġnajb":143244,"Ġnajbli":143245,"Ġnajbliż":143246,"Ġnajbliższ":143247,"ĠиÑģполÑĮзÑĥеÑĤÑģÑı":143248,"ĠcientÃŃf":143249,"ĠcientÃŃfico":143250,"×¢×ŀ×§":143251,"Ġgợi":143252,"Ø´ØŃÙĨ":143253,"ĠÅĽm":143254,"ĠÅĽmier":143255,"ĠÅĽmierci":143256,"à¸Ħาสิà¹Ĥà¸Ļà¸Ńà¸Ńà¸Ļà¹Ħลà¸Ļà¹Į":143257,"×Ĺש×ijת×Ļ":143258,"Ġningu":143259,"Ġninguém":143260,"è¾¼ãĤģ":143261,"ãģ·":143262,"ĠÑĥг":143263,"ĠÑĥгол":143264,"ï½°":143265,"פת×Ļ×Ĺ":143266,"פת×Ļ×Ĺת":143267,"Ġ×Ķר×IJש×ķ׳×Ļ×Ŀ":143268,"pósito":143269,"ãĤŃãĥ¬ãĤ¤":143270,"ãģ©ãģĵãĤį":143271,"à¹Ģà¸Ĺà¹Īาà¹Ħ":143272,"à¹Ģà¸Ĺà¹Īาà¹Ħหร":143273,"à¹Ģà¸Ĺà¹Īาà¹Ħหรà¹Ī":143274,"ĠинÑĤеÑĢÑĮеÑĢ":143275,"ĠØŃاج":143276,"ĠØŃاجة":143277,"สีà¸Ĥาว":143278,"ìĸ¼":143279,"Ġná»Ļ":143280,"Ġná»Ļp":143281,"ĠÃŃnd":143282,"ĠÃŃndice":143283,"สำรวà¸Ī":143284,"Ġкаждой":143285,"Ġhotéis":143286,"ĠnastÄĻ":143287,"ĠnastÄĻpn":143288,"Ġ×Ķ×§×ķ×ĵ":143289,"Ġ×Ķ×§×ķ×ĵ×Ŀ":143290,"פ×ķפ":143291,"פ×ķפ×ķ׾":143292,"פ×ķפ×ķ׾ר×Ļ":143293,"вÑĪей":143294,"ãĤ·ãĥ³ãĥĹ":143295,"ãĤ·ãĥ³ãĥĹãĥ«":143296,"ĠzdjÄĻÄĩ":143297,"ĠгÑĢÑĥппа":143298,"ĠпомеÑī":143299,"ĠпомеÑīениÑı":143300,"ãģ©ãģĨãģĦãģĨ":143301,"ĠиÑģпÑĭÑĤа":143302,"ĠogÅĤ":143303,"ĠogÅĤos":143304,"ĠogÅĤoszen":143305,"ĠogÅĤoszeni":143306,"สรà¹īาà¸ĩสรร":143307,"สรà¹īาà¸ĩสรรà¸Ħà¹Į":143308,"à¸ŀรรà¸ĵ":143309,"ĠçıkÄ±ÅŁ":143310,"ĠÑĩаÑģÑĤноÑģÑĤи":143311,"Ġ×ķ×Ļ×ķתר":143312,"ç¶ļãģįãĤĴ":143313,"ç¶ļãģįãĤĴèªŃ":143314,"ç¶ļãģįãĤĴèªŃãĤĢ":143315,"à¸ģรั":143316,"à¸ģรัม":143317,"гÑĢаÑĦ":143318,"Ġвлад":143319,"ĠвладелÑĮ":143320,"ĠвладелÑĮÑĨ":143321,"ĠistediÄŁ":143322,"ĠistediÄŁiniz":143323,"×ij×ľ×¢":143324,"×ij×ľ×¢×ĵ×Ļ":143325,"ÙħÙĪØ§Ùģ":143326,"ÙħÙĪØ§ÙģÙĤØ©":143327,"Ġ×Ļ×ķר":143328,"Ġ×Ļ×ķרק":143329,"ãĤ«ãĥ¼ãĥīãĥŃãĥ¼ãĥ³":143330,"ĠاÙĦÙħØ´ÙĥÙĦ":143331,"ĠاÙĦÙħØ´ÙĥÙĦØ©":143332,"ĠêµŃíļĮ":143333,"ספ×ĺ":143334,"ספ×ĺ×ŀ":143335,"ספ×ĺ×ŀ×ijר":143336,"Ġìĸ´ëłµ":143337,"ÙĥاÙħ":143338,"ÙĥاÙħÙĬرا":143339,"schlü":143340,"schlüsse":143341,"ĠØ«ÙĨ":143342,"ĠØ«ÙĨائÙĬ":143343,"ìī½":143344,"ĠÐŀÑģоб":143345,"ĠÐŀÑģобенно":143346,"ĠинвеÑģÑĤи":143347,"ĠинвеÑģÑĤиÑĨи":143348,"اØŃتÙħ":143349,"اØŃتÙħاÙĦ":143350,"EÄŀ":143351,"EÄŀİ":143352,"íķĺê²łëĭ¤":143353,"Ġ×IJ×ijר×Ķ":143354,"Ġ×IJ×ijר×Ķ×Ŀ":143355,"Ġ×ij×Ĺ×Ļ׳×Ŀ":143356,"Ø£ÙĪØ¶":143357,"Ø£ÙĪØ¶Ø§Ø¹":143358,"Ġdél":143359,"Ġdélai":143360,"Ġ×IJ×ķ×Ķ×ij×Ļ×Ŀ":143361,"ĠÑģоÑħ":143362,"ĠÑģоÑħÑĢ":143363,"ĠÑģоÑħÑĢани":143364,"ĠдоÑģÑĤиж":143365,"ĠдоÑģÑĤижени":143366,"สิà¹Īà¸ĩà¹ģ":143367,"สิà¹Īà¸ĩà¹ģวà¸Ķ":143368,"สิà¹Īà¸ĩà¹ģวà¸Ķล":143369,"สิà¹Īà¸ĩà¹ģวà¸Ķลà¹īà¸Ńม":143370,"ĠاÙĦÙħباشر":143371,"ĠÑĦиг":143372,"ĠÑĦигÑĥÑĢ":143373,"можем":143374,"׾×ŀ×Ļ×ĵ×Ķ":143375,"Ġciné":143376,"Ġcinéma":143377,"Ġbada":143378,"ĠbadaÅĦ":143379,"جبÙĩØ©":143380,"Ġдеп":143381,"ĠдепÑĥÑĤ":143382,"ĠдепÑĥÑĤаÑĤ":143383,"Ġdistância":143384,"ĠاÙĦÙħعار":143385,"ĠاÙĦÙħعارضة":143386,"thèse":143387,"ünc":143388,"üncü":143389,"Ġданного":143390,"ĠBelgi":143391,"ĠBelgië":143392,"Ġ×ij×ij×§":143393,"Ġ×ij×ijקש×Ķ":143394,"ยà¹Īาà¸Ļ":143395,"Ġsolução":143396,"Ġ×Ķצ×ĺר":143397,"Ġ×Ķצ×ĺרפ×ķ":143398,"ĠØ£ÙĨØŃ":143399,"ĠØ£ÙĨØŃاء":143400,"ĠدÙħØ´":143401,"ĠدÙħØ´ÙĤ":143402,"มัà¹ī":143403,"มัà¹īย":143404,"Ùħغرب":143405,"استعÙħاÙĦ":143406,"ĠSÅĤow":143407,"ĠëıĻìĭľ":143408,"ĠëıĻìĭľìĹIJ":143409,"ĠÑģоÑģ":143410,"ĠÑģоÑģед":143411,"ì²ŃìĨĮ":143412,"ì²ŃìĨĮëħĦ":143413,"ĠгÑĢаÑĦ":143414,"ĠгÑĢаÑĦик":143415,"ĠìŀijìĿĢ":143416,"Ġyeti":143417,"ĠyetiÅŁtir":143418,"ĠìĿ´ê²ĥìĿ´":143419,"หà¹Īาà¸ĩ":143420,"Ø¥ÙħÙĥاÙĨ":143421,"Ø¥ÙħÙĥاÙĨÙĬØ©":143422,"استعراض":143423,"Ùħخدر":143424,"ĠÑĩÑĥÑĤÑĮ":143425,"ÙħدÙĬر":143426,"ÙħدÙĬرÙĬØ©":143427,"Ġà¹Ģมษ":143428,"Ġà¹Ģมษายà¸Ļ":143429,"ĠмеÑħ":143430,"ĠмеÑħаниз":143431,"ĠмеÑħанизм":143432,"ĠÑģÑĥм":143433,"ĠÑģÑĥммÑĥ":143434,"Ġvö":143435,"Ġvöll":143436,"Ġvöllig":143437,"ĠдÑĢÑĥз":143438,"ĠдÑĢÑĥзÑĮÑı":143439,"ãĤĴåĪ©ç͍ãģĹãģ¦":143440,"à¸ļรรà¸Īุ":143441,"pożycz":143442,"×ŀש׼":143443,"×ŀ×©×Ľ×ł×ª":143444,"×ŀ×©×Ľ×ł×ª×IJ":143445,"Ġeuropéen":143446,"Ġproprié":143447,"Ġpropriétaire":143448,"Ġkhấu":143449,"ãģĦãģŁãģłãģijãĤĭ":143450,"Ġtecrü":143451,"Ġtecrübe":143452,"×Ķ×ij":143453,"×Ķ×ij׳×Ķ":143454,"ĠcuÌ":143455,"ĠcuÌī":143456,"ĠcuÌīa":143457,"×IJ×ķ×ķ":143458,"×IJ×ķ×ķ×Ļר×Ķ":143459,"Ġ׼×ķ׾×ķ":143460,"Ulus":143461,"Uluslararası":143462,"Ġ׳×ķת":143463,"Ġ׳×ķ×ª×Ł":143464,"ãģ«åIJij":143465,"ãģ«åIJijãģijãģ¦":143466,"ë¹Ľ":143467,"à¸Ĺัà¸ģษ":143468,"à¸Ĺัà¸ģษะ":143469,"سÙĤÙĪ":143470,"سÙĤÙĪØ·":143471,"Ġвн":143472,"ĠвнеÑĪ":143473,"ĠвнеÑĪне":143474,"Ġurz":143475,"ĠurzÄĻd":143476,"Ġámb":143477,"Ġámbito":143478,"à¸Ńà¸ĺิ":143479,"à¸Ńà¸ĺิà¸ļาย":143480,"ĠÅĤad":143481,"ĠÅĤadn":143482,"ê±´ì¶ķ":143483,"wództ":143484,"wództw":143485,"Ġquestões":143486,"Ġשק":143487,"Ġשק×Ļ×ij׾":143488,"ĠmiejscowoÅĽci":143489,"Ġвал":143490,"ĠвалÑİÑĤ":143491,"häuser":143492,"หà¸Ļà¸Ńà¸ĩ":143493,"ãģ¨åħ±":143494,"ãģ¨åħ±ãģ«":143495,"ãĥıãĥ¼ãĥī":143496,"Ġê°ľìµľ":143497,"ĠоÑģновном":143498,"ĠмÑıÑģ":143499,"اعت":143500,"اعتÙĤاÙĦ":143501,"สà¸ĸิ":143502,"สà¸ĸิà¸ķิ":143503,"Ngu":143504,"Nguá»ĵn":143505,"ĠÙħجÙĦ":143506,"ĠÙħجÙĦØ©":143507,"à¹ģà¸Ĥà¸Ļ":143508,"ĠاÙĦÙĦÙĬبÙĬ":143509,"פע×Ļ׾×ķ×Ļ×ķת":143510,"Ġ×Ķרפ×ķ×IJ×Ļ":143511,"פר×ķפ":143512,"פר×ķפ×Ļ׾":143513,"ק׾×IJ":143514,"ק׾×IJס×Ļ":143515,"ÙĥتشÙģ":143516,"ãģ«ãģªãģ£ãģ¦ãģĹãģ¾ãģĨ":143517,"à¹Ģà¸Ħลà¹ĩà¸Ķ":143518,"à¹Ģà¸Ħลà¹ĩà¸Ķลัà¸ļ":143519,"Ġì»´":143520,"Ġì»´íĵ¨":143521,"Ġì»´íĵ¨íĦ°":143522,"Ġ×Ĺ×Ļ×ķ×ij×Ļ":143523,"Ġnäm":143524,"Ġnämlich":143525,"åij¼ãģ°":143526,"åij¼ãģ°ãĤĮ":143527,"ĠÑĢол":143528,"ĠÑĢоли":143529,"Ġspécialisé":143530,"à¸Ļวัà¸ķ":143531,"à¸Ļวัà¸ķà¸ģรรม":143532,"ÙĨصÙĪØµ":143533,"пеÑĢед":143534,"пеÑĢедаÑĩ":143535,"thèque":143536,"Ġר×IJ×Ļת×Ļ":143537,"ãĥĢãĤ¦ãĥ³":143538,"ãĤıãģĭ":143539,"ãĤıãģĭãģ£ãģ¦":143540,"беÑĢеж":143541,"ĠÑģек":143542,"ĠÑģекÑĢ":143543,"ĠÑģекÑĢеÑĤ":143544,"ĠпоÑģÑĤоÑıнн":143545,"à¸Ĥà¸Ļสà¹Īà¸ĩ":143546,"Ġmük":143547,"Ġmükem":143548,"Ġmükemmel":143549,"еÑĤеÑģÑĮ":143550,"ĠاÙĦسÙĨÙĪØ§Øª":143551,"ĠìłĦíĺĢ":143552,"Ġ×Ķ×ŀ×§×ķר×Ļ":143553,"Ġmüd":143554,"Ġmüdah":143555,"Ġmüdahale":143556,"Ġwyb":143557,"Ġwybór":143558,"Ġtendência":143559,"إدار":143560,"إدارÙĬØ©":143561,"Ġunterstützen":143562,"ת×ijר":143563,"ת×ijרר":143564,"Ġdiá":143565,"Ġdiálogo":143566,"ĠÃĸnce":143567,"ĠÃĸnceki":143568,"ãĤ¹ãĥĿãĥĥãĥĪ":143569,"ëĦ£":143570,"ĠGeli":143571,"ĠGeliÅŁ":143572,"ãĤĴéĢļ":143573,"ãĤĴéĢļãģĹãģ¦":143574,"ĠFuÃŁball":143575,"Ġsalari":143576,"Ġsalarié":143577,"ĠпÑĢодÑĥкÑĤов":143578,"صÙģÙĤØ©":143579,"รวà¸ļ":143580,"รวà¸ļรวม":143581,"à¹ĥà¸Ļà¸IJาà¸Ļ":143582,"à¹ĥà¸Ļà¸IJาà¸Ļะ":143583,"Ġkayna":143584,"Ġkaynaģı":143585,"ĠìŀijíĴĪ":143586,"ĠвÑĭÑĢаж":143587,"ĠвÑĭÑĢажен":143588,"ĠÑģÑĤеп":143589,"ĠÑģÑĤепени":143590,"ĠاÙĦÙħÙĪØ¬ÙĪØ¯":143591,"ĠاÙĦÙħÙĪØ¬ÙĪØ¯Ø©":143592,"ลà¹īม":143593,"ĠnajczÄĻ":143594,"ĠnajczÄĻÅĽcie":143595,"ĠnajczÄĻÅĽciej":143596,"Ġzwy":143597,"Ġzwyk":143598,"ĠzwykÅĤ":143599,"Ġê·¸ëłĩì§Ģ":143600,"à¸ģระà¸Ī":143601,"à¸ģระà¸Īาย":143602,"Ġëĭµ":143603,"Ġëĭµë³Ģ":143604,"ĠÑĢеак":143605,"ĠÑĢеакÑĨи":143606,"ĠÅĽwież":143607,"ĠÑģÑĤоимоÑģÑĤи":143608,"ÙħÙĨاÙĤ":143609,"ÙħÙĨاÙĤØ´":143610,"ÙħÙĨاÙĤشة":143611,"ĠÑħоÑĩÑĥ":143612,"ãĥľãĥ¼ãĥī":143613,"Ġróżnic":143614,"ĠкÑĢÑĭ":143615,"ĠкÑĢÑĭÑĪ":143616,"âľĵ":143617,"ãĤ³ãĥ³ãĥĨãĥ³":143618,"ãĤ³ãĥ³ãĥĨãĥ³ãĥĦ":143619,"ĠпÑĢедпоÑĩ":143620,"×ŀר×ij×Ļת":143621,"ĠØ´Ùĥ":143622,"ĠØ´Ùĥرا":143623,"Ġдал":143624,"Ġдалек":143625,"Ġдалеко":143626,"برÙĬØ·":143627,"برÙĬطاÙĨÙĬا":143628,"عÙĨا":143629,"عÙĨاÙĬØ©":143630,"ĠÑĢаÑģÑģказ":143631,"ĠÑĢаÑģÑģказÑĭва":143632,"Ø£ÙĦÙĪ":143633,"Ø£ÙĦÙĪØ§ÙĨ":143634,"æĮģãģ£ãģ¦":143635,"æĮģãģ£ãģ¦ãģĦ":143636,"Ùħبادئ":143637,"×Ķ×¢×ijר":143638,"×Ķ×¢×ijרת":143639,"Ġyayı":143640,"Ġyayıml":143641,"Ġyayımla":143642,"mát":143643,"máticos":143644,"à¸ģัà¸ĩ":143645,"à¸ģัà¸ĩวล":143646,"Ġ×ľ×¤×ª":143647,"Ġ×ľ×¤×ª×ķ×Ĺ":143648,"à¸ŀฤà¸ķิ":143649,"à¸ŀฤà¸ķิà¸ģรรม":143650,"íĤ¬":143651,"ĠокÑĢÑĥг":143652,"Ġ×ŀצ×ķ×ķ×Ķ":143653,"ÐĽÐµÐ½Ð¸":143654,"ÐĽÐµÐ½Ð¸Ð½":143655,"ĠTriá»ģu":143656,"ãĤ³ãĥŁãĥ¥":143657,"ãĤ³ãĥŁãĥ¥ãĥĭ":143658,"ãĤ³ãĥŁãĥ¥ãĥĭãĤ±":143659,"ãĤ³ãĥŁãĥ¥ãĥĭãĤ±ãĥ¼ãĤ·ãĥ§ãĥ³":143660,"ÙĥÙĨÙĬ":143661,"ÙĥÙĨÙĬسة":143662,"ãĤĴä¸Ńå¿ĥ":143663,"ãĤĴä¸Ńå¿ĥãģ«":143664,"ĠmiÄĻdz":143665,"ĠmiÄĻdzyn":143666,"ĠmiÄĻdzynar":143667,"ĠmiÄĻdzynarod":143668,"ĠmiÄĻdzynarodow":143669,"ÙĦÙĨ":143670,"ÙĦÙĨدا":143671,"برش":143672,"برشÙĦÙĪÙĨ":143673,"برشÙĦÙĪÙĨØ©":143674,"à¸ģระà¸ķุ":143675,"à¸ģระà¸ķุà¹īà¸Ļ":143676,"Ġgı":143677,"Ġgıda":143678,"à¸Ľà¸£à¸°à¸Ĺัà¸ļ":143679,"à¸Ľà¸£à¸°à¸Ĺัà¸ļà¹ĥà¸Ī":143680,"Ġë¶Ī구":143681,"Ġë¶Ī구íķĺê³ł":143682,"ĠÙĨØ·":143683,"ĠÙĨطاÙĤ":143684,"ĠÐľÐ¾Ð¶ÐµÑĤ":143685,"Präs":143686,"Präsident":143687,"ĠÑģкоÑĢ":143688,"ĠÑģкоÑĢоÑģÑĤÑĮ":143689,"Ġ×Ķ×ij×ķקר":143690,"еÑħаÑĤÑĮ":143691,"Ġgạo":143692,"Ġש×IJ×Ļ׳×Ŀ":143693,"Ġ×ij׳×ķ×Ĵ":143694,"Ġ×ij׳×ķ×Ĵ×¢":143695,"ĠопиÑģание":143696,"Ġuczni":143697,"Ġuczniów":143698,"à¹Ģà¸Ńà¹ĩà¸Ļ":143699,"Ġتش":143700,"ĠتشرÙĬÙĨ":143701,"Ġnhãn":143702,"빨":143703,"Ġcaractère":143704,"×¢×ľ×Ļ":143705,"×¢×ľ×Ļ×Ļ×Ķ":143706,"楽ãģĹãĤģãĤĭ":143707,"ĠÑģаÑħ":143708,"ĠÑģаÑħаÑĢ":143709,"дÑĥмаÑĤÑĮ":143710,"ĠÐĴозможно":143711,"صÙĬاÙĨ":143712,"صÙĬاÙĨØ©":143713,"ömür":143714,"สล":143715,"สลà¹ĩ":143716,"สลà¹ĩà¸Ń":143717,"สลà¹ĩà¸Ńà¸ķ":143718,"롯":143719,"Ġthói":143720,"grÃ¶ÃŁe":143721,"ĠksiÄĻ":143722,"ĠksiÄĻg":143723,"ĠÑĢом":143724,"ĠÑĢоман":143725,"ÙĤاسÙħ":143726,"×ŀ×ij×ķ×Ĵ":143727,"×ŀ×ij×ķ×Ĵר×Ļ×Ŀ":143728,"besch":143729,"beschäft":143730,"beschäftig":143731,"×Ķצע×Ķ":143732,"ĠÃģrea":143733,"ĠзаÑıвк":143734,"Ĺ":143735,"ĠлÑİбого":143736,"Ġม":143737,"Ġมà¸ģร":143738,"Ġมà¸ģราà¸Ħม":143739,"ÑĦиз":143740,"ÑĦизиÑĩеÑģк":143741,"инÑĦ":143742,"инÑĦек":143743,"инÑĦекÑĨи":143744,"اÙĦØ·":143745,"اÙĦطائÙģ":143746,"Ġколл":143747,"ĠколлекÑĤив":143748,"езжа":143749,"ĠسبØŃ":143750,"ĠسبØŃاÙĨ":143751,"ĠسبØŃاÙĨÙĩ":143752,"schlä":143753,"schläge":143754,"Ġди":143755,"Ġдиаг":143756,"ĠдиагноÑģÑĤ":143757,"ĠоÑĤмеÑĤиÑĤÑĮ":143758,"ТЬ":143759,"ĠاÙĦدر":143760,"ĠاÙĦدراسÙĬ":143761,"עצ×ŀ":143762,"עצ×ŀ×IJ×ķת":143763,"Ġdémarch":143764,"Ġdémarche":143765,"Ġ×ĺ×ķ×¢":143766,"Ġ×ĺ×ķ×¢×Ł":143767,"Ġfuncionários":143768,"ỵ":143769,"׾׼×IJ":143770,"׾׼×IJ×ķר×Ķ":143771,"à¸ĭà¹Ī":143772,"à¸ĭà¹Īà¸Ńม":143773,"ĠÑĩÑĥв":143774,"ĠÑĩÑĥвÑģÑĤво":143775,"âĸ¼":143776,"пÑĥÑī":143777,"пÑĥÑīен":143778,"ĠмеÑĢ":143779,"ĠмеÑĢоп":143780,"ĠмеÑĢопÑĢи":143781,"ĠмеÑĢопÑĢиÑıÑĤиÑı":143782,"Ġuçu":143783,"ĠuçuÅŁ":143784,"ãĤĴåĪ©ç͍ãģĻãĤĭ":143785,"aÄŁ":143786,"aÄŁlı":143787,"ìĺĪìĪł":143788,"à¹ģยà¹Ī":143789,"ĠاÙĦÙĥÙħ":143790,"ĠاÙĦÙĥÙħبÙĬ":143791,"ĠاÙĦÙĥÙħبÙĬÙĪØªØ±":143792,"تÙĪÙĬ":143793,"تÙĪÙĬتر":143794,"à¹Ģà¸Ĭีà¹Īยว":143795,"à¹Ģà¸Ĭีà¹Īยวà¸Ĭา":143796,"à¹Ģà¸Ĭีà¹Īยวà¸Ĭาà¸į":143797,"á»Ķ":143798,"Ġhiếm":143799,"ذاÙĥرة":143800,"Ġ×Ķ×ŀ×Ļ×ķ×Ĺ×ĵ":143801,"ĠìĪľ":143802,"ĠìĪľê°Ħ":143803,"ĠKı":143804,"ĠKısa":143805,"ĠgeleceÄŁi":143806,"пÑĢоÑĦеÑģÑģиона":143807,"пÑĢоÑĦеÑģÑģионал":143808,"Ġogó":143809,"Ġogóle":143810,"ĠgÅĤów":143811,"ĠgÅĤówne":143812,"ĠÑģÑĤилÑĮ":143813,"×IJפ׾":143814,"×IJפ׾×Ļ×§":143815,"×IJפ׾×Ļקצ×Ļ×Ķ":143816,"สมารà¹Į":143817,"สมารà¹Įà¸Ĺ":143818,"สมารà¹Įà¸Ĺà¹Ĥà¸Ł":143819,"สมารà¹Įà¸Ĺà¹Ĥà¸Łà¸Ļ":143820,"Ġthánh":143821,"ÐŁÐ¾Ð´":143822,"ÐŁÐ¾Ð´ÑĢоб":143823,"ÐŁÐ¾Ð´ÑĢобнее":143824,"ĠاÙĦتÙĪÙĨ":143825,"ĠاÙĦتÙĪÙĨسÙĬ":143826,"Ġbahçe":143827,"à¹ģà¸ģà¹īà¸Ľà¸±à¸įหา":143828,"éducation":143829,"europ":143830,"europä":143831,"europäische":143832,"ĠKsi":143833,"ĠKsiÄĻ":143834,"ĠëĦĺ":143835,"ĠëĦĺìĸ´":143836,"Ġvüc":143837,"Ġvücud":143838,"Ġyayg":143839,"Ġyaygın":143840,"Ġniekt":143841,"Ġniektóry":143842,"Ġniektórych":143843,"ãģŃãģĩ":143844,"Ġкаж":143845,"ĠкажеÑĤÑģÑı":143846,"каж":143847,"кажеÑĤ":143848,"ĠاÙĦدÙĬÙħÙĤرا":143849,"ĠاÙĦدÙĬÙħÙĤراط":143850,"ĠاÙĦدÙĬÙħÙĤراطÙĬØ©":143851,"æŃ©":143852,"æŃ©ãģĦãģ¦":143853,"Ġvaz":143854,"Ġvazge":143855,"Ġvazgeç":143856,"ĠминималÑĮ":143857,"ĠминималÑĮн":143858,"ãĥijãĤ¿":143859,"ãĥijãĤ¿ãĥ¼ãĥ³":143860,"ĠëĬ":143861,"ĠëĬIJ":143862,"ĠëĬIJëĤĮ":143863,"ãģ¡ãĤĩãģĨ":143864,"ãģ¡ãĤĩãģĨãģ©":143865,"Ġà¸ģร":143866,"Ġà¸ģรà¸ģà¸İ":143867,"Ġà¸ģรà¸ģà¸İาà¸Ħม":143868,"تجدÙĬد":143869,"ĠشاÙħÙĦ":143870,"หลัà¸ģà¸IJาà¸Ļ":143871,"ĠмаÑĢÑĪ":143872,"ĠмаÑĢÑĪÑĢÑĥÑĤ":143873,"ĠvÃŃt":143874,"ĠvÃŃtima":143875,"Ġquizá":143876,"aygı":143877,"×ĵ×ijר×Ļ×ķ":143878,"Ġизд":143879,"Ġиздели":143880,"ĠизделиÑı":143881,"пла":143882,"плаÑĩ":143883,"плаÑĩива":143884,"ä»»ãģĽ":143885,"Ġéquipé":143886,"ä¹ħãģĹãģ":143887,"ä¹ħãģĹãģ¶":143888,"ä¹ħãģĹãģ¶ãĤĬ":143889,"ĠкаÑĤ":143890,"ĠкаÑĤал":143891,"ĠкаÑĤалог":143892,"สà¹īม":143893,"ĠÑĢей":143894,"ĠÑĢейÑĤ":143895,"ĠÑĢейÑĤинг":143896,"Ġthuyá»ģn":143897,"ĠاÙĦÙħÙĤدس":143898,"espère":143899,"ãģ«åħ¥ãģ£ãģŁ":143900,"หมายà¹Ģลà¸Ĥ":143901,"ת×Ĺ×ķשת":143902,"à¸Ļà¹Īะ":143903,"ĠpeÅĤ":143904,"ĠpeÅĤne":143905,"Ġpérd":143906,"Ġpérdida":143907,"หมวà¸Ķ":143908,"หมวà¸Ķหมูà¹Ī":143909,"иÑĩеÑģкÑĥÑİ":143910,"çµĤãĤı":143911,"çµĤãĤıãģ£ãģŁ":143912,"Ġ×Ĵ×ķ×Ĵ׾":143913,"à¸Ĺำà¸Ħวาม":143914,"à¸Ĺำà¸Ħวามสะà¸Ńาà¸Ķ":143915,"Hotéis":143916,"ĠзаÑĢ":143917,"ĠзаÑĢегиÑģÑĤ":143918,"ĠзаÑĢегиÑģÑĤÑĢи":143919,"ĠзаÑĢегиÑģÑĤÑĢиÑĢова":143920,"ĠÑģобÑĭÑĤи":143921,"ĠÑģобÑĭÑĤиÑı":143922,"Ġ×ĸ׼×IJ":143923,"ÙħÙĨظÙĪÙħØ©":143924,"Ġ×Ķ×ŀצ":143925,"Ġ×Ķ×ŀצ×Ļ×IJ×ķת":143926,"ÙħÙĥÙĪÙĨ":143927,"ÙħÙĥÙĪÙĨات":143928,"ä¸ĬãģĮãĤĭ":143929,"ĠmÄĻ":143930,"ĠmÄĻsk":143931,"หรืà¸Ńà¹Ģà¸Ľà¸¥à¹Īา":143932,"ëĤ®":143933,"Ġnoktas":143934,"Ġnoktası":143935,"ĠболÑĮÑĪим":143936,"ĠлÑĥÑĩÑĪиÑħ":143937,"Ø´ÙĩÙĬد":143938,"à¸Ńำà¸Ļ":143939,"à¸Ńำà¸Ļวย":143940,"à¸Ńำà¸Ļวยà¸Ħวาม":143941,"à¸Ńำà¸Ļวยà¸Ħวามสะà¸Ķวà¸ģ":143942,"Ġев":143943,"ĠевÑĢ":143944,"ĠевÑĢоп":143945,"ĠевÑĢопей":143946,"à¸īาย":143947,"ìĦŃ":143948,"ÙħÙ쨧":143949,"ÙħÙ쨧ÙĪØ¶":143950,"ÙħÙ쨧ÙĪØ¶Ø§Øª":143951,"ë¹Į":143952,"赤ãģ¡ãĤĥãĤĵ":143953,"ĠÑĥдалоÑģÑĮ":143954,"ĠХоÑĤ":143955,"ĠХоÑĤÑı":143956,"przedsiÄĻbiorc":143957,"ĠHôm":143958,"íķĺìĺĢìĬµëĭĪëĭ¤":143959,"Ġнаг":143960,"ĠнагÑĢÑĥз":143961,"ĠнагÑĢÑĥзк":143962,"Ġ×ij×Ļ׳׾×IJ×ķ×ŀ×Ļ":143963,"Ġê°ĢëĬ¥íķľ":143964,"ĠHữu":143965,"à¸Ńุà¸Ķ":143966,"à¸Ńุà¸Ķม":143967,"ת×ķפ":143968,"ת×ķפע×Ķ":143969,"ĠmiÅĤo":143970,"ĠmiÅĤoÅĽci":143971,"ksiÄħż":143972,"ksiÄħżka":143973,"ĠاÙĦÙĦعبة":143974,"à¸īาà¸ģ":143975,"สะสม":143976,"×ŀתר":143977,"×ŀתר×Ĺש":143978,"Ġlégère":143979,"Ġ׾צפ":143980,"Ġ׾צפ×Ļ×Ķ":143981,"ĠиÑģÑĤоÑĢиÑı":143982,"ĠãĥĪãĥ©":143983,"ĠãĥĪãĥ©ãĥĥãĤ¯":143984,"ĠãĥĪãĥ©ãĥĥãĤ¯ãĥIJãĥĥãĤ¯":143985,"Ġка":143986,"ĠкаÑĦе":143987,"×ŀס×ŀ×ļ":143988,"Ġcüm":143989,"Ġcümle":143990,"à¹Ģà¸Ħลืà¹Īà¸Ńà¸Ļà¹Ħหว":143991,"ãģĬãģĿ":143992,"ãģĬãģĿãĤīãģı":143993,"ìŀIJëıĻ":143994,"ìŀIJëıĻì°¨":143995,"à¸Ńัà¸ķ":143996,"à¸Ńัà¸ķà¹Ĥà¸Ļ":143997,"à¸Ńัà¸ķà¹Ĥà¸Ļมั":143998,"à¸Ńัà¸ķà¹Ĥà¸Ļมัà¸ķิ":143999,"ĠÅŁik":144000,"ĠÅŁikay":144001,"ĠÅŁikayet":144002,"extrême":144003,"krä":144004,"kräfte":144005,"ëĤĻ":144006,"íķij":144007,"ì²Ļ":144008,"íĺĪ":144009,"ì°į":144010,"âĻ¡":144011,"ìŀĶ":144012,"뢰":144013,"íĿĶ":144014,"íĿIJ":144015,"âĩĴ":144016,"ë§Ľ":144017,"ìĬĪ":144018,"á»Ĵ":144019,"ìĺµ":144020,"âĹİ":144021,"íĤ¨":144022,"ê¿Ī":144023,"ì΍":144024,"ìĽ¨":144025,"ë§¥":144026,"ï½Ģ":144027,"J":144028,"Ẩ":144029,"ãħİ":144030,"ÑĹ":144031,"ìĦ¬":144032,"ì¹¼":144033,"ï¼¶":144034,"ìĽł":144035,"룴":144036,"Åĥ":144037,"ëĤ¼":144038,"ëĭIJ":144039,"â̹":144040,"ë¦Ń":144041,"ì§IJ":144042,"â̤":144043,"Ãħ":144044,"뾨":144045,"íĦ¸":144046,"íľĺ":144047,"ê²ģ":144048,"ë´ħ":144049,"Ãĺ":144050,"ëŃĶ":144051,"ëĺij":144052,"âĹĩ":144053,"ìĹĺ":144054,"ï»´":144055,"ë§¹":144056,"ï¾Ŀ":144057,"ìĬ·":144058,"íĥķ":144059,"ï¼ł":144060,"ì»´":144061,"ëłĮ":144062,"ì½ľ":144063,"ﻹ":144064,"ãħł":144065,"졸":144066,"ëħ¹":144067,"âĤº":144068,"âĸ¶":144069,"íĥIJ":144070,"êµ´":144071,"íij¸":144072,"ÑĶ":144073,"íͽ":144074,"Ðħ":144075,"ë°¤":144076,"Ôģ":144077,"첨":144078,"ì¶ĺ":144079,"ë²Ĺ":144080,"멸":144081,"ï¼»":144082,"ï¼½":144083,"ï¼·":144084,"ì°Į":144085,"ÃĴ":144086,"íı´":144087,"ìĵ¸":144088,"ì´Į":144089,"ëģĶ":144090,"ëĶ©":144091,"ëĩĮ":144092,"ë©Ģ":144093,"벨":144094,"ï¼µ":144095,"ë§¡":144096,"ëĭ«":144097,"฿":144098,"ãģ±":144099,"ìĩ¼":144100,"ìºł":144101,"뮤":144102,"ê±±":144103,"컬":144104,"âĦĥ":144105,"ëͱ":144106,"ëĥĪ":144107,"ìĭ±":144108,"íĻĪ":144109,"ëŀIJ":144110,"ìħĢ":144111,"ìłł":144112,"ÐĨ":144113,"ëłī":144114,"ï½ħ":144115,"ï½ı":144116,"íĻĢ":144117,"뼰":144118,"á»®":144119,"íĤ¹":144120,"ê½ĥ":144121,"ﻤ":144122,"ïºĶ":144123,"꺼":144124,"ìķī":144125,"âϦ":144126,"ï½ģ":144127,"ìĵ´":144128,"ãĢī":144129,"ì°®":144130,"ì¤ĺ":144131,"Ừ":144132,"ëģĦ":144133,"ëIJ¨":144134,"ìķĮ":144135,"íĿĺ":144136,"íħIJ":144137,"ãĢĪ":144138,"겪":144139,"ëĭ¥":144140,"ê²¼":144141,"á»Į":144142,"맨":144143,"ëģĬ":144144,"벤":144145,"ëijĶ":144146,"íĿ¡":144147,"Ử":144148,"ë¬ĺ":144149,"ãģī":144150,"ëŀ«":144151,"íĶĪ":144152,"íħį":144153,"ìŀĥ":144154,"ï½ī":144155,"ìģľ":144156,"âĸ½":144157,"묻":144158,"âĸ³":144159,"X":144160,"ìģĺ":144161,"ì¶°":144162,"ìĬ´":144163,"ìķ±":144164,"ìĩĦ":144165,"Ắ":144166,"ï´¿":144167,"ï´¾":144168,"âĤ½":144169,"ëĦĵ":144170,"룩":144171,"쳤":144172,"ê´ľ":144173,"ÃĻ":144174,"Ỿ":144175,"ï¿£":144176,"ëĵŃ":144177,"ë©ĺ":144178,"ê»´":144179,"ëł´":144180,"Ðĥ":144181,"묵":144182,"ì§Ŀ":144183,"ãģº":144184,"ðŁĺĤ":144185,"ëŀ¬":144186,"ìłĬ":144187,"ê´Ħ":144188,"ìŀĬ":144189,"íŀĮ":144190,"ìĦ¯":144191,"âĪĢ":144192,"âĸ¡":144193,"ëĢĮ":144194,"ëŀĻ":144195,"ï½ĥ":144196,"Ặ":144197,"ï¾Ħ":144198,"ïºĺ":144199,"ë¹¼":144200,"ÃĮ":144201,"âĸ·":144202,"ê¸į":144203,"ë©ĭ":144204,"ãģĥ":144205,"ìĺĨ":144206,"ìĺ®":144207,"몬":144208,"롤":144209,"볬":144210,"ëĬ¦":144211,"âĸª":144212,"ì¼ĵ":144213,"ìľĪ":144214,"ì§§":144215,"ï½½":144216,"ëĥī":144217,"ï¾Į":144218,"ëĺIJ":144219,"ï¼ĥ":144220,"á»Ħ":144221,"ì´¬":144222,"춤":144223,"ï¼¹":144224,"ï»Ń":144225,"âĤ«":144226,"ï½ĩ":144227,"ìĺ·":144228,"ëĸ¨":144229,"âī«":144230,"릿":144231,"⾨":144232,"Ù±":144233,"쯤":144234,"ê¹Ķ":144235,"ðŁĺĬ":144236,"ìĪ«":144237,"ê³±":144238,"êµ³":144239,"ï½ĭ":144240,"à¸Į":144241,"Äł":144242,"ë͏":144243,"ë°ij":144244,"ìħĭ":144245,"íİ´":144246,"âľħ":144247,"íĥij":144248,"ëĪĩ":144249,"íı¼":144250,"ðŁĺį":144251,"ìĺĽ":144252,"ﻣ":144253,"Ñĺ":144254,"ì©Į":144255,"ë¦ħ":144256,"ìĿį":144257,"ク":144258,"ëįľ":144259,"ãģħ":144260,"íݼ":144261,"ëĭĿ":144262,"ë¿Į":144263,"ì¼°":144264,"ìĭ«":144265,"ë°¥":144266,"íĽĮ":144267,"ì¨Į":144268,"ë¹Ļ":144269,"ï½İ":144270,"ë´Ħ":144271,"ìĦ¹":144272,"ï½²":144273,"ìĮĵ":144274,"Òij":144275,"ë°į":144276,"ëłĢ":144277,"íĨ¤":144278,"ッ":144279,"ë¤Ħ":144280,"꽤":144281,"ï½Ĵ":144282,"ìķ¨":144283,"ï½¼":144284,"ê¹IJ":144285,"íģIJ":144286,"âĦĸ":144287,"맺":144288,"ﺮ":144289,"ëħģ":144290,"겸":144291,"ï»ł":144292,"íĬľ":144293,"Ź":144294,"ë¥Ń":144295,"ëĪī":144296,"ï½Ķ":144297,"íĮ¬":144298,"ìŀĩ":144299,"ï¬ģ":144300,"ﻨ":144301,"ëij¥":144302,"ëŀĦ":144303,"Ù¬":144304,"íĭ´":144305,"ìŀī":144306,"Ú¾":144307,"ìĽħ":144308,"ï»®":144309,"ëĭī":144310,"âīª":144311,"âĹĦ":144312,"ëĪĮ":144313,"íĽ¼":144314,"ì¤į":144315,"Ÿ":144316,"줬":144317,"ì¾Į":144318,"ï½ĵ":144319,"ï¾Ĭ":144320,"ðŁı»":144321,"ï¾ī":144322,"Ðģ":144323,"íĺIJ":144324,"ï¾Ļ":144325,"꼬":144326,"íŀIJ":144327,"âĢ¥":144328,"ëŁŃ":144329,"ë§ŀ":144330,"ìĥ¤":144331,"ïºĴ":144332,"íĭ±":144333,"ë½ij":144334,"Ãķ":144335,"âĪļ":144336,"ëĤĦ":144337,"ê¹Ŀ":144338,"ëĨĪ":144339,"Ẻ":144340,"ìħĪ":144341,"ìĮį":144342,"âĢ¡":144343,"ï¼±":144344,"ìģ¨":144345,"âĺº":144346,"ëĴ·":144347,"ìĺ³":144348,"ðŁijį":144349,"몽":144350,"ëĤŃ":144351,"ïºŃ":144352,"ë©Ī":144353,"á»Ī":144354,"íķĢ":144355,"ëĭĻ":144356,"ë¦ĩ":144357,"ìķ¤":144358,"ìį¼":144359,"ãĥµ":144360,"Ñ£":144361,"ìľĹ":144362,"âŃIJ":144363,"ï¾ĺ":144364,"íŬ":144365,"ê¾¼":144366,"ìķĹ":144367,"ï»Į":144368,"ê±·":144369,"ëħķ":144370,"롱":144371,"ìķĬ":144372,"ï¾Ģ":144373,"ìĩł":144374,"íĮ©":144375,"ﺪ":144376,"ë§Ļ":144377,"_":144378,"ê¿Ķ":144379,"íİľ":144380,"룸":144381,"íĶĶ":144382,"ﻳ":144383,"ëıķ":144384,"ìĭ¼":144385,"á»İ":144386,"ë§ĺ":144387,"ì¢ĭ":144388,"íĨ¡":144389,"ï½±":144390,"íĿij":144391,"Ỹ":144392,"ì¦Į":144393,"칸":144394,"ëŃĺ":144395,"ï¾Ĺ":144396,"ï»ĭ":144397,"íĬĢ":144398,"ë¥Ļ":144399,"콩":144400,"ëģĹ":144401,"ëį´":144402,"ìħľ":144403,"¸":144404,"ë»IJ":144405,"ìĥµ":144406,"ê²IJ":144407,"ëĵ¬":144408,"룰":144409,"ãħĭ":144410,"ìĹī":144411,"á»ĸ":144412,"ëĦĮ":144413,"ï½¶":144414,"ë´ĩ":144415,"ëĤ³":144416,"ãĤľ":144417,"ëĸ»":144418,"íİĢ":144419,"ëį©":144420,"íķ¸":144421,"÷":144422,"ê¼¼":144423,"ëĶľ":144424,"ë°´":144425,"ë©į":144426,"âĹ¯":144427,"ìĹij":144428,"ìϼ":144429,"ïºij":144430,"ë¶ķ":144431,"롬":144432,"ï½Į":144433,"íĨ¨":144434,"ﺴ":144435,"ëłĺ":144436,"ê°¤":144437,"ìβ":144438,"Ñĵ":144439,"ìħī":144440,"ï»ĵ":144441,"ëĪĶ":144442,"ëį§":144443,"â̼":144444,"ﻲ":144445,"ê°±":144446,"ê¿Ģ":144447,"ëĭ·":144448,"Ẹ":144449,"Ẫ":144450,"ÆĴ":144451,"ëį¤":144452,"ìĪŃ":144453,"ï½Ĥ":144454,"ï½Ī":144455,"Åł":144456,"룬":144457,"ѵ":144458,"ëĸ¡":144459,"ëĥĦ":144460,"ìĦ°":144461,"ëĵĪ":144462,"ï¾ĥ":144463,"ëĩ¨":144464,"ï½IJ":144465,"êµ½":144466,"ìĹ½":144467,"ëĤĢ":144468,"묶":144469,"ï½·":144470,"ìıŁ":144471,"íĺĶ":144472,"ê¼Ī":144473,"ëģĪ":144474,"ì¥IJ":144475,"ïºĹ":144476,"ÄĮ":144477,"ëĪł":144478,"ëĸ¼":144479,"íĢ´":144480,"âī¥":144481,"ëĭŃ":144482,"ì±Ļ":144483,"ê»ı":144484,"멤":144485,"ìĥĺ":144486,"ëį®":144487,"룡":144488,"ìĤ½":144489,"ãĪľ":144490,"Ĩ":144491,"â̧":144492,"コ":144493,"Ä£":144494,"ì¦ī":144495,"ï¼¼":144496,"Û©":144497,"âĪĻ":144498,"ë°ı":144499,"ë¹ħ":144500,"ðŁĺĽ":144501,"íĪ´":144502,"ðŁĴķ":144503,"ãĢĴ":144504,"ìŀĺ":144505,"ﺤ":144506,"ï½ĸ":144507,"멾":144508,"ë²¼":144509,"ëĿĦ":144510,"ëļľ":144511,"ï»ĺ":144512,"ìĥĮ":144513,"ï½Ħ":144514,"ì©Ķ":144515,"ï½Ļ":144516,"ﺩ":144517,"Ûŀ":144518,"âĺİ":144519,"ìł¤":144520,"ëIJ©":144521,"ÅĿ":144522,"âŀ¡":144523,"ï»§":144524,"Ðı":144525,"ì«ĵ":144526,"ê³½":144527,"Éij":144528,"ãĥ²":144529,"ëĤ«":144530,"ë¦ī":144531,"ì¢ģ":144532,"ë°Ń":144533,"ðŁĺģ":144534,"ë¹µ":144535,"첩":144536,"컵":144537,"ðŁĺĺ":144538,"ë±ħ":144539,"âīĪ":144540,"ë¹ļ":144541,"ï»ľ":144542,"ðŁĻı":144543,"íģ°":144544,"ìĦŀ":144545,"ï¾ļ":144546,"ìĺ¹":144547,"ë¼Ī":144548,"ëĤ¯":144549,"ëŀ©":144550,"íļ¡":144551,"ï½ķ":144552,"íĥĵ":144553,"ëĿł":144554,"ê³ģ":144555,"ëĵĢ":144556,"ìĹł":144557,"Z":144558,"ë§ij":144559,"ëĭ¿":144560,"쿨":144561,"ãİ¡":144562,"ÐĬ":144563,"íĦ±":144564,"Ũ":144565,"ﺳ":144566,"ï¾ı":144567,"âĭħ":144568,"ê¼´":144569,"âī¤":144570,"íĮģ":144571,"Ω":144572,"궤":144573,"ìĪį":144574,"âľ¿":144575,"콤":144576,"ëĪħ":144577,"íĨ±":144578,"ãħľ":144579,"áIJħ":144580,"ÅĴ":144581,"ðŁijī":144582,"ﻦ":144583,"Ъ":144584,"ë¥ľ":144585,"íķ«":144586,"ï¾ĭ":144587,"âĻ«":144588,"ê¹ľ":144589,"ë°¸":144590,"ëĶĺ":144591,"íĿī":144592,"ï¾ģ":144593,"ï¾Ľ":144594,"볼":144595,"ê²¹":144596,"쿼":144597,"ﻬ":144598,"âŀ¤":144599,"ðŁĻģ":144600,"ïºł":144601,"ëĨ¨":144602,"믹":144603,"ê¸ĭ":144604,"ë»Ķ":144605,"ê¹ĥ":144606,"ëijij":144607,"íĭ¸":144608,"íİĻ":144609,"âŀĸ":144610,"ãĥ½":144611,"ì§ļ":144612,"ャ":144613,"ﻥ":144614,"íĮ½":144615,"âĢĴ":144616,"ìĮĢ":144617,"ìŃī":144618,"ëļ±":144619,"ãĤŀ":144620,"íĭĪ":144621,"ãĤIJ":144622,"ëīĺ":144623,"Σ":144624,"ê³°":144625,"ë¹Ĺ":144626,"ï¾İ":144627,"ðŁĺŃ":144628,"íĿł":144629,"ìĹ¿":144630,"ê°ļ":144631,"ì¤Į":144632,"ë§µ":144633,"ï½³":144634,"ãģ¢":144635,"ï»Ĺ":144636,"âī¦":144637,"Ú¤":144638,"ëłģ":144639,"ê¼½":144640,"ﻫ":144641,"âī§":144642,"ì´Ľ":144643,"ìłĿ":144644,"Ằ":144645,"âĻ£":144646,"ìºĺ":144647,"âĪĩ":144648,"ê²ī":144649,"ë°Ł":144650,"ï»Ķ":144651,"íĸĩ":144652,"âĸĴ":144653,"ðŁijı":144654,"Ãŀ":144655,"ðŁĺĨ":144656,"ﺼ":144657,"âĿĹ":144658,"ìºĶ":144659,"칩":144660,"ëĸ¤":144661,"ëĥħ":144662,"âĶľ":144663,"ï½»":144664,"ÎĶ":144665,"áĥ¦":144666,"ìŀİ":144667,"âĺĢ":144668,"âμ":144669,"ðŁĶ¥":144670,"ë°Į":144671,"ìłĸ":144672,"íĹĽ":144673,"Îķ":144674,"ïºĥ":144675,"ë¶ī":144676,"âĪŀ":144677,"íĥŃ":144678,"Ãĭ":144679,"âģĦ":144680,"ãħĩ":144681,"ëĦ¥":144682,"ëĭ®":144683,"ëł·":144684,"íĮĿ":144685,"캡":144686,"ë·Ķ":144687,"ì©į":144688,"íĤ´":144689,"ëļ«":144690,"âĵĴ":144691,"íķį":144692,"âĻĤ":144693,"ï¾Ĩ":144694,"âĨ©":144695,"ìį©":144696,"ïºķ":144697,"íĿĻ":144698,"Ñľ":144699,"íĤ·":144700,"íĿ°":144701,"íĥ±":144702,"ëķIJ":144703,"ï¾Ĵ":144704,"×ĥ":144705,"ëĮĦ":144706,"ìĺ´":144707,"ìķµ":144708,"ê¹¥":144709,"ëŀŃ":144710,"쪼":144711,"ãİĿ":144712,"ðŁĺħ":144713,"ëıĭ":144714,"몫":144715,"ﺸ":144716,"뮬":144717,"ë²ħ":144718,"ëijł":144719,"ìħ°":144720,"ì»·":144721,"ëĶª":144722,"ëħĶ":144723,"ãħ¡":144724,"ìĶ»":144725,"íķı":144726,"ëį±":144727,"ﺨ":144728,"ï¾į":144729,"ï½µ":144730,"ì¢Ģ":144731,"íİĮ":144732,"ï»°":144733,"ﺣ":144734,"Æ£":144735,"ðŁ¤£":144736,"ï·º":144737,"ëĤļ":144738,"âĭĨ":144739,"ë³į":144740,"ðŁĺĦ":144741,"ìĸĢ":144742,"ìĻł":144743,"ëĨĶ":144744,"íŨ":144745,"ï»Ľ":144746,"ï»Ŀ":144747,"á»¶":144748,"ìĸĺ":144749,"ìİĦ":144750,"ÚĨ":144751,"ï»ŀ":144752,"ëĢIJ":144753,"ê²Ķ":144754,"ﻵ":144755,"âŦ":144756,"íļŁ":144757,"ê¹ģ":144758,"ê°ĵ":144759,"ëĶ´":144760,"ìıĺ":144761,"ëļĿ":144762,"ỳ":144763,"ëŀ´":144764,"ëĦī":144765,"âĺŀ":144766,"ï½ĺ":144767,"Ž":144768,"ë¦İ":144769,"âĸ¬":144770,"ëŃī":144771,"âĩĽ":144772,"ìį¬":144773,"ïºŁ":144774,"Ëľ":144775,"ë¶ĵ":144776,"ìĽ°":144777,"Åľ":144778,"ëŃĩ":144779,"Ỳ":144780,"Ëļ":144781,"ëķĢ":144782,"âĺij":144783,"ðŁı¼":144784,"ìĸ½":144785,"âĮĴ":144786,"Ðİ":144787,"ɾ":144788,"íĮ¡":144789,"ï¾ħ":144790,"ìŀŃ":144791,"ィ":144792,"칫":144793,"ìľĮ":144794,"ÒĽ":144795,"굿":144796,"ëĭ¦":144797,"âĶĶ":144798,"ï¾ij":144799,"ì§ĸ":144800,"ìºĦ":144801,"ãĢĥ":144802,"ʼ":144803,"ê²Ł":144804,"ï½§":144805,"Ä¢":144806,"íİł":144807,"ë§·":144808,"ê°ĩ":144809,"ìĭ¹":144810,"ðŁĴ¦":144811,"ï¾ľ":144812,"ëĬĻ":144813,"벡":144814,"Å¿":144815,"ðŁĺĭ":144816,"ðŁĴª":144817,"ì¿Ħ":144818,"ë©ķ":144819,"ìѤ":144820,"ëĬĦ":144821,"ðŁĮ¸":144822,"ãĤĿ":144823,"Çİ":144824,"ï½ļ":144825,"ÄĹ":144826,"ëģĵ":144827,"ê¶IJ":144828,"áµī":144829,"ãĥĤ":144830,"ê»į":144831,"ðŁĺ¦":144832,"ãĢĿ":144833,"ð٤Ĺ":144834,"ÑŁ":144835,"ìĹİ":144836,"âľĮ":144837,"ìīIJ":144838,"ÃĨ":144839,"íĹIJ":144840,"ðŁİī":144841,"Îij":144842,"ï½Ń":144843,"ðŁĴĻ":144844,"ìĽ¬":144845,"íĢĺ":144846,"ﻢ":144847,"ðŁĺİ":144848,"íij¼":144849,"íĿ©":144850,"ï»Ħ":144851,"íħĢ":144852,"ëłIJ":144853,"쥬":144854,"Ðĭ":144855,"ìĥ·":144856,"뾬":144857,"ðŁĺĥ":144858,"ëĦ¬":144859,"륨":144860,"ìĽį":144861,"ï½Ĩ":144862,"ï½´":144863,"ãĥħ":144864,"Ãı":144865,"ﻪ":144866,"âĻł":144867,"ëĬ¬":144868,"ë±Ģ":144869,"ë°ĭ":144870,"ìĥĢ":144871,"ï½¾":144872,"ëĤ±":144873,"컸":144874,"ðŁĴĸ":144875,"ðŁijĮ":144876,"Ñŀ":144877,"ì§±":144878,"ËĨ":144879,"ðŁĵļ":144880,"âŃķ":144881,"ï¬Ĥ":144882,"ﻡ":144883,"ëij¬":144884,"íμ":144885,"âĸ¸":144886,"ê°¯":144887,"ê¹ħ":144888,"ï½®":144889,"ëĺ¥":144890,"Ä¡":144891,"íĮŁ":144892,"ÐĮ":144893,"ìĨŁ":144894,"ïºĵ":144895,"ﻼ":144896,"ÃĽ":144897,"ãĥ¾":144898,"ëĮĵ":144899,"íĴĭ":144900,"ìķĵ":144901,"ï½¹":144902,"ëĤ¡":144903,"ðŁijĩ":144904,"Ẽ":144905,"ãĢŁ":144906,"ðŁĮŁ":144907,"íĥł":144908,"ãĢĨ":144909,"âĢŁ":144910,"ë¸IJ":144911,"ðŁĮ¹":144912,"ìł¼":144913,"ðŁĵĮ":144914,"ìͬ":144915,"âĹĢ":144916,"ðŁĴĵ":144917,"ê¹İ":144918,"ìĤIJ":144919,"ìĶĮ":144920,"ÑĽ":144921,"âĶĪ":144922,"ë²³":144923,"ãİŀ":144924,"Õ¡":144925,"íĤµ":144926,"ð٤Ķ":144927,"ëĢĶ":144928,"ìĬIJ":144929,"íĻī":144930,"⾦":144931,"ëľ¯":144932,"ìł¯":144933,"ëͧ":144934,"Φ":144935,"ËĪ":144936,"ìī¼":144937,"âĹĬ":144938,"ëľ©":144939,"ëľ°":144940,"ï¾IJ":144941,"ë¿Ķ":144942,"ìĹ®":144943,"ì·Į":144944,"ﺧ":144945,"ÎĴ":144946,"ëµĻ":144947,"ï»Ĭ":144948,"ì°Ķ":144949,"íİĦ":144950,"ðŁĴĹ":144951,"Ẵ":144952,"ì°¢":144953,"íľ¼":144954,"ê½Ĥ":144955,"ì±Ķ":144956,"ìī´":144957,"âĸ¾":144958,"íΰ":144959,"ëĭĽ":144960,"âĿ£":144961,"ェ":144962,"ðŁĴľ":144963,"Ëĺ":144964,"ãħ¤":144965,"âĨĹ":144966,"íĸĦ":144967,"âϬ":144968,"ìķ°":144969,"ïºľ":144970,"âī¡":144971,"ãĢĵ":144972,"ìij¥":144973,"íĮį":144974,"íīģ":144975,"ë»Ĺ":144976,"íľł":144977,"íľ©":144978,"âľĪ":144979,"íĢĦ":144980,"ìĸĩ":144981,"ì¢ĩ":144982,"íŀĻ":144983,"몹":144984,"ãĤĽ":144985,"ðŁĺ±":144986,"ëįŁ":144987,"à¹ħ":144988,"êµ¶":144989,"Ù«":144990,"ìĶģ":144991,"âľª":144992,"ï¾Ī":144993,"ðŁĻĮ":144994,"âļ¡":144995,"Îļ":144996,"ì¼Ī":144997,"ï¾Ķ":144998,"ï¾Ĥ":144999,"êµī":145000,"ﺻ":145001,"ðŁĴĭ":145002,"á¹£":145003,"ÓĻ":145004,"ìĨľ":145005,"ìĹ£":145006,"âľ©":145007,"ìľĻ":145008,"ﺰ":145009,"Ẳ":145010,"ìŀ£":145011,"âĿĮ":145012,"âĺģ":145013,"ìķİ":145014,"Ľ":145015,"Ûģ":145016,"ãĦ±":145017,"ëŁ¿":145018,"íĮ¸":145019,"ê½ī":145020,"ìıł":145021,"ðŁįĢ":145022,"âĨĶ":145023,"ëŃ¡":145024,"ï»ģ":145025,"ï¼Ħ":145026,"ðŁĴ¥":145027,"âĺĽ":145028,"íĹ·":145029,"ëij¡":145030,"Îł":145031,"Τ":145032,"âĦĵ":145033,"ﺷ":145034,"ÎĻ":145035,"ëıĶ":145036,"짤":145037,"âĶĥ":145038,"ãĦ·":145039,"ÇĴ":145040,"ðŁ¥°":145041,"ëĶķ":145042,"ìļ¥":145043,"ì¸Ħ":145044,"íĽĶ":145045,"ïºĩ":145046,"ﺬ":145047,"ðŁĺ¢":145048,"빡":145049,"ì͹":145050,"ų":145051,"ËĿ":145052,"íİij":145053,"ï¾ĵ":145054,"ðŁĴļ":145055,"ëĬij":145056,"꺾":145057,"íĨ°":145058,"ÿ":145059,"ÐĦ":145060,"ëĮIJ":145061,"ë½Ģ":145062,"ì·Ħ":145063,"ðŁĵį":145064,"ðŁĻĪ":145065,"âĹĪ":145066,"ê¿ĩ":145067,"ì¼Ħ":145068,"íİ«":145069,"ðŁĩ·":145070,"âĶĭ":145071,"âļł":145072,"ë±ī":145073,"ìį°":145074,"ìĻĪ":145075,"ɪ":145076,"ïºĭ":145077,"ðŁĺľ":145078,"ÎŁ":145079,"ðŁĻĤ":145080,"âļ½":145081,"ÅĪ":145082,"ë¹Ķ":145083,"íĮľ":145084,"à¹ı":145085,"ìĸ¹":145086,"íĪŃ":145087,"ðŁ¥ĩ":145088,"ãĦ´":145089,"ëĶ¥":145090,"ìŃĪ":145091,"âĪĨ":145092,"ëĸ³":145093,"ë±ĥ":145094,"ìŀ¦":145095,"ï»IJ":145096,"Îľ":145097,"âľ§":145098,"Ïį":145099,"ìłĵ":145100,"âĹķ":145101,"ëĴĢ":145102,"ï»Ģ":145103,"ðŁĶ´":145104,"ê½ģ":145105,"ëĮĪ":145106,"ëİĮ":145107,"ãĤİ":145108,"â¦ģ":145109,"ì½§":145110,"ﯾ":145111,"âĿ¯":145112,"à¸ħ":145113,"ðŁĻĦ":145114,"âĿĢ":145115,"ðŁĶ¹":145116,"âĩIJ":145117,"êµµ":145118,"âĩĶ":145119,"ë¶IJ":145120,"ðŁĴĽ":145121,"ξ":145122,"íĥ¬":145123,"âĿĦ":145124,"Ò£":145125,"ã̰":145126,"âĪij":145127,"âĺ¼":145128,"âīł":145129,"Ò¯":145130,"ﺯ":145131,"꿨":145132,"âľĸ":145133,"Êĸ":145134,"íĢĢ":145135,"ê¾Ģ":145136,"íĹĿ":145137,"âĶ£":145138,"ãİľ":145139,"ëĶĽ":145140,"뾸":145141,"ﺫ":145142,"ê¿°":145143,"ðŁĩ¹":145144,"ÇIJ":145145,"ÛĴ":145146,"룻":145147,"ïºĸ":145148,"Ñļ":145149,"ëĬł":145150,"Ûķ":145151,"깡":145152,"ë¿ľ":145153,"ì²¼":145154,"ï¨ij":145155,"륵":145156,"ìį¸":145157,"íħħ":145158,"íij¹":145159,"ÖĢ":145160,"ï³Į":145161,"ãħ£":145162,"ìij¤":145163,"ì½ķ":145164,"ëķł":145165,"ðŁĮ¿":145166,"íĥĶ":145167,"ìĽģ":145168,"ζ":145169,"âŀľ":145170,"ìĬĺ":145171,"íĽĹ":145172,"ë©§":145173,"ìīĺ":145174,"Õ¶":145175,"á¹ĩ":145176,"ðŁİģ":145177,"ソ":145178,"ï¼Ĥ":145179,"á¼IJ":145180,"âľķ":145181,"âŀ¢":145182,"ëĦ¨":145183,"컫":145184,"ì¯Ķ":145185,"ì°ľ":145186,"ðŁĴ°":145187,"íħĿ":145188,"ãİı":145189,"ë³¶":145190,"Òĵ":145191,"âĨ³":145192,"ìĥ´":145193,"íģĺ":145194,"âĸĢ":145195,"ë²Ļ":145196,"à¸ĥ":145197,"á½¶":145198,"Äķ":145199,"â¬ĩ":145200,"ë¤ĺ":145201,"ðŁİµ":145202,"âľļ":145203,"ïºı":145204,"Ρ":145205,"âĹī":145206,"ðŁĴ«":145207,"ÐĪ":145208,"ìĸĦ":145209,"ì§Ļ":145210,"ï»ĥ":145211,"ðĿijĴ":145212,"ëŃĦ":145213,"âĿ¥":145214,"âĿĸ":145215,"âĺĿ":145216,"ʹ":145217,"ḥ":145218,"âĢ¿":145219,"ãħħ":145220,"ê¸ģ":145221,"ëķ¡":145222,"ëį¥":145223,"âĪ©":145224,"ê»Ħ":145225,"ë®Į":145226,"Ò±":145227,"âĪĹ":145228,"ëłĻ":145229,"ïºĮ":145230,"ËIJ":145231,"ðŁĺ³":145232,"ðŁij©":145233,"ðŁİ¶":145234,"쿵":145235,"ðŁ¤©":145236,"ê·¤":145237,"ëĮĶ":145238,"ïºIJ":145239,"Ïİ":145240,"ì¶¥":145241,"ï½Ĭ":145242,"á¹Ń":145243,"뤼":145244,"âĸ«":145245,"ì§ł":145246,"á¼Ģ":145247,"ê»ij":145248,"ëĮģ":145249,"í̏":145250,"âĻĽ":145251,"ðŁĴŀ":145252,"âĸ°":145253,"ðĿijĸ":145254,"ëĿ¤":145255,"द":145256,"ì´ĺ":145257,"ðŁĺĩ":145258,"ëͤ":145259,"ÎĹ":145260,"ðŁĻĩ":145261,"ËĽ":145262,"ì©¡":145263,"âΧ":145264,"Õ¥":145265,"ÑĻ":145266,"ëIJ¬":145267,"ëĸĦ":145268,"ðŁĮ·":145269,"ìĹĮ":145270,"ðŁĺ¥":145271,"ëĪ´":145272,"ï»ļ":145273,"ÉĽ":145274,"ïºĦ":145275,"ï»ı":145276,"ÅĮ":145277,"ë²ļ":145278,"ìĭ£":145279,"ïºĢ":145280,"Îĵ":145281,"ðŁĺĮ":145282,"ËĻ":145283,"ëŀı":145284,"ðŁĶ¸":145285,"ðŁĵ·":145286,"ëģ½":145287,"íģ½":145288,"ðŁĴ¡":145289,"ðŁĮ±":145290,"ëºı":145291,"ìģł":145292,"ìĥIJ":145293,"ëıĹ":145294,"츰":145295,"ëĪķ":145296,"ÎĿ":145297,"âģī":145298,"ðŁĮ¼":145299,"íĮł":145300,"âĭ¯":145301,"áĥĺ":145302,"⾤":145303,"ê±Ķ":145304,"íĮİ":145305,"ðŁĴ¯":145306,"ìıĻ":145307,"íĹī":145308,"ÙŃ":145309,"ì½°":145310,"ﺿ":145311,"ï»±":145312,"ì±Į":145313,"âĺķ":145314,"ðŁİĢ":145315,"ÄĿ":145316,"ë°§":145317,"ìĤ¿":145318,"áijķ":145319,"ðŁįĥ":145320,"âĩ¨":145321,"ÎĽ":145322,"ë§´":145323,"ë³ķ":145324,"áijIJ":145325,"âĸĵ":145326,"ðĿijľ":145327,"âĻ»":145328,"íĤ¥":145329,"Õ¸":145330,"ãα":145331,"ëºĢ":145332,"첸":145333,"ïºĽ":145334,"ðŁıĨ":145335,"ðŁĩª":145336,"âĿĵ":145337,"ÄĢ":145338,"ì½¥":145339,"ðŁĩ§":145340,"á½·":145341,"âľĤ":145342,"ìŀ¼":145343,"ï§¡":145344,"ðŁĵ¸":145345,"âϝ":145346,"ÉĶ":145347,"ὸ":145348,"âĮª":145349,"ï»ĸ":145350,"不":145351,"âļ«":145352,"âĶĹ":145353,"ðŁĮĪ":145354,"ﻩ":145355,"ðŁĵ²":145356,"ÏĪ":145357,"ðŁĺ¡":145358,"ðĿijİ":145359,"ìľ½":145360,"짬":145361,"ì§Ĭ":145362,"á½³":145363,"ìĮ¤":145364,"ëĤį":145365,"âīĴ":145366,"ðŁij¨":145367,"âĺĺ":145368,"Ó©":145369,"âĤĵ":145370,"âĪĤ":145371,"ï¹ģ":145372,"ðŁĴIJ":145373,"íħĥ":145374,"ðŁı½":145375,"ê·Ħ":145376,"ðŁĺı":145377,"ðŁĮº":145378,"ðŁĺĶ":145379,"ォ":145380,"âľİ":145381,"ëµĪ":145382,"ðŁĩ¸":145383,"âĢ£":145384,"âŀĶ":145385,"ëĺĺ":145386,"ìĥ¬":145387,"Êĥ":145388,"â¬ħ":145389,"ì©IJ":145390,"ðŁĻĨ":145391,"ðŁİĦ":145392,"ľ":145393,"⣶":145394,"áĥIJ":145395,"âĺ»":145396,"ì±ķ":145397,"ìģ©":145398,"ë½ķ":145399,"캣":145400,"ðŁijĪ":145401,"ðŁĻĭ":145402,"ï¾ĸ":145403,"Òļ":145404,"Õ«":145405,"ìĮĪ":145406,"ë²§":145407,"ðŁĩ®":145408,"ï½Ŀ":145409,"ðŁįģ":145410,"ìĹ¥":145411,"ij":145412,"ë½IJ":145413,"íį½":145414,"íĽij":145415,"âĤ¹":145416,"ãħģ":145417,"ìͽ":145418,"ðŁĶģ":145419,"य":145420,"ê¾¹":145421,"ëīľ":145422,"âĹ¡":145423,"íķĮ":145424,"Îĺ":145425,"룹":145426,"ìĻĵ":145427,"ðŁĩ¦":145428,"ðŁijĢ":145429,"âĶĮ":145430,"ῦ":145431,"ëĦĽ":145432,"ìĦ£":145433,"ìŃĻ":145434,"ï±ł":145435,"Îŀ":145436,"Ê»":145437,"á¿¶":145438,"âĿĿ":145439,"ê±Ģ":145440,"ëĸ´":145441,"ãĦ¹":145442,"ðŁĴİ":145443,"Ϲ":145444,"âĽħ":145445,"ï»ķ":145446,"ãĥ±":145447,"ï½Ľ":145448,"ëĮķ":145449,"ë¹½":145450,"ì¥Ķ":145451,"쿤":145452,"ðŁĸ¤":145453,"ÑĴ":145454,"ê¹į":145455,"ëİĢ":145456,"ìĭ¯":145457,"뻤":145458,"ðŁĵŀ":145459,"ðŁĵ£":145460,"ðŁĺĿ":145461,"ìį¹":145462,"ìĹ¡":145463,"ì°IJ":145464,"á½IJ":145465,"ï»Ī":145466,"âľį":145467,"Äı":145468,"ðŁĮŀ":145469,"âĦ¦":145470,"ê½Ŀ":145471,"ë»ĺ":145472,"ìα":145473,"âĶĺ":145474,"ðŁĮ»":145475,"âĤ´":145476,"âŀ¨":145477,"íIJģ":145478,"ê¶Ī":145479,"âĺ¢":145480,"ðŁĺĪ":145481,"ゥ":145482,"âĦĹ":145483,"ê°Ń":145484,"ê°¸":145485,"ë»ij":145486,"쥴":145487,"컥":145488,"ï¤Ĭ":145489,"ï»Ĵ":145490,"ðŁĺķ":145491,"âĺĶ":145492,"ìĺIJ":145493,"ðŁļĹ":145494,"ëĹĦ":145495,"ë§ı":145496,"Õ½":145497,"âĸ»":145498,"⣵":145499,"ìī°":145500,"ï»ij":145501,"âĻ©":145502,"Î¥":145503,"ðŁĺ£":145504,"âĬĤ":145505,"ãħĤ":145506,"ìħ¸":145507,"íıĦ":145508,"âľ½":145509,"ì¦Ļ":145510,"âĸ£":145511,"ê±į":145512,"ê¿ĭ":145513,"ì«Ħ":145514,"ìºĩ":145515,"ðŁĩµ":145516,"ðŁijij":145517,"âľĺ":145518,"ðĿijĽ":145519,"ìį½":145520,"ìºī":145521,"וּ":145522,"ðŁĶº":145523,"âĦ®":145524,"íĥ¤":145525,"ðŁĩº":145526,"ðŁĴµ":145527,"íħ¨":145528,"ï½ij":145529,"Ψ":145530,"ìĥ¹":145531,"ìĸķ":145532,"ì¹µ":145533,"ðŁĵ±":145534,"व":145535,"ðŁijĬ":145536,"ðŁĴĦ":145537,"ðŁĴĿ":145538,"ãĮĶ":145539,"ìĻģ":145540,"Ðĩ":145541,"à®IJ":145542,"âĸ¹":145543,"á´Ľ":145544,"âĹĺ":145545,"뺨":145546,"íĥī":145547,"ìĸĮ":145548,"ðŁIJ¶":145549,"ãĤij":145550,"Ëĩ":145551,"Åı":145552,"á½¹":145553,"ìħ§":145554,"ï¹°":145555,"ðĿij¡":145556,"ðŁĶĿ":145557,"ðŁĺ»":145558,"ðŁĴĥ":145559,"ðŁ¤¦":145560,"ðŁįĴ":145561,"í̵":145562,"âľĨ":145563,"ë¹´":145564,"理":145565,"ï»Ļ":145566,"á´Ĺ":145567,"ðŁĮ´":145568,";":145569,"ëĮij":145570,"ì¨ĭ":145571,"쵸":145572,"ðŁİĪ":145573,"ðŁıł":145574,"á½±":145575,"ÛĨ":145576,"á¿ĸ":145577,"âĢĽ":145578,"ì°¼":145579,"íķ¥":145580,"íĹ´":145581,"ðŁĩ¬":145582,"ì°Ŀ":145583,"âĪł":145584,"ï¼ĩ":145585,"âĬĻ":145586,"âĿij":145587,"ëĦĭ":145588,"ëŀĹ":145589,"ë°ī":145590,"ìĹĬ":145591,"ì¢Ĩ":145592,"íĮ¥":145593,"ï°²":145594,"ðŁĵĸ":145595,"ðŁĺ®":145596,"âļª":145597,"ðŁĺļ":145598,"âĿŀ":145599,"ðĿijŁ":145600,"ðŁİĤ":145601,"Åķ":145602,"áIJĪ":145603,"꺽":145604,"ì±ł":145605,"ïºĿ":145606,"ê¿ī":145607,"áĥł":145608,"ðŁıĥ":145609,"ðŁĴ¸":145610,"âĿģ":145611,"âĹ¾":145612,"Úª":145613,"á¹ĥ":145614,"íĬ¬":145615,"ðŁĩ±":145616,"íİŃ":145617,"ðŁĺŀ":145618,"ë¾°":145619,"á¹Ľ":145620,"뼸":145621,"âĿĤ":145622,"êĴ³":145623,"âĶIJ":145624,"íĵ°":145625,"âŀł":145626,"ê´ĺ":145627,"ëħĺ":145628,"뻥":145629,"ì¾ħ":145630,"ðŁĺIJ":145631,"âĪª":145632,"ðŁijģ":145633,"âĪ´":145634,"âĹģ":145635,"ëºIJ":145636,"ìŀ¤":145637,"ì±Ĺ":145638,"ðŁı¾":145639,"Χ":145640,"á½»":145641,"âŀ¥":145642,"ìŁĪ":145643,"ï»ī":145644,"âĸĮ":145645,"ãĥ®":145646,"ðŁ¤¤":145647,"âĩĵ":145648,"ì¼ł":145649,"á´ı":145650,"맬":145651,"뻣":145652,"ðŁĴ¬":145653,"ðŁįĵ":145654,"ĸ":145655,"Ù¹":145656,"Ê¿":145657,"á½°":145658,"ëķľ":145659,"ì°¡":145660,"ì°»":145661,"íİį":145662,"ðŁİ¯":145663,"ðŁįĤ":145664,"ðŁij§":145665,"âĻ¢":145666,"áĨŀ":145667,"âϧ":145668,"âļľ":145669,"âľī":145670,"ëĵ¦":145671,"ëŃ£":145672,"ìĪı":145673,"ìĵ±":145674,"ÅŃ":145675,"ÊĬ":145676,"âĴ¸":145677,"âĩ©":145678,"ðŁĴĶ":145679,"Õµ":145680,"Ðī":145681,"Ò»":145682,"ë§£":145683,"ìĽľ":145684,"ì¿¡":145685,"íĽħ":145686,"íĽ¤":145687,"ﺢ":145688,"âľĭ":145689,"âĪĪ":145690,"ðŁĮį":145691,"Êľ":145692,"ëĬª":145693,"ëĴ¹":145694,"ﺲ":145695,"âĸĦ":145696,"ãħĪ":145697,"ëļ¤":145698,"íİ©":145699,"â΍":145700,"ðŁ¤ª":145701,"áĥļ":145702,"ê³¶":145703,"íĬķ":145704,"ðŁĺ¬":145705,"âĪ«":145706,"ðŁijĭ":145707,"ÒIJ":145708,"íĬ¿":145709,"ðŁĶµ":145710,"ðŁĴ¨":145711,"ðŁĮĻ":145712,"ëĩ©":145713,"âľ³":145714,"ë¨ģ":145715,"ëºĦ":145716,"ìĻij":145717,"ìºħ":145718,"íıĪ":145719,"ðĿijĻ":145720,"ðŁĴĺ":145721,"ãİ¥":145722,"âĿı":145723,"âľ°":145724,"ﯿ":145725,"ëµIJ":145726,"ì¼IJ":145727,"ﺱ":145728,"Õ´":145729,"ï¬Ģ":145730,"âľ´":145731,"ð٤Ń":145732,"ðŁijĨ":145733,"âĽĶ":145734,"ê·ĵ":145735,"ìĮĮ":145736,"ðŁ¤·":145737,"ÛĶ":145738,"ðŁ§¡":145739,"ðŁĺĵ":145740,"Îĸ":145741,"âı°":145742,"ê²ľ":145743,"ëĭ³":145744,"ëİħ":145745,"ë°Ī":145746,"ï®IJ":145747,"ðŁı¡":145748,"âĨª":145749,"âĵĶ":145750,"âľĬ":145751,"ϲ":145752,"ÜIJ":145753,"ðŁĩ³":145754,"ÖĤ":145755,"âľı":145756,"ìĸĹ":145757,"ì«Ļ":145758,"ðŁĺ²":145759,"ÄŃ":145760,"âĻŃ":145761,"âĶı":145762,"âĹĮ":145763,"ðŁĺ¯":145764,"áµĴ":145765,"íĬł":145766,"Ä·":145767,"Êģ":145768,"à¤Ł":145769,"á¹ģ":145770,"á¼°":145771,"á¿Ĩ":145772,"â«":145773,"⫸":145774,"ëį«":145775,"ì³ĩ":145776,"켤":145777,"íĽ¨":145778,"ðŁĴŁ":145779,"ÊĢ":145780,"ʳ":145781,"ëĵIJ":145782,"âķ°":145783,"âĿĩ":145784,"ÇĢ":145785,"ÇĶ":145786,"É´":145787,"âĺļ":145788,"âĺľ":145789,"ê¶Ĥ":145790,"ì«Ĵ":145791,"ì±Ī":145792,"ðŁĩ¨":145793,"ðŁİ¥":145794,"ðŁĵĿ":145795,"ħ":145796,"ðĿijIJ":145797,"ÛĪ":145798,"ब":145799,"ì¬IJ":145800,"íĹ¥":145801,"âύ":145802,"ðŁį´":145803,"ï¹ı":145804,"Ëĭ":145805,"ðŁ¥º":145806,"âĸ¨":145807,"íĻĭ":145808,"âĪħ":145809,"ëģĻ":145810,"ëŀł":145811,"ìĨ¥":145812,"âĢĸ":145813,"ð٤ĺ":145814,"ðŁIJ»":145815,"áµķ":145816,"ÇĿ":145817,"âĺı":145818,"ïºļ":145819,"ï»Ĥ":145820,"ðŁļ©":145821,"ìĪŁ":145822,"ËĬ":145823,"⤵":145824,"ðŁĴ§":145825,"ãħį":145826,"ë©©":145827,"Ƭ":145828,"Îĩ":145829,"âĩ§":145830,"âĵļ":145831,"ìĤ¯":145832,"ìΝ":145833,"ëĨĭ":145834,"âľ¯":145835,"ðŁļĢ":145836,"Úĺ":145837,"Ú¨":145838,"âľŃ":145839,"ê²ħ":145840,"íĮ°":145841,"íľĻ":145842,"ðŁĮĬ":145843,"ðŁİĵ":145844,"ðŁĺĻ":145845,"Ëĥ":145846,"ðŁĴģ":145847,"ðŁijİ":145848,"âĺ¹":145849,"ðŁĺ«":145850,"ðŁĴ»":145851,"ëĤµ":145852,"ìĿĬ":145853,"íĮ»":145854,"Ò³":145855,"á½²":145856,"âŀŀ":145857,"ëĤij":145858,"ëĿĪ":145859,"죤":145860,"ﻯ":145861,"ðŁĩ©":145862,"ðŁ¥³":145863,"âĴ¼":145864,"ð٦ĭ":145865,"âĺĤ":145866,"ðŁĺ°":145867,"ðŁĻĥ":145868,"ðŁĺĴ":145869,"Ûİ":145870,"Ïķ":145871,"Ḥ":145872,"룽":145873,"ìĬ¥":145874,"ðĿijī":145875,"ÉIJ":145876,"ðŁįİ":145877,"âķ¯":145878,"âķ¹":145879,"າ":145880,"ï¾ł":145881,"ë¹ķ":145882,"ïºĨ":145883,"ʺ":145884,"Ó§":145885,"âĨł":145886,"ëĥĩ":145887,"ìİĪ":145888,"ìŁ¤":145889,"ï±¢":145890,"âķ¬":145891,"âĺł":145892,"ðŁİĬ":145893,"ãįį":145894,"ãİİ":145895,"âĺ°":145896,"âľĥ":145897,"ãħī":145898,"ë¯Ī":145899,"빤":145900,"ìıŃ":145901,"ðĿij¢":145902,"ðŁIJ¾":145903,"Åĭ":145904,"ðŁij¶":145905,"âĶĽ":145906,"ï¿¢":145907,"áĥ¡":145908,"ļ":145909,"ÅĨ":145910,"ÑIJ":145911,"ìĥĽ":145912,"ìĺĮ":145913,"챤":145914,"íħģ":145915,"íļĥ":145916,"ï³Ĭ":145917,"ðĿijĶ":145918,"ðŁĩ«":145919,"âĭ°":145920,"ðŁĺ¨":145921,"âĤ©":145922,"Õ¬":145923,"á¸į":145924,"á»´":145925,"âĨĺ":145926,"âĺ¯":145927,"ãħı":145928,"ìł¬":145929,"âĻĶ":145930,"ðŁĶĶ":145931,"ðŁĺł":145932,"ðŁĻĬ":145933,"à®ľ":145934,"á¹ħ":145935,"âĹIJ":145936,"âĿĪ":145937,"âŀ½":145938,"ìĥħ":145939,"ðĿijł":145940,"Æ¢":145941,"âĭĻ":145942,"ê°Ľ":145943,"ëĿµ":145944,"ë£Ł":145945,"ìıľ":145946,"ïºģ":145947,"ðŁĴŃ":145948,"âĬĥ":145949,"ðŁIJ°":145950,"ãħĮ":145951,"Üĵ":145952,"âŀķ":145953,"á½ģ":145954,"ìķ³":145955,"ðĿijĿ":145956,"ðŁİ¬":145957,"É¡":145958,"à¤Ĺ":145959,"áIJī":145960,"ì©ľ":145961,"ì¶§":145962,"ï³ī":145963,"ï»ħ":145964,"ðĿIJŀ":145965,"श":145966,"ðŁĵ¢":145967,"ðŁįĭ":145968,"ðŁĴħ":145969,"ï¾ķ":145970,"â¬Ĩ":145971,"âε":145972,"ð٤ij":145973,"áĥ£":145974,"ÆĦ":145975,"ѹ":145976,"á¼Ķ":145977,"ê°ł":145978,"ê´Į":145979,"ê·IJ":145980,"뼴":145981,"ì±ĺ":145982,"ï®Ń":145983,"ﺹ":145984,"ﺾ":145985,"âľĹ":145986,"âĿ¦":145987,"ðŁij¦":145988,"áĥĹ":145989,"Ù²":145990,"á½´":145991,"âĪı":145992,"âľ®":145993,"ê¹°":145994,"ë²µ":145995,"ìĦĢ":145996,"ì©Ŀ":145997,"ïºŀ":145998,"ﺽ":145999,"ðŁĩŃ":146000,"ËĤ":146001,"ðŁįij":146002,"ðŁįĮ":146003,"ðŁĶ»":146004,"깬":146005,"ìĬŃ":146006,"ìľ·":146007,"ðŁĽij":146008,"ǧ":146009,"ë¼Ľ":146010,"ﺡ":146011,"ﺺ":146012,"ðĿijļ":146013,"ðŁĵ¦":146014,"ðŁĶİ":146015,"ðŁĹĵ":146016,"áĥĶ":146017,"âľĴ":146018,"âľ¡":146019,"ðŁĮµ":146020,"âĶķ":146021,"ëĢĿ":146022,"ðŁįĬ":146023,"âĺĥ":146024,"ìĺħ":146025,"ব":146026,"ð٦ģ":146027,"âݯ":146028,"ðŁIJķ":146029,"Ñ¿":146030,"।":146031,"à¼ĭ":146032,"ê·Ī":146033,"ì«Į":146034,"ðŁĩ°":146035,"âĿī":146036,"ì«Ģ":146037,"íĿĦ":146038,"ðĿIJ¢":146039,"ðŁļ¨":146040,"âϤ":146041,"ðŁĺ©":146042,"ðŁįį":146043,"ðŁĺij":146044,"ðŁļļ":146045,"ÖĦ":146046,"ë«":146047,"뫼":146048,"à¤ı":146049,"á¿·":146050,"âĮ©":146051,"âĺIJ":146052,"âŀ£":146053,"긱":146054,"꼿":146055,"ëĦĿ":146056,"ìı´":146057,"ìļ¤":146058,"쿱":146059,"íİIJ":146060,"ðŁĴ¢":146061,"ì´IJ":146062,"âĩij":146063,"âĶĵ":146064,"âģ¾":146065,"ÜĿ":146066,"ðŁį°":146067,"â´°":146068,"Æı":146069,"ÏŁ":146070,"Úº":146071,"Ûĥ":146072,"áĦĴ":146073,"âĪŁ":146074,"âĿį":146075,"ãĦ²":146076,"ìľħ":146077,"ì¤ı":146078,"ðŁĩ²":146079,"êºĦ":146080,"ðŁİ¤":146081,"âľ£":146082,"â¸Ŀ":146083,"︵":146084,"ວ":146085,"áĢĻ":146086,"âķł":146087,"Õ¯":146088,"âı©":146089,"ðĿij£":146090,"ðŁĴ£":146091,"Åĺ":146092,"à¥IJ":146093,"âģĥ":146094,"âĮĺ":146095,"ê»Į":146096,"ìĮĶ":146097,"ðĿijĺ":146098,"ð٤ĵ":146099,"Õ¿":146100,"à¤Ń":146101,"âĮļ":146102,"âľĿ":146103,"ðŁIJ¼":146104,"ËĮ":146105,"âķļ":146106,"ï¦Ĺ":146107,"âĿķ":146108,"âķ£":146109,"ðŁIJ±":146110,"த":146111,"Ѿ":146112,"à¤ļ":146113,"à¤ľ":146114,"ìĪĦ":146115,"ìļľ":146116,"ðŁİ®":146117,"ÉĴ":146118,"Ú·":146119,"àºį":146120,"âĨµ":146121,"âĪĺ":146122,"âĿĬ":146123,"ë¿į":146124,"ìIJĪ":146125,"ìļĺ":146126,"쯧":146127,"íĥ¯":146128,"ìĸı":146129,"︰":146130,"ðŁĩ¯":146131,"ð٧ļ":146132,"ðŁĺµ":146133,"ðŁĺ·":146134,"ðŁĮ³":146135,"ລ":146136,"Äī":146137,"Ä¥":146138,"âľ¶":146139,"῾":146140,"âĬ±":146141,"âĺ¾":146142,"ê°ī":146143,"ê¼°":146144,"ëºij":146145,"ðŁĶĬ":146146,"ðŁĸIJ":146147,"Ť":146148,"Ò«":146149,"à®®":146150,"âĮĪ":146151,"âĹĹ":146152,"ëĦµ":146153,"ëħľ":146154,"ëľ¹":146155,"ðĿij¥":146156,"ðŁĴ¿":146157,"ðŁĽĴ":146158,"ÊĴ":146159,"áŀĵ":146160,"ðŁIJĿ":146161,"ð٦Ħ":146162,"ðŁį·":146163,"âĺŁ":146164,"︶":146165,"ðŁ¤Ł":146166,"Ô±":146167,"âĨ²":146168,"âĪİ":146169,"âľ«":146170,"ëĩ½":146171,"ëıIJ":146172,"ëķĦ":146173,"靈":146174,"ï§Ŀ":146175,"ïºĻ":146176,"ðŁij»":146177,"ðŁĵº":146178,"êµ¼":146179,"ìĮ©":146180,"ðŁĮ²":146181,"ȱ":146182,"íĶķ":146183,"ðŁĺ¤":146184,"ãĮ¢":146185,"ÊĶ":146186,"ड":146187,"á¼Ī":146188,"ëİĥ":146189,"멱":146190,"ë®Ī":146191,"ðĿIJ«":146192,"âĬķ":146193,"ëĥł":146194,"뻬":146195,"íĭĶ":146196,"Õ¤":146197,"á¼±":146198,"âľ¥":146199,"âĺĦ":146200,"âĪ¥":146201,"âļķ":146202,"ðŁijĦ":146203,"ðŁİħ":146204,"àºĻ":146205,"âͬ":146206,"á½µ":146207,"Õ¾":146208,"Öģ":146209,"âĹĶ":146210,"ê¿į":146211,"ëĸµ":146212,"ë©İ":146213,"ë®´":146214,"ìķ´":146215,"áĥľ":146216,"ἡ":146217,"âĶĬ":146218,"âķ®":146219,"âĹ¼":146220,"ðŁį¾":146221,"ðŁĽį":146222,"ðŁijĹ":146223,"ð٤ŀ":146224,"âľĦ":146225,"ÕĢ":146226,"ল":146227,"Ëī":146228,"⣨":146229,"į":146230,"ÏĬ":146231,"á´ľ":146232,"ë¹³":146233,"ï³ĭ":146234,"ï¿ł":146235,"Ī":146236,"âĤ¸":146237,"âľ±":146238,"ê»IJ":146239,"ëĭ»":146240,"맸":146241,"ìŀ¿":146242,"쩨":146243,"ìŃIJ":146244,"ì°¿":146245,"íħŁ":146246,"ðĿIJ§":146247,"ðĿijij":146248,"ðŁĮİ":146249,"ðŁĵ®":146250,"ðŁķĶ":146251,"âĹĻ":146252,"âĹ»":146253,"âŀ§":146254,"ìŁĿ":146255,"⾬":146256,"ãĥ°":146257,"âģĪ":146258,"âĵĺ":146259,"ðŁĴĮ":146260,"ï¬ĥ":146261,"àºĶ":146262,"ìͰ":146263,"ðŁĺª":146264,"×Ģ":146265,"ìĥ¨":146266,"ïŃĭ":146267,"ðŁįķ":146268,"ðŁĺ´":146269,"ϳ":146270,"á¼Ħ":146271,"á½ħ":146272,"âĩ¢":146273,"âķŃ":146274,"ìĺ»":146275,"íĬ¤":146276,"Üĺ":146277,"⤴":146278,"âĹį":146279,"áŀŁ":146280,"ðŁįº":146281,"áŀļ":146282,"ðŁıĬ":146283,"ðŁIJ·":146284,"ÊĮ":146285,"ὺ":146286,"âģ»":146287,"ê½Į":146288,"ëĪĹ":146289,"ëĹı":146290,"ì¿°":146291,"í̼":146292,"íįħ":146293,"ï·²":146294,"ðŁĮı":146295,"ðŁį«":146296,"ðŁį³":146297,"ðŁİ°":146298,"ðŁij°":146299,"ðŁĴ²":146300,"á¥Ļ":146301,"ðŁIJŁ":146302,"ï¿¡":146303,"ðŁĹ£":146304,"ðŁįľ":146305,"âľ²":146306,"ãİ¢":146307,"ðŁĶ°":146308,"Ἰ":146309,"á½ij":146310,"Äİ":146311,"áĦĢ":146312,"âĻķ":146313,"ëłĿ":146314,"ìĪ´":146315,"ïŃŃ":146316,"Óľ":146317,"ÔĢ":146318,"ëĢľ":146319,"ëĥĶ":146320,"ìĬĽ":146321,"ì«ij":146322,"캥":146323,"캬":146324,"ðĿij¦":146325,"ðŁĶ¶":146326,"쾨":146327,"ðĿIJļ":146328,"ðŁį»":146329,"ðŁĴį":146330,"ðŁ¤¡":146331,"ðŁķĬ":146332,"â½ĩ":146333,"âĵIJ":146334,"ðŁįŃ":146335,"ðŁįª":146336,"ðŁĶĨ":146337,"Ò¡":146338,"á´ĩ":146339,"ÉĹ":146340,"ÜĶ":146341,"âĦİ":146342,"âĿĥ":146343,"ëĹĢ":146344,"ï²Ķ":146345,"ïºĪ":146346,"ðĿIJ»":146347,"ðŁĴĬ":146348,"ðŁļ«":146349,"Ѱ":146350,"ѳ":146351,"ष":146352,"âĹł":146353,"ðŁij¤":146354,"ï¾ĩ":146355,"âĺĵ":146356,"ðŁįµ":146357,"ðŁ¤¨":146358,"âĸŃ":146359,"à®´":146360,"Ü¢":146361,"ܬ":146362,"à´®":146363,"ðŁķº":146364,"Ô¹":146365,"Õ£":146366,"à´¯":146367,"á´Ģ":146368,"âĮī":146369,"âľIJ":146370,"âŀ¦":146371,"ê¹½":146372,"ëĮľ":146373,"ðŁı¥":146374,"ðŁĵ©":146375,"Ò¹":146376,"Óĺ":146377,"à¤ħ":146378,"âĿ§":146379,"ÆĹ":146380,"âĹ½":146381,"ðŁij«":146382,"ðŁİ§":146383,"ðŁij£":146384,"âľ»":146385,"ðŁĻħ":146386,"ðŁĺĸ":146387,"ðŁĴ®":146388,"ະ":146389,"ðŁĶľ":146390,"ðŁįĦ":146391,"ð٤Ŀ":146392,"áĥĿ":146393,"áŀĢ":146394,"âĩ¦":146395,"ʾ":146396,"Ò®":146397,"Õ¼":146398,"à¤Ĩ":146399,"âĹħ":146400,"âļĵ":146401,"âļĸ":146402,"ê¿©":146403,"ë¯Ħ":146404,"ìIJIJ":146405,"ìŀ°":146406,"ì§Ń":146407,"íĭĭ":146408,"íݨ":146409,"íϧ":146410,"ï²ij":146411,"ðŁİĹ":146412,"Ù³":146413,"ðŁij¸":146414,"ম":146415,"ðŁijķ":146416,"Úµ":146417,"â̾":146418,"âŀ°":146419,"ðŁij¯":146420,"ðŁİ¼":146421,"ðŁıģ":146422,"ĺ":146423,"Êı":146424,"Ú³":146425,"âı±":146426,"ê½Ī":146427,"ëĿĮ":146428,"ìĮī":146429,"ìĹ·":146430,"ìŀ´":146431,"íĹ¹":146432,"íľ¨":146433,"ðĿĹ²":146434,"ðŁĮIJ":146435,"ðŁİĻ":146436,"ðŁıµ":146437,"íĽĻ":146438,"ðĿijħ":146439,"ðŁĺ¶":146440,"âĵħ":146441,"âķ¥":146442,"ðŁįı":146443,"ï¦İ":146444,"Õ©":146445,"ðĿIJĦ":146446,"Ó£":146447,"Ú¿":146448,"âĻļ":146449,"ðŁĶĹ":146450,"ḫ":146451,"âĭ®":146452,"âĸ¦":146453,"âĽ½":146454,"âľµ":146455,"ãħĨ":146456,"ãħĬ":146457,"ëĦĻ":146458,"ëĿ¨":146459,"ë¥Ħ":146460,"ìĦ¦":146461,"ì§°":146462,"ì§¹":146463,"íīĪ":146464,"ï§ij":146465,"ï»ĩ":146466,"ðŁĮ¾":146467,"ðŁıĸ":146468,"ðŁIJij":146469,"ðŁĴ³":146470,"ðŁĵĨ":146471,"Ûĩ":146472,"Üķ":146473,"á½½":146474,"ëĦľ":146475,"à´²":146476,"à´³":146477,"àºŃ":146478,"áĥĽ":146479,"âĿĶ":146480,"âijħ":146481,"áĥ¥":146482,"ðŁĵħ":146483,"âŀ³":146484,"á´µ":146485,"﹡":146486,"ï¹¶":146487,"ÎĨ":146488,"थ":146489,"áīµ":146490,"âĿĻ":146491,"âĿ±":146492,"ëīł":146493,"ëİł":146494,"ëıĽ":146495,"ë¿ħ":146496,"ì͏":146497,"íij¯":146498,"íŀī":146499,"íŀĽ":146500,"ï§Ħ":146501,"ïŃĺ":146502,"ﺦ":146503,"ﻸ":146504,"ðĿijĤ":146505,"ðĿijı":146506,"Ïij":146507,"Úł":146508,"áĢĶ":146509,"áŀĶ":146510,"á¹¢":146511,"ëĦ¸":146512,"ðĿIJ¨":146513,"ðŁĩ´":146514,"Õ°":146515,"ðŁijł":146516,"ðŁįĨ":146517,"ðŁıĢ":146518,"ðŁijIJ":146519,"ðŁįĩ":146520,"ðŁIJ£":146521,"áĪŃ":146522,"ܪ":146523,"ðŁĮĢ":146524,"áŀĺ":146525,"âĩĦ":146526,"ðĿIJĢ":146527,"ÊĻ":146528,"âͼ":146529,"ðŁı¿":146530,"Æ·":146531,"Èł":146532,"ѽ":146533,"âĤ¨":146534,"ê´Ń":146535,"ê¹»":146536,"ë͍":146537,"ìĪĢ":146538,"ì¾°":146539,"íĨĪ":146540,"ï®§":146541,"ﯽ":146542,"ðŁĶħ":146543,"ðŁĶ®":146544,"Å¢":146545,"ʰ":146546,"Ѹ":146547,"ण":146548,"âĬĹ":146549,"ëªĦ":146550,"ï¹·":146551,"ïºħ":146552,"ðĿIJµ":146553,"ðŁĮ¶":146554,"ðŁĵ°":146555,"ðŁĶ·":146556,"ðŁĸĴ":146557,"ðŁ¤²":146558,"ëī©":146559,"ðŁİĨ":146560,"ð٧IJ":146561,"ðŁį®":146562,"âĨº":146563,"âĿ¢":146564,"ðŁijª":146565,"ðŁij±":146566,"âĨ¡":146567,"áŀı":146568,"Úķ":146569,"ðŁį¹":146570,"ðŁĴĢ":146571,"Ë®":146572,"Ó¨":146573,"Öħ":146574,"à¤ĩ":146575,"âĤ¡":146576,"âĪķ":146577,"âĺī":146578,"ê¹¼":146579,"ê¼IJ":146580,"콸":146581,"ðĿIJ¬":146582,"ðŁıħ":146583,"ðŁijĻ":146584,"ðŁĴī":146585,"ð٤Ļ":146586,"Èĺ":146587,"ɳ":146588,"ɹ":146589,"Ùº":146590,"áĢĦ":146591,"ῳ":146592,"âļĺ":146593,"âĿĨ":146594,"ëĨī":146595,"ìĸį":146596,"ìĺĩ":146597,"ì¥ĺ":146598,"íĸħ":146599,"íĻij":146600,"ï®Ĭ":146601,"ï¿Ń":146602,"ðĿĴIJ":146603,"ðĿĹ¢":146604,"ðŁĶĸ":146605,"ðŁĶ¨":146606,"ðŁļij":146607,"ðŁļ²":146608,"Ƹ":146609,"âĹ¥":146610,"ðĿIJŃ":146611,"ðŁį½":146612,"âĹij":146613,"âĵĩ":146614,"ðŁĶ±":146615,"âľ¼":146616,"ï¹ĥ":146617,"âķ±":146618,"ãĢĹ":146619,"ðŁıĭ":146620,"ðŁļ´":146621,"ðĿIJ®":146622,"Äļ":146623,"Õı":146624,"Ķ":146625,"áĥij":146626,"Ṭ":146627,"ÄĪ":146628,"ÄĴ":146629,"Ò°":146630,"Óķ":146631,"âIJ":146632,"âIJ£":146633,"âĹ¢":146634,"âļĻ":146635,"ãħĹ":146636,"ê°¬":146637,"곪":146638,"ê»Ģ":146639,"ëĦ´":146640,"ëİģ":146641,"ëĿĶ":146642,"묽":146643,"ëŃį":146644,"ìĩ³":146645,"ì°¹":146646,"íĮ¹":146647,"íŀĿ":146648,"ï®ĭ":146649,"ï¶Ī":146650,"ðĿĴĤ":146651,"ðŁ¥Ģ":146652,"ð٦ħ":146653,"Êĺ":146654,"á¼ij":146655,"âģİ":146656,"ðŁįŀ":146657,"âĨĸ":146658,"âĨĻ":146659,"ðŁİĥ":146660,"âĦ¡":146661,"âĭ±":146662,"ðŁĶį":146663,"ನ":146664,"áµĥ":146665,"âĶ«":146666,"⦿":146667,"ðŁĩ»":146668,"Ƥ":146669,"Òı":146670,"Ò·":146671,"Ûī":146672,"à®ķ":146673,"ḳ":146674,"בּ":146675,"ðŁĨĶ":146676,"ÚŃ":146677,"Û¦":146678,"áħ¡":146679,"âĦ¹":146680,"ê¿İ":146681,"ëķĶ":146682,"ë¼ī":146683,"ìļ§":146684,"ì²µ":146685,"ì´¨":146686,"íĬĪ":146687,"íĸIJ":146688,"ðĿĹĺ":146689,"ðŁĩ¿":146690,"ðŁİĸ":146691,"ðŁijħ":146692,"ðŁĵĺ":146693,"ðŁļĻ":146694,"ðŁĽµ":146695,"à¶½":146696,"⼵":146697,"ðĿIJ³":146698,"ðĿIJ¸":146699,"âļĶ":146700,"ðŁijŃ":146701,"Óij":146702,"â͝":146703,"ðŁħ¿":146704,"ðŁĺ¹":146705,"ï¿«":146706,"⼤":146707,"ðŁĴĩ":146708,"ðŁĵİ":146709,"ðŁĸĭ":146710,"স":146711,"ðĿIJį":146712,"IJ":146713,"Ïĭ":146714,"Ѭ":146715,"Ú¬":146716,"ÜĴ":146717,"á´¬":146718,"ï¨Ħ":146719,"É£":146720,"Ëij":146721,"ϵ":146722,"ÒĿ":146723,"Û¥":146724,"Üł":146725,"à¹Ľ":146726,"áĥķ":146727,"áĬķ":146728,"á¾¶":146729,"âĤ·":146730,"âĩ¾":146731,"âķ©":146732,"âĸIJ":146733,"âĺª":146734,"âĺ®":146735,"âĿļ":146736,"âĿŃ":146737,"âŀ±":146738,"âµİ":146739,"ãıĬ":146740,"ë©ĵ":146741,"ìĹ¾":146742,"ìªĦ":146743,"íĵĮ":146744,"íķ¼":146745,"ïѬ":146746,"ðĿijĨ":146747,"ðĿijŀ":146748,"ðĿĸĬ":146749,"ðŁİ¸":146750,"ðŁıĦ":146751,"ðŁijµ":146752,"ðŁĴł":146753,"ðŁĶĺ":146754,"ðŁ¥Ĥ":146755,"Ū":146756,"à·ĥ":146757,"á´¼":146758,"âĬ°":146759,"ë³ı":146760,"ë´£":146761,"ï¥ľ":146762,"ðŁĵĪ":146763,"ðŁķ¯":146764,"ð٧Ģ":146765,"âĻIJ":146766,"ðŁĨĹ":146767,"ðŁĵķ":146768,"ð٧ģ":146769,"Ü«":146770,"âĿIJ":146771,"Õķ":146772,"à½ķ":146773,"âŀĿ":146774,"à¦ķ":146775,"ðĿIJ¶":146776,"É¢":146777,"ÎĦ":146778,"áĨ¢":146779,"âĤ±":146780,"Õį":146781,"à¡ķ":146782,"á´°":146783,"ḩ":146784,"⼷":146785,"âĿ®":146786,"ê¡ĵ":146787,"ëı¤":146788,"ëĹIJ":146789,"ëµĮ":146790,"ìijĪ":146791,"íı¿":146792,"íŵ":146793,"ðĿIJİ":146794,"ðŁĨĺ":146795,"ðŁıŁ":146796,"É¥":146797,"Õ»":146798,"à¡Ķ":146799,"à¤ĸ":146800,"á´¸":146801,"âİĻ":146802,"âİ¥":146803,"âı³":146804,"ëģķ":146805,"ëĬī":146806,"ì¡į":146807,"칡":146808,"禮":146809,"ï¬Ł":146810,"ﮫ":146811,"ﮯ":146812,"ï±ĥ":146813,"ï·»":146814,"ﺵ":146815,"ðĿĹĶ":146816,"ðĿĹ¡":146817,"ðŁİ¨":146818,"ðŁĶĴ":146819,"ÚĽ":146820,"ध":146821,"âŀ¹":146822,"áĢĢ":146823,"ðŁįħ":146824,"âŤ":146825,"à¤ł":146826,"ðŁIJ¥":146827,"áĥĴ":146828,"ðŁıĿ":146829,"ðŁį¼":146830,"ãĮ§":146831,"âĿĽ":146832,"ðŁIJĪ":146833,"য":146834,"áĢŀ":146835,"ãĢĸ":146836,"áŀĻ":146837,"প":146838,"ÕĨ":146839,"âĬĨ":146840,"âľ¾":146841,"ðŁIJĹ":146842,"ﹿ":146843,"Ħ":146844,"ÜŁ":146845,"à²ł":146846,"ಥ":146847,"áŀī":146848,"á´¥":146849,"á´©":146850,"á½Ģ":146851,"ὡ":146852,"âĨķ":146853,"âŀ¯":146854,"ê¡ij":146855,"ëij£":146856,"ë±Į":146857,"ìĪij":146858,"ìľĶ":146859,"ìŀ½":146860,"ì¨į":146861,"ðĿijĢ":146862,"ðŁĮĮ":146863,"ðŁį¦":146864,"ðŁį©":146865,"ðŁIJļ":146866,"ðŁĵĴ":146867,"ðŁĵ¹":146868,"ðŁ¥ij":146869,"Äĭ":146870,"ËĹ":146871,"Ñ«":146872,"Õ¢":146873,"Ú°":146874,"âĮĢ":146875,"âĹĤ":146876,"âĹ£":146877,"⾼":146878,"âĿĴ":146879,"âĿĺ":146880,"âŀĻ":146881,"âŀ²":146882,"ãİį":146883,"ê¡IJ":146884,"ëŀĸ":146885,"ìĬĿ":146886,"ìĽ¤":146887,"ì¡ĭ":146888,"쨰":146889,"íĹĻ":146890,"兩":146891,"ï³į":146892,"ï»İ":146893,"ðĿijĵ":146894,"ðŁĵĬ":146895,"ðŁļ¼":146896,"ï¦ģ":146897,"ðĿķĴ":146898,"ðŁijľ":146899,"ðŁij¿":146900,"ðŁĩ½":146901,"à·Ħ":146902,"âĸ´":146903,"ãįī":146904,"âĬĩ":146905,"ðŁ§¸":146906,"Ú¡":146907,"â¾ĥ":146908,"ðŁĹ»":146909,"âĵij":146910,"ðŁ¤¸":146911,"ðŁ¤¯":146912,"êĴ°":146913,"ðĿIJĵ":146914,"âĶ´":146915,"êĴ±":146916,"áĢĺ":146917,"âĽĦ":146918,"ï¹¹":146919,"ÓĶ":146920,"áĥ±":146921,"Ü¡":146922,"ßŀ":146923,"âĻı":146924,"⾸":146925,"ìij¨":146926,"ðĿIJĿ":146927,"ðĿIJ¥":146928,"ðŁįī":146929,"ðŁij¼":146930,"ðŁ¥Ŀ":146931,"ÆĶ":146932,"ݬ":146933,"फ":146934,"àºļ":146935,"á´´":146936,"á½ĸ":146937,"âĤ¶":146938,"âİ¢":146939,"âĿħ":146940,"⣫":146941,"ãİĽ":146942,"뮨":146943,"ëºĮ":146944,"ë¼ĺ":146945,"ìĨĿ":146946,"ìľ³":146947,"ìŀĮ":146948,"ì£Ĺ":146949,"ìªĺ":146950,"컹":146951,"ï·¼":146952,"ïºĤ":146953,"ðĿIJ´":146954,"ðĿIJ¼":146955,"ðŁĮļ":146956,"ðŁı«":146957,"ðŁĴ¤":146958,"ðŁĴ¶":146959,"ðŁĴ¼":146960,"Êķ":146961,"ʽ":146962,"â²Ł":146963,"ãīł":146964,"ê¡Ĵ":146965,"ëľĢ":146966,"ìĥ¾":146967,"츤":146968,"ï¥ģ":146969,"ðĿļĬ":146970,"ðŁļĥ":146971,"âŀĽ":146972,"ìħ´":146973,"áĦĭ":146974,"âĩĹ":146975,"ï§·":146976,"âĺĸ":146977,"ðŁIJ¦":146978,"⸾":146979,"ðŁĴ´":146980,"ð٤ļ":146981,"ãĬĹ":146982,"âĮĽ":146983,"áĪĽ":146984,"༺":146985,"â½ī":146986,"ðŁı¢":146987,"âĵŀ":146988,"âĺ½":146989,"ãĢĻ":146990,"ðŁ¤®":146991,"ÅIJ":146992,"áĥ¬":146993,"ðĿĹ»":146994,"ðŁįĸ":146995,"ÆĬ":146996,"ÊŁ":146997,"ßĭ":146998,"à¤ĭ":146999,"áµĶ":147000,"á¿ĥ":147001,"âĦī":147002,"âĮĭ":147003,"âı²":147004,"âĵĪ":147005,"âĵ¢":147006,"âķĶ":147007,"âļij":147008,"âĿĭ":147009,"âĿİ":147010,"⵾":147011,"âµ£":147012,"ëĴĪ":147013,"ëľģ":147014,"ë¶ĩ":147015,"ìį»":147016,"ìĺŃ":147017,"ì§¢":147018,"íĹĢ":147019,"ï§Ĭ":147020,"טּ":147021,"ﱡ":147022,"ðĿIJº":147023,"ðĿij§":147024,"ðĿĺ¦":147025,"ðŁĵ¥":147026,"ðŁĺŁ":147027,"ðŁ¥IJ":147028,"Äĸ":147029,"ɨ":147030,"áĢIJ":147031,"áĥĵ":147032,"áºĵ":147033,"á¼¶":147034,"á½Ħ":147035,"âĤ¤":147036,"âĮľ":147037,"âĮŁ":147038,"âİł":147039,"⼸":147040,"âµį":147041,"âµı":147042,"âµĵ":147043,"ãĢĺ":147044,"ë·¸":147045,"íħ¼":147046,"ï¦Į":147047,"ïŃĦ":147048,"ïŃİ":147049,"ðĿĻļ":147050,"ðĿļĺ":147051,"à¼ĵ":147052,"ëŃħ":147053,"áIJĽ":147054,"ãݾ":147055,"ï¨Ģ":147056,"ðŁĹ½":147057,"âĻŀ":147058,"Ëĸ":147059,"âĹŀ":147060,"ðŁ¤«":147061,"ðŁĺĹ":147062,"ヲ":147063,"ðŁ¤¢":147064,"âģĩ":147065,"ã̵":147066,"ðŁįĶ":147067,"áĬł":147068,"ðŁĺ¼":147069,"ðĿĹ®":147070,"ðŁIJ³":147071,"ðĿIJĭ":147072,"ðŁĨļ":147073,"ðŁĶĽ":147074,"Ñ»":147075,"ܨ":147076,"ல":147077,"âľŀ":147078,"âµĻ":147079,"êµ£":147080,"츨":147081,"ðĿIJľ":147082,"ðĿĺ°":147083,"ðŁĶ½":147084,"Ç»":147085,"Ç¿":147086,"Êĩ":147087,"ÎIJ":147088,"ÐĢ":147089,"Ñ¡":147090,"Ѳ":147091,"ÒĴ":147092,"Ù¶":147093,"ßķ":147094,"à¶±":147095,"áIJģ":147096,"âģŀ":147097,"âĸ§":147098,"âĽĪ":147099,"âľľ":147100,"âľ¹":147101,"âŁ¹":147102,"â¤ĩ":147103,"ê²Ĭ":147104,"ê¾ľ":147105,"ë¯IJ":147106,"ë³IJ":147107,"ìħ©":147108,"ìIJ¬":147109,"ìij¹":147110,"ï¤Ķ":147111,"ï¦ļ":147112,"ï¬ł":147113,"ïŃĶ":147114,"ﺶ":147115,"ðĿĴı":147116,"ðĿĸĨ":147117,"ðĿŶ":147118,"ðŁıĤ":147119,"ðŁIJ½":147120,"ðŁĴ©":147121,"ðŁĵ½":147122,"ðŁĹ¨":147123,"ðŁĹº":147124,"ðŁĺ¸":147125,"ðŁ¥§":147126,"ÅĹ":147127,"Êİ":147128,"ÒĻ":147129,"ײ":147130,"à¤Ī":147131,"á¼´":147132,"á¿ij":147133,"âµī":147134,"ãħĵ":147135,"ì½´":147136,"ðĿĸĵ":147137,"ðŁĵĹ":147138,"ðŁĶª":147139,"ðŁĸį":147140,"ÏĴ":147141,"ðŁij¬":147142,"áĥĻ":147143,"âĨ¬":147144,"âͤ":147145,"âĽ¹":147146,"âĻŁ":147147,"ðŁļ¶":147148,"ðŁij¾":147149,"âĪĭ":147150,"ðŁIJ¯":147151,"à¼İ":147152,"âľ·":147153,"ï¨Ļ":147154,"âĶ»":147155,"ðŁij¹":147156,"áĦī":147157,"ສ":147158,"â¾ı":147159,"â½ħ":147160,"ãİĸ":147161,"Ñ´":147162,"Õ®":147163,"Ú¼":147164,"áĢķ":147165,"áĨ¼":147166,"ëŃı":147167,"ðŁIJ¸":147168,"ðŁļ£":147169,"ÆĿ":147170,"Ô»":147171,"áĥ¢":147172,"ðŁį¯":147173,"ɦ":147174,"Õ¦":147175,"âĻĭ":147176,"שׂ":147177,"ðĿŦ":147178,"Çļ":147179,"ɱ":147180,"à¤ī":147181,"á´Ħ":147182,"âĻĵ":147183,"⼰":147184,"âŁª":147185,"ëĥĺ":147186,"뢸":147187,"ìĤij":147188,"ï®Ķ":147189,"ðĿķĸ":147190,"ðĿŧ":147191,"ðŁĩ¼":147192,"ðŁĵĭ":147193,"ðŁļľ":147194,"ðŁ¥¤":147195,"Ä®":147196,"Å·":147197,"ßĬ":147198,"॥":147199,"ப":147200,"áŀĦ":147201,"áµĢ":147202,"á¸ħ":147203,"á¼¢":147204,"âĪĿ":147205,"âĬ¹":147206,"âĴ¶":147207,"âķ´":147208,"⼱":147209,"âĽ³":147210,"âĽº":147211,"âŀŁ":147212,"ãıĦ":147213,"ê¸Ķ":147214,"ê¹Ł":147215,"ëĩ°":147216,"ë¹»":147217,"ìĤ¥":147218,"ìĽ»":147219,"ì°Ł":147220,"íĥ°":147221,"íĨº":147222,"íļ½":147223,"老":147224,"量":147225,"ï³Ŀ":147226,"ðĿIJ¦":147227,"ðĿĴľ":147228,"ðĿĴŁ":147229,"ðĿļĹ":147230,"ðŁİŃ":147231,"ðŁıĵ":147232,"ðŁı³":147233,"ðŁıº":147234,"ðŁIJį":147235,"ðŁijĥ":147236,"ðŁĴı":147237,"ð٤ĸ":147238,"ðŁ¤µ":147239,"Õ²":147240,"âµĶ":147241,"ëĺ¬":147242,"念":147243,"ÊĤ":147244,"áĨ«":147245,"áŀij":147246,"ðĿĸİ":147247,"ðĿĹĸ":147248,"áĦĥ":147249,"âĩł":147250,"áĢ¡":147251,"à½Ħ":147252,"âŀ¸":147253,"ï¦Ļ":147254,"âĩļ":147255,"ðŁIJ¬":147256,"ðŁIJ¢":147257,"â¾Ĵ":147258,"ðŁIJ¤":147259,"ðŁĶ«":147260,"ãĢŀ":147261,"︺":147262,"ðŁĺº":147263,"â½´":147264,"ðŁĨķ":147265,"âģ¿":147266,"ðŁį¨":147267,"à²ķ":147268,"ðŁļĺ":147269,"áŀħ":147270,"à¦ħ":147271,"áŀ¢":147272,"à¨ľ":147273,"âļĮ":147274,"ã̽":147275,"à·´":147276,"âĵĽ":147277,"áĢľ":147278,"ìĨ¨":147279,"Ë©":147280,"ÜĹ":147281,"âĭ¼":147282,"ðŁĻī":147283,"ÅĬ":147284,"Éĵ":147285,"ʲ":147286,"ΰ":147287,"Ѽ":147288,"Ô¿":147289,"à¡IJ":147290,"à¼ľ":147291,"ས":147292,"á¶ľ":147293,"âĤ²":147294,"âĨ¨":147295,"âĬ¥":147296,"âķ§":147297,"âĻľ":147298,"ãĭ¡":147299,"ë´¬":147300,"ë¶ij":147301,"ìī¿":147302,"ìİħ":147303,"ìł±":147304,"ì°§":147305,"ﲡ":147306,"ðĿĴĽ":147307,"ðĿķ£":147308,"ðĿĹľ":147309,"ðŁį²":147310,"ðŁİ©":147311,"ðŁIJIJ":147312,"ðŁIJł":147313,"ðŁij½":147314,"ðŁĴij":147315,"ðŁĵľ":147316,"ðŁķµ":147317,"ðŁļĮ":147318,"ðŁĽ£":147319,"Êĭ":147320,"Ó¯":147321,"Ù¸":147322,"ßĶ":147323,"ßĻ":147324,"à¡ĵ":147325,"á´į":147326,"ḿ":147327,"âıº":147328,"âĸ¥":147329,"뤽":147330,"íľij":147331,"ðĿIJ¹":147332,"ðĿĸĶ":147333,"ðĿļİ":147334,"ðŁĵĦ":147335,"ðŁ¦·":147336,"Æĥ":147337,"à¦Ł":147338,"âĮĤ":147339,"âĺŃ":147340,"â²ļ":147341,"ëĿķ":147342,"ðŁİ£":147343,"à®ĩ":147344,"à½Ĩ":147345,"áħµ":147346,"áĹľ":147347,"â̽":147348,"âĮ£":147349,"âģ½":147350,"ðŁĵ¬":147351,"ðŁ¤§":147352,"âĩª":147353,"â½£":147354,"âĹŁ":147355,"ï¨Ĺ":147356,"êĴª":147357,"ðŁĽĢ":147358,"ÇĤ":147359,"ðŁ¥¶":147360,"ðŁİį":147361,"ï¿©":147362,"ðŁijĴ":147363,"áµĪ":147364,"︿":147365,"áħ©":147366,"⾦":147367,"à°¤":147368,"á´ĸ":147369,"ਬ":147370,"àºĹ":147371,"༻":147372,"Ѻ":147373,"ਪ":147374,"á´³":147375,"ðĿIJĪ":147376,"à»Ģ":147377,"á´¿":147378,"âĤį":147379,"âĩ¡":147380,"âĽª":147381,"ðĿIJĤ":147382,"ðĿĴķ":147383,"ðŁIJľ":147384,"Êį":147385,"ѱ":147386,"à½ĥ":147387,"ë®IJ":147388,"ìĽ¡":147389,"ìľģ":147390,"ðĿIJ¿":147391,"ðĿķł":147392,"ðŁijĽ":147393,"ƪ":147394,"Ϻ":147395,"Ó¬":147396,"Ù¿":147397,"Ý£":147398,"àªī":147399,"ஹ":147400,"à½ij":147401,"áĨ¯":147402,"áµĩ":147403,"âĩ¥":147404,"âıª":147405,"âϰ":147406,"âļŃ":147407,"âļ¾":147408,"ãħĦ":147409,"ḛ̂":147410,"ê°Ĺ":147411,"ê²ĭ":147412,"ê²»":147413,"ê¶ľ":147414,"ê¼ĩ":147415,"ê½¹":147416,"ëĤŁ":147417,"ëħĪ":147418,"ëĭ¢":147419,"ë§Ł":147420,"ëªĨ":147421,"ëµĢ":147422,"ì½±":147423,"íĩĺ":147424,"íľľ":147425,"ï§¾":147426,"ï±µ":147427,"ï²¢":147428,"ﲤ":147429,"ðĿĴĬ":147430,"ðĿĺ¯":147431,"ðŁįĹ":147432,"ðŁıį":147433,"ðŁIJĺ":147434,"ðŁĵ¡":147435,"ðŁĶŀ":147436,"ðŁ¤³":147437,"ðŁ¥ģ":147438,"ðŁ¥Ĺ":147439,"ð٦Ĭ":147440,"ĵ":147441,"Ʀ":147442,"ǵ":147443,"ɯ":147444,"Îı":147445,"ÕĦ":147446,"Ü¥":147447,"à½ģ":147448,"ᨳ":147449,"âķ«":147450,"ãİī":147451,"ë·´":147452,"ìĨİ":147453,"ìİĮ":147454,"죵":147455,"íĽł":147456,"離":147457,"ï³ı":147458,"ﻺ":147459,"ðĿijģ":147460,"ðĿijĩ":147461,"ðĿĴĨ":147462,"ðŁİł":147463,"ðŁIJĶ":147464,"ðŁijŁ":147465,"Åĸ":147466,"à¤Į":147467,"á¾½":147468,"ê¦Ĵ":147469,"à®Ł":147470,"á´±":147471,"ðŁı°":147472,"ðŁIJŀ":147473,"à½Ģ":147474,"áĢħ":147475,"âĬ¿":147476,"ðŁIJ§":147477,"áĽģ":147478,"â¼Ī":147479,"âĶ¿":147480,"ðŁ¥´":147481,"⼿":147482,"ðŁ§ľ":147483,"ãħ¿":147484,"âĦ«":147485,"ã̳":147486,"ãĬĻ":147487,"â¼Ģ":147488,"怜":147489,"ðŁı¬":147490,"ðŁĵ»":147491,"áĬĽ":147492,"áĦħ":147493,"àºĬ":147494,"àºĽ":147495,"áħ³":147496,"ðŁij®":147497,"à®±":147498,"âĺĩ":147499,"ðĿIJı":147500,"à´µ":147501,"à»ģ":147502,"à½ı":147503,"ར":147504,"ᥱ":147505,"âĤ£":147506,"復":147507,"ïŃĻ":147508,"ï´©":147509,"ï¹Ĥ":147510,"ðŁį£":147511,"ðŁķ¹":147512,"Ïĸ":147513,"ම":147514,"ຢ":147515,"áĭŃ":147516,"âİĿ":147517,"âĹĿ":147518,"âĻĪ":147519,"âĻİ":147520,"ê½¥":147521,"ì³Ķ":147522,"ì¼ij":147523,"ï±°":147524,"ðĿijĥ":147525,"ðŁĮª":147526,"ðŁį¡":147527,"Åİ":147528,"ʦ":147529,"ѧ":147530,"Óİ":147531,"Ô´":147532,"ÚĪ":147533,"ßĵ":147534,"ß§":147535,"à¤Ķ":147536,"áĪ«":147537,"áε":147538,"áĹ©":147539,"á´ł":147540,"á¼ł":147541,"âĢĹ":147542,"âģij":147543,"âĦı":147544,"âĸĩ":147545,"â²£":147546,"ãĦ³":147547,"ãī®":147548,"ê³Ĺ":147549,"ëĦĴ":147550,"ëĸ«":147551,"ë¡Ħ":147552,"ë¹°":147553,"ë½ģ":147554,"ìĦģ":147555,"ìĮĺ":147556,"ìŁĮ":147557,"ì³ī":147558,"ì¼ķ":147559,"כּ":147560,"ï³İ":147561,"ﹸ":147562,"ï¹¾":147563,"ðĿIJĨ":147564,"ðĿij·":147565,"ðĿĽ¼":147566,"ðŁİı":147567,"ðŁİŀ":147568,"ðŁIJĻ":147569,"ðŁijĤ":147570,"ðŁĵģ":147571,"ðŁĸ±":147572,"ðŁļį":147573,"ðŁļ§":147574,"ðŁĽ¡":147575,"ð٤Ĵ":147576,"ðŁ¥ŀ":147577,"ðŁ¥©":147578,"ð٦Ģ":147579,"ð٦ĸ":147580,"Ë¢":147581,"Üļ":147582,"வ":147583,"áĢģ":147584,"áī°":147585,"âıŃ":147586,"âĻ¿":147587,"ê³ĺ":147588,"ëıĿ":147589,"ëķĥ":147590,"ìħĮ":147591,"ìĴ¸":147592,"ìĽŁ":147593,"íħĦ":147594,"íľ«":147595,"ï§ĺ":147596,"↓":147597,"ðŁı·":147598,"ðŁĶ§":147599,"ðŁ¥Ī":147600,"Æĸ":147601,"áŀĩ":147602,"áŀĸ":147603,"âģº":147604,"âĹľ":147605,"âŀ©":147606,"ê¦Ń":147607,"ëϤ":147608,"ïѼ":147609,"ðĿĻĸ":147610,"ðĿĻ£":147611,"ðĿϤ":147612,"ðŁĮĿ":147613,"ðŁĶij":147614,"ðŁĽł":147615,"àºĩ":147616,"âĺ£":147617,"ãĦ¨":147618,"ðĿĸĹ":147619,"Óĵ":147620,"âĨ£":147621,"ðŁ¥ī":147622,"ðŁĮł":147623,"ðŁĺ½":147624,"ãİł":147625,"ŧ":147626,"ðŁIJĴ":147627,"ï§IJ":147628,"ðŁĺ¿":147629,"âά":147630,"ðŁIJ®":147631,"⣱":147632,"ಡ":147633,"â¾¼":147634,"à°²":147635,"˶":147636,"âĸ¿":147637,"ÕĪ":147638,"áŀİ":147639,"áħ¥":147640,"áŀĹ":147641,"Õ§":147642,"ð٤IJ":147643,"ðŁįł":147644,"ত":147645,"ය":147646,"âĻį":147647,"ìĺĻ":147648,"íĺĵ":147649,"ﹺ":147650,"ðŁĽ³":147651,"Åī":147652,"á´İ":147653,"âıľ":147654,"âͳ":147655,"긷":147656,"ì¡Ķ":147657,"ðĿĴĪ":147658,"ðĿĴį":147659,"ðĿĴ¹":147660,"ðĿĵĩ":147661,"ðĿķŁ":147662,"ðĿĹ¹":147663,"ðŁĮħ":147664,"ðŁı´":147665,"ÄĶ":147666,"Ĥ":147667,"ŵ":147668,"Ǿ":147669,"Ïŀ":147670,"϶":147671,"Ô³":147672,"ÜĨ":147673,"ß©":147674,"à¡Ĵ":147675,"à¤ĺ":147676,"à¶ļ":147677,"à½ĸ":147678,"áģĬ":147679,"áĥŀ":147680,"áĦĤ":147681,"áĭ«":147682,"á´º":147683,"ḣ":147684,"Ḫ":147685,"á¹Ĥ":147686,"á¼·":147687,"á¿ĩ":147688,"âĩĮ":147689,"âı¬":147690,"âĻĮ":147691,"⮣":147692,"â´»":147693,"ⵣ":147694,"ê¦ķ":147695,"ꦪ":147696,"ꦮ":147697,"ê²Ħ":147698,"ê¾IJ":147699,"ëĥij":147700,"ëķĭ":147701,"롸":147702,"ë¬Ģ":147703,"ìĩ¤":147704,"ìĪ©":147705,"ìľķ":147706,"ìŃĺ":147707,"ì·°":147708,"ì·¸":147709,"íľĢ":147710,"藍":147711,"ï§į":147712,"ï±Ħ":147713,"ï³ij":147714,"ðĿIJ¤":147715,"ðĿĴĵ":147716,"ðĿĴ¶":147717,"ðĿĹ¼":147718,"ðĿĻĬ":147719,"ðŁĩ¾":147720,"ðŁĮĽ":147721,"ðŁĮ®":147722,"ðŁİĩ":147723,"ðŁİ²":147724,"ðŁıĽ":147725,"ðŁij¥":147726,"ðŁij´":147727,"ðŁĴĨ":147728,"ðŁĵĤ":147729,"ðŁĵ§":147730,"ðŁķIJ":147731,"ðŁĸķ":147732,"ðŁĺ§":147733,"ðŁĻĢ":147734,"ðŁļĴ":147735,"ðŁĽ«":147736,"ðŁ¤ł":147737,"ðŁ¥ļ":147738,"ðŁ¥Ľ":147739,"ðŁ¥£":147740,"ǯ":147741,"ȧ":147742,"ÎĬ":147743,"Ò²":147744,"×°":147745,"Ûij":147746,"áĥ©":147747,"áĦĮ":147748,"áĪį":147749,"áī¥":147750,"áıĤ":147751,"âģ±":147752,"âĬ¢":147753,"âĹĵ":147754,"âĿ°":147755,"ë¿¡":147756,"ìĽ©":147757,"íģŃ":147758,"íĨ³":147759,"íĬĦ":147760,"íĵ¸":147761,"北":147762,"若":147763,"ï±IJ":147764,"ﱯ":147765,"ï³ļ":147766,"ðĿĸĺ":147767,"ðĿĺĢ":147768,"ðŁIJĬ":147769,"ðŁIJĮ":147770,"ðŁijļ":147771,"ðŁĵĥ":147772,"ðŁļĽ":147773,"ðŁļª":147774,"ðŁ¤°":147775,"Ä´":147776,"áĥ®":147777,"áŨ":147778,"âĻ®":147779,"â²ŀ":147780,"ãĪĶ":147781,"ìħį":147782,"ãħĥ":147783,"率":147784,"ມ":147785,"Õİ":147786,"Õº":147787,"⬼":147788,"⽤":147789,"ðĿIJ²":147790,"âŀµ":147791,"áĢĽ":147792,"âĶħ":147793,"âĨŁ":147794,"â¼Ĭ":147795,"ðŁĮ½":147796,"ðŁļ¿":147797,"ï¦Ĭ":147798,"ãĦ£":147799,"⼩":147800,"ï©Ľ":147801,"ðŁį±":147802,"⾨":147803,"à´¤":147804,"áŀģ":147805,"àºŀ":147806,"Êļ":147807,"ðĿIJĴ":147808,"à´±":147809,"áŀľ":147810,"ன":147811,"à°Ĺ":147812,"à´ļ":147813,"âĩ£":147814,"ï¦ķ":147815,"Õħ":147816,"Æĺ":147817,"âĤ¦":147818,"âĶĦ":147819,"ï¦Ł":147820,"嶺":147821,"ðĿIJģ":147822,"ðĿIJĥ":147823,"ðŁį¸":147824,"ðŁIJ²":147825,"Ŷ":147826,"Éĸ":147827,"ßĺ":147828,"ฦ":147829,"à½Ķ":147830,"áĨ·":147831,"âģķ":147832,"âĵĤ":147833,"âĿľ":147834,"便":147835,"אַ":147836,"ðĿĹĿ":147837,"ðĿĹ¿":147838,"ðŁİ¾":147839,"ðŁĹĿ":147840,"ð٦Į":147841,"Æħ":147842,"Ǫ":147843,"ÒĹ":147844,"ÜĽ":147845,"ßł":147846,"à¡ij":147847,"áī£":147848,"áĬŃ":147849,"ṡ":147850,"âŀ¼":147851,"âŀ¾":147852,"â´±":147853,"ãī¡":147854,"곯":147855,"ë½Ī":147856,"ìĤĺ":147857,"ìīij":147858,"ì«ĺ":147859,"íĮĥ":147860,"íϰ":147861,"ï¤Ĺ":147862,"ðŁĮ¬":147863,"ðŁĮ°":147864,"ðŁį¤":147865,"Ä»":147866,"Åĩ":147867,"ƨ":147868,"Éķ":147869,"Ò¢":147870,"Òº":147871,"Öį":147872,"×±":147873,"Ú±":147874,"Ú½":147875,"ÛIJ":147876,"à¤Ľ":147877,"à·Ģ":147878,"à¹ļ":147879,"ຫ":147880,"á´¹":147881,"á½Ķ":147882,"á¾³":147883,"âĤĴ":147884,"âĨ´":147885,"âĩĿ":147886,"âīħ":147887,"âĮ¨":147888,"âĵĵ":147889,"âĸ¢":147890,"âļ¬":147891,"âŀŃ":147892,"â²Ĵ":147893,"ãİ¿":147894,"ê¿´":147895,"ëα":147896,"ëį¬":147897,"ëİIJ":147898,"ëIJ«":147899,"ëĶ«":147900,"ë±ģ":147901,"ìĥ¥":147902,"íĮ¼":147903,"ïŃĵ":147904,"ﮥ":147905,"ï²°":147906,"ðĿIJĩ":147907,"ðĿIJij":147908,"ðĿijĮ":147909,"ðĿĵª":147910,"ðĿķļ":147911,"ðĿĺª":147912,"ðĿĺ¼":147913,"ðĿļĽ":147914,"ðŁĩ¶":147915,"ðŁĮĦ":147916,"ðŁĮķ":147917,"ðŁĮ¤":147918,"ðŁĮ§":147919,"ðŁį¬":147920,"ðŁİĭ":147921,"ðŁİ»":147922,"ðŁı¨":147923,"ðŁIJĩ":147924,"ðŁijĵ":147925,"ðŁĵIJ":147926,"ðŁĵĻ":147927,"ðŁĶ¼":147928,"ðŁķĴ":147929,"ðŁĸı":147930,"ðŁĸ¥":147931,"ðŁ¤¬":147932,"ðŁ¥Ĭ":147933,"ðŁ¥Ĵ":147934,"ßĮ":147935,"àºĦ":147936,"á¼µ":147937,"âķ¡":147938,"Ⲥ":147939,"â´¼":147940,"âµ¢":147941,"ãΝ":147942,"ëĵ¸":147943,"ëŁĩ":147944,"ëºį":147945,"ðĿϧ":147946,"ðŁįĪ":147947,"ðŁĶ¬":147948,"ðŁĸĬ":147949,"ðŁ¤¾":147950,"Ë¡":147951,"Ü©":147952,"âĮ¡":147953,"âŃij":147954,"Ⲧ":147955,"ë©ī":147956,"ì¼Ń":147957,"¦":147958,"ðĿĴİ":147959,"ðĿĹ¥":147960,"ðŁIJµ":147961,"ðŁķ¶":147962,"ðŁķ¸":147963,"ðŁ¤ľ":147964,"Õª":147965,"áĪĭ":147966,"ðŁ¥µ":147967,"ï°ģ":147968,"áµIJ":147969,"âķĵ":147970,"áĢĸ":147971,"âĭĪ":147972,"Éŀ":147973,"âŀ®":147974,"॰":147975,"ãĨģ":147976,"ðŁĴ±":147977,"ðŁıŃ":147978,"áĨ¨":147979,"ðŁįļ":147980,"ð٦IJ":147981,"á´»":147982,"âĺĮ":147983,"à´ķ":147984,"Õ±":147985,"áħ®":147986,"ðĿIJĮ":147987,"Ŧ":147988,"àºķ":147989,"âľĻ":147990,"˳":147991,"Ôµ":147992,"âķĴ":147993,"ðĿĹĹ":147994,"ðĿĹł":147995,"Úļ":147996,"ধ":147997,"âĨĿ":147998,"âĻī":147999,"ãĮ»":148000,"ì¹Ĭ":148001,"ðĿĹº":148002,"ð٧ĺ":148003,"ì³£":148004,"ï¬Ŀ":148005,"ðŁijº":148006,"ÇŁ":148007,"ÎĪ":148008,"Ϋ":148009,"Ñ¥":148010,"Ô²":148011,"Õ¨":148012,"ܦ":148013,"à¦Ĩ":148014,"থ":148015,"áIJ¢":148016,"á¼ģ":148017,"á¼ĺ":148018,"ἦ":148019,"âĵĿ":148020,"ãΰ":148021,"ãİĹ":148022,"겡":148023,"ë¨Ģ":148024,"ì£Ķ":148025,"ì´¤":148026,"ìµĿ":148027,"ï§´":148028,"ïŃĬ":148029,"ï²Ł":148030,"ðĿIJ·":148031,"ðĿijĭ":148032,"ðĿĵī":148033,"ðĿĺµ":148034,"ðŁĴ·":148035,"ðŁĽ©":148036,"ðŁ§¹":148037,"ÅĶ":148038,"Êŀ":148039,"Ë¥":148040,"ÎĮ":148041,"Ñ©":148042,"ÓIJ":148043,"Ół":148044,"Úij":148045,"ÚĴ":148046,"ߨ":148047,"àªĪ":148048,"áIJĥ":148049,"ṯ":148050,"âĤĭ":148051,"âĤµ":148052,"âĦħ":148053,"âĦł":148054,"âĪ£":148055,"âīº":148056,"âī»":148057,"âĬĽ":148058,"âĮIJ":148059,"âİĵ":148060,"âĺ¸":148061,"âĻĴ":148062,"âļĴ":148063,"âľĩ":148064,"âľł":148065,"â´·":148066,"âµĸ":148067,"ãĦ¸":148068,"ãī¢":148069,"ãī°":148070,"êĩ´":148071,"ê´¸":148072,"êºł":148073,"ëĤı":148074,"ëĤ¢":148075,"ëIJĢ":148076,"뺴":148077,"ìĥľ":148078,"ìįħ":148079,"줫":148080,"챦":148081,"ìºij":148082,"ì¼ģ":148083,"쿳":148084,"íĤģ":148085,"íħ¡":148086,"íĴĤ":148087,"íĴī":148088,"íľĦ":148089,"ïŃª":148090,"ﮬ":148091,"ﯦ":148092,"ﱪ":148093,"ï²ı":148094,"ï´Ģ":148095,"ï»Ĩ":148096,"₩":148097,"ðĿijĹ":148098,"ðĿĸĻ":148099,"ðŁĮ¡":148100,"ðŁįĿ":148101,"ðŁį§":148102,"ðŁİ«":148103,"ðŁıĺ":148104,"ðŁıª":148105,"ðŁIJĭ":148106,"ðŁIJĽ":148107,"ðŁIJº":148108,"ðŁijĸ":148109,"ðŁijŀ":148110,"ðŁij·":148111,"ðŁĵĢ":148112,"ðŁĶĦ":148113,"ðŁĶĮ":148114,"ðŁķĻ":148115,"ðŁĻį":148116,"ðŁĻİ":148117,"ð٦į":148118,"ǰ":148119,"ÉŁ":148120,"ÊĨ":148121,"Ô¼":148122,"Úľ":148123,"ড":148124,"শ":148125,"áĴĥ":148126,"Ἡ":148127,"âĵķ":148128,"â²Ī":148129,"ê°°":148130,"ê¹ł":148131,"êºħ":148132,"ëĦ¹":148133,"ë¯ĵ":148134,"íIJĪ":148135,"ï§¶":148136,"ï®ij":148137,"ﲨ":148138,"ðĿĴī":148139,"ðĿĴĶ":148140,"ðĿŨ":148141,"ðĿĻŀ":148142,"ðĿļĴ":148143,"ðĿļķ":148144,"ðŁIJİ":148145,"ð٤ķ":148146,"ð٧Ķ":148147,"ϰ":148148,"ÔĿ":148149,"âĮĬ":148150,"âĴ¾":148151,"ãī£":148152,"ïŃ©":148153,"ðĿļŀ":148154,"Êij":148155,"দ":148156,"áĦĩ":148157,"âīĥ":148158,"â²Ģ":148159,"ìŁİ":148160,"ðĿij¶":148161,"ðĿĵ²":148162,"ðŁİ·":148163,"ðŁļ¹":148164,"àºģ":148165,"áłł":148166,"ãĦļ":148167,"ðŁIJ¿":148168,"áĽļ":148169,"âķ³":148170,"ðŁIJŃ":148171,"âĴ¹":148172,"ðĿĸļ":148173,"âĻĸ":148174,"ãβ":148175,"âĨ¾":148176,"áĦĨ":148177,"âķĽ":148178,"ð٤į":148179,"â½¥":148180,"ðŁĮ¨":148181,"âĪ®":148182,"ãĮĺ":148183,"ãįij":148184,"ï¹Ģ":148185,"âĵĹ":148186,"âĬĦ":148187,"ðŁı¹":148188,"ËĴ":148189,"ðŁ¤±":148190,"ãıľ":148191,"ðŁİĮ":148192,"ï¥Ń":148193,"ণ":148194,"ðŁİ¹":148195,"ãĬŁ":148196,"à´°":148197,"ðĿIJĶ":148198,"à´¨":148199,"à½ļ":148200,"âľº":148201,"Õ·":148202,"ðŁij³":148203,"à¦ľ":148204,"âĺĭ":148205,"âĻĬ":148206,"ãĢĽ":148207,"Èĭ":148208,"à®°":148209,"áĥ¨":148210,"âĦķ":148211,"íijĢ":148212,"ðĿĵĥ":148213,"ð٦Ķ":148214,"Ä¿":148215,"ÅĢ":148216,"Ƴ":148217,"Éļ":148218,"Öĥ":148219,"Ü£":148220,"ߣ":148221,"à¦Ń":148222,"à§¡":148223,"à¶»":148224,"ຣ":148225,"à½ĩ":148226,"Ḩ":148227,"á½Ī":148228,"⽬":148229,"ê¡Ķ":148230,"ì³Ħ":148231,"ï¨ī":148232,"ðĿIJ¡":148233,"ðĿĺ¢":148234,"ðŁį¿":148235,"ðŁİŁ":148236,"ðŁıī":148237,"ðŁĶIJ":148238,"ðŁļħ":148239,"ðŁ¤½":148240,"Æį":148241,"Ç«":148242,"ǽ":148243,"Èļ":148244,"Îī":148245,"Ó¤":148246,"Óª":148247,"ÕĬ":148248,"Ù¼":148249,"Ú´":148250,"ßĿ":148251,"à¶ľ":148252,"á¼ķ":148253,"á¿¥":148254,"âİŀ":148255,"ãĢļ":148256,"ãī¤":148257,"곸":148258,"ê·ģ":148259,"ëĵĦ":148260,"ëĵķ":148261,"ì¨Ķ":148262,"챨":148263,"ðĿIJ¾":148264,"ðĿij»":148265,"ðĿͼ":148266,"ðĿķĿ":148267,"ðĿĺŃ":148268,"ðŁĨĻ":148269,"ðŁĵ¤":148270,"ðŁĶŁ":148271,"ðŁĹ¼":148272,"Äľ":148273,"Æģ":148274,"Æ¿":148275,"dz":148276,"Ç·":148277,"Éĥ":148278,"Éł":148279,"Êī":148280,"ʧ":148281,"˲":148282,"Ï´":148283,"Õģ":148284,"Õŀ":148285,"Öĩ":148286,"ÛĤ":148287,"Ûĵ":148288,"ßĹ":148289,"ߦ":148290,"হ":148291,"ள":148292,"à´¸":148293,"à»Ĥ":148294,"áĪĿ":148295,"áĪª":148296,"áĭµ":148297,"áIJĬ":148298,"áĴª":148299,"áļĸ":148300,"áŀĽ":148301,"á´¢":148302,"áµı":148303,"áµŃ":148304,"á¶«":148305,"á¸ı":148306,"áºĴ":148307,"á¼¥":148308,"á½ķ":148309,"á½¼":148310,"âĤĬ":148311,"âĦĤ":148312,"âĦ©":148313,"âĩī":148314,"âī£":148315,"âĮł":148316,"âİŁ":148317,"âı®":148318,"âķĺ":148319,"âĹĸ":148320,"âĺ©":148321,"âĻij":148322,"âϲ":148323,"âļĽ":148324,"ãĦŁ":148325,"ãī±":148326,"ãİļ":148327,"ê¡ķ":148328,"êªĸ":148329,"ê°¹":148330,"ê²Ĩ":148331,"êµĦ":148332,"ëĩ¬":148333,"ëĭ¯":148334,"ëıł":148335,"ëĴ¬":148336,"ëĸĪ":148337,"ëĸ½":148338,"ëĺĶ":148339,"ëŀ¸":148340,"ë¸ħ":148341,"뻳":148342,"ë¿Ł":148343,"ìĤµ":148344,"ìĬī":148345,"ìľ°":148346,"ìłĭ":148347,"ìłĶ":148348,"쥡":148349,"ìŃĿ":148350,"켬":148351,"íĪĩ":148352,"íīľ":148353,"íįĦ":148354,"íĽ¾":148355,"íĿ£":148356,"朗":148357,"勞":148358,"ï¦ľ":148359,"獵":148360,"ï§ľ":148361,"ï¨Ī":148362,"שׁ":148363,"הּ":148364,"ïѽ":148365,"ï®ī":148366,"ï¯ŀ":148367,"ï°Ĵ":148368,"ï±ĩ":148369,"ï¿Ħ":148370,"ðĿIJħ":148371,"ðĿijĦ":148372,"ðĿijº":148373,"ðĿĴĹ":148374,"ðĿĵ®":148375,"ðĿķĽ":148376,"ðĿķŀ":148377,"ðĿĸij":148378,"ðĿĺģ":148379,"ðĿĺĨ":148380,"ðĿĺ¶":148381,"ðĿĻ¢":148382,"ðĿļľ":148383,"ðŁĮĥ":148384,"ðŁĮ¦":148385,"ðŁįŁ":148386,"ðŁİİ":148387,"ðŁıĻ":148388,"ðŁIJ©":148389,"ðŁIJ«":148390,"ðŁIJ´":148391,"ðŁijĶ":148392,"ðŁĵī":148393,"ðŁĵĽ":148394,"ðŁĶī":148395,"ðŁĸ¼":148396,"ðŁĹĥ":148397,"ðŁĹ¯":148398,"ðŁļĩ":148399,"ðŁļIJ":148400,"ðŁļµ":148401,"ðŁ¤¶":148402,"ðŁ¥ĭ":148403,"ðŁ¥ĵ":148404,"ðŁ¥®":148405,"ð٦İ":148406,"ðŁ¦ł":148407,"ð٧Ĵ":148408,"ðŁ§¨":148409,"ÆIJ":148410,"Çį":148411,"ÓĢ":148412,"ÔĽ":148413,"ರ":148414,"à´Ļ":148415,"áĢĴ":148416,"ê²Ŀ":148417,"ê¹¹":148418,"ë©¥":148419,"ìĸĶ":148420,"ï¤ģ":148421,"ï¤ı":148422,"ï¦ī":148423,"ï¦ĵ":148424,"ï§ī":148425,"ï²Ŀ":148426,"ðĿĹŀ":148427,"ðĿű":148428,"ðŁĮĭ":148429,"ðŁį¶":148430,"à¦ļ":148431,"ìķľ":148432,"ðĿIJ¯":148433,"ðĿļĿ":148434,"à°¨":148435,"à½ĺ":148436,"à½ł":148437,"á¡¥":148438,"á¾°":148439,"âģį":148440,"âͰ":148441,"⬾":148442,"ðĿIJł":148443,"ðĿij¯":148444,"ðĿĹĽ":148445,"ðĿĵ»":148446,"ðĿĸĪ":148447,"âŀ»":148448,"áŀł":148449,"⡱":148450,"â»ij":148451,"ðŁ§µ":148452,"廉":148453,"ðŁijĺ":148454,"ãĤĶ":148455,"â¼Ł":148456,"ãĬ¤":148457,"ï¦Ŀ":148458,"ãĮ¦":148459,"â̏":148460,"ðŁĶĻ":148461,"ã¹":148462,"㹦":148463,"ï¹ħ":148464,"ï©Į":148465,"ãī¨":148466,"︽":148467,"âį¥":148468,"ðŁļī":148469,"ðŁ¥ľ":148470,"âĵľ":148471,"â»Ŀ":148472,"ï¨ľ":148473,"ðŁĴĴ":148474,"áĦij":148475,"â¾ŀ":148476,"ï¨ģ":148477,"à´ª":148478,"áĦİ":148479,"âŀ´":148480,"ষ":148481,"áħ¬":148482,"áŀ§":148483,"âĨ¢":148484,"âķ¦":148485,"âľij":148486,"ˬ":148487,"ÕIJ":148488,"à¼Ķ":148489,"ʤ":148490,"˨":148491,"à¤ŀ":148492,"à»ĥ":148493,"à¼ļ":148494,"âĵ¥":148495,"âķľ":148496,"ðŁIJĸ":148497,"á¼Ļ":148498,"ἤ":148499,"ìĨ°":148500,"ÈĤ":148501,"ʱ":148502,"à®ļ":148503,"áĥ§":148504,"á´ĭ":148505,"á´®":148506,"âĿ¡":148507,"âŀ·":148508,"ëĿ¡":148509,"ï§¢":148510,"ﯡ":148511,"ðĿķķ":148512,"ðŁħ°":148513,"ðŁ¦¸":148514,"Ǹ":148515,"Óŀ":148516,"Ô¶":148517,"ÖĨ":148518,"Úģ":148519,"Ûĭ":148520,"áİ¥":148521,"᾿":148522,"âĶŃ":148523,"âĶ®":148524,"êĢĢ":148525,"ê±ĺ":148526,"ëIJŃ":148527,"ë½Ħ":148528,"ìĶIJ":148529,"ì¸Į":148530,"íģł":148531,"íϱ":148532,"ï¥ī":148533,"ï¨ĸ":148534,"ðĿij´":148535,"ðĿĸĴ":148536,"ðĿĺ¨":148537,"ðĿļĮ":148538,"ðŁIJ¡":148539,"ðŁij¢":148540,"ðŁĵĶ":148541,"Åħ":148542,"Æİ":148543,"È©":148544,"Òª":148545,"Ôĥ":148546,"áĥ«":148547,"á¸ĩ":148548,"⼣":148549,"ê»Ń":148550,"ë¨Ħ":148551,"ìŁĢ":148552,"줴":148553,"íļIJ":148554,"盧":148555,"ðŁŁ¢":148556,"Ƨ":148557,"ȼ":148558,"ÊĿ":148559,"ËĦ":148560,"Ëħ":148561,"Ëį":148562,"˧":148563,"Ò¥":148564,"ÕĶ":148565,"Øı":148566,"ؼ":148567,"ßIJ":148568,"ßľ":148569,"à¤ĵ":148570,"à¦Ļ":148571,"à®ĵ":148572,"à¶´":148573,"à¼į":148574,"à¼Ĵ":148575,"ལ":148576,"áĢĤ":148577,"áĢĬ":148578,"áĦĦ":148579,"áĪĺ":148580,"áĭĬ":148581,"áĮį":148582,"áijĭ":148583,"áŀĤ":148584,"áł¢":148585,"á¡Ŀ":148586,"á´¦":148587,"áµį":148588,"ᵨ":148589,"ḡ":148590,"ḯ":148591,"á¼£":148592,"âģĤ":148593,"âĦĺ":148594,"âĦľ":148595,"âĦ³":148596,"âĦµ":148597,"âĨ¦":148598,"âĩĨ":148599,"âĪ·":148600,"âĬļ":148601,"âĮ«":148602,"âĮ¯":148603,"âİĽ":148604,"âİľ":148605,"âݤ":148606,"âݦ":148607,"âİ®":148608,"âijī":148609,"âĶī":148610,"âķĻ":148611,"âĸĤ":148612,"âĹŃ":148613,"âĺĬ":148614,"âĺį":148615,"âĺĴ":148616,"âļĨ":148617,"⼧":148618,"âĽ²":148619,"âŀĺ":148620,"â¥Ħ":148621,"â´³":148622,"â´½":148623,"âµĪ":148624,"ãī¯":148625,"ãİij":148626,"㧬":148627,"êϬ":148628,"ê§ģ":148629,"곬":148630,"ê´ŀ":148631,"ê»ľ":148632,"ëħĵ":148633,"ëĭ¼":148634,"ëįĸ":148635,"ëĸ±":148636,"ëĿ°":148637,"롹":148638,"뢴":148639,"ë£Ģ":148640,"뤳":148641,"ë¨ķ":148642,"ëŃ¥":148643,"ìĦ¶":148644,"ìħ¤":148645,"ìĮķ":148646,"ìįª":148647,"ìı©":148648,"ìĴĢ":148649,"ì͝":148650,"ìĿĶ":148651,"ìĿľ":148652,"ìłŃ":148653,"짦":148654,"쨩":148655,"첬":148656,"ì³¥":148657,"켯":148658,"íĢ«":148659,"íĢŃ":148660,"íĥ¸":148661,"íĵģ":148662,"íķ¬":148663,"íŸ":148664,"íĽķ":148665,"íľŃ":148666,"íĿĹ":148667,"ï¤Į":148668,"浪":148669,"ï§¿":148670,"ï¬Ħ":148671,"ï¬ħ":148672,"ïŃij":148673,"ïŃ«":148674,"ïŃº":148675,"ï®Ĥ":148676,"ﮢ":148677,"ﮨ":148678,"ï°İ":148679,"ï°ł":148680,"ï²£":148681,"ï³IJ":148682,"ï³Ĵ":148683,"ï³ĺ":148684,"ï³ľ":148685,"ï¹¼":148686,"│":148687,"ðĿIJ©":148688,"ðĿĴļ":148689,"ðĿķĶ":148690,"ðĿķ¤":148691,"ðĿĸĮ":148692,"ðĿĹ£":148693,"ðĿŰ":148694,"ðĿĹ´":148695,"ðĿĺĤ":148696,"ðĿĺ¥":148697,"ðĿĺ®":148698,"ðĿĺ¸":148699,"ðĿĻĢ":148700,"ðĿĽ¾":148701,"ðĿľı":148702,"ðŁĮģ":148703,"ðŁĮľ":148704,"ðŁĮ¥":148705,"ðŁĮ¯":148706,"ðŁįIJ":148707,"ðŁİĴ":148708,"ðŁıĶ":148709,"ðŁıķ":148710,"ðŁı®":148711,"ðŁIJĤ":148712,"ðŁIJī":148713,"ðŁIJ¹":148714,"ðŁĶķ":148715,"ðŁĶļ":148716,"ðŁķij":148717,"ðŁķ£":148718,"ðŁĹŀ":148719,"ðŁĹ¡":148720,"ðŁĹ¿":148721,"ðŁļĨ":148722,"ðŁļĬ":148723,"ðŁļĵ":148724,"ðŁļķ":148725,"ðŁļ¾":148726,"ðŁĽģ":148727,"ðŁĽİ":148728,"ðŁĽı":148729,"ðŁ¤´":148730,"ðŁ¥ķ":148731,"ðŁ¥ĸ":148732,"ðŁ¥ł":148733,"ðŁ¥¥":148734,"ð٦Ĩ":148735,"ð٦ī":148736,"ð٦ļ":148737,"ð٧ij":148738,"ðŁ§¥":148739,"ðŁ§¿":148740,"Ű":148741,"ƺ":148742,"ɧ":148743,"àªĩ":148744,"ண":148745,"áĪĪ":148746,"áĬ¤":148747,"áĭ®":148748,"áĮĪ":148749,"áĮµ":148750,"ᥲ":148751,"âĵŁ":148752,"êϳ":148753,"ê°Ĭ":148754,"ëķģ":148755,"ëķ¨":148756,"ìĬģ":148757,"例":148758,"גּ":148759,"ðĿĸį":148760,"ðĿĺĮ":148761,"ðĿĺ³":148762,"ðĿĻ©":148763,"ðŁįĻ":148764,"ðŁĸĸ":148765,"áī³":148766,"áĭ¨":148767,"áĸĩ":148768,"áŀĮ":148769,"á¹§":148770,"âķª":148771,"âŀļ":148772,"â²ĺ":148773,"êķ":148774,"êķ¥":148775,"路":148776,"ﮣ":148777,"ï¯ł":148778,"ðĿĴĸ":148779,"ðĿķĺ":148780,"ðĿĸĩ":148781,"ðĿĹŁ":148782,"ðĿĹª":148783,"ðĿĹ¯":148784,"ðĿĻł":148785,"ðŁĵı":148786,"à¦Ĺ":148787,"âĴ»":148788,"â²ł":148789,"ðĿĵµ":148790,"Ê£":148791,"à°ľ":148792,"áĬ¢":148793,"áŀIJ":148794,"ḷ":148795,"âĦĽ":148796,"âĩĢ":148797,"âĩĬ":148798,"êĴ¦":148799,"ê¦ł":148800,"ﮤ":148801,"ðŁįĽ":148802,"ðŁ¤Ľ":148803,"ᨾ":148804,"âŀº":148805,"áķ¯":148806,"áĽı":148807,"âĩĤ":148808,"â͹":148809,"âĻĹ":148810,"ðŁĸ¨":148811,"ê¦ı":148812,"ર":148813,"áļ¨":148814,"ðŁ¤¥":148815,"ðŁ§¢":148816,"ãIJĤ":148817,"ãĦ¥":148818,"ðŁĸĮ":148819,"â¼Ĵ":148820,"ãĬ§":148821,"âį©":148822,"ð٦ij":148823,"âĶ·":148824,"ï©IJ":148825,"ï©¡":148826,"ðĵĪ":148827,"ðĵĪĴ":148828,"â»Ħ":148829,"ï¨Ĵ":148830,"âĦª":148831,"Ò§":148832,"ÚĮ":148833,"â̶":148834,"âºł":148835,"â»ģ":148836,"âĨ¸":148837,"áĦIJ":148838,"ãħIJ":148839,"à»Ħ":148840,"áĹª":148841,"âĨ¼":148842,"âĩĭ":148843,"âĩĺ":148844,"âĮij":148845,"âĸ©":148846,"ðĿIJĹ":148847,"ÄĬ":148848,"à¦ī":148849,"ìīł":148850,"ɤ":148851,"ßį":148852,"ßı":148853,"áµĹ":148854,"âĤ¥":148855,"âĵī":148856,"âĶł":148857,"â͍":148858,"âķĦ":148859,"ä¤":148860,"ä¤Ģ":148861,"껸":148862,"ï®ģ":148863,"ðĵĤ":148864,"ðĵĤĥ":148865,"ð٦ķ":148866,"ÆĽ":148867,"à¦ĩ":148868,"ãıĺ":148869,"﮼":148870,"Úĵ":148871,"ÚĿ":148872,"à¦ĵ":148873,"ද":148874,"á´ħ":148875,"á½Ļ":148876,"âģ¼":148877,"âĸİ":148878,"⼩":148879,"äĶ":148880,"äĶĢ":148881,"뻡":148882,"ìĽ½":148883,"íģĦ":148884,"良":148885,"ï±ī":148886,"ï¹»":148887,"ðĿĸĭ":148888,"ðĿĻĪ":148889,"ðĿĻª":148890,"ðĿ϶":148891,"ðŁIJĦ":148892,"ðŁIJĨ":148893,"áİ¢":148894,"á¸Į":148895,"âĿ´":148896,"ðŁı¸":148897,"ÈĿ":148898,"ɸ":148899,"Îħ":148900,"Ïľ":148901,"Ó¢":148902,"Õ¹":148903,"à´ħ":148904,"àºĪ":148905,"áĭ°":148906,"áijİ":148907,"áłµ":148908,"á¡ł":148909,"á´ī":148910,"ḵ":148911,"á¿´":148912,"âĵ£":148913,"âͶ":148914,"⽯":148915,"ê²¥":148916,"ê¿ĺ":148917,"ëģİ":148918,"ëİĪ":148919,"ë͝":148920,"ë²°":148921,"ìĺ¯":148922,"ìĽ¸":148923,"ìŀĹ":148924,"ì§ĺ":148925,"쬬":148926,"ì·¬":148927,"íģħ":148928,"íĵĶ":148929,"íĽĿ":148930,"冷":148931,"魯":148932,"沈":148933,"ï¯ĸ":148934,"ðĿĵħ":148935,"ðĿĻĦ":148936,"ðŁĵ¶":148937,"ðŁĹĴ":148938,"ðŁ¥Ķ":148939,"ðŁ¥Ń":148940,"Å®":148941,"Å´":148942,"Æī":148943,"Æ«":148944,"Çģ":148945,"Ç£":148946,"Ǻ":148947,"Ǽ":148948,"Èį":148949,"ȯ":148950,"Éľ":148951,"ʬ":148952,"Ëģ":148953,"ˤ":148954,"˵":148955,"ÏĽ":148956,"Ò¤":148957,"Ò¬":148958,"Óı":148959,"ÓĽ":148960,"Ó¡":148961,"Ó³":148962,"ÔĮ":148963,"Ô¬":148964,"Õ³":148965,"Ù»":148966,"Úī":148967,"Ú§":148968,"Üľ":148969,"ߪ":148970,"à¤Ŀ":148971,"à¦Ľ":148972,"à¨Ĩ":148973,"àªķ":148974,"ડ":148975,"à®İ":148976,"à°¬":148977,"ൻ":148978,"ർ":148979,"à¶ł":148980,"à¶Ń":148981,"à¶¶":148982,"à·Ĩ":148983,"༽":148984,"áĢļ":148985,"áħ¢":148986,"áĨ¸":148987,"áĪĢ":148988,"áĪķ":148989,"áΰ":148990,"áī¡":148991,"áī¤":148992,"áĬ¦":148993,"áĬ«":148994,"áĭĭ":148995,"áĭį":148996,"áݯ":148997,"áijŃ":148998,"áķĹ":148999,"᣼":149000,"á¥Ĵ":149001,"á©ī":149002,"áŃº":149003,"á´¡":149004,"áµĺ":149005,"ᵼ":149006,"á¶ł":149007,"á¸ģ":149008,"á¸ĭ":149009,"á¹Ļ":149010,"á¹Ŀ":149011,"Ṧ":149012,"áºħ":149013,"á¼Ĥ":149014,"á½ĥ":149015,"á½į":149016,"á½§":149017,"á¾·":149018,"â̵":149019,"âĤİ":149020,"âĦĿ":149021,"âħĢ":149022,"âĨŀ":149023,"âĨ§":149024,"âĩħ":149025,"âĪĥ":149026,"âīı":149027,"âī½":149028,"âĬŀ":149029,"âĬ¡":149030,"âĬ§":149031,"âĬ¶":149032,"âĭĦ":149033,"âİĴ":149034,"âİ¡":149035,"âİ£":149036,"âݪ":149037,"âıİ":149038,"âĵĥ":149039,"âĵĸ":149040,"âĵ¨":149041,"âķĭ":149042,"âķĸ":149043,"âķ¢":149044,"âķ²":149045,"âĸĨ":149046,"âĸĬ":149047,"âĸį":149048,"âĸ®":149049,"âĺ¡":149050,"âĺ¦":149051,"âĺ±":149052,"âĺ¿":149053,"âĻĺ":149054,"âĻĿ":149055,"âļ°":149056,"âĽij":149057,"âŀª":149058,"â¤Ŀ":149059,"⤢":149060,"⤷":149061,"â§«":149062,"â¨Ń":149063,"⨯":149064,"â±£":149065,"â²İ":149066,"⵼":149067,"ãħĶ":149068,"ãĪı":149069,"ãī²":149070,"ãī³":149071,"ãĬij":149072,"ãĭĽ":149073,"ãİIJ":149074,"겤":149075,"ê·¿":149076,"ê¹ŀ":149077,"껨":149078,"ê¼į":149079,"꿸":149080,"ëĥ¬":149081,"ëĩIJ":149082,"ëĭł":149083,"ëį¯":149084,"ëĹĮ":149085,"ëĹij":149086,"ë¥Ģ":149087,"ëªĥ":149088,"몯":149089,"뱡":149090,"ë³ĵ":149091,"ë³½":149092,"뵾":149093,"ìĤ³":149094,"ìħ¥":149095,"ìĩ½":149096,"ìı¨":149097,"ìı¸":149098,"ìķį":149099,"ìĸĸ":149100,"ìŁ¨":149101,"ì¢ĥ":149102,"ì¢į":149103,"ì¥ij":149104,"ì§¼":149105,"ì©ĥ":149106,"ì®ľ":149107,"쮸":149108,"ì³ij":149109,"ì´¥":149110,"ì¾ĥ":149111,"íħ¦":149112,"íĪ¿":149113,"íĵ½":149114,"íķ³":149115,"íĸı":149116,"íĹł":149117,"íĿ«":149118,"ï¤ĵ":149119,"ï¤ĺ":149120,"ï¥İ":149121,"略":149122,"ï¦ħ":149123,"尿":149124,"ï§ĩ":149125,"ï¬Ĩ":149126,"דּ":149127,"ï®ĩ":149128,"ï®Ī":149129,"ï®Ŀ":149130,"ﮩ":149131,"ï®±":149132,"ï¯ĺ":149133,"ï¯Ļ":149134,"ﯢ":149135,"ﯣ":149136,"ﯤ":149137,"ﯥ":149138,"ï±Ĥ":149139,"ï²Ĩ":149140,"ﲪ":149141,"ï´¼":149142,"ïºī":149143,"ïºĬ":149144,"ﺥ":149145,"ðĿij¨":149146,"ðĿij©":149147,"ðĿij²":149148,"ðĿĴĮ":149149,"ðĿĴª":149150,"ðĿĴ®":149151,"ðĿĵĤ":149152,"ðĿĵĪ":149153,"ðĿĵ¯":149154,"ðĿ͍":149155,"ðĿķĢ":149156,"ðĿķĨ":149157,"ðĿķ¦":149158,"ðĿķ§":149159,"ðĿķ«":149160,"ðĿķ·":149161,"ðĿŵ":149162,"ðĿŸ":149163,"ðĿĺĦ":149164,"ðĿĺĻ":149165,"ðĿĺł":149166,"ðĿĺ¬":149167,"ðĿĻį":149168,"ðĿĻij":149169,"ðĿĻ¡":149170,"ðĿύ":149171,"ðĿĻ·":149172,"ðĿļį":149173,"ðĿĽ¿":149174,"ðŁĥ":149175,"ðŁĥı":149176,"ðŁħĺ":149177,"ðŁī":149178,"ðŁīij":149179,"ðŁİ¡":149180,"ðŁİª":149181,"ðŁİ±":149182,"ðŁİ³":149183,"ðŁİº":149184,"ðŁıİ":149185,"ðŁıĹ":149186,"ðŁıļ":149187,"ðŁıŀ":149188,"ðŁı¦":149189,"ðŁı§":149190,"ðŁIJģ":149191,"ðŁIJħ":149192,"ðŁIJĵ":149193,"ðŁĴĤ":149194,"ðŁĵij":149195,"ðŁĵĵ":149196,"ðŁĵ¨":149197,"ðŁĵ«":149198,"ðŁĶĭ":149199,"ðŁĶŃ":149200,"ðŁĶ¯":149201,"ðŁķĹ":149202,"ðŁļĤ":149203,"ðŁļ¢":149204,"ðŁļ¦":149205,"ðŁļ¬":149206,"ðŁĽĭ":149207,"ðŁĽĮ":149208,"ðŁĽ¬":149209,"ðŁĽ¶":149210,"ðŁŁ¡":149211,"ðŁ¥ĺ":149212,"ðŁ¥Ł":149213,"ðŁ¥¦":149214,"ð٦ĩ":149215,"ð٦Ī":149216,"ð٧Ĭ":149217,"ð٧Ĺ":149218,"ðŁ§¤":149219,"Ê·":149220,"˹":149221,"á¹ļ":149222,"á½¥":149223,"âĦŁ":149224,"겯":149225,"껫":149226,"ë°·":149227,"ìĥĨ":149228,"ìĽĿ":149229,"ì¨ī":149230,"ì«ı":149231,"ï¯ķ":149232,"ðĿľĭ":149233,"ɲ":149234,"ÒŃ":149235,"ÓĪ":149236,"à½Ľ":149237,"áĭĵ":149238,"áĻŃ":149239,"áł©":149240,"á¹®":149241,"âĦĴ":149242,"âĨ»":149243,"âµĥ":149244,"ë̍":149245,"ëł§":149246,"ìī¥":149247,"ìĮľ":149248,"ìŶ":149249,"ì¨Ī":149250,"쪾":149251,"íı½":149252,"íļĶ":149253,"íĽµ":149254,"露":149255,"ï¦IJ":149256,"ï§Ĺ":149257,"ï§ļ":149258,"אָ":149259,"ðĿIJĬ":149260,"ðĿķĹ":149261,"ðĿĹļ":149262,"ðĿļĸ":149263,"ðŁħ´":149264,"Èĥ":149265,"ÉĿ":149266,"ϱ":149267,"ÓĹ":149268,"ढ":149269,"áħł":149270,"áī¦":149271,"áijĮ":149272,"áĴ¼":149273,"áŀ¡":149274,"᳨":149275,"áłŃ":149276,"á¨ħ":149277,"á¨Ķ":149278,"á´ĺ":149279,"ᶦ":149280,"á¸İ":149281,"á¼ħ":149282,"á¼¹":149283,"âĨ¯":149284,"âĵİ":149285,"ãıĮ":149286,"êī":149287,"êīĤ":149288,"ëĨ§":149289,"ëĿ±":149290,"좡":149291,"íν":149292,"ï¤ĩ":149293,"ï¤Ľ":149294,"ðĿIJķ":149295,"ðĿĵ¸":149296,"ðĿĵ¼":149297,"ðĿĹķ":149298,"ðĿĺĪ":149299,"ðŁı£":149300,"ðŁı¤":149301,"ðŁĹĦ":149302,"Ñ·":149303,"Òł":149304,"áµĸ":149305,"Ἠ":149306,"ë¬Ħ":149307,"ï°´":149308,"âν":149309,"ÕŃ":149310,"Ú¹":149311,"à¥Ł":149312,"áĢĨ":149313,"áŀĴ":149314,"ã̶":149315,"ꦫ":149316,"ï¸ĵ":149317,"ðĿIJĽ":149318,"ðĿĺĹ":149319,"ðŁıľ":149320,"ì«Ń":149321,"ð٧ŀ":149322,"à½Ĥ":149323,"âĨ¿":149324,"âĩı":149325,"âĵģ":149326,"âͧ":149327,"âķģ":149328,"âķ¤":149329,"ê¦Ĺ":149330,"ꦤ":149331,"ðŁıĪ":149332,"áŀķ":149333,"Ô½":149334,"àªĹ":149335,"à¬Ĩ":149336,"âķķ":149337,"ï½ł":149338,"⼦":149339,"⼯":149340,"â¾·":149341,"âĶĸ":149342,"à¬ĵ":149343,"âĺĹ":149344,"âįĭ":149345,"ï¨Ŀ":149346,"â¼¥":149347,"寧":149348,"âĦĬ":149349,"ãĢ´":149350,"âį¢":149351,"ð¡Ī":149352,"ð¡Ī½":149353,"難":149354,"ãĢ»":149355,"ãıĥ":149356,"說":149357,"ï¨ĺ":149358,"ðŁIJĥ":149359,"ðŁĨĸ":149360,"ðŁĹ¾":149361,"ãĦĩ":149362,"Þĭ":149363,"â¼¼":149364,"ï¨Ń":149365,"ÞĢ":149366,"ÞĦ":149367,"ÞĪ":149368,"ÞIJ":149369,"âĮĦ":149370,"â»ĺ":149371,"ãŁ¢":149372,"áħ§":149373,"ðIJĮ¿":149374,"Ë»":149375,"à²Ĺ":149376,"áĢĩ":149377,"áŀĬ":149378,"âķĩ":149379,"ãĩ¼":149380,"ãݰ":149381,"ÕĴ":149382,"ÜĪ":149383,"ߥ":149384,"à¿IJ":149385,"áĢŁ":149386,"âĨ¥":149387,"âķĮ":149388,"â½Ģ":149389,"â½°":149390,"â¾Ĭ":149391,"äĦ":149392,"äĦĢ":149393,"ðĵIJ":149394,"ðĵIJį":149395,"ðŁİ¦":149396,"âĤ¯":149397,"âĬĺ":149398,"âĦį":149399,"ʵ":149400,"Ѷ":149401,"Úĥ":149402,"à¦Ķ":149403,"à´¦":149404,"áݶ":149405,"áĵķ":149406,"Ṩ":149407,"âĤł":149408,"âĩ°":149409,"âĹĴ":149410,"â¿Ĭ":149411,"ê·±":149412,"ì¹ķ":149413,"íĪ©":149414,"ïŃĢ":149415,"ðĿĴ¸":149416,"ðĿĵĬ":149417,"ðĿĺ©":149418,"Ǧ":149419,"É«":149420,"áĬ¨":149421,"ȹ":149422,"ʯ":149423,"Ϊ":149424,"ÚĢ":149425,"áĮ¸":149426,"áİ»":149427,"áıķ":149428,"áı´":149429,"á²Ĥ":149430,"Ὠ":149431,"âıĿ":149432,"âĺĻ":149433,"ëĥ¨":149434,"ëĦ¼":149435,"ëĪĻ":149436,"ë£ħ":149437,"ìͼ":149438,"ìķĿ":149439,"ìļ¬":149440,"ìľ±":149441,"ï¥Ĥ":149442,"惡":149443,"יּ":149444,"ïŃģ":149445,"ï³Ī":149446,"ðĿĶħ":149447,"ðĿĺ¤":149448,"ðĿĻı":149449,"ðĿĻĻ":149450,"ðŁķī":149451,"ð٧Ļ":149452,"á¸ij":149453,"ê´¼":149454,"ëģį":149455,"ëĹ´":149456,"ëĿ³":149457,"ë°ŀ":149458,"ë°¢":149459,"ëµĺ":149460,"ìĤĶ":149461,"ìĦĦ":149462,"ì¼ļ":149463,"íĢł":149464,"íĬ±":149465,"íĮĸ":149466,"ï¤ij":149467,"領":149468,"隸":149469,"ï´į":149470,"ðĿĺ·":149471,"Ĭ":149472,"Ŭ":149473,"ÆĢ":149474,"Æĭ":149475,"Æľ":149476,"Çij":149477,"Çĺ":149478,"Çŀ":149479,"Ç¥":149480,"Ç®":149481,"ɰ":149482,"ɶ":149483,"É·":149484,"ɽ":149485,"ÊĪ":149486,"ÊIJ":149487,"Ëİ":149488,"ËŁ":149489,"˦":149490,"˯":149491,"ÏIJ":149492,"Ïĵ":149493,"Ï¢":149494,"Ϥ":149495,"Ϫ":149496,"ÏŃ":149497,"Ï®":149498,"Ï»":149499,"Ñł":149500,"ÑŃ":149501,"Ò¨":149502,"ÓĿ":149503,"Ô¡":149504,"Ô·":149505,"Õī":149506,"Õĵ":149507,"Õĸ":149508,"Õļ":149509,"ÕĿ":149510,"Öİ":149511,"Ø¿":149512,"Úħ":149513,"Úį":149514,"ÚĶ":149515,"ÛĬ":149516,"Û¾":149517,"ÜĻ":149518,"ÝĴ":149519,"Ýĺ":149520,"ßĴ":149521,"ßĸ":149522,"à¤Ĭ":149523,"à¤IJ":149524,"à¦ı":149525,"à¦ĸ":149526,"à§Ł":149527,"મ":149528,"હ":149529,"à®ħ":149530,"à®Ĩ":149531,"à°¡":149532,"à°°":149533,"à²ļ":149534,"ಮ":149535,"ಯ":149536,"à´Ł":149537,"à´·":149538,"ൾ":149539,"à¶ij":149540,"à¶ŀ":149541,"༼":149542,"à½ĵ":149543,"áĢĵ":149544,"áĤ¦":149545,"áĥĸ":149546,"áĥŃ":149547,"áĥ¯":149548,"áħ¨":149549,"áħª":149550,"áĨ°":149551,"áĪģ":149552,"áĪİ":149553,"áĪĵ":149554,"áĪ¥":149555,"áβ":149556,"áĪ´":149557,"áĪ»":149558,"áīł":149559,"áī²":149560,"áī¶":149561,"áĬ£":149562,"áĬ¥":149563,"áĬª":149564,"áĭĺ":149565,"áĭ²":149566,"áĭ¶":149567,"áĮ£":149568,"áį¡":149569,"áį£":149570,"áݬ":149571,"áݾ":149572,"áIJ¡":149573,"áķķ":149574,"áĸ±":149575,"áĹIJ":149576,"áĹŃ":149577,"áĺī":149578,"áļ±":149579,"ἣ":149580,"áŀ¥":149581,"áŁĶ":149582,"áł£":149583,"áłª":149584,"áł°":149585,"áł´":149586,"á¤ĸ":149587,"ᥣ":149588,"á®":149589,"᮳":149590,"á¯":149591,"á¯Ļ":149592,"á°":149593,"á°į":149594,"á´Ĭ":149595,"á´¾":149596,"áµģ":149597,"áµİ":149598,"áµŀ":149599,"ᵤ":149600,"á¶ħ":149601,"á¶ĺ":149602,"á¶Ł":149603,"á¶¢":149604,"ᶤ":149605,"á¶±":149606,"á¶»":149607,"á¸ī":149608,"á¸ŀ":149609,"Ḻ":149610,"á¹ĵ":149611,"á¹Ĺ":149612,"Ṫ":149613,"áºĬ":149614,"áºı":149615,"áºĽ":149616,"á¼ĥ":149617,"á¼Į":149618,"Ἷ":149619,"á½Ĥ":149620,"á½ĵ":149621,"á½Ĺ":149622,"ὦ":149623,"á¾±":149624,"á¾´":149625,"á¿ĺ":149626,"á¿Ł":149627,"Ὸ":149628,"âģĺ":149629,"âĤij":149630,"âĤĽ":149631,"âĤ¿":149632,"âĦĩ":149633,"âĦŀ":149634,"âĦ±":149635,"âĩŁ":149636,"âĩ²":149637,"âΤ":149638,"âζ":149639,"âīĤ":149640,"âī¾":149641,"âĬ¨":149642,"âĬ³":149643,"âĬ·":149644,"âĭĮ":149645,"âĭĺ":149646,"âĮķ":149647,"âĮ¥":149648,"âĮµ":149649,"âĮº":149650,"âį£":149651,"âį²":149652,"âįµ":149653,"âİĩ":149654,"âıĥ":149655,"âıIJ":149656,"âıł":149657,"âı¤":149658,"âı¶":149659,"âı¸":149660,"âı¹":149661,"âijĤ":149662,"âĴ·":149663,"âĴº":149664,"âĵ¡":149665,"âĵ¤":149666,"â;":149667,"âĸĺ":149668,"âĸµ":149669,"âĹª":149670,"âĹ·":149671,"âĺ¨":149672,"âĺ«":149673,"âĺ²":149674,"âĺ³":149675,"âĻĨ":149676,"âļ¤":149677,"âļ¥":149678,"âĽĵ":149679,"⼴":149680,"âĽ¾":149681,"âŀ«":149682,"âŀ¿":149683,"⣷":149684,"â¤ij":149685,"⤫":149686,"⤶":149687,"⤽":149688,"⧪":149689,"â¨Ģ":149690,"⩽":149691,"⬡":149692,"⬢":149693,"⬤":149694,"â²ĸ":149695,"Ⲫ":149696,"âµĢ":149697,"⸮":149698,"⸽":149699,"ãĢł":149700,"ãĢ·":149701,"ãĦĮ":149702,"ãĦĺ":149703,"ãħij":149704,"ãĪİ":149705,"ãĪIJ":149706,"ãĬľ":149707,"ãĮĵ":149708,"ãĮł":149709,"ãİŁ":149710,"ãݤ":149711,"ãݧ":149712,"㬮":149713,"äĪ":149714,"äĪĢ":149715,"ä°":149716,"ä°Ģ":149717,"êħ":149718,"êħī":149719,"êĩĹ":149720,"êĪ":149721,"êĪį":149722,"ê§Ĥ":149723,"ê§Ĭ":149724,"êªĢ":149725,"ê²Ī":149726,"ê²į":149727,"ê³Ģ":149728,"êµł":149729,"ê½IJ":149730,"ê¾Ī":149731,"꿱":149732,"ëĥı":149733,"ëĦij":149734,"ëħ¤":149735,"ëĩ¸":149736,"ëμ":149737,"ëīħ":149738,"ëĬ£":149739,"ëĭº":149740,"ëįŀ":149741,"ëIJĮ":149742,"ëķ¸":149743,"ëĺł":149744,"ëĻĩ":149745,"ëĻĪ":149746,"ëľ½":149747,"ëŀĶ":149748,"ëłľ":149749,"ë£IJ":149750,"ë§Ģ":149751,"ë§Ĭ":149752,"ëªĢ":149753,"ë¬Ń":149754,"믾":149755,"ë³ľ":149756,"ë´Ĭ":149757,"ëµī":149758,"ë·ľ":149759,"ë¸Ģ":149760,"ë¹ĭ":149761,"ìģĦ":149762,"ìĤ£":149763,"ìĤ»":149764,"ìĦµ":149765,"ìħĴ":149766,"ìīĪ":149767,"ìīĶ":149768,"ìĬĮ":149769,"ìĬĻ":149770,"ìIJ´":149771,"ìĵº":149772,"ìķļ":149773,"ìķº":149774,"ìĸľ":149775,"ìĹª":149776,"ìĺľ":149777,"ìϤ":149778,"ìļĽ":149779,"ìļº":149780,"ìĿħ":149781,"ìĿı":149782,"ìĿŃ":149783,"ìĿ¶":149784,"ìłĽ":149785,"ì¡Ī":149786,"ì¢ī":149787,"ì¢Ķ":149788,"ì©ł":149789,"ìŃĮ":149790,"쯩":149791,"ì´£":149792,"ì¸ķ":149793,"ì¹Ł":149794,"쾡":149795,"ì¿Ļ":149796,"íģĩ":149797,"íģī":149798,"íĩĢ":149799,"íζ":149800,"íĸij":149801,"íĸ¤":149802,"íĹħ":149803,"íľı":149804,"íĿĿ":149805,"ï¤Ĵ":149806,"ï¤ķ":149807,"郎":149808,"ï¥ħ":149809,"ï¥ĩ":149810,"ï¥ı":149811,"ï¥ļ":149812,"ï¥Ł":149813,"ï¦Ħ":149814,"ï¦Ī":149815,"令":149816,"囹":149817,"零":149818,"ï§ģ":149819,"ï§ĥ":149820,"ï§Ķ":149821,"ï§ł":149822,"ï§£":149823,"ï§®":149824,"ïŃIJ":149825,"ïŃĸ":149826,"ïѦ":149827,"ïŃ´":149828,"ïѵ":149829,"ïѶ":149830,"ïѸ":149831,"ï®Į":149832,"ï®İ":149833,"ï®ŀ":149834,"ï®Ł":149835,"ﮡ":149836,"ﮪ":149837,"ï¯Ķ":149838,"ï¯Ĺ":149839,"ï¯ļ":149840,"ï¯Ľ":149841,"ï¯Ŀ":149842,"ï¯Ł":149843,"ﯧ":149844,"ﯨ":149845,"ﯫ":149846,"ﯯ":149847,"ﯰ":149848,"ﯱ":149849,"ﯲ":149850,"ﯳ":149851,"ﯴ":149852,"ﯵ":149853,"ﯶ":149854,"ï°Ģ":149855,"ï±ħ":149856,"ï±Ķ":149857,"ï±´":149858,"ï²ģ":149859,"ï³ķ":149860,"ï·½":149861,"ï¸ķ":149862,"︱":149863,"ï¹£":149864,"ï¹½":149865,"ï»į":149866,"ï¾±":149867,"ðĿIJĻ":149868,"ðĿIJ½":149869,"ðĿij¤":149870,"ðĿij®":149871,"ðĿijµ":149872,"ðĿĴĥ":149873,"ðĿĴĦ":149874,"ðĿĵŃ":149875,"ðĿĵ·":149876,"ðĿĶĸ":149877,"ðĿĶŀ":149878,"ðĿĶ¢":149879,"ðĿͦ":149880,"ðĿͬ":149881,"ðĿķĦ":149882,"ðĿķĬ":149883,"ðĿķİ":149884,"ðĿķĻ":149885,"ðĿķľ":149886,"ðĿķŃ":149887,"ðĿķ³":149888,"ðĿķ¸":149889,"ðĿķ¾":149890,"ðĿĸī":149891,"ðĿĸı":149892,"ðĿĺĩ":149893,"ðĿĺī":149894,"ðĿĺĸ":149895,"ðĿĺĽ":149896,"ðĿĺŀ":149897,"ðĿĺ«":149898,"ðĿĺ¾":149899,"ðĿĻĩ":149900,"ðĿĻī":149901,"ðĿĻĭ":149902,"ðĿĻİ":149903,"ðĿĻĺ":149904,"ðĿĻ¥":149905,"ðĿļĥ":149906,"ðĿļIJ":149907,"ðĿļĶ":149908,"ðĿľĥ":149909,"ðŁĦ·":149910,"ðŁħĿ":149911,"ðŁħ¾":149912,"ðŁĨĤ":149913,"ðŁĨĵ":149914,"ðŁĮĤ":149915,"ðŁĮĨ":149916,"ðŁĮī":149917,"ðŁĮij":149918,"ðŁĮĺ":149919,"ðŁĮ©":149920,"ðŁĮ«":149921,"ðŁį¢":149922,"ðŁį¥":149923,"ðŁİĽ":149924,"ðŁİ¢":149925,"ðŁİ´":149926,"ðŁij¡":149927,"ðŁĴ¾":149928,"ðŁĵŃ":149929,"ðŁĶĪ":149930,"ðŁĶ¦":149931,"ðŁĶ²":149932,"ðŁĶ³":149933,"ðŁķĵ":149934,"ðŁķķ":149935,"ðŁķĺ":149936,"ðŁķŁ":149937,"ðŁķ·":149938,"ðŁĹ³":149939,"ðŁļĦ":149940,"ðŁļĶ":149941,"ðŁļĸ":149942,"ðŁĽIJ":149943,"ðŁĽ¤":149944,"ðŁĽ¸":149945,"ðŁł":149946,"ðŁł³":149947,"ðŁ¤¹":149948,"ðŁ¥ĥ":149949,"ðŁ¥¨":149950,"ðŁ¥ª":149951,"ðŁ¥¾":149952,"ð٦ĥ":149953,"ð٦Ĵ":149954,"ð٦Ļ":149955,"ðŁ¦¶":149956,"ðŁ§ł":149957,"ðŁ§ª":149958,"ð٧Ń":149959,"ðŁ§²":149960,"ð£·":149961,"ð£·Ń":149962,"ð¦ĺ":149963,"ð¦ĺĴ":149964,"Æij":149965,"ÇĻ":149966,"È®":149967,"Øł":149968,"ÚĦ":149969,"ÜĢ":149970,"ߢ":149971,"áīĢ":149972,"áĬIJ":149973,"áİł":149974,"áºŀ":149975,"ëĪŀ":149976,"ëķŁ":149977,"ë£ģ":149978,"ë¤Ĺ":149979,"ìĦ¥":149980,"ìħij":149981,"ìĸIJ":149982,"ìĽĽ":149983,"ì£ķ":149984,"íİı":149985,"íĽĵ":149986,"梁":149987,"ï³Ľ":149988,"ï´«":149989,"ðĸ§":149990,"ðĸ§·":149991,"ðĿķģ":149992,"ðŁIJª":149993,"ðŁĴĪ":149994,"ðŁĵł":149995,"ðŁķĽ":149996,"ðŁķ´":149997,"ÑĿ":149998,"ÓĬ":149999,"ॲ":150000,"પ":150001,"áĥ¤":150002,"áįIJ":150003,"á¶°":150004,"á¼Ŀ":150005,"Ὡ":150006,"âĭĭ":150007,"âĴ½":150008,"âϾ":150009,"â½Ķ":150010,"⾯":150011,"ãĦĴ":150012,"ãħļ":150013,"ëIJį":150014,"ë·ģ":150015,"ìĭĢ":150016,"ìļĿ":150017,"쥰":150018,"캴":150019,"íĭī":150020,"íĿ½":150021,"ï¦Ģ":150022,"樂":150023,"ï§ħ":150024,"ï§ĵ":150025,"ïѝ":150026,"ï®Ĩ":150027,"ðIJ¤ķ":150028,"ðĿIJŁ":150029,"ðĿĴħ":150030,"ðĿĵľ":150031,"ðĿͰ":150032,"ðĿĶ»":150033,"ðĿĺį":150034,"ðĿϝ":150035,"ðŁĦ½":150036,"ðŁħĤ":150037,"ðŁħĶ":150038,"ðŁħ½":150039,"ðŁĵ´":150040,"ð٧ĸ":150041,"ÓĴ":150042,"Ḳ":150043,"ëī¼":150044,"Çı":150045,"Èĵ":150046,"ʸ":150047,"ÕĤ":150048,"Ûħ":150049,"ß¡":150050,"ߣ":150051,"ய":150052,"à°Ī":150053,"ಸ":150054,"ຮ":150055,"à¼ķ":150056,"áĢİ":150057,"áĨ¡":150058,"áIJĭ":150059,"áIJķ":150060,"áij¯":150061,"áŀĨ":150062,"á¨ķ":150063,"á©Ī":150064,"âģħ":150065,"âĨļ":150066,"âĶİ":150067,"âł©":150068,"â²Ĥ":150069,"â²Ķ":150070,"Ⲩ":150071,"ãĬļ":150072,"íĵ²":150073,"ðĿijĪ":150074,"ðĿij¬":150075,"ðĿij¹":150076,"ðĿĴ¾":150077,"ðĿĵ±":150078,"ðĿĵ½":150079,"ðĿķ¯":150080,"ðĿķ»":150081,"ðĿĺ½":150082,"ðĿļĨ":150083,"ðŁĦ°":150084,"ðŁIJ¨":150085,"Òķ":150086,"à²ħ":150087,"ï¨Ĩ":150088,"ðĿij°":150089,"ðŁĦ¸":150090,"Ôİ":150091,"Øį":150092,"Ùµ":150093,"ಶ":150094,"áĢĪ":150095,"áĺĹ":150096,"᳸":150097,"á¡¡":150098,"ᨲ":150099,"á©ģ":150100,"á´·":150101,"áµ§":150102,"âķ¨":150103,"âļģ":150104,"â¾Ŀ":150105,"ã̼":150106,"ãĦı":150107,"êĴ«":150108,"ꦥ":150109,"ꦩ":150110,"ꦲ":150111,"ìĺ¼":150112,"íĵIJ":150113,"ðĵĩ":150114,"ðĵĩ¼":150115,"ðĿķ¿":150116,"ðŁĽ´":150117,"먾":150118,"ವ":150119,"à´İ":150120,"à¼Ģ":150121,"âĩĸ":150122,"ãĪ«":150123,"âĵĢ":150124,"áħ´":150125,"áļ¾":150126,"áĽŀ":150127,"Ἣ":150128,"ᥴ":150129,"âĨĽ":150130,"âĨ¶":150131,"âĩ¤":150132,"âķŁ":150133,"âĺ·":150134,"âļIJ":150135,"ðŁ§´":150136,"á¹³":150137,"âĶį":150138,"âĶĴ":150139,"âĶ©":150140,"âͦ":150141,"â¾µ":150142,"àªľ":150143,"ત":150144,"âĩĻ":150145,"âͱ":150146,"âķĢ":150147,"â½Ĭ":150148,"ï½Ł":150149,"ଡ":150150,"ðł®":150151,"ðł®·":150152,"âķĥ":150153,"â°Ķ":150154,"ãĬ¦":150155,"ðŁİIJ":150156,"ãĩ°":150157,"â¼Ŀ":150158,"â¾Ķ":150159,"â½Ĵ":150160,"âłĴ":150161,"都":150162,"ï©Ĵ":150163,"免":150164,"ï©ĸ":150165,"ðĵı¸":150166,"ãĮĥ":150167,"ðĸ¤":150168,"ðĸ¤IJ":150169,"ï¦Ń":150170,"âĬħ":150171,"â¾³":150172,"ä´¥":150173,"ï©ķ":150174,"ðŁĮĶ":150175,"áŀĭ":150176,"âļį":150177,"â¼ĭ":150178,"ãİĺ":150179,"ðIJĮ²":150180,"É©":150181,"áİij":150182,"âĨ®":150183,"âĩĥ":150184,"âļİ":150185,"ãĩ±":150186,"ãĭ©":150187,"ãĮ¶":150188,"êĻª":150189,"ëݬ":150190,"ï¨IJ":150191,"ï¨Ľ":150192,"ï©Ĭ":150193,"ï©į":150194,"ðĵħ":150195,"ðĵħº":150196,"Ï¡":150197,"Èij":150198,"ÉĤ":150199,"Ôĵ":150200,"ßİ":150201,"à´§":150202,"áĢī":150203,"áĢĭ":150204,"áĢij":150205,"áĢł":150206,"áļĻ":150207,"á¨Ħ":150208,"ᨩ":150209,"ᨹ":150210,"á©ĵ":150211,"ᬾ":150212,"á´Ļ":150213,"áµij":150214,"âĤŃ":150215,"âĨ°":150216,"âľģ":150217,"â½IJ":150218,"ãĭ¯":150219,"ãĮ½":150220,"íĨ¢":150221,"錄":150222,"ðŁĤ":150223,"ðŁĤ»":150224,"ÈĴ":150225,"ͺ":150226,"Ô¥":150227,"Õij":150228,"Ú¶":150229,"à§İ":150230,"à¶®":150231,"àºĸ":150232,"àºľ":150233,"ຽ":150234,"áĥ»":150235,"áħ¯":150236,"áĭŀ":150237,"áĸķ":150238,"á´Ī":150239,"á¶Ĩ":150240,"Ḿ":150241,"á¹¼":150242,"Ῠ":150243,"âĦĭ":150244,"âĦŃ":150245,"âα":150246,"âĮĵ":150247,"âĶĩ":150248,"âĶ¢":150249,"â±®":150250,"â²Ħ":150251,"ãĩ¾":150252,"ãά":150253,"븡":150254,"ìIJī":150255,"íĻĽ":150256,"ðĿķª":150257,"ƹ":150258,"Ͳ":150259,"Óģ":150260,"Û¼":150261,"ফ":150262,"áħŁ":150263,"áīĨ":150264,"áįĪ":150265,"áºĸ":150266,"á½ī":150267,"â͏":150268,"⽩":150269,"êľ":150270,"êľ¥":150271,"êµħ":150272,"ëĤĶ":150273,"ëĦł":150274,"ëĩĹ":150275,"ëĻĿ":150276,"ìļ¯":150277,"ìļ·":150278,"ìŁĽ":150279,"ì·IJ":150280,"íŁ¬":150281,"íŁ®":150282,"íŁ°":150283,"ï¦Ĩ":150284,"鈴":150285,"ï²ŀ":150286,"ﳤ":150287,"ï³¥":150288,"ðIJĮ¸":150289,"ðĿĶı":150290,"ðĿķ®":150291,"ðĿĺ£":150292,"à¦Ī":150293,"âıı":150294,"ãĦĸ":150295,"ê²ĩ":150296,"ëĸĺ":150297,"ëľ·":150298,"ëŀĴ":150299,"ë¡ĵ":150300,"ë¢ī":150301,"ë£ĥ":150302,"ë§ĭ":150303,"ë²ĭ":150304,"ìĤ·":150305,"ìĪķ":150306,"ìĮ¨":150307,"ìĵ»":150308,"ìĸĬ":150309,"ìϬ":150310,"ìĿ»":150311,"ì¦ģ":150312,"쵤":150313,"ì·ĥ":150314,"íĢľ":150315,"íħī":150316,"íįł":150317,"íıħ":150318,"íij±":150319,"íķķ":150320,"íĸł":150321,"íĿķ":150322,"ÆĻ":150323,"Æļ":150324,"Æŀ":150325,"Çĥ":150326,"ÇĬ":150327,"Çľ":150328,"Ǥ":150329,"ÇŃ":150330,"ǹ":150331,"ÈĢ":150332,"Èģ":150333,"Èħ":150334,"Èī":150335,"ÈĹ":150336,"ÈŁ":150337,"Ȥ":150338,"È¥":150339,"Ȩ":150340,"ȵ":150341,"Ⱥ":150342,"È»":150343,"ÉĮ":150344,"É®":150345,"Êħ":150346,"Ê¥":150347,"ʨ":150348,"Ëĵ":150349,"ËĶ":150350,"Ëł":150351,"Ë£":150352,"˸":150353,"Í´":150354,"ÏĹ":150355,"Ïĺ":150356,"ÏĻ":150357,"Ïļ":150358,"ÏĿ":150359,"Ϩ":150360,"Ϭ":150361,"Ͼ":150362,"Ï¿":150363,"Ѫ":150364,"ÒĢ":150365,"Òľ":150366,"Ò¼":150367,"Ò½":150368,"ÓĤ":150369,"Óħ":150370,"Óĩ":150371,"Óį":150372,"Óĸ":150373,"ÓŁ":150374,"Ó«":150375,"Ó±":150376,"ÔĨ":150377,"Ôĩ":150378,"Ôº":150379,"Õĭ":150380,"Öī":150381,"ØĪ":150382,"ØĬ":150383,"ؽ":150384,"ؾ":150385,"Ù·":150386,"ÚĤ":150387,"ÚĬ":150388,"Úĸ":150389,"ÚĹ":150390,"Ú£":150391,"Ú«":150392,"Ú¸":150393,"ÛĢ":150394,"Ûį":150395,"Û½":150396,"Üī":150397,"ܤ":150398,"ݧ":150399,"Ý´":150400,"Þĥ":150401,"Þ¤":150402,"Þ¥":150403,"ßļ":150404,"߼":150405,"ߤ":150406,"àłį":150407,"àłĵ":150408,"àł³":150409,"à¡¢":150410,"à¥ł":150411,"à§ł":150412,"৺":150413,"à¨Ĭ":150414,"à¨IJ":150415,"ਮ":150416,"ਯ":150417,"ਰ":150418,"ਸ":150419,"àªĨ":150420,"ળ":150421,"વ":150422,"ઽ":150423,"à¬Į":150424,"à¬ĺ":150425,"ଽ":150426,"à®ĥ":150427,"ஸ":150428,"à°Ĩ":150429,"à°ķ":150430,"à°¦":150431,"à²Ĩ":150432,"à²Ĭ":150433,"à²Į":150434,"à²IJ":150435,"à²Ľ":150436,"ತ":150437,"ದ":150438,"ಪ":150439,"ಲ":150440,"ಹ":150441,"à´Ĩ":150442,"à´ı":150443,"à´Ĺ":150444,"à´«":150445,"à´¹":150446,"ൺ":150447,"ൽ":150448,"à¶ħ":150449,"à¶Ĭ":150450,"à¶Ķ":150451,"à¶§":150452,"à¶«":150453,"à¶°":150454,"à¼Ħ":150455,"à¼ħ":150456,"à¼Ĭ":150457,"à½Ļ":150458,"ཡ":150459,"ཧ":150460,"à¿Ģ":150461,"à¿Ļ":150462,"áĢĿ":150463,"á̧":150464,"áĢ©":150465,"áĢ¿":150466,"áģµ":150467,"áĤģ":150468,"áĤ½":150469,"áĥĤ":150470,"áĥª":150471,"áĦĬ":150472,"áĦ¢":150473,"áħ¦":150474,"áħŃ":150475,"áĨ®":150476,"áĨ±":150477,"áĨ»":150478,"áĩ":150479,"áĩĤ":150480,"áĪħ":150481,"áĪī":150482,"áĪĮ":150483,"áĪIJ":150484,"áĪĴ":150485,"áĪĻ":150486,"áĪļ":150487,"áĪľ":150488,"áĪŀ":150489,"áĪ©":150490,"áγ":150491,"áĪº":150492,"áν":150493,"áīħ":150494,"áī¢":150495,"áī±":150496,"áī´":150497,"áĬĥ":150498,"áĬį":150499,"áĬĸ":150500,"áĬ®":150501,"áĬ¸":150502,"áĭĽ":150503,"áĭĿ":150504,"áĭ³":150505,"áĮģ":150506,"áĮħ":150507,"áĮ¥":150508,"áĮ¦":150509,"áĮ¨":150510,"áįĬ":150511,"áįį":150512,"áįķ":150513,"áįĸ":150514,"áį¢":150515,"áį¤":150516,"áİĴ":150517,"áݪ":150518,"áıģ":150519,"áıIJ":150520,"áıŁ":150521,"áIJĤ":150522,"áIJĸ":150523,"áIJĿ":150524,"áIJŀ":150525,"áIJŁ":150526,"áIJł":150527,"áijĸ":150528,"áĴĭ":150529,"áĴį":150530,"áĴ¡":150531,"áĵ«":150532,"áĶķ":150533,"áķĭ":150534,"áķij":150535,"áķĻ":150536,"áķļ":150537,"áķĽ":150538,"áķ¤":150539,"áķ¦":150540,"áķ®":150541,"áķ¼":150542,"áĸĵ":150543,"áĹĹ":150544,"áĹ¢":150545,"áĹ¯":150546,"áĹ·":150547,"áĺĦ":150548,"áĺij":150549,"áĽĤ":150550,"áĽĻ":150551,"áŀį":150552,"áłĨ":150553,"áł¡":150554,"᳦":150555,"áł®":150556,"áł¯":150557,"áł²":150558,"áł·":150559,"á¡į":150560,"á¡ŀ":150561,"ᡤ":150562,"á¡´":150563,"ᡵ":150564,"á¤ĵ":150565,"á¥ĸ":150566,"ᥰ":150567,"ᨦ":150568,"ᨧ":150569,"ᨨ":150570,"ᨪ":150571,"ᨬ":150572,"ᨯ":150573,"ᨳ":150574,"ᨵ":150575,"á©ĥ":150576,"á¬ķ":150577,"áŃ£":150578,"á±":150579,"á±ļ":150580,"á²ł":150581,"á´ĵ":150582,"á´¶":150583,"áµĤ":150584,"áµĮ":150585,"áµ¥":150586,"áµ´":150587,"á¶ĩ":150588,"á¸Ī":150589,"ḳ":150590,"ḧ":150591,"Ḵ":150592,"Ḿ":150593,"á¹Ģ":150594,"á¹ĸ":150595,"á¹Ł":150596,"á¹ł":150597,"ṫ":150598,"á¹±":150599,"á¹·":150600,"ṿ":150601,"áºĦ":150602,"áºį":150603,"áºij":150604,"áºĹ":150605,"á¼ī":150606,"á¼ĵ":150607,"á¼Ń":150608,"á½ĭ":150609,"á½Ĵ":150610,"á½ł":150611,"á½£":150612,"á¾Ħ":150613,"á¾ı":150614,"á¾ij":150615,"á¾Ĺ":150616,"ᾦ":150617,"á¾§":150618,"á¾¾":150619,"á¿Ħ":150620,"á¿ĵ":150621,"á¿¡":150622,"Ῥ":150623,"âģļ":150624,"âĤĮ":150625,"âĦģ":150626,"âĦĶ":150627,"âĦ£":150628,"âĦ§":150629,"âĦ¯":150630,"âĦ°":150631,"âĦ´":150632,"âħħ":150633,"âĨľ":150634,"âĨ«":150635,"âĨŃ":150636,"âĨ±":150637,"âĨ¹":150638,"âĨ½":150639,"âĩĩ":150640,"âĩľ":150641,"âĩµ":150642,"âĪī":150643,"âĪĬ":150644,"âĪĸ":150645,"âĪľ":150646,"âξ":150647,"âīĢ":150648,"âīĭ":150649,"âīĮ":150650,"âīĵ":150651,"âīľ":150652,"âī´":150653,"âī¿":150654,"âĬĬ":150655,"âĬĭ":150656,"âĬĶ":150657,"âĬĸ":150658,"âĬ£":150659,"âĬ¦":150660,"âĭİ":150661,"âĭª":150662,"âĭ²":150663,"âĮ¦":150664,"âĮ§":150665,"âįº":150666,"âİĪ":150667,"âݨ":150668,"âݬ":150669,"âݳ":150670,"âݼ":150671,"âݾ":150672,"âıĮ":150673,"âıļ":150674,"âı«":150675,"âı¯":150676,"âıµ":150677,"âĴľ":150678,"âĴĿ":150679,"âĴ«":150680,"âĵĦ":150681,"âĵĬ":150682,"âĵĻ":150683,"âĵ©":150684,"âĶij":150685,"âĶĻ":150686,"âĶļ":150687,"âĶ¥":150688,"âķħ":150689,"âķī":150690,"âķį":150691,"âķı":150692,"âķŀ":150693,"âĸļ":150694,"âĸ¯":150695,"âĹĥ":150696,"âĹļ":150697,"âŬ":150698,"âĹ´":150699,"âĺĪ":150700,"âĺ¤":150701,"âĺ¥":150702,"âĺ§":150703,"âĺ¬":150704,"âĻģ":150705,"âϱ":150706,"âļĥ":150707,"âļĦ":150708,"âļħ":150709,"âļı":150710,"âļļ":150711,"âļŀ":150712,"âļŁ":150713,"âļ±":150714,"âļ²":150715,"âľĢ":150716,"⾣":150717,"âľ¢":150718,"âĿµ":150719,"âŁ¡":150720,"⣦":150721,"⣧":150722,"âŁ³":150723,"âŁ¾":150724,"âŁ¿":150725,"âłĩ":150726,"â¤Ħ":150727,"⤺":150728,"â¥Ĥ":150729,"⥹":150730,"â§ī":150731,"â§¼":150732,"â§½":150733,"â¨į":150734,"â¬Ĭ":150735,"⬣":150736,"âŃŀ":150737,"â®ŀ":150738,"⮳":150739,"â¯Ī":150740,"â¯ij":150741,"ⱳ":150742,"â±±":150743,"â²Ń":150744,"â´¹":150745,"âµķ":150746,"⸾":150747,"⺫":150748,"â¼Ĩ":150749,"â¼ł":150750,"â½Ł":150751,"â½¼":150752,"â¾Ľ":150753,"â¾§":150754,"â¿ĥ":150755,"â¿»":150756,"ãĤķ":150757,"ãĤŁ":150758,"ãĦĽ":150759,"ãĦ¡":150760,"ãĦ¶":150761,"ãĦº":150762,"ãħĴ":150763,"ãħŁ":150764,"ãĨĢ":150765,"ãĩ»":150766,"ãĪij":150767,"ãĪŃ":150768,"ãĪ®":150769,"ãγ":150770,"ãι":150771,"ãī¥":150772,"ãī¦":150773,"ãī¹":150774,"ãī¿":150775,"ãĬŀ":150776,"ãĬ¨":150777,"ãĭij":150778,"ãĭ¥":150779,"ãĭ´":150780,"ãĭº":150781,"ãİĦ":150782,"ãİķ":150783,"ãݯ":150784,"ãıĤ":150785,"ãıĪ":150786,"ãıĵ":150787,"ãıĸ":150788,"ãı±":150789,"ãIJ±":150790,"ãŁģ":150791,"ã¢":150792,"㢨":150793,"ã¨":150794,"㨳":150795,"㫪":150796,"ã«´":150797,"ã¶³":150798,"㺾":150799,"äĢ":150800,"äĢĢ":150801,"äĭ":150802,"äĭĮ":150803,"äĮĢ":150804,"äIJĢ":150805,"äłĢ":150806,"äł":150807,"äł¼":150808,"ä§":150809,"ä§ŀ":150810,"䨰":150811,"䨺":150812,"ä´Ģ":150813,"ä·":150814,"ä·ħ":150815,"ä·¸":150816,"êĤ":150817,"êĤ«":150818,"êĮ":150819,"êĮ¼":150820,"êį":150821,"êį²":150822,"êĴµ":150823,"êĵ":150824,"êĵ½":150825,"êĻŃ":150826,"êĿĽ":150827,"êĿ¥":150828,"êŀ":150829,"êŀĬ":150830,"ê¦Ĩ":150831,"ê¦ĩ":150832,"ê¦Ł":150833,"ꦨ":150834,"ê§Ī":150835,"ê©":150836,"ê©Ł":150837,"êªĭ":150838,"êªij":150839,"êªķ":150840,"êªĹ":150841,"êªľ":150842,"ꪮ":150843,"ꪱ":150844,"ꪻ":150845,"ꪼ":150846,"ê«Ģ":150847,"ê«Ŀ":150848,"ê°ĥ":150849,"ê°ĺ":150850,"ê±ľ":150851,"ê²ĵ":150852,"ê²ļ":150853,"ê³Ļ":150854,"ê³¾":150855,"ê´Ĺ":150856,"ê´Ļ":150857,"êµĽ":150858,"ê¶ĥ":150859,"ê¶ķ":150860,"궨":150861,"긩":150862,"긿":150863,"ê¹Ħ":150864,"ê¹Ĩ":150865,"ê¹ī":150866,"ê¹ĵ":150867,"ê¹¢":150868,"ê¹£":150869,"깸":150870,"꺳":150871,"ê¿ı":150872,"ê¿ķ":150873,"ê¿§":150874,"ëĢ©":150875,"ëģħ":150876,"ëĥµ":150877,"ëĦĸ":150878,"ëĦĹ":150879,"ëĦ¢":150880,"ëħĤ":150881,"ëĨIJ":150882,"ëĩľ":150883,"ëĪĭ":150884,"ëĪļ":150885,"ëīį":150886,"ëī¨":150887,"ëĬļ":150888,"ëĬ¡":150889,"ëĭľ":150890,"ëĭª":150891,"ëĮĺ":150892,"ëĮ¤":150893,"ëĮ¸":150894,"ëİŁ":150895,"ëı¨":150896,"ëIJĦ":150897,"ëIJı":150898,"ëIJ´":150899,"ëIJ¸":150900,"ëijģ":150901,"ëij¿":150902,"ëĴ¨":150903,"ëĵ·":150904,"ëĶ®":150905,"ëͲ":150906,"ëķ§":150907,"ëĸĶ":150908,"ëĸª":150909,"ëĺŃ":150910,"ëļĢ":150911,"ëļł":150912,"ëĽĶ":150913,"뼩":150914,"ëľħ":150915,"ëŀķ":150916,"ëŀ°":150917,"ëŁIJ":150918,"ëł¡":150919,"ë¡ŀ":150920,"ë¡£":150921,"롵":150922,"ë£Ħ":150923,"ë£į":150924,"뤳":150925,"ë¦į":150926,"ë¦ı":150927,"릳":150928,"ë§Ħ":150929,"ë§Ĩ":150930,"ë§į":150931,"ë§ľ":150932,"ë§«":150933,"ë§»":150934,"먮":150935,"ë©Ĥ":150936,"ë©Ń":150937,"몴":150938,"묾":150939,"묳":150940,"묫":150941,"묾":150942,"ëѬ":150943,"ë®ĺ":150944,"뮹":150945,"ë¯ķ":150946,"ë¯ľ":150947,"ë°¨":150948,"ë°ª":150949,"ë±Ķ":150950,"ë²ĺ":150951,"ë²Ľ":150952,"ë²±":150953,"ë²´":150954,"ë´½":150955,"뵤":150956,"뵨":150957,"ë·Ĺ":150958,"ë·ĺ":150959,"ë¸ĵ":150960,"븾":150961,"빪":150962,"ëºĥ":150963,"ëºĺ":150964,"뺵":150965,"ë»´":150966,"ë¼IJ":150967,"ë¾Ķ":150968,"ìģŃ":150969,"ìĤł":150970,"ìĤ®":150971,"ìĥı":150972,"ìĥĻ":150973,"ìĦº":150974,"ìħ¢":150975,"ìĨĢ":150976,"ìĨħ":150977,"ìĨ¤":150978,"ìĨ¦":150979,"ìĨ¬":150980,"ìĩ±":150981,"ìε":150982,"ìĭ¨":150983,"ìĭ´":150984,"ìĮ°":150985,"ìįľ":150986,"ìİĹ":150987,"ìİĺ":150988,"ìݼ":150989,"ìijī":150990,"ìijĿ":150991,"ìij»":150992,"ìĴĶ":150993,"ìĴ¯":150994,"ìĵ©":150995,"ìķIJ":150996,"ìķĸ":150997,"ìĸł":150998,"ìĸ¾":150999,"ìĹĥ":151000,"ìĹĹ":151001,"ìĹľ":151002,"ìŨ":151003,"ìĺĤ":151004,"ìĺĦ":151005,"ìĺı":151006,"ìĺ¾":151007,"ìĺ¿":151008,"ìľ§":151009,"ìĿIJ":151010,"ìĿĸ":151011,"ìĿ·":151012,"ìŀį":151013,"ìŀı":151014,"ìŀ¨":151015,"ìŀª":151016,"ìŀ³":151017,"ìł¡":151018,"ìł´":151019,"ìł¹":151020,"ì¡Ģ":151021,"졪":151022,"졵":151023,"ì¢IJ":151024,"좨":151025,"ì£Į":151026,"ì£Ļ":151027,"죳":151028,"ì¦ij":151029,"ì§¥":151030,"ì§´":151031,"ì§¾":151032,"ì¨ĵ":151033,"ì¨ķ":151034,"ì©°":151035,"ì©»":151036,"쩼":151037,"ìªĹ":151038,"ì¬Ķ":151039,"ì¬ĺ":151040,"ì®®":151041,"ì¯ķ":151042,"ì¯ĺ":151043,"ì°İ":151044,"ì°¯":151045,"ì±ĥ":151046,"ì±µ":151047,"ì²§":151048,"ì²®":151049,"첯":151050,"쳬":151051,"ì´ĭ":151052,"ì´¢":151053,"ìµ¥":151054,"ì¶£":151055,"ì¸Ī":151056,"ì¸Ļ":151057,"캤":151058,"ìºŃ":151059,"컽":151060,"ì¼Ļ":151061,"콬":151062,"ì¾Ģ":151063,"ì¿ħ":151064,"쿽":151065,"íĢħ":151066,"íģ¦":151067,"íĤħ":151068,"íĥ¶":151069,"íĥ¹":151070,"íĦĶ":151071,"íħ£":151072,"íĨĦ":151073,"íĨ§":151074,"íĨ¹":151075,"íĩ¼":151076,"íī¤":151077,"íĬ½":151078,"íĭĤ":151079,"íĭij":151080,"íįĪ":151081,"íįĻ":151082,"íį¿":151083,"íݶ":151084,"íIJĿ":151085,"íĴľ":151086,"íĵĿ":151087,"íĵª":151088,"íĵ±":151089,"íĵ·":151090,"íĵ¼":151091,"íĶĻ":151092,"íĶł":151093,"íķļ":151094,"íķĽ":151095,"íķŀ":151096,"íķŁ":151097,"íķ§":151098,"íķ¶":151099,"íĸĬ":151100,"íĸĭ":151101,"íĸį":151102,"íĸĶ":151103,"íĸĺ":151104,"íĸ¡":151105,"íĸ¬":151106,"íĹ£":151107,"íĹ¿":151108,"íĺĸ":151109,"íĺŃ":151110,"íļ°":151111,"íĽį":151112,"íĽ½":151113,"íĿŁ":151114,"íĿŃ":151115,"íĿ´":151116,"íŀľ":151117,"ï¤ī":151118,"ï¤Ń":151119,"爐":151120,"蘆":151121,"祿":151122,"ï¥Ģ":151123,"ï¥ij":151124,"ï¥Ĵ":151125,"ï¥ķ":151126,"ï¥ĺ":151127,"ï¥Ļ":151128,"參":151129,"塞":151130,"殺":151131,"勵":151132,"ï¦ĭ":151133,"ï¦ı":151134,"ï¦Ķ":151135,"ï¦ĸ":151136,"ï¦ĺ":151137,"ï¦Ľ":151138,"ï¦ł":151139,"瑩":151140,"羚":151141,"了":151142,"僚":151143,"料":151144,"ï§Ĩ":151145,"ï§ĸ":151146,"ï§Ľ":151147,"ï§ŀ":151148,"ï§Ł":151149,"ï§§":151150,"ï§³":151151,"狀":151152,"ï§½":151153,"ï¨ĥ":151154,"ï¨ļ":151155,"諸":151156,"ï©Ł":151157,"ﬤ":151158,"שּׁ":151159,"לּ":151160,"ïŃĴ":151161,"ïŃķ":151162,"ïŃĽ":151163,"ïŃĿ":151164,"ïŃŀ":151165,"ïŃŁ":151166,"ïѤ":151167,"ïѧ":151168,"ïѨ":151169,"ïŃ®":151170,"ïѰ":151171,"ïѱ":151172,"ïŃ·":151173,"ïѹ":151174,"ïŃ»":151175,"ï®Ģ":151176,"ï®ĥ":151177,"ï®Ħ":151178,"ï®ħ":151179,"ï®į":151180,"ï®Ĵ":151181,"ï®ĵ":151182,"ï®ķ":151183,"ﮦ":151184,"ï®®":151185,"ï®°":151186,"ï¯ĵ":151187,"ï¯ľ":151188,"ﯩ":151189,"ﯪ":151190,"ﯬ":151191,"ï¯Ń":151192,"ﯮ":151193,"ﯷ":151194,"ﯹ":151195,"ﯻ":151196,"ﯼ":151197,"ï°ĥ":151198,"ï°Į":151199,"ï°IJ":151200,"ï°ĺ":151201,"ï°Ļ":151202,"ï°ľ":151203,"ï°ŀ":151204,"ï°¢":151205,"ï°®":151206,"ï°°":151207,"ï°¼":151208,"ï°¿":151209,"ï±Ģ":151210,"ï±ģ":151211,"ï±Ī":151212,"ï±ĭ":151213,"ï±ı":151214,"ï±Ń":151215,"ï²Ģ":151216,"ï²ĩ":151217,"ï²Ī":151218,"ï²ĭ":151219,"ï²İ":151220,"ï²Ĵ":151221,"ï²ľ":151222,"ï²ł":151223,"ﲬ":151224,"ï²»":151225,"ï³ĩ":151226,"ï³Ķ":151227,"ï³£":151228,"ﳫ":151229,"ï´ĺ":151230,"ï´°":151231,"ï´½":151232,"ï¶":151233,"ï¶°":151234,"ï¸ĸ":151235,"︴":151236,"︹":151237,"ï¹į":151238,"ï¹Ĺ":151239,"ï¹¢":151240,"﹤":151241,"﹩":151242,"ï¹±":151243,"ï¾°":151244,"ï¿Ĥ":151245,"ï¿®":151246,"ðIJĮ°":151247,"ðIJĮ¹":151248,"ðIJĮº":151249,"ðIJĮ½":151250,"ðIJįĤ":151251,"ðIJįĥ":151252,"ðIJįĦ":151253,"ðIJİ":151254,"ðIJݹ":151255,"ðIJ¤Ĥ":151256,"ðIJ¤į":151257,"ðIJ¤ı":151258,"ðIJ¤ĵ":151259,"ðIJŃī":151260,"ðIJŃį":151261,"ðIJ°ĩ":151262,"ðIJ°°":151263,"ðijĤ":151264,"ðijĤĦ":151265,"ðijĺ":151266,"ðijĺģ":151267,"ðĴĢ":151268,"ðĴ̏":151269,"ðĴģ":151270,"ðĴģº":151271,"ðĴĦ":151272,"ðĴĦ·":151273,"ðĴĬ":151274,"ðĴĬij":151275,"ðĴĭ":151276,"ðĴĭĹ":151277,"ðĴĮ":151278,"ðĴĮ¨":151279,"ðĵĥ¢":151280,"ðĵĥ°":151281,"ðĸł":151282,"ðĸłļ":151283,"ðĿĦĥ":151284,"ðĿĦħ":151285,"ðĿĦķ":151286,"ðĿĦĻ":151287,"ðĿĦ±":151288,"ðĿĦ´":151289,"ðĿĦ¹":151290,"ðĿħİ":151291,"ðĿħª":151292,"ðĿĨ£":151293,"ðĿĨ³":151294,"ðĿĨ¹":151295,"ðĿĩĬ":151296,"ðĿĩĹ":151297,"ðĿĩļ":151298,"ðĿĩľ":151299,"ðĿĩł":151300,"ðĿIJī":151301,"ðĿIJĸ":151302,"ðĿIJĺ":151303,"ðĿIJ£":151304,"ðĿIJ±":151305,"ðĿijĬ":151306,"ðĿijŃ":151307,"ðĿij¼":151308,"ðĿij½":151309,"ðĿĴ°":151310,"ðĿĴ·":151311,"ðĿĴ¿":151312,"ðĿĵģ":151313,"ðĿĵĭ":151314,"ðĿĵİ":151315,"ðĿĵĴ":151316,"ðĿĵĺ":151317,"ðĿĵ¢":151318,"ðĿĵ¦":151319,"ðĿĵ«":151320,"ðĿĵ¿":151321,"ðĿĶİ":151322,"ðĿͱ":151323,"ðĿĶ´":151324,"ðĿĶ·":151325,"ðĿ͏":151326,"ðĿͽ":151327,"ðĿķĤ":151328,"ðĿķĥ":151329,"ðĿķĭ":151330,"ðĿķı":151331,"ðĿķIJ":151332,"ðĿķ¥":151333,"ðĿķ´":151334,"ðĿķº":151335,"ðĿĸIJ":151336,"ðĿĸĽ":151337,"ðĿĸĿ":151338,"ðĿĸŀ":151339,"ðĿĹ©":151340,"ðĿĹ³":151341,"ðĿĹ½":151342,"ðĿĺĬ":151343,"ðĿĺĭ":151344,"ðĿĺĶ":151345,"ðĿĺ±":151346,"ðĿĺ´":151347,"ðĿĺ¿":151348,"ðĿĻĴ":151349,"ðĿĻĿ":151350,"ðĿĻŁ":151351,"ðĿϬ":151352,"ðĿĻŃ":151353,"ðĿĻ»":151354,"ðĿϾ":151355,"ðĿļĪ":151356,"ðĿļĭ":151357,"ðĿļij":151358,"ðĿļŁ":151359,"ðĿļł":151360,"ðĿļ£":151361,"ðĿĽ½":151362,"ðĿľĤ":151363,"ðĿľĶ":151364,"ðĿľĻ":151365,"ðŁĢ":151366,"ðŁĢĦ":151367,"ðŁĦ²":151368,"ðŁĦ¶":151369,"ðŁħIJ":151370,"ðŁħĸ":151371,"ðŁħļ":151372,"ðŁħĽ":151373,"ðŁħ¦":151374,"ðŁħ¶":151375,"ðŁħ»":151376,"ðŁħ¼":151377,"ðŁĨĥ":151378,"ðŁĨĨ":151379,"ðŁĨİ":151380,"ðŁĪ¯":151381,"ðŁĪ²":151382,"ðŁĪ¹":151383,"ðŁĮĩ":151384,"ðŁĮĵ":151385,"ðŁįĺ":151386,"ðŁİij":151387,"ðŁİ¿":151388,"ðŁıı":151389,"ðŁıĴ":151390,"ðŁı©":151391,"ðŁı¯":151392,"ðŁIJĢ":151393,"ðŁijĿ":151394,"ðŁĴ¹":151395,"ðŁĴº":151396,"ðŁĵŁ":151397,"ðŁĵª":151398,"ðŁĵ¼":151399,"ðŁĶĢ":151400,"ðŁĶĤ":151401,"ðŁĶĥ":151402,"ðŁĶĩ":151403,"ðŁĶĵ":151404,"ðŁĶ¢":151405,"ðŁĶ¤":151406,"ðŁĶ©":151407,"ðŁķĸ":151408,"ðŁķļ":151409,"ðŁķľ":151410,"ðŁķĿ":151411,"ðŁķŀ":151412,"ðŁķł":151413,"ðŁķ¢":151414,"ðŁķ³":151415,"ðŁĸĩ":151416,"ðŁĸij":151417,"ðŁĸ¶":151418,"ðŁĹģ":151419,"Ѩ":151420,"Úİ":151421,"á¡Į":151422,"Ḱ":151423,"áºĢ":151424,"á¼®":151425,"á½Ŀ":151426,"âĦ¬":151427,"âļ§":151428,"⼤":151429,"㳬":151430,"êĻĭ":151431,"ê¸ij":151432,"ëĶī":151433,"ëĹį":151434,"ë¡ij":151435,"ë¯ij":151436,"ë»ħ":151437,"ë¼Ŀ":151438,"ìĦIJ":151439,"ìī¡":151440,"ìĭ²":151441,"ìı±":151442,"ìŤ":151443,"ìĿ©":151444,"ìĿ¿":151445,"ìŁĻ":151446,"ìł°":151447,"ì¥ī":151448,"íĬŃ":151449,"íķ®":151450,"ï®ı":151451,"ðŁħ±":151452,"ðŁĨĴ":151453,"ðŁķĭ":151454,"Éĺ":151455,"Êĵ":151456,"Õĥ":151457,"à´´":151458,"à½ħ":151459,"áĨº":151460,"áĪĬ":151461,"á΍":151462,"áξ":151463,"áīIJ":151464,"áĮĥ":151465,"áĮ½":151466,"áĶŃ":151467,"áłĤ":151468,"ᳬ":151469,"ᨸ":151470,"á©ĭ":151471,"á¶ı":151472,"á¾Ķ":151473,"á¿IJ":151474,"á¿ļ":151475,"âĻĻ":151476,"âļĤ":151477,"âļĹ":151478,"â¡¢":151479,"⤦":151480,"ëĸ°":151481,"ë¤Ĥ":151482,"ë§ł":151483,"ë±ĭ":151484,"ë±IJ":151485,"ìĽ¢":151486,"ìľ¾":151487,"ì³ħ":151488,"ì»ģ":151489,"íģ»":151490,"íĥĻ":151491,"íĵĸ":151492,"íĵŃ":151493,"íķ±":151494,"íĽľ":151495,"ï¤ħ":151496,"ï¤Ĩ":151497,"ï¦ĥ":151498,"ï§©":151499,"ï¨Ĥ":151500,"ðIJ¤Ķ":151501,"ðIJŃĵ":151502,"ðIJ°¼":151503,"ðĿĵŀ":151504,"ðĿĵ°":151505,"ðĿĻľ":151506,"ðĿļģ":151507,"ðŁħ¢":151508,"ðŁıĩ":151509,"Ȳ":151510,"ʶ":151511,"ÔĪ":151512,"Ôij":151513,"Ýĵ":151514,"Ý¥":151515,"à¤ij":151516,"ॱ":151517,"à¬ī":151518,"à°³":151519,"à°µ":151520,"à²Ł":151521,"áĢı":151522,"áģ¼":151523,"áī¨":151524,"áĬĴ":151525,"áĭ©":151526,"áĮĦ":151527,"áĮĶ":151528,"áIJ§":151529,"áĴĮ":151530,"áĶħ":151531,"áĶĬ":151532,"áłĦ":151533,"á¨ģ":151534,"á¸ĥ":151535,"ḻ":151536,"âĶŀ":151537,"âĺµ":151538,"âļ£":151539,"â²¢":151540,"ãĪª":151541,"ä¶µ":151542,"ê²Ļ":151543,"ê²´":151544,"ê³Ĥ":151545,"롼":151546,"ìĨĬ":151547,"ì¼ĩ":151548,"íĭį":151549,"íĵ¬":151550,"íĵ®":151551,"íĵ¶":151552,"íĵ»":151553,"臘":151554,"ï¥ł":151555,"辰":151556,"ïѲ":151557,"ðIJŃĬ":151558,"ðIJ±ħ":151559,"ðĸ¥":151560,"ðĸ¥¨":151561,"ðĿij³":151562,"ðĿĵķ":151563,"ðĿĵ¬":151564,"ðĿĵ¹":151565,"ðĿĵ¾":151566,"ðĿĶĵ":151567,"ðĿķį":151568,"ðĿķ¡":151569,"ðĿķ±":151570,"ðĿĸĸ":151571,"ðĿĺı":151572,"ðĿĺIJ":151573,"ðĿĺļ":151574,"ðĿĻ®":151575,"ðĿϰ":151576,"ðĿϏ":151577,"ðĿĻº":151578,"ðĿϼ":151579,"ðĿϽ":151580,"ðĿĻ¿":151581,"ðĿļĦ":151582,"ðĿļı":151583,"ðŁħħ":151584,"ðŁħĵ":151585,"ÆĪ":151586,"àłĮ":151587,"áϳ":151588,"áļĮ":151589,"áĽħ":151590,"áĽIJ":151591,"á¤Ĭ":151592,"á¸Ĭ":151593,"âͽ":151594,"âķĬ":151595,"âĽĩ":151596,"âĽı":151597,"âĿª":151598,"âĿ«":151599,"⣰":151600,"ãĦį":151601,"ãĦĵ":151602,"ãĦ§":151603,"ãħĸ":151604,"ãī«":151605,"ê¦Ķ":151606,"ï±Ĭ":151607,"àºĤ":151608,"áħ£":151609,"á¥Ķ":151610,"ᥤ":151611,"âĨ¤":151612,"âĨ·":151613,"âĩŀ":151614,"âĸ¤":151615,"âŀ¶":151616,"ãμ":151617,"嘆":151618,"ðĵı§":151619,"âͲ":151620,"âĢ´":151621,"âĴŁ":151622,"âĴ¡":151623,"â°Ĥ":151624,"â°į":151625,"â°İ":151626,"â°IJ":151627,"â°ij":151628,"â°Ł":151629,"â°ł":151630,"â°¡":151631,"â¼Ń":151632,"ãĬ¥":151633,"âĴł":151634,"⽺":151635,"ãĩº":151636,"ãĩ½":151637,"ï¨Ĭ":151638,"áķ·":151639,"âį¨":151640,"âºŁ":151641,"â½Ĺ":151642} \ No newline at end of file diff --git a/README.md b/README.md index 7be5fc7f47d5db027d120b8024982df93db95b74..c4f3c9bff605d6c2237b73106db6833c99816abd 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,213 @@ +# Explore–Execute Chain (E2C) + +This repository provides the implementation of our paper: +**Explore–Execute Chain: Towards an Efficient Structured Reasoning Paradigm** +*Kaisen Yang, Lixuan He, Rushi Shah, Kaicheng Yang, Qinwei Ma, Dianbo Liu, Alex Lamb* + +> *Under review at ICLR 2026* + --- -license: mit + +## 📖 Overview + +Large Language Models (LLMs) often intertwine high-level planning with low-level execution, limiting both efficiency and interpretability. +The **Explore–Execute Chain (E2C)** framework addresses this challenge by **decoupling reasoning into two distinct stages**: + +1. **Exploration** — Generate lightweight reasoning sketches (plans). +2. **Execution** — Select one or a few promising plans and execute them faithfully. + +This separation offers multiple benefits: + +* **Efficiency** — Short explorations reduce computation while preserving reasoning coverage. +* **Interpretability** — Exploration traces are explicit and auditable. +* **Adaptability** — Exploration can be domain-adapted with minimal supervision. + --- + +## 📝 Current Progress + +**Released** + +* ✅ Public release of **datasets** and **pretrained models** + +**Ongoing** + +* 🔄 Exploring more diverse **exploration strategies** and more reliable **execution mechanisms** +* 🔄 Cleaning and organizing **experimental code** for broader reproducibility + +Stay tuned for further updates as we expand features and experiments. + +--- + +## 🚀 Key Features + +* **Two-stage training pipeline**: + + * **E2C-SFT** — Supervised fine-tuning on synthetic exploration–execution pairs. + * **E2C-RL** — Reinforcement learning with token-level weighting tailored to exploration. + +* **Efficient domain adaptation (EF-SFT)** — Adapt models with exploration-only data at minimal cost. + +* **Test-time scaling** — Sample multiple explorations and aggregate them via clustering or LLM-based fusion. + +* **Extensive benchmarks**: + + * **Mathematical reasoning**: AIME’24/25, MATH500, Olympiad, Minerva. + * **Medical reasoning**: MedQA, MedMCQA, MMLU medical subsets. + +--- + +## 📂 Repository Structure + +``` +. +├── data/ # Dataset preparation scripts and instructions +├── e2c/ # Core E2C framework +│ ├── sft/ # Supervised fine-tuning (E2C-SFT, EF-SFT) +│ ├── rl/ # Reinforcement learning (E2C-RL, GRPO baseline) +│ └── inference/ # Test-time scaling and evaluation +├── prompts/ # Prompt templates +├── scripts/ # Training / evaluation entrypoints +└── README.md +``` + +--- + +## ⚙️ Setup + +### Requirements + +See `e2c/verl/README.md` for details. + +* Python 3.9+ +* PyTorch ≥ 2.0 +* HuggingFace Transformers +* sentence-transformers +* Additional packages in `requirements.txt` + +Install dependencies via: + +```bash +pip install -r requirements.txt +``` + +--- + +## 📥 Model Download + +Pretrained and fine-tuned E2C models are available on Hugging Face. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_name = "your-org/e2c-model" +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained(model_name) +``` + +Or via CLI: + +```bash +huggingface-cli download your-org/e2c-model +``` + +Replace `your-org/e2c-model` with the actual repository name. + +--- + +## 🏋️ Training + +### Stage 1: Supervised Fine-Tuning (E2C-SFT) + +```bash +bash e2c/sft/train_sft.sh +``` + +### Stage 2: Reinforcement Learning (E2C-RL) + +```bash +bash e2c/rl/train_rl.sh +``` + +### Efficient Adaptation (EF-SFT) + +```bash +bash e2c/sft/train_ef_sft.sh +``` + +--- + +## 🔍 Evaluation + +Run evaluation and test-time scaling: + +```bash +bash e2c/inference/evaluate.sh +``` + +Supported strategies include: + +* **E2C-Select (Self LM-Judge)** +* **E2C-Select (Semantic Cluster)** +* **E2C-SC (Self-Consistency)** +* **E2C-RP (Random Plan)** + +--- + +## 📊 Results + +E2C consistently improves over GRPO baselines with reduced cost. + +* On **AIME’24**, Qwen3-4B + E2C-(SFT+RL) yields **+8.7% accuracy gain** over GRPO. +* On **medical reasoning**, EF-SFT adapts with only **3.5% of training tokens**, surpassing standard SFT. + +See the paper for detailed results and ablations. + +--- + +## 🙏 Credits + +This work builds upon the **VERL** framework, which has been instrumental for training and experimentation. +We thank the authors for releasing such a powerful and flexible platform. + +--- + +## 📌 Notes + +* This repository is **work in progress**; some datasets and scripts are placeholders. +* Full reproduction may require additional setup. +* Hyperparameters and prompt templates are provided in the paper appendices. +* We are still improving the training process and codebase for better usability. + +--- + +## 📜 Citation + +If you use this work, please cite: + +**BibTeX** + +```bibtex +@inproceedings{yang2026explore, + title={Explore--Execute Chain: Towards an Efficient Structured Reasoning Paradigm}, + author={Yang, Kaisen and He, Lixuan and Shah, Rushi and Yang, Kaicheng and Ma, Qinwei and Liu, Dianbo and Lamb, Alex}, + booktitle={International Conference on Learning Representations (ICLR)}, + year={2026}, + note={under review} +} +``` + +**APA** + +> Yang, K., He, L., Shah, R., Yang, K., Ma, Q., Liu, D., & Lamb, A. (2026). *Explore–Execute Chain: Towards an Efficient Structured Reasoning Paradigm*. International Conference on Learning Representations (ICLR). (Under review) + +**MLA** + +> Yang, Kaisen, et al. "Explore–Execute Chain: Towards an Efficient Structured Reasoning Paradigm." *International Conference on Learning Representations (ICLR)*, 2026. Under review. + +--- + +## 🧾 License + +This project is licensed under the **MIT License**. +See the [LICENSE](LICENSE) file for details. diff --git a/e2c/config/__init__.py b/e2c/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a5105f7f340251cb44c8245bda8d8a1eeaaaeeff --- /dev/null +++ b/e2c/config/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .algorithm import * # noqa +from .config import * # noqa +from . import config, algorithm + +__all__ = config.__all__ + algorithm.__all__ diff --git a/e2c/config/_generated_ppo_megatron_trainer.yaml b/e2c/config/_generated_ppo_megatron_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ff14b9134fd49a57c44f6764127cf0527b73230 --- /dev/null +++ b/e2c/config/_generated_ppo_megatron_trainer.yaml @@ -0,0 +1,541 @@ +# This reference configration yaml is automatically generated via 'scripts/generate_trainer_config.sh' +# in which it invokes 'python3 scripts/print_cfg.py --cfg job --config-name=ppo_megatron_trainer.yaml' to flatten the 'verl/trainer/config/ppo_megatron_trainer.yaml' config fields into a single file. +# Do not modify this file directly. +# The file is usually only for reference and never used. + +actor_rollout_ref: + actor: + optim: + _target_: verl.workers.config.McoreOptimizerConfig + lr: 1.0e-06 + lr_warmup_steps_ratio: 0.0 + total_training_steps: -1 + weight_decay: 0.01 + lr_warmup_steps: -1 + betas: + - 0.9 + - 0.999 + clip_grad: 1.0 + optimizer: adam + lr_warmup_init: 0.0 + lr_decay_steps: null + lr_decay_style: constant + min_lr: 0.0 + weight_decay_incr_style: constant + lr_wsd_decay_style: exponential + lr_wsd_decay_steps: null + use_checkpoint_opt_param_scheduler: false + override_optimizer_config: {} + megatron: + _target_: verl.workers.config.McoreEngineConfig + param_offload: false + grad_offload: false + optimizer_offload: false + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null + context_parallel_size: 1 + sequence_parallel: true + use_distributed_optimizer: true + use_dist_checkpointing: false + dist_checkpointing_path: null + seed: 42 + override_ddp_config: {} + override_transformer_config: + recompute_granularity: null + recompute_modules: + - core_attn + recompute_method: null + recompute_num_layers: null + attention_backend: flash + override_mcore_model_config: {} + use_mbridge: false + forward_only: false + _target_: verl.workers.config.McoreActorConfig + strategy: megatron + ppo_mini_batch_size: 256 + ppo_micro_batch_size: null + ppo_micro_batch_size_per_gpu: null + use_dynamic_bsz: false + ppo_max_token_len_per_gpu: 16384 + clip_ratio: 0.2 + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + freeze_vision_tower: false + policy_loss: + _target_: verl.workers.config.PolicyLossConfig + loss_mode: vanilla + clip_cov_ratio: 0.0002 + clip_cov_lb: 1.0 + clip_cov_ub: 5.0 + kl_cov_ratio: 0.0002 + ppo_kl_coef: 0.1 + clip_ratio_c: 3.0 + loss_agg_mode: token-mean + entropy_coeff: 0 + tis_imp_ratio_cap: -1 + use_kl_loss: false + use_torch_compile: true + kl_loss_coef: 0.001 + kl_loss_type: low_var_kl + ppo_epochs: 1 + shuffle: false + checkpoint: + _target_: verl.trainer.config.CheckpointConfig + save_contents: + - model + - optimizer + - extra + load_contents: ${.save_contents} + async_save: false + use_fused_kernels: ${oc.select:actor_rollout_ref.model.use_fused_kernels,false} + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + data_loader_seed: null + load_weight: true + ref: + strategy: megatron + use_torch_compile: ${oc.select:actor_rollout_ref.actor.use_torch_compile,true} + log_prob_micro_batch_size: null + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + megatron: + _target_: verl.workers.config.MegatronEngineConfig + param_offload: false + grad_offload: false + optimizer_offload: false + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null + context_parallel_size: 1 + sequence_parallel: true + use_distributed_optimizer: true + use_dist_checkpointing: false + dist_checkpointing_path: null + seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42} + override_ddp_config: {} + override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}} + override_mcore_model_config: {} + use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False} + forward_only: false + load_weight: true + rollout: + _target_: verl.workers.config.RolloutConfig + name: ??? + mode: sync + temperature: 1.0 + top_k: -1 + top_p: 1 + prompt_length: ${oc.select:data.max_prompt_length,512} + response_length: ${oc.select:data.max_response_length,512} + dtype: bfloat16 + gpu_memory_utilization: 0.5 + ignore_eos: false + enforce_eager: false + cudagraph_capture_sizes: null + free_cache_engine: true + tensor_model_parallel_size: 2 + data_parallel_size: 1 + expert_parallel_size: 1 + max_num_batched_tokens: 8192 + max_model_len: null + max_num_seqs: 1024 + enable_chunked_prefill: true + enable_prefix_caching: true + load_format: dummy + log_prob_micro_batch_size: null + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + disable_log_stats: true + do_sample: true + 'n': 1 + over_sample_rate: 0 + multi_stage_wake_up: false + engine_kwargs: + vllm: {} + sglang: {} + val_kwargs: + _target_: verl.workers.config.SamplingConfig + top_k: -1 + top_p: 1.0 + temperature: 0 + 'n': 1 + do_sample: false + multi_turn: + _target_: verl.workers.config.MultiTurnConfig + enable: false + max_assistant_turns: null + tool_config_path: null + max_user_turns: null + max_parallel_calls: 1 + max_tool_response_length: 256 + tool_response_truncate_side: middle + interaction_config_path: null + use_inference_chat_template: false + tokenization_sanity_check_mode: strict + format: hermes + num_repeat_rollouts: null + calculate_log_probs: false + agent: + _target_: verl.workers.config.AgentLoopConfig + num_workers: 8 + agent_loop_config_path: null + custom_async_server: + _target_: verl.workers.config.CustomAsyncServerConfig + path: null + name: null + update_weights_bucket_megabytes: 512 + trace: + _target_: verl.workers.config.TraceConfig + backend: null + token2text: false + skip_rollout: false + skip_dump_dir: /tmp/rollout_dump + skip_tokenizer_init: true + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: ${oc.select:actor_rollout_ref.actor.profiler.enable,false} + all_ranks: ${oc.select:actor_rollout_ref.actor.profiler.all_ranks,false} + ranks: ${oc.select:actor_rollout_ref.actor.profiler.ranks,[]} + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} + layer_name_map: + qkv_layer_name: qkv + gate_proj_layer_name: gate_up + hybrid_engine: true + nccl_timeout: 600 + model: + path: ~/models/deepseek-llm-7b-chat + custom_chat_template: null + external_lib: null + override_config: + model_config: {} + moe_config: + freeze_moe_router: false + use_fused_kernels: false + trust_remote_code: false + use_remove_padding: false +data: + tokenizer: null + use_shm: false + train_files: ~/data/rlhf/gsm8k/train.parquet + val_files: ~/data/rlhf/gsm8k/test.parquet + prompt_key: prompt + reward_fn_key: data_source + max_prompt_length: 512 + max_response_length: 512 + train_batch_size: 1024 + val_batch_size: null + return_raw_input_ids: false + return_raw_chat: false + return_full_prompt: false + shuffle: true + dataloader_num_workers: 8 + validation_shuffle: false + filter_overlong_prompts: false + filter_overlong_prompts_workers: 1 + truncation: error + image_key: images + video_key: videos + trust_remote_code: false + custom_cls: + path: null + name: null + return_multi_modal_inputs: true + sampler: + class_path: null + class_name: null + datagen: + path: null + name: null + apply_chat_template_kwargs: {} +critic: + optim: + _target_: verl.workers.config.McoreOptimizerConfig + lr: 1.0e-05 + lr_warmup_steps_ratio: 0.0 + total_training_steps: -1 + weight_decay: 0.01 + lr_warmup_steps: -1 + betas: + - 0.9 + - 0.999 + clip_grad: 1.0 + optimizer: adam + lr_warmup_init: 0.0 + lr_decay_steps: null + lr_decay_style: constant + min_lr: 0.0 + weight_decay_incr_style: constant + lr_wsd_decay_style: exponential + lr_wsd_decay_steps: null + use_checkpoint_opt_param_scheduler: false + override_optimizer_config: {} + megatron: + _target_: verl.workers.config.McoreEngineConfig + param_offload: false + grad_offload: false + optimizer_offload: false + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null + context_parallel_size: 1 + sequence_parallel: true + use_distributed_optimizer: true + use_dist_checkpointing: false + dist_checkpointing_path: null + seed: 42 + override_ddp_config: {} + override_transformer_config: + recompute_granularity: null + recompute_modules: + - core_attn + recompute_method: null + recompute_num_layers: null + attention_backend: flash + override_mcore_model_config: {} + use_mbridge: false + forward_only: false + _target_: verl.workers.config.McoreCriticConfig + rollout_n: ${oc.select:actor_rollout_ref.rollout.n,1} + strategy: megatron + enable: null + model: + path: ~/models/deepseek-llm-7b-chat + tokenizer_path: ${oc.select:actor_rollout_ref.model.path,"~/models/deepseek-llm-7b-chat"} + override_config: + model_config: {} + moe_config: + freeze_moe_router: false + external_lib: ${oc.select:actor_rollout_ref.model.external_lib,null} + trust_remote_code: ${oc.select:actor_rollout_ref.model.trust_remote_code,false} + _target_: verl.trainer.config.BaseModelConfig + ppo_mini_batch_size: ${oc.select:actor_rollout_ref.actor.ppo_mini_batch_size,256} + ppo_micro_batch_size: null + ppo_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size,null} + use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + ppo_max_token_len_per_gpu: 32768 + forward_max_token_len_per_gpu: ${.ppo_max_token_len_per_gpu} + ppo_epochs: ${oc.select:actor_rollout_ref.actor.ppo_epochs,1} + shuffle: ${oc.select:actor_rollout_ref.actor.shuffle,false} + cliprange_value: 0.5 + loss_agg_mode: ${oc.select:actor_rollout_ref.actor.loss_agg_mode,token-mean} + checkpoint: + _target_: verl.trainer.config.CheckpointConfig + save_contents: + - model + - optimizer + - extra + load_contents: ${.save_contents} + async_save: false + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + nccl_timeout: 600 + load_weight: true + data_loader_seed: ${oc.select:actor_rollout_ref.actor.data_loader_seed,null} +reward_model: + enable: false + enable_resource_pool: false + n_gpus_per_node: 0 + nnodes: 0 + strategy: megatron + model: + input_tokenizer: ${actor_rollout_ref.model.path} + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + external_lib: ${actor_rollout_ref.model.external_lib} + trust_remote_code: false + micro_batch_size: null + micro_batch_size_per_gpu: null + max_length: null + use_dynamic_bsz: ${critic.use_dynamic_bsz} + forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + reward_manager: naive + launch_reward_fn_async: false + sandbox_fusion: + url: null + max_concurrent: 64 + memory_limit_mb: 1024 + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} + nccl_timeout: 600 + megatron: + _target_: verl.workers.config.MegatronEngineConfig + param_offload: false + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null + context_parallel_size: 1 + sequence_parallel: true + use_distributed_optimizer: false + use_dist_checkpointing: false + dist_checkpointing_path: null + seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42} + override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}} + use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False} + load_weight: true +custom_reward_function: + path: null + name: compute_score +algorithm: + _target_: verl.trainer.config.AlgoConfig + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + norm_adv_by_std_in_grpo: true + use_kl_in_reward: false + kl_penalty: kl + kl_ctrl: + _target_: verl.trainer.config.KLControlConfig + type: fixed + kl_coef: 0.001 + horizon: 10000 + target_kl: 0.1 + use_pf_ppo: false + pf_ppo: + reweight_method: pow + weight_pow: 2.0 +trainer: + balance_batch: true + total_epochs: 30 + total_training_steps: null + project_name: verl_examples + experiment_name: gsm8k + logger: + - console + - wandb + log_val_generations: 0 + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + esi_redundant_time: 0 + resume_mode: auto + resume_from_path: null + del_local_ckpt_after_load: false + val_before_train: true + test_freq: -1 + critic_warmup: 0 + default_hdfs_dir: null + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + max_actor_ckpt_to_keep: null + max_critic_ckpt_to_keep: null + ray_wait_register_center_timeout: 300 + device: cuda + rollout_data_dir: null +global_profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: null + steps: null + profile_continuous_steps: false + save_path: outputs/profile + global_tool_config: + nsys: + discrete: false + controller_nsight_options: + trace: cuda,nvtx,cublas,ucx + cuda-memory-usage: 'true' + cuda-graph-trace: graph + worker_nsight_options: + trace: cuda,nvtx,cublas,ucx + cuda-memory-usage: 'true' + cuda-graph-trace: graph + capture-range: cudaProfilerApi + capture-range-end: null + kill: none + torch_memory: + trace_alloc_max_entries: 100000 + stack_depth: 32 + context: all + stacks: all + kw_args: {} +ray_kwargs: + ray_init: + num_cpus: null + timeline_json_file: null diff --git a/e2c/config/_generated_ppo_trainer.yaml b/e2c/config/_generated_ppo_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61e5efb54501d7b747eeed43a3c4d103a1440b61 --- /dev/null +++ b/e2c/config/_generated_ppo_trainer.yaml @@ -0,0 +1,523 @@ +# This reference configration yaml is automatically generated via 'scripts/generate_trainer_config.sh' +# in which it invokes 'python3 scripts/print_cfg.py --cfg job ' to flatten the 'verl/trainer/config/ppo_trainer.yaml' config fields into a single file. +# Do not modify this file directly. +# The file is usually only for reference and never used. + +actor_rollout_ref: + actor: + optim: + _target_: verl.workers.config.FSDPOptimizerConfig + lr: 1.0e-06 + lr_warmup_steps_ratio: 0.0 + total_training_steps: -1 + weight_decay: 0.01 + lr_warmup_steps: -1 + betas: + - 0.9 + - 0.999 + clip_grad: 1.0 + min_lr_ratio: 0.0 + num_cycles: 0.5 + warmup_style: constant + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + wrap_policy: + min_num_params: 0 + param_offload: false + optimizer_offload: false + offload_policy: false + reshard_after_forward: true + fsdp_size: -1 + forward_prefetch: false + model_dtype: fp32 + use_orig_params: false + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + use_torch_compile: true + entropy_checkpointing: false + forward_only: false + strategy: fsdp + _target_: verl.workers.config.FSDPActorConfig + strategy: fsdp + ppo_mini_batch_size: 256 + ppo_micro_batch_size: null + ppo_micro_batch_size_per_gpu: null + use_dynamic_bsz: false + ppo_max_token_len_per_gpu: 16384 + clip_ratio: 0.2 + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + freeze_vision_tower: false + policy_loss: + _target_: verl.workers.config.PolicyLossConfig + loss_mode: vanilla + clip_cov_ratio: 0.0002 + clip_cov_lb: 1.0 + clip_cov_ub: 5.0 + kl_cov_ratio: 0.0002 + ppo_kl_coef: 0.1 + clip_ratio_c: 3.0 + loss_agg_mode: token-mean + entropy_coeff: 0 + tis_imp_ratio_cap: -1 + use_kl_loss: false + use_torch_compile: true + kl_loss_coef: 0.001 + kl_loss_type: low_var_kl + ppo_epochs: 1 + shuffle: false + checkpoint: + _target_: verl.trainer.config.CheckpointConfig + save_contents: + - model + - optimizer + - extra + load_contents: ${.save_contents} + async_save: false + use_fused_kernels: ${oc.select:actor_rollout_ref.model.use_fused_kernels,false} + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + grad_clip: 1.0 + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + entropy_checkpointing: false + use_remove_padding: ${oc.select:actor_rollout_ref.model.use_remove_padding,false} + ref: + strategy: ${actor_rollout_ref.actor.strategy} + use_torch_compile: ${oc.select:actor_rollout_ref.actor.use_torch_compile,true} + log_prob_micro_batch_size: null + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + wrap_policy: + min_num_params: 0 + param_offload: false + optimizer_offload: false + offload_policy: false + reshard_after_forward: true + fsdp_size: -1 + forward_prefetch: false + model_dtype: fp32 + use_orig_params: false + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + use_torch_compile: true + entropy_checkpointing: false + forward_only: false + strategy: fsdp + model: null + ulysses_sequence_parallel_size: ${oc.select:actor_rollout_ref.actor.ulysses_sequence_parallel_size,1} + entropy_from_logits_with_chunking: false + entropy_checkpointing: false + rollout: + _target_: verl.workers.config.RolloutConfig + name: ??? + mode: sync + temperature: 1.0 + top_k: -1 + top_p: 1 + prompt_length: ${oc.select:data.max_prompt_length,512} + response_length: ${oc.select:data.max_response_length,512} + dtype: bfloat16 + gpu_memory_utilization: 0.5 + ignore_eos: false + enforce_eager: false + cudagraph_capture_sizes: null + free_cache_engine: true + tensor_model_parallel_size: 2 + data_parallel_size: 1 + expert_parallel_size: 1 + max_num_batched_tokens: 8192 + max_model_len: null + max_num_seqs: 1024 + enable_chunked_prefill: true + enable_prefix_caching: true + load_format: dummy + log_prob_micro_batch_size: null + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + disable_log_stats: true + do_sample: true + 'n': 1 + over_sample_rate: 0 + multi_stage_wake_up: false + engine_kwargs: + vllm: {} + sglang: {} + val_kwargs: + _target_: verl.workers.config.SamplingConfig + top_k: -1 + top_p: 1.0 + temperature: 0 + 'n': 1 + do_sample: false + multi_turn: + _target_: verl.workers.config.MultiTurnConfig + enable: false + max_assistant_turns: null + tool_config_path: null + max_user_turns: null + max_parallel_calls: 1 + max_tool_response_length: 256 + tool_response_truncate_side: middle + interaction_config_path: null + use_inference_chat_template: false + tokenization_sanity_check_mode: strict + format: hermes + num_repeat_rollouts: null + calculate_log_probs: false + agent: + _target_: verl.workers.config.AgentLoopConfig + num_workers: 8 + agent_loop_config_path: null + custom_async_server: + _target_: verl.workers.config.CustomAsyncServerConfig + path: null + name: null + update_weights_bucket_megabytes: 512 + trace: + _target_: verl.workers.config.TraceConfig + backend: null + token2text: false + skip_rollout: false + skip_dump_dir: /tmp/rollout_dump + skip_tokenizer_init: true + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: ${oc.select:actor_rollout_ref.actor.profiler.enable,false} + all_ranks: ${oc.select:actor_rollout_ref.actor.profiler.all_ranks,false} + ranks: ${oc.select:actor_rollout_ref.actor.profiler.ranks,[]} + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} + layered_summon: false + model: + _target_: verl.workers.config.HFModelConfig + path: ~/models/deepseek-llm-7b-chat + hf_config_path: null + tokenizer_path: null + use_shm: false + trust_remote_code: false + custom_chat_template: null + external_lib: null + override_config: {} + enable_gradient_checkpointing: true + enable_activation_offload: false + use_remove_padding: false + lora_rank: 0 + lora_alpha: 16 + target_modules: all-linear + exclude_modules: null + use_liger: false + use_fused_kernels: false + fused_kernel_options: + impl_backend: torch + hybrid_engine: true + nccl_timeout: 600 +data: + tokenizer: null + use_shm: false + train_files: ~/data/rlhf/gsm8k/train.parquet + val_files: ~/data/rlhf/gsm8k/test.parquet + prompt_key: prompt + reward_fn_key: data_source + max_prompt_length: 512 + max_response_length: 512 + train_batch_size: 1024 + val_batch_size: null + return_raw_input_ids: false + return_raw_chat: false + return_full_prompt: false + shuffle: true + dataloader_num_workers: 8 + validation_shuffle: false + filter_overlong_prompts: false + filter_overlong_prompts_workers: 1 + truncation: error + image_key: images + video_key: videos + trust_remote_code: false + custom_cls: + path: null + name: null + return_multi_modal_inputs: true + sampler: + class_path: null + class_name: null + datagen: + path: null + name: null + apply_chat_template_kwargs: {} +critic: + optim: + _target_: verl.workers.config.FSDPOptimizerConfig + lr: 1.0e-05 + lr_warmup_steps_ratio: 0.0 + total_training_steps: -1 + weight_decay: 0.01 + lr_warmup_steps: -1 + betas: + - 0.9 + - 0.999 + clip_grad: 1.0 + min_lr_ratio: 0.0 + num_cycles: 0.5 + warmup_style: constant + model: + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + wrap_policy: + min_num_params: 0 + param_offload: false + optimizer_offload: false + offload_policy: false + reshard_after_forward: true + fsdp_size: -1 + forward_prefetch: false + model_dtype: fp32 + use_orig_params: false + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + use_torch_compile: true + entropy_checkpointing: false + forward_only: false + strategy: fsdp + path: ~/models/deepseek-llm-7b-chat + tokenizer_path: ${oc.select:actor_rollout_ref.model.path,"~/models/deepseek-llm-7b-chat"} + override_config: {} + external_lib: ${oc.select:actor_rollout_ref.model.external_lib,null} + trust_remote_code: ${oc.select:actor_rollout_ref.model.trust_remote_code,false} + _target_: verl.workers.config.FSDPCriticModelCfg + use_shm: false + enable_gradient_checkpointing: true + enable_activation_offload: false + use_remove_padding: false + lora_rank: 0 + lora_alpha: 16 + target_modules: all-linear + _target_: verl.workers.config.FSDPCriticConfig + rollout_n: ${oc.select:actor_rollout_ref.rollout.n,1} + strategy: fsdp + enable: null + ppo_mini_batch_size: ${oc.select:actor_rollout_ref.actor.ppo_mini_batch_size,256} + ppo_micro_batch_size: null + ppo_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size,null} + use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + ppo_max_token_len_per_gpu: 32768 + forward_max_token_len_per_gpu: ${.ppo_max_token_len_per_gpu} + ppo_epochs: ${oc.select:actor_rollout_ref.actor.ppo_epochs,1} + shuffle: ${oc.select:actor_rollout_ref.actor.shuffle,false} + cliprange_value: 0.5 + loss_agg_mode: ${oc.select:actor_rollout_ref.actor.loss_agg_mode,token-mean} + checkpoint: + _target_: verl.trainer.config.CheckpointConfig + save_contents: + - model + - optimizer + - extra + load_contents: ${.save_contents} + async_save: false + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + forward_micro_batch_size: ${oc.select:.ppo_micro_batch_size,null} + forward_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size_per_gpu,null} + ulysses_sequence_parallel_size: 1 + grad_clip: 1.0 +reward_model: + enable: false + enable_resource_pool: false + n_gpus_per_node: 0 + nnodes: 0 + strategy: fsdp + model: + input_tokenizer: ${actor_rollout_ref.model.path} + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + external_lib: ${actor_rollout_ref.model.external_lib} + trust_remote_code: false + use_shm: false + use_remove_padding: false + use_fused_kernels: ${actor_rollout_ref.model.use_fused_kernels} + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + wrap_policy: + min_num_params: 0 + param_offload: false + reshard_after_forward: true + fsdp_size: -1 + forward_prefetch: false + micro_batch_size: null + micro_batch_size_per_gpu: null + max_length: null + use_dynamic_bsz: ${critic.use_dynamic_bsz} + forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + reward_manager: naive + launch_reward_fn_async: false + sandbox_fusion: + url: null + max_concurrent: 64 + memory_limit_mb: 1024 + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} + ulysses_sequence_parallel_size: 1 +custom_reward_function: + path: null + name: compute_score +algorithm: + _target_: verl.trainer.config.AlgoConfig + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + norm_adv_by_std_in_grpo: true + use_kl_in_reward: false + kl_penalty: kl + kl_ctrl: + _target_: verl.trainer.config.KLControlConfig + type: fixed + kl_coef: 0.001 + horizon: 10000 + target_kl: 0.1 + use_pf_ppo: false + pf_ppo: + reweight_method: pow + weight_pow: 2.0 +trainer: + balance_batch: true + total_epochs: 30 + total_training_steps: null + project_name: verl_examples + experiment_name: gsm8k + logger: + - console + - wandb + log_val_generations: 0 + rollout_data_dir: null + validation_data_dir: null + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + esi_redundant_time: 0 + resume_mode: auto + resume_from_path: null + val_before_train: true + val_only: false + test_freq: -1 + critic_warmup: 0 + default_hdfs_dir: null + del_local_ckpt_after_load: false + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + max_actor_ckpt_to_keep: null + max_critic_ckpt_to_keep: null + ray_wait_register_center_timeout: 300 + device: cuda + use_legacy_worker_impl: auto +global_profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: null + steps: null + profile_continuous_steps: false + save_path: outputs/profile + global_tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: false + controller_nsight_options: + trace: cuda,nvtx,cublas,ucx + cuda-memory-usage: 'true' + cuda-graph-trace: graph + worker_nsight_options: + trace: cuda,nvtx,cublas,ucx + cuda-memory-usage: 'true' + cuda-graph-trace: graph + capture-range: cudaProfilerApi + capture-range-end: null + kill: none + torch_memory: + trace_alloc_max_entries: 100000 + stack_depth: 32 + context: all + stacks: all + kw_args: {} +ray_kwargs: + ray_init: + num_cpus: null + timeline_json_file: null diff --git a/e2c/config/actor/actor.yaml b/e2c/config/actor/actor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b5764e4a39bec650966babcd1c3699bbb038dd7 --- /dev/null +++ b/e2c/config/actor/actor.yaml @@ -0,0 +1,216 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# Target class for this configuration +_target_: verl.workers.config.ActorConfig + +# the abstract actor configs +# fsdp, fsdp2 or megatron. must be set. +strategy: ??? + +# Split each sample into sub-batches of this size for PPO +ppo_mini_batch_size: 256 + +# [Deprecated] Global micro batch size +ppo_micro_batch_size: null + +# Local per-GPU micro batch size +ppo_micro_batch_size_per_gpu: null + +# Whether to automatically adjust batch size at runtime +# oc.select: the default val for ref.log_prob_use_dynamic_bsz +use_dynamic_bsz: false + +# Max tokens per GPU in one PPO batch; affects gradient accumulation +# Typically it should be: n * ${data.max_prompt_length} + ${data.max_response_length} +# oc.select: the default val for ref.log_prob_max_token_len_per_gpu +ppo_max_token_len_per_gpu: 16384 + +# PPO clip ratio +clip_ratio: 0.2 + +# Lower bound for asymmetric clipping (used in dual-clip PPO) +clip_ratio_low: 0.2 + +# Upper bound for asymmetric clipping (used in dual-clip PPO) +clip_ratio_high: 0.2 + +# Whether to freeze vision model, if set true, it will be freeze vision model +freeze_vision_tower: false + +# policy loss config +policy_loss: + + # # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.PolicyLossConfig + + # Loss function mode: vanilla / clip-cov / kl-cov /gpg from https://arxiv.org/abs/2505.22617 + loss_mode: "vanilla" + + # Ratio of tokens to be clipped for clip-cov loss + clip_cov_ratio: 0.0002 + + # Lower bound for clip-cov loss + clip_cov_lb: 1.0 + + # Upper bound for clip-cov loss + clip_cov_ub: 5.0 + + # Ratio of tokens to be applied kl penalty for kl-cov loss + kl_cov_ratio: 0.0002 + + # KL divergence penalty coefficient + ppo_kl_coef: 0.1 + +# Constant C in Dual-clip PPO; clips when advantage < 0 and ratio > C +clip_ratio_c: 3.0 + +# Loss aggregation mode: "token-mean", "seq-mean-token-sum", or "seq-mean-token-mean" +loss_agg_mode: token-mean + +# Entropy regularization coefficient in PPO loss +entropy_coeff: 0 + +# Truncated Importance Sampling (TIS): https://fengyao.notion.site/off-policy-rl +# the truncation value C of truncated Importance Sampling (-1 for disable TIS) +tis_imp_ratio_cap: -1 + +# Whether to use KL loss instead of KL reward penalty. True for GRPO +use_kl_loss: false + +# Whether to use torch.compile() +# oc.select: the default val for ref.use_torch_compile +use_torch_compile: true + +# KL loss coefficient when use_kl_loss is enabled. For GRPO +kl_loss_coef: 0.001 + +# Type of KL divergence loss. Options: "kl"(k1), "abs", "mse"(k2), "low_var_kl"(k3), "full" +kl_loss_type: low_var_kl + +# Number of PPO epochs per batch +ppo_epochs: 1 + +# Shuffle training data across PPO epochs +shuffle: false + +# checkpoint configs +checkpoint: + + # Target dataclass for this configuration + _target_: verl.trainer.config.CheckpointConfig + + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + + # For more flexibility, you can specify the contents to load from the checkpoint. + # .xxx refers to the local variable xxx from the same level of hierarchy similar to python pkg + load_contents: ${.save_contents} + + # Whether to save checkpoints asynchronously. Only effective for Megatron as of now. + async_save: False + +# optimizer configs +optim: + + # Learning rate + lr: 1e-6 + + # Warmup steps ratio (used if lr_warmup_steps is 0 or negative) + lr_warmup_steps_ratio: 0.0 + + # Total training steps (must be overridden at runtime) + total_training_steps: -1 + + # Weight decay + weight_decay: 0.01 + + # Prioritized. None, 0 or Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps: -1 + + +# Whether to use custom fused kernels (e.g., FlashAttention, fused MLP) +use_fused_kernels: ${oc.select:actor_rollout_ref.model.use_fused_kernels,false} + +# profile the actor model in `update_policy` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # profiler tool, default same as profiler.tool in global config + # choices: nsys, npu, torch + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on Actor + enable: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config which only related to the role + tool_config: + + # nsys tool config + nsys: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NsightToolConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + + # npu config + npu: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NPUToolConfig + + # Contents to profile, can be empty + # options: npu, cpu, memory, shapes, module, stack + contents: [] + + # Collection level, optional values: level_none, level0, level1, level2. + level: "level1" + + # Whether to automatically parse the data. + analysis: True + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # torch profiler config + torch: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + + # start profile mini-batch in training + # NOTICE: different with global steps config which refers to iteration + # This field only related with mini-batch + step_start: 0 + + # stop profile mini-batch in training + step_end: null + + # torch memory profiler config + torch_memory: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + + # Maximum number of memory allocation entries to track + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + + # Stack trace depth for memory allocations + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} diff --git a/e2c/config/actor/dp_actor.yaml b/e2c/config/actor/dp_actor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab27304f7364f5cd8919a3253396fb6b736d64bf --- /dev/null +++ b/e2c/config/actor/dp_actor.yaml @@ -0,0 +1,42 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# defaults specify the default config from each component +defaults: + + # fsdp optimizer config + - ../optim@optim: fsdp + + # fsdp engine config + - ../engine@fsdp_config: fsdp + + # dp actor config, inheriting from trainer/config/actor/actor.yaml + - actor + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +# Target class for this configuration +_target_: verl.workers.config.FSDPActorConfig + +# TODO(haibin.lin): switch to fsdp2 +strategy: fsdp + +# Gradient clipping for actor updates, specific to the strategy. +grad_clip: 1.0 + +# Sequence parallelism size for Ulysses-style model parallelism +# oc.select: the default val for ref.ulysses_sequence_parallel_size +ulysses_sequence_parallel_size: 1 + +# calculate entropy with chunking to reduce memory peak +entropy_from_logits_with_chunking: False + +# recompute entropy +entropy_checkpointing: False + +# Whether to remove padding tokens in inputs during training +use_remove_padding: ${oc.select:actor_rollout_ref.model.use_remove_padding,false} diff --git a/e2c/config/actor/megatron_actor.yaml b/e2c/config/actor/megatron_actor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfb38467de220f3f7e522838105607d1d3e5f193 --- /dev/null +++ b/e2c/config/actor/megatron_actor.yaml @@ -0,0 +1,20 @@ +# megatron actor config, inheriting from trainer/config/actor/actor.yaml +defaults: + # megatron optimizer config + - ../optim@optim: megatron + + # megatron engine config + - ../engine@megatron: megatron + + - actor + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +_target_: verl.workers.config.McoreActorConfig + +strategy: megatron + +data_loader_seed: null + +load_weight: True diff --git a/e2c/config/algorithm.py b/e2c/config/algorithm.py new file mode 100644 index 0000000000000000000000000000000000000000..b66648b6ffb926a2586f1df1ad3b55a865f5f819 --- /dev/null +++ b/e2c/config/algorithm.py @@ -0,0 +1,87 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Any, Optional + +from verl.base_config import BaseConfig + +__all__ = ["AlgoConfig", "FilterGroupsConfig", "KLControlConfig"] + + +@dataclass +class KLControlConfig(BaseConfig): + """Configuration for KL control. + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + type (str): Type of KL control. Can be "fixed" or "adaptive". + kl_coef (float): Initial coefficient for KL penalty. + horizon (int): Horizon value for adaptive controller. + target_kl (float): Target KL divergence for adaptive controller. + """ + + type: str = "fixed" + kl_coef: float = 0.001 + horizon: int = 10000 + target_kl: float = 0.1 + + +@dataclass +class FilterGroupsConfig(BaseConfig): + """Configuration for filter groups (used in DAPO and Entropy). + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + enable (bool): Whether to enable filter groups. + metric (Optional[str]): Metric to use for filtering: "acc", "score", "seq_reward", "seq_final_reward", etc. + max_num_gen_batches (int): Non-positive values mean no upper limit. + """ + + enable: bool = False + metric: Optional[str] = None + max_num_gen_batches: int = 0 + + +@dataclass +class AlgoConfig(BaseConfig): + """Configuration for the algorithm. + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + gamma (float): Discount factor for future rewards. + lam (float): Trade-off between bias and variance in the GAE estimator. + adv_estimator (str): Advantage estimator type: "gae", "grpo", "reinforce_plus_plus", etc. + norm_adv_by_std_in_grpo (bool): Whether to normalize advantages by std (specific to GRPO). + use_kl_in_reward (bool): Whether to enable in-reward KL penalty. + kl_penalty (str): How to estimate KL divergence: "kl", "abs", "mse", "low_var_kl", or "full". + kl_ctrl (KLControlConfig): KL control configuration. + use_pf_ppo (bool): Whether to enable preference feedback PPO. + pf_ppo (dict[str, Any]): Preference feedback PPO settings. + filter_groups (Optional[FilterGroupsConfig]): Filter groups configuration, used in DAPO and Entropy + """ + + gamma: float = 1.0 + lam: float = 1.0 + adv_estimator: str = "gae" + norm_adv_by_std_in_grpo: bool = True + use_kl_in_reward: bool = False + kl_penalty: str = "kl" + kl_ctrl: KLControlConfig = field(default_factory=KLControlConfig) + use_pf_ppo: bool = False + pf_ppo: dict[str, Any] = field(default_factory=dict) + filter_groups: Optional[FilterGroupsConfig] = None diff --git a/e2c/config/config.py b/e2c/config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..3c326d0145b1f3122668057a36203591e782b4fd --- /dev/null +++ b/e2c/config/config.py @@ -0,0 +1,77 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Any, Optional + +from verl.base_config import BaseConfig + +__all__ = ["CheckpointConfig", "ProfileConfig", "BaseModelConfig"] + + +@dataclass +class CheckpointConfig(BaseConfig): + """Configuration for model checkpointing. + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + save_contents (list[str]): What to include in saved checkpoints. + Options: 'model', 'optimizer', 'extra', 'hf_model'. + load_contents (list[str]): Contents to load from checkpoint. Defaults to same as save_contents. + async_save (bool): Whether to save checkpoints asynchronously. Only implemented for Megatron as of now. + """ + + save_contents: list[str] = field(default_factory=lambda: ["model", "optimizer", "extra"]) + load_contents: list[str] = field(default_factory=lambda: ["model", "optimizer", "extra"]) + async_save: bool = False + + +@dataclass +class ProfileConfig(BaseConfig): + """Configuration for profiling. + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + profile_ranks (Optional[list[int]]): List of ranks to profile. None means all ranks. + step_start (int): Starting step for profiling. + step_end (int): Ending step for profiling. + save_path (Optional[str]): Path to save profiling results. + """ + + profile_ranks: Optional[list[int]] = None + step_start: int = -1 + step_end: int = -1 + save_path: Optional[str] = None + + +@dataclass +class BaseModelConfig(BaseConfig): + """Base configuration for a model. + Contains core settings for loading and initializing a pretrained model checkpoint. + + Args: + path (str): Path to pretrained model weights. + tokenizer_path (Optional[str]): Tokenizer path (defaults to actor's model path if not set). + override_config (dict): Hugging Face config override. + external_lib (Optional[str]): External model implementation (optional). + trust_remote_code (bool): Whether to trust remote code from Hugging Face models. + """ + + path: str = "~/models/deepseek-llm-7b-chat" + tokenizer_path: Optional[str] = None + override_config: dict[str, Any] = field(default_factory=dict) + external_lib: Optional[str] = None + trust_remote_code: bool = False diff --git a/e2c/config/critic/critic.yaml b/e2c/config/critic/critic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f201a34b40c36c5a45f3dda61084bc7dc78dd70a --- /dev/null +++ b/e2c/config/critic/critic.yaml @@ -0,0 +1,176 @@ +# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs +_target_: verl.workers.config.CriticConfig + +# Number of rollouts per update (mirrors actor rollout_n) +rollout_n: ${oc.select:actor_rollout_ref.rollout.n,1} + +# fsdp or fsdp2 strategy used for critic model training +strategy: ??? + +# whether to enable the critic worker. +# by default it is only enabled if advantage estimator is gae +# set it to True manually if you always want to enable critic worker +enable: null + +# optimizer configs +optim: + + # Learning rate + lr: 1e-5 + + # Warmup steps ratio; total steps will be injected at runtime + lr_warmup_steps_ratio: 0.0 + + # Total training steps (must be overridden at runtime) + total_training_steps: -1 + + # Weight decay + weight_decay: 0.01 + + # Prioritized. None, 0 or Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps: -1 + + +# model config for the critic +model: + + # Path to pretrained model weights + path: ~/models/deepseek-llm-7b-chat + + # Tokenizer path (defaults to actor's model path) + tokenizer_path: ${oc.select:actor_rollout_ref.model.path,"~/models/deepseek-llm-7b-chat"} + + # Hugging Face config override + override_config: {} + + # External model implementation (optional) + external_lib: ${oc.select:actor_rollout_ref.model.external_lib,null} + + # Whether to trust remote code from Hugging Face models + trust_remote_code: ${oc.select:actor_rollout_ref.model.trust_remote_code,false} + +# PPO mini-batch size per update +ppo_mini_batch_size: ${oc.select:actor_rollout_ref.actor.ppo_mini_batch_size,256} + +# [Deprecated] Global micro batch size +ppo_micro_batch_size: null + +# Local per-GPU micro batch size +ppo_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size,null} + +# Whether to automatically adjust batch size at runtime +use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + +# Max tokens per GPU in one PPO batch (doubled for critic) +ppo_max_token_len_per_gpu: 32768 + +# Max token length per GPU in forward pass +forward_max_token_len_per_gpu: ${.ppo_max_token_len_per_gpu} + +# Number of PPO epochs per batch +ppo_epochs: ${oc.select:actor_rollout_ref.actor.ppo_epochs,1} + +# Shuffle training data across PPO epochs +shuffle: ${oc.select:actor_rollout_ref.actor.shuffle,false} + +# PPO value function clipping range +cliprange_value: 0.5 + +# Loss aggregation mode: "token-mean", "seq-mean-token-sum", or "seq-mean-token-mean" +loss_agg_mode: ${oc.select:actor_rollout_ref.actor.loss_agg_mode,token-mean} + +# checkpoint configs +checkpoint: + + # Target dataclass for this configuration + _target_: verl.trainer.config.CheckpointConfig + + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + + # What to include when loading checkpoints + load_contents: ${.save_contents} + + # Whether to save checkpoints asynchronously. Only effective for Megatron as of now. + async_save: False + +# profile the critic model in `update_critic` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # profiler tool, default same as profiler.tool in global config + # choices: nsys, npu, torch, torch_memory + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on Critic + enable: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config which only related to the role + tool_config: + + # nsys tool config + nsys: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NsightToolConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + + # npu config + npu: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NPUToolConfig + + # Contents to profile, can be empty + # options: npu, cpu, memory, shapes, module, stack + contents: [] + + # Collection level, optional values: level_none, level0, level1, level2. + level: "level1" + + # Whether to automatically parse the data. + analysis: True + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # torch profiler config + torch: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + + # start profile mini-batch in training + # NOTICE: different with global steps config which refers to iteration + # This field only related with mini-batch + step_start: 0 + + # stop profile mini-batch in training + step_end: null + + # torch memory profiler config + torch_memory: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + + # Maximum number of memory allocation entries to track + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + + # Stack trace depth for memory allocations + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + \ No newline at end of file diff --git a/e2c/config/critic/dp_critic.yaml b/e2c/config/critic/dp_critic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..240cc2de0a7f8f322be2aaca34811e9bf66e953e --- /dev/null +++ b/e2c/config/critic/dp_critic.yaml @@ -0,0 +1,65 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# defaults specify the default config from each component +defaults: + + # fsdp optimizer config + - ../optim@optim: fsdp + + # fsdp engine config + - ../engine@model.fsdp_config: fsdp + + # dp actor config, inheriting from trainer/config/critic/critic.yaml + - critic + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs +_target_: verl.workers.config.FSDPCriticConfig + +# distribution strategy. Options: fsdp (deprecating), fsdp2 +strategy: fsdp + +# model config for the critic +model: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.FSDPCriticModelCfg + + # Whether to use shared memory for loading the model + use_shm: False + + # Enable gradient checkpointing to save memory + enable_gradient_checkpointing: True + + # Offload activations to CPU to reduce GPU memory usage + enable_activation_offload: False + + # Use remove padding optimization (saves compute) + use_remove_padding: False + + # Set to positive value to enable LoRA (e.g., 32) + lora_rank: 0 + + # LoRA scaling factor + lora_alpha: 16 + + # LoRA target modules: "all-linear" or list of linear projection layers + target_modules: all-linear + +# Forward-only batch size during inference (global) +forward_micro_batch_size: ${oc.select:.ppo_micro_batch_size,null} + +# Forward-only batch size during inference (per GPU) +forward_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size_per_gpu,null} + +# Sequence parallelism size for Ulysses-style model parallelism +ulysses_sequence_parallel_size: 1 + +# Gradient clipping for critic updates +grad_clip: 1.0 diff --git a/e2c/config/critic/megatron_critic.yaml b/e2c/config/critic/megatron_critic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4a8509f2278a7ad845b7659196792683b6eaeb5 --- /dev/null +++ b/e2c/config/critic/megatron_critic.yaml @@ -0,0 +1,43 @@ +# defaults specify the default config from each component +defaults: + + # megatron optimizer config + - ../optim@optim: megatron + + # megatron engine config + - ../engine@megatron: megatron + + # dp actor config, inheriting from trainer/config/critic/critic.yaml + - critic + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs +_target_: verl.workers.config.McoreCriticConfig + +strategy: megatron + +# seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using megatron +nccl_timeout: 600 + +# model config for the critic +model: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.BaseModelConfig + + # override default empty mapping + override_config: + + model_config: {} + + moe_config: + + freeze_moe_router: False + +# Whether to load initial weights +load_weight: True + +# seed for data loader +data_loader_seed: ${oc.select:actor_rollout_ref.actor.data_loader_seed,null} diff --git a/e2c/config/data/legacy_data.yaml b/e2c/config/data/legacy_data.yaml new file mode 100644 index 0000000000000000000000000000000000000000..028405b4238e5e344fb1fa3f1f0a4b1a83403b69 --- /dev/null +++ b/e2c/config/data/legacy_data.yaml @@ -0,0 +1,112 @@ +# Tokenizer class or path. If null, it will be inferred from the model. +tokenizer: null + +# Whether to use shared memory for data loading. +use_shm: False + +# Training set parquet. Can be a list or a single file. +# The program will read all files into memory, so it can't be too large (< 100GB). +# The path can be either a local path or an HDFS path. +# For HDFS path, we provide utils to download it to DRAM and convert it to a local path. +train_files: ~/data/rlhf/gsm8k/train.parquet + +# Validation parquet. Can be a list or a single file. +val_files: ~/data/rlhf/gsm8k/test.parquet + +# The field in the dataset where the prompt is located. Default is 'prompt'. +prompt_key: prompt + +# The field used to select the reward function (if using different ones per example). +reward_fn_key: data_source + +# Maximum prompt length. All prompts will be left-padded to this length. +# An error will be reported if the length is too long. +# oc.select: default val for rollout.prompt_length +max_prompt_length: 512 + +# Maximum response length. Rollout in RL algorithms (e.g. PPO) generates up to this length. +# oc.select: default val for rollout.response_length +max_response_length: 512 + +# Batch size sampled for one training iteration of different RL algorithms. +train_batch_size: 1024 + +# Batch size used during validation. Can be null. +val_batch_size: null + +# Whether to return the original input_ids without adding chat template. +# This is used when the reward model's chat template differs from the policy. +# If using a model-based RM with different templates, this should be True. +return_raw_input_ids: False + +# Whether to return the original chat (prompt) without applying chat template. +return_raw_chat: False + +# Whether to return the full prompt with chat template. +return_full_prompt: False + +# Whether to shuffle the data in the dataloader. +shuffle: True + +# num dataloader workers +dataloader_num_workers: 8 + +# Whether to shuffle the validation set. +validation_shuffle: False + +# Whether to filter overlong prompts. +filter_overlong_prompts: False + +# Number of workers for filtering overlong prompts. +# For large-scale datasets, filtering can be time-consuming. +# Use multiprocessing to speed up. Default is 1. +filter_overlong_prompts_workers: 1 + +# Truncate the input_ids or prompt if they exceed max_prompt_length. +# Options: 'error', 'left', 'right', 'middle'. Default is 'error'. +truncation: error + +# The field in the multi-modal dataset where the image is located. Default is 'images'. +image_key: images + +# The field in the multi-modal dataset where the video is located. +video_key: videos + +# If the remote tokenizer has a Python file, this flag determines whether to allow using it. +trust_remote_code: False + +# Optional: specify a custom dataset class path and name if overriding default loading behavior. +custom_cls: + + # The path to the file containing your customized dataset class. If not specified, pre-implemented dataset will be used. + path: null + + # The name of the dataset class within the specified file. + name: null + +# Whether to return multi-modal inputs in the dataset. Set to False if rollout generates new multi-modal inputs. +return_multi_modal_inputs: True + +# settings related to data sampler +sampler: + + # the path to the module containing a curriculum class which implements the + # AbstractSampler interface + class_path: null + + # the name of the curriculum class like `MySampler` + class_name: null + +# Data generation configuration for augmenting the dataset. +datagen: + + # The path to the file containing your customized data generation class. + # E.g. 'pkg://verl.experimental.dynamic_dataset.dynamicgen_dataset' + path: null + + # The class name of the data generation class within the specified file. + # E.g. 'MockDataGenerator' + name: null + +# Additional kwargs when calling tokenizer.apply_chat_template +apply_chat_template_kwargs: {} diff --git a/e2c/config/engine/fsdp.yaml b/e2c/config/engine/fsdp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3f551adb669bff28a4254ef45d97b8d0a57d398 --- /dev/null +++ b/e2c/config/engine/fsdp.yaml @@ -0,0 +1,53 @@ +# Target class for this configuration +_target_: verl.workers.config.FSDPEngineConfig + +# policy for wrapping the model +wrap_policy: + + # Minimum number of parameters to trigger wrapping a layer with FSDP + min_num_params: 0 + +# Whether to offload model parameters to CPU (trades speed for memory) +# Note that this differs from the offload_policy in FSDP +param_offload: false + +# Whether to offload optimizer state to CPU +# Note that this differs from the offload_policy in FSDP +optimizer_offload: false + +# Only for FSDP2: offload param/grad/optimizer during train +offload_policy: false + +# Only for FSDP2: Reshard after forward pass to reduce memory footprint +reshard_after_forward: true + +# Number of GPUs in each FSDP shard group; -1 means auto +fsdp_size: -1 + +# Only for FSDP1: FSDP1 configuration, prefetch the next forward-pass all-gather +# before the current forward computation. +forward_prefetch: False + +# model dtype of fsdp +model_dtype: fp32 + +# Whether to use original parameters in fsdp. Only avaiable in fsdp1 +use_orig_params: false + +# ulysses sequence parallel size +ulysses_sequence_parallel_size: 1 + +# Whether to use entropy_from_logits_with_chunking in fsdp. +entropy_from_logits_with_chunking: false + +# Whether to use torch compile in fsdp. +use_torch_compile: true + +# Whether to use entropy checkpointing in fsdp. +entropy_checkpointing: false + +# Whether to use forward only in fsdp. +forward_only: false + +# fsdp or fsdp2 +strategy: fsdp diff --git a/e2c/config/engine/megatron.yaml b/e2c/config/engine/megatron.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9bb4ef01fb1e31eea6e645bbb2a7bcbc3824b59 --- /dev/null +++ b/e2c/config/engine/megatron.yaml @@ -0,0 +1,78 @@ +# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs +_target_: verl.workers.config.McoreEngineConfig + +# Whether to offload model parameters to CPU +param_offload: False + +# Whether to offload gradients to CPU +grad_offload: False + +# Whether to offload optimizer state to CPU +optimizer_offload: False + +# tensor model parallel size +tensor_model_parallel_size: 1 + +# expert model parallel size +expert_model_parallel_size: 1 + +# expert tensor parallel size +expert_tensor_parallel_size: 1 + +# pipeline model parallel size +pipeline_model_parallel_size: 1 + +# virtual pipeline model parallel size +virtual_pipeline_model_parallel_size: null + +# context parallel size +context_parallel_size: 1 + +# sequence parallel +sequence_parallel: True + +# Whether to use distributed optimizer +use_distributed_optimizer: True + +# Whether to use distributed checkpointing +use_dist_checkpointing: False + +# distributed checkpointing path +dist_checkpointing_path: null + +# oc.select: default val for ref.megatron.seed +seed: 42 + +# Allow to override Distributed Data Parallel (DDP) config +override_ddp_config: {} + +# additional transformer config like: num_layers_in_first(/last)_pipeline_stage +# oc.select: default val for ref.megatron.override_transformer_config +override_transformer_config: + # Recompute configuration, same as in megatron.training.arguments + # default use minimal performance-interference recompute methods + # Recompute granualarity, choices: ["full", "selective"] + recompute_granularity: null + + # Recompute modules, multiple choices: ["core_attn", "moe_act", "layernorm", "mla_up_proj", "mlp", "moe"] + # Please use correct module in matched model + recompute_modules: ["core_attn"] + + # 'uniform', 'block' + # 'uniform' divides the total number of transformer layers and checkpoints the input activation of each chunk + # 'block' checkpoints the specified number of layers per pipeline stage at the specified granularity + recompute_method: null + + # 'full' will checkpoint the entire transformer layer and 'selective' only checkpoints memory intensive part of attention + recompute_num_layers: null + + # Attention backend to use (flash,fused,unfused,local,auto). Defaults to auto in mcore, flash in verl + attention_backend: flash + +override_mcore_model_config: {} + +# oc.select: default val for ref.megatron.use_mbridge +use_mbridge: False + +# whether to use forward only +forward_only: False diff --git a/e2c/config/evaluation.yaml b/e2c/config/evaluation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a88d77f1e73b6c3cce1972f639fcafb412669fa --- /dev/null +++ b/e2c/config/evaluation.yaml @@ -0,0 +1,15 @@ +data: + path: /tmp/math_Qwen2-7B-Instruct.parquet + prompt_key: prompt + response_key: responses + data_source_key: data_source + reward_model_key: reward_model + +custom_reward_function: + path: null + name: compute_score + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. + timeline_json_file: null diff --git a/e2c/config/generation.yaml b/e2c/config/generation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a57858ff4bd65a88b370cffe7450b962f21b24fe --- /dev/null +++ b/e2c/config/generation.yaml @@ -0,0 +1,57 @@ +trainer: + nnodes: 1 + n_gpus_per_node: 8 + device: cuda + +data: + path: ~/data/rlhf/math/test.parquet + prompt_key: prompt + n_samples: 5 + output_path: /opt/tiger/math_Qwen2-7B-Instruct.parquet + batch_size: 128 + +model: + path: ~/models/Qwen2-7B-Instruct + external_lib: null +rollout: + _target_: verl.workers.config.RolloutConfig + name: vllm + mode: sync # sync: LLM, async: AsyncLLM + temperature: 1.0 + top_k: 50 # 0 for hf rollout, -1 for vllm rollout + top_p: 0.7 + prompt_length: 1536 + response_length: 512 + # for vllm rollout + dtype: bfloat16 # should align with FSDP + gpu_memory_utilization: 0.5 + ignore_eos: False + enforce_eager: True + free_cache_engine: True + load_format: auto + tensor_model_parallel_size: 1 + max_num_batched_tokens: 8192 + max_model_len: null + max_num_seqs: 1024 + log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu + log_prob_micro_batch_size_per_gpu: 8 + # for hf rollout + do_sample: True + disable_log_stats: True + enable_chunked_prefill: True + n: 1 + # support logging rollout prob for debugging purpose + calculate_log_probs: False +actor: + strategy: fsdp # This is for backward-compatibility + ulysses_sequence_parallel_size: 1 # sp size + entropy_from_logits_with_chunking: False # calculate entropy with chunking to reduce memory peak + entropy_checkpointing: False # recompute entropy + fsdp_config: + fsdp_size: -1 + forward_prefetch: False # FSDP1 forward_prefetch configuration + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. + timeline_json_file: null diff --git a/e2c/config/model/hf_model.yaml b/e2c/config/model/hf_model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac08c1bb7fd46b9bbd47a3f114690ed23a8daddb --- /dev/null +++ b/e2c/config/model/hf_model.yaml @@ -0,0 +1,64 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +_target_: verl.workers.config.HFModelConfig + +# path to the huggingface model +path: ~/models/deepseek-llm-7b-chat + +# config to the huggingface config. In case it is not the same as path +hf_config_path: null + +# path to the huggingface tokenizer. In case it is not the same as path +tokenizer_path: null + +# whether to use shared memory for model loading +use_shm: False + +# whether to trust remote code. +trust_remote_code: False + +# custom chat template for the model +custom_chat_template: null + +# whether to use external libs for the model +external_lib: null + +# override hf config +override_config: {} + +# whether to enable gradient checkpointing. Only valid when we use hf model definition +enable_gradient_checkpointing: True + +# whether to enable activation offload. Only valid when we use hf model definition +enable_activation_offload: False + +# whether to use remove padding. Only valid when we use hf model definition +use_remove_padding: False + +# Set to positive value to enable LoRA (e.g., 32) +lora_rank: 0 + +# LoRA scaling factor +lora_alpha: 16 + +# Target modules for LoRA adaptation +target_modules: all-linear + +# Exclude modules from LoRA adaptation +exclude_modules: null + +# whether to use liger. Only valid when we use hf model definition +use_liger: False + +# whether to use fused kernels. +use_fused_kernels: False + +# fused kernel options. +fused_kernel_options: + + # the implementation backend for fused kernels. + impl_backend: torch diff --git a/e2c/config/npu_profile/npu_profile.yaml b/e2c/config/npu_profile/npu_profile.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52bb52d3f40d7d6695708b7414c82c0136d8fba2 --- /dev/null +++ b/e2c/config/npu_profile/npu_profile.yaml @@ -0,0 +1,34 @@ +# Options for the npu profiler +options: + + # Storage path of collected data. + save_path: ./profiler_data + + # The roles that will be profiled. Only takes effect in discrete mode. + # optional values: all, rollout_generate, actor_compute_log_prob, actor_update and ref_compute_log_prob. + # "all" means all roles will be profiled. + roles: ["all"] + + # Collection level, optional values: level_none, level0, level1, level2. + level: level1 + + # Whether to enable memory analysis. + with_memory: False + + # Whether to record tensor shape. + record_shapes: False + + # Whether to record Device-side performance data. + with_npu: True + + # Whether to record Host-side performance data. + with_cpu: True + + # Whether to record Python call stack information. + with_module: False + + # Whether to record operator call stack information. + with_stack: False + + # Whether to automatically parse the data. + analysis: True \ No newline at end of file diff --git a/e2c/config/optim/fsdp.yaml b/e2c/config/optim/fsdp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2ca6b2e8da31fbd060f262a56e4c2e20fb17092 --- /dev/null +++ b/e2c/config/optim/fsdp.yaml @@ -0,0 +1,33 @@ +# Target class for this configuration +_target_: verl.workers.config.FSDPOptimizerConfig + +# Learning rate +lr: 1e-3 + +# LR warmup steps ratio +lr_warmup_steps_ratio: 0.0 + +# Total training steps +total_training_steps: -1 + +# Weight decay +weight_decay: 0.01 + +# LR warmup steps +lr_warmup_steps: -1 + +# Betas for Adam optimizer +betas: [0.9, 0.999] + +# Clip gradient +clip_grad: 1.0 + +# Minimum LR ratio for cosine schedule +min_lr_ratio: 0.0 + +# Number of cosine cycles in LR schedule +num_cycles: 0.5 + +# LR warmup style: "constant" or "cosine" +warmup_style: constant + diff --git a/e2c/config/optim/megatron.yaml b/e2c/config/optim/megatron.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3e49b7df8e59d33f51b50b943d9353af66d296c --- /dev/null +++ b/e2c/config/optim/megatron.yaml @@ -0,0 +1,49 @@ +_target_: verl.workers.config.McoreOptimizerConfig + +# Learning rate +lr: 1e-3 + +# LR warmup steps ratio +lr_warmup_steps_ratio: 0.0 + +# Total training steps +total_training_steps: -1 + +# Weight decay +weight_decay: 0.01 + +# LR warmup steps +lr_warmup_steps: -1 + +# Betas for Adam optimizer +betas: [0.9, 0.999] + +# Clip gradient +clip_grad: 1.0 + +# optimizer type +optimizer: adam + +# initial learning rate for warmup, default to 0.0 +lr_warmup_init: 0.0 + +lr_decay_steps: null + +# select from constant/linear/cosine/inverse_square_root +lr_decay_style: constant + +# minimum learning rate, default to 0.0 +min_lr: 0.0 + +# select from constant/linear/cosine +weight_decay_incr_style: constant + +# select from constant/exponential/cosine +lr_wsd_decay_style: exponential + +lr_wsd_decay_steps: null + +# use checkpoint optimizer parameter scheduler +use_checkpoint_opt_param_scheduler: False + +override_optimizer_config: {} diff --git a/e2c/config/ppo_megatron_trainer.yaml b/e2c/config/ppo_megatron_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..238522ebcecfb77c13c44c5bd5e1ed4e6e8a1b86 --- /dev/null +++ b/e2c/config/ppo_megatron_trainer.yaml @@ -0,0 +1,173 @@ +# specify the default per-component configs +defaults: + # @.: + # actor_rollout_ref.actor: trainer/config/actor/megatron_actor.yaml + - actor@actor_rollout_ref.actor: megatron_actor + # data: trainer/config/data/legacy_data.yaml + - data@data: legacy_data + # load the reference default config, then apply the fields in the current yaml + # Reference model config. + # Reference model will be enabled when actor.use_kl_loss or/and algorithm.use_kl_in_reward is/are True. + - ref@actor_rollout_ref.ref: megatron_ref + # Rollout model config. + - rollout@actor_rollout_ref.rollout: rollout + # Critic model config. + - critic@critic: megatron_critic + # Reward model config. + - reward_model@reward_model: megatron_reward_model + - _self_ + +actor_rollout_ref: + hybrid_engine: True + + nccl_timeout: 600 # seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using megatron + + model: + + path: ~/models/deepseek-llm-7b-chat + + custom_chat_template: null + + external_lib: null + + override_config: + model_config: {} + + moe_config: + freeze_moe_router: False + + use_fused_kernels: False # Whether to use custom fused kernels (PostProcessing, for memory efficiency) + + trust_remote_code: False + + # Whether to remove padding tokens in inputs during training + use_remove_padding: false + + rollout: + layer_name_map: + qkv_layer_name: qkv + gate_proj_layer_name: gate_up + +custom_reward_function: + path: null + name: compute_score + +algorithm: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.AlgoConfig + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + norm_adv_by_std_in_grpo: True + use_kl_in_reward: False + kl_penalty: kl # how to estimate kl divergence + kl_ctrl: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.KLControlConfig + type: fixed + kl_coef: 0.001 + horizon: 10000 + target_kl: 0.1 + use_pf_ppo: False + pf_ppo: + reweight_method: pow # ["pow", "max_min", "max_random"] + weight_pow: 2.0 + +trainer: + balance_batch: True + total_epochs: 30 + total_training_steps: null + project_name: verl_examples + experiment_name: gsm8k + logger: ["console", "wandb"] + log_val_generations: 0 + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + esi_redundant_time: 0 + + # auto: find the last ckpt to resume. If can't find, start from scratch + resume_mode: auto # or disable or resume_path if resume_from_path is set + resume_from_path: null + del_local_ckpt_after_load: False + val_before_train: True + test_freq: -1 + critic_warmup: 0 + default_hdfs_dir: null + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + max_actor_ckpt_to_keep: null + max_critic_ckpt_to_keep: null + # The timeout for ray worker group to wait for the register center to be ready + ray_wait_register_center_timeout: 300 + device: cuda + # Directory for logging rollout data; no dump if null + rollout_data_dir: null + +global_profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: null # choose between nsys, npu, torch, torch_memory + steps: null # profile steps + profile_continuous_steps: False + save_path: "outputs/profile" # profiler saving path + # Specific tool configs, can use +profiler.tool_config.[tool].xxx to config + global_tool_config: + # nsys config + nsys: + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # controller Nvidia Nsight Systems Options. Must set when profile_steps is not None. + ## reference https://docs.nvidia.com/nsight-systems/UserGuide/index.html + ## reference https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html + controller_nsight_options: + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # worker Nvidia Nsight Systems Options. Must set when profile_steps is not None. + worker_nsight_options: + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # Profiling only in a range of torch.cuda.profiler.start and stop. Do not change this config. + capture-range: "cudaProfilerApi" + + # Specify the desired behavior when a capture range ends. + # In verl we need the torch.cuda.profiler.start/stop pair to repeats n times. + # valid values are "repeat-shutdown:n" or null. + # For normal whole step profiling, n = len(profile_steps); + # but for discrete profiling, n = len(profile_steps) * Number(subtasks). + # Or you can just leave it null and the program will use n = len(profile_steps) * 6; + capture-range-end: null + + # Send signal to the target application's process group. We let the program to exit by itself. + kill: none + + # enable memory visualization for debugging memory usage + torch_memory: + # Maximum number of allocation entries to record + trace_alloc_max_entries: 100_000 + # The depth of the call stack to capture for each allocation + stack_depth: 32 + # 'alloc': records only allocation events || 'state': records memory state changes || 'all': records both. + context: "all" + # 'python': records Python stacks || 'cpp': records C++ stacks (available in some versions) || 'all': records both. + stacks: "all" + # devices, record_context etc. + kw_args: {} + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. + timeline_json_file: null diff --git a/e2c/config/ppo_trainer.yaml b/e2c/config/ppo_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8214833545f10ba92fcbc57afabaac4feaf32ac --- /dev/null +++ b/e2c/config/ppo_trainer.yaml @@ -0,0 +1,308 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# specify the default per-component configs +defaults: + + # @.: + # actor_rollout_ref.actor: trainer/config/actor/dp_actor.yaml + - actor@actor_rollout_ref.actor: dp_actor + + # data: trainer/config/data/legacy_data.yaml + - data@data: legacy_data + + # Reference model config. + # Reference model will be enabled when actor.use_kl_loss or/and algorithm.use_kl_in_reward is/are True. + - ref@actor_rollout_ref.ref: dp_ref + + # Rollout model config. + - rollout@actor_rollout_ref.rollout: rollout + + # Model config. + - model@actor_rollout_ref.model: hf_model + + # Critic model config. + - critic@critic: dp_critic + + # Reward model config. + - reward_model@reward_model: dp_reward_model + + # load the reference default config, then apply the fields in the current yaml + # self config override anything above + - _self_ + +# config for actor, rollout and reference model +actor_rollout_ref: + + # Whether it's a hybrid engine, currently only supports hybrid engine + hybrid_engine: true + + # Timeout for operations executed against the process group + nccl_timeout: 600 + + # Rollout model config. + rollout: + + # for huge model, layered summon can save memory (prevent OOM) but make it slower + layered_summon: False + +# custom reward function definition +custom_reward_function: + + # The path to the file containing your customized reward function. + # If not specified, pre-implemented reward functions will be used. + path: null + + # The name of the reward function within the specified file. Default is 'compute_score'. + name: compute_score + +# config for the algorithm +algorithm: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.AlgoConfig + + # Discount factor for future rewards + gamma: 1.0 + + # Trade-off between bias and variance in the GAE estimator + lam: 1.0 + + # Advantage estimator type: "gae", "grpo", "reinforce_plus_plus", etc. + adv_estimator: gae + + # Whether to normalize advantages by std (specific to GRPO) + norm_adv_by_std_in_grpo: True + + # Whether to enable in-reward KL penalty + use_kl_in_reward: False + + # How to estimate KL divergence: "kl", "abs", "mse", "low_var_kl", or "full" + kl_penalty: kl + + # KL control configuration + kl_ctrl: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.KLControlConfig + + # KL control type: "fixed" or "adaptive" + type: fixed + + # Initial coefficient for KL penalty + kl_coef: 0.001 + + # Horizon value for adaptive controller (if enabled) + horizon: 10000 + + # Target KL divergence (used for adaptive controller) + target_kl: 0.1 + + # Whether to enable preference feedback PPO + use_pf_ppo: False + + # Preference feedback PPO settings + pf_ppo: + + # Method for reweighting samples: "pow", "max_min", or "max_random" + reweight_method: pow + + # Power used for weight scaling in "pow" method + weight_pow: 2.0 + +# config for the trainer +trainer: + + # Whether to balance batch sizes across distributed workers + balance_batch: True + + # Number of epochs in training + total_epochs: 30 + + # Total training steps (can be set explicitly or derived from epochs) + total_training_steps: null + + # Project name for experiment tracking (e.g., wandb) + project_name: verl_examples + + # Experiment name for run identification in tracking tools + experiment_name: gsm8k + + # Logging backends to use: "console", "wandb", etc. + logger: ["console", "wandb"] + + # Number of generations to log during validation + log_val_generations: 0 + + # Directory for logging rollout data; no dump if null + rollout_data_dir: null + + # Directory for logging validation data; no dump if null + validation_data_dir: null + + # Number of nodes used in the training + nnodes: 1 + + # Number of GPUs per node + n_gpus_per_node: 8 + + # Save frequency (by iteration) for model checkpoints + save_freq: -1 + + # ESI refers to the elastic server instance used during training, similar to the training plan. For example, + # if you purchase 10 hours of computing power, the ESI will automatically shut down after 10 hours of training. + # To ensure a checkpoint is saved before ESI shuts down, the system will start saving a checkpoint in advance. + # The advance time is calculated as: Advance Time = Longest historical step duration + Checkpoint save duration + esi_redundant_time. + # Here, esi_redundant_time is a user-defined value that further extends the advance time for added safety. + esi_redundant_time: 0 + + # Resume mode: "auto", "disable", or "resume_path" + # "auto": resume from last checkpoint if available + # "disable": start from scratch + # "resume_path": resume from a user-defined path + resume_mode: auto + + # Path to resume training from (only used when resume_mode is "resume_path") + resume_from_path: null + + # Whether to run validation before training begins + val_before_train: True + + # Whether to run validation only + val_only: False + + # Validation frequency (in training iterations) + test_freq: -1 + + # Number of iterations to warm up the critic before updating policy + critic_warmup: 0 + + # Default path to distributed filesystem for saving checkpoints + default_hdfs_dir: null + + # Whether to delete local checkpoints after loading + del_local_ckpt_after_load: False + + # Default local directory for saving checkpoints + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + + # Maximum number of actor checkpoints to keep + max_actor_ckpt_to_keep: null + + # Maximum number of critic checkpoints to keep + max_critic_ckpt_to_keep: null + + # Timeout (in seconds) for Ray worker to wait for registration + ray_wait_register_center_timeout: 300 + + # Device to run training on (e.g., "cuda", "cpu") + device: cuda + + # whether to use legacy worker implementation + # mode: "auto", "enable", or "disable" + use_legacy_worker_impl: auto + +# profiler configs +global_profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # Profiling tool: choose between nsys, npu, torch, torch_memory + tool: null + + # profile steps + steps: null + + # Whether to combine continuous steps into one database. + ## If True, worker.profiler.discrete must be False, [1,2] in one, [5] in another. + ## If False, [1] in one, [2] in another, [5] in another. + profile_continuous_steps: False + + # Path to save profiling contents + save_path: "outputs/profile" + + # Specific tool configs, can use +profiler.tool_config.[tool].xxx to config + global_tool_config: + + # nsys config + nsys: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NsightToolConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # controller Nvidia Nsight Systems Options. Must set when profile_steps is not None. + ## reference https://docs.nvidia.com/nsight-systems/UserGuide/index.html + ## reference https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html + controller_nsight_options: + + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # worker Nvidia Nsight Systems Options. Must set when profile_steps is not None. + worker_nsight_options: + + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # Profiling only in a range of torch.cuda.profiler.start and stop. Do not change this config. + capture-range: "cudaProfilerApi" + + # Specify the desired behavior when a capture range ends. + # In verl we need the torch.cuda.profiler.start/stop pair to repeats n times. + # valid values are "repeat-shutdown:n" or null. + # For normal whole step profiling, n = len(profile_steps); + # but for discrete profiling, n = len(profile_steps) * Number(subtasks). + # Or you can just leave it null and the program will use n = len(profile_steps) * 6; + capture-range-end: null + + # Send signal to the target application's process group. We let the program to exit by itself. + kill: none + + # enable memory visualization for debugging memory usage + torch_memory: + + # Maximum number of allocation entries to record + trace_alloc_max_entries: 100_000 + + # The depth of the call stack to capture for each allocation + stack_depth: 32 + + # 'alloc': records only allocation events || 'state': records memory state changes || 'all': records both. + context: "all" + + # 'python': records Python stacks || 'cpp': records C++ stacks (available in some versions) || 'all': records both. + stacks: "all" + + # devices, record_context etc. + kw_args: {} + +# configs related to ray +ray_kwargs: + + # configs related to ray initialization + ray_init: + + # Number of CPUs for Ray. Use a fixed number instead of null when using SLURM. + num_cpus: null + + # Path to save Ray timeline JSON for performance profiling + timeline_json_file: null diff --git a/e2c/config/ref/dp_ref.yaml b/e2c/config/ref/dp_ref.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d4ae0c882df83c0f166fd5497f9178ccd6bcd7 --- /dev/null +++ b/e2c/config/ref/dp_ref.yaml @@ -0,0 +1,26 @@ +# defaults specify the default config from each component +defaults: + + # dp ref config, inheriting from trainer/config/ref/ref.yaml + - ref + + # fsdp engine config + - ../engine@fsdp_config: fsdp + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +# ref model is assumed to be identical to actor model. Specify model.path for using a different ref model. +# Potential use case involves on policy distillation where we calculate KL divergence between student actor +# and teacher ref +model: null + +# sequence parallel size +# same as actor_rollout_ref.actor.ulysses_sequence_parallel_size if it exists, otherwise 1 +ulysses_sequence_parallel_size: ${oc.select:actor_rollout_ref.actor.ulysses_sequence_parallel_size,1} + +# calculate entropy with chunking to reduce memory peak +entropy_from_logits_with_chunking: False + +# recompute entropy +entropy_checkpointing: False diff --git a/e2c/config/ref/megatron_ref.yaml b/e2c/config/ref/megatron_ref.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e2769f2b7ae159cb26dd01b41ccf25148900fb2 --- /dev/null +++ b/e2c/config/ref/megatron_ref.yaml @@ -0,0 +1,19 @@ +# megatron ref config, inheriting from trainer/config/ref/ref.yaml +defaults: + - ref + + # megatron engine config + - ../engine@megatron: megatron + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +strategy: megatron + +megatron: + _target_: verl.workers.config.MegatronEngineConfig + seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42} + override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}} + use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False} + +load_weight: True \ No newline at end of file diff --git a/e2c/config/ref/ref.yaml b/e2c/config/ref/ref.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb4c811683d8fb43d8428bb765c72af2f2d9b171 --- /dev/null +++ b/e2c/config/ref/ref.yaml @@ -0,0 +1,99 @@ +# actor_rollout_ref.ref: FSDP config same as actor. For models larger than 7B, it’s recommended to turn on offload for ref by default +strategy: ${actor_rollout_ref.actor.strategy} + +# whether to enable torch.compile +# same as actor_rollout_ref.actor.use_torch_compile if it exists, otherwise 1 +use_torch_compile: ${oc.select:actor_rollout_ref.actor.use_torch_compile,true} + +# [Will be deprecated, use log_prob_micro_batch_size_per_gpu] +# The batch size for one forward pass in the computation of log_prob. Global batch size. +log_prob_micro_batch_size: null + +# The batch size for one forward pass in the computation of log_prob. Local batch size per GPU. +log_prob_micro_batch_size_per_gpu: null + +# enable dynamic batch size (sequence packing) for log_prob computation +# same as actor_rollout_ref.actor.use_dynamic_bsz if it exists, otherwise false +log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + +# the max token length per GPU +# same as actor_rollout_ref.actor.ppo_max_token_len_per_gpu if it exists, otherwise 16384 +log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + +# profile the ref model in `compute_log_prob` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # choices: nsys, npu, torch, torch_memory + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on Ref + enable: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config which only related to the role + tool_config: + + # nsys tool config + nsys: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NsightToolConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + + # npu config + npu: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NPUToolConfig + + # Contents to profile, can be empty + # options: npu, cpu, memory, shapes, module, stack + contents: [] + + # Collection level, optional values: level_none, level0, level1, level2. + level: "level1" + + # Whether to automatically parse the data. + analysis: True + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # torch profiler config + torch: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + + # start profile mini-batch in training + # NOTICE: different with global steps config which refers to iteration + # This field only related with mini-batch + step_start: 0 + + # stop profile mini-batch in training + step_end: null + + # torch memory profiler config + torch_memory: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + + # Maximum number of memory allocation entries to track + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + + # Stack trace depth for memory allocations + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} \ No newline at end of file diff --git a/e2c/config/reward_model/dp_reward_model.yaml b/e2c/config/reward_model/dp_reward_model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fff1f9f1f1d32100e77357781ee29a5728ef298c --- /dev/null +++ b/e2c/config/reward_model/dp_reward_model.yaml @@ -0,0 +1,55 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# defaults specify the default config from each component +defaults: + + # dp actor config, inheriting from trainer/config/reward_model/reward_model.yaml + - reward_model + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +strategy: fsdp + +model: + + # Whether to use shared memory for loading the model + use_shm: False + + # Use remove padding optimization (saves compute) + use_remove_padding: False + + # Whether to use fused reward kernels for speedup + use_fused_kernels: ${actor_rollout_ref.model.use_fused_kernels} + + # FSDP-specific config + fsdp_config: + + # Target configuration dataclass + _target_: verl.workers.config.FSDPEngineConfig + + # Policy for wrapping layers with FSDP + wrap_policy: + + # Minimum number of parameters to trigger wrapping + min_num_params: 0 + + # Whether to offload model parameters to CPU + param_offload: False + + # Only for FSDP2: Reshard after forward pass to reduce memory footprint + reshard_after_forward: True + + # Number of GPUs in each FSDP shard group; -1 means auto + fsdp_size: -1 + + # Only for FSDP1: FSDP1 configuration, prefetch the next forward-pass all-gather + # before the current forward computation. + forward_prefetch: False + +# Sequence parallelism size for Ulysses-style model parallelism +ulysses_sequence_parallel_size: 1 \ No newline at end of file diff --git a/e2c/config/reward_model/megatron_reward_model.yaml b/e2c/config/reward_model/megatron_reward_model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62a58d4b2dd31388560d8157980784c525ac659c --- /dev/null +++ b/e2c/config/reward_model/megatron_reward_model.yaml @@ -0,0 +1,65 @@ +# defaults specify the default config from each component +defaults: + + # dp actor config, inheriting from trainer/config/reward_model/reward_model.yaml + - reward_model + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +strategy: megatron + +# seconds, default is 10 minutes for torch, you can set it to a larger value +# if you have long-running operations like 32B or 72B model using megatron +nccl_timeout: 600 + +# Megatron parallelism & checkpointing config +megatron: + + # Target configuration dataclass + _target_: verl.workers.config.MegatronEngineConfig + + # Whether to offload model parameters to CPU + param_offload: False + + # Number of GPUs in tensor model parallel group + tensor_model_parallel_size: 1 + + # Number of GPUs in expert model parallel group + expert_model_parallel_size: 1 + + # Expert tensor parallel size + expert_tensor_parallel_size: 1 + + # Number of pipeline model parallel stages + pipeline_model_parallel_size: 1 + + # change VPP interface for parallelism tests + virtual_pipeline_model_parallel_size: null + + # Context parallel size + context_parallel_size: 1 + + # Whether to use sequence parallelism + sequence_parallel: True + + # Whether to use distributed optimizer + use_distributed_optimizer: False + + # Whether to enable distributed checkpointing + use_dist_checkpointing: False + + # Path for distributed checkpoints + dist_checkpointing_path: null + + # RNG seed for megatron + seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42} + + # Any overrides to transformer config + override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}} + + # Whether to use mbridge for faster comms + use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False} + +# Whether to load weights (default True) +load_weight: True \ No newline at end of file diff --git a/e2c/config/reward_model/reward_model.yaml b/e2c/config/reward_model/reward_model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9ffc60fbc614377b81579454e7f88023db70d91 --- /dev/null +++ b/e2c/config/reward_model/reward_model.yaml @@ -0,0 +1,97 @@ +# configs for the reward model + +# Whether to enable reward model. If False, we compute the reward only with the user-defined reward functions. +# In GSM8K and Math examples, we disable reward model. +# For RLHF alignment example using full_hh_rlhf, we utilize reward model to assess the responses. +# If False, the following parameters are not effective +enable: False + +# Whether to deploy the model to a separate resource pool. +# If true, n_gpus_per_node & nnodes will be used to determine the resource node. +enable_resource_pool: False +n_gpus_per_node: 0 +nnodes: 0 + +# FSDP strategy: "fsdp" or "fsdp2" +strategy: ??? + +# model config for reward scoring +model: + + # Input tokenizer. If the reward model's chat template is inconsistent with the policy, + # we need to first decode to plaintext, then apply the rm's chat_template. + # Then score with RM. If chat_templates are consistent, it can be set to null. + # set this to null if the chat template is identical + input_tokenizer: ${actor_rollout_ref.model.path} + + # RM’s HDFS path or local path. Note that RM only supports AutoModelForSequenceClassification. + # Other model types need to define their own RewardModelWorker and pass it from the code. + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + + # External model implementation (optional) + external_lib: ${actor_rollout_ref.model.external_lib} + + # Whether to enable loading a remote code model, default to False + trust_remote_code: False + +# [Deprecated] Global micro batch size +# will be deprecated, use micro_batch_size_per_gpu +micro_batch_size: null + +# Local per-GPU micro batch size +micro_batch_size_per_gpu: null + +# Maximum sequence length to process for scoring +max_length: null + +# Whether to dynamically adjust batch size at runtime +use_dynamic_bsz: ${critic.use_dynamic_bsz} + +# Maximum number of tokens per GPU in one forward pass +forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + +# Reward Manager. This defines the mechanism of computing rule-based reward and handling different reward sources. +# Default is naive. If all verification functions are multiprocessing-safe, +# the reward manager can be set to prime for parallel verification. +reward_manager: naive + +# Whether to launch custom reward function asynchronously during log_prob +# custom reward function executed async on CPU, during log_prob +launch_reward_fn_async: False + +# Cloud/local sandbox fusion configuration for custom reward logic +sandbox_fusion: + + # Cloud /local function URL for sandbox execution + url: null + + # Max concurrent requests allowed to sandbox + max_concurrent: 64 + + # Max memory limit for each sandbox process in MB + memory_limit_mb: 1024 + +# profile the reward model in `compute_reward` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # profiler tool, default same as profiler.tool in global config + # choices: nsys, npu, torch + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on ref + enable: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} \ No newline at end of file diff --git a/e2c/config/rollout/rollout.yaml b/e2c/config/rollout/rollout.yaml new file mode 100644 index 0000000000000000000000000000000000000000..366b553d840a51be8b4bda9cfefb5b1e18a16b66 --- /dev/null +++ b/e2c/config/rollout/rollout.yaml @@ -0,0 +1,291 @@ +# Target class for this configuration +_target_: verl.workers.config.RolloutConfig + +# actor_rollout_ref.rollout.name: hf/vllm/sglang. The default value will be removed in the future +name: ??? + +# sync: LLM, async: AsyncLLM +mode: sync + +# Sampling temperature for rollout. +temperature: 1.0 + +# Top-k sampling parameter. -1 for vLLM rollout, 0 for HF rollout. +top_k: -1 + +# Top-p sampling parameter. Default 1.0. +top_p: 1 + +# typically the same as data max prompt length +# same as data.max_prompt_length if it exists +prompt_length: ${oc.select:data.max_prompt_length,512} + +# typically the same as data max response length +# same as data.max_response_length if it exists +response_length: ${oc.select:data.max_response_length,512} + +# for vllm rollout +# Rollout model parameters type. Align with actor model's FSDP/Megatron type. +dtype: bfloat16 + +# Fraction of GPU memory used by vLLM/SGLang for KV cache. +gpu_memory_utilization: 0.5 + +# Whether to ignore EOS and continue generating after EOS is hit. +ignore_eos: False + +# Whether to disable CUDA graph. Default False to best performance. +enforce_eager: False + +# batch size of cudagraph to capture. Require enforce_eager: False to use this option +# Since cudagraph in inference engine can not be offloaded during update policy, +# you can use smaller batch size to save memory used in cuda graph, eg: [1 ,2, 4, 8, 16, 32] +# supported engines: vllm +cudagraph_capture_sizes: null + +# Whether to free engine KVCache after generation. +free_cache_engine: True + +# TP size for rollout. Not effective for hf +tensor_model_parallel_size: 2 + +# DP size for rollout +data_parallel_size: 1 + +# EP size for rollout +expert_parallel_size: 1 + +# max number of tokens in a batch +max_num_batched_tokens: 8192 + +# max length for rollout +max_model_len: null + +# max length of sequences +max_num_seqs: 1024 + +# may get higher throughput when set to True. When activated, Please increase max_num_batched_tokens or decrease max_model_len. +enable_chunked_prefill: True + +# Prefix caching kv-cache blocks is a popular optimization in LLM inference to avoid redundant prompt computations. +enable_prefix_caching: True + +# Which loader to use for rollout model weights: dummy, hf, megatron, etc. +# safetensors (for huge model, and set use_shm=True); dummy: randomly init model weight +load_format: dummy + +# [Will be deprecated, use log_prob_micro_batch_size_per_gpu] The batch size for one forward pass in the computation of log_prob. Global batch size. +log_prob_micro_batch_size: null + +# The batch size for one forward pass in the computation of log_prob. Local batch size per GPU. +log_prob_micro_batch_size_per_gpu: null + +# enable dynamic batch size (sequence packing) for log_prob computation +# same as actor_rollout_ref.actor.use_dynamic_bsz if it exists, otherwise false +log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + +# max token length for log_prob computation +# same as actor_rollout_ref.actor.ppo_max_token_len_per_gpu if it exists, otherwise 16384 +log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + +# disable logging statistics +disable_log_stats: True + +# for hf rollout +# Whether to sample during training rollout. False uses greedy sampling. +do_sample: True + +# number of responses (i.e. num sample times). > 1 for grpo +n: 1 + +# The over_sample_rate parameter controls the early termination threshold for training rollouts, +# where the system will abort remaining requests when (1 - over_sample_rate) * total_requests completions are reached. +over_sample_rate: 0 + +# Whether to wake up inference engine in multi-stage for SGLang +# to reduce peak memory during training-rollout transition. +# This is only effective for SGLang rollout. +multi_stage_wake_up: false + +# Extra inference engine arguments (vllm, sglang), please refer vllm/sglang official doc for detail +engine_kwargs: + + # vllm engine config + vllm: {} + + # sglang engine config + sglang: {} + +# Sampling parameters used during validation. +val_kwargs: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.SamplingConfig + + # sampling parameters for validation + # Top-k sampling parameter. -1 for vLLM rollout, 0 for HF rollout. + top_k: -1 + + # Top-p sampling parameter. Default 1.0. + top_p: 1.0 + + # Sampling temperature for rollout. + temperature: 0 + + # whether to repeat n times for validation + n: 1 + + # Whether to sample during training rollout. False uses greedy sampling. + do_sample: False + +# Multi-turn interaction config for tools or chat. +multi_turn: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.MultiTurnConfig + + # set to True for multi-turn tool interaction tasks; should set rollout.name to sglang as well + enable: False + + # null for no limit (default max_length // 3) + max_assistant_turns: null + + # null for no tool + tool_config_path: null + + # null for no limit (default max_length // 3) + max_user_turns: null + + # max parallel call for tools in single turn + max_parallel_calls: 1 + + # max length of tool response + max_tool_response_length: 256 + + # truncate side of tool response: left, middle, right + tool_response_truncate_side: middle + + # null for no interaction + interaction_config_path: null + + # - When set to True, the model's default chat template is used for multi-turn rollout, which typically matches production behavior. + # - When set to False, the token ids recorded for training are used instead; unlike the default chat template, these always include the model's full output, + # which may contain additional content such as reasoning content. This maintains the consistency between training and rollout, but it will lead to longer prompts. + use_inference_chat_template: False + + # Tokenization is performed turn by turn and the resulting token ids are concatenated to form the full conversation. + # To ensure this matches the result of tokenizing the entire conversation at once, a sanity check is run at the end of each multi-turn rollout to compare the two sets of token ids. + # Some models are known to produce different tokenization results when tokenizing turn by turn vs. all at once. aThis behavior has already been validated for them. + # To reduce excessive warnings, you can turn off the sanity check for these models if you are using their default chat template: + # Qwen/QwQ-32B, Qwen/Qwen3-xxB + # - disable: disable tokenization sanity check + # - strict: enable strict tokenization sanity check (default) + # - ignore_strippable: ignore strippable tokens when checking tokenization sanity + tokenization_sanity_check_mode: strict + + # Format of the multi-turn interaction. Options: hermes, llama3_json, ... + format: hermes + + # Number of repeat rollouts for each interaction + num_repeat_rollouts: null + +# support logging rollout prob for debugging purpose +# "Truncated importance sampling" requires rollout log probs, set to True when turning on Truncated importance sampling +calculate_log_probs: False + +# [Experimental] agent loop based rollout configs +agent: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.AgentLoopConfig + + # Number of agent loop workers + num_workers: 8 + + # custom agent loop config path, which should contain list of configs to intialize AgentLoop instances. + # https://hydra.cc/docs/advanced/instantiate_objects/overview/ + # + # - name: react_agent + # _target_: recipe.langgraph_agent.react_agent_loop.ReactAgentLoop + # tools: ["get_current_temperature"] + # - name: math_expression + # _target_: recipe.langgraph_agent.example.math_expression.MathExpressionReactAgentLoop + # min_terms: 2 + # max_terms: 6 + agent_loop_config_path: null + + # custom async server configs + custom_async_server: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.CustomAsyncServerConfig + + # Path to the custom async server implementation + path: null + + # Class name of the custom async server class (e.g. AsyncvLLMServer) + name: null + +# Specifies the tensor bucket size (in megabytes) for batch weight updates during rollout operations. +# This parameter controls the maximum payload size for a single weight update request. +# Reference: https://github.com/volcengine/verl/pull/2418 +# Currently only supported in SGLang rollout implementations +# Larger values may improve throughput but increase memory overhead +# Detailed performance comparison: +# https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/issues/169#issuecomment-3070686720 +# Default value (512MB) is optimized for typical GPU memory configurations +# For the best performance of `rebuild_cuda_tensor`, it is recommended to: +# 1. Enable `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES` +# 2. Manually set `CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7` +# when using Tensor Parallelism (TP) >= 8. +update_weights_bucket_megabytes: 512 + +# trace rollout data +trace: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.TraceConfig + + # trace backend, support mlflow, weave + backend: null + + # whether translate token id to text in output + token2text: False + +# When enabled (True), the trainer will attempt to load previously generated rollout data from the specified directory instead of computing new rollouts. +# If no cached data is found or loading fails, new rollouts will be generated and automatically saved. +# This feature is useful for debugging or when you want to reuse computation results across multiple runs. +skip_rollout: False + +# Specifies the filesystem path where rollout data should be cached when skip_rollout is enabled. +# Note: Giving path under /tmp/ray/session* is not recommended as these are temporary Ray cluster directories. +skip_dump_dir: /tmp/rollout_dump + +# Whether to skip tokenizer initialization for rollout engine +# When enabled (True), the rollout assume token in token out for generation +skip_tokenizer_init: True + +# profile the rollout model in `generate_sequence` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # profiler tool, default same as profiler.tool in global config + # choices: nsys, npu, torch + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on ref + enable: ${oc.select:actor_rollout_ref.actor.profiler.enable,false} + + # Whether to profile all ranks. + all_ranks: ${oc.select:actor_rollout_ref.actor.profiler.all_ranks,false} + + # The ranks that will be profiled. [] or [0,1,...] + ranks: ${oc.select:actor_rollout_ref.actor.profiler.ranks,[]} + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} diff --git a/e2c/config/sft_trainer.yaml b/e2c/config/sft_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..343fe7bfc156de54b1d5473fc7cde358cd733466 --- /dev/null +++ b/e2c/config/sft_trainer.yaml @@ -0,0 +1,83 @@ +data: + train_batch_size: 160 + micro_batch_size: null + micro_batch_size_per_gpu: 20 + train_files: ./data/e2c-sft-train.parquet + val_files: ./data/e2c-sft-val.parquet + prompt_key: question + response_key: solution + prompt_dict_keys: null + response_dict_keys: null + multiturn: + enable: false + messages_key: messages + tools_key: tools + enable_thinking_key: enable_thinking + max_length: 1000 + truncation: right + balance_dp_token: false + chat_template: null + custom_cls: + path: null + name: null + use_shm: false + apply_chat_template_kwargs: {} +model: + partial_pretrain: "Path to the pretrained model" + use_shm: false + fsdp_config: + model_dtype: bf16 + wrap_policy: + min_num_params: 0 + cpu_offload: false + offload_params: false + external_lib: null + enable_gradient_checkpointing: true + trust_remote_code: false + lora_rank: -1 + lora_alpha: 16 + target_modules: all-linear + use_liger: false + strategy: fsdp2 +optim: + lr: 1.0e-05 + betas: + - 0.9 + - 0.95 + weight_decay: 0.01 + warmup_steps_ratio: 0.1 + clip_grad: 1.0 + lr_scheduler: cosine +ulysses_sequence_parallel_size: 1 +use_remove_padding: false +trainer: + base_dir: ./training + default_local_dir: ./${trainer.base_dir}/${trainer.project_name}/${trainer.experiment_name} + default_hdfs_dir: null + project_name: e2c-sft + experiment_name: run1 + total_epochs: 3 + total_training_steps: null + logger: + - console + seed: 114514 + save_freq: 100 + test_freq: -1 + kl_coeff: 0 + nnodes: 1 + n_gpus_per_node: 8 + max_ckpt_to_keep: null + resume_mode: auto + resume_from_path: null + checkpoint: + save_contents: + - model + - optimizer + - extra + - hf_model + load_contents: + - model + - optimizer + - extra + device: cuda +global_step: 0 diff --git a/e2c/config/sft_trainer_engine.yaml b/e2c/config/sft_trainer_engine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cc4b32fe51451eff71525b7faa0b53746b2e692 --- /dev/null +++ b/e2c/config/sft_trainer_engine.yaml @@ -0,0 +1,71 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# @.: + +defaults: + - model@model: hf_model + - engine@engine: fsdp + - optim@optim: fsdp + - _self_ + +data: + train_batch_size: 256 # global batch size + micro_batch_size_per_gpu: 4 # this is also val batch size + max_token_len_per_gpu: 8192 + use_dynamic_bsz: True + train_files: ~/data/gsm8k/train.parquet + val_files: ~/data/gsm8k/test.parquet + # Multi-turn settings + messages_key: messages # Key for messages list in multi-turn mode + tools_key: tools # Key for tools list in multi-turn mode + enable_thinking_key: enable_thinking # Whether to enable thinking in multi-turn mode + pad_mode: left_right + # for right padding + max_length: 1024 + # for left right padding + max_prompt_length: 512 + max_response_length: 512 + truncation: error + balance_dp_token: False # to be implement + custom_cls: + path: null + name: null + use_shm: False + apply_chat_template_kwargs: {} + +# Checkpoint configuration +checkpoint: + _target_: verl.trainer.config.CheckpointConfig + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ["model", "optimizer", "extra"] + + # For more flexibility, you can specify the contents to load from the checkpoint. + load_contents: ${checkpoint.save_contents} + +trainer: + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + default_hdfs_dir: null + project_name: gsm8k-sft + experiment_name: test + total_epochs: 4 + total_training_steps: null + logger: [ 'console', 'wandb' ] + seed: 1 + save_freq: -1 + test_freq: -1 + max_ckpt_to_keep: null # Maximum number of checkpoints to keep, set to null to keep all + + # Resume mode: "auto", "disable", or "resume_path" + # "auto": resume from last checkpoint if available + # "disable": start from scratch + # "resume_path": resume from a user-defined path + resume_mode: auto + + # Path to resume training from (used when resume_mode is "resume_path" or "auto") + resume_from_path: null + device: cuda diff --git a/e2c/data/README.md b/e2c/data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8cdfc18697e8f767af1938dfc92d2fc71cb2472e --- /dev/null +++ b/e2c/data/README.md @@ -0,0 +1,6 @@ +The Datas +Please download the benchmark datasets from the following links: +and put the evaluation data in data/eval + +Please download the training datasets from: +and put the training data in data/train diff --git a/e2c/rl/e2c_rl.sh b/e2c/rl/e2c_rl.sh new file mode 100644 index 0000000000000000000000000000000000000000..1880031cf9910ee1d01e5b7e59fffeb3310a8a03 --- /dev/null +++ b/e2c/rl/e2c_rl.sh @@ -0,0 +1,78 @@ +# Tested successfully on the hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.4-flashinfer0.2.2-cxx11abi0 image. +# It outperforms the Qwen2 7B base model by two percentage points on the test set of GSM8K. +# ray start --head --port=6379 --num-gpus=4 +ray start --head --port=6379 --num-gpus=8 +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +CUDA_NUM=8 +MASTER_PORT=12345 +MODEL_PATH="Path to the pretrained model" +base_dir="./training" +total_epochs=2 +train_file=./data/e2c-rl-train.parquet +val_file=./data/e2c-rl-val.parquet +adv_coeff=2.0 +project_name=E2C-RL +experiment_name=run1 +set -x +# ROLL_TEMP=1.3 +python3 -m verl.trainer.main_ppo \ + --config-path ./config --config-name ppo_trainer.yaml \ + algorithm.adv_estimator=grpo \ + data.train_files="/home/fit/alex/Kaisen.Yang/CoT Decomposition/dataset/dapo/dapo-diverse.parquet" \ + data.val_files="/home/fit/alex/Kaisen.Yang/CoT Decomposition/dataset/dapo/qwen-rl-valid.parquet" \ + data.train_batch_size=256 \ + data.question_suffix="" \ + data.solution_prefix="''" \ + data.max_prompt_length=712 \ + data.max_response_length=8192 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + reward_model.use_constrain_reward=True \ + actor_rollout_ref.model.path="/WORK/fit/alex/Kaisen/checkpoints/qwen/combine/global_step_500/huggingface" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=14000 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.adv_coeff=$adv_coeff \ + actor_rollout_ref.actor.log_part_entropy=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console"]' \ + trainer.project_name=$project_name \ + trainer.experiment_name=$experiment_name \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=30 \ + trainer.test_freq=10 \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=True \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=4096 \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=1.0 \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=8192 \ + trainer.total_epochs=3 $@ + +# change rl_qwen +# change naive for decoding +# change rl_dataset for loading + + +# temp sample epoch fix \ No newline at end of file diff --git a/e2c/sft/e2c_sft.sh b/e2c/sft/e2c_sft.sh new file mode 100644 index 0000000000000000000000000000000000000000..b4765e07d254c78641cdc52d9d38d111c58bdb47 --- /dev/null +++ b/e2c/sft/e2c_sft.sh @@ -0,0 +1,18 @@ +export train_file=./data/e2c-sft-train.parquet +export val_file=./data/e2c-sft-val.parquet +export CUDA_VISIBLE_DEVICES=0,1,2,3 +export CUDA_NUM=4 +export MASTER_PORT=12345 +export MODEL_PATH="Path to the pretrained model" +export base_dir="" +# 1000 iter is enough +export total_training_steps=1000 +torchrun --nproc_per_node $CUDA_NUM ".\verl\verl\trainer\fsdp_sft_trainer.py" --config-path config --config-name sft_trainer.yaml \ + model.partial_pretrain=$MODEL_PATH \ + data.train_files=$train_file \ + data.val_files=$val_file \ + trainer.project_name=e2c-sft \ + trainer.experiment_name=run1 \ + trainer.base_dir=$base_dir \ + trainer.total_training_steps=$total_training_steps \ + trainer.n_gpus_per_node=$CUDA_NUM \ diff --git a/e2c/sft/ef-sft.sh b/e2c/sft/ef-sft.sh new file mode 100644 index 0000000000000000000000000000000000000000..e20d4472b123e3042a69b25636c06b99f3182449 --- /dev/null +++ b/e2c/sft/ef-sft.sh @@ -0,0 +1,18 @@ +export train_file=./data/ef-sft-train.parquet +export val_file=./data/ef-sft-val.parquet +export CUDA_VISIBLE_DEVICES=0,1,2,3 +export CUDA_NUM=4 +export MASTER_PORT=12345 +export MODEL_PATH="Path to the pretrained model" +export base_dir="" +# 500 iter is enough +export total_training_steps=500 +torchrun --nproc_per_node $CUDA_NUM ".\verl\verl\trainer\fsdp_sft_trainer.py" --config-path config --config-name sft_trainer.yaml \ + model.partial_pretrain=$MODEL_PATH \ + data.train_files=$train_file \ + data.val_files=$val_file \ + trainer.project_name=e2c-sft \ + trainer.experiment_name=run1 \ + trainer.base_dir=$base_dir \ + trainer.total_training_steps=$total_training_steps \ + trainer.n_gpus_per_node=$CUDA_NUM \ diff --git a/e2c/verl/.gemini/config.yaml b/e2c/verl/.gemini/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66015ad30ed6768e06dceb91f11004be8a74bb04 --- /dev/null +++ b/e2c/verl/.gemini/config.yaml @@ -0,0 +1,10 @@ +have_fun: false +code_review: + disable: false + comment_severity_threshold: HIGH + max_review_comments: -1 + pull_request_opened: + help: false + summary: false + code_review: true +ignore_patterns: [] diff --git a/e2c/verl/.github/CODEOWNERS b/e2c/verl/.github/CODEOWNERS new file mode 100644 index 0000000000000000000000000000000000000000..a44436d719126c6322ae4c495f74087359af5aa2 --- /dev/null +++ b/e2c/verl/.github/CODEOWNERS @@ -0,0 +1,30 @@ +/docs @eric-haibin-lin @zhaochenyang20 @hongpeng-guo +/docs/amd_tutorial @yushengsu-thu +/docs/slang_multiturn @zhaochenyang20 @SwordFaith +/docs/ascend_tutorial @FightingZhen + +/recipe/dapo @tongyx361 @PeterSH6 @vermouth1992 @tardis-key @FightingZhen @ji-huazhong +/recipe/spin @zhaochenyang20 +/recipe/sppo @zhaochenyang20 + +/third_party/sglang @zhaochenyang20 @SwordFaith +/third_party/vllm @PeterSH6 @wuxibin89 + +/examples/grpo_trainer @vermouth1992 @PeterSH6 @tardis-key @FightingZhen @ji-huazhong + +/verl/single_controller @zw0610 @wuxibin89 @hongpeng-guo +/verl/trainer @eric-haibin-lin @vermouth1992 @tongyx361 @PeterSH6 +/verl/models/mcore @ISEEKYAN @vermouth1992 +/verl/models/transformers @vermouth1992 @PeterSH6 @tardis-key @FightingZhen @ji-huazhong +/verl/workers/engine @eric-haibin-lin @vermouth1992 @ZihengJiang +/verl/workers/roles @eric-haibin-lin @vermouth1992 @ZihengJiang +/verl/workers/engine/fsdp @eric-haibin-lin @vermouth1992 @ZihengJiang +/verl/workers/rollout/vllm_rollout @wuxibin89 @PeterSH6 @chenhaiq +/verl/workers/rollout/sglang_rollout @zhaochenyang20 @SwordFaith @chenhaiq +/verl/workers/actor/megatron_actor.py @ISEEKYAN @vermouth1992 +/verl/workers/critic/megatron_critic.py @ISEEKYAN @vermouth1992 +/verl/workers/megatron_workers.py @ISEEKYAN @vermouth1992 + +/tests/single_controller @zw0610 @wuxibin89 +/tests/trainer @eric-haibin-lin @vermouth1992 @tongyx361 @PeterSH6 +/tests/workers/rollout/vllm_rollout @wuxibin89 @PeterSH6 @chenhaiq diff --git a/e2c/verl/.github/ISSUE_TEMPLATE/bug-report.yml b/e2c/verl/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000000000000000000000000000000000000..67341f4139d9a5348f3887242b7c6accf10abc7b --- /dev/null +++ b/e2c/verl/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,65 @@ +# modified from https://github.com/huggingface/transformers/blob/main/.github/ISSUE_TEMPLATE/bug-report.yml?plain=1 +name: "\U0001F41B Bug Report" +description: Submit a bug report to help us improve verl +labels: [ "bug" ] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! 🤗 + + - type: textarea + id: system-info + attributes: + label: System Info + description: Please share your system info with us. You can run the command `python scripts/diagnose.py` and copy-paste its output below. + placeholder: verl version, platform, python version, ... + validations: + required: true + + - type: checkboxes + id: information-scripts-examples + attributes: + label: Information + description: 'The problem arises when using:' + options: + - label: "The official example scripts" + - label: "My own modified scripts" + + - type: checkboxes + id: information-tasks + attributes: + label: Tasks + description: "The tasks I am working on are:" + options: + - label: "An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)" + - label: "My own task or dataset (give details below)" + + - type: textarea + id: reproduction + validations: + required: true + attributes: + label: Reproduction + description: | + Please provide a code sample that reproduces the problem you ran into. It can be a Colab link or just a code snippet. + Please include relevant config information with your code. + If you have code snippets, error messages, stack traces please provide them here as well. + Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting + Do not use screenshots, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code. + + placeholder: | + Steps to reproduce the behavior: + + 1. + 2. + 3. + + + - type: textarea + id: expected-behavior + validations: + required: true + attributes: + label: Expected behavior + description: "A clear and concise description of what you would expect to happen." \ No newline at end of file diff --git a/e2c/verl/.github/ISSUE_TEMPLATE/config.yml b/e2c/verl/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..ac09e8636d7a7577999a55ffdf7095dd0b656e52 --- /dev/null +++ b/e2c/verl/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,2 @@ +blank_issues_enabled: true +version: 0.1 diff --git a/e2c/verl/.github/ISSUE_TEMPLATE/feature-request.yml b/e2c/verl/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000000000000000000000000000000000000..18e0615b9ecd01bf1cead1155ff8124865e07d24 --- /dev/null +++ b/e2c/verl/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,32 @@ +# modified from https://github.com/huggingface/transformers/blob/main/.github/ISSUE_TEMPLATE/feature-request.yml?plain=1 +name: "\U0001F680 Feature request" +description: Submit a proposal/request for a new verl feature +labels: [ "Feature request" ] +body: + - type: textarea + id: feature-request + validations: + required: true + attributes: + label: Feature request + description: | + A clear and concise description of the feature proposal. Please provide a link to the paper and code in case they exist. + + - type: textarea + id: motivation + validations: + required: true + attributes: + label: Motivation + description: | + Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too. + + + - type: textarea + id: contribution + validations: + required: true + attributes: + label: Your contribution + description: | + Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/volcengine/verl/blob/main/CONTRIBUTING.md) \ No newline at end of file diff --git a/e2c/verl/.github/PULL_REQUEST_TEMPLATE.md b/e2c/verl/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..4f092f174f01b08144069b5004b328e9562cb1a2 --- /dev/null +++ b/e2c/verl/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,40 @@ +### What does this PR do? + +> Add **concise** overview of what this PR aims to achieve or accomplish. Reference related GitHub issues and PRs that help with the review. + +### Checklist Before Starting + +- [ ] Search for similar PRs. Paste at least one query link here: ... +- [ ] Format the PR title as `[{modules}] {type}: {description}` (This will be checked by the CI) + - `{modules}` include `fsdp`, `megatron`, `sglang`, `vllm`, `rollout`, `trainer`, `ci`, `training_utils`, `recipe`, `hardware`, `deployment`, `ray`, `worker`, `single_controller`, `misc`, `perf`, `model`, `algo`, `env`, `tool`, `ckpt`, `doc`, `data` + - If this PR involves multiple modules, separate them with `,` like `[megatron, fsdp, doc]` + - `{type}` is in `feat`, `fix`, `refactor`, `chore`, `test` + - If this PR breaks any API (CLI arguments, config, function signature, etc.), add `[BREAKING]` to the beginning of the title. + - Example: `[BREAKING][fsdp, megatron] feat: dynamic batching` + +### Test + +> For changes that can not be tested by CI (e.g., algorithm implementation, new model support), validate by experiment(s) and show results like training curve plots, evaluation results, etc. + +### API and Usage Example + +> Demonstrate how the API changes if any, and provide usage example(s) if possible. + +```python +# Add code snippet or script demonstrating how to use this +``` + +### Design & Code Changes + +> Demonstrate the high-level design if this PR is complex, and list the specific changes. + +### Checklist Before Submitting + +> [!IMPORTANT] +> Please check all the following items before requesting a review, otherwise the reviewer might deprioritize this PR for review. + +- [ ] Read the [Contribute Guide](https://github.com/volcengine/verl/blob/main/CONTRIBUTING.md). +- [ ] Apply [pre-commit checks](https://github.com/volcengine/verl/blob/main/CONTRIBUTING.md#code-linting-and-formatting): `pre-commit install && pre-commit run --all-files --show-diff-on-failure --color=always` +- [ ] Add / Update [the documentation](https://github.com/volcengine/verl/tree/main/docs). +- [ ] Add unit or end-to-end test(s) to [the CI workflow](https://github.com/volcengine/verl/tree/main/.github/workflows) to cover all the code. If not feasible, explain why: ... +- [ ] Once your PR is ready for CI, send a message in [the `ci-request` channel](https://verl-project.slack.com/archives/C091TCESWB1) in [the `verl` Slack workspace](https://join.slack.com/t/verl-project/shared_invite/zt-3855yhg8g-CTkqXu~hKojPCmo7k_yXTQ). (If not accessible, please try [the Feishu group (飞书群)](https://applink.larkoffice.com/client/chat/chatter/add_by_link?link_token=772jd4f1-cd91-441e-a820-498c6614126a).) diff --git a/e2c/verl/.github/dependabot.yml b/e2c/verl/.github/dependabot.yml new file mode 100644 index 0000000000000000000000000000000000000000..24a3571c620c9c1e5ef7b4011851124c7a020626 --- /dev/null +++ b/e2c/verl/.github/dependabot.yml @@ -0,0 +1,9 @@ +## Enabled the dependabot to check the dependencies of the project +## Dependabot will open pull requests to update dependencies automatically + +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly \ No newline at end of file diff --git a/e2c/verl/.github/workflows/.deprecate/e2e_eval_aime24.yml b/e2c/verl/.github/workflows/.deprecate/e2e_eval_aime24.yml new file mode 100644 index 0000000000000000000000000000000000000000..e674fd10cbde674d94053e9ceecf0ca3eddfeb76 --- /dev/null +++ b/e2c/verl/.github/workflows/.deprecate/e2e_eval_aime24.yml @@ -0,0 +1,147 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +name: e2e_eval_aime24 + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!*.md" + - "!docker/**" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + - "!recipe/**" + - "recipe/r1" + - "!recipe/r1/README.md" + pull_request: + branches: + - main + paths: + - "**/*.py" + # Other entrypoints + - "!*.md" + - "!docker/**" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Home + - "recipe/r1" + - "!recipe/r1/README.md" + # Other recipes + - "!recipe/**" + # Entrypoints + - ".github/workflows/e2e_eval_aime24.yml" + - "tests/special_e2e/run_r1_distill_qwen_aime24_eval.sh" + - "verl/trainer/main_generation.py" + - "verl/trainer/config/generation.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_eval_aime24: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test,gpu,math] + pip3 install math-verify transformers==4.56.2 + - name: Prepare aime24 dataset + run: | + ray stop --force + python3 recipe/r1/data_process.py --task aime2024 + - name: Running generation and evaluation in AIME 2024 + run: | + ray stop --force + bash tests/special_e2e/run_r1_distill_qwen_aime24_eval.sh + + cleanup: + runs-on: ubuntu-latest + needs: [setup, e2e_eval_aime24] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/.deprecate/e2e_ppo_trainer.yml b/e2c/verl/.github/workflows/.deprecate/e2e_ppo_trainer.yml new file mode 100644 index 0000000000000000000000000000000000000000..fa6fef0bd144b64cc02179aae4ea711f6e477c63 --- /dev/null +++ b/e2c/verl/.github/workflows/.deprecate/e2e_ppo_trainer.yml @@ -0,0 +1,133 @@ +name: e2e_ppo_trainer_deprecate + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - disabled_ci + pull_request: + branches: + - disabled_ci + paths: + - "**/*.py" + # Other entrypoints + - "!**/*.md" + - "!docker/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Docs + - "!docs/**" + # Recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/ppo_trainer" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + pre_commit_for_ppo: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip install -e . + - name: Set ruff --output-format=github + run: | + sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml + git add .pre-commit-config.yaml + - uses: pre-commit/action@v3.0.1 + with: + extra_args: "" # Overriding default "--all-files" + + e2e_ppo_trainer_sglang_multiturn_with_tool: + runs-on: [L20x8] + needs: pre_commit_for_ppo + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + container: + image: verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2 + options: --gpus all --shm-size=10g + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test,gpu,sglang] + - name: Prepare gsm8k dataset with tool + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k_multiturn_w_tool.py --local_save_dir $HOME/data/gsm8k_verl_sgl_multi_turn_preprocessed + - name: Running GSM8K with tool E2E training tests on 8 L20 GPUs with rmpad using function rm and save ckpt with sglang + run: | + ray stop --force + bash tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_w_tool.sh + - name: Running GSM8K with tool E2E training tests with FSDP2 + run: | + ray stop --force + FSDP_STRATEGY=fsdp2 bash tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_w_tool.sh + + e2e_ppo_trainer_sglang_vlm_multiturn_with_tool: + runs-on: [L20x8] + needs: pre_commit_for_ppo + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + container: + image: verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2 + options: --gpus all --shm-size=10g + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test,geo,gpu,sglang] + - name: Prepare geo3k dataset with tool + run: | + ray stop --force + python3 examples/data_preprocess/geo3k_multiturn_w_tool.py --local_dir $HOME/data/geo3k_verl_sgl_multi_turn_preprocessed + - name: Running GEO3K with tool E2E training tests on 8 L20 GPUs with rmpad using function rm and save ckpt with sglang + run: | + ray stop --force + bash tests/special_e2e/run_geo3k_fsdp_sgl_multiturn_w_tool.sh + - name: Running GEO3K with tool E2E training tests with FSDP2 + run: | + ray stop --force + FSDP_STRATEGY=fsdp2 bash tests/special_e2e/run_geo3k_fsdp_sgl_multiturn_w_tool.sh diff --git a/e2c/verl/.github/workflows/.deprecate/e2e_ppo_trainer_megatron_sglang.yml b/e2c/verl/.github/workflows/.deprecate/e2e_ppo_trainer_megatron_sglang.yml new file mode 100644 index 0000000000000000000000000000000000000000..30c22d9948226055e255b9c4a71d3fa9bec07c74 --- /dev/null +++ b/e2c/verl/.github/workflows/.deprecate/e2e_ppo_trainer_megatron_sglang.yml @@ -0,0 +1,155 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_sglang_deprecate + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - disabled_ci + pull_request: + branches: + - disabled_ci + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer_megatron_sglang.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_megatron-qwen3: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) with validation and saving + run: | + ray stop --force + ENGINE=sglang ALL_OFFLOAD=True VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) testing learning rate scheduler + run: | + ray stop --force + ENGINE=sglang LR_WARMUP_STEPS=1 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh + + - name: Test Megatron checkpoints merging function (Qwen3 Actor and Critic) + run: | + exp_name="qwen3-0.6b-megatron-gsm8k-minimal" + python -m verl.model_merger test --backend megatron --tie-word-embedding --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + python -m verl.model_merger test --backend megatron --is-value-model --local_dir checkpoints/verl-test/${exp_name}/global_step_1/critic --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/critic/huggingface + - name: clean up + run: | + rm -rf checkpoints + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_ppo_trainer_megatron-deepseek, + e2e_ppo_trainer_megatron-qwen3, + e2e_ppo_trainer_megatron-different-train-infer-tp-qwen-tie-embedding, + e2e_ppo_trainer_megatron-qwen-override-transformer-config, + e2e_ppo_trainer_megatron-deepseek-override-transformer-config, + e2e_ppo_trainer_megatron-moe-expert-parallel, + e2e_ppo_trainer_megatron-qwen2_5vl-3b, + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/.deprecate/e2e_prime.yml b/e2c/verl/.github/workflows/.deprecate/e2e_prime.yml new file mode 100644 index 0000000000000000000000000000000000000000..694591b9c6c810d4b61e48767ccb6e7d791fa369 --- /dev/null +++ b/e2c/verl/.github/workflows/.deprecate/e2e_prime.yml @@ -0,0 +1,66 @@ +name: e2e_prime_deprecate + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - disabled_ci + pull_request: + branches: + - disabled_ci + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Other recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Home + - "recipe/prime" + # Entrypoints + - ".github/workflows/e2e_prime.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_prime.sh" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + e2e_prime: + runs-on: [L20x8] + timeout-minutes: 50 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + container: + image: whatcanyousee/verl:ngc-cu124-vllm0.8.5-sglang0.4.6.post5-mcore0.12.0-te2.3 + options: --gpus all --shm-size=10g + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test,gpu] + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py + - name: Running GSM8K E2E with prime alg + run: | + ray stop --force + bash tests/special_e2e/run_prime.sh diff --git a/e2c/verl/.github/workflows/.deprecate/e2e_spin.yml b/e2c/verl/.github/workflows/.deprecate/e2e_spin.yml new file mode 100644 index 0000000000000000000000000000000000000000..b3c8a85e070168ac4d0cc5d3df6ce8c7d2754c06 --- /dev/null +++ b/e2c/verl/.github/workflows/.deprecate/e2e_spin.yml @@ -0,0 +1,119 @@ +name: e2e_spin + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Other recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Home + - "recipe/spin" + # Entrypoints + - ".github/workflows/e2e_spin.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_spin.sh" + - "!examples" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Other recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Home + - "recipe/spin" + # Entrypoints + - ".github/workflows/e2e_spin.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_spin.sh" + - "!examples" + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_spin: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test,gpu,sglang] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with the spin algorithm + run: | + ray stop --force + bash tests/special_e2e/run_spin.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_spin + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/.deprecate/e2e_sppo.yml b/e2c/verl/.github/workflows/.deprecate/e2e_sppo.yml new file mode 100644 index 0000000000000000000000000000000000000000..0dddd849cb0df51f394db60aa27eec65dced1e8e --- /dev/null +++ b/e2c/verl/.github/workflows/.deprecate/e2e_sppo.yml @@ -0,0 +1,118 @@ +name: e2e_sppo + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Other recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Home + - "recipe/sppo" + # Entrypoints + - ".github/workflows/e2e_sppo.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_sppo.sh" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Other recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Home + - "recipe/sppo" + # Entrypoints + - ".github/workflows/e2e_sppo.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_sppo.sh" + +# Declare permissions just read content. +permissions: + contents: read + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + TRANSFORMERS_VERSION: "4.56.2" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_sppo: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test,gpu,sglang] + - name: Prepare MATH dataset + run: | + python3 examples/data_preprocess/math_dataset.py --local_dataset_path $HOME/models/hf_data/DigitalLearningGmbH/MATH-lighteval + - name: Running the E2E test with the SPPO algorithm + run: | + ray stop --force + bash tests/special_e2e/run_sppo.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_sppo + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/README.md b/e2c/verl/.github/workflows/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aab56302a3000a36353cd8a0fcacdcb254ab7508 --- /dev/null +++ b/e2c/verl/.github/workflows/README.md @@ -0,0 +1,73 @@ +### Adding a New Workflow + +When adding a new workflow for continuous integration (CI), you have two runner options: a fixed runner or a machine from the vemlp. + +- **Fixed Runner**: To use a fixed runner, specify it in your workflow using the `runs-on` keyword, like `runs-on: [L20x8]`. +- **Vemlp Runner**: Opting for a Vemlp machine allows you to launch tasks elastically. + +Here is a template to assist you. This template is designed for using Vemlp machines. Currently, for each workflow, you need to create a `setup` and a `cleanup` job. When using this template, the main parts you need to modify are the `IMAGE` environment variable and the specific `job steps`. + +```yaml +name: Your Default Workflow + +on: + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - ".github/workflows/template.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +env: + IMAGE: "your vemlp image" # e.g. "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.4-vllm0.8.5-mcore0.12.2" + DYNAMIC_RUNNER_URL: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" # public veFaas api + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + task-id: ${{ steps.create-runner.outputs.task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_URL }}" + image: "${{ env.DEFAULT_IMAGE }}" + + your_job: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'default-runner' }}"] + steps: + xxxx # your jobs + + cleanup: + runs-on: ubuntu-latest + needs: [setup, your_job] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_URL }}" + task-id: "${{ needs.setup.outputs.task-id }}" +``` + +### Model and Dataset +To avoid CI relies on network, we pre-download dataset on a NFS on the CI machine. The path for models are \${HOME}/models and the path for dataset is \${HOME}/models/hf_data. \ No newline at end of file diff --git a/e2c/verl/.github/workflows/check-pr-title.yml b/e2c/verl/.github/workflows/check-pr-title.yml new file mode 100644 index 0000000000000000000000000000000000000000..948ce5e3f01498ce4f569230cdb2dd384fc0cbfd --- /dev/null +++ b/e2c/verl/.github/workflows/check-pr-title.yml @@ -0,0 +1,58 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +on: + pull_request: + types: [opened, edited, synchronize] + +jobs: + check-title: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Run PR title checker + run: python3 tests/special_sanity/check_pr_title.py + env: + PR_TITLE: ${{ github.event.pull_request.title }} + + - name: Run PR description checker + run: python3 tests/special_sanity/check_pr_description.py + env: + PR_TITLE: ${{ github.event.pull_request.title }} + GITHUB_EVENT_PATH: ${{ github.event_path }} diff --git a/e2c/verl/.github/workflows/checkpoint_converter.yml b/e2c/verl/.github/workflows/checkpoint_converter.yml new file mode 100644 index 0000000000000000000000000000000000000000..e716709b6cc963ecdd87e7e0c67f5018c40104ce --- /dev/null +++ b/e2c/verl/.github/workflows/checkpoint_converter.yml @@ -0,0 +1,142 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: checkpoint_converter +# latest version: Megatron-LM core_r0.11.0 https://github.com/NVIDIA/Megatron-LM/tree/core_r0.11.0 + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Entrypoints + - ".github/workflows/checkpoint_converter.yml" + - ".github/workflows/e2e_ppo_trainer_megatron.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + checkpoint_converter: + runs-on: [L20x8] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + container: + image: verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2 + options: --gpus all --shm-size=10g + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test] + - name: Download Model to Use + run: | +# huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir ${HOME}/models/Qwen/Qwen2.5-0.5B +# huggingface-cli download deepseek-ai/deepseek-coder-1.3b-instruct --local-dir ${HOME}/models/deepseek-ai/deepseek-coder-1.3b-instruct + export HF_HUB_OFFLINE=1 + - name: Running Huggingface to Megatron dist_ckpt converter (Qwen/Qwen2.5-0.5B) + run: | + ray stop --force + python scripts/converter_hf_to_mcore.py --hf_model_path=${HOME}/models/Qwen/Qwen2.5-0.5B --output_path checkpoints/Qwen/Qwen2.5-0.5B --test + - name: Running Huggingface to Megatron dist_ckpt converter (deepseek-ai/deepseek-coder-1.3b-instruct) + run: | + ray stop --force + python scripts/converter_hf_to_mcore.py --hf_model_path=${HOME}/models/deepseek-ai/deepseek-coder-1.3b-instruct --output_path checkpoints/deepseek-ai/deepseek-coder-1.3b-instruct --test + - name: Clean up + run: | + rm -rf checkpoints + checkpoint_converter_large_moe_models: + runs-on: [L20x8] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + HF_ENDPOINT: "https://hf-mirror.com" + container: + image: verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2 + options: --gpus all --shm-size=10g + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test] + - name: Download Model to Use + run: | +# huggingface-cli download Qwen/Qwen1.5-MoE-A2.7B-Chat --local-dir ${HOME}/models/Qwen/Qwen1.5-MoE-A2.7B-Chat + export HF_HUB_OFFLINE=1 + - name: Running Huggingface to Megatron dist_ckpt CPU converter (Qwen/Qwen1.5-MoE-A2.7B-Chat) + run: | + ray stop --force + python scripts/converter_hf_to_mcore.py --hf_model_path=${HOME}/models/Qwen/Qwen1.5-MoE-A2.7B-Chat --output_path checkpoints/Qwen/Qwen1.5-MoE-A2.7B-Chat --use_cpu_initialization + - name: Running distributed Huggingface to Megatron dist_ckpt CPU converter (Qwen/Qwen1.5-MoE-A2.7B-Chat) + run: | + ray stop --force + torchrun --nproc_per_node 8 --nnodes 1 scripts/converter_hf_to_mcore.py --hf_model_path=${HOME}/models/Qwen/Qwen1.5-MoE-A2.7B-Chat --output_path checkpoints/Qwen/Qwen1.5-MoE-A2.7B-Chat_dist --use_cpu_initialization + - name: clean up + run: | + rm -rf checkpoints diff --git a/e2c/verl/.github/workflows/cpu_unit_tests.yml b/e2c/verl/.github/workflows/cpu_unit_tests.yml new file mode 100644 index 0000000000000000000000000000000000000000..95cbc8be370bfe2356cca87ced753142f1d749f1 --- /dev/null +++ b/e2c/verl/.github/workflows/cpu_unit_tests.yml @@ -0,0 +1,88 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +name: cpu_unit_tests + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - .github/workflows/cpu_unit_tests.yml + - "!recipe/**/*.py" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + cpu_unit_tests: + runs-on: [L20x8] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + container: + image: verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip install -e .[test,prime,geo] + pip install --upgrade "ray>=2.40.0" pillow + - name: Download datasets + run: | + huggingface-cli download verl-team/gsm8k-v0.4.1 --repo-type dataset --local-dir ~/verl-data/gsm8k + python3 examples/data_preprocess/geo3k.py + - name: Running CPU unit tests + run: | + echo '[pytest]' > pytest.ini + echo 'python_files = *_on_cpu.py' >> pytest.ini + pytest -s -x --asyncio-mode=auto tests/ \ No newline at end of file diff --git a/e2c/verl/.github/workflows/doc.yml b/e2c/verl/.github/workflows/doc.yml new file mode 100644 index 0000000000000000000000000000000000000000..f006b737c10292471068f9d0e83781e9e9ae1370 --- /dev/null +++ b/e2c/verl/.github/workflows/doc.yml @@ -0,0 +1,100 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +name: doc_test + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "docs/**" + - .github/workflows/doc.yml + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read # for checkout + pages: write # for deploy-pages + id-token: write # for deploy-pages + +jobs: + doc_test: + runs-on: ubuntu-latest + timeout-minutes: 5 # Increase this timeout value as needed + strategy: + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip install -e .[test] --no-deps + pip install -r docs/requirements-docs.txt + + - name: Run doc make html + run: | + cd docs + make clean + make html SPHINXOPTS="--keep-going -w _build/sphinx.log" + if grep -q ": ERROR:" _build/sphinx.log; then + echo "🚨 Sphinx doc build contained ERRORs - see _build/sphinx.log" + exit 1 + fi + if grep -q "WARNING: document isn't included in any toctree" _build/sphinx.log; then + echo "🚨 Sphinx doc build contained WARNING. Please include newly added docs in index.rst. See _build/sphinx.log for details" + exit 1 + fi + if grep -q "WARNING: Inline emphasis" _build/sphinx.log; then + echo "🚨 Sphinx doc build contained WARNING. Please check inline emphasis is correct. See _build/sphinx.log for details" + exit 1 + fi + if grep -q "WARNING: Definition list ends without a blank line" _build/sphinx.log; then + echo "🚨 Sphinx doc build contained WARNING. Please check if the indentation is correct. See _build/sphinx.log for details" + exit 1 + fi diff --git a/e2c/verl/.github/workflows/e2e_ascend.yml b/e2c/verl/.github/workflows/e2e_ascend.yml new file mode 100644 index 0000000000000000000000000000000000000000..5d04d76c12c108f1c0b48d90cd40bba3957bbe81 --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_ascend.yml @@ -0,0 +1,155 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +name: e2e_ascend + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + paths: + - ".github/workflows/e2e_ascend.yml" + - "**/*.py" + - "docs/ascend_tutorial/**" + - "examples/**" + - "recipe/**" + - "tests/special_npu/**" + - "tests/special_sanity/**" + - "verl/**" + - "pyproject.toml" + - "requirements-npu.txt" + - "setup.py" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +jobs: + test: + name: verl Ascend test (self-host) + runs-on: [self-hosted, npu-0] + timeout-minutes: 40 # Increase this timeout value as needed + container: + image: crispig/verl_npu:cann8.1rc1-py3.10-torch2.5.1-vllm-ascend0.7.3.post1-mindspeed0121-250731 + volumes: + - /usr/local/dcmi:/usr/local/dcmi + - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi + - /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ + - /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info + - /etc/ascend_install.info:/etc/ascend_install.info + - /data00/dataset:/github/home/dataset + - /data00/models:/github/home/models + # Use self-host cache speed up pip and model download + # - /home/action/actions-runner/_work/cache:/github/home/.cache/ + options: >- + --device /dev/davinci0 + --device /dev/davinci_manager + --device /dev/devmm_svm + --device /dev/hisi_hdc + --network host + --privileged + --shm-size 16g + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Checkout volcengine/verl repo + uses: actions/checkout@v4 + - name: Install the current repository + run: | + pip3 install hf_transfer peft + pip3 install -r requirements-npu.txt + pip install -e . + - name: Install torchvision + run: | + pip install torchvision==0.20.1+cpu --index-url https://download.pytorch.org/whl/cpu + - name: Uninstall Triton + run: | + pip uninstall -y triton + - name: Preprocess gsm8k dataset + run: | + python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/dataset/openai/gsm8k + - name: Preprocess geo3k dataset + run: | + python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/dataset/hiyouga/geometry3k + - name: Running gsm8k e2e qwen3 training tests with PPO on ASCEND NPU + run: | + ray stop --force + bash tests/special_npu/run_qwen3_06b_ppo.sh + rm -rf $HOME/ckpts + - name: Running gsm8k e2e training tests with peft sft on ASCEND NPU + run: | + ray stop --force + bash tests/special_npu/run_qwen2_5_05b_sft_peft_sp2.sh + rm -rf $HOME/ckpts + - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU + run: | + ray stop --force + bash tests/special_npu/run_qwen2_5_05b_grpo.sh + rm -rf $HOME/ckpts + - name: Running geo3k e2e training tests with GRPO on ASCEND NPU + run: | + ray stop --force + bash tests/special_npu/run_qwen2_5_vl_3b_npu.sh + rm -rf $HOME/ckpts + - name: Running gsm8k e2e training tests with DAPO on ASCEND NPU + run: | + ray stop --force + bash tests/special_npu/run_qwen2_5_05b_dapo.sh + rm -rf $HOME/ckpts + - name: Running gsm8k e2e training tests with GRPO MindSpeed on ASCEND NPU + run: | + ray stop --force + USE_DIST_CKPT=True bash tests/special_npu/run_qwen2_5_05b_grpo_mindspeed.sh + rm -rf $HOME/dist_ckpt/qwen2_5_05b_grpo_mindspeed + rm -rf $HOME/ckpts + - name: Running NPU profiling unit tests + run: | + ray stop --force + pytest -s -x tests/utils/test_special_mstx_profile.py \ No newline at end of file diff --git a/e2c/verl/.github/workflows/e2e_dapo.yml b/e2c/verl/.github/workflows/e2e_dapo.yml new file mode 100644 index 0000000000000000000000000000000000000000..0c1e8e3478dc37646e114288e37dc55a656cffd8 --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_dapo.yml @@ -0,0 +1,145 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +name: e2e_dapo + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "verl/*.py" + # Other entrypoints + - "!examples/*trainer*" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Megatron + - "!verl/workers/**/megatron_*.py" + - "!recipe/**" + - "recipe/dapo" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Other recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Home + - "recipe/dapo" + # Entrypoints + - ".github/workflows/e2e_dapo.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_dapo.sh" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_dapo: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test,gpu] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with the DAPO algorithm + run: | + ray stop --force + bash tests/special_e2e/run_dapo.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_dapo + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/e2e_genrm_remote.yml b/e2c/verl/.github/workflows/e2e_genrm_remote.yml new file mode 100644 index 0000000000000000000000000000000000000000..48544bbe5bafb7301d669d4b8aa4438242c1deda --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_genrm_remote.yml @@ -0,0 +1,138 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +name: e2e_genrm_remote + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "tests/**" + - "!recipe/**" + - "recipe/genrm_remote" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Other recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Home + - "recipe/genrm_remote" + - "!recipe/genrm_remote/README.md" + # Entrypoints + - ".github/workflows/e2e_genrm_remote.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_genrm_remote.sh" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_genrm_remote: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test,gpu] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with the Generative Reward Model + run: | + ray stop --force + bash tests/special_e2e/run_genrm_remote.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_genrm_remote + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/e2e_one_step_off_policy.yml b/e2c/verl/.github/workflows/e2e_one_step_off_policy.yml new file mode 100644 index 0000000000000000000000000000000000000000..025a86f0b36428dd9f03ed3cabb62173d2b629bc --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_one_step_off_policy.yml @@ -0,0 +1,178 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +name: e2e_one_step_off_policy + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "!**/*.md" + - "!**/*.sh" + # Other entrypoints + - "!examples/*trainer*" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + - "!recipe/**" + - "recipe/one_step_off_policy" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "!**/*.md" + - "!**/*.sh" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Other recipes + - "!recipe/**" + # Home + - "recipe/one_step_off_policy" + # Entrypoints + - ".github/workflows/e2e_one_step_off_policy.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_one_step_off_policy.sh" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + TRANSFORMERS_VERSION: "4.56.2" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + # Test FSDP2 strategy + e2e_one_step_off_policy_fsdp2: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 10 # Increase timeout for async training + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ACTOR_STRATEGY: "fsdp2" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test,gpu] + pip3 install transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with one_step_off_policy algorithm (FSDP2) + run: | + ray stop --force + bash tests/special_e2e/run_one_step_off_policy.sh + + # Test Megatron strategy + e2e_one_step_off_policy_megatron: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 10 # Increase timeout for async training + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ACTOR_STRATEGY: "megatron" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test,gpu] + pip3 install transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with one_step_off_policy algorithm (Megatron) + run: | + ray stop --force + bash tests/special_e2e/run_one_step_off_policy.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_one_step_off_policy_fsdp2, + e2e_one_step_off_policy_megatron + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/e2e_ppo_trainer.yml b/e2c/verl/.github/workflows/e2e_ppo_trainer.yml new file mode 100644 index 0000000000000000000000000000000000000000..7a7b062cf606b612e05922e23ca55a896311115b --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_ppo_trainer.yml @@ -0,0 +1,79 @@ +name: e2e_ppo_trainer + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!**/*.md" + - "!docker/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Docs + - "!docs/**" + # Recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/ppo_trainer" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + pre_commit_for_ppo: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip install -e . + - name: Set ruff --output-format=github + run: | + sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml + git add .pre-commit-config.yaml + - uses: pre-commit/action@v3.0.1 + with: + extra_args: "" # Overriding default "--all-files" + diff --git a/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml b/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml new file mode 100644 index 0000000000000000000000000000000000000000..cf7b2599d42235657875fe9a592b3bafd3e67a35 --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml @@ -0,0 +1,281 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_sglang + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Entrypoints + - "verl/worksers/rollout/sglang_rollout/*" + - ".github/workflows/e2e_ppo_trainer_megatron_sglang.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_megatron-deepseek: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + OPTIM_MEMORY_EFFICIENT=True ENGINE=sglang SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + export VLLM_USE_V1=1 + ray start --head + ENGINE=sglang MODE=async RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Test Megatron checkpoints merging function (DeepSeek Actor and Critic) + run: | + exp_name="deepseek-coder-1.3b-instruct-megatron-gsm8k-minimal" + python -m verl.model_merger test --backend megatron --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + python -m verl.model_merger test --backend megatron --is-value-model --local_dir checkpoints/verl-test/${exp_name}/global_step_1/critic --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/critic/huggingface + - name: Profiling GRPO GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Deepseek) + run: | + ray stop --force + PROFILE_ENABLE=True ENGINE=sglang ADV_ESTIMATOR=grpo USE_DYNAMIC_BSZ=False MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh + if [ -z "$( ls -A '/tmp/ray/session_latest/logs/nsight/' )" ]; then + echo "[ERROR] not found any profiling files" + exit 1 + else + echo "[SUCCESS] profile success" + fi + - name: clean up + run: | + rm -rf checkpoints + + e2e_ppo_trainer_megatron-different-train-infer-tp-qwen-tie-embedding: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with tie-embedding Megatron (Qwen) with train tp > infer tp + run: | + ray stop --force + ENGINE=sglang VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 TRAIN_TP=2 INFER_TP=1 MODEL_ID=Qwen/Qwen2.5-1.5B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen) with train tp < infer tp + run: | + ray stop --force + ENGINE=sglang VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 TRAIN_TP=1 INFER_TP=2 MODEL_ID=Qwen/Qwen2.5-1.5B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + + e2e_ppo_trainer_megatron-qwen-override-transformer-config: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k +# - name: Download Model to Use +# run: | +# huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir ${HOME}/models/Qwen/Qwen2.5-0.5B +# export HF_HUB_OFFLINE=1 + - name: Prepare dist_ckpt of Qwen2.5-0.5B, uneven layer distribution only supports dist_ckpt + run: | + python3 scripts/converter_hf_to_mcore.py --hf_model_path ${HOME}/models/Qwen/Qwen2.5-0.5B --output_path checkpoints/verl-test/qwen2.5-0.5b-megatron + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen) + run: | + ray stop --force + ENGINE=sglang SAVE_FREQ=1 COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 SKIP_SAVE_HF_MODEL=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_first_pipeline_stage=8 +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_last_pipeline_stage=4 actor_rollout_ref.actor.megatron.use_dist_checkpointing=true actor_rollout_ref.actor.megatron.dist_checkpointing_path=checkpoints/verl-test/qwen2.5-0.5b-megatron actor_rollout_ref.ref.megatron.use_dist_checkpointing=true actor_rollout_ref.ref.megatron.dist_checkpointing_path=checkpoints/verl-test/qwen2.5-0.5b-megatron critic.megatron.use_dist_checkpointing=true critic.megatron.dist_checkpointing_path=checkpoints/verl-test/qwen2.5-0.5b-megatron reward_model.megatron.use_dist_checkpointing=true reward_model.megatron.dist_checkpointing_path=checkpoints/verl-test/qwen2.5-0.5b-megatron + cp -r checkpoints checkpoints-dut + ENGINE=sglang SAVE_FREQ=1 COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Test Megatron checkpoints merging function (Qwen Actor and Critic) + run: | + exp_name="qwen2.5-0.5b-megatron-gsm8k-minimal" + python -m verl.model_merger test --backend megatron --tie-word-embedding --local_dir checkpoints-dut/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + python -m verl.model_merger test --backend megatron --is-value-model --local_dir checkpoints-dut/verl-test/${exp_name}/global_step_1/critic --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/critic/huggingface + - name: clean up + run: | + rm -rf checkpoints + + e2e_ppo_trainer_megatron-deepseek-override-transformer-config: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + ENGINE=sglang SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct COMMON_PP=2 COMMON_VPP=null bash tests/special_e2e/run_ppo_trainer_megatron.sh +actor_rollout_ref.actor.megatron.override_transformer_config.account_for_embedding_in_pipeline_split=true +actor_rollout_ref.actor.megatron.override_transformer_config.account_for_loss_in_pipeline_split=true + - name: Test Megatron checkpoints merging function (DeepSeek Actor and Critic) + run: | + exp_name="deepseek-coder-1.3b-instruct-megatron-gsm8k-minimal" + python -m verl.model_merger test --backend megatron --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + python -m verl.model_merger test --backend megatron --is-value-model --local_dir checkpoints/verl-test/${exp_name}/global_step_1/critic --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/critic/huggingface + - name: clean up + run: | + rm -rf checkpoints + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_ppo_trainer_megatron-deepseek, + e2e_ppo_trainer_megatron-different-train-infer-tp-qwen-tie-embedding, + e2e_ppo_trainer_megatron-qwen-override-transformer-config, + e2e_ppo_trainer_megatron-deepseek-override-transformer-config, + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml b/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml new file mode 100644 index 0000000000000000000000000000000000000000..7c4cba92c1d19cf7af5780cd8bf63187dec0f1a8 --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml @@ -0,0 +1,275 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_sglang_2 + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Entrypoints + - "verl/worksers/rollout/sglang_rollout/*" + - ".github/workflows/e2e_ppo_trainer_megatron_sglang.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_megatron-moe-expert-parallel: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + MEGATRON_CI_DISABLE_EXPANDABLE_SEGMENTS=1 \ + ADV_ESTIMATOR=grpo USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json \ + PPO_MAX_TOKEN_LEN=512 FWD_MAX_TOKEN_LEN=512 \ + MAX_PROMPT_LENGTH=256 MAX_RESPONSE_LENGTH=256 \ + MODEL_ID=Qwen/Qwen1.5-MoE-A2.7B-Chat \ + ENGINE=sglang COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=4 COMMON_ETP=1 INFER_TP=8 \ + USE_DIST_CKPT=True ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + + e2e_ppo_trainer_megatron-qwen2_5vl-3b: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + - name: Prepare Geo3k dataset + run: | + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ + - name: Prepare dist_ckpt of Qwen2.5-VL-3B, only supports dist_ckpt + run: | + python3 scripts/converter_hf_to_mcore.py --hf_model_path ${HOME}/models/Qwen/Qwen2.5-VL-3B-Instruct --output_path checkpoints/verl-test/qwen2.5-vl-3b-megatron + - name: Running Geo3k E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen) + run: | + ray stop --force + ENGINE=sglang TRAIN_FILES=${HOME}/data/geo3k/train.parquet VAL_FILES=${HOME}/data/geo3k/test.parquet MAX_PROMPT_LENGTH=1024 MAX_RESPONSE_LENGTH=2048 MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct ADV_ESTIMATOR=grpo USE_DYNAMIC_BSZ=False SKIP_SAVE_HF_MODEL=1 COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 COMMON_TP=2 USE_DIST_CKPT=true DIST_CKPT_PATH=checkpoints/verl-test/qwen2.5-vl-3b-megatron bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + + e2e_ppo_trainer_sglang: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test,gpu,sglang] + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm and save ckpt + run: | + ray stop --force + ENGINE=sglang bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on sglang async + run: | + ray stop --force + TOTAL_TRAIN_STEPS=2 ENGINE=sglang ROLLOUT_MODE=async bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + e2e_ppo_trainer_sglang_vlm: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test,geo,gpu,sglang] --no-deps + # Geo3k + - name: Prepare GEO3K dataset + run: | + ray stop --force + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ + - name: Running GEO3K VLM E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2-VL-2B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + ENGINE=sglang GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ + ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GEO3K VLM E2E with rmpad using torch fused kernel (Qwen2.5-VL) + run: | + ray stop --force + FUSED_KERNELS=True TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + ENGINE=sglang GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ + ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GEO3K VLM E2E with rmpad using triton fused kernel (Qwen2.5-VL) + run: | + ray stop --force + FUSED_KERNELS=True FUSED_KERNEL_BACKEND=triton \ + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + ENGINE=sglang GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ + ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_ppo_trainer_megatron-moe-expert-parallel, + e2e_ppo_trainer_megatron-qwen2_5vl-3b, + e2e_ppo_trainer_sglang, + e2e_ppo_trainer_sglang_vlm + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml b/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml new file mode 100644 index 0000000000000000000000000000000000000000..b2966e795c3476f64d44c9bd71bfbb15d1837603 --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml @@ -0,0 +1,292 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_vllm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer_megatron_vllm.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + TRANSFORMERS_VERSION: "4.56.2" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_megatron-deepseek: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip3 install math-verify transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use mbridge e2e to pre-load and save (Deepseek) + run: | + ray stop --force + ALL_OFFLOAD=True SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True USE_DIST_CKPT=False \ + bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use mbridge e2e to pre-load and save (Deepseek) + run: | + ray stop --force + RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 SAVE_FREQ=1 COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True USE_DIST_CKPT=False \ + bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + export VLLM_USE_V1=1 + ray start --head + MODE=async USE_FUSED_KERNELS=True MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 SAVE_FREQ=2 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Test Megatron checkpoints merging function (DeepSeek Actor and Critic) + run: | + exp_name="deepseek-coder-1.3b-instruct-megatron-gsm8k-minimal" + python -m verl.model_merger test --backend megatron --local_dir checkpoints/verl-test/${exp_name}/global_step_2/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_2/actor/huggingface + python -m verl.model_merger test --backend megatron --is-value-model --local_dir checkpoints/verl-test/${exp_name}/global_step_2/critic --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_2/critic/huggingface + - name: Test Megatron distributed checkpoints merging function (DeepSeek) + run: | + exp_name="deepseek-coder-1.3b-instruct-megatron-gsm8k-minimal" + torchrun --nproc_per_node 4 --nnodes 1 -m verl.model_merger merge --backend megatron --local_dir checkpoints/verl-test/${exp_name}/global_step_2/actor --target_dir checkpoints/verl-test/${exp_name}/global_step_2/actor/hf_model + - name: Running GRPO GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Deepseek) + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_DYNAMIC_BSZ=False MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + e2e_ppo_trainer_megatron-qwen3: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip3 install math-verify transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) with validation and saving + run: | + ray stop --force + ALL_OFFLOAD=True VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) testing learning rate scheduler + run: | + ray stop --force + LR_WARMUP_STEPS=1 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh + + - name: Test Megatron checkpoints merging function (Qwen3 Actor and Critic) + run: | + exp_name="qwen3-0.6b-megatron-gsm8k-minimal" + python -m verl.model_merger test --backend megatron --tie-word-embedding --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + python -m verl.model_merger test --backend megatron --is-value-model --local_dir checkpoints/verl-test/${exp_name}/global_step_1/critic --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/critic/huggingface + - name: clean up + run: | + rm -rf checkpoints + e2e_ppo_trainer_megatron-different-train-infer-tp-qwen-tie-embedding: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip3 install math-verify transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with tie-embedding Megatron (Qwen) with train tp > infer tp + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 TRAIN_TP=2 INFER_TP=1 MODEL_ID=Qwen/Qwen2.5-1.5B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen) with train tp < infer tp + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 TRAIN_TP=1 INFER_TP=2 ALL_OFFLOAD=True MODEL_ID=Qwen/Qwen2.5-1.5B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + e2e_ppo_trainer_megatron-qwen-override-transformer-config: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip3 install math-verify transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k +# - name: Download Model to Use +# run: | +# huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir ${HOME}/models/Qwen/Qwen2.5-0.5B +# export HF_HUB_OFFLINE=1 + - name: Prepare dist_ckpt of Qwen2.5-0.5B, uneven layer distribution only supports dist_ckpt + run: | + python3 scripts/converter_hf_to_mcore.py --hf_model_path ${HOME}/models/Qwen/Qwen2.5-0.5B --output_path checkpoints/verl-test/qwen2.5-0.5b-megatron + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen) + run: | + ray stop --force + SAVE_FREQ=1 COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 SKIP_SAVE_HF_MODEL=1 USE_DIST_CKPT=True DIST_CKPT_PATH=checkpoints/verl-test/qwen2.5-0.5b-megatron \ + bash tests/special_e2e/run_ppo_trainer_megatron.sh +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_first_pipeline_stage=8 +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_last_pipeline_stage=4 + cp -r checkpoints checkpoints-dut + SAVE_FREQ=1 COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Test Megatron checkpoints merging function (Qwen Actor and Critic) + run: | + exp_name="qwen2.5-0.5b-megatron-gsm8k-minimal" + python -m verl.model_merger test --backend megatron --tie-word-embedding --local_dir checkpoints-dut/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + python -m verl.model_merger test --backend megatron --is-value-model --local_dir checkpoints-dut/verl-test/${exp_name}/global_step_1/critic --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/critic/huggingface + - name: clean up + run: | + rm -rf checkpoints + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_ppo_trainer_megatron-deepseek, + e2e_ppo_trainer_megatron-qwen3, + e2e_ppo_trainer_megatron-different-train-infer-tp-qwen-tie-embedding, + e2e_ppo_trainer_megatron-qwen-override-transformer-config, + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml b/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml new file mode 100644 index 0000000000000000000000000000000000000000..3183210de1f9f50164379ace6f1b9a256586b84d --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml @@ -0,0 +1,420 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_vllm_2 + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer_megatron_vllm.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + TRANSFORMERS_VERSION: "4.56.2" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_megatron-deepseek-override-transformer-config: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip3 install transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct COMMON_PP=2 COMMON_VPP=null bash tests/special_e2e/run_ppo_trainer_megatron.sh +actor_rollout_ref.actor.megatron.override_transformer_config.account_for_embedding_in_pipeline_split=true +actor_rollout_ref.actor.megatron.override_transformer_config.account_for_loss_in_pipeline_split=true + - name: Test Megatron checkpoints merging function (DeepSeek Actor and Critic) + run: | + exp_name="deepseek-coder-1.3b-instruct-megatron-gsm8k-minimal" + python -m verl.model_merger test --backend megatron --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + python -m verl.model_merger test --backend megatron --is-value-model --local_dir checkpoints/verl-test/${exp_name}/global_step_1/critic --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/critic/huggingface + - name: clean up + run: | + rm -rf checkpoints + e2e_ppo_trainer_megatron-moe-expert-parallel: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip3 install mbridge + pip3 install transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json \ + PPO_MAX_TOKEN_LEN=512 FWD_MAX_TOKEN_LEN=512 \ + MAX_PROMPT_LENGTH=256 MAX_RESPONSE_LENGTH=256 \ + MODEL_ID=Qwen/Qwen1.5-MoE-A2.7B-Chat USE_MBRIDGE=True \ + COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=4 COMMON_ETP=1 INFER_TP=8 \ + USE_DIST_CKPT=True ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + e2e_ppo_trainer_megatron-qwen2_5vl-3b: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip3 install transformers==$TRANSFORMERS_VERSION + - name: Prepare Geo3k dataset + run: | + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ + - name: Prepare dist_ckpt of Qwen2.5-VL-3B, only supports dist_ckpt + run: | + python3 scripts/converter_hf_to_mcore.py --hf_model_path ${HOME}/models/Qwen/Qwen2.5-VL-3B-Instruct --output_path checkpoints/verl-test/qwen2.5-vl-3b-megatron + - name: Running Geo3k E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen) + run: | + ray stop --force + TRAIN_FILES=${HOME}/data/geo3k/train.parquet VAL_FILES=${HOME}/data/geo3k/test.parquet \ + MAX_PROMPT_LENGTH=1024 MAX_RESPONSE_LENGTH=2048 MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct ADV_ESTIMATOR=grpo \ + USE_DYNAMIC_BSZ=False USE_FUSED_KERNELS=True SKIP_SAVE_HF_MODEL=1 \ + COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 COMMON_TP=2 USE_DIST_CKPT=true \ + DIST_CKPT_PATH=checkpoints/verl-test/qwen2.5-vl-3b-megatron bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + e2e_ppo_trainer_vllm: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test,vllm] + pip3 install transformers==$TRANSFORMERS_VERSION + - name: Prepare GSM8K dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + # HF sanity +# - name: Running GSM8K E2E training tests on 1 L20 GPU with hf for sanity +# run: | +# ray stop --force +# bash tests/special_e2e/ppo_trainer/run_single_gpu.sh +# # HF sanity +# - name: Running GSM8K E2E training tests on 1 L20 GPU with engine interface for sanity. +# run: | +# ray stop --force +# bash tests/special_e2e/ppo_trainer/run_single_gpu_with_engine.sh + # Function RM + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP_SIZE=8) + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm after resuming + run: | + ray stop --force + RESUME_MODE=auto VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test merging FSDP checkpoints (Qwen Actor) + run: | + exp_name="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" + python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (DDP_SIZE=2, FSDP_SIZE=4) + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True FSDP_SIZE=4 VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test merging DDP+FSDP checkpoints (Qwen Actor) + run: | + exp_name="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" + python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP2) + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test merging FSDP2 checkpoints (Qwen Actor) + run: | + exp_name="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" + python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + - name: Running GSM8K E2E without rmpad using function rm + run: | + ray stop --force + RM_PAD=False bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm (GRPO) + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm (ReMax) + run: | + ray stop --force + ADV_ESTIMATOR=remax USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using customized reward function + run: | + ray stop --force + CUSTOM_REWARD_FN=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with in-reward kl and kl loss + run: | + ray stop --force + USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh + # LoRA tests + - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True TOTAL_TRAIN_STEPS=1 SAVE_FREQ=1 FSDP_SIZE=4 VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test GRPO LoRA checkpoints merging function + run: | + export EXP_NAME="qwen2.5-0.5b-function-reward-minimal" + ls checkpoints/verl-test/${EXP_NAME}/global_step_1/actor + cat checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface/config.json + python3 -m verl.model_merger merge --backend fsdp --local_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/ --target_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface + - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon with fsdp2 + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh + # Model RM + - name: Running GRPO GSM8K E2E training tests with FSDP on 8 L20 GPUs (DeepSeek) + run: | + ray stop --force + MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E with rmpad using model rm + run: | + ray stop --force + bash tests/special_e2e/ppo_trainer/run_model_reward.sh + - name: Running GSM8K E2E without rmpad using model rm + run: | + ray stop --force + RM_PAD=False bash tests/special_e2e/ppo_trainer/run_model_reward.sh + - name: Running GSM8K E2E with rmpad using model rm and ulysses sp=2 + run: | + ray stop --force + SP_SIZE=2 bash tests/special_e2e/ppo_trainer/run_model_reward.sh + - name: Running GSM8K E2E with rmpad using model rm and dynamic batch size + run: | + ray stop --force + SEQ_BALANCE=True bash tests/special_e2e/ppo_trainer/run_model_reward.sh + - name: Running GSM8K E2E with rmpad using model rm with Liger Kernel enabled + run: | + ray stop --force + LIGER=True bash tests/special_e2e/ppo_trainer/run_model_reward.sh + - name: Running GSM8K E2E with rmpad using model rm with Fused Kernel enabled + run: | + ray stop --force + FUSED_KERNELS=True bash tests/special_e2e/ppo_trainer/run_model_reward.sh + - name: Running GSM8K E2E with rmpad using model rm with Fused Kernel enabled + run: | + ray stop --force + FUSED_KERNEL=True FUSED_KERNEL_BACKEND=triton bash tests/special_e2e/ppo_trainer/run_model_reward.sh + - name: Running GSM8K E2E training tests on vllm async + run: | + ray stop --force + export VLLM_USE_V1=1 + ray start --head + TOTAL_TRAIN_STEPS=2 ENGINE=vllm ROLLOUT_MODE=async bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + e2e_ppo_trainer_vllm_vlm: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test,gpu,vllm,geo,trl] + pip3 install transformers==$TRANSFORMERS_VERSION + # Geo3k + - name: Prepare GEO3K dataset + run: | + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ + - name: Running GEO3K VLM GRPO E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2-VL-2B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + - name: Running GEO3K VLM PPO E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2-VL-2B-Instruct \ + ADV_ESTIMATOR=gae RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GEO3K VLM GRPO E2E lora training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2-VL-2B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + LORA_RANK=32 LORA_EXCLUDE=".*visual.*" \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_ppo_trainer_megatron-deepseek-override-transformer-config, + e2e_ppo_trainer_megatron-moe-expert-parallel, + e2e_ppo_trainer_megatron-qwen2_5vl-3b, + e2e_ppo_trainer_vllm, + e2e_ppo_trainer_vllm_vlm + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/e2e_sft.yml b/e2c/verl/.github/workflows/e2e_sft.yml new file mode 100644 index 0000000000000000000000000000000000000000..75153d06c25772ecc6d57b165e5d539e76a3d3d3 --- /dev/null +++ b/e2c/verl/.github/workflows/e2e_sft.yml @@ -0,0 +1,161 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_sft + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Entrypoints + - ".github/workflows/e2e_sft.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/sft" + - "verl/trainer/fsdp_sft_trainer.py" + - "verl/trainer/config/sft_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + e2e_sft: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 25 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install peft + pip3 install --no-deps -e .[test,gpu] + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs w/o rmpad using function rm + run: | + ray stop --force + RM_PAD=False bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with sequence parallism + run: | + ray stop --force + SP_SIZE=2 bash tests/special_e2e/sft/run_sft.sh + - name: Check loss difference between sequence parallel vs. default implementation + run: | + ray stop --force + ENTRYPOINT="tests/special_e2e/sft/test_sp_loss_match.py" SP_SIZE=2 bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with sequence parallism and liger + run: | + ray stop --force + SP_SIZE=2 LIGER=True bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests with LoRA + run: | + ray stop --force + LORA_RANK=32 bash tests/special_e2e/sft/run_sft.sh + - name: Run GSM8K E2E training and resume tests resuming from the checkpoint manager + run: | + ray stop --force + LORA_RANK=32 RESUME_MODE=auto TOTAL_TRAIN_STEP=2 bash tests/special_e2e/sft/run_sft.sh + # TODO: multiturn + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k_multiturn_sft.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with multiturn and various configs and compare results + run: | + bash tests/special_e2e/sft/test_sft_engine_all.sh + + + cleanup: + runs-on: ubuntu-latest + needs: [setup, e2e_sft] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/e2c/verl/.github/workflows/gpu_unit_tests.yml b/e2c/verl/.github/workflows/gpu_unit_tests.yml new file mode 100644 index 0000000000000000000000000000000000000000..828ce9b59d6dedbda3e00864eed7e5bec16d7057 --- /dev/null +++ b/e2c/verl/.github/workflows/gpu_unit_tests.yml @@ -0,0 +1,112 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: GPU unit tests + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.4.x + paths: + - "**/*.py" + - .github/workflows/gpu_unit_tests.yml + pull_request: + branches: + - main + - v0.4.x + paths: + # The order that you define paths patterns matters: + # A matching negative pattern (prefixed with !) after a positive match will exclude the path. + # A matching positive pattern after a negative match will include the path again. + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + - "!recipe/**" + # Entrypoints + - .github/workflows/gpu_unit_tests.yml + - "tests/**test_*.py" + # Ignore CPU tests + - "!tests/*_on_cpu.py" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + gpu_unit_tests: + runs-on: [L20x8] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1" + HF_HUB_ENABLE_HF_TRANSFER: 1 + container: + image: verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2 + options: --gpus all --shm-size=10g + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install hf_transfer + pip3 install --no-deps -e .[test] + pip3 install --upgrade "ray>=2.40.0" + pip3 install cupy-cuda12x + - name: Download Model to Use + run: | + huggingface-cli download Qwen/Qwen2.5-0.5B-Instruct + huggingface-cli download Qwen/Qwen2.5-1.5B-Instruct + export HF_HUB_OFFLINE=1 + # Disable requests to avoid network errors + - name: Run all GPU unit tests + run: | + pytest -s -x --ignore-glob="*test_special_*.py" --ignore-glob='*on_cpu.py' --ignore-glob="*test_vllm*" --ignore-glob="*_sglang*" --ignore-glob="*_hf_rollout*" --ignore-glob="tests/models/" --ignore-glob='tests/special*' --ignore-glob="tests/experimental" --ignore-glob="tests/workers/reward_model" tests/ + - name: Testing LinearCrossEntropyTP Correctness, Computation Time and Memory Consumption + run: | + LOW_MEMORY=True torchrun --standalone --nnodes=1 --nproc-per-node=8 tests/utils/test_special_linear_cross_entropy_tp.py + - name: Testing FSDP2 actor functionality + run: | + torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/actor/test_special_dp_actor.py + - name: Testing FSDP2 critic functionality + run: | + torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/critic/test_special_dp_critic.py diff --git a/e2c/verl/.github/workflows/model.yml b/e2c/verl/.github/workflows/model.yml new file mode 100644 index 0000000000000000000000000000000000000000..54856b8d914dbe5aa3d087ebc057d89ac384c84b --- /dev/null +++ b/e2c/verl/.github/workflows/model.yml @@ -0,0 +1,230 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: model + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "verl/**/*.py" + # Entrypoints + - ".github/workflows/model.yml" + - "tests/special_distributed/test_fsdp_ckpt.py" + - "tests/special_distributed/test_mcore_config_converter.py" + - "tests/special_distributed/test_tensor_dict.py" + - "tests/models/**" + - "tests/special_distributed/run_all.sh" + +# Declare permissions just read content. +permissions: + contents: read + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + model_rmpad: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository and upgrade to latest transformers(4.54.0)/flash_attn, transformers 4.55.0 has strange behavior with model backward + run: | + pip3 install --no-deps -e .[test] + pip3 install --upgrade transformers + - name: Running rmpad model tests on 8 L20 GPUs + flash_attn 2.5.8 + run: | + pytest -s tests/models/test_transformer.py + - name: Running rmpad model tests on 8 L20 GPUs + latest flash_attn + run: | + pytest -s tests/models/test_transformer.py + - name: Running FSDP rmpad model tests on 8 L20 GPUs + latest flash_attn + run: | + STRATEGY=fsdp torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py + - name: Running transformers ulysses tests on 8 L20 GPUs + latest transformers + run: | + torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py + - name: Running transformers ulysses tests on 8 L20 GPUs + transformers 4.54.1 + run: | + pip3 install transformers==4.54.1 + torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py + - name: Running transformers ulysses tests on 8 L20 GPUs + transformers 4.53.2 + run: | + pip3 install transformers==4.53.2 + torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py + - name: Running transformers ulysses tests on 8 L20 GPUs + transformers 4.52.0 + run: | + pip3 install transformers==4.52.0 + torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py + - name: Run distributed test + run: | + bash tests/special_distributed/run_all.sh + + # TODO: Move this back to model_rmpad once FSDP2 is stable. + # NOTE: List as an independent job to make rerun easier. + model_rmpad_fsdp2_unstable: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository and upgrade to latest transformers/flash_attn + run: | + pip3 install --no-deps -e .[test] + pip3 install --upgrade transformers + - name: Running FSDP2 rmpad model tests on 8 L20 GPUs + latest flash_attn + run: | + STRATEGY=fsdp2 torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py + + mcore_config_converter: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip install --upgrade "huggingface_hub[cli]" +# - name: Download model config files +# run: | +# hf download Qwen/Qwen2.5-7B config.json --local-dir $HOME/configs/Qwen/Qwen2.5-7B +# hf download Qwen/Qwen3-8B config.json --local-dir $HOME/configs/Qwen/Qwen3-8B +# hf download deepseek-ai/deepseek-coder-1.3b-instruct config.json --local-dir $HOME/configs/deepseek-ai/deepseek-coder-1.3b-instruct +# hf download Qwen/Qwen2-57B-A14B config.json --local-dir $HOME/configs/Qwen/Qwen2-57B-A14B +# hf download Qwen/Qwen3-30B-A3B config.json --local-dir $HOME/configs/Qwen/Qwen3-30B-A3B +# hf download deepseek-ai/DeepSeek-V3-Base config.json --local-dir $HOME/configs/deepseek-ai/DeepSeek-V3-Base + - name: Running mcore config converter tests on 8 L20 GPUs + run: | + torchrun --nproc_per_node=8 tests/special_distributed/test_mcore_config_converter.py + + model_engine: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install --no-deps -e .[test] + pip3 install --upgrade tensordict transformers + pip install --upgrade "huggingface_hub[cli]" + - name: Download model config files + run: | + hf download Qwen/Qwen2.5-0.5B-Instruct --local-dir $HOME/models/Qwen/Qwen2.5-0.5B-Instruct + + - name: Running mcore engine tests on 8 L20 GPUs + run: | + pytest -s -x tests/models/test_engine.py + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + model_rmpad, + model_rmpad_fsdp2_unstable, + mcore_config_converter, + model_engine + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/pre-commit-full.yml b/e2c/verl/.github/workflows/pre-commit-full.yml new file mode 100644 index 0000000000000000000000000000000000000000..be2dea6da98dd5cddfcdc950a53b341412a7d63b --- /dev/null +++ b/e2c/verl/.github/workflows/pre-commit-full.yml @@ -0,0 +1,30 @@ +name: pre-commit-full + +# Run weekly on Sunday at 00:00 UTC +on: + schedule: + - cron: "0 0 * * 0" + # Allow manual triggering + workflow_dispatch: + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + pre-commit-full: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Set ruff --output-format=github + run: | + sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml + git add .pre-commit-config.yaml + - uses: pre-commit/action@v3.0.1 diff --git a/e2c/verl/.github/workflows/pre-commit.yml b/e2c/verl/.github/workflows/pre-commit.yml new file mode 100644 index 0000000000000000000000000000000000000000..80cfa0945664576a33639d7a16b547b82145389f --- /dev/null +++ b/e2c/verl/.github/workflows/pre-commit.yml @@ -0,0 +1,36 @@ +# c.f. https://github.com/pre-commit/action?tab=readme-ov-file#using-this-action +name: pre-commit + +# No need to avoid / cancel lightweight pre-commit jobs +on: + pull_request: + push: + branches: + - main + - v0.* + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + pre-commit: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip install -e . + - name: Set ruff --output-format=github + run: | + sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml + git add .pre-commit-config.yaml + # Check "--all-files" by default + - uses: pre-commit/action@v3.0.1 diff --git a/e2c/verl/.github/workflows/reward_model.yml b/e2c/verl/.github/workflows/reward_model.yml new file mode 100644 index 0000000000000000000000000000000000000000..7ba377194e0fd154d56e6246885f293a19fcb283 --- /dev/null +++ b/e2c/verl/.github/workflows/reward_model.yml @@ -0,0 +1,131 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: reward_model + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "verl/**/*.py" + # Entrypoints + - ".github/workflows/reward_model.yml" + - "tests/workers/reward_model/**" + +# Declare permissions just read content. +permissions: + contents: read + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-sglang0.4.10.post2-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + TRANSFORMERS_VERSION: "4.56.2" + + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + reward_model: + needs: setup + runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" + NCCL_SHM_DISABLE: "1" + NCCL_P2P_DISABLE: "1" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test] +# - name: Download model config files +# run: | +# hf download Skywork/Skywork-Reward-V2-Llama-3.2-1B --local-dir $HOME/models/Skywork/Skywork-Reward-V2-Llama-3.2-1B +# hf download verl-team/GenRM-CI-Test-1.5B --local-dir $HOME/models/verl-team/GenRM-CI-Test-1.5B + - name: Running discriminative reward model tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + pytest -s -x tests/workers/reward_model/test_discriminative_reward_model.py + - name: Running generative reward model tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + pytest -s -x tests/workers/reward_model/test_generative_reward_model.py + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + reward_model + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" \ No newline at end of file diff --git a/e2c/verl/.github/workflows/sanity.yml b/e2c/verl/.github/workflows/sanity.yml new file mode 100644 index 0000000000000000000000000000000000000000..f3bd7766135aaf8ee5d3aab7ce16c6272265152d --- /dev/null +++ b/e2c/verl/.github/workflows/sanity.yml @@ -0,0 +1,109 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: sanity + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - .github/workflows/sanity.yml + - "tests/special_sanity/**" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + sanity: + runs-on: ubuntu-latest + timeout-minutes: 5 # Increase this timeout value as needed + strategy: + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu + pip3 install -r requirements.txt + pip install -e .[test] + - name: Run sanity test + run: | + pytest -s -x tests/special_sanity + - name: Run license test + run: | + python3 tests/special_sanity/check_license.py --directories . + - name: Assert naming convention + run: | + if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=__pycache__ 'veRL' .; then + echo "Please use verl instead of veRL in the codebase" + exit 1 + fi + - name: Assert SGLang naming convention + run: | + if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=__pycache__ -E 'Sglang|sgLang|sglAng|sglaNg|sglanG' .; then + echo "Please use SGLang or sglang as the formal name of SGLang rollout engine" + exit 1 + fi + - name: Validate test folder structure + run: python3 tests/special_sanity/validate_structure.py + - name: Assert documentation requirement for functions + run: python3 tests/special_sanity/validate_imported_docs.py + - name: Assert device api usage in verl/recipe + run: python3 tests/special_sanity/check_device_api_usage.py --directory ./recipe + - name: Assert device api usage in verl/verl + run: python3 tests/special_sanity/check_device_api_usage.py --directory ./verl + - name: Assert documentation time info + run: python3 tests/special_sanity/check_docs_time_info.py + - name: Check docstrings for specified files + run: python3 tests/special_sanity/check_docstrings.py + - name: Check DataProto for specified folders + run: python3 tests/special_sanity/check_dataproto_usage.py -d ./verl/workers/engine diff --git a/e2c/verl/.github/workflows/scorecard.yml b/e2c/verl/.github/workflows/scorecard.yml new file mode 100644 index 0000000000000000000000000000000000000000..176d15ae2bd470752daaf138fa5aaa90641738e9 --- /dev/null +++ b/e2c/verl/.github/workflows/scorecard.yml @@ -0,0 +1,66 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: "27 7 * * 1" + push: + branches: + - main + - v0.* + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + # contents: read + # actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecard on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results to GitHub's code scanning dashboard (optional). + # Commenting out will disable upload of results to your repo's Code Scanning dashboard + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 #v3.28.9 + with: + sarif_file: results.sarif diff --git a/e2c/verl/.github/workflows/secrets_scan.yml b/e2c/verl/.github/workflows/secrets_scan.yml new file mode 100644 index 0000000000000000000000000000000000000000..298ed16c668c67facdb6af2119878da576f5bdf5 --- /dev/null +++ b/e2c/verl/.github/workflows/secrets_scan.yml @@ -0,0 +1,22 @@ +on: + push: + branches: + - main + - v0.* + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + - name: Secret Scanning + uses: trufflesecurity/trufflehog@7dc056a193116ba8d82154bf0549381c8fb8545c # v3.88.14 + with: + extra_args: --results=verified,unknown diff --git a/e2c/verl/.github/workflows/sgl.yml b/e2c/verl/.github/workflows/sgl.yml new file mode 100644 index 0000000000000000000000000000000000000000..1f490ddfb8f025f116e4e975ba67dead4d779322 --- /dev/null +++ b/e2c/verl/.github/workflows/sgl.yml @@ -0,0 +1,178 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: sgl + +on: +# workflow_dispatch: # Manual + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + - .github/workflows/sgl.yml + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Megatron + - "!verl/workers/**/megatron_*.py" + # vLLM + - "!**/*vllm*" + # Recipes + - "!recipe/**" + # Entrypoints + - ".github/workflows/sgl.yml" + - "tests/rollout/*sglang*" + - "tests/rollout/async_rollout_utils.py" + - "tests/workers/rollout/*interaction*" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + sgl: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 35 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: 1 + SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" + NCCL_SHM_DISABLE: "1" + NCCL_P2P_DISABLE: "1" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install hf_transfer fastmcp + pip3 install -e .[test] +# - name: Download Model to Use +# run: | +# huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir ${HOME}/models/Qwen/Qwen2.5-0.5B +# huggingface-cli download Qwen/Qwen2.5-1.5B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-1.5B-Instruct +# huggingface-cli download Qwen/Qwen2.5-VL-3B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-VL-3B-Instruct +# export HF_HUB_OFFLINE=1 + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Test the latest SGLang Rollout async with agent loop + run: | + ROLLOUT_NAME=sglang pytest -svvv tests/experimental/agent_loop +# huggingface-cli download verl-team/gsm8k-v0.4.1 --repo-type dataset --local-dir ~/verl-data/gsm8k + - name: Test the latest SGLang + run: | + cd tests/workers/rollout + torchrun --nnodes=1 --nproc_per_node=2 $(which pytest) -s test_sglang_spmd.py + - name: Test the latest SGLang Rollout async with interaction + run: | + cd tests/workers/rollout + torchrun --nnodes=1 --nproc_per_node=2 $(which pytest) -s test_sglang_async_rollout_w_interaction.py + - name: Test the latest SGLang Multi Interaction + run: | + cd tests/workers/rollout + torchrun --nnodes=1 --nproc_per_node=2 $(which pytest) -s test_sglang_multi_interaction.py + - name: Test the latest SGLang Rollout async with tool + run: | + cd tests/workers/rollout + torchrun --nnodes=1 --nproc_per_node=2 $(which pytest) -s test_sglang_async_rollout_w_tools.py + - name: Test the latest SGLang Rollout async with sandbox fusion tool + run: | + cd tests/workers/rollout + pytest -s test_sglang_async_rollout_sf_tools.py + - name: Test the latest SGLang Rollout async with search tool + run: | + cd tests/workers/rollout + pytest -s test_sglang_async_rollout_search_tools.py + - name: Test the latest SGLang Rollout async with mcp search tool + run: | + cd tests/workers/rollout + pytest -s test_sglang_async_rollout_mcp_tools.py + # Note(haibin.lin): for any new test, please update gpu_unit_tests.yaml to avoid repeated tests + - name: Test the latest SGLang Rollout async with multimodal delta + run: | + cd tests/workers/rollout + pytest -s test_sglang_async_rollout_multimodal_delta.py + + cleanup: + runs-on: ubuntu-latest + needs: [setup, sgl] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/e2c/verl/.github/workflows/type-coverage-check.yml b/e2c/verl/.github/workflows/type-coverage-check.yml new file mode 100644 index 0000000000000000000000000000000000000000..aa8c03a54b62f02292152e58d04c89087b9454b5 --- /dev/null +++ b/e2c/verl/.github/workflows/type-coverage-check.yml @@ -0,0 +1,31 @@ +name: Type Annotation and Docstring Coverage + +on: + pull_request: + paths: + - '**/*.py' + - '.github/workflows/type-coverage-check.yml' + +jobs: + type-coverage-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # 🚨 Important: fetch full history so `origin/main` is available + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu + pip3 install -r requirements.txt + pip3 install -e . --no-deps + - name: Run type annotation coverage check + run: | + python3 tests/special_sanity/type_coverage_check.py + - name: Run docstring coverage check + run: | + python3 tests/special_sanity/check_api_docs.py verl diff --git a/e2c/verl/.github/workflows/vllm.yml b/e2c/verl/.github/workflows/vllm.yml new file mode 100644 index 0000000000000000000000000000000000000000..7e2bc563fd34148d9bd333ea19544688611bb7c6 --- /dev/null +++ b/e2c/verl/.github/workflows/vllm.yml @@ -0,0 +1,145 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: vllm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Megatron + - "!verl/workers/**/megatron_*.py" + # SGLang + - "!**/*sglang*" + # Entrypoints + - ".github/workflows/vllm.yml" + - "tests/special_e2e/generation" + - "tests/workers/rollout" + - "verl/trainer/main_generation.py" + - "verl/trainer/config/generation.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'volcengine' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + vllm: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 35 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -e .[test] +# - name: Download Model to Use +# run: | +# huggingface-cli download Qwen/Qwen2.5-0.5B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-0.5B-Instruct +# huggingface-cli download Qwen/Qwen2.5-1.5B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-1.5B-Instruct +# huggingface-cli download Qwen/Qwen2.5-VL-3B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-VL-3B-Instruct +# huggingface-cli download OldKingMeister/Qwen2.5-1.5B-Instruct-YaRN --local-dir ${HOME}/models/OldKingMeister/Qwen2.5-1.5B-Instruct-YaRN +# export HF_HUB_OFFLINE=1 + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Test the latest vLLM Rollout async with agent loop + run: | + ROLLOUT_NAME=vllm pytest -svvv tests/experimental/agent_loop + - name: Test the latest vLLM + run: | + torchrun --standalone --nnodes=1 --nproc_per_node=4 $(which pytest) -s tests/workers/rollout/rollout_vllm/test_vllm_spmd.py + - name: Test the latest vLLM on model with rope scaling + run: | + torchrun --standalone --nnodes=1 --nproc_per_node=4 $(which pytest) -s tests/workers/rollout/rollout_vllm/test_vllm_model_rope_scaling.py + # Note(haibin.lin): for any new test, please update gpu_unit_tests.yaml to avoid repeated tests + + cleanup: + runs-on: ubuntu-latest + needs: [setup, vllm] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/e2c/verl/.gitignore b/e2c/verl/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..d77a5b43ffc6887b2569fcde32bfa4bb86f7ef20 --- /dev/null +++ b/e2c/verl/.gitignore @@ -0,0 +1,128 @@ + +**/*.pt +**/checkpoints +**/wget-log +**/_build/ +**/*.ckpt +**/outputs +**/*.tar.gz +**/playground +**/wandb + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +dataset/* +tensorflow/my_graph/* +.idea/ +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +tmp/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +pytest.ini +output.txt + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +.venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# vscode +.vscode + +# Mac +.DS_Store + +# vim +*.swp + +# ckpt +*.lock + +# data +*.parquet + + +# local logs +logs +log +outputs +.history diff --git a/e2c/verl/.pre-commit-config.yaml b/e2c/verl/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd77c362015f6e97767a18aa63cc50916fae12c0 --- /dev/null +++ b/e2c/verl/.pre-commit-config.yaml @@ -0,0 +1,37 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.12.2" + hooks: + - id: ruff + args: ["--fix", "--show-fixes", "--output-format=full"] + exclude: ^.*\.(ipynb)$ + - id: ruff-format + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.17.0' + hooks: + - id: mypy + + - repo: local + hooks: + - id: autogen-trainer-cfg + name: Generate and verify verl/trainer/config/_generated_*.yaml + entry: scripts/generate_trainer_config.sh + language: script + pass_filenames: false + + - repo: local + hooks: + - id: check-docstrings + name: Check doc string coverage + entry: python3 tests/special_sanity/check_docstrings.py + language: python + pass_filenames: false + + - repo: local + hooks: + - id: check-license + name: Check license + entry: python3 tests/special_sanity/check_license.py --directories examples recipe scripts tests verl setup.py + language: python + pass_filenames: false diff --git a/e2c/verl/.readthedocs.yaml b/e2c/verl/.readthedocs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0016868541a2a0667ef40ae6a9d861bcd26b9316 --- /dev/null +++ b/e2c/verl/.readthedocs.yaml @@ -0,0 +1,19 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + rust: "1.70" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: docs/requirements-docs.txt + - method: pip + path: . diff --git a/e2c/verl/.vscode/settings.json b/e2c/verl/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..705533538d75025d58f108b0d5ae1ec7b5a470b5 --- /dev/null +++ b/e2c/verl/.vscode/settings.json @@ -0,0 +1,15 @@ +{ + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.codeActionsOnSave": { + "source.organizeImports": "always", + } + }, + "files.associations": { + "array": "cpp", + "string_view": "cpp", + "initializer_list": "cpp", + "utility": "cpp" + }, + "iis.configDir": "" +} \ No newline at end of file diff --git a/e2c/verl/CONTRIBUTING.md b/e2c/verl/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..e953f113ed2e5665acd03d5ee93774baeb0049c9 --- /dev/null +++ b/e2c/verl/CONTRIBUTING.md @@ -0,0 +1,89 @@ +# Contributing to verl + +Thank you for considering a contribution to verl! We welcome contributions of any kind - bug fixes, enhancements, documentation improvements, or even just feedback. Whether you're an experienced developer or this is your first open-source project, your help is invaluable. + +Your support can take many forms: +- Report issues or unexpected behaviors. +- Suggest or implement new features. +- Improve or expand documentation. +- Review pull requests and assist other contributors. +- Spread the word: share verl in blog posts, social media, or give the repo a ⭐. + +## Finding Issues to Contribute + +Looking for ways to dive in? Check out these issues: +- [Good first issues](https://github.com/volcengine/verl/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22good%20first%20issue%22) +- [Call for contribution](https://github.com/volcengine/verl/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22call%20for%20contribution%22) +Furthermore, you can learn the development plan and roadmap via [RFC](https://github.com/volcengine/verl/issues?q=is%3Aissue%20state%3Aopen%20label%3ARFC) and [Roadmap](https://github.com/volcengine/verl/issues?q=state%3Aopen%20label%3A%22roadmap%22). + + +## Developing + +- **Python-only**: install verl via `pip install -e .[test,vllm]` or `pip install -e .[test,sglang]` and iterate quickly. For full dependency setup, check out the verl [installation doc](https://verl.readthedocs.io/en/latest/start/install.html). + +## Code Linting and Formatting + +We rely on pre-commit to keep our code consistent. To set it up: + +```bash +pip install pre-commit +pre-commit install +# for staged changes +pre-commit run +# for all files in the repo +pre-commit run --all-files +# run a specific hook with pre-commit +# pre-commit run --all-files --show-diff-on-failure --color=always +pre-commit run --all-files --show-diff-on-failure --color=always ruff +pre-commit run --all-files --show-diff-on-failure --color=always autogen-trainer-cfg +``` + +## Testing + +Our test suites run on GitHub Actions. Check these workflows for details: +- [GPU unit tests](https://github.com/volcengine/verl/blob/main/.github/workflows/gpu_unit_tests.yml) +- [CPU unit tests](https://github.com/volcengine/verl/blob/main/.github/workflows/cpu_unit_tests.yml) +- [vLLM tests](https://github.com/volcengine/verl/blob/main/.github/workflows/vllm.yml) +- [SGLang tests](https://github.com/volcengine/verl/blob/main/.github/workflows/sgl.yml) + +### Adding CI tests + +If possible, please add CI test(s) for your new feature: + +1. Find the most relevant workflow yml file, which usually corresponds to a `hydra` default config (e.g. `ppo_trainer`, `ppo_megatron_trainer`, `sft_trainer`, etc). +2. Add related path patterns to the `paths` section if not already included. +3. Minimize the workload of the test script(s) (see existing scripts for examples). + +## Building the Docs +``` +# Ensure verl is on your PYTHONPATH, e.g.: +pip install -e .[test] + +# Install documentation dependencies +pip install -r requirements-docs.txt + +# Generate HTML docs +make clean +make html + +# Preview locally +python -m http.server -d _build/html/ +``` +Open your browser at http://localhost:8000 to explore the docs. + +## Pull Requests & Code Reviews + +Thanks for submitting a PR! To streamline reviews: +- Follow our Pull Request Template for title format and checklist. +- Adhere to our pre-commit lint rules and ensure all checks pass. +- Update docs for any user-facing changes. +- Add or update tests in the CI workflows, or explain why tests aren't applicable. + +## License + +See the [LICENSE](https://github.com/volcengine/verl/blob/main/LICENSE) file for full details. + +## Thank You + +We appreciate your contributions to verl. Your efforts help make the project stronger and more user-friendly. Happy coding! + diff --git a/e2c/verl/LICENSE b/e2c/verl/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/e2c/verl/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/e2c/verl/Notice.txt b/e2c/verl/Notice.txt new file mode 100644 index 0000000000000000000000000000000000000000..ade439da525ac3f82936e131a1ae386f43207fd8 --- /dev/null +++ b/e2c/verl/Notice.txt @@ -0,0 +1 @@ +Copyright 2023-2024 Bytedance Ltd. and/or its affiliates \ No newline at end of file diff --git a/e2c/verl/README.md b/e2c/verl/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f41bbedc8aa5b05b4eafe02814d8cdd14335c4a6 --- /dev/null +++ b/e2c/verl/README.md @@ -0,0 +1,264 @@ +
+ 👋 Hi, everyone! + verl is a RL training library initiated by ByteDance Seed team and maintained by the verl community. +
+
+
+ +
+ +Ask DeepWiki.com +[![GitHub Repo stars](https://img.shields.io/github/stars/volcengine/verl)](https://github.com/volcengine/verl/stargazers) +[![Twitter](https://img.shields.io/twitter/follow/verl_project)](https://twitter.com/verl_project) + + +[![Documentation](https://img.shields.io/badge/documentation-blue)](https://verl.readthedocs.io/en/latest/) + + +
+ +![seed logo](https://github.com/user-attachments/assets/c42e675e-497c-4508-8bb9-093ad4d1f216) + +

verl: Volcano Engine Reinforcement Learning for LLMs

+ +verl is a flexible, efficient and production-ready RL training library for large language models (LLMs). + +verl is the open-source version of **[HybridFlow: A Flexible and Efficient RLHF Framework](https://arxiv.org/abs/2409.19256v2)** paper. + +verl is flexible and easy to use with: + +- **Easy extension of diverse RL algorithms**: The hybrid-controller programming model enables flexible representation and efficient execution of complex post-training dataflows. Build RL dataflows such as GRPO, PPO in a few lines of code. + +- **Seamless integration of existing LLM infra with modular APIs**: Decouples computation and data dependencies, enabling seamless integration with existing LLM frameworks, such as FSDP, Megatron-LM, vLLM, SGLang, etc + +- **Flexible device mapping**: Supports various placement of models onto different sets of GPUs for efficient resource utilization and scalability across different cluster sizes. + +- Ready integration with popular HuggingFace models + +verl is fast with: + +- **State-of-the-art throughput**: SOTA LLM training and inference engine integrations and SOTA RL throughput. + +- **Efficient actor model resharding with 3D-HybridEngine**: Eliminates memory redundancy and significantly reduces communication overhead during transitions between training and generation phases. + +

+ +## News +- [2025/08] verl is presented in the [PyTorch Expert Exchange Webinar](https://www.youtube.com/watch?v=Vd79NmmqY3Q&t=2s). [Slides](https://github.com/eric-haibin-lin/verl-community/blob/main/slides/verl_talk_pytorch_2025_08.pdf) available. +- [2025/07] The [ReTool](https://arxiv.org/pdf/2504.11536) recipe is fully open sourced. [Blog](https://www.notion.so/verl-reTool-recipe-Using-multi-round-conversations-and-code-sandboxing-to-improve-the-math-of-large-23a8b5b7feba80b386b2e5b5e3c1cde0) +- [2025/07] The first verl meetup will be held at ICML Vancouver on July 16th! Please [join us](https://lu.ma/0ek2nyao) if you are at ICML! (onsite only) +- [2025/06] verl with Megatron backend enables large MoE models such as [DeepSeek-671B and Qwen3-235B](https://verl.readthedocs.io/en/latest/perf/dpsk.html). +- [2025/03] [DAPO](https://dapo-sia.github.io/) is the open-sourced SOTA RL algorithm that achieves 50 points on AIME 2024 based on the Qwen2.5-32B pre-trained model, surpassing the previous SOTA achieved by DeepSeek's GRPO (DeepSeek-R1-Zero-Qwen-32B). DAPO's training is fully powered by verl and the reproduction code is available in `recipe/dapo` now. +
more... +
    +
  • [2025/04] [Seed-Thinking-v1.5](https://github.com/ByteDance-Seed/Seed-Thinking-v1.5/blob/main/seed-thinking-v1.5.pdf) tech report is released! Trained with verl, Seed-Thinking-v1.5 achieves 86.7 on AIME 2024, 55.0 on Codeforces and 77.3 on GPQA, demonstrating excellent reasoning abilities in STEM and coding. Beyond reasoning tasks, the method demonstrates notable generalization across diverse domains.
  • +
  • [2025/07] verl keynote at [AWS AI Hours Singapore](https://pages.awscloud.com/aws-ai-hours-sg.html#agenda) on 7/8, verl & verl-agent project updates at [Agent for SWE meetup](https://lu.ma/e498qhsi) by LF AI & Data Singapore on 7/11.
  • +
  • [2025/06] verl team will provide latest project updates at [PyTorch Day China](https://www.lfasiallc.com/pytorch-day-china/) on June 7th. Meet our dev team in Beijing!
  • +
  • [2025/04] [VAPO](https://arxiv.org/pdf/2504.05118) (value-based augmented PPO) paper covers our latest RL method for reasoning models. Trained from Qwen-32B-base model, VAPO achieves 60.4 on AIME 2024, outperforming DAPO-32B.
  • +
  • [2025/05] [PF-PPO](https://arxiv.org/abs/2409.06957), accepted to ICML 2025, is now supported in verl! PF-PPO enhances policy learning efficiency and robustness by filtering potentially noisy reward signals and reusing high-quality experiences via a replay buffer.
  • +
  • [2025/04] We will give a tutorial about latest post-training techniques and programming guide for verl at [ICLR 2025 Expo](https://iclr.cc/virtual/2025/calendar?filter_events=Expo+Talk+Panel&filter_rooms=), [SCI-FM workshop](https://open-foundation-model.github.io/) and [LMSys afterparty](https://lu.ma/d23nyynm). Talk materials available [here](https://github.com/eric-haibin-lin/verl-community/tree/main/iclr25).
  • +
  • [2025/03] verl v0.3.0.post1 is released! See [release note](https://github.com/volcengine/verl/releases/) for details. It achieves [~1.4x speedup](https://tongyx361.github.io/blogs/posts/verl-intro/#/verl-flexible-and-efficient-rl-for-llms) compared to prev versions.
  • +
  • [2025/05] verl will be presented at [A2M Shanghai](https://a2m.msup.com.cn/home/?aid=4488&city=shanghai) on 5/16 - 5/17.
  • +
  • [2025/05] verl will be presented at [GOSIM x PyTorch Day 2025](https://paris2025.gosim.org/). See you in Paris!
  • +
  • [2025/03] We introduced the programming model of verl at the [vLLM Beijing Meetup](https://mp.weixin.qq.com/s/n77GibL2corAtQHtVEAzfg) and [verl intro and updates](https://github.com/eric-haibin-lin/verl-community/blob/main/slides/verl-lmsys-meetup.pdf) at the [SGLang-LMSYS Org Meetup](https://lu.ma/ntjrr7ig) in Sunnyvale mid-March.
  • +
  • [2025/03] We will present verl(HybridFlow) at EuroSys 2025. See you in Rotterdam!
  • +
  • [2025/02] verl v0.2.0.post2 is released!
  • +
  • [2025/02] We presented verl in the Bytedance/NVIDIA/Anyscale Ray Meetup. See you in San Jose!
  • +
  • [2025/01] [Doubao-1.5-pro](https://team.doubao.com/zh/special/doubao_1_5_pro) is released with SOTA-level performance on LLM & VLM. The RL scaling preview model is trained using verl, reaching OpenAI O1-level performance on math benchmarks (70.0 pass@1 on AIME).
  • +
  • [2024/12] verl is presented at Ray Forward 2024. Slides available here
  • +
  • [2024/12] The team presented Post-training LLMs: From Algorithms to Infrastructure at NeurIPS 2024. Slides and video available.
  • +
  • [2024/10] verl is presented at Ray Summit. Youtube video available.
  • +
  • [2024/08] HybridFlow (verl) is accepted to EuroSys 2025.
  • +
+
+ +## Key Features + +- **FSDP**, **FSDP2** and **Megatron-LM** for training. +- **vLLM**, **SGLang** and **HF Transformers** for rollout generation. +- Compatible with Hugging Face Transformers and Modelscope Hub: [Qwen-3](https://github.com/volcengine/verl/blob/main/examples/grpo_trainer/run_qwen3-8b.sh), Qwen-2.5, Llama3.1, Gemma2, DeepSeek-LLM, etc +- Supervised fine-tuning. +- Reinforcement learning with [PPO](examples/ppo_trainer/), [GRPO](examples/grpo_trainer/), [GSPO](recipe/gspo/), [ReMax](examples/remax_trainer/), [REINFORCE++](https://verl.readthedocs.io/en/latest/examples/config.html#algorithm), [RLOO](examples/rloo_trainer/), [PRIME](recipe/prime/), [DAPO](recipe/dapo/), [DrGRPO](recipe/drgrpo), [KL_Cov & Clip_Cov](recipe/entropy) etc. + - Support model-based reward and function-based reward (verifiable reward) for math, [coding](https://github.com/volcengine/verl/tree/main/recipe/dapo), etc + - Support vision-language models (VLMs) and [multi-modal RL](examples/grpo_trainer/run_qwen2_5_vl-7b.sh) with Qwen2.5-vl, Kimi-VL + - [Multi-turn with tool calling](https://github.com/volcengine/verl/tree/main/examples/sglang_multiturn) +- LLM alignment recipes such as [Self-play preference optimization (SPPO)](https://github.com/volcengine/verl/tree/main/recipe/sppo) +- Flash attention 2, [sequence packing](examples/ppo_trainer/run_qwen2-7b_seq_balance.sh), [sequence parallelism](examples/ppo_trainer/run_deepseek7b_llm_sp2.sh) support via DeepSpeed Ulysses, [LoRA](examples/sft/gsm8k/run_qwen_05_peft.sh), [Liger-kernel](examples/sft/gsm8k/run_qwen_05_sp2_liger.sh). +- Scales up to 671B models and hundreds of GPUs with [expert parallelism](https://github.com/volcengine/verl/pull/1467) +- Multi-gpu [LoRA RL](https://verl.readthedocs.io/en/latest/advance/ppo_lora.html) support to save memory. +- Experiment tracking with wandb, swanlab, mlflow and tensorboard. + +## Upcoming Features and Changes + +- Q3 Roadmap https://github.com/volcengine/verl/issues/2388 +- DeepSeek 671b optimizations with Megatron https://github.com/volcengine/verl/issues/1033 +- Multi-turn rollout and tools using optimizations https://github.com/volcengine/verl/issues/1882 +- [Agent integration](https://github.com/volcengine/verl/tree/main/verl/experimental/agent_loop) +- Async and off-policy architecture https://github.com/volcengine/verl/pull/2231 +- List of breaking changes since v0.4 https://github.com/volcengine/verl/discussions/2270 + +## Getting Started + +Documentation + +**Quickstart:** + +- [Installation](https://verl.readthedocs.io/en/latest/start/install.html) +- [Quickstart](https://verl.readthedocs.io/en/latest/start/quickstart.html) +- [Programming Guide](https://verl.readthedocs.io/en/latest/hybrid_flow.html) & [Tech Talk](https://hcqnc.xetlk.com/sl/3vACOK) (in Chinese) +- [PPO in verl](https://verl.readthedocs.io/en/latest/algo/ppo.html) +- [GRPO in verl](https://verl.readthedocs.io/en/latest/algo/grpo.html) + +**Running a PPO example step-by-step:** + +- [Prepare Data for Post-Training](https://verl.readthedocs.io/en/latest/preparation/prepare_data.html) +- [Implement Reward Function for Dataset](https://verl.readthedocs.io/en/latest/preparation/reward_function.html) +- [PPO Example Architecture](https://verl.readthedocs.io/en/latest/examples/ppo_code_architecture.html) +- [Config Explanation](https://verl.readthedocs.io/en/latest/examples/config.html) + +**Reproducible algorithm baselines:** + +- [RL performance on coding, math](https://verl.readthedocs.io/en/latest/algo/baseline.html) + +**For code explanation and advance usage (extension):** + +- PPO Trainer and Workers + - [PPO Ray Trainer](https://verl.readthedocs.io/en/latest/workers/ray_trainer.html) + - [PyTorch FSDP Backend](https://verl.readthedocs.io/en/latest/workers/fsdp_workers.html) + - [Megatron-LM Backend](https://verl.readthedocs.io/en/latest/index.html) + +- Advanced Usage and Extension + - [Add Models with the FSDP Backend](https://verl.readthedocs.io/en/latest/advance/fsdp_extension.html) + - [Add Models with the Megatron-LM Backend](https://verl.readthedocs.io/en/latest/advance/megatron_extension.html) + - [Multi-turn Rollout Support](https://verl.readthedocs.io/en/latest/sglang_multiturn/multiturn.html) + - [Search Tool Integration](https://verl.readthedocs.io/en/latest/sglang_multiturn/search_tool_example.html) + - [Sandbox Fusion Integration](https://verl.readthedocs.io/en/latest/examples/sandbox_fusion_example.html) + - [Deployment using Separate GPU Resources](https://github.com/volcengine/verl/tree/main/examples/split_placement) + - [Extend to Other RL(HF) algorithms](https://verl.readthedocs.io/en/latest/advance/dpo_extension.html) + - [Ray API design tutorial](https://verl.readthedocs.io/en/latest/advance/placement.html) + +**Blogs from the community** + +- [When Reasoning Models Break Tokenization: The Hidden Complexity of Multiturn Training](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/multi-turn/fast_tokenization/multiturn_tokenization_and_masking.md) +- [verl deployment on AWS SageMaker](https://medium.com/@kaige.yang0110/run-verl-on-sagemaker-using-4x8-l40s-gpus-8e6d5c3c61d3) +- [verl x SGLang Multi-turn Code Walkthrough](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/multi-turn/code-walk-through/readme_EN.md) +- [Optimizing SGLang Memory Usage in verl](https://hebiao064.github.io/rl-memory-management) +- [SGLang, verl, OpenBMB and Tsinghua University: Pioneering End-to-End Multi-Turn RLHF](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/multi-turn/verl-multiturn-rollout-Release.md) +- [Reinforcement Learning from Human Feedback on AMD GPUs with verl and ROCm Integration](https://rocm.blogs.amd.com/artificial-intelligence/verl-large-scale/README.html) +- [veMLP x verl :玩转强化学习训练](https://mp.weixin.qq.com/s/7nbqxk4knMGd-hQE9ls2tA) +- [使用 verl 进行 GRPO 分布式强化学习训练最佳实践](https://www.volcengine.com/docs/6459/1463942) +- [HybridFlow verl 原文浅析](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/readme.md) +- [最高提升 20 倍吞吐量!豆包大模型团队发布全新 RLHF 框架,现已开源!](https://team.doubao.com/en/blog/%E6%9C%80%E9%AB%98%E6%8F%90%E5%8D%8720%E5%80%8D%E5%90%9E%E5%90%90%E9%87%8F-%E8%B1%86%E5%8C%85%E5%A4%A7%E6%A8%A1%E5%9E%8B%E5%9B%A2%E9%98%9F%E5%8F%91%E5%B8%83%E5%85%A8%E6%96%B0-rlhf-%E6%A1%86%E6%9E%B6-%E7%8E%B0%E5%B7%B2%E5%BC%80%E6%BA%90) + +## Performance Tuning Guide + +The performance is essential for on-policy RL algorithm. We have written a detailed [performance tuning guide](https://verl.readthedocs.io/en/latest/perf/perf_tuning.html) to help you optimize performance. + +## Upgrade to vLLM >= v0.8.2 + +verl now supports vLLM>=0.8.2 when using FSDP as the training backend. Please refer to [this document](https://github.com/volcengine/verl/blob/main/docs/README_vllm0.8.md) for the installation guide and more information. Please avoid vllm 0.7.x, which contains bugs that may lead to OOMs and unexpected errors. + +## Use Latest SGLang + +SGLang is fully supported with verl, and SGLang RL Group is working extensively on building unique features, including multi-turn agentic RL, VLM RLHF, server-based RL, and partial rollout. Please refer to [this document](https://verl.readthedocs.io/en/latest/workers/sglang_worker.html) for the installation guide and more information. + +## Upgrade to FSDP2 + +verl is fully embracing FSDP2! FSDP2 is recommended by torch distributed team, providing better throughput and memory usage, and is composible with other features (e.g. torch.compile). To enable FSDP2, simply use verl main and set the following options: +``` +actor_rollout_ref.ref.strategy=fsdp2 +actor_rollout_ref.actor.strategy=fsdp2 +critic.strategy=fsdp2 +reward_model.strategy=fsdp2 +``` +Furthermore, FSDP2 cpu offloading is compatible with gradient accumulation. You can turn it on to save memory with `actor_rollout_ref.actor.fsdp_config.offload_policy=True`. For more details, see https://github.com/volcengine/verl/pull/1026 + +## AMD Support (ROCm Kernel) + +verl now supports FSDP as the training engine (Megatron support coming soon) and both integrates with vLLM and SGLang as inference engines. Please refer to [this document](https://github.com/volcengine/verl/blob/main/docs/amd_tutorial/amd_build_dockerfile_page.rst) for the installation guide and more information, and [this document](https://github.com/volcengine/verl/blob/main/docs/amd_tutorial/amd_vllm_page.rst) for the vLLM performance tuning for ROCm. + + +## Citation and acknowledgement + +If you find the project helpful, please cite: + +- [HybridFlow: A Flexible and Efficient RLHF Framework](https://arxiv.org/abs/2409.19256v2) +- [A Framework for Training Large Language Models for Code Generation via Proximal Policy Optimization](https://i.cs.hku.hk/~cwu/papers/gmsheng-NL2Code24.pdf) + +```bibtex +@article{sheng2024hybridflow, + title = {HybridFlow: A Flexible and Efficient RLHF Framework}, + author = {Guangming Sheng and Chi Zhang and Zilingfeng Ye and Xibin Wu and Wang Zhang and Ru Zhang and Yanghua Peng and Haibin Lin and Chuan Wu}, + year = {2024}, + journal = {arXiv preprint arXiv: 2409.19256} +} +``` + +verl is inspired by the design of Nemo-Aligner, Deepspeed-chat and OpenRLHF. The project is adopted and contributed by Bytedance, Anyscale, LMSys.org, [Alibaba Qwen team](https://github.com/QwenLM/), Shanghai AI Lab, Tsinghua University, UC Berkeley, UCLA, UIUC, University of Hong Kong, ke.com, [All Hands AI](https://www.all-hands.dev/), [ModelBest](http://modelbest.cn/), JD AI Lab, Microsoft Research, [StepFun](https://www.stepfun.com/), Amazon, LinkedIn, Meituan, [Camel-AI](https://www.camel-ai.org/), [OpenManus](https://github.com/OpenManus), Xiaomi, NVIDIA research, [Baichuan](https://www.baichuan-ai.com/home), [RedNote](https://www.xiaohongshu.com/), [SwissAI](https://www.swiss-ai.org/), [Moonshot AI (Kimi)](https://www.moonshot-ai.com/), Baidu, Snowflake, Skywork.ai, JetBrains, [IceSword Lab](https://www.iceswordlab.com), and many more. + +## Awesome work using verl + +- [TinyZero](https://github.com/Jiayi-Pan/TinyZero): a reproduction of **DeepSeek R1 Zero** recipe for reasoning tasks ![GitHub Repo stars](https://img.shields.io/github/stars/Jiayi-Pan/TinyZero) +- [SkyThought](https://github.com/NovaSky-AI/SkyThought): RL training for Sky-T1-7B by NovaSky AI team. ![GitHub Repo stars](https://img.shields.io/github/stars/NovaSky-AI/SkyThought) +- [simpleRL-reason](https://github.com/hkust-nlp/simpleRL-reason): SimpleRL-Zoo: Investigating and Taming Zero Reinforcement Learning for Open Base Models in the Wild ![GitHub Repo stars](https://img.shields.io/github/stars/hkust-nlp/simpleRL-reason) +- [Easy-R1](https://github.com/hiyouga/EasyR1): **Multi-modal** RL training framework ![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/EasyR1) +- [OpenManus-RL](https://github.com/OpenManus/OpenManus-RL): LLM Agents RL tunning framework for multiple agent environments. ![GitHub Repo stars](https://img.shields.io/github/stars/OpenManus/OpenManus-RL) +- [rllm](https://github.com/agentica-project/rllm): async RL training with [verl-pipeline](https://github.com/agentica-project/verl-pipeline) ![GitHub Repo stars](https://img.shields.io/github/stars/agentica-project/rllm) +- [RAGEN](https://github.com/ZihanWang314/ragen): a general-purpose reasoning **agent** training framework ![GitHub Repo stars](https://img.shields.io/github/stars/ZihanWang314/ragen) +- [Search-R1](https://github.com/PeterGriffinJin/Search-R1): RL with reasoning and **searching (tool-call)** interleaved LLMs ![GitHub Repo stars](https://img.shields.io/github/stars/PeterGriffinJin/Search-R1) +- [ReSearch](https://github.com/Agent-RL/ReSearch): Learning to **Re**ason with **Search** for LLMs via Reinforcement Learning ![GitHub Repo stars](https://img.shields.io/github/stars/Agent-RL/ReSearch) +- [Skywork-OR1](https://github.com/SkyworkAI/Skywork-OR1): Skywork open reaonser series ![GitHub Repo stars](https://img.shields.io/github/stars/SkyworkAI/Skywork-OR1) +- [ToRL](https://github.com/GAIR-NLP/ToRL): Scaling tool-integrated RL ![GitHub Repo stars](https://img.shields.io/github/stars/GAIR-NLP/ToRL) +- [Absolute Zero Reasoner](https://github.com/LeapLabTHU/Absolute-Zero-Reasoner): [A no human curated data self-play framework for reasoning](https://arxiv.org/abs/2505.03335) ![GitHub Repo stars](https://img.shields.io/github/stars/LeapLabTHU/Absolute-Zero-Reasoner) +- [verl-agent](https://github.com/langfengQ/verl-agent): A scalable training framework for **long-horizon LLM/VLM agents**, along with a new algorithm **GiGPO** ![GitHub Repo stars](https://img.shields.io/github/stars/langfengQ/verl-agent) +- [RL-Factory](https://github.com/Simple-Efficient/RL-Factory): An easy and efficient RL post-training framework for Agentic Learning ![GitHub Repo stars](https://img.shields.io/github/stars/Simple-Efficient/RL-Factory) +- [ReTool](https://retool-rl.github.io/): ReTool: reinforcement learning for strategic tool use in LLMs. Code release is in progress... +- [verl-tool](https://github.com/TIGER-AI-Lab/verl-tool): An unified and easy-to-extend tool-agent training framework based on verl![GitHub Repo stars](https://img.shields.io/github/stars/TIGER-AI-Lab/verl-tool) +- [PRIME](https://github.com/PRIME-RL/PRIME): Process reinforcement through implicit rewards ![GitHub Repo stars](https://img.shields.io/github/stars/PRIME-RL/PRIME) +- [MemAgent](https://github.com/BytedTsinghua-SIA/MemAgent): MemAgent: Reshaping Long-Context LLM with Multi-Conv RL based Memory Agent ![GitHub Repo stars](https://img.shields.io/github/stars/BytedTsinghua-SIA/MemAgent) +- [POLARIS](https://github.com/ChenxinAn-fdu/POLARIS): A Post-training recipe for scaling RL on Advanced Reasoning models ![GitHub Repo stars](https://img.shields.io/github/stars/ChenxinAn-fdu/POLARIS) +- [GUI-R1](https://github.com/ritzz-ai/GUI-R1): **GUI-R1**: A Generalist R1-style Vision-Language Action Model For **GUI Agents** ![GitHub Repo stars](https://img.shields.io/github/stars/ritzz-ai/GUI-R1) +- [DeepRetrieval](https://github.com/pat-jj/DeepRetrieval): RL Training of **Search Agent** with **Search/Retrieval Outcome** ![GitHub Repo stars](https://img.shields.io/github/stars/pat-jj/DeepRetrieval) +- [Code-R1](https://github.com/ganler/code-r1): Reproducing R1 for **Code** with Reliable Rewards ![GitHub Repo stars](https://img.shields.io/github/stars/ganler/code-r1) +- [DeepResearcher](https://github.com/GAIR-NLP/DeepResearcher): Scaling deep research via reinforcement learning in real-world environments ![GitHub Repo stars](https://img.shields.io/github/stars/GAIR-NLP/DeepResearcher) +- [VAGEN](https://github.com/RAGEN-AI/VAGEN): Training VLM agents with multi-turn reinforcement learning ![GitHub Repo stars](https://img.shields.io/github/stars/RAGEN-AI/VAGEN) +- [RM-R1](https://arxiv.org/abs/2505.02387): RL training of reasoning reward models ![GitHub Repo stars](https://img.shields.io/github/stars/RM-R1-UIUC/RM-R1) +- [LUFFY](https://arxiv.org/pdf/2504.14945): Learning to Reason under Off-Policy Guidance![GitHub Repo stars](https://img.shields.io/github/stars/ElliottYan/LUFFY) +- [DeepMath](https://github.com/zwhe99/DeepMath): DeepMath-103K data and series models for math reasoning![GitHub Repo stars](https://img.shields.io/github/stars/zwhe99/DeepMath) +- [PACS](https://github.com/ritzz-ai/PACS): Implicit Actor Critic Coupling via a Supervised Learning Framework for RLVR ![GitHub Repo stars](https://img.shields.io/github/stars/ritzz-ai/PACS) +- [Entropy Mechanism of RL](https://github.com/PRIME-RL/Entropy-Mechanism-of-RL): The Entropy Mechanism of Reinforcement Learning for Large Language Model Reasoning![GitHub Repo stars](https://img.shields.io/github/stars/PRIME-RL/Entropy-Mechanism-of-RL) +- [LLaSA-TTS-GRPO](https://github.com/channel-io/ch-tts-llasa-rl-grpo): TTS fine-tuning with GRPO optimization based on LLASA models ![GitHub Repo stars](https://img.shields.io/github/stars/channel-io/ch-tts-llasa-rl-grpo) +- [PF-PPO](https://arxiv.org/abs/2409.06957): Policy Filtration for PPO based on the reliability of reward signals for more efficient and robust RLHF. +- [RACRO](https://github.com/gyhdog99/RACRO2): Build multi-modal reasoning models via decoupling it into query-conditioned captioning and text-only reasoning ![GitHub Repo stars](https://img.shields.io/github/stars/gyhdog99/RACRO2) +- [Agent Lightning](https://github.com/microsoft/agent-lightning): A flexible and extensible framework that enables seamless agent optimization for any existing agent framework. ![GitHub Repo stars](https://img.shields.io/github/stars/microsoft/agent-lightning) +- [VTool-R1](https://github.com/VTOOL-R1/vtool-r1): VLMs Learn to Think with Images via Reinforcement Learning on Multimodal Tool Use. ![GitHub Repo stars](https://img.shields.io/github/stars/VTOOL-R1/vtool-r1) +- [Kimina-Prover-RL](https://github.com/project-numina/kimina-prover-rl/tree/main/recipe/kimina_prover_rl): Training pipeline for formal theorem proving, based on a paradigm inspired by DeepSeek-R1. +- [RL-PLUS](https://github.com/YihongDong/RL-PLUS): Countering Capability Boundary Collapse of LLMs in Reinforcement Learning with Hybrid-policy Optimization. +- [rStar2-Agent](https://github.com/microsoft/rStar): Using reinforcement learning with multi-step tool-calling for math tasks, rStar2-Agent-14B reaches frontier-level math reasoning in just 510 RL training steps ![GitHub Repo stars](https://img.shields.io/github/stars/microsoft/rStar) +- [Vision-SR1](https://github.com/zli12321/Vision-SR1): Self-Rewarding Vision-Language Model via Reasoning Decomposition ![GitHub Repo stars](https://img.shields.io/github/stars/zli12321/Vision-SR1) +- [SimpleVLA-RL](https://github.com/PRIME-RL/SimpleVLA-RL): SimpleVLA-RL: A Simple yet Effective Vision-Language Action Model for Reinforcement Learning ![GitHub Repo stars](https://img.shields.io/github/stars/PRIME-RL/SimpleVLA-RL) +- [Table-R1](https://github.com/Table-R1/Table-R1): Table-R1: Inference-Time Scaling for Table Reasoning ![GitHub Repo stars](https://img.shields.io/github/stars/Table-R1/Table-R1) + +and many more awesome work listed in [recipe](recipe/README.md). + +## Contribution Guide + +See [contributions guide](CONTRIBUTING.md) + +## About [ByteDance Seed Team](https://team.doubao.com/) + +Founded in 2023, ByteDance Seed Team is dedicated to crafting the industry's most advanced AI foundation models. The team aspires to become a world-class research team and make significant contributions to the advancement of science and society. You can get to know Bytedance Seed better through the following channels👇 +
+ + + + + + + + + +
+--- + +We are HIRING! Send us an [email](mailto:the.verl.project@gmail.com) if you are interested in internship/FTE opportunities in RL for agents. diff --git a/e2c/verl/docker/Apptainerfile.rocm b/e2c/verl/docker/Apptainerfile.rocm new file mode 100644 index 0000000000000000000000000000000000000000..02596218726a60fc105bad7dc3653655ae164e93 --- /dev/null +++ b/e2c/verl/docker/Apptainerfile.rocm @@ -0,0 +1,57 @@ +Bootstrap: docker + +# Support - Traing: fsdp; Inference: vllm +# FROM: rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 +# Support - Traing: fsdp; Inference: vllm, sglang +FROM lmsysorg/sglang:v0.4.5-rocm630 + +%environment + export PYTORCH_ROCM_ARCH="gfx90a;gfx942" + + export HIPCC_COMPILE_FLAGS_APPEND="--amdgpu-target=gfx90a;gfx942 -D__HIP_PLATFORM_AMD__" + export CFLAGS="-D__HIP_PLATFORM_AMD__" + export CXXFLAGS="-D__HIP_PLATFORM_AMD__" + +%post + # Create source directory + mkdir -p /opt/src + + # Uninstall and reinstall vllm + pip uninstall -y vllm + cd /opt/src + git clone -b v0.6.3 https://github.com/vllm-project/vllm.git + cd vllm + MAX_JOBS=$(nproc) python3 setup.py install + cd /opt + rm -rf /opt/src/vllm + + # Install dependencies + pip install "tensordict<0.6" --no-deps + pip install accelerate \ + codetiming \ + datasets \ + dill \ + hydra-core \ + liger-kernel \ + numpy \ + pandas \ + peft \ + "pyarrow>=15.0.0" \ + pylatexenc \ + "ray[data,train,tune,serve]" \ + torchdata \ + transformers \ + wandb \ + orjson \ + pybind11 + + # Clone and install verl from GitHub + cd /opt + git clone https://github.com/volcengine/verl.git + cd verl + # Uncomment to use a specific version + # git checkout v0.3.0.post0 + pip install -e . --no-deps + + # Install torch_memory_saver + pip install git+https://github.com/ExtremeViscent/torch_memory_saver.git --no-deps \ No newline at end of file diff --git a/e2c/verl/docker/Dockerfile.extention.awsefa b/e2c/verl/docker/Dockerfile.extention.awsefa new file mode 100644 index 0000000000000000000000000000000000000000..ba3bc67311977050d1d368d2bfac38fa0278cb0f --- /dev/null +++ b/e2c/verl/docker/Dockerfile.extention.awsefa @@ -0,0 +1,55 @@ +# Base Image support aws EFA +# Build Image with frameworks based on this +FROM verlai/verl:app-verl0.5-sglang0.4.6.post5-mcore0.12.2 + +# For aws instances with EFA net interface (Sagemaker AI Pod) +# install EFA driver: +######## AWS EFA ############ +ENV NCCL_VERSION=2.25.1-1 +ENV DEBIAN_FRONTEND=noninteractive +ENV EFA_INSTALLER_VERSION=1.40.0 +ENV AWS_OFI_NCCL_VERSION=1.14.2 +ENV FI_EFA_SET_CUDA_SYNC_MEMOPS=0 +ENV FI_PROVIDER=efa + +RUN apt update && apt install -y linux-image-generic libhwloc-dev + +RUN cd /tmp && \ + curl -O https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz && \ + tar -xf aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz && \ + cd aws-efa-installer && \ + ./efa_installer.sh -y -g --skip-kmod --skip-limit-conf --no-verify && \ + ldconfig && \ + rm -rf /tmp/aws-efa-installer /var/lib/apt/lists/* + +# NCCL EFA Plugin +RUN cd /tmp && \ + curl -LO https://github.com/aws/aws-ofi-nccl/archive/refs/tags/v${AWS_OFI_NCCL_VERSION}.tar.gz && \ + tar -xzf /tmp/v${AWS_OFI_NCCL_VERSION}.tar.gz && \ + rm /tmp/v${AWS_OFI_NCCL_VERSION}.tar.gz && \ + mv aws-ofi-nccl-${AWS_OFI_NCCL_VERSION} aws-ofi-nccl && \ + cd /tmp/aws-ofi-nccl && \ + ./autogen.sh && \ + ./configure --prefix=/opt/amazon/efa \ + --with-libfabric=/opt/amazon/efa \ + --with-cuda=/usr/local/cuda \ + --enable-platform-aws \ + --with-mpi=/opt/amazon/openmpi && \ + make -j$(nproc) install && \ + rm -rf /tmp/aws-ofi/nccl + +# NCCL +RUN echo "/usr/local/lib" >> /etc/ld.so.conf.d/local.conf && \ + echo "/opt/amazon/openmpi/lib" >> /etc/ld.so.conf.d/efa.conf && \ + ldconfig + +ENV OMPI_MCA_pml=^cm,ucx \ + OMPI_MCA_btl=tcp,self \ + OMPI_MCA_btl_tcp_if_exclude=lo,docker0,veth_def_agent \ + OPAL_PREFIX=/opt/amazon/openmpi \ + NCCL_SOCKET_IFNAME=^docker,lo,veth_def_agent \ + FI_EFA_USE_HUGE_PAGE=0 + +# docker build -t verl:awsefa --label "commit=$(git rev-parse --short HEAD)" . +# on aws: +# docker run --ipc=host --privileged --name verldev --gpus all --network=host --shm-size=1800gb -itd verl:awsefa diff --git a/e2c/verl/docker/Dockerfile.ngc.vllm b/e2c/verl/docker/Dockerfile.ngc.vllm new file mode 100644 index 0000000000000000000000000000000000000000..fa0ebbb2c04ed842c729f91ddbce9dd7803bd87d --- /dev/null +++ b/e2c/verl/docker/Dockerfile.ngc.vllm @@ -0,0 +1,48 @@ +# docker buildx build --platform linux/x86_64 -t "verlai/verl:ngc-th2.4.0-cu124-vllm0.6.3-ray2.4-te1.7-v0.0.6" -f docker/Dockerfile.ngc.vllm . --builder cloud-verlai-verl-builder --progress=plain --push +FROM nvcr.io/nvidia/pytorch:24.05-py3 + +# uninstall nv-pytorch fork +RUN pip3 uninstall pytorch-quantization \ + pytorch-triton \ + torch \ + torch-tensorrt \ + torchvision \ + xgboost transformer_engine flash_attn \ + apex megatron-core -y + +RUN pip3 install torch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 --index-url https://download.pytorch.org/whl/cu124 + +# =============== Megatron dependencies (optional) ================= +# install apex, set MAX_JOBS to avoid OOMs +RUN MAX_JOBS=4 pip3 install -v --disable-pip-version-check --no-cache-dir --no-build-isolation \ + --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" \ + git+https://github.com/NVIDIA/apex +# =============== End of Megatron dependencies (optional) ================= + +RUN pip3 install --no-cache-dir \ + accelerate \ + codetiming \ + datasets \ + dill \ + hydra-core \ + numpy \ + 'pandas' \ + 'peft' \ + 'pyarrow>=15.0.0' \ + 'pybind11' \ + 'pylatexenc' \ + 'ray>=2.10' \ + 'tensordict<0.6' \ + 'transformers' \ + 'vllm==0.6.3.post1' \ + 'wandb' \ + 'tensorboard' + +# full dependencies +RUN pip3 install pytest pre-commit py-spy pyext liger-kernel + +# =============== Megatron dependencies (optional) ================= +# install Transformer Engine, which requires FA 2.5.8. Do it in a separate step for docker cache +RUN MAX_JOBS=4 NINJA_FLAGS="-j4" pip3 install flash-attn==2.5.8 --no-cache-dir --no-build-isolation +RUN MAX_JOBS=1 NINJA_FLAGS="-j1" TE_BUILD_WITH_NINJA=0 pip3 install git+https://github.com/eric-haibin-lin/TransformerEngine.git@v1.7.0 +# =============== End of Megatron dependencies (optional) ================= diff --git a/e2c/verl/docker/Dockerfile.ngc.vllm0.8 b/e2c/verl/docker/Dockerfile.ngc.vllm0.8 new file mode 100644 index 0000000000000000000000000000000000000000..f6ff8fe80b24a413ee57321296f60cedada14563 --- /dev/null +++ b/e2c/verl/docker/Dockerfile.ngc.vllm0.8 @@ -0,0 +1,75 @@ +# Start from the NVIDIA official image (ubuntu-22.04 + cuda-12.6 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:24.08-py3 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini && \ + apt-get clean + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + xgboost transformer_engine flash_attn apex megatron-core grpcio + +# Install torch-2.6.0+cu124 + vllm-0.8.3 +# torch-2.6.0+cu124: cxx11abi=False +# torch-2.6.0+cu126: cxx11abi=True +# see https://github.com/flashinfer-ai/flashinfer/issues/911 +RUN pip install --no-cache-dir "vllm==0.8.3" "torch==2.6.0" "torchvision==0.21.0" "torchaudio==2.6.0" "tensordict==0.6.2" torchdata \ + "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=15.0.0" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler \ + pytest py-spy pyext pre-commit ruff tensorboard + +# Install flash-attn-2.7.4.post1 (cxx11abi=False) +RUN wget -nv https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl && \ + pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl + +# Install flashinfer-0.2.2.post1+cu124 (cxx11abi=False) +# vllm-0.8.3 does not support flashinfer>=0.2.3 +# see https://github.com/vllm-project/vllm/pull/15777 +RUN wget -nv https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.2.post1/flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + pip install --no-cache-dir flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl + +# Fix packages +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +# Install verl +RUN pip install --no-cache-dir verl[vllm] -U + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url diff --git a/e2c/verl/docker/Dockerfile.ngc.vllm0.8.sagemaker b/e2c/verl/docker/Dockerfile.ngc.vllm0.8.sagemaker new file mode 100644 index 0000000000000000000000000000000000000000..2746fa9f8e115c0c6ad43627b2d4200df75d597b --- /dev/null +++ b/e2c/verl/docker/Dockerfile.ngc.vllm0.8.sagemaker @@ -0,0 +1,46 @@ +# Using a pre-built image from AWS DLC which contains the current version of python (3.10) and supported cuda version (12.1) +FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:2.1.0-transformers4.36.0-gpu-py310-cu121-ubuntu20.04 + +# uninstall nv-pytorch fork +RUN pip3 uninstall -y pytorch-quantization \ + pytorch-triton torch torch-tensorrt torchvision \ + xgboost transformer_engine flash_attn apex megatron-core + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini && \ + apt-get clean + +# Install torch-2.6.0 + vllm-0.8.2 +RUN pip install --no-cache-dir vllm==0.8.2 torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 tensordict torchdata==0.11.0 \ + transformers>=4.49.0 accelerate datasets peft hf-transfer \ + ray[default] codetiming hydra-core pandas pyarrow>=15.0.0 pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler \ + pytest pre-commit py-spy pyext ruff tensorboard + +# Install flash_attn-2.7.4.post1 +RUN pip uninstall -y transformer-engine flash-attn && \ + pip install flash-attn==2.7.4.post1 --no-build-isolation + +# Fix cv2 +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir nvidia-ml-py>=12.560.30 opencv-python-headless==4.8.0.74 fastapi==0.115.6 && \ + pip install --no-cache-dir --upgrade optree>=0.13.0 + +# Install verl +RUN pip install --no-cache-dir verl[vllm] -U + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url diff --git a/e2c/verl/docker/Dockerfile.rocm b/e2c/verl/docker/Dockerfile.rocm new file mode 100644 index 0000000000000000000000000000000000000000..16c99aa0540169e0165bc47394f6faa45e007093 --- /dev/null +++ b/e2c/verl/docker/Dockerfile.rocm @@ -0,0 +1,322 @@ +# FROM "compute-artifactory.amd.com:5000/rocm-plus-docker/framework/compute-rocm-rel-6.4:94_ubuntu22.04_py3.10_pytorch_release-2.7_575e247" +# FROM "rlfoundation.azurecr.io/rocm6.3.4:vllm-0.8.5-numa-patch-ubuntu-22.04" +FROM "rlsys/rocm-6.3.4-patch:rocm6.3.4-numa-patch_ubuntu-22.04" + +SHELL ["/bin/bash", "-ceuxo", "pipefail"] + +ENV MAX_JOBS=512 + +ENV PATH="/usr/local/python3.12/bin:$PATH" +RUN ln -sf /usr/bin/python3.12 /usr/bin/python && \ + ln -sf /usr/bin/pip3.12 /usr/bin/pip + +############################################ +############################################ +RUN apt-get update +RUN apt-get install -y pkg-config liblzma-dev +############################################ +############################################ + + +########################################### +##########Install TransformerEngine######## +########################################### +WORKDIR /workspace/ +# transformer-engine install +# https://github.com/ROCm/TransformerEngine + +RUN rm -rf TransformerEngine +RUN git clone --recursive https://github.com/ROCm/TransformerEngine.git +WORKDIR /workspace/TransformerEngine +RUN git checkout 236178e5 +# git checkout bb061ade +# git checkout 864405c + +ENV NVTE_FRAMEWORK=pytorch +ENV NVTE_ROCM_ARCH=gfx942 +ENV NVTE_USE_HIPBLASLT=1 +ENV NVTE_USE_ROCM=1 + +# export CMAKE_PREFIX_PATH="/opt/rocm:/opt/rocm/hip:/usr/local:/usr:${CMAKE_PREFIX_PATH:-}" +ENV CMAKE_PREFIX_PATH="/opt/rocm:/opt/rocm/hip:/usr/local:/usr" + + +# ENV NVTE_BUILD_MAX_JOBS=$(MAX_JOBS) + +RUN MAX_JOBS=$(MAX_JOBS) pip install . -vvv + +WORKDIR /workspace/ +########################################### +########################################### +########################################### + + + + + +#################################################################################### +################Install vllm - sglang require vllm 0.6.7 dependency################# +#################################################################################### +#### Require vllm 0.6.7 - checkout 113274a0 +WORKDIR /workspace/ +RUN rm -rf vllm +RUN pip uninstall -y vllm +# Refer to here (down-grade vllm to 0.6.3): https://docs.vllm.ai/en/v0.6.3/getting_started/amd-installation.html +RUN git clone https://github.com/ROCm/vllm.git +# git clone https://github.com/vllm-project/vllm.git +WORKDIR /workspace/vllm +RUN git checkout 113274a0 +ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" +#ENV MAX_JOBS=512 +ENV MAX_JOBS=${MAX_JOBS} +RUN pip install "boto3>=1.26.0" +RUN pip install setuptools_scm +# will add src into py. You can delete the repo +RUN python3 setup.py install +WORKDIR /workspace/ +#################################################################################### +#################################################################################### +#################################################################################### + + + +########################################### +############For hack docker################ +########################################### +RUN pip install setuptools==75.8.0 +########################################### +########################################### +########################################### + + + +########################################### +############build sgalng################### +########################################### +# Set environment variables +ENV BASE_DIR=/sgl-workspace +ENV BUILD_TYPE=all +ENV SGL_REPO=https://github.com/sgl-project/sglang +ENV SGL_BRANCH=v0.4.6.post5 +ENV TRITON_REPO=https://github.com/ROCm/triton.git +ENV TRITON_COMMIT=improve_fa_decode_3.0.0 +ENV AITER_REPO=https://github.com/ROCm/aiter.git +ENV AITER_COMMIT=v0.1.2 +# v0.1.2 version - commit id: 9d11f47 +# ENV AITER_COMMIT=9d11f47 + +ENV HIP_FORCE_DEV_KERNARG=1 +ENV HSA_NO_SCRATCH_RECLAIM=1 +ENV SGLANG_SET_CPU_AFFINITY=1 +ENV SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 +ENV NCCL_MIN_NCHANNELS=112 +ENV MOE_PADDING=1 +ENV VLLM_FP8_PADDING=1 +ENV VLLM_FP8_ACT_PADDING=1 +ENV VLLM_FP8_WEIGHT_PADDING=1 +ENV VLLM_FP8_REDUCE_CONV=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=1 +ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942" +ENV AMDGPU_TARGETS=gfx942 +ENV ROCM_ARCH=gfx942 +ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" + +# Switch to working directory +WORKDIR /sgl-workspace + +# Clean and create directory +RUN rm -rf /sgl-workspace && mkdir -p /sgl-workspace + +# Clone and build sglang +RUN git clone ${SGL_REPO} \ + && cd sglang \ + && git checkout ${SGL_BRANCH} || echo "Using default branch" \ + && cd sgl-kernel \ + && rm -f pyproject.toml \ + && mv pyproject_rocm.toml pyproject.toml \ + && python setup_rocm.py install \ + && cd .. \ + && if [ "$BUILD_TYPE" = "srt" ]; then \ + python -m pip --no-cache-dir install -e "python[srt_hip]"; \ + else \ + python -m pip --no-cache-dir install -e "python[all_hip]"; \ + fi \ + && cd /sgl-workspace \ + && cp -r /sgl-workspace/sglang /sglang \ + && python -m pip cache purge + +# Install common Python packages +RUN pip install IPython orjson python-multipart torchao pybind11 + +# Rebuild Triton +RUN pip uninstall -y triton || true \ + && git clone ${TRITON_REPO} \ + && cd triton \ + && git checkout ${TRITON_COMMIT} \ + && cd python \ + && python3 setup.py install \ + && cd /sgl-workspace + + +# ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942 --amdgpu-lower-module-lds-strategy=1" +# ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942" + +# Build aiter +#version: Commit 9d11f47 + # && git checkout ${AITER_COMMIT} \ +RUN pip uninstall -y aiter || true +RUN git clone ${AITER_REPO} \ + && cd aiter \ + && git checkout ${AITER_COMMIT} \ + && git submodule sync \ + && git submodule update --init --recursive \ + && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py install \ + && cd /sgl-workspace + # && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py develop \ + # && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py develop \ + +# Copy MI300X config +RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ + /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ + -type f -name '*MI300X*' | \ + xargs -I {} sh -c 'vf_config=$(echo "$1" | sed "s/MI300X/MI300X_VF/"); cp "$1" "$vf_config"' -- {} + +# Environment setup complete. +RUN echo "Environment setup complete." + +WORKDIR /workspace/ +########################################### +########################################### +########################################### + + + + + + +########################################### +###############vllm v0.8.5################# +########################################### +# ENV GITHUB_USERNAME=yushengsu-thu +# ENV GITHUB_MAIL=yushengsu@gmail.com + +# RUN git config --global user.name "${GITHUB_USERNAME}" \ +# && git config --global user.email "${GITHUB_MAIL}" + +WORKDIR /workspace/ + +ENV VLLM_TARGET_DEVICE=rocm +ENV ROCM_PATH=/opt/rocm +ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev + +# Find the repo path in: DockerFile/Dockerfile.rocm_yang +# RUN git clone https://github.com/RLFoundation/vllm-patch.git +RUN pip uninstall -y vllm || true +RUN rm -rf vllm-patch +RUN git clone https://github.com/RLFoundation/vllm-patch.git \ + && cd vllm-patch \ + && git checkout v0.8.5-sleep-numa \ + && rm -rf build/ dist/ *.egg-info \ + && ln -sf /opt/rocm/lib/libamdhip64.so /usr/lib/libamdhip64.so \ + && SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev PYTORCH_ROCM_ARCH="gfx90a;gfx942" MAX_JOBS=${MAX_JOBS} python3 setup.py install + # RUN SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev PYTORCH_ROCM_ARCH="gfx90a;gfx942" MAX_JOBS=${MAX_JOBS} python3 setup.py develop + +WORKDIR /workspace/ +########################################### +########################################### +########################################### + + + + +######################################### +#### Install megatron-core############### +######################################### +RUN pip uninstall -y megatron-core && \ + git clone https://github.com/yushengsu-thu/Megatron-LM-amd_version.git && \ + cd Megatron-LM-amd_version && \ + pip install -vvv -e . && \ + cd /workspace/ +######################################### +######################################### +######################################### + + + + +####################################### +################apex################### +####################################### +WORKDIR /workspace/ +RUN pip uninstall -y apex && \ + git clone https://github.com/ROCm/apex.git && \ + cd apex && \ + python setup.py install && \ + cd /workspace/ +####################################### +####################################### +####################################### + + + + +################################################################################ +###########################Add torch_memory_saver############################### +################################################################################ +# Set environment variables +ENV HIPCC_COMPILE_FLAGS_APPEND="--amdgpu-target=gfx90a;gfx942 -D__HIP_PLATFORM_AMD__" +ENV CFLAGS="-D__HIP_PLATFORM_AMD__" +ENV CXXFLAGS="-D__HIP_PLATFORM_AMD__" +RUN pip install "git+https://github.com/YangWang92/torch_memory_saver_numa.git@numa" +################################################################################ +################################################################################ +################################################################################ + + + +######################################## +######Install ray####################### +######################################## +# need to add this patch: https://github.com/ray-project/ray/pull/53531/files +RUN pip uninstall ray -y +RUN pip install "ray[data,train,tune,serve]>=2.47.0" +######################################## +######################################## +######################################## + + + +########################################## +#######Install other dependencies######### +########################################## +RUN pip install "tensordict==0.6.2" --no-deps && \ + pip install accelerate \ + codetiming \ + datasets \ + dill \ + hydra-core \ + liger-kernel \ + numpy \ + pandas \ + peft \ + "pyarrow>=15.0.0" \ + pylatexenc \ + torchdata \ + wandb \ + orjson \ + pybind11 + +WORKDIR /workspace/ +RUN git clone https://github.com/volcengine/verl.git && \ + cd verl && \ + pip install -e . +########################################## +########################################## +########################################## + + + +WORKDIR /workspace/ + +CMD ["/usr/bin/bash"] diff --git a/e2c/verl/docker/Dockerfile.rocm7 b/e2c/verl/docker/Dockerfile.rocm7 new file mode 100644 index 0000000000000000000000000000000000000000..d92001b3f98b263887b2989872bbd35c436ed28a --- /dev/null +++ b/e2c/verl/docker/Dockerfile.rocm7 @@ -0,0 +1,141 @@ +# default base image +ARG REMOTE_VLLM="1" +ARG COMMON_WORKDIR=/app +ARG BASE_IMAGE=rocm/vllm-dev:base_rocm7_0930_rc1_20250916_tuned_20250917 + +FROM ${BASE_IMAGE} AS base + +ARG ARG_PYTORCH_ROCM_ARCH +ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}} + +# Install some basic utilities +RUN apt-get update -q -y && apt-get install -q -y \ + sqlite3 libsqlite3-dev libfmt-dev libmsgpack-dev libsuitesparse-dev \ + apt-transport-https ca-certificates wget curl +# Remove sccache +RUN python3 -m pip install --upgrade pip +RUN apt-get purge -y sccache; python3 -m pip uninstall -y sccache; rm -f "$(which sccache)" +ARG COMMON_WORKDIR +WORKDIR ${COMMON_WORKDIR} + + +# ----------------------- +# vLLM fetch stages +FROM base AS fetch_vllm_0 +ONBUILD COPY ./ vllm/ +FROM base AS fetch_vllm_1 +#ARG VLLM_REPO="https://github.com/ROCm/vllm.git" +#ARG VLLM_BRANCH="main" +ARG VLLM_REPO=https://github.com/HollowMan6/vllm.git +ARG VLLM_BRANCH="sleep_amd" +ONBUILD RUN git clone ${VLLM_REPO} \ + && cd vllm \ + && git checkout ${VLLM_BRANCH} +FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm + +# ----------------------- +# vLLM build stages +FROM fetch_vllm AS build_vllm +# Build vLLM +RUN cd vllm \ + && python3 -m pip install -r requirements/rocm.txt \ + && python3 setup.py clean --all \ + && ln -sf /opt/rocm/lib/libamdhip64.so /usr/lib/libamdhip64.so \ + && VLLM_TARGET_DEVICE=rocm ROCM_PATH=/opt/rocm/ VLLM_GPU_LANG=HIP SETUPTOOLS_SCM_PRETEND_VERSION=0.8.4.dev python3 setup.py bdist_wheel --dist-dir=dist + #&& python3 setup.py bdist_wheel --dist-dir=dist +FROM scratch AS export_vllm +ARG COMMON_WORKDIR +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/dist/*.whl / +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite + +# ----------------------- +# Test vLLM image +FROM base AS test + +RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/* + +# Install vLLM +RUN --mount=type=bind,from=export_vllm,src=/,target=/install \ + cd /install \ + && pip install -U -r requirements/rocm.txt \ + && pip install -U -r requirements/rocm-test.txt \ + && pip uninstall -y vllm \ + && pip install *.whl + +WORKDIR /vllm-workspace +ARG COMMON_WORKDIR +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace + +# install development dependencies (for testing) +RUN cd /vllm-workspace \ + && rm -rf vllm \ + && python3 -m pip install -e tests/vllm_test_utils \ + && python3 -m pip install lm-eval[api]==0.4.4 \ + && python3 -m pip install pytest-shard + +# ----------------------- +# Final vLLM image +FROM base AS final + +RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/* +# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt. +# Manually remove it so that later steps of numpy upgrade can continue +RUN case "$(which python3)" in \ + *"/opt/conda/envs/py_3.9"*) \ + rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/;; \ + *) ;; esac + +RUN python3 -m pip install --upgrade huggingface-hub[cli] + +# Install vLLM +RUN --mount=type=bind,from=export_vllm,src=/,target=/install \ + cd /install \ + && pip install -U -r requirements/rocm.txt \ + && pip uninstall -y vllm \ + && pip install *.whl + +ARG COMMON_WORKDIR + +# Copy over the benchmark scripts as well +COPY --from=export_vllm /benchmarks ${COMMON_WORKDIR}/vllm/benchmarks +COPY --from=export_vllm /examples ${COMMON_WORKDIR}/vllm/examples + +ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1 +ENV TOKENIZERS_PARALLELISM=false + +# ENV that can improve safe tensor loading, and end-to-end time +ENV SAFETENSORS_FAST_GPU=1 + +# Performance environment variable. +ENV HIP_FORCE_DEV_KERNARG=1 + +# ----------------------- +# Install verl +RUN pip install "tensordict==0.6.2" --no-deps && \ + pip install accelerate \ + codetiming \ + datasets \ + dill \ + hydra-core \ + liger-kernel \ + numpy \ + pandas \ + peft \ + "pyarrow>=15.0.0" \ + pylatexenc \ + torchdata \ + wandb \ + orjson \ + pybind11 + +WORKDIR /workspace/ +RUN git clone https://github.com/volcengine/verl.git && \ + cd verl && \ + pip install -e . + +CMD ["/bin/bash"] + diff --git a/e2c/verl/docker/Dockerfile.rocm_verl-0.3.0.post1 b/e2c/verl/docker/Dockerfile.rocm_verl-0.3.0.post1 new file mode 100644 index 0000000000000000000000000000000000000000..185096d9d2e83fc235f04e70260fd3dbfca7dd1c --- /dev/null +++ b/e2c/verl/docker/Dockerfile.rocm_verl-0.3.0.post1 @@ -0,0 +1,58 @@ +# Build the docker in the repo dir: +# docker build -f docker/Dockerfile.rocm -t verl-rocm:03.04.2015 . +# docker images # you can find your built docker + + +# Support - Traing: fsdp; Inference: vllm +# FROM rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 +# Support - Traing: fsdp; Inference: vllm, sglang +FROM lmsysorg/sglang:v0.4.6.post5-rocm630 + +# Set working directory +# WORKDIR $PWD/app + +# Set environment variables +ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" + +ENV HIPCC_COMPILE_FLAGS_APPEND="--amdgpu-target=gfx90a;gfx942 -D__HIP_PLATFORM_AMD__" +ENV CFLAGS="-D__HIP_PLATFORM_AMD__" +ENV CXXFLAGS="-D__HIP_PLATFORM_AMD__" + +# Install vllm +RUN pip uninstall -y vllm && \ + rm -rf vllm && \ + git clone -b v0.6.3 https://github.com/vllm-project/vllm.git && \ + cd vllm && \ + MAX_JOBS=$(nproc) python3 setup.py install && \ + cd .. && \ + rm -rf vllm + +# Copy the entire project directory +COPY . . + +# Install dependencies +RUN pip install "tensordict==0.6.2" --no-deps && \ + pip install accelerate \ + codetiming \ + datasets \ + dill \ + hydra-core \ + liger-kernel \ + numpy \ + pandas \ + peft \ + "pyarrow>=15.0.0" \ + pylatexenc \ + "ray[data,train,tune,serve]<2.45.0" \ + torchdata \ + transformers \ + wandb \ + orjson \ + pybind11 + +RUN git clone https://github.com/volcengine/verl.git && \ + cd verl && \ + pip install -e . + +# Install torch_memory_saver +RUN pip install git+https://github.com/ExtremeViscent/torch_memory_saver.git --no-deps diff --git a/e2c/verl/docker/Dockerfile.rocm_verl-0.4.1 b/e2c/verl/docker/Dockerfile.rocm_verl-0.4.1 new file mode 100644 index 0000000000000000000000000000000000000000..57621b248ce8cab819c748df42e0e056b489062f --- /dev/null +++ b/e2c/verl/docker/Dockerfile.rocm_verl-0.4.1 @@ -0,0 +1,323 @@ +# FROM "compute-artifactory.amd.com:5000/rocm-plus-docker/framework/compute-rocm-rel-6.4:94_ubuntu22.04_py3.10_pytorch_release-2.7_575e247" +# FROM "rlfoundation.azurecr.io/rocm6.3.4:vllm-0.8.5-numa-patch-ubuntu-22.04" +FROM "rlsys/rocm-6.3.4-patch:rocm6.3.4-numa-patch_ubuntu-22.04" + +SHELL ["/bin/bash", "-ceuxo", "pipefail"] + +ENV MAX_JOBS=512 + +ENV PATH="/usr/local/python3.12/bin:$PATH" +RUN ln -sf /usr/bin/python3.12 /usr/bin/python && \ + ln -sf /usr/bin/pip3.12 /usr/bin/pip + +############################################ +############################################ +RUN apt-get update +RUN apt-get install -y pkg-config liblzma-dev +############################################ +############################################ + + +########################################### +##########Install TransformerEngine######## +########################################### +WORKDIR /workspace/ +# transformer-engine install +# https://github.com/ROCm/TransformerEngine + +RUN rm -rf TransformerEngine +RUN git clone --recursive https://github.com/ROCm/TransformerEngine.git +WORKDIR /workspace/TransformerEngine +RUN git checkout 236178e5 +# git checkout bb061ade +# git checkout 864405c + +ENV NVTE_FRAMEWORK=pytorch +ENV NVTE_ROCM_ARCH=gfx942 +ENV NVTE_USE_HIPBLASLT=1 +ENV NVTE_USE_ROCM=1 + +# export CMAKE_PREFIX_PATH="/opt/rocm:/opt/rocm/hip:/usr/local:/usr:${CMAKE_PREFIX_PATH:-}" +ENV CMAKE_PREFIX_PATH="/opt/rocm:/opt/rocm/hip:/usr/local:/usr" + + +# ENV NVTE_BUILD_MAX_JOBS=$(MAX_JOBS) + +RUN MAX_JOBS=$(MAX_JOBS) pip install . -vvv + +WORKDIR /workspace/ +########################################### +########################################### +########################################### + + + + + +#################################################################################### +################Install vllm - sglang require vllm 0.6.7 dependency################# +#################################################################################### +#### Require vllm 0.6.7 - checkout 113274a0 +WORKDIR /workspace/ +RUN rm -rf vllm +RUN pip uninstall -y vllm +# Refer to here (down-grade vllm to 0.6.3): https://docs.vllm.ai/en/v0.6.3/getting_started/amd-installation.html +RUN git clone https://github.com/ROCm/vllm.git +# git clone https://github.com/vllm-project/vllm.git +WORKDIR /workspace/vllm +RUN git checkout 113274a0 +ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" +#ENV MAX_JOBS=512 +ENV MAX_JOBS=${MAX_JOBS} +RUN pip install "boto3>=1.26.0" +RUN pip install setuptools_scm +# will add src into py. You can delete the repo +RUN python3 setup.py install +WORKDIR /workspace/ +#################################################################################### +#################################################################################### +#################################################################################### + + + +########################################### +############For hack docker################ +########################################### +RUN pip install setuptools==75.8.0 +########################################### +########################################### +########################################### + + + +########################################### +############build sgalng################### +########################################### +# Set environment variables +ENV BASE_DIR=/sgl-workspace +ENV BUILD_TYPE=all +ENV SGL_REPO=https://github.com/sgl-project/sglang +ENV SGL_BRANCH=v0.4.6.post5 +ENV TRITON_REPO=https://github.com/ROCm/triton.git +ENV TRITON_COMMIT=improve_fa_decode_3.0.0 +ENV AITER_REPO=https://github.com/ROCm/aiter.git +ENV AITER_COMMIT=v0.1.2 +# v0.1.2 version - commit id: 9d11f47 +# ENV AITER_COMMIT=9d11f47 + +ENV HIP_FORCE_DEV_KERNARG=1 +ENV HSA_NO_SCRATCH_RECLAIM=1 +ENV SGLANG_SET_CPU_AFFINITY=1 +ENV SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 +ENV NCCL_MIN_NCHANNELS=112 +ENV MOE_PADDING=1 +ENV VLLM_FP8_PADDING=1 +ENV VLLM_FP8_ACT_PADDING=1 +ENV VLLM_FP8_WEIGHT_PADDING=1 +ENV VLLM_FP8_REDUCE_CONV=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=1 +ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942" +ENV AMDGPU_TARGETS=gfx942 +ENV ROCM_ARCH=gfx942 +ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" + +# Switch to working directory +WORKDIR /sgl-workspace + +# Clean and create directory +RUN rm -rf /sgl-workspace && mkdir -p /sgl-workspace + +# Clone and build sglang +RUN git clone ${SGL_REPO} \ + && cd sglang \ + && git checkout ${SGL_BRANCH} || echo "Using default branch" \ + && cd sgl-kernel \ + && rm -f pyproject.toml \ + && mv pyproject_rocm.toml pyproject.toml \ + && python setup_rocm.py install \ + && cd .. \ + && if [ "$BUILD_TYPE" = "srt" ]; then \ + python -m pip --no-cache-dir install -e "python[srt_hip]"; \ + else \ + python -m pip --no-cache-dir install -e "python[all_hip]"; \ + fi \ + && cd /sgl-workspace \ + && cp -r /sgl-workspace/sglang /sglang \ + && python -m pip cache purge + +# Install common Python packages +RUN pip install IPython orjson python-multipart torchao pybind11 + +# Rebuild Triton +RUN pip uninstall -y triton || true \ + && git clone ${TRITON_REPO} \ + && cd triton \ + && git checkout ${TRITON_COMMIT} \ + && cd python \ + && python3 setup.py install \ + && cd /sgl-workspace + + +# ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942 --amdgpu-lower-module-lds-strategy=1" +# ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942" + +# Build aiter +#version: Commit 9d11f47 + # && git checkout ${AITER_COMMIT} \ +RUN pip uninstall -y aiter || true +RUN git clone ${AITER_REPO} \ + && cd aiter \ + && git checkout ${AITER_COMMIT} \ + && git submodule sync \ + && git submodule update --init --recursive \ + && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py install \ + && cd /sgl-workspace + # && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py develop \ + # && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py develop \ + +# Copy MI300X config +RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ + /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ + -type f -name '*MI300X*' | \ + xargs -I {} sh -c 'vf_config=$(echo "$1" | sed "s/MI300X/MI300X_VF/"); cp "$1" "$vf_config"' -- {} + +# Environment setup complete. +RUN echo "Environment setup complete." + +WORKDIR /workspace/ +########################################### +########################################### +########################################### + + + + + + +########################################### +###############vllm v0.8.5################# +########################################### +# ENV GITHUB_USERNAME=yushengsu-thu +# ENV GITHUB_MAIL=yushengsu@gmail.com + +# RUN git config --global user.name "${GITHUB_USERNAME}" \ +# && git config --global user.email "${GITHUB_MAIL}" + +WORKDIR /workspace/ + +ENV VLLM_TARGET_DEVICE=rocm +ENV ROCM_PATH=/opt/rocm +ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev + +# Find the repo path in: DockerFile/Dockerfile.rocm_yang +# RUN git clone https://github.com/RLFoundation/vllm-patch.git +RUN pip uninstall -y vllm || true +RUN rm -rf vllm-patch +RUN git clone https://github.com/RLFoundation/vllm-patch.git \ + && cd vllm-patch \ + && git checkout v0.8.5-sleep-numa \ + && rm -rf build/ dist/ *.egg-info \ + && ln -sf /opt/rocm/lib/libamdhip64.so /usr/lib/libamdhip64.so \ + && SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev PYTORCH_ROCM_ARCH="gfx90a;gfx942" MAX_JOBS=${MAX_JOBS} python3 setup.py install + # RUN SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev PYTORCH_ROCM_ARCH="gfx90a;gfx942" MAX_JOBS=${MAX_JOBS} python3 setup.py develop + +WORKDIR /workspace/ +########################################### +########################################### +########################################### + + + + +######################################### +#### Install megatron-core############### +######################################### +RUN pip uninstall -y megatron-core && \ + git clone https://github.com/yushengsu-thu/Megatron-LM-amd_version.git && \ + cd Megatron-LM-amd_version && \ + pip install -vvv -e . && \ + cd /workspace/ +######################################### +######################################### +######################################### + + + + +####################################### +################apex################### +####################################### +WORKDIR /workspace/ +RUN pip uninstall -y apex && \ + git clone https://github.com/ROCm/apex.git && \ + cd apex && \ + python setup.py install && \ + cd /workspace/ +####################################### +####################################### +####################################### + + + + +################################################################################ +###########################Add torch_memory_saver############################### +################################################################################ +# Set environment variables +ENV HIPCC_COMPILE_FLAGS_APPEND="--amdgpu-target=gfx90a;gfx942 -D__HIP_PLATFORM_AMD__" +ENV CFLAGS="-D__HIP_PLATFORM_AMD__" +ENV CXXFLAGS="-D__HIP_PLATFORM_AMD__" +RUN pip install "git+https://github.com/YangWang92/torch_memory_saver_numa.git@numa" +################################################################################ +################################################################################ +################################################################################ + + + +######################################## +######Install ray####################### +######################################## +# need to add this patch: https://github.com/ray-project/ray/pull/53531/files +RUN pip uninstall ray -y +RUN pip install "ray[data,train,tune,serve]>=2.47.0" +######################################## +######################################## +######################################## + + + +########################################## +#######Install other dependencies######### +########################################## +RUN pip install "tensordict==0.6.2" --no-deps && \ + pip install accelerate \ + codetiming \ + datasets \ + dill \ + hydra-core \ + liger-kernel \ + numpy \ + pandas \ + peft \ + "pyarrow>=15.0.0" \ + pylatexenc \ + torchdata \ + wandb \ + orjson \ + pybind11 + +WORKDIR /workspace/ +RUN git clone https://github.com/volcengine/verl.git && \ + cd verl && \ + pip install -e . +########################################## +########################################## +########################################## + + + +WORKDIR /workspace/ + +CMD ["/usr/bin/bash"] +CMD ["/usr/bin/bash"] diff --git a/e2c/verl/docker/Dockerfile.sglang b/e2c/verl/docker/Dockerfile.sglang new file mode 100644 index 0000000000000000000000000000000000000000..11ad4a77da682e1f0de33bbed1007e9aefed7b21 --- /dev/null +++ b/e2c/verl/docker/Dockerfile.sglang @@ -0,0 +1,55 @@ +# Start from the NVIDIA official image (ubuntu-22.04 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:24.08-py3 + +# Define environments +ENV MAX_JOBS=32 +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.ustc.edu.cn/ubuntu/ + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini && \ + apt-get clean + +# Change pip source +ARG PIP_INDEX=https://mirrors.aliyun.com/pypi/simple/ + +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Install sglang-0.4.6.post5 and torch-memory-saver +RUN pip uninstall -y cuda-python && pip install "sglang[all]==0.4.6.post5" --no-cache-dir --find-links https://flashinfer.ai/whl/cu124/torch2.6/flashinfer-python && pip install torch-memory-saver --no-cache-dir + +# Install torch-2.6.0 +RUN pip install --no-cache-dir torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 tensordict torchdata \ + transformers>=4.49.0 accelerate datasets peft hf_transfer \ + ray[default] codetiming hydra-core pandas pyarrow>=15.0.0 pylatexenc qwen-vl-utils wandb liger-kernel \ + pytest pre-commit py-spy pyext + +# Install flash_attn-2.7.4.post1 +RUN pip uninstall -y transformer-engine flash-attn && \ + wget -v https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl && \ + pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl + +# Fix cv2 +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir nvidia-ml-py>=12.560.30 opencv-python-headless==4.8.0.74 fastapi==0.115.6 diff --git a/e2c/verl/docker/Dockerfile.vemlp.vllm.te b/e2c/verl/docker/Dockerfile.vemlp.vllm.te new file mode 100644 index 0000000000000000000000000000000000000000..361fb2084ff4ecea1f48d22acda487d620080ca4 --- /dev/null +++ b/e2c/verl/docker/Dockerfile.vemlp.vllm.te @@ -0,0 +1,41 @@ +# docker buildx build --platform linux/x86_64 -t "verlai/verl:$TAG" -f docker/$FILE . + +# the one in docker.io is an alias for the one veturbo +# FROM vemlp-cn-beijing.cr.volces.com/veturbo/pytorch:2.4-cu124 +FROM docker.io/haibinlin/verl:v0.0.5-th2.4.0-cu124-base + +# only config pip index with https://pypi.tuna.tsinghua.edu.cn/simple if needed +# unset for now +RUN pip3 config unset global.index-url + +# transformers 4.47.0 contains the following bug: +# AttributeError: 'Gemma2Attention' object has no attribute '_flash_attn_uses_top_left_mask' +RUN pip3 install --no-cache-dir \ + torch==2.4.0 \ + accelerate \ + codetiming \ + dill \ + hydra-core \ + numpy \ + pybind11 \ + tensordict \ + "transformers <= 4.46.0" + +RUN pip3 install --no-cache-dir flash-attn==2.7.0.post2 --no-build-isolation + +# vllm depends on ray +RUN pip3 install --no-cache-dir vllm==0.6.3 ray==2.10 + +# install apex +RUN MAX_JOBS=4 pip3 install -v --disable-pip-version-check --no-cache-dir --no-build-isolation \ + --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" \ + git+https://github.com/NVIDIA/apex + +# install Transformer Engine +# - flash-attn pinned to 2.5.3 by TransformerEngine, switch to eric-haibin-lin/TransformerEngine.git@v1.7.0 to relax version req +# - install with: MAX_JOBS=1 NINJA_FLAGS="-j1" TE_BUILD_WITH_NINJA=0 to avoid OOM +# - cudnn is required by TransformerEngine +# RUN CUDNN_PATH=/opt/conda/lib/python3.11/site-packages/nvidia/cudnn \ +# pip3 install git+https://github.com/eric-haibin-lin/TransformerEngine.git@v1.7.0 +RUN MAX_JOBS=1 NINJA_FLAGS="-j1" pip3 install flash-attn==2.5.3 --no-cache-dir --no-build-isolation +RUN MAX_JOBS=1 NINJA_FLAGS="-j1" pip3 install git+https://github.com/NVIDIA/TransformerEngine.git@v1.7 diff --git a/e2c/verl/docker/Dockerfile.vllm.sglang.megatron.deepseek b/e2c/verl/docker/Dockerfile.vllm.sglang.megatron.deepseek new file mode 100644 index 0000000000000000000000000000000000000000..366286f882f7b393ee2c79304b1b659449d4b1ae --- /dev/null +++ b/e2c/verl/docker/Dockerfile.vllm.sglang.megatron.deepseek @@ -0,0 +1,115 @@ +# Start from the NVIDIA official image (ubuntu-22.04 + cuda-12.6 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:24.08-py3 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini aria2 && \ + apt-get clean + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + xgboost transformer_engine flash_attn apex megatron-core grpcio + +# Reinstall CUDA 12.4 +RUN aria2c https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin && \ + mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 + +RUN aria2c --always-resume=true --max-tries=99999 https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda-repo-ubuntu2204-12-4-local_12.4.1-550.54.15-1_amd64.deb && \ + dpkg -i cuda-repo-ubuntu2204-12-4-local_12.4.1-550.54.15-1_amd64.deb && \ + cp /var/cuda-repo-ubuntu2204-12-4-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + apt-get -y install cuda-toolkit-12-4 && \ + rm cuda-repo-ubuntu2204-12-4-local_12.4.1-550.54.15-1_amd64.deb && \ + update-alternatives --set cuda /usr/local/cuda-12.4 && \ + rm -rf /usr/local/cuda-12.6 + +# Install torch-2.6.0+cu124 + vllm-0.8.5.post1 + sglang-0.4.6.post5 +# torch-2.6.0+cu124: cxx11abi=False +# torch-2.6.0+cu126: cxx11abi=True +# see https://github.com/flashinfer-ai/flashinfer/issues/911 +# Install sglang-0.4.6.post1 and torch-memory-saver +RUN pip install --resume-retries 999 "sglang[all]==0.4.6.post5" --no-cache-dir --find-links https://flashinfer.ai/whl/cu124/torch2.6/flashinfer-python && pip install --resume-retries 999 torch-memory-saver --no-cache-dir + +RUN pip install --resume-retries 999 --no-cache-dir "vllm==0.8.5.post1" "torch==2.6.0" "torchvision==0.21.0" "torchaudio==2.6.0" "tensordict==0.6.2" torchdata + +RUN pip install --resume-retries 999 --no-cache-dir "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=15.0.0" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile \ + pytest py-spy pyext pre-commit ruff + +# Install flash-attn-2.7.4.post1 (cxx11abi=False) +RUN wget -nv https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl && \ + pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl + +# Fix packages +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +# Install cudnn +RUN aria2c --max-tries=9999 https://developer.download.nvidia.com/compute/cudnn/9.8.0/local_installers/cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + dpkg -i cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + cp /var/cudnn-local-repo-ubuntu2204-9.8.0/cudnn-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + apt-get -y install cudnn-cuda-12 && \ + rm cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install Apex +RUN git clone https://github.com/NVIDIA/apex.git && \ + cd apex && \ + pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --no-deps --no-cache-dir git+https://github.com/NVIDIA/TransformerEngine.git@v2.3 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 + +# Fix opencv +RUN pip install opencv-python + +RUN pip install opencv-fixer && \ + python -c "from opencv_fixer import AutoFix; AutoFix()" + +# Install verl + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url + + RUN apt-get update && \ + apt-get install -y aria2 libfreeimage3 libfreeimage-dev zlib1g \ No newline at end of file diff --git a/e2c/verl/docker/README.md b/e2c/verl/docker/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c392923bbfbc3ebcd8516c4ed22afdfab64d1a05 --- /dev/null +++ b/e2c/verl/docker/README.md @@ -0,0 +1,70 @@ +# Dockerfiles of verl + +We provide pre-built Docker images for quick setup. And from this version, we utilize a new image release hierarchy for productivity and stability. + +The image types are divided into three large categories: + +- **Base Image**: Without inference and training frameworks, only basic dependencies are installed. Can directly install vllm or SGLang on top of it, without need of reinstall torch or CUDA. +- **Application Image**: Stable version with inference and training frameworks installed. +- **Preview Image**: Unstable version with the latest frameworks and features. + +The first two types of images are hosted on dockerhub [verlai/verl](https://hub.docker.com/r/verlai/verl) repository, while the preview images are hosted on community repository. + +> The image versions are mapped with verl releases, for example, image with tag ``verl0.4`` is built for verl release ``v0.4.x``. + +## Base Image + +The stable base image is ``verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.7.4`` with different CUDA versions. + +The update of base image is not frequent, and the app image can be built on top of it without reinstalling base packages. + +## Application Image + +From this version, we divide images built for vLLM and SGLang as the divergence of dependent packages like FlashInfer. +There are 2 types of application images available: + +- **vLLM with FSDP and Megatron**: ``verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2`` +- **SGLang with FSDP and Megatron**: `verlai/verl:app-verl0.5-transformers4.55.4-sglang0.4.10.post2-mcore0.13.0-te2.2` + +Docker images with Megatron backends are runnable with large language model like ``Qwen/Qwen3-235B-A22B``, ``deepseek-ai/DeepSeek-V3-0324`` post-training. Refer to the :doc:`Large Language Model Post-Training documentation<../perf/dpsk>` for more details. + +Application images can be updated frequently, and the Dockerfile can be found in ``docker/verl[version]-[packages]/Dockerfile.app.[frameworks]``. Based on the base image, it is easy to build your own application image with the desired inference and training frameworks. + +## Community Image + +For vLLM with FSDP, please refer to [hiyouga/verl](https://hub.docker.com/r/hiyouga/verl) repository and the latest version is ``hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.4-flashinfer0.2.2-cxx11abi0``. + +For SGLang with FSDP, please refer to [ocss884/verl-sglang](https://hub.docker.com/r/ocss884/verl-sglang) repository and the latest version is ``ocss884/verl-sglang:ngc-th2.6.0-cu126-sglang0.4.6.post5`` which is provided by SGLang RL Group. + +See files under ``docker/`` for NGC-based image or if you want to build your own. + +Note that For aws instances with EFA net interface (Sagemaker AI Pod), you need to install EFA driver as shown in ``docker/Dockerfile.extenstion.awsefa`` + +## Installation from Docker + +After pulling the desired Docker image and installing desired inference and training frameworks, you can run it with the following steps: + +1. Launch the desired Docker image and attach into it: + +```sh +docker create --runtime=nvidia --gpus all --net=host --shm-size="10g" --cap-add=SYS_ADMIN -v .:/workspace/verl --name verl sleep infinity +docker start verl +docker exec -it verl bash +``` + +2. If you use the images provided, you only need to install verl itself without dependencies: + +```sh +# install the nightly version (recommended) +git clone https://github.com/volcengine/verl && cd verl +pip3 install --no-deps -e . +``` + +[Optional] If you hope to switch between different frameworks, you can install verl with the following command: + +```sh +# install the nightly version (recommended) +git clone https://github.com/volcengine/verl && cd verl +pip3 install -e .[vllm] +pip3 install -e .[sglang] +``` diff --git a/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.12 b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.12 new file mode 100644 index 0000000000000000000000000000000000000000..68c3e1a043b3955f3773ca1665de5d8d1dd4fc17 --- /dev/null +++ b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.12 @@ -0,0 +1,41 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.4-cu124-cudnn9.8-torch2.6-fa2.7.4 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install sglang-0.4.6.post5 and torch-memory-saver +RUN pip install --resume-retries 999 "sglang[all]==0.4.6.post5" --no-cache-dir --find-links https://flashinfer.ai/whl/cu124/torch2.6/flashinfer-python && pip install torch-memory-saver --no-cache-dir + +# Some sglang operations in 0.4.6.post5 require vllm +# [Warning] vllm can have some packages not compatible with sglang, for example, flashinfer +RUN pip install --resume-retries 999 --no-cache-dir vllm==0.8.5.post1 + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 + +# Fix for transformers 4.53.0 +RUN pip3 install --no-cache-dir "transformers[hf_xet]<4.52.0" + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge \ No newline at end of file diff --git a/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.12.deepep b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.12.deepep new file mode 100644 index 0000000000000000000000000000000000000000..f08aeee9c7389f036e4b8ca032209495d9f5bffe --- /dev/null +++ b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.12.deepep @@ -0,0 +1,82 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.4-cu124-cudnn9.8-torch2.6-fa2.7.4 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install sglang-0.4.6.post5 and torch-memory-saver +RUN pip install --resume-retries 999 "sglang[all]==0.4.6.post5" --no-cache-dir --find-links https://flashinfer.ai/whl/cu124/torch2.6/flashinfer-python && pip install torch-memory-saver --no-cache-dir + +# Some sglang operations in 0.4.6.post5 require vllm +# [Warning] vllm can have some packages not compatible with sglang, for example, flashinfer +RUN pip install --resume-retries 999 --no-cache-dir vllm==0.8.5.post1 + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 + +# Fix for transformers 4.53.0 +RUN pip3 install --no-cache-dir "transformers[hf_xet]<4.52.0" + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge + +# Install DeepEP +## the dependency of IBGDA +RUN ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +## Clone and build deepep and deepep-nvshmem +RUN git clone -b v2.3.1 https://github.com/NVIDIA/gdrcopy.git && \ + git clone https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout a84a248 + +# Prepare nvshmem +RUN wget https://developer.nvidia.com/downloads/assets/secure/nvshmem/nvshmem_src_3.2.5-1.txz && \ + tar -xvf nvshmem_src_3.2.5-1.txz && mv nvshmem_src deepep-nvshmem && \ + cd deepep-nvshmem && git apply ../DeepEP/third-party/nvshmem.patch + +ENV CUDA_HOME=/usr/local/cuda +### Set MPI environment variables. Having errors when not set. +ENV CPATH=/usr/local/mpi/include:$CPATH +ENV LD_LIBRARY_PATH=/usr/local/mpi/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV GDRCOPY_HOME=/workspace/gdrcopy + +## Build deepep-nvshmem +RUN cd deepep-nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -G Ninja -S . -B build/ -DCMAKE_INSTALL_PREFIX=/workspace/deepep-nvshmem/install && cmake --build build/ --target install + +ENV NVSHMEM_DIR=/workspace/deepep-nvshmem/install +ENV LD_LIBRARY_PATH=$NVSHMEM_DIR/lib:$LD_LIBRARY_PATH +ENV PATH=$NVSHMEM_DIR/bin:$PATH + +## Build deepep +RUN cd DeepEP && \ + python setup.py install \ No newline at end of file diff --git a/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.13.preview b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.13.preview new file mode 100644 index 0000000000000000000000000000000000000000..0e0bdd43fec1eac3e4a85de5416e34fad4ac7c69 --- /dev/null +++ b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.sglang.vllm.mcore0.13.preview @@ -0,0 +1,82 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.4-cu124-cudnn9.8-torch2.6-fa2.7.4 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install sglang-0.4.6.post5 and torch-memory-saver +RUN pip install --resume-retries 999 "sglang[all]==0.4.6.post5" --no-cache-dir --find-links https://flashinfer.ai/whl/cu124/torch2.6/flashinfer-python && pip install torch-memory-saver --no-cache-dir + +# Some sglang operations in 0.4.6.post5 require vllm +# [Warning] vllm can have some packages not compatible with sglang, for example, flashinfer +RUN pip install --resume-retries 999 --no-cache-dir vllm==0.8.5.post1 + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@release_v2.5 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_r0.13.0 + +# Fix for transformers 4.53.0 +RUN pip3 install --no-cache-dir "transformers[hf_xet]<4.52.0" + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge + +# Install DeepEP +## the dependency of IBGDA +RUN ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +## Clone and build deepep and deepep-nvshmem +RUN git clone -b v2.3.1 https://github.com/NVIDIA/gdrcopy.git && \ + git clone https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout a84a248 + +# Prepare nvshmem +RUN wget https://developer.nvidia.com/downloads/assets/secure/nvshmem/nvshmem_src_3.2.5-1.txz && \ + tar -xvf nvshmem_src_3.2.5-1.txz && mv nvshmem_src deepep-nvshmem && \ + cd deepep-nvshmem && git apply ../DeepEP/third-party/nvshmem.patch + +ENV CUDA_HOME=/usr/local/cuda +### Set MPI environment variables. Having errors when not set. +ENV CPATH=/usr/local/mpi/include:$CPATH +ENV LD_LIBRARY_PATH=/usr/local/mpi/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV GDRCOPY_HOME=/workspace/gdrcopy + +## Build deepep-nvshmem +RUN cd deepep-nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -G Ninja -S . -B build/ -DCMAKE_INSTALL_PREFIX=/workspace/deepep-nvshmem/install && cmake --build build/ --target install + +ENV NVSHMEM_DIR=/workspace/deepep-nvshmem/install +ENV LD_LIBRARY_PATH=$NVSHMEM_DIR/lib:$LD_LIBRARY_PATH +ENV PATH=$NVSHMEM_DIR/bin:$PATH + +## Build deepep +RUN cd DeepEP && \ + python setup.py install \ No newline at end of file diff --git a/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.12 b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.12 new file mode 100644 index 0000000000000000000000000000000000000000..616c701c523a76eb11cfe8364a40ca7329023f73 --- /dev/null +++ b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.12 @@ -0,0 +1,47 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.4-cu124-cudnn9.8-torch2.6-fa2.7.4 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install torch-2.6.0+cu124 + vllm-0.8.5.post1 +# torch-2.6.0+cu124: cxx11abi=False +# torch-2.6.0+cu126: cxx11abi=True +# see https://github.com/flashinfer-ai/flashinfer/issues/911 +RUN pip install --resume-retries 999 --no-cache-dir vllm==0.8.5.post1 + +# Install flashinfer-0.2.2.post1+cu126 (cxx11abi=True) +# vllm-0.8.3 does not support flashinfer>=0.2.3 +# see https://github.com/vllm-project/vllm/pull/15777 +RUN aria2c --max-tries=9999 https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.2.post1/flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + pip install --no-cache-dir flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + rm flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 + +# Fix for transformers 4.53.0 +RUN pip3 install --no-cache-dir "transformers[hf_xet]<4.52.0" + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge \ No newline at end of file diff --git a/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.12.deepep b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.12.deepep new file mode 100644 index 0000000000000000000000000000000000000000..a2be5e998dd32a2bcfbdcc63bedd2dd1ae634943 --- /dev/null +++ b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.12.deepep @@ -0,0 +1,88 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.4-cu124-cudnn9.8-torch2.6-fa2.7.4 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install torch-2.6.0+cu124 + vllm-0.8.5.post1 +# torch-2.6.0+cu124: cxx11abi=False +# torch-2.6.0+cu126: cxx11abi=True +# see https://github.com/flashinfer-ai/flashinfer/issues/911 +RUN pip install --resume-retries 999 --no-cache-dir vllm==0.8.5.post1 + +# Install flashinfer-0.2.2.post1+cu126 (cxx11abi=True) +# vllm-0.8.3 does not support flashinfer>=0.2.3 +# see https://github.com/vllm-project/vllm/pull/15777 +RUN aria2c --max-tries=9999 https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.2.post1/flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + pip install --no-cache-dir flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + rm flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 + +# Fix for transformers 4.53.0 +RUN pip3 install --no-cache-dir "transformers[hf_xet]<4.52.0" + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge + +# Install DeepEP +## the dependency of IBGDA +RUN ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +## Clone and build deepep and deepep-nvshmem +RUN git clone -b v2.3.1 https://github.com/NVIDIA/gdrcopy.git && \ + git clone https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout a84a248 + +# Prepare nvshmem +RUN wget https://developer.nvidia.com/downloads/assets/secure/nvshmem/nvshmem_src_3.2.5-1.txz && \ + tar -xvf nvshmem_src_3.2.5-1.txz && mv nvshmem_src deepep-nvshmem && \ + cd deepep-nvshmem && git apply ../DeepEP/third-party/nvshmem.patch + +ENV CUDA_HOME=/usr/local/cuda +### Set MPI environment variables. Having errors when not set. +ENV CPATH=/usr/local/mpi/include:$CPATH +ENV LD_LIBRARY_PATH=/usr/local/mpi/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV GDRCOPY_HOME=/workspace/gdrcopy + +## Build deepep-nvshmem +RUN cd deepep-nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -G Ninja -S . -B build/ -DCMAKE_INSTALL_PREFIX=/workspace/deepep-nvshmem/install && cmake --build build/ --target install + +ENV NVSHMEM_DIR=/workspace/deepep-nvshmem/install +ENV LD_LIBRARY_PATH=$NVSHMEM_DIR/lib:$LD_LIBRARY_PATH +ENV PATH=$NVSHMEM_DIR/bin:$PATH + +## Build deepep +RUN cd DeepEP && \ + python setup.py install \ No newline at end of file diff --git a/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.13.preview b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.13.preview new file mode 100644 index 0000000000000000000000000000000000000000..be6183a8479fe3e379f7ddaa06034c3e1ff64278 --- /dev/null +++ b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.13.preview @@ -0,0 +1,85 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.4-cu124-cudnn9.8-torch2.6-fa2.7.4 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install torch-2.6.0+cu124 + vllm-0.8.5.post1 +# torch-2.6.0+cu124: cxx11abi=False +# torch-2.6.0+cu126: cxx11abi=True +# see https://github.com/flashinfer-ai/flashinfer/issues/911 +RUN pip install --resume-retries 999 --no-cache-dir vllm==0.8.5.post1 + +# Install flashinfer-0.2.2.post1+cu126 (cxx11abi=True) +# vllm-0.8.3 does not support flashinfer>=0.2.3 +# see https://github.com/vllm-project/vllm/pull/15777 +RUN aria2c --max-tries=9999 https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.2.post1/flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + pip install --no-cache-dir flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + rm flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@release_v2.5 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge + +# Install DeepEP +## the dependency of IBGDA +RUN ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +## Clone and build deepep and deepep-nvshmem +RUN git clone -b v2.3.1 https://github.com/NVIDIA/gdrcopy.git && \ + git clone https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout a84a248 + +# Prepare nvshmem +RUN wget https://developer.nvidia.com/downloads/assets/secure/nvshmem/nvshmem_src_3.2.5-1.txz && \ + tar -xvf nvshmem_src_3.2.5-1.txz && mv nvshmem_src deepep-nvshmem && \ + cd deepep-nvshmem && git apply ../DeepEP/third-party/nvshmem.patch + +ENV CUDA_HOME=/usr/local/cuda +### Set MPI environment variables. Having errors when not set. +ENV CPATH=/usr/local/mpi/include:$CPATH +ENV LD_LIBRARY_PATH=/usr/local/mpi/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV GDRCOPY_HOME=/workspace/gdrcopy + +## Build deepep-nvshmem +RUN cd deepep-nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -G Ninja -S . -B build/ -DCMAKE_INSTALL_PREFIX=/workspace/deepep-nvshmem/install && cmake --build build/ --target install + +ENV NVSHMEM_DIR=/workspace/deepep-nvshmem/install +ENV LD_LIBRARY_PATH=$NVSHMEM_DIR/lib:$LD_LIBRARY_PATH +ENV PATH=$NVSHMEM_DIR/bin:$PATH + +## Build deepep +RUN cd DeepEP && \ + python setup.py install \ No newline at end of file diff --git a/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.base b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.base new file mode 100644 index 0000000000000000000000000000000000000000..25b1d9431e237287bfd720788974894d1c07873e --- /dev/null +++ b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.base @@ -0,0 +1,113 @@ +# Base Docker Image of verl, with CUDA/Torch/FlashAttn/Apex/TransformerEngine, without other frameworks +# Target: verlai/verl:base-v2-cu124-cudnn9.8-torch2.6-fa2.8.0-te2.3 +# Start from the NVIDIA official image (ubuntu-22.04 + cuda-12.6 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:24.08-py3 + +# Define environments +ENV MAX_JOBS=16 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini aria2 && \ + apt-get clean + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + xgboost transformer_engine flash_attn apex megatron-core grpcio + +# Reinstall CUDA 12.4 +RUN aria2c https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin && \ + mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 + +RUN aria2c --always-resume=true --max-tries=99999 https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda-repo-ubuntu2204-12-4-local_12.4.1-550.54.15-1_amd64.deb && \ + dpkg -i cuda-repo-ubuntu2204-12-4-local_12.4.1-550.54.15-1_amd64.deb && \ + cp /var/cuda-repo-ubuntu2204-12-4-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + apt-get -y install cuda-toolkit-12-4 && \ + rm cuda-repo-ubuntu2204-12-4-local_12.4.1-550.54.15-1_amd64.deb && \ + update-alternatives --set cuda /usr/local/cuda-12.4 && \ + rm -rf /usr/local/cuda-12.6 + +RUN pip install --resume-retries 999 --no-cache-dir torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 + +RUN pip install --resume-retries 999 --no-cache-dir "tensordict==0.6.2" torchdata "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +# Install flash-attn-2.7.4.post1 (cxx11abi=False) +RUN wget -nv https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl && \ + pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl + +# Fix packages +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +# Install cudnn +RUN aria2c --max-tries=9999 https://developer.download.nvidia.com/compute/cudnn/9.8.0/local_installers/cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + dpkg -i cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + cp /var/cudnn-local-repo-ubuntu2204-9.8.0/cudnn-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + apt-get -y install cudnn-cuda-12 && \ + rm cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb + +# Install Apex +RUN git clone https://github.com/NVIDIA/apex.git && \ + cd apex && \ + pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ + +# Profiling tools +RUN aria2c --always-resume=true --max-tries=99999 https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2025_3/nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb && \ + apt-get update && apt-get install -y libxcb-cursor0 && \ + dpkg -i ./nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb && \ + rm -rf /usr/local/cuda/bin/nsys && \ + ln -s /opt/nvidia/nsight-systems/2025.3.1/target-linux-x64/nsys /usr/local/cuda/bin/nsys && \ + rm -rf /usr/local/cuda/bin/nsys-ui && \ + ln -s /opt/nvidia/nsight-systems/2025.3.1/target-linux-x64/nsys-ui /usr/local/cuda/bin/nsys-ui && \ + rm nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb + +# Fix opencv +RUN pip install --resume-retries 999 --no-cache-dir opencv-python + +RUN pip install --resume-retries 999 --no-cache-dir opencv-fixer && \ + python -c "from opencv_fixer import AutoFix; AutoFix()" + +RUN pip install --resume-retries 999 --no-cache-dir cuda-bindings + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url + +RUN apt-get update && \ + apt-get install -y libfreeimage3 libfreeimage-dev zlib1g htop + diff --git a/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/README.md b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/README.md new file mode 100644 index 0000000000000000000000000000000000000000..022fb47784730b723f2113eedd370dc20085f4bc --- /dev/null +++ b/e2c/verl/docker/verl0.4-cu124-torch2.6-fa2.7.4/README.md @@ -0,0 +1,31 @@ +# verl image with verl v0.4.x + +## Important packages version + +```txt +cuda==12.4 +cudnn==9.8.0 +torch==2.6.0 +flash_attn=2.7.4 +sglang==0.4.6.post5 +vllm==0.8.5.post1 +vidia-cudnn-cu12==9.8.0.87 +transformer_engine==2.3 +megatron.core==core_v0.12.2 +# Preview +transformer_engine==2.5 +megatron.core==core_r0.13.0 +``` + +## Target + +- Base image: + - `verlai/verl:base-verl0.4-cu124-cudnn9.8-torch2.6-fa2.7.4` +- App image: + - `verlai/verl:app-verl0.4-sglang0.4.6.post5-vllm0.8.5-mcore0.12.2-te2.2`: SGLang requires vLLM in 0.4.6.post5 version, vLLM can have some package conflicts with SGLang + - `verlai/verl:app-verl0.4-sglang0.4.6.post5-vllm0.8.5-mcore0.12.2-te2.2-deepep`: Built with deepep + - `verlai/verl:app-verl0.4-vllm0.8.5-mcore0.12.2-te2.2` + - `verlai/verl:app-verl0.4-vllm0.8.5-mcore0.12.2-te2.2-deepep`: Built with deepep +- Preview image: + - `verlai/verl:app-verl0.4-sglang0.4.6.post5-vllm0.8.5-mcore0.13.0-te2.2-preview` + - `verlai/verl:app-verl0.4-vllm0.8.5-mcore0.13.0-te2.2-preview` \ No newline at end of file diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.sglang0.4.10.post2.mcore0.13 b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.sglang0.4.10.post2.mcore0.13 new file mode 100644 index 0000000000000000000000000000000000000000..d1be8fb450bf0fe62fcf3d2cc3a140867ce99827 --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.sglang0.4.10.post2.mcore0.13 @@ -0,0 +1,37 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.7.4 + +# Define environments +ENV MAX_JOBS=8 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install sglang-0.4.10 +# Install FlashInfer Python package +RUN pip install --upgrade pip setuptools packaging +RUN pip install --resume-retries 999 --no-cache-dir --no-build-isolation flashinfer-python==0.2.9rc1 +RUN pip install --resume-retries 999 --no-cache-dir --no-build-isolation "sglang[all]==0.4.10.post2" + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]==4.55.4" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.13.0 + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.sglang0.4.9.post6.mcore0.13 b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.sglang0.4.9.post6.mcore0.13 new file mode 100644 index 0000000000000000000000000000000000000000..d79201a92eec3d778e32a023cfa3a3b32bf3ac55 --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.sglang0.4.9.post6.mcore0.13 @@ -0,0 +1,37 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.7.4 + +# Define environments +ENV MAX_JOBS=8 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install sglang-0.4.10 +# Install FlashInfer Python package +RUN pip install --upgrade pip setuptools packaging +RUN pip install --resume-retries 999 --no-cache-dir --no-build-isolation flashinfer-python==0.2.9rc1 +RUN pip install --resume-retries 999 --no-cache-dir --no-build-isolation "sglang[all]==0.4.9.post6" + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]==4.55.4" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.13.0 + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge \ No newline at end of file diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.vllm.mcore0.13 b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.vllm.mcore0.13 new file mode 100644 index 0000000000000000000000000000000000000000..9d73e0ffeeb1c8d17c6d9022d5e5411dc5d70cc1 --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.vllm.mcore0.13 @@ -0,0 +1,38 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.7.4 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install torch-2.7.1+cu126 + vllm-0.10.0 +RUN pip install --resume-retries 999 --no-cache-dir vllm==0.10.0 + +# Fix packages +# transformers 4.54.0 still not support +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.55.4" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.13.0 + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge + +# Fix qwen vl +RUN pip3 install --no-cache-dir --no-deps trl \ No newline at end of file diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.vllm.mcore0.15 b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.vllm.mcore0.15 new file mode 100644 index 0000000000000000000000000000000000000000..296fd3b74f641efce2c90d6aec81fdcb81b8a867 --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.app.vllm.mcore0.15 @@ -0,0 +1,39 @@ +# Start from the verl base image +# Dockerfile.base +FROM iseekyan/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.7.4-h100 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install torch-2.7.1+cu126 + vllm-0.10.0 +RUN pip install --resume-retries 999 --no-cache-dir vllm==0.10.0 + +# Fix packages +# transformers 4.54.0 still not support +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.55.4" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@release_v2.7 +RUN pip install onnxscript + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.15.0rc4 + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge==v0.15.0 + +# Fix qwen vl +RUN pip3 install --no-cache-dir --no-deps trl \ No newline at end of file diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.base.torch2.7.1 b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.base.torch2.7.1 new file mode 100644 index 0000000000000000000000000000000000000000..6d85cf512016e4cc0988da3d099230d5d0c2b5d7 --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/Dockerfile.base.torch2.7.1 @@ -0,0 +1,133 @@ +# Base Docker Image of verl, with CUDA/Torch/FlashAttn/Apex/TransformerEngine, without other frameworks +# Target: verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.8.0-fi0.2.6 +# Start from the NVIDIA official image (ubuntu-22.04 + cuda-12.6 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:24.08-py3 + +# Define environments +ENV MAX_JOBS=16 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini aria2 libfreeimage3 libfreeimage-dev zlib1g htop && \ + apt-get clean + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + xgboost transformer_engine flash_attn apex megatron-core grpcio + +RUN pip install --resume-retries 999 --no-cache-dir torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 + +# Install flash-attn-2.7.4.post1, although built with torch2.6, it is compatible with torch2.7 +# https://github.com/Dao-AILab/flash-attention/issues/1644#issuecomment-2899396361 +RUN ABI_FLAG=$(python -c "import torch; print('TRUE' if torch._C._GLIBCXX_USE_CXX11_ABI else 'FALSE')") && \ + URL="https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abi${ABI_FLAG}-cp310-cp310-linux_x86_64.whl" && \ + FILE="flash_attn-2.7.4.post1+cu12torch2.6cxx11abi${ABI_FLAG}-cp310-cp310-linux_x86_64.whl" && \ + wget -nv "${URL}" && \ + pip install --no-cache-dir "${FILE}" + +# Fix packages +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +# Install cudnn +RUN aria2c --max-tries=9999 https://developer.download.nvidia.com/compute/cudnn/9.8.0/local_installers/cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + dpkg -i cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + cp /var/cudnn-local-repo-ubuntu2204-9.8.0/cudnn-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + apt-get -y install cudnn-cuda-12 && \ + rm cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb + +# Install Apex +RUN pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" --resume-retries 999 git+https://github.com/NVIDIA/apex.git + +# Profiling tools +RUN aria2c --always-resume=true --max-tries=99999 https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2025_3/nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb && \ + apt-get update && apt-get install -y libxcb-cursor0 + +RUN apt-get install -y ./nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb && \ + rm -rf /usr/local/cuda/bin/nsys && \ + ln -s /opt/nvidia/nsight-systems/2025.3.1/target-linux-x64/nsys /usr/local/cuda/bin/nsys && \ + rm -rf /usr/local/cuda/bin/nsys-ui && \ + ln -s /opt/nvidia/nsight-systems/2025.3.1/target-linux-x64/nsys-ui /usr/local/cuda/bin/nsys-ui && \ + rm nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb + +RUN pip install --resume-retries 999 --no-cache-dir "tensordict==0.6.2" torchdata "transformers[hf_xet]>=4.52.3" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas cuda-bindings \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +# Install DeepEP +## the dependency of IBGDA +RUN ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +## Clone and build deepep and deepep-nvshmem +RUN git clone -b v2.3.1 https://github.com/NVIDIA/gdrcopy.git && \ + git clone https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout a84a248 + +# Prepare nvshmem +RUN wget https://developer.nvidia.com/downloads/assets/secure/nvshmem/nvshmem_src_3.2.5-1.txz && \ + tar -xvf nvshmem_src_3.2.5-1.txz && mv nvshmem_src deepep-nvshmem && \ + cd deepep-nvshmem && git apply ../DeepEP/third-party/nvshmem.patch + +ENV CUDA_HOME=/usr/local/cuda +### Set MPI environment variables. Having errors when not set. +ENV CPATH=/usr/local/mpi/include:$CPATH +ENV LD_LIBRARY_PATH=/usr/local/mpi/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV GDRCOPY_HOME=/workspace/gdrcopy + +## Build deepep-nvshmem +RUN cd deepep-nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -G Ninja -S . -B build/ -DCMAKE_INSTALL_PREFIX=/workspace/deepep-nvshmem/install && cmake --build build/ --target install + +ENV NVSHMEM_DIR=/workspace/deepep-nvshmem/install +ENV LD_LIBRARY_PATH=$NVSHMEM_DIR/lib:$LD_LIBRARY_PATH +ENV PATH=$NVSHMEM_DIR/bin:$PATH + +## Build deepep +RUN cd DeepEP && \ + python setup.py install + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url + diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/README.md b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3327050e4f2a4a44751e9f483a635b580b2630e4 --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7-fa2.7.4/README.md @@ -0,0 +1,27 @@ +# verl image with verl v0.5 + +## Important packages version + +```txt +cuda==12.6 +cudnn==9.8.0 +torch==2.7.1 +flash_attn=2.7.4.post1 +sglang==0.4.9.post6 +vllm==0.8.5.post1 +vidia-cudnn-cu12==9.8.0.87 +transformer_engine==2.3 +megatron.core==core_v0.12.2 +# Preview +transformer_engine==2.5 +megatron.core==core_r0.13.0 +``` + +## Target + +- Base image: + - `verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.7.4`: We offer a base image with deep ep built in, for vllm/sglang +- App image: + - `verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2` + - `verlai/verl:app-verl0.5-transformers4.55.4-sglang0.4.10.post2-mcore0.13.0-te2.2` + - `iseekyan/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.15.0-te2.7` diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.mcore0.12 b/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.mcore0.12 new file mode 100644 index 0000000000000000000000000000000000000000..205814a234bf4d1162a2fa89f1cd051d6463fb47 --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.mcore0.12 @@ -0,0 +1,37 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.8.0 + +# Define environments +ENV MAX_JOBS=8 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install sglang-0.4.8 and torch-memory-saver +# Install FlashInfer Python package +RUN pip install --upgrade pip setuptools packaging +RUN pip install --resume-retries 999 --no-cache-dir --no-build-isolation flashinfer-python==0.2.6.post1 +RUN pip install --resume-retries 999 --no-cache-dir "sglang[all]==0.4.8" && pip install torch-memory-saver --no-cache-dir + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.3 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge \ No newline at end of file diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.mcore0.13.preview b/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.mcore0.13.preview new file mode 100644 index 0000000000000000000000000000000000000000..5704813d9eee80a7ca790b204a836e81d58cbf1a --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.mcore0.13.preview @@ -0,0 +1,37 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.8.0 + +# Define environments +ENV MAX_JOBS=8 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install sglang-0.4.8 and torch-memory-saver +# Install FlashInfer Python package +RUN pip install --upgrade pip setuptools packaging +RUN pip install --resume-retries 999 --no-cache-dir --no-build-isolation flashinfer-python==0.2.6.post1 +RUN pip install --resume-retries 999 --no-cache-dir "sglang[all]==0.4.8" && pip install torch-memory-saver --no-cache-dir + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@release_v2.5 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge \ No newline at end of file diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.base b/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.base new file mode 100644 index 0000000000000000000000000000000000000000..53089fba400d56a3a37e1e943439c367ee40df55 --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/Dockerfile.base @@ -0,0 +1,132 @@ +# Base Docker Image of verl, with CUDA/Torch/FlashAttn/Apex/TransformerEngine, without other frameworks +# Target: verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.8.0-fi0.2.6 +# Start from the NVIDIA official image (ubuntu-22.04 + cuda-12.6 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:24.08-py3 + +# Define environments +ENV MAX_JOBS=16 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini aria2 libfreeimage3 libfreeimage-dev zlib1g htop && \ + apt-get clean + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + xgboost transformer_engine flash_attn apex megatron-core grpcio + +RUN pip install --resume-retries 999 --no-cache-dir torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 + +# Install flash-attn-2.8.0.post2 (cxx11abi=True) +RUN ABI_FLAG=$(python -c "import torch; print('TRUE' if torch._C._GLIBCXX_USE_CXX11_ABI else 'FALSE')") && \ + URL="https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.0.post2/flash_attn-2.8.0.post2+cu12torch2.7cxx11abi${ABI_FLAG}-cp310-cp310-linux_x86_64.whl" && \ + FILE="flash_attn-2.8.0.post2+cu12torch2.7cxx11abi${ABI_FLAG}-cp310-cp310-linux_x86_64.whl" && \ + wget -nv "${URL}" && \ + pip install --no-cache-dir "${FILE}" + +# Fix packages +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +# Install cudnn +RUN aria2c --max-tries=9999 https://developer.download.nvidia.com/compute/cudnn/9.8.0/local_installers/cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + dpkg -i cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + cp /var/cudnn-local-repo-ubuntu2204-9.8.0/cudnn-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + apt-get -y install cudnn-cuda-12 && \ + rm cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb + +# Install Apex +RUN pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" --resume-retries 999 git+https://github.com/NVIDIA/apex.git + +# Profiling tools +RUN aria2c --always-resume=true --max-tries=99999 https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2025_3/nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb && \ + apt-get update && apt-get install -y libxcb-cursor0 + +RUN apt-get install -y ./nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb && \ + rm -rf /usr/local/cuda/bin/nsys && \ + ln -s /opt/nvidia/nsight-systems/2025.3.1/target-linux-x64/nsys /usr/local/cuda/bin/nsys && \ + rm -rf /usr/local/cuda/bin/nsys-ui && \ + ln -s /opt/nvidia/nsight-systems/2025.3.1/target-linux-x64/nsys-ui /usr/local/cuda/bin/nsys-ui && \ + rm nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb + +RUN pip install --resume-retries 999 --no-cache-dir "tensordict==0.6.2" torchdata "transformers[hf_xet]>=4.53" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas cuda-bindings \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pyext pre-commit ruff + +# Install DeepEP +## the dependency of IBGDA +RUN ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +## Clone and build deepep and deepep-nvshmem +RUN git clone -b v2.3.1 https://github.com/NVIDIA/gdrcopy.git && \ + git clone https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout a84a248 + +# Prepare nvshmem +RUN wget https://developer.nvidia.com/downloads/assets/secure/nvshmem/nvshmem_src_3.2.5-1.txz && \ + tar -xvf nvshmem_src_3.2.5-1.txz && mv nvshmem_src deepep-nvshmem && \ + cd deepep-nvshmem && git apply ../DeepEP/third-party/nvshmem.patch + +ENV CUDA_HOME=/usr/local/cuda +### Set MPI environment variables. Having errors when not set. +ENV CPATH=/usr/local/mpi/include:$CPATH +ENV LD_LIBRARY_PATH=/usr/local/mpi/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV GDRCOPY_HOME=/workspace/gdrcopy + +## Build deepep-nvshmem +RUN cd deepep-nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -G Ninja -S . -B build/ -DCMAKE_INSTALL_PREFIX=/workspace/deepep-nvshmem/install && cmake --build build/ --target install + +ENV NVSHMEM_DIR=/workspace/deepep-nvshmem/install +ENV LD_LIBRARY_PATH=$NVSHMEM_DIR/lib:$LD_LIBRARY_PATH +ENV PATH=$NVSHMEM_DIR/bin:$PATH + +## Build deepep +RUN cd DeepEP && \ + python setup.py install + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url + diff --git a/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/README.md b/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f8c5459a29e31dd91d3ddb1d1eb418bdb21e05eb --- /dev/null +++ b/e2c/verl/docker/verl0.5-cu126-torch2.7.1-fa2.8.0/README.md @@ -0,0 +1,27 @@ +# verl image with verl v0.5 + +## Important packages version + +```txt +cuda==12.6 +cudnn==9.8.0 +torch==2.7.1 +flash_attn=2.8.0 ## +sglang==0.4.8 +vllm==0.8.5.post1 +vidia-cudnn-cu12==9.8.0.87 +transformer_engine==2.3 +megatron.core==core_v0.12.2 +# Preview +transformer_engine==2.5 +megatron.core==core_r0.13.0 +``` + +## Target + +- Base image: + - `verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.8.0`: We offer a base image with deep ep built in +- App image: + - `verlai/verl:app-verl0.5-sglang0.4.9-mcore0.12.2` + - `verlai/verl:app-verl0.5-sglang0.4.9-mcore0.13.0-preview` +- vllm temporarily not support latest version \ No newline at end of file diff --git a/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.megatron b/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.megatron new file mode 100644 index 0000000000000000000000000000000000000000..d41ea19d69bccb39436a79a0798a9cb0cbefec04 --- /dev/null +++ b/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/Dockerfile.app.sglang.megatron @@ -0,0 +1,36 @@ +# Start from the verl base image +# Dockerfile.base +FROM verlai/verl:base-verl0.5-preview-cu128-cudnn9.8-torch2.7.1-fa2.8.0-fi0.2.6 + +# Define environments +ENV MAX_JOBS=8 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Install sglang-0.4.8 and torch-memory-saver +# Install FlashInfer Python package +RUN pip install --resume-retries 999 --no-cache-dir --no-build-isolation flashinfer-python==0.2.6.post1 +RUN pip install --resume-retries 999 --no-cache-dir "sglang[all]==0.4.8" && pip install torch-memory-saver --no-cache-dir + +# Fix packages +RUN pip install --no-cache-dir "tensordict==0.6.2" "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pre-commit ruff + +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --resume-retries 999 --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +RUN pip install --resume-retries 999 --no-cache-dir nvidia-cudnn-cu12==9.8.0.87 + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --resume-retries 999 --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@release_v2.5 + +# Install Megatron-LM +RUN pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/Megatron-LM.git@core_r0.13.0 + +# Install mbridge +RUN pip3 install --no-cache-dir mbridge \ No newline at end of file diff --git a/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/Dockerfile.base b/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/Dockerfile.base new file mode 100644 index 0000000000000000000000000000000000000000..29c49faa848c0e15350abaf48cad7f8a899f4eb6 --- /dev/null +++ b/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/Dockerfile.base @@ -0,0 +1,91 @@ +# Base Docker Image of verl, with CUDA/Torch/FlashAttn/Apex/TransformerEngine, without other frameworks +# Target: verlai/verl:base-verl0.5-preview-cu128-cudnn9.8-torch2.7.1-fa2.8.0-fi0.2.6 +# Start from the NVIDIA official image (ubuntu-22.04 + cuda-12.6 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:25.02-py3 + +# Define environments +ENV MAX_JOBS=16 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini aria2 libfreeimage3 libfreeimage-dev zlib1g htop && \ + apt-get clean + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + xgboost transformer_engine flash_attn apex megatron-core grpcio + +RUN pip install --resume-retries 999 --no-cache-dir torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cu128 + +# Install flash-attn-2.8.0.post2 (cxx11abi=True) +RUN ABI_FLAG=$(python -c "import torch; print('TRUE' if torch._C._GLIBCXX_USE_CXX11_ABI else 'FALSE')") && \ + URL="https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.0.post2/flash_attn-2.8.0.post2+cu12torch2.7cxx11abi${ABI_FLAG}-cp312-cp312-linux_x86_64.whl" && \ + FILE="flash_attn-2.8.0.post2+cu12torch2.7cxx11abi${ABI_FLAG}-cp312-cp312-linux_x86_64.whl" && \ + wget -nv "${URL}" && \ + pip install --no-cache-dir "${FILE}" + +# Fix packages +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +# Install cudnn +RUN aria2c --max-tries=9999 https://developer.download.nvidia.com/compute/cudnn/9.8.0/local_installers/cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + dpkg -i cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb && \ + cp /var/cudnn-local-repo-ubuntu2204-9.8.0/cudnn-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + apt-get -y install cudnn-cuda-12 && \ + rm cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb + +# Install Apex +RUN pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" --resume-retries 999 git+https://github.com/NVIDIA/apex.git + +# Profiling tools +RUN aria2c --always-resume=true --max-tries=99999 https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2025_3/nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb && \ + apt-get update && apt-get install -y libxcb-cursor0 + +RUN apt-get install -y ./nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb && \ + rm -rf /usr/local/cuda/bin/nsys && \ + ln -s /opt/nvidia/nsight-systems/2025.3.1/target-linux-x64/nsys /usr/local/cuda/bin/nsys && \ + rm -rf /usr/local/cuda/bin/nsys-ui && \ + ln -s /opt/nvidia/nsight-systems/2025.3.1/target-linux-x64/nsys-ui /usr/local/cuda/bin/nsys-ui && \ + rm nsight-systems-2025.3.1_2025.3.1.90-1_amd64.deb + +RUN pip install --resume-retries 999 --no-cache-dir "tensordict==0.6.2" torchdata "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas cuda-bindings \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pre-commit ruff + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url + diff --git a/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/README.md b/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/README.md new file mode 100644 index 0000000000000000000000000000000000000000..07d68977f25c788132845c4051566eafbb902b18 --- /dev/null +++ b/e2c/verl/docker/verl0.5-preview-cu128-torch2.7.1-fa2.8.0/README.md @@ -0,0 +1,26 @@ +# verl image with verl v0.5 + +## Important packages version + +```txt +cuda==12.8 +cudnn==9.8.0 +torch==2.7.1 +flash_attn=2.8.0 ## +sglang==0.4.8 +transformer_engine==2.5 +megatron.core==core_r0.13.0 +vidia-cudnn-cu12==9.8.0.87 +``` + +## Target + +- Base image: + - `verlai/verl:base-verl0.5-preview-cu128-cudnn9.8-torch2.7.1-fa2.8.0`: We offer a base image with flash infer 0.2.6.post1 built in +- App image: + - `verlai/verl:app-verl0.5-preview-sglang0.4.8-mcore0.13.0-preview` +- vllm temporarily not support latest version + +## !!!Notice!!! + +- pyext is lack of maintainace and cannot work with python 3.12, consider using replacement and deprecating this package. \ No newline at end of file diff --git a/e2c/verl/docker/verl0.6-cu128-torch2.8.0-fa2.7.4/Dockerfile.app.sglang b/e2c/verl/docker/verl0.6-cu128-torch2.8.0-fa2.7.4/Dockerfile.app.sglang new file mode 100644 index 0000000000000000000000000000000000000000..23dbea72ccce3030beb76691ba1832fc04dcdc76 --- /dev/null +++ b/e2c/verl/docker/verl0.6-cu128-torch2.8.0-fa2.7.4/Dockerfile.app.sglang @@ -0,0 +1,4 @@ +FROM verlai/verl:base-verl0.6-cu128-cudnn9.8-torch2.8.0-fa2.7.4 + +RUN pip install --no-cache-dir "sglang[all]==0.5.2" +RUN pip install --no-cache-dir "torch-memory-saver==0.0.9rc1" diff --git a/e2c/verl/docker/verl0.6-cu128-torch2.8.0-fa2.7.4/Dockerfile.base b/e2c/verl/docker/verl0.6-cu128-torch2.8.0-fa2.7.4/Dockerfile.base new file mode 100644 index 0000000000000000000000000000000000000000..7bfd48169cc0ed666507d6560669e4c6f3511a11 --- /dev/null +++ b/e2c/verl/docker/verl0.6-cu128-torch2.8.0-fa2.7.4/Dockerfile.base @@ -0,0 +1,108 @@ +# Start from the NVIDIA official image (ubuntu-24.04 + cuda-12.8 + python-3.12) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-25-03.html +FROM nvcr.io/nvidia/pytorch:25.03-py3 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" +ENV PIP_CONSTRAINT="" + +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + pip config set global.no-cache-dir "true" && \ + python -m pip install --upgrade pip + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install libxml2 +RUN apt-get update && \ + apt-get install -y libxml2 aria2 && \ + apt-get clean + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + transformer_engine flash_attn apex megatron-core \ + xgboost opencv grpcio + +# Fix packages +RUN pip install --no-cache-dir tensordict torchdata "transformers[hf_xet]==4.55.4" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=19.0.1" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler blobfile xgrammar \ + pytest py-spy pre-commit ruff + +# Fix cv2 +RUN rm -rf /usr/local/lib/python3.11/dist-packages/cv2 + +# Install torch +RUN pip install --no-cache-dir torch==2.8.0 --index-url https://download.pytorch.org/whl/cu128 + +# Install flash-attn +RUN pip install --no-cache-dir --no-build-isolation flash_attn==2.7.4.post1 + +# Install DeepEP +# the dependency of IBGDA +RUN ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +# Clone and build deepep and deepep-nvshmem +RUN git clone -b v2.3.1 https://github.com/NVIDIA/gdrcopy.git && \ + git clone https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout a84a248 + +# Prepare nvshmem +RUN wget https://developer.nvidia.com/downloads/assets/secure/nvshmem/nvshmem_src_3.2.5-1.txz && \ + tar -xvf nvshmem_src_3.2.5-1.txz && mv nvshmem_src deepep-nvshmem && \ + cd deepep-nvshmem && git apply ../DeepEP/third-party/nvshmem.patch + +## Build deepep-nvshmem +RUN apt-get install -y ninja-build cmake + +ENV CUDA_HOME=/usr/local/cuda +### Set MPI environment variables. Having errors when not set. +ENV CPATH=/usr/local/mpi/include:$CPATH +ENV LD_LIBRARY_PATH=/usr/local/mpi/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV GDRCOPY_HOME=/workspace/gdrcopy +ENV GDRCOPY_INCLUDE=/workspace/gdrcopy/include + +RUN cd deepep-nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -G Ninja -S . -B build/ -DCMAKE_INSTALL_PREFIX=/workspace/deepep-nvshmem/install && cmake --build build/ --target install + +ENV NVSHMEM_DIR=/workspace/deepep-nvshmem/install +ENV LD_LIBRARY_PATH=$NVSHMEM_DIR/lib:$LD_LIBRARY_PATH +ENV PATH=$NVSHMEM_DIR/bin:$PATH + +## Build deepep +RUN cd DeepEP && \ + python setup.py install + +# Install Apex +RUN pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" git+https://github.com/NVIDIA/apex.git + +# Install TransformerEngine +RUN export NVTE_FRAMEWORK=pytorch && pip3 install --no-deps --no-cache-dir --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + +# Install Megatron-LM +RUN git clone -b core_v0.13.0 https://github.com/NVIDIA/Megatron-LM.git && \ + cd Megatron-LM && pip3 install --no-deps -e . + +# Install mbridge +RUN pip3 install --no-cache-dir git+https://github.com/ISEEKYAN/mbridge.git diff --git a/e2c/verl/docs/Makefile b/e2c/verl/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..8bda904a9b0b29dfcf538cb52b806dd910710a4a --- /dev/null +++ b/e2c/verl/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = verl +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/e2c/verl/docs/README.md b/e2c/verl/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8c5db04874138435ef986342a7b8be668b81d0b0 --- /dev/null +++ b/e2c/verl/docs/README.md @@ -0,0 +1,22 @@ +# verl documentations + +## Build the docs + +```bash +# If you want to view auto-generated API docstring, please make sure verl is available in python path. For instance, install verl via: +# pip install .. -e[test] + +# Install dependencies needed for building docs. +pip install -r requirements-docs.txt + +# Build the docs. +make clean +make html +``` + +## Open the docs with your browser + +```bash +python -m http.server -d _build/html/ +``` +Launch your browser and navigate to http://localhost:8000 to view the documentation. Alternatively you could drag the file `_build/html/index.html` to your local browser and view directly. diff --git a/e2c/verl/docs/README_vllm0.7.md b/e2c/verl/docs/README_vllm0.7.md new file mode 100644 index 0000000000000000000000000000000000000000..e84feddd7537b0cadb1157993a3819bfc5e52042 --- /dev/null +++ b/e2c/verl/docs/README_vllm0.7.md @@ -0,0 +1,73 @@ +# Upgrading to vllm >= 0.7 + +Note: verl+vllm 0.8.3 is now stable. Please see ``docs/README_vllm0.8.md`` for upgrade guide. + +## Installation + +Note: At time of writing, verl+vllm 0.7.x supports **FSDP** for training and **vLLM** for rollout. + +``` +# Create the conda environment +conda create -n verl python==3.10 +conda activate verl + +# Install verl +git clone https://github.com/volcengine/verl.git +cd verl +pip3 install -e . + +# Install the latest stable version of vLLM +pip3 install vllm==0.7.3 + +# Install flash-attn +pip3 install flash-attn --no-build-isolation + +``` + +Note that if you are installing lower versions of vLLM (0.7.0, 0.7.1, 0.7.2), you need to make some tiny patches manually on vllm (/path/to/site-packages/vllm after installation) after the above steps: + +- vllm/distributed/parallel_state.py: Remove the assertion below: + +``` +if (world_size + != tensor_model_parallel_size * pipeline_model_parallel_size): + raise RuntimeError( + f"world_size ({world_size}) is not equal to " + f"tensor_model_parallel_size ({tensor_model_parallel_size}) x " + f"pipeline_model_parallel_size ({pipeline_model_parallel_size})") + +``` + +- vllm/executor/uniproc_executor.py: change `local_rank = rank` to `local_rank = int(os.environ["LOCAL_RANK"])` +- vllm/model_executor/model_loader/weight_utils.py: remove the `torch.cuda.empty_cache()` in `pt_weights_iterator` + +## Features + +### Use cuda graph + +After installation, examples using FSDP as training backends can be used. By default, the `enforce_eager` is set to True, which disables the cuda graph. To enjoy cuda graphs and the sleep mode of vLLM>=0.7, add the following lines to the bash script: + +``` +actor_rollout_ref.rollout.enforce_eager=False \ +actor_rollout_ref.rollout.free_cache_engine=True \ + +``` + +For a typical job like examples/ppo_trainer/run_qwen2-7b_seq_balance.sh, the rollout generation time is 85 seconds with vLLM0.7.0. By enabling the cudagraph, the generation duration is further reduced to 62 seconds. + +**Note:** Currently, if the `n` is greater than 1 in `SamplingParams` in vLLM>=0.7, there is a potential performance issue on the stability of rollout generation time (Some iterations would see generation time bursts) using vLLM's V0 Engine. + +### Use vLLM V1 Engine + +Using the vLLM V1 engine can avoid instability issues and achieve additional performance improvements. To use the V1 engine, you can first uninstall the previously installed vLLM and then follow the steps below to install the newer version. + +``` +git clone https://github.com/vllm-project/vllm.git +cd vllm +git checkout 2275784 +sed -i "903a\ data_parallel_size = world_size // pipeline_model_parallel_size // tensor_model_parallel_size" ./vllm/distributed/parallel_state.py +VLLM_USE_PRECOMPILED=1 pip install --editable . +``` + +Then you can enable the V1 engine by setting `export VLLM_USE_V1=1`. In some benchmark tests, the V1 engine demonstrates a 1.5x speed improvement over the vLLM V0 engine. +The stable support of the vLLM V1 engine is available on verl main. diff --git a/e2c/verl/docs/README_vllm0.8.md b/e2c/verl/docs/README_vllm0.8.md new file mode 100644 index 0000000000000000000000000000000000000000..d4f509f19f780a4e8b3edec6bb256d2aa964639a --- /dev/null +++ b/e2c/verl/docs/README_vllm0.8.md @@ -0,0 +1,52 @@ +# Upgrading to vLLM >= 0.8 + +Last updated: 05/04/2025. + +## Installation + +Note: This version of verl+vLLM 0.8+ supports **FSDP** for training and **vLLM** for rollout. + +```bash +# Create the conda environment +conda create -n verl python==3.10 +conda activate verl + +# Install verl +git clone https://github.com/volcengine/verl.git +cd verl +pip3 install -e . + +# Install the latest stable version of vLLM +pip3 install vllm==0.8.3 + +# Install flash-attn +pip3 install flash-attn --no-build-isolation + +``` + +We have a pre-built docker image for verl+vLLM 0.8.3. You can direct import it with the following command: + +```bash +docker pull hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.3-flashinfer0.2.2-cxx11abi0 +``` + +## Features + +vLLM 0.8+ supports cuda graph and V1 engine by default in verl. To enable these features, remember to add the following lines to the bash script: + +```bash +actor_rollout_ref.rollout.enforce_eager=False \ +actor_rollout_ref.rollout.free_cache_engine=True \ +``` + +and also **remove** the environment variable if it exists: + +## Notes + +When you just directly upgrade vllm>=0.8, some dependency packages may undergo version changes. If you encounter the following problems: + +```bash +in from torch.multiprocessing.reductions import ForkingPickler ImportError: cannot import name 'ForkingPickler' from 'torch.multiprocessing.reductions' (/opt/conda/lib/python3.11/site-packages/torch/multiprocessing/reductions.py) +``` + +You need to upgrade `tensordict` to version 0.6.2 using the command `pip install tensordict==0.6.2`. diff --git a/e2c/verl/docs/_static/custom.css b/e2c/verl/docs/_static/custom.css new file mode 100644 index 0000000000000000000000000000000000000000..32f08475754bc280bca407d1643ec3aa68eeacf3 --- /dev/null +++ b/e2c/verl/docs/_static/custom.css @@ -0,0 +1,217 @@ +/* Make the documentation use full screen width */ +.wy-nav-content { + max-width: none !important; + width: 100% !important; + padding: 1.618em 3.236em !important; +} + +/* Adjust the content wrapper - will be set by JavaScript */ +.wy-nav-content-wrap { + margin-left: 300px; + transition: margin-left 0.2s ease; + width: auto !important; + position: relative !important; + background: white !important; + min-height: 100vh !important; +} + +/* Make the main content area responsive */ +.rst-content { + max-width: none !important; + width: 100% !important; +} + +/* Optional: Adjust table widths to prevent overflow */ +.rst-content table.docutils { + width: 100% !important; + table-layout: auto !important; +} + +/* Optional: Better code block width handling */ +.rst-content .highlight { + width: 100% !important; +} + +/* Content area positioning already handled above */ + +/* Optional: Improve readability with some margin on very wide screens */ +@media (min-width: 1400px) { + .wy-nav-content { + max-width: none !important; + margin: 0 auto !important; + } +} + +/* Resizable sidebar styles */ +.wy-nav-side { + position: fixed !important; + top: 0 !important; + bottom: 0 !important; + left: 0 !important; + width: 300px; + min-width: 200px; + max-width: 600px; + display: flex; + flex-direction: column; + z-index: 200 !important; +} + +/* Ensure sidebar header (logo, search) adapts to width */ +.wy-side-nav-search { + width: 100% !important; + box-sizing: border-box !important; + padding: 0.809em 0.809em !important; +} + +.wy-side-nav-search input[type="text"] { + width: 100% !important; + box-sizing: border-box !important; +} + +/* Make logo/title area responsive */ +.wy-side-nav-search > div.version { + width: 100% !important; +} + +.wy-side-nav-search > a { + width: 100% !important; + display: block !important; + white-space: nowrap !important; + overflow: hidden !important; + text-overflow: ellipsis !important; +} + +/* Responsive adjustments for narrow sidebar */ +@media (max-width: 300px) { + .wy-side-nav-search > a { + font-size: 0.9em !important; + } + + .wy-side-nav-search input[type="text"] { + font-size: 0.8em !important; + } +} + +/* Ensure search input doesn't overflow */ +.wy-side-nav-search form { + width: 100% !important; + margin: 0 !important; +} + +/* Make search icon responsive */ +.wy-side-nav-search .wy-dropdown { + width: 100% !important; +} + +/* Adjust search results dropdown width */ +.wy-side-nav-search .wy-dropdown-menu { + width: 100% !important; + max-width: none !important; + left: 0 !important; + right: 0 !important; +} + +/* Resize handle is created by JavaScript */ + +/* Make sure the sidebar content doesn't overflow */ +.wy-side-scroll { + width: 100% !important; + flex: 1 !important; + overflow-y: auto !important; + overflow-x: hidden !important; + padding-right: 10px !important; + box-sizing: border-box !important; + scroll-behavior: auto !important; /* Prevent smooth scrolling on sidebar itself */ +} + +/* Ensure proper scroll behavior for main content area */ +html { + scroll-behavior: smooth !important; +} + +/* Ensure anchor links work properly in main content */ +.wy-nav-content-wrap { + scroll-behavior: smooth !important; +} + +/* Fix scroll to target for anchor links */ +.rst-content { + scroll-behavior: smooth !important; +} + +/* Fix anchor scroll offset to account for fixed header */ +.rst-content .section { + scroll-margin-top: 60px; +} + +/* Fix anchor scroll offset for headers */ +.rst-content h1, .rst-content h2, .rst-content h3, .rst-content h4, .rst-content h5, .rst-content h6 { + scroll-margin-top: 60px; +} + +/* Fix anchor scroll offset for specific scroll targets */ +.rst-content .headerlink { + scroll-margin-top: 60px; +} + +/* Fix sidebar navigation styling */ +.wy-menu-vertical { + width: 100% !important; +} + +.wy-menu-vertical li { + width: 100% !important; +} + +.wy-menu-vertical a { + width: 100% !important; + word-wrap: break-word !important; + white-space: normal !important; +} + +/* Content area margin is handled by JavaScript */ + +/* Custom drag handle (more visible) */ +.resize-handle { + position: absolute; + top: 0; + right: 0; + width: 8px; + height: 100%; + background: #ccc; + cursor: col-resize; + z-index: 1001; + opacity: 0.3; + transition: opacity 0.2s ease; +} + +.resize-handle:hover { + opacity: 0.8; + background: #999; +} + +.resize-handle::before { + content: ''; + position: absolute; + top: 50%; + left: 50%; + width: 2px; + height: 20px; + background: #666; + transform: translate(-50%, -50%); + border-radius: 1px; +} + +.resize-handle:hover::before { + background: #333; +} + +/* Ensure smooth resizing */ +.wy-nav-side.resizing { + user-select: none; + pointer-events: none; +} + +.wy-nav-side.resizing .wy-side-scroll { + overflow: hidden; +} \ No newline at end of file diff --git a/e2c/verl/docs/_static/js/resizable-sidebar.js b/e2c/verl/docs/_static/js/resizable-sidebar.js new file mode 100644 index 0000000000000000000000000000000000000000..2a51fa90043bb0ecf78149b092fd3447740fdaee --- /dev/null +++ b/e2c/verl/docs/_static/js/resizable-sidebar.js @@ -0,0 +1,251 @@ +// Resizable sidebar functionality +document.addEventListener('DOMContentLoaded', function() { + const sidebar = document.querySelector('.wy-nav-side'); + const content = document.querySelector('.wy-nav-content-wrap'); + + if (!sidebar || !content) return; + + // Create resize handle + const resizeHandle = document.createElement('div'); + resizeHandle.className = 'resize-handle'; + sidebar.appendChild(resizeHandle); + + let isResizing = false; + let startX = 0; + let startWidth = 0; + + // Get initial width + const getInitialWidth = () => { + return 300; // Default width + }; + + // Save width to localStorage + const saveWidth = (width) => { + localStorage.setItem('sidebar-width', width); + }; + + // Load width from localStorage + const loadWidth = () => { + const savedWidth = localStorage.getItem('sidebar-width'); + if (savedWidth) { + const width = parseInt(savedWidth, 10); + if (width >= 200 && width <= 600) { + return width; + } + } + return getInitialWidth(); + }; + + // Apply width to sidebar and content + const applyWidth = (width) => { + // Update sidebar width + sidebar.style.width = width + 'px'; + + // Update content margin with !important to override any CSS + content.style.setProperty('margin-left', width + 'px', 'important'); + + // Also update any other content wrapper that might exist + const contentInner = document.querySelector('.wy-nav-content'); + if (contentInner) { + contentInner.style.setProperty('margin-left', '0px', 'important'); + } + + // Force reflow and repaint + sidebar.offsetHeight; + content.offsetHeight; + + // Trigger window resize event to notify other components + window.dispatchEvent(new Event('resize')); + }; + + // Initialize with saved width + const initialWidth = loadWidth(); + applyWidth(initialWidth); + + // Mouse down on resize handle + resizeHandle.addEventListener('mousedown', (e) => { + isResizing = true; + startX = e.clientX; + startWidth = parseInt(window.getComputedStyle(sidebar).width, 10); + + sidebar.classList.add('resizing'); + document.body.style.cursor = 'col-resize'; + document.body.style.userSelect = 'none'; + + // Add overlay to prevent iframe issues + const overlay = document.createElement('div'); + overlay.style.cssText = ` + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + z-index: 9999; + cursor: col-resize; + `; + overlay.id = 'resize-overlay'; + document.body.appendChild(overlay); + + e.preventDefault(); + }); + + // Mouse move + document.addEventListener('mousemove', (e) => { + if (!isResizing) return; + + const width = startWidth + e.clientX - startX; + const clampedWidth = Math.max(200, Math.min(600, width)); + applyWidth(clampedWidth); + }); + + // Mouse up + document.addEventListener('mouseup', () => { + if (!isResizing) return; + + isResizing = false; + sidebar.classList.remove('resizing'); + document.body.style.cursor = ''; + document.body.style.userSelect = ''; + + // Remove overlay + const overlay = document.getElementById('resize-overlay'); + if (overlay) { + overlay.remove(); + } + + // Save the current width + const currentWidth = parseInt(window.getComputedStyle(sidebar).width, 10); + saveWidth(currentWidth); + }); + + // Handle window resize - removed to prevent infinite loop + // The sidebar width is fixed and managed by drag functionality, no need to recalculate on window resize + + // Double-click to reset to default width + resizeHandle.addEventListener('dblclick', () => { + const defaultWidth = 300; + applyWidth(defaultWidth); + saveWidth(defaultWidth); + }); +}); + +// Fix navigation issues - Using MutationObserver for reliable initialization +document.addEventListener('DOMContentLoaded', function() { + let navigationFixed = false; + + function setupNavigationFix() { + if (navigationFixed) return; + + // Find all links in the sidebar + const sidebarLinks = document.querySelectorAll('.wy-menu-vertical a'); + + // Only proceed if we have sidebar links + if (sidebarLinks.length === 0) return; + + console.log('Setting up navigation fix...'); + + sidebarLinks.forEach(function(link) { + const href = link.getAttribute('href'); + + // Clone the link to remove all existing event listeners + const newLink = link.cloneNode(true); + + // Add our own click handler + newLink.addEventListener('click', function(e) { + console.log('Link clicked:', href); + + // If it's an anchor link within the same page + if (href && href.startsWith('#') && href !== '#') { + e.preventDefault(); + e.stopPropagation(); + + const targetId = href.substring(1); + const targetElement = document.getElementById(targetId); + + if (targetElement) { + // Calculate offset for fixed header + const headerHeight = 60; + const elementPosition = targetElement.getBoundingClientRect().top; + const offsetPosition = elementPosition + window.pageYOffset - headerHeight; + + window.scrollTo({ + top: offsetPosition, + behavior: 'smooth' + }); + + // Update URL hash + if (history.pushState) { + history.pushState(null, null, '#' + targetId); + } else { + location.hash = '#' + targetId; + } + } + } + // For external links, navigate normally + else if (href && !href.startsWith('#') && !href.startsWith('javascript:')) { + console.log('Navigating to external link:', href); + window.location.href = href; + } + }); + + // Replace the old link with the new one + link.parentNode.replaceChild(newLink, link); + }); + + navigationFixed = true; + + // Handle initial page load with hash + if (window.location.hash) { + // Use requestAnimationFrame for better timing + requestAnimationFrame(() => { + const targetId = window.location.hash.substring(1); + const targetElement = document.getElementById(targetId); + if (targetElement) { + const headerHeight = 60; + const elementPosition = targetElement.getBoundingClientRect().top; + const offsetPosition = elementPosition + window.pageYOffset - headerHeight; + + window.scrollTo({ + top: offsetPosition, + behavior: 'smooth' + }); + } + }); + } + } + + // Try to set up navigation fix immediately + setupNavigationFix(); + + // If it didn't work, use MutationObserver to watch for when sidebar links are added + if (!navigationFixed) { + const observer = new MutationObserver(function(mutations) { + mutations.forEach(function(mutation) { + if (mutation.type === 'childList' && mutation.addedNodes.length > 0) { + // Check if sidebar links were added + const sidebarLinks = document.querySelectorAll('.wy-menu-vertical a'); + if (sidebarLinks.length > 0) { + setupNavigationFix(); + if (navigationFixed) { + observer.disconnect(); + } + } + } + }); + }); + + // Start observing the document for changes + observer.observe(document.body, { + childList: true, + subtree: true + }); + + // Fallback timeout in case MutationObserver doesn't work + setTimeout(function() { + if (!navigationFixed) { + setupNavigationFix(); + } + observer.disconnect(); + }, 5000); + } +}); \ No newline at end of file diff --git a/e2c/verl/docs/_static/js/runllm-widget.js b/e2c/verl/docs/_static/js/runllm-widget.js new file mode 100644 index 0000000000000000000000000000000000000000..bec345cacc5b943693e1bf1973a7a6d863b0d85e --- /dev/null +++ b/e2c/verl/docs/_static/js/runllm-widget.js @@ -0,0 +1,14 @@ +document.addEventListener("DOMContentLoaded", function () { + var script = document.createElement("script"); + script.type = "module"; + script.id = "runllm-widget-script"; + script.src = "https://widget.runllm.com"; + script.setAttribute("version", "stable"); + script.setAttribute("crossorigin", "true"); + script.setAttribute("runllm-keyboard-shortcut", "Mod+j"); + script.setAttribute("runllm-name", "verl Chatbot"); + script.setAttribute("runllm-position", "TOP_RIGHT"); + script.setAttribute("runllm-assistant-id", "679"); + script.async = true; + document.head.appendChild(script); + }); \ No newline at end of file diff --git a/e2c/verl/docs/_static/logo.png b/e2c/verl/docs/_static/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..6a3b61308d73eb72071eaec8c95544ab36cd3970 Binary files /dev/null and b/e2c/verl/docs/_static/logo.png differ diff --git a/e2c/verl/docs/advance/agent_loop.rst b/e2c/verl/docs/advance/agent_loop.rst new file mode 100644 index 0000000000000000000000000000000000000000..cb07c62f5736e0ccf95c901cf95cbd2106f0c74a --- /dev/null +++ b/e2c/verl/docs/advance/agent_loop.rst @@ -0,0 +1,238 @@ +Agent Loop +========== + +Last updated: 07/17/2025. + +.. versionadded:: 0.4.2 + [status: alpha] + +.. warning:: + Agent Loop is ready for use, but the API may change in future releaes. + +Agent Loop is designed as general interface for multi-turn rollout and agentic reinforcement learning. + +**Design goal**: + +- Plugable user defined agent loop +- Provide standard request generate api with different inference frameworks +- Provide request level load balance between multiple inference servers + +**Non-goal**: + +- How tool is defined and how to call tool + +In high level overview, agent loop is given a prompt, run user defined loop: call LLM generate api, call tools, ... +and return the final output. The final output is then calculated reward and used as trajectory for RL training. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/agent_loop_overview.svg?raw=true + + +API Design +---------- + +``AgentLoopBase`` class is the abstraction of agent loop, and ``run`` method is the only interface that user need to implement. +The run method, given prompt messages in format: [{"role": "user"}, {"content": "..."}], and additional sampling params, +could do whatever user wants, such as + +- call LLM generate api +- call tools: web search, database query, code sandbox, ... +- environment interaction +- reflection +- ... + +.. code:: python + + class AgentLoopBase(ABC): + @abstractmethod + async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput: + """Run agent loop to interact with LLM server and environment. + + Args: + sampling_params (Dict[str, Any]): LLM sampling params. + **kwargs: dataset fields from `verl.utils.dataset.RLHFDataset`. + + Returns: + AgentLoopOutput: Agent loop output. + """ + raise NotImplementedError + +After running user defined loop, run method should return ``AgentLoopOutput``, including prompt token ids, +response token ids, and response mask. + +.. code:: python + + class AgentLoopOutput(BaseModel): + """Agent loop output.""" + + prompt_ids: list[int] + """Prompt token ids.""" + response_ids: list[int] + """Response token ids including LLM generated token, tool response token.""" + response_mask: list[int] + """Response mask, 1 for LLM generated token, 0 for tool response token.""" + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/agent_loop_output.svg?raw=true + +.. note:: AgentLoopOutput only output one trajectory for a given prompt, multiple trajectories output is still under discussion. + +Architecture Design +------------------- + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/agent_loop_architecture.png?raw=true + +A single PPO step contain two phase: rollout and train. In rollout phase: + +1. PPOTrainer sample a batch from dataset and call ``AgentLoopManager.generate_sequences``. +2. AgentLoopManager ``wake_up`` all async LLM server instances, which will sync weights between inference engine(vLLM/SGLang) and training engine(FSDP/Megatron-LM). +3. AgentLoopManager split batch into chunks and send each chunk to ``AgentLoopWorker``. +4. AgentLoopWorker receive chunk and for each prompt, spawn a user defined ``AgentLoopBase`` instance, run ``run`` coroutine until end and get ``AgentLoopOutput``. + +.. tip:: + AgentLoopWorker schedules multiple coroutines concurrently. If number of AgentLoopWorker equals batch_size, then each worker is response for one prompt. + +In agent loop, when user need LLM generate response: + +5. Call ``AsyncLLMServerManager.generate`` with prompt_ids. +6. AsyncLLMServerManager select a server instance with least request in first turn and send request to it. (In following turns, the request will be sent to the same server instance). +7. AsyncLLMServer receive a request, issue ipc/rpc with model_runner, and generate response. (There's slight differences between vLLM and SGLang, see below). + +When all prompts in all AgentLoopWorker finish, AgentLoopManager gather results and return to PPOTrainer. + +8. AgentLoopManager ``sleep`` all server instances, which will free kv cache and offload weights to CPU memory. + +AsyncLLMServer +~~~~~~~~~~~~~~ + +AsyncLLMServer is the abstraction of LLM server with two types of generation api: + +- `OpenAI chat completion `_: generate response for the given chat conversation. +- Token in token out: generate response ids for the given token ids. + +We have officially supported vLLM and SGLang AsyncLLMServer, both of them implement the two api and are well tested. +Other inference engine should be easy to plug-in by implement the ``AsyncServerBase`` class. + +.. code:: python + + class AsyncServerBase(ABC): + @abstractmethod + async def chat_completion(self, raw_request: Request) -> JSONResponse: + """OpenAI chat completion API. + + Args: + raw_request (Request): raw json request + + Returns: + JSONResponse: json response + + API reference: https://platform.openai.com/docs/api-reference/chat/create + """ + raise NotImplementedError + + @abstractmethod + async def generate(self, prompt_ids: list[int], sampling_params: dict[str, Any], request_id: str) -> list[int]: + """Generate response ids given prompt ids. + + Args: + prompt_ids (List[int]): prompt ids + sampling_params (Dict[str, Any]): sampling params + request_id (str): request id + + Returns: + List[int]: response ids + """ + raise NotImplementedError + + +Chat completion vs Token in token out +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + The following conclusion is based on our recent experience and is still open to investigation and discussion. + +Almost all agent frameworks (LangGraph, CrewAI, LlamaIndex, etc) call LLM with OpenAI chat completion api, and +keep chat history as messages. So user may expect that we should use the chat completion api in multi-turn rollout. + +But based on our recent experience on single-turn training on DAPO and multi-turn training on `retool `_, +we found the token_ids from apply the final messages may not equal to the token_ids by concat prompt_ids and response_ids in each turn. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/multi_turn.png?raw=true + +**Where does this inconsistency happened?** + +First, the tool parser may alter the content. For example + +.. code:: json + + {"role": "assistant", "content": "Let me call a ... and get the result"} + +After tool_calls extraction, the messages is like this: + +.. code:: json + + {"role": "assistant", "content": "Let me call a and get the result", "tool_calls": [{"name": "foo", "arguments": "{}"}]} + +Encode the extracted message back is not equal to the original LLM generated response_ids. + +Second, the `decode-encode` may also lead to inconsistency: `Agent-R1 issue#30 `_. + +**What is the impact of this inconsistency?** + +This inconsistency is not a big problem for serving/agent system, but is critical to RL training. +It causes the trajectory deviate from the policy model distribution. We have observed that apply_chat_template +to the final chat history messages make PPO training not even converged in single-turn. + +vLLM +^^^^ + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/async_vllm.png?raw=true + +For vLLM, the Async LLM Engine is running in same process as the server, and ModelRunner is running in same process as FSDP/Megatron-LM workers. +Async LLM Engine communicate with ModelRunner through ZeroMQ. When server receive a request, it directly call engine to generate response_ids. + +SGLang +^^^^^^ + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/async_sglang.png?raw=true + +For SGLang, the Async LLM Engine is running in same process as FSDP/Megatron-LM worker-0, and it spawn multiple subprocesses as ModelRunner. +Also, Async LLM Engine communicate with ModelRunner through ZeroMQ. When server receive a request, it remote call the worker-0 and get response_ids. + +AsyncLLMServerManager +~~~~~~~~~~~~~~~~~~~~~ + +AsyncLLMServerManager serve as proxy to multiple AsyncLLMServer instances, provides: + +- load balance: select a server instance with least request in first turn and send request to it. +- sticky session: bind request_id to server instance, so that the same request_id will be sent to the same server instance in following turns. + +AsyncLLMServerManager is passed to ``AgentLoopBase.__init__``, whenever user want to interact with LLM in agent loop, +they can call ``AsyncLLMServerManager.generate`` to generate response_ids. + +.. code:: python + + class AsyncLLMServerManager: + async def generate( + self, + request_id, + *, + prompt_ids: list[int], + sampling_params: dict[str, Any], + ) -> list[int]: + """Generate tokens from prompt ids. + + Args: + request_id (str): request id for sticky session. + prompt_ids (List[int]): List of prompt token ids. + sampling_params (Dict[str, Any]): Sampling parameters for the chat completion. + + Returns: + List[int]: List of generated token ids. + """ + ... + +Next +---- + +- :doc:`Agentic RL Training<../start/agentic_rl>`: Quick start agentic RL training with gsm8k dataset. +- `LangGraph MathExpression `_: Demonstrate how to use LangGraph to build agent loop. +- `Retool `_: End-to-end retool paper reproduction using tool agent. diff --git a/e2c/verl/docs/advance/checkpoint.rst b/e2c/verl/docs/advance/checkpoint.rst new file mode 100644 index 0000000000000000000000000000000000000000..56bec4a75c3192802e553faca8cb2246ad14f35d --- /dev/null +++ b/e2c/verl/docs/advance/checkpoint.rst @@ -0,0 +1,183 @@ +.. _checkpoint-page: + +Using Checkpoints to Support Fault Tolerance Training +===================================================== + +Last updated: 06/25/2025. + +There could be training errors or machine failure during the whole RLHF training process, +so it is recommended to enable checkpoints to minimize your loss. + +The API Interface has already been listed in :ref:`config-explain-page`, +and we will not repeat them. But there are still some technique details +we hope to clarify. + +.. note:: + + Notice that the ``checkpoint.contents`` field has no effect to FSDP checkpoint except ``hf_model``, + the other 3 fields are binded together to save and load. We recommend to include ``model``, ``optimizer`` and ``extra`` all. + +Checkpoint Saving Directory Structure +------------------------------------- + +Commonly, we use the ``default_local_dir`` declared in ``ppo_trainer.yaml`` or ``ppo_megatron_trainer.yml`` +to work as preffix when saving checkpoints, which is ``checkpoints/${trainer.project_name}/${trainer.experiment_name}``. + +So the inner checkpoint structure of **FSDP** is like: + +.. code:: + + checkpoints/${trainer.project_name}/${trainer.experiment_name} + ├── global_steps_${i} + │ ├── actor + │ │ ├── huggingface # default save config and tokenizer, save huggingface model if include ``hf_model`` in checkpoint.contents + │ │ └── fsdp_config.json # FSDP config file, including world_size and fsdp version + │ │ ├── model_world_size_{self.world_size}_rank_{self.rank}.pt + │ │ ├── optim_world_size_{self.world_size}_rank_{self.rank}.pt + │ │ └── extra_state_world_size_{self.world_size}_rank_{self.rank}.pt + │ ├── critic + │ │ ├── huggingface + │ │ └── fsdp_config.json + │ │ ├── model_world_size_{self.world_size}_rank_{self.rank}.pt + │ │ ├── optim_world_size_{self.world_size}_rank_{self.rank}.pt + │ │ └── extra_state_world_size_{self.world_size}_rank_{self.rank}.pt + └── latest_checkpointed_iteration.txt + +All model shards, optimizers and extra states are stored together, in a sharded and distributed way. + +While **Megatron** current checkpoint structure is: + +.. code:: + + checkpoints/${trainer.project_name}/${trainer.experiment_name} + ├── global_steps_${i} + │ ├── actor + │ │ ├── huggingface # default save config and tokenizer, save huggingface model if include ``hf_mode`` in checkpoint.contents + │ │ └── dist_ckpt # save sharded model/optimizer/rng_states, naming the same as Megatron + │ └── critic + │ │ ├── huggingface + │ │ └── dist_ckpt + └── latest_checkpointed_iteration.txt + +Convert FSDP and Megatron Checkpoints to HuggingFace Format Model +----------------------------------------------------------------- + +We provide a tool to convert the FSDP and Megatron checkpoints to HuggingFace format model. +The tool is located in ``verl/model_merger``. For older versions of verl that don't include fsdp_config.json in checkpoints, you can use the legacy model merger located at ``verl/scripts/legacy_model_merger.py``. + +The script supports two main sub-commands: `merge` (to convert and save checkpoints) and `test` (to validate merged checkpoints against a reference model). +The arguments for the `merge` sub-command are as follows: + +.. code:: bash + + usage: python -m verl.model_merger merge [-h] --backend {fsdp,megatron} [--local_dir LOCAL_DIR] [--tie-word-embedding] [--is-value-model] [--use_cpu_initialization] [--target_dir TARGET_DIR] + [--hf_upload_path HF_UPLOAD_PATH] [--private] + + options: + -h, --help show this help message and exit + --backend {fsdp,megatron} + The backend of the model + --local_dir LOCAL_DIR + Path to the saved model checkpoints + --tie-word-embedding Whether to tie word embedding weights (currently only Megatron supported) + --is-value-model Whether the model is a value model (currently only Megatron supported) + --use_cpu_initialization + Whether to use CPU initialization for the model. This is useful for large models that cannot fit into GPU memory during initialization. + --target_dir TARGET_DIR + Directory to save the merged huggingface model + --hf_upload_path HF_UPLOAD_PATH + Hugging Face repository ID to upload the model + --private Whether to upload the model to a private Hugging Face repository + +Example usage for merging Megatron checkpoints: + +.. code:: bash + + python -m verl.model_merger merge \ + --backend megatron \ + --tie-word-embedding \ + --local_dir checkpoints/verl_megatron_gsm8k_examples/qwen2_5_0b5_megatron_saveload/global_step_1/actor \ + --target_dir /path/to/merged_hf_model + +Example usage for distributed merging Megatron checkpoints: + +.. code:: bash + + torchrun --nproc_per_node 1 --nnodes 8 --node_rank ${RANK} -m verl.model_merger merge \ + --backend megatron \ + --tie-word-embedding \ + --local_dir checkpoints/verl_megatron_gsm8k_examples/qwen2_5_0b5_megatron_saveload/global_step_1/actor \ + --target_dir /path/to/merged_hf_model + +Example usage for merging FSDP checkpoints: + +.. code:: bash + + python -m verl.model_merger merge \ + --backend fsdp \ + --local_dir checkpoints/verl_fsdp_gsm8k_examples/qwen2_5_0b5_fsdp_saveload/global_step_1/actor \ + --target_dir /path/to/merged_hf_model + + +Megatron Merger details +----------------------- + +Current implement of decoder layers uses ``nn.ModuleList`` to store the layers, +and thus the model layers on every PP rank and VPP rank starts their index from 0. + +There are 3 ways to correct this behavior: + +1. Modify the decoder layer's state_dict, add ``offset`` to each layer's index, thus rewrite ``nn.ModuleList`` implementation. +2. Modify the layer index when saving checkpoint and recover them when loading checkpoint. +3. The Checkpoint merger do this work, calculate the actual ``offset`` from ``state_dict`` only, a little complex. + +Current implementation use solution 2. + + +HuggingFace to Megatron DistCheckpoint details +---------------------------------------------- + +If your model is quite huge, we recommend you to use Megatron dist-checkpoint to load the model. +Megatron dist-checkpoint supports loading with different kinds of model parallelism, +and it is much faster than the original checkpoint loading. + +To convert original HuggingFace model to Megatron dist-checkpoint, +you can use the ``scripts/converter_hf_to_mcore.py`` script. Large MoE models are temporarily supported with CPU initialization, +which is a little slower. While we are working on a better solution to support large models. + +Example command to convert the model is as follows: + +.. code:: bash + + python scripts/converter_hf_to_mcore.py \ + --hf_model_path Qwen/Qwen1.5-MoE-A2.7B-Chat \ + --output_path /mnt/disk/Qwen/Qwen1.5-MoE-A2.7B-Chat \ + --use_cpu_initialization # Only work for MoE models + + +Example command to distributed convert the huge model like deepseekv3 671B is as follows: + +.. code:: bash + + torchrun --nproc_per_node 1 --nnodes 8 --node_rank ${RANK} scripts/converter_hf_to_mcore.py \ + --hf_model_path deepseek-ai/DeepSeek-V3 \ + --output_path /mnt/disk/deepseek-ai/DeepSeek-V3 \ + --use_cpu_initialization # Only work for MoE models + +Original Checkpoint Utils +------------------------- + +Original Checkpoint Utils refer to original checkpoint implementation in ``verl/models/[model]/megatron/checkpoint_utils``. + +We only need ``[model]_loader.py`` in original checkpoint utils now, since we get rid of storing ``hf_model`` every time (which is not recommended for large model training, try only saving sharded models if you can). + +.. note:: + + Note that ``[model]_loader`` only support environments where **storage clusters are able to connect with every calculation nodes**. + Because it utilizes **sharded load way to minimize the loading checkpoint overhead**. + Every rank loads its own data from ``state_dict`` which can be accessed by all of them. + While there is also no need to broadcast among DP ranks, since the saved state_dict is only produced by DP rank 0. + + For users who can **only place the huggingface model on one device**, we keep the original costly implementation in ``[model]_loader_deprecated``. In this implementation, rank 0 broadcast all weights to each tp and pp rank, and then dp rank 0 broadcast to all dp ranks. There may be at risks of OOM. + + To use deprecated loader, change the import package of ``load_state_dict_to_megatron_llama``. diff --git a/e2c/verl/docs/advance/dpo_extension.rst b/e2c/verl/docs/advance/dpo_extension.rst new file mode 100644 index 0000000000000000000000000000000000000000..ee9ac619dde1ebfe3390d0b409b92252cb4e4104 --- /dev/null +++ b/e2c/verl/docs/advance/dpo_extension.rst @@ -0,0 +1,273 @@ +Extend to other RL(HF) algorithms +================================= + +Last updated: 02/25/2025. + +We already implemented the complete training pipeline of the PPO +algorithms. To extend to other algorithms, we analyze the high-level +principle to use verl and provide a tutorial to implement the DPO +algorithm. Users can follow the similar paradigm to extend to other RL algorithms. + +.. note:: **Key ideas**: Single process drives multi-process computation and data communication. + +Overall Approach +---------------- + +Step 1: Consider what multi-machine multi-GPU computations are needed +for each model, such as ``generate_sequence`` , ``compute_log_prob`` and +``update_policy`` in the actor_rollout model. Implement distributed +single-process-multiple-data (SPMD) computation and encapsulate them +into APIs + +Step 2: Based on different distributed scenarios, including FSDP and 3D +parallelism in Megatron-LM, implement single-process control of data +interaction among multi-process computations. + +Step 3: Utilize the encapsulated APIs to implement the control flow + +Example: Online DPO +------------------- + +We use verl to implement a simple online DPO algorithm. The algorithm +flow of Online DPO is as follows: + +1. There is a prompt (rollout) generator which has the same weight as + the actor model. After a batch of prompts are fed into the generator, + it generates N responses for each prompt. +2. Send all the prompts + responses to a verifier for scoring, which can + be reward model or a rule-based function. Then sort them in pairs to + form a training batch. +3. Use this training batch to train the actor model using DPO. During + the process, a reference policy is needed. + +Step 1: What are the multi-machine multi-GPU computations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Sample Generator** + +Implementation details: + +.. code:: python + + from verl.single_controller.base import Worker + from verl.single_controller.ray import RayWorkerGroup, RayClassWithInitArgs, RayResourcePool + import ray + + @ray.remote + class SampleGenerator(Worker): + def __init__(self, config): + super().__init__() + self.config = config + + def generate_sequences(self, data): + pass + +Here, ``SampleGenerator`` can be viewed as a multi-process pulled up by +``torchrun``, with each process running the same code (SPMD). +``SampleGenerator`` needs to implement a ``generate_sequences`` API for +the control flow to call. The implementation details inside can use any +inference engine including vllm, sglang and huggingface. Users can +largely reuse the code in +verl/verl/workers/rollout/vllm_rollout/vllm_rollout.py and we won't +go into details here. + +**ReferencePolicy inference** + +API: compute reference log probability + +.. code:: python + + from verl.single_controller.base import Worker + import ray + + @ray.remote + class ReferencePolicy(Worker): + def __init__(self): + super().__init__() + self.model = Model() + + def infer(self, data): + return self.model(data) + +**Actor update** + +API: Update actor model parameters + +.. code:: python + + from verl.single_controller.base import Worker + import ray + + @ray.remote + class DPOActor(Worker): + def __init__(self): + super().__init__() + self.model = Model() + self.model = FSDP(self.model) # or other distributed strategy + self.optimizer = optim.Adam(self.model.parameters(), lr=1e-3) + self.loss_fn = xxx + + def update(self, data): + self.optimizer.zero_grad() + logits = self.model(data) + loss = self.loss_fn(logits) + loss.backward() + self.optimizer.step() + +**Notes: How to distinguish between control processes and distributed computation processes** +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Control processes are generally functions directly decorated with + ``@ray.remote`` +- Computation processes are all wrapped into a ``RayWorkerGroup``. + +Users can reuse most of the distribtued computation logics implemented +in PPO algorithm, including FSDP and Megatron-LM backend in +verl/verl/trainer/ppo. + +Step 2: Based on different distributed scenarios, implement single-process control of multi-process data interaction +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**The core problem to solve here is how a single process sends data to +multiple processes, drives multi-process computation, and how the +control process obtains the results of multi-process computation.** +First, we initialize the multi-process ``WorkerGroup`` in the control +process. + +.. code:: python + + @ray.remote(num_cpus=1) + def main_task(config): + # construct SampleGenerator + resource_pool = RayResourcePool(process_on_nodes=[8] * 2) # 16 GPUs + ray_cls = RayClassWithInitArgs(SampleGenerator, config=config) + # put SampleGenerator onto resource pool + worker_group = RayWorkerGroup(resource_pool, ray_cls) + + # construct reference policy + +As we can see, in the control process, multiple processes are wrapped +into a ``RayWorkerGroup``. Inside this ``WorkerGroup``, there is a +``self._workers`` member, where each worker is a RayActor +(https://docs.ray.io/en/latest/ray-core/actors.html) of SampleGenerator. +ray_trainer.md also provide an implementation of +``MegatronRayWorkerGroup``. + +Assuming the model is distributed using FSDP, and there is a batch of +data on the control process, for data parallelism, the underlying +calling process is: + +.. code:: python + + data = xxx + data_list = data.chunk(dp_size) + + output = [] + for d in data_list: + # worker_group._workers[i] is a SampleGenerator + output.append(worker_group._workers[i].generate_sequences.remote(d)) + + output = ray.get(output) + output = torch.cat(output) + +Single process calling multiple processes involves the following 3 +steps: + +1. Split the data into DP parts on the control process. +2. Send the data to remote, call the remote computation through RPC, and + utilize multi-process computation. +3. Obtain the computation results of each worker on the control process + and merge them. + +Frequently calling these 3 steps on the controller process greatly hurts +code readability. **In verl, we have abstracted and encapsulated these 3 +steps, so that the worker's method + dispatch + collect can be +registered into the worker_group** + +.. code:: python + + from verl.single_controller.base.decorator import register + + def dispatch_data(worker_group, data): + return data.chunk(worker_group.world_size) + + def collect_data(worker_group, data): + return torch.cat(data) + + dispatch_mode = { + 'dispatch_fn': dispatch_data, + 'collect_fn': collect_data + } + + @register(dispatch_mode=dispatch_mode) + def generate_sequences(self, data): + pass + +In this way, we can directly call the method inside the worker through +the ``worker_group`` on the control (driver) process (which is a single +process): + +.. code:: python + + output = worker_group.generate_sequences(data) + +This single line includes data splitting, data distribution and +computation, and data collection. + +Furthermore, the model parallelism size of each model is usually fixed, +including dp, tp, pp. So for these common distributed scenarios, we have +pre-implemented specific dispatch and collect methods,in `decorator.py `_, which can be directly used to wrap the computations. + +.. code:: python + + from verl.single_controller.base.decorator import register, Dispatch + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def generate_sequences(self, data: DataProto) -> DataProto: + pass + +Here it requires the data interface to be ``DataProto``. Definition of +``DataProto`` is in `protocol.py `_. + +Step 3: Main training loop +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +With the above training flows, we can implement the algorithm's control +flow. It is recommended that ``main_task`` is also a ray remote process. + +.. code:: python + + @ray.remote(num_cpus=1) + def main_task(config): + # construct SampleGenerator + resource_pool = RayResourcePool(process_on_nodes=[8] * 2) # 16 GPUs + ray_cls = RayClassWithInitArgs(SampleGenerator, config=config) + # put SampleGenerator onto resource pool + sample_gen = RayWorkerGroup(resource_pool, ray_cls) + + # construct reference policy + ray_cls = RayClassWithInitArgs(ReferencePolicy) + ref_policy = RayWorkerGroup(resource_pool, ray_cls) + + # construct actor + ray_cls = RayClassWithInitArgs(DPOActor) + dpo_policy = RayWorkerGroup(resource_pool, ray_cls) + + dataloader = DataLoader() + + for data in dataloader: + # generate data + data = sample_gen.generate_sequences(data) + # generate scores for each data + data = generate_scores(data) + # generate pairwise data using scores + data = generate_pairwise_data(data) + # generate ref_log_prob + data.batch['ref_log_prob'] = ref_policy.infer(data) + # update using dpo + dpo_policy.update(data) + # logging + +Here, different ``WorkerGroups`` can be placed in the same resource pool or +in different resource pools using ``create_colocated_worker_cls`` +similar as in `ray_trainer.py `_. diff --git a/e2c/verl/docs/advance/fsdp_extension.rst b/e2c/verl/docs/advance/fsdp_extension.rst new file mode 100644 index 0000000000000000000000000000000000000000..181e109082262f26334034337c5915d522049759 --- /dev/null +++ b/e2c/verl/docs/advance/fsdp_extension.rst @@ -0,0 +1,97 @@ + +Add models with the FSDP backend +================================== + +Last updated: 02/09/2025. + +Model +-------------------------- + +In principle, our FSDP backend can support any HF model and we can +sychronoize the actor model weight with vLLM using `hf_weight_loader.py` under `third_party/vllm`. +However, ``hf_weight_loader`` is will gather the full state_dict of a +model during synchronization, which may cause OOM. We suggest using +``dtensor_weight_loader`` which gather the full model parameter layer by +layer to reduce the peak memory usage. We already support dtensor weight +loader for the models below in `dtensor_weight_loader.py` under `third_party/vllm`: + +- ``GPT2LMHeadModel`` +- ``LlamaForCausalLM`` +- ``LLaMAForCausalLM`` +- ``MistralForCausalLM`` +- ``InternLMForCausalLM`` +- ``AquilaModel`` +- ``AquilaForCausalLM`` +- ``Phi3ForCausalLM`` +- ``GemmaForCausalLM`` +- ``Gemma2ForCausalLM`` +- ``GPTBigCodeForCausalLM`` +- ``Starcoder2ForCausalLM`` +- ``Qwen2ForCausalLM`` +- ``DeepseekV2ForCausalLM`` + +To implement ``dtensor_weight_loader`` of a model that's supported in +vLLM, follow the guide of gemma model below: + +1. Copy the + ``load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]])`` from the vllm model class + to ``dtensor_weight_loaders.py`` +2. Modify the arguments to + ``(actor_weights: Dict, vllm_model: nn.Module)`` +3. Replace the ``self`` to ``vllm_model`` +4. Add the + ``local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight)`` + before each ``param = params_dict[name]`` and modify the following + weight loading using ``local_loaded_weight``. +5. Register the implemented dtensor weight loader to ``__MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__``. + +.. code-block:: diff + + - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + + def gemma_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + - params_dict = dict(self.named_parameters()) + + params_dict = dict(vllm_model.named_parameters()) + loaded_params = set() + - for name, loaded_weight in weights: + + for name, loaded_weight in actor_weights.items(): + for (param_name, shard_name, shard_id) in stacked_params_mapping: + if shard_name not in name: + continue + name = name.replace(shard_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) + param = params_dict[name] + weight_loader = param.weight_loader + - weight_loader(param, loaded_weight, shard_id) + + weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) + break + else: + # lm_head is not used in vllm as it is tied with embed_token. + # To prevent errors, skip loading lm_head.weight. + if "lm_head.weight" in name: + continue + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + - weight_loader(param, loaded_weight) + + weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) + loaded_params.add(name) + unloaded_params = params_dict.keys() - loaded_params + if unloaded_params: + raise RuntimeError( + "Some weights are not initialized from checkpoints: " + f"{unloaded_params}") \ No newline at end of file diff --git a/e2c/verl/docs/advance/megatron_extension.rst b/e2c/verl/docs/advance/megatron_extension.rst new file mode 100644 index 0000000000000000000000000000000000000000..9a52e6017b7adc77b404398501587aff0e045129 --- /dev/null +++ b/e2c/verl/docs/advance/megatron_extension.rst @@ -0,0 +1,20 @@ +Add models with the Megatron-LM backend +========================================= + +Last updated: 04/25/2025. + +Model +----------- + + +If use latest verl, we have direct support of ``GPTModel`` for Megatron backend. +You can use the similar way of using Megatron to pretrain custom models. +We list the steps here: + +1. Find `model_initializer.py `_ +2. If your model is configurable by ``TransformerLayerSpec`` , you can + directly use ``GPTModel``. Otherwise, Please implement a new + ``ModelLayerSpec`` and ``ModelLayer`` here. +3. Use the right ``LayerSpec`` , ``TransformerConfig`` and ``HuggingfaceConfig`` + as arguments to initialize the GPTModel. +4. Return the model at last. diff --git a/e2c/verl/docs/advance/one_step_off.md b/e2c/verl/docs/advance/one_step_off.md new file mode 100644 index 0000000000000000000000000000000000000000..31cf5246783fac6f56e7b30140e53a5cf1fb81de --- /dev/null +++ b/e2c/verl/docs/advance/one_step_off.md @@ -0,0 +1,308 @@ +# Recipe: One Step Off Policy Async Trainer + +**Author:** `https://github.com/meituan-search` + +Last updated: 07/17/2025. + +## Introduction + +### Background + +The current reinforcement learning training process implemented by verl is synchronous, adhering to the algorithmic +workflows of established methods like PPO, GRPO, and DAPO. In each step, training samples are generated by the latest +model, and the model is updated after training completes. While this approach aligns with off-policy reinforcement +learning and stabilizes RL training, but it suffers from severe efficiency issues. +Model updates must wait for the longest output in the generation phase to complete. +During the generation of long-tail samples, GPUs remain idle, resulting in significant underutilization. +The more severe the long-tail problem in sample generation, the lower the overall training efficiency. +For example, in DAPO 32B training, the Rollout phase accounts for approximately 70% of the total time, +and increasing resources does not reduce the Rollout duration. + +![DAPO 32B Math Performance]( +https://raw.githubusercontent.com/eric-haibin-lin/verl-community/refs/heads/main/docs/dapo_32b_math.png) +> source data: https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=nwusertongyuxuan361 + +### Solution + +We have implemented the **One Step Off Async Trainer** to help alleviate this issue. This approach parallelizes the +generation and training processes, utilizing samples generated in the previous step for current training. +It also involves appropriately partitioning resources, allocating dedicated resources for generation while automatically +assigning the remainder to training. By reducing resources allocated to the generation phase, we mitigate GPU idle time +during long-tail sample generation. Throughout this process, generation and training parameters maintain a one-step off +policy. + +![One Step Off Policy Diagram]( +https://raw.githubusercontent.com/eric-haibin-lin/verl-community/refs/heads/main/docs/one_step_off_policy.png) +> reference: [AReaL: A Large-Scale Asynchronous Reinforcement Learning System for Language Reasoning]( +> https://arxiv.org/abs/2505.24298) + +Our core contributions include: + +1. **Parallel Generation and Training**: + Samples for the next batch are asynchronously generated while the current batch is being trained. + +2. **Resource Isolation**: + Unlike `hybrid_engine`, this method requires explicit resource allocation for rollout, with remaining resources + automatically assigned to training. + +3. **NCCL Parameter Synchronization**: + Employs NCCL communication primitives for seamless parameter transfer between generation and training modules. + +### Experimental Results + +- **Machine Configuration**: 2 nodes with 16 H20 GPUs each + - Generation: 4 GPUs + - Training: 12 GPUs +- **Model**: Qwen2.5-Math-7B +- **Rollout Configuration**: +- **Max Response Length**: FSDP2: 20,480 tokens; Megatron: 8,192 tokens +- **Algorithm**: DAPO +- **Rollout Engine**: vLLM + +| training mode | engine | step | gen | wait_prev_gen | generate_sequences | old_log_prob | update_actor | total time | acc/best@32/mean | acc/maj@32/mean | +|------------------------|---------------|------|-----|---------------|--------------------|--------------|--------------|---------------|------------------|-----------------| +| colocate sync | VLLM+FSDP2 | 749 | 321 | - | 247 | 88 | 286 | 19h18m | 0.5948 | 0.417 | +| one-step-overlap async | VLLM+FSDP2 | 520 | - | 45 | 458 | 108 | 337 | 15h34m(+23%) | 0.6165 | 0.494 | +| colocate sync | VLLM+Megatron | 699 | 207 | - | 162 | 119 | 344 | 18h21m | 0.605 | 0.4217 | +| one-step-overlap async | VLLM+Megatron | 566 | - | 59 | 501 | 120 | 347 | 13h06m (+40%) | 0.6569 | 0.4038 | + +* colocate sync: step ≈ gen + old_log_prob + update_actor +* one-step-overlap async: step ≈ wait_prev_gen + old_log_prob + update_actor + +![One Step Off Megatron Performance]( +https://raw.githubusercontent.com/eric-haibin-lin/verl-community/refs/heads/main/docs/one_step_off_megatron.png) + +> source data: https://wandb.ai/hou-zg-meituan/one-step-off-policy?nw=nwuserhouzg + +## Implementation + +### One Step Off Policy Async Pipline + +Our implemented **One Step Off Policy Async Pipeline** integrates seamlessly into existing training logic at minimal +cost, +eliminating the need for additional sample storage management. The core mechanism uses `async_gen_next_batch` +for asynchronous rollout generation while maintaining continuous operation during epoch transitions +via `create_continuous_iterator`. + +```python +# iterator generator, simplify one-step integration of the training process +def _create_continuous_iterator(self): + for epoch in range(self.config.trainer.total_epochs): + iterator = iter(self.train_dataloader) + for batch_dict in iterator: + yield epoch, batch_dict + + +# read next batch samples, parameters sync and launch asyn gen_seq +def _async_gen_next_batch(self, continuous_iterator): + # read train_data + try: + epoch, batch_dict = next(continuous_iterator) + except StopIteration: + return None + batch = DataProto.from_single_dict(batch_dict) + gen_batch = batch_pocess(batch) + # sync weights from actor to rollout + self.sync_rollout_weights() + # async generation + gen_batch_output = self.rollout_wg.async_generate_sequences(gen_batch) + # future encapsulated + return GenerationBatchFuture(epoch, batch, gen_batch_output) + + +continuous_iterator = self._create_continuous_iterator() +# run rollout first to achieve one-step-off +batch_data_future = self._async_gen_next_batch(continuous_iterator) + +while batch_data_future is not None: + # wait for the gen_seq result from the previous step + batch = batch_data_future.get() + # launch the next async call to generate sequences + batch_data_future = self._async_gen_next_batch(continuous_iterator) + + # compute advantages + batch = critic.compute_values(batch) + batch = reference.compute_log_prob(batch) + batch = reward.compute_reward(batch) + batch = compute_advantages(batch) + + # model update + critic_metrics = critic.update_critic(batch) + actor_metrics = actor.update_actor(batch) +``` + +### Parameter Synchronization + +The exciting point is that our nccl based weights updating for rollout model has great performance. +At most of time, the latency is under 300ms, which is negligible for RLHF. + +> **sync_rollout_weights**:The time for synchronizing parameters from actor to rollout is extremely fast and can almost +> be ignored because it is implemented with nccl. + +```python +class ActorRolloutRefWorker: + # actor acquires the meta-info of model parameters for parameter sync + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def get_actor_weights_info(self): + params = self._get_actor_params() + ret = [] + for key, tensor in params.items(): + ret.append((key, tensor.size(), tensor.dtype)) + self._weights_info = ret + return ret + + # rollout sets the meta-info of model parameters for parameter sync + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_actor_weights_info(self, weights_info): + self._weights_info = weights_info + + +class AsyncRayPPOTrainer(RayPPOTrainer): + def init_workers(self): + ... + # rollout obtains the meta-info of model parameters from the actor for parameter sync + weights_info = self.actor_wg.get_actor_weights_info()[0] + self.rollout_wg.set_actor_weights_info(weights_info) + + # Create an actor-rollout communication group for parameter sync + self.create_weight_sync_group +``` + +```python +# The driving process invokes the actor and rollout respectively to create a weight synchronization group based on nccl/hccl. +def create_weight_sync_group(self): + master_address = ray.get(self.actor_wg.workers[0]._get_node_ip.remote()) + master_port = ray.get(self.actor_wg.workers[0]._get_free_port.remote()) + world_size = len(self.actor_wg.workers + self.rollout_wg.workers) + self.actor_wg.create_weight_sync_group( + master_address, + master_port, + 0, + world_size, + ) + ray.get( + self.rollout_wg.create_weight_sync_group( + master_address, + master_port, + len(self.actor_wg.workers), + world_size, + ) + ) + +# drive process call the actor and rollout respectively to sync parameters by nccl +def sync_rollout_weights(self): + self.actor_wg.sync_rollout_weights() + ray.get(self.rollout_wg.sync_rollout_weights()) + + +# fsdp model parameter sync +@register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) +def sync_rollout_weights(self): + params = self._get_actor_params() if self._is_actor else None + if self._is_rollout: + inference_model = ( + self.rollout.inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner.model + ) + from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader + patch_vllm_moe_model_weight_loader(inference_model) + # Model parameters are broadcast tensor-by-tensor from actor to rollout + for key, shape, dtype in self._weights_info: + tensor = torch.empty(shape, dtype=dtype, device=get_torch_device().current_device()) + if self._is_actor: + assert key in params + origin_data = params[key] + if hasattr(origin_data, "full_tensor"): + origin_data = origin_data.full_tensor() + if torch.distributed.get_rank() == 0: + tensor.copy_(origin_data) + from ray.util.collective import collective + + collective.broadcast(tensor, src_rank=0, group_name="actor_rollout") + if self._is_rollout: + inference_model.load_weights([(key, tensor)]) +``` + +## Usage + +### FSDP2 Configuration Example + +```shell +python3 -m recipe.one_step_off_policy.async_main_ppo \ + --config-path=config \ + --config-name='one_step_off_ppo_trainer.yaml' \ + actor_rollout_ref.actor.strategy=fsdp2 \ + # actor and rollout are placed separately + actor_rollout_ref.hybrid_engine=False \ + # actor and rollout resource + trainer.nnodes=1 \ + trainer.n_gpus_per_node=6 \ + rollout.nnodes=1 \ + rollout.n_gpus_per_node=2 +``` + +### Megatron Configuration Example + +```shell +python3 -m recipe.one_step_off_policy.async_main_ppo \ + --config-path=config \ + --config-name='one_step_off_ppo_megatron_trainer.yaml' \ + actor_rollout_ref.actor.strategy=megatron \ + # actor and rollout are placed separately + actor_rollout_ref.hybrid_engine=False \ + # actor and rollout resource + trainer.nnodes=1 \ + trainer.n_gpus_per_node=6 \ + rollout.nnodes=1 \ + rollout.n_gpus_per_node=2 +``` + +### Configuration Guidelines + +1. **Card Number Relationships** + Maintain either of these relationships for optimal batch distribution: + - `actor_rollout_ref.rollout.n` should be an integer divisor of: + `trainer.n_gpus_per_node * trainer.nnodes` + - `actor_rollout_ref.rollout.n * data.train_batch_size` should be evenly divisible by: + `trainer.n_gpus_per_node * trainer.nnodes` + + > Rationale: Ensures training samples can be evenly distributed across training GPUs when using partial resources for + generation. + +2. **Dynamic Resource Tuning** + Adjust `trainer.nnodes` `trainer.n_gpus_per_node` `rollout.nnodes` `rollout.n_gpus_per_node` based on phase + durations: + - **Ideal state**: Rollout and training phases have comparable durations + - **Diagnostic metrics**: + - Monitor `wait_prev_gen` duration + - Analyze `sequence_length` distribution + - **Adjustment strategy**: + - High `wait_prev_gen` + uniform sequence lengths → Increase rollout resources + - High `wait_prev_gen` + long-tail sequences → Optimize stopping criteria (resource increase won't help) + > **wait_prev_gen**:The time consumed waiting for the previous rollout to end (the part that is not fully + overlapped). + **Resource Configuration Strategies:** + - **Resource-constrained scenario**: Optimize resource utilization by adjusting GPU allocation ratios, + keeping the number of nodes equal to allow training and rollout to share nodes; + - Configure `trainer.nnodes = rollout.nnodes` with + `trainer.n_gpus_per_node + rollout.n_gpus_per_node = physical_gpus_per_node`. Control rollout resource + allocation by adjusting `n_gpus_per_node`. + - **Resource-abundant scenario**: Optimize performance by adjusting the number of nodes, + keeping the number of GPUs per node equal to enable independent scaling of training and rollout + parallelism. + - Configure `trainer.n_gpus_per_node = rollout.n_gpus_per_node` and control rollout resource allocation by + adjusting `trainer.nnodes` and `rollout.nnodes`to achieve optimal performance. + > **Note**: The total number of nodes required by the system is not simply `trainer.nnodes + rollout.nnodes`. The + > actual calculation depends on GPU capacity: + > - When `trainer.n_gpus_per_node + rollout.n_gpus_per_node <= physical_gpus_per_node`, + > the required node count is `max(trainer.nnodes, rollout.nnodes)` + > - When `trainer.n_gpus_per_node + rollout.n_gpus_per_node > physical_gpus_per_node`, + > the required node count is `trainer.nnodes + rollout.nnodes` + +## Functional Support + +| Category | Support Situation | +|--------------------|-----------------------------------------------------------------------------------------------------------------| +| train engine | FSDP2
Megatron | +| rollout engine | vLLM | +| AdvantageEstimator | GRPO
GRPO_PASSK
REINFORCE_PLUS_PLUS
RLOO
OPO
REINFORCE_PLUS_PLUS_BASELINE
GPG | +| Reward | all | diff --git a/e2c/verl/docs/advance/placement.rst b/e2c/verl/docs/advance/placement.rst new file mode 100644 index 0000000000000000000000000000000000000000..43ba761f76d86591d31b447c0ac5140149dd1082 --- /dev/null +++ b/e2c/verl/docs/advance/placement.rst @@ -0,0 +1,13 @@ +Ray API Design Tutorial +======================================= + +Last updated: 10/30/2024. + +We provide a tutorial for our Ray API design, including: + +- Ray basic concepts +- Resource Pool and RayWorkerGroup +- Data Dispatch, Execution and Collection +- Initialize the RayWorkerGroup and execute the distributed computation in the given Resource Pool + +See details in `tutorial.ipynb `_. \ No newline at end of file diff --git a/e2c/verl/docs/advance/ppo_lora.rst b/e2c/verl/docs/advance/ppo_lora.rst new file mode 100644 index 0000000000000000000000000000000000000000..baf3ab90a73c748e999b1467faed51aa7cc54410 --- /dev/null +++ b/e2c/verl/docs/advance/ppo_lora.rst @@ -0,0 +1,87 @@ +RL(HF) algorithms with LoRA Support +=========================================== + +Last updated: 06/05/2025. + +We support LoRA (Low-Rank Adaptation) for reinforcement learning algorithms such as PPO, GRPO, and others. + +LoRA is a parameter-efficient fine-tuning technique that injects trainable low-rank matrices into pre-trained weights (typically linear layers). This reduces memory footprint and compute cost, making it possible to fine-tune large models with limited hardware. + +The benefits this brings include: + +- reinforcement learning with very large models (e.g. 70B+) with modest hardware (e.g. 8x80G GPUs), +- enable larger batch sizes due to reduced memory usage, +- simplify model transfer and deployment, as only LoRA adapters need to be saved, +- Combine with techniques like `SLoRA `_ or `CCoE `_ to serve multiple LoRA adapters efficiently + +This guide explains how to enable LoRA in RL training and configure related parameters. + +Usage Guide +------------------------ +1. Lora is available in the `verl.trainer.ppo.ray_trainer.RayPPOTrainer`. Examples are provided via the `verl.trainer.main_ppo` entry point. + +2. Currently, LoRA is supported via huggingface peft, only with fsdp/fsdp2 and vllm backend (sglang support coming soon). + +- `strategy=fsdp` or `strategy=fsdp2` +- `rollout.name=vllm` + +3. Required configurations for LoRA: + +- `actor_rollout_ref.model.lora_rank`: int, set to a reasonable value greater than 0 (e.g., 8, 16, 32, 64) +- `actor_rollout_ref.model.lora_alpha`: float, the alpha term in LoRA +- `actor_rollout_ref.rollout.load_format="safetensors"`: required. This enables vLLM to load the base model. +- `actor_rollout_ref.model.target_modules`: the target modules for LoRA. Typically set to "all-linear". + +4. Recommend options: + +- `actor_rollout_ref.model.use_shm=True`: preload the model into `/dev/shm` to improve model loading speed. +- `actor_rollout_ref.rollout.layered_summon=True`: this enables the actor-model to gather the FSDP shards per layers when synchronizing the LoRA Adapter to vLLM, thereby reducing GPU peak memory. Recommended if the model is very large (70B+) or the GPU memory is limited (< 48GB) + + +Best Practices and Notes +------------------------- + +1. **Learning rate**: it is recommended to increase the value of learning rate by an order of magnitude. + +2. **LoRA Rank**: + +- Too small a rank can hurt convergence. +- LoRA rank recommendation from @thelongestusernameofall: + + - A very small lora_rank can lead to slower convergence or worse training performance. It is recommended to set lora_rank to be>=32. Tests have shown that for a 0.5B model, with lora_rank=32,the training convergence speed and final performance are almost identical to non-LoRA training + - For a 32B model,with lora_rank=128,the training convergence speed and final performance are also almost identical to non-LoRA training. + - More comprehensive reference results are coming soon. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/f2b80b8b26829124dd393b7a795a0640eff11644/docs/lora.jpg?raw=true + +3. Reference configuration for RL training with the Qwen2.5-72B model using 8 x 80GB GPUs (increase lora_rank if needed): + +.. code-block:: + + data.train_batch_size=64 \ + actor_rollout_ref.model.use_shm=True \ + actor_rollout_ref.model.lora_rank=32 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.actor.optim.lr=3e-5 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=8 \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=8 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.max_num_seqs=64 \ + actor_rollout_ref.rollout.max_model_len=1536 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1536 \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + +Example Script +------------------- + +For an end-to-end example, refer to the script below: + +examples/grpo_trainer/run_qwen2_5-3b_gsm8k_grpo_lora.sh diff --git a/e2c/verl/docs/advance/rollout_skip.rst b/e2c/verl/docs/advance/rollout_skip.rst new file mode 100644 index 0000000000000000000000000000000000000000..1839beed3e46805293cc7cdf9836571b4525c7fe --- /dev/null +++ b/e2c/verl/docs/advance/rollout_skip.rst @@ -0,0 +1,61 @@ +RolloutSkip Function Usage Documentation +======================================== + +Last updated: 08/01/2025. + +Applicable Scenarios +-------------------- + +The RolloutSkip functionality is designed to accelerate the rollout process in reinforcement learning training by caching and reusing previously generated sequences. This feature is particularly useful when: + +1. You need to repeatedly run experiments with the same configuration + +2. You want to save time by avoiding redundant sequence generation to come close to the optimal policy + + +API and Usage Example +---------------------- + +2.1 Trainer Adaptation +~~~~~~~~~~~~~~~~~~~~~~ + +Both`RayDAPOTrainer()` (in `verl/recipe/dapo/dapo_ray_trainer.py`) and `RayPPOTrainer()`(in `verl/trainer/ppo/ray_trainer.py``) have already been adapted. + +This is an example of how to patch rollout_skip in RayPPOTrainer. + +.. code-block:: python + + #* Import the RolloutSkip class + from verl.utils.rollout_skip import RolloutSkip + + ... + class RayPPOTrainer: + ... + def fit(self): + ... + + #* Add code as follow: + rollout_skip = RolloutSkip(self.config, self.actor_rollout_wg) + rollout_skip.wrap_generate_sequences() + + ... + + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + ... + +2.2 Basic Configuration +~~~~~~~~~~~~~~~~~~~~~~~ + +Then, you should add the following parameters to your config to enable the RolloutSkip feature: + +.. code-block:: bash + + actor_rollout_ref.rollout.skip_rollout=True \ + actor_rollout_ref.rollout.skip_dump_dir="/tmp/rollout_dump" \ + + +Note: + +1. The `skip_dump_dir` is the directory where the cached sequences will be stored. Ensure that this directory is writable and accessible by your training process. And make sure that `skip_dump_dir` is not relative path because ray will store the data in `/tmp/ray/session_/` and the relative path will not be found in the worker. +2. The dumped data path follows this naming pattern `{experiment_name}_{project_name}_TrainGBS{train_gbs}__InferGBS{gen_gbs}__N{n}`, once you change the `experiment_name`, `project_name`, `train_gbs`, `gen_gbs`, or `n`, the cached data will be stored in a new directory. diff --git a/e2c/verl/docs/advance/rollout_trace.rst b/e2c/verl/docs/advance/rollout_trace.rst new file mode 100644 index 0000000000000000000000000000000000000000..ea203bbc02c03534c09ddda39a435909874703d8 --- /dev/null +++ b/e2c/verl/docs/advance/rollout_trace.rst @@ -0,0 +1,125 @@ +Trace Function Usage Instructions +======================================== + +Last updated: 07/10/2025. + +Applicable Scenarios +-------------------- + +Agentic RL involves multiple turns of conversations, tool invocations, and user interactions during the rollout process. During the Model Training process, it is necessary to track function calls, inputs, and outputs to understand the flow path of data within the application. The Trace feature helps, in complex multi-round conversations, to view the transformation of data during each interaction and the entire process leading to the final output by recording the inputs, outputs, and corresponding timestamps of functions, which is conducive to understanding the details of how the model processes data and optimizing the training results. + +The Trace feature integrates commonly used Agent trace tools, including wandb weave and mlflow, which are already supported. Users can choose the appropriate trace tool according to their own needs and preferences. Here, we introduce the usage of each tool. + + +Trace Parameter Configuration +----------------------------- + +- ``actor_rollout_ref.rollout.trace.backend=mlflow|weave`` # the trace backend type +- ``actor_rollout_ref.rollout.trace.token2text=True`` # To show decoded text in trace view + + +Glossary +-------- + ++----------------+------------------------------------------------------------------------------------------------------+ +| Object | Explaination | ++================+======================================================================================================+ +| trajectory | A complete multi-turn conversation includes: | +| | 1. LLM output at least once | +| | 2. Tool Call | ++----------------+------------------------------------------------------------------------------------------------------+ +| step | The training step corresponds to the global_steps variable in the trainer | ++----------------+------------------------------------------------------------------------------------------------------+ +| sample_index | The identifier of the sample, defined in the extra_info.index of the dataset. It is usually a number,| +| | but may also be a uuid in some cases. | ++----------------+------------------------------------------------------------------------------------------------------+ +| rollout_n | In the GROP algorithm, each sample is rolled out n times. rollout_n represents the serial number of | +| | the rollout. | ++----------------+------------------------------------------------------------------------------------------------------+ +| validate | Whether the test dataset is used for evaluation? | ++----------------+------------------------------------------------------------------------------------------------------+ + +Rollout trace functions +----------------------- + +There are 2 functions used for tracing: + +1. ``rollout_trace_op``: This is a decorator function used to mark the functions to trace. In default, only few method has it, you can add it to more functions to trace more infor. +2. ``rollout_trace_attr``: This function is used to mark the entry of a trajectory and input some info to trace. If you add new type of agent, you may need to add it to enable trace. + + +Usage of wandb weave +-------------------- + +1.1 Basic Configuration +~~~~~~~~~~~~~~~~~~~~~~~ + +1. Set the ``WANDB_API_KEY`` environment variable +2. Configuration Parameters + + 1. ``actor_rollout_ref.rollout.trace.backend=weave`` + 2. ``trainer.logger=['console', 'wandb']``: This item is optional. Trace and logger are independent functions. When using Weave, it is recommended to also enable the wandb logger to implement both functions in one system. + 3. ``trainer.project_name=$project_name`` + 4. ``trainer.experiment_name=$experiment_name`` + 5. ``actor_rollout_ref.rollout.mode=async``: Since trace is mainly used for agentic RL, need to enable agent toop using async mode for either vllm or sglang. + +Note: +The Weave Free Plan comes with a default monthly network traffic allowance of 1GB. During the training process, the amount of trace data generated is substantial, reaching dozens of gigabytes per day, so it is necessary to select an appropriate wandb plan. + + +1.2 View Trace Logs +~~~~~~~~~~~~~~~~~~~ + +After executing the training, on the project page, you can see the WEAVE sidebar. Click Traces to view it. + +Each Trace project corresponds to a trajectory. You can filter and select the trajectories you need to view by step, sample_index, rollout_n, and experiment_name. + +After enabling token2text, prompt_text and response_text will be automatically added to the output of ToolAgentLoop.run, making it convenient to view the input and output content. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/weave_trace_list.png?raw=true + +1.3 Compare Trace Logs +~~~~~~~~~~~~~~~~~~~~~~ + +Weave can select multiple trace items and then compare the differences among them. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/weave_trace_compare.png?raw=true + +Usage of mlflow +--------------- + +1. Basic Configuration +~~~~~~~~~~~~~~~~~~~~~~ + +1. Set the ``MLFLOW_TRACKING_URI`` environment variable, which can be: + + 1. Http and https URLs corresponding to online services + 2. Local files or directories, such as ``sqlite:////tmp/mlruns.db``, indicate that data is stored in ``/tmp/mlruns.db``. When using local files, it is necessary to initialize the file first (e.g., start the UI: ``mlflow ui --backend-store-uri sqlite:////tmp/mlruns.db``) to avoid conflicts when multiple workers create files simultaneously. + +2. Configuration Parameters + + 1. ``actor_rollout_ref.rollout.trace.backend=mlflow`` + 2. ``trainer.logger=['console', 'mlflow']``. This item is optional. Trace and logger are independent functions. When using mlflow, it is recommended to also enable the mlflow logger to implement both functions in one system. + 3. ``trainer.project_name=$project_name`` + 4. ``trainer.experiment_name=$experiment_name`` + + +2. View Log +~~~~~~~~~~~ + +Since ``trainer.project_name`` corresponds to Experiments in mlflow, in the mlflow view, you need to select the corresponding project name, then click the "Traces" tab to view traces. Among them, ``trainer.experiment_name`` corresponds to the experiment_name of tags, and tags corresponding to step, sample_index, rollout_n, etc., are used for filtering and viewing. + +For example, searching for ``"tags.step = '1'"`` can display all trajectories of step 1. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/mlflow_trace_list.png?raw=true + +Opening one of the trajectories allows you to view each function call process within it. + +After enabling token2text, prompt_text and response_text will be automatically added to the output of ToolAgentLoop.run, making it convenient to view the content. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/mlflow_trace_view.png?raw=true + +Note: + +1. mlflow does not support comparing multiple traces +2. rollout_trace can not associate the mlflow trace with the run, so the trace content cannot be seen in the mlflow run logs. diff --git a/e2c/verl/docs/advance/rope.rst b/e2c/verl/docs/advance/rope.rst new file mode 100644 index 0000000000000000000000000000000000000000..9463549e47d055552a273e83a851fc76f93f9d1a --- /dev/null +++ b/e2c/verl/docs/advance/rope.rst @@ -0,0 +1,39 @@ +RoPE Scaling override +======================================= + +Last updated: 05/14/2025. + +Some models such as `Qwen/Qwen2.5-7B-Instruct `_ support RoPE Scaling but don't have it defined in their config.json file. +For example, this model supports this configuration: + +.. code:: python + + { + ..., + "rope_scaling": { + "factor": 4.0, + "original_max_position_embeddings": 32768, + "type": "yarn" + } + } + + + +In order to support a longer context for such models, you must override the model configs when starting the trainer. + +PPO example: + +.. code:: bash + + +actor_rollout_ref.model.override_config.rope_scaling.type=yarn \ + +actor_rollout_ref.model.override_config.rope_scaling.factor=4.0 \ + +actor_rollout_ref.model.override_config.rope_scaling.original_max_position_embeddings=32768 \ + + +And for the critic model + +.. code:: bash + + +critic.model.override_config.rope_scaling.type=yarn \ + +critic.model.override_config.rope_scaling.factor=4.0 \ + +critic.model.override_config.rope_scaling.original_max_position_embeddings=32768 \ diff --git a/e2c/verl/docs/algo/baseline.md b/e2c/verl/docs/algo/baseline.md new file mode 100644 index 0000000000000000000000000000000000000000..bbf241ca31669cf05a0846791c05cbdc1b51e38a --- /dev/null +++ b/e2c/verl/docs/algo/baseline.md @@ -0,0 +1,77 @@ +# Algorithm Baselines + +Last updated: 06/18/2025. + +## Math related datasets + +### GSM8k + +Assuming GSM8k/math dataset is preprocessed via: + +```bash +python3 examples/data_preprocess/*.py +``` + +Refer to the table below to reproduce RL training from different pre-trained checkpoints. Below is the performance on the GSM8k dataset if not specified otherwise. More comprehensive benchmark results areavailable in the recipe folder. + + +| Hardware | Model | Method | Test score | Details | +|-------------|----------------------------------|-------------------|--------------|---------| +| NVIDIA GPU | google/gemma-2-2b-it | hf checkpoint | 23.9 | [Huggingface](https://huggingface.co/google/gemma-2-2b-it#benchmark-results) | +| NVIDIA GPU | google/gemma-2-2b-it | SFT | 52.06 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/gemma-2-2b-it-sft-0.411.log) | +| NVIDIA GPU | google/gemma-2-2b-it | SFT + PPO | 64.02 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/gemma-2-2b-it-ppo-bsz512_4-prompt1024-resp-512-0.640.log), [wandb](https://api.wandb.ai/links/verl-team/h7ux8602) | +| NVIDIA GPU | Qwen/Qwen2.5-0.5B-Instruct | hf checkpoint | 36.4 | [Qwen blog](https://qwenlm.github.io/blog/qwen2.5-llm/) | +| NVIDIA GPU | Qwen/Qwen2.5-0.5B-Instruct | PPO | 56.7 | [command and log](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-0.5B-bsz256_2-prompt1024-resp512-0.567.log) | +| NVIDIA GPU | Qwen/Qwen2.5-0.5B-Instruct | PRIME | 58.7 | [script](https://github.com/volcengine/verl/blob/main/recipe/prime/run_prime_qwen.sh), [wandb](https://api.wandb.ai/links/zefan-wang-thu-tsinghua-university/rxd1btvb) | +| NVIDIA GPU | Qwen/Qwen2.5-0.5B-Instruct | GRPO-LoRA | 54.3 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-0.5B-bsz64_2-prompt512-resp1024-lorarank32-score0.543.log)| +| NVIDIA GPU | Qwen/Qwen2.5-1.5B-Instruct | GRPO-LoRA | 77.9 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-1.5B-bsz64_2-prompt512-resp1024-lorarank32-score0.779.log)| +| NVIDIA GPU | Qwen/Qwen2.5-3B-Instruct | GRPO-LoRA | 86.1 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-3B-bsz64_2-prompt512-resp1024-lorarank32-score0.861.log)| +| NVIDIA GPU | deepseek-ai/deepseek-llm-7b-chat | PPO (Megatron) | 69.5 [1] | [log](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/deepseek-llm-7b-chat-megatron-bsz256_4-prompt512-resp512-0.695.log), [wandb](https://wandb.ai/verl-team/verl_megatron_gsm8k_examples/runs/10fetyr3) | +| NVIDIA GPU | Qwen/Qwen2-7B-Instruct | GRPO | 89 | [script](https://github.com/volcengine/verl/blob/a65c9157bc0b85b64cd753de19f94e80a11bd871/examples/grpo_trainer/run_qwen2-7b_seq_balance.sh) | +| NVIDIA GPU | Qwen/Qwen2-7B-Instruct | GRPO (FSDP2) | 89.8 | [log](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/qwen2-7b-fsdp2.log) | +| NVIDIA GPU | Qwen/Qwen2-7B-Instruct | GRPO (Megatron) | 89.6 | [log](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/qwen2-7b_math_megatron.log) | +| NVIDIA GPU | Qwen/Qwen2.5-7B-Instruct | ReMax | 97 | [script](https://github.com/eric-haibin-lin/verl/blob/main/examples/remax_trainer/run_qwen2.5-3b_seq_balance.sh), [wandb](https://wandb.ai/liziniu1997/verl_remax_example_gsm8k/runs/vxl10pln) | +| NVIDIA GPU | Qwen/Qwen2.5-7B-Instruct | SPPO | 65.6 (MATH) | [SPPO script](https://github.com/volcengine/verl/tree/main/recipe/sppo/README.md) | +| NVIDIA GPU | Qwen/Qwen2.5-7B-Instruct | GRPO-LoRA | 93.4 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-7B-bsz64_8-prompt512-resp1024-lorarank32-score0.934.log)| +| NVIDIA GPU | Mixtral-8x22B-Instruct-v0.1 | Instruct model | 83.7 | [Qwen Blog](https://qwenlm.github.io/blog/qwen2.5-llm/) | +| NVIDIA GPU | Mixtral-8x22B-Instruct-v0.1 | RLOO (Megatron) | 92.3 | [wandb](https://api.wandb.ai/links/ppo_dev/sbuiuf2d) | +| NVIDIA GPU | Qwen/Qwen2.5-7B-Instruct | SPIN | 92 | [script](https://github.com/volcengine/verl/tree/main/recipe/spin/README.md) | +| NVIDIA GPU | Qwen/Qwen2-7B-Instruct | GPG | 88 | [log](https://github.com/diqiuzhuanzhuan/verldata/blob/main/run_logs/qwen2-7b_math.log), [wandb](https://wandb.ai/diqiuzhuanzhuan/verl_gpg_example_gsm8k_math/runs/ab86c4va) | +| NVIDIA GPU | Qwen/Qwen2-7B-Instruct | GPG (Megatron) | 88 | [log](https://github.com/diqiuzhuanzhuan/verldata/blob/main/run_logs/qwen2-7b_math_megatron.log), [wandb](https://wandb.ai/diqiuzhuanzhuan/verl_gpg_example_gsm8k_math/runs/yy8bheu8) | +| NVIDIA GPU | Qwen/Qwen2.5-VL-7B-Instruct | GRPO (Megatron) | 65.4 (GEO3k) | [script](https://github.com/volcengine/verl/blob/main/examples/grpo_trainer/run_qwen2_5_vl-7b-megatron.sh), [wandb](https://api.wandb.ai/links/megatron-core-moe-dev/1yngvkek) | +| AMD MI300 | deepseek-ai/deepseek-llm-7b-chat | PPO | 70.5 [1] | [log](https://github.com/yushengsu-thu/verl_training_log/blob/main/gsm8k/ppo_run_deepseek7b_llm.log) | +| AMD MI300 | deepseek-ai/deepseek-llm-7b-chat | GRPO | 71.4 [1] | [log](https://github.com/yushengsu-thu/verl_training_log/blob/main/gsm8k/grpo_run_deepseek7b_llm.log) | +| NVIDIA GPU | Qwen/Qwen2.5-14B-Instruct | GRPO-LoRA | 94.6 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-14B-bsz64_8-prompt512-resp1024-lorarank32-score0.946.log)| +| NVIDIA GPU | Qwen/Qwen2.5-32B-Instruct | GRPO-LoRA | 95.8 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-32B-bsz64_8-prompt512-resp1024-lorarank32-score0.958.log)| +| NVIDIA GPU | Qwen/Qwen2.5-72B-Instruct | GRPO-LoRA | 96.0 | [command and logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-72B-bs64_8-prompt512-resp1024-lorarank32-score0.960.log)| + +### DAPO math-17k + +- Training DAPO math-17k dataset: https://huggingface.co/datasets/BytedTsinghua-SIA/DAPO-Math-17k +- Testing: AIME'24: https://huggingface.co/datasets/BytedTsinghua-SIA/AIME-2024 + +Note: +- For Qwen/Qwen2.5-Math-7B, we directly modify the max_position_embeddings to 32768 without observing performance degradation in order to train longer response length. + +| Hardware | Model | Method | Test score | Details | +|-------------|-----------------------------|-------------------------|------------|---------| +| NVIDIA GPU | Qwen/Qwen2.5-Math-7B (32k) | DAPO | 36.3 | [command](https://github.com/volcengine/verl/blob/main/recipe/dapo/test_dapo_7b_math.sh), [logs](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/runs/ow47vvon?nw=nwusertongyuxuan361)| +| NVIDIA GPU | Qwen/Qwen2.5-7B-Instruct | DAPO + Code Interpreter | 40.0 | [command](https://github.com/volcengine/verl/blob/main/recipe/retool/run_qwen2_7b_dapo.sh)| + + + + +## Coding related datasets + +Below is the result on leetcode if not specified otherwise. + +| Hardware | Model | Method | Test score | Details | +|-------------|----------------------------------|-------------------|--------------|---------| +| NVIDIA GPU | PRIME-RL/Eurus-2-7B-SFT | RPIME | 36.1 | [script](https://github.com/volcengine/verl/blob/main/recipe/prime/run_prime_qwen_code.sh), [swanlab](https://swanlab.cn/@wangzefan/prime_example/runs/7f541qhspgmy8nmhdlx35/chart) | + + +### Notes + +[1] During evaluation, we have only extracted answers following the format `"####"`. A more flexible answer extraction, longer response length, and better prompt engineering may lead to a higher score. + +[2] The default value of `actor_rollout_ref.actor.entropy_coeff` is set to `0.0` since verl 0.3.x on 2025-05-30, which is different from previous versions. diff --git a/e2c/verl/docs/algo/collabllm.md b/e2c/verl/docs/algo/collabllm.md new file mode 100644 index 0000000000000000000000000000000000000000..3279e0ff3a43b4154c9ee54ed80452ea997408e0 --- /dev/null +++ b/e2c/verl/docs/algo/collabllm.md @@ -0,0 +1,105 @@ +# Recipe: CollabLLM + +Last updated: 09/22/2025. + +> Open-Source Algorithm Implementation & Expriement Running: [Haiquan Chen](https://github.com/chenhaiq), [Shirley Wu](https://github.com/Wuyxin) + +🏠 [Homepage](https://aka.ms/CollabLLM) | 📝 [Paper](https://arxiv.org/pdf/2502.00640) | 🤗 [Datasets & Models](https://huggingface.co/collabllm) | ⭐️ [Original Implementation](https://github.com/Wuyxin/collabllm) + +`verl` provides a recipe for the Outstanding Paper at ICML 2025, **"CollabLLM: From Passive Responders to Active Collaborators"**. [CollabLLM](https://aka.ms/CollabLLM) is a unified fine-tuning framework that optimizes LLMs for effective and efficient multiturn collaboration with users. + +**Core Idea:** Models are rewarded based on how well their responses enable effective *future* collaboration with users. + +Paper Authors: [Shirley Wu](https://cs.stanford.edu/~shirwu/), [Michel Galley](https://www.microsoft.com/en-us/research/people/mgalley/), Baolin Peng, Hao Cheng, Gavin Li, Yao Dou, Weixin Cai, [James Zou](https://www.james-zou.com/), [Jure Leskovec](https://cs.stanford.edu/people/jure/), [Jianfeng Gao](https://www.microsoft.com/en-us/research/people/jfgao/) + + +--- +## Quick Start + +### 0. Environment +Make sure the required packages for `verl` are installed. Additionally, install `litellm` and export the required API keys. The API model will be used for user simulators and, optionally, LLM Judges (see the Configuration section below). + +### 1. Prepare Your Dataset + +First, process your dataset using the provided script (see example commands and usage in `process_dataset.py`): + +```bash +python process_dataset.py --dataset <> ... --dataset_type +``` + + +**Requirements:** +- Input: A Hugging Face multiturn dataset. Existing datasets: `collabllm/collabllm-multiturn-$DATASET`, with `DATASET` in one of [`math-hard(-large)`, `medium(-large)`, `bigcodebench(-large)`] (*-large are the datasets used in the CollabLLM paper) +- Example format: See [collabllm-multiturn-math-hard](https://huggingface.co/datasets/collabllm/collabllm-multiturn-math-hard) +- To generate your own dataset: Use [build_dataset.py](https://github.com/Wuyxin/collabllm/blob/main/scripts/engine/build_dataset.py) from the original CollabLLM repository + + +### 2. Train Your Model + +**(Optional) For Supervised Fine-Tuning (SFT):** +```bash +bash train_sft_collabllm.sh +``` + +**For Reinforcement Learning (RL):** + +```bash +bash train_rl_collabllm.sh +``` + +The RL script shows an example to train CollabLLM on `math-hard-large`. + +- The config to sample future conversations are in `recipe/collabllm/config/collabllm_interaction_config.yaml`. +- The Multiturn-aware Reward is aggregated from these three conversational-level rewards: + + ``` + +reward_model.reward_kwargs.metric_weights.accuracy=1 \ + +reward_model.reward_kwargs.metric_weights.interactivity=1 \ + +reward_model.reward_kwargs.metric_weights.token_amount=-0.0001 \ + ``` + + You can remove, add, or modify the weights depending on your task. A list of implemented metrics you can already add are under `recipe/collabllm/metrics`. For example, on `medium-large`, you can replace `accuracy` with `bleu_score` via + ``` + +reward_model.reward_kwargs.metric_weights.bleu_score=1 + ``` + which will instead apply bleu score on the sampled future conversations. + +## Algorithm + +| Step | Name | Description | +|------|-------------------------------|-----------------------------------------------------------------------------| +| 1 | Model response generation | The model generates multiple responses for each prompt in a batch. | +| 2 | Collaborative simulation | A user simulator (e.g., GPT or Claude) samples `num_repeat_rollouts` conversations for up to `max_user_turns` additional turns. | +| 3 | Compute Multiturn-aware Reward | Customized conversational reward functions are applied to the sampled conversations. Rewards are aggregated, then averaged across rollouts. | +| 4 | Update model | The model weights are updated using the computed multiturn-aware rewards. | + +--- + +## Configuration + +The primary configuration is managed through the launch script `train_rl_collabllm.sh` and the YAML file `recipe/collabllm/config/collabllm_interaction_config.yaml`. Key configuration sections: + +| Section | Key Parameters / Notes | +|----------------------|-----------------------------------------------------------------------------------------| +| `data` | Paths to training/validation files, batch sizes, sequence lengths. | +| `actor_rollout_ref` (common) | Base model path (used for actor + initial reference), FSDP settings, optimization (LR, scheduler). | +| `actor_rollout_ref` (CollabLLM-specific) | Hyperparameters under `actor_rollout_ref.rollout.multi_turn`: `max_user_turns`, `max_assistant_turns`, `num_repeat_rollouts`. | +| `interaction` | Defined in `collabllm_interaction_config.yaml`. Specifies user simulator and hyperparameters. Requires exported API keys. | +| `reward_model` | Manager set to `collabllm` by default. Modify `reward_model.reward_kwargs.metric_weights` for conversational rewards and weights. LLM Judge hyperparameters (e.g., `model`, `temperature`) go under `reward_model.reward_kwargs.llm_judge_kwargs`. | +| `algorithm` | GRPO-specific hyperparameters such as `actor_rollout_ref.rollout.n`. | +| `trainer` | Distributed training (nodes, GPUs per node), logging (WandB), checkpointing frequency. | + +--- + +## Key Files + +| File Path | Purpose | +|-----------|---------| +| `recipe/collabllm/collabllm_agent_loop.py` | Main logic to sample future conversations, using `CollabLLMInteraction` from `verl/interactions/collabllm_interaction.py`. | +| `verl/workers/reward_manager/collabllm.py` | Computes rewards for future conversations, leveraging `recipe/collabllm/reward_function.py` to apply each metric. | + +--- + +## Acknowledgement + +We sincerely thank the `verl` community and advisors for their contributions and guidance! diff --git a/e2c/verl/docs/algo/dapo.md b/e2c/verl/docs/algo/dapo.md new file mode 100644 index 0000000000000000000000000000000000000000..96f242eaa86add8ab598af583975eb808c6a78d6 --- /dev/null +++ b/e2c/verl/docs/algo/dapo.md @@ -0,0 +1,187 @@ +# Recipe: Decoupled Clip and Dynamic Sampling Policy Optimization (DAPO) + +Last updated: 06/19/2025. + +> Open-Source Algorithm Implementation & Expriement Running: [Yuxuan Tong](https://tongyx361.github.io/), [Guangming Sheng](https://hk.linkedin.com/in/guangming-sheng-b50640211) + +🏠 [Homepage](https://dapo-sia.github.io/) | 📝 [Paper@arXiv](https://arxiv.org/abs/2503.14476) | 🤗 [Datasets&Models@HF](https://huggingface.co/collections/BytedTsinghua-SIA/dapo-67d7f1517ee33c8aed059da0) | 🐱 [Code@GitHub](https://github.com/volcengine/verl/tree/recipe/dapo/recipe/dapo) | 🐱 [Repo@GitHub](https://github.com/BytedTsinghua-SIA/DAPO) + +> We propose the **D**ecoupled Clip and Dynamic s**A**mpling **P**olicy **O**ptimization (DAPO) algorithm. By making our work publicly available, we provide the broader research community and society with practical access to scalable reinforcement learning, enabling all to benefit from these advancements. Our system is based on the awesome [verl](https://github.com/volcengine/verl) framework. Thanks for their great work! Applying DAPO training to Qwen2.5-32B base model proves to outperform the previous state-of-the-art DeepSeek-R1-Zero-Qwen-32B on AIME 2024, achieving **50%** accuracy with **50%** less training steps. +> +> ![dapo-main-result](https://dapo-sia.github.io/static/images/score.png) + +## Quickstart + +1. Prepare the datasets **on the Ray cluster**: + +```bash +bash prepare_dapo_data.sh # This downloads the datasets to ${HOME}/verl/data by default +``` + +2. Submit the job to the Ray cluster **from any machine**: + +```bash +cd verl # Repo root +export RAY_ADDRESS="http://${RAY_IP:-localhost}:8265" # The Ray cluster address to connect to +export WORKING_DIR="${PWD}" # The local directory to package to the Ray cluster +# Set the runtime environment like env vars and pip packages for the Ray cluster in yaml +export RUNTIME_ENV="./recipe/dapo/runtime_env.yaml" # This sets environment variables for the Ray cluster +bash recipe/dapo/run_dapo_qwen2.5_32b.sh # or other scripts +``` + +## Reproduction Runs + +| Setup | AIME 2024 Acc. | Hardware | Image | Commit | Environment Variables | Training Script | Training Record | +| -------------------------------------------- | -------------- | --------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| DAPO | 52% | 16x8xH800 | `hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.3-flashinfer0.2.2-cxx11abi0` | [`4f80e4`](https://github.com/volcengine/verl/tree/4f80e465c2ec79ab9c3c30ec74b9745de61d0490) | [runtime_env.yaml](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/runtime_env.yaml) | [run_dapo_qwen2.5_32b.sh](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/run_dapo_qwen2.5_32b.sh) | [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=wmb4qxfht0n) | +| DAPO w/o Dynamic Sampling | 50% | 16x8xH800 | `hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.3-flashinfer0.2.2-cxx11abi0` | [`4f80e4`](https://github.com/volcengine/verl/tree/4f80e465c2ec79ab9c3c30ec74b9745de61d0490) | [runtime_env.yaml](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/runtime_env.yaml) | [run_dapo_wo_ds_qwen2.5_32b.sh](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/run_dapo_wo_ds_qwen2.5_32b.sh) | [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=wmb4qxfht0n) | +| DAPO w/o Token-level Loss & Dynamic Sampling | 44% | 16x8xH20 | `hiyouga/verl:ngc-th2.5.1-cu120-vllm0.7.4-hotfix` | [`4f80e4`](https://github.com/volcengine/verl/tree/4f80e465c2ec79ab9c3c30ec74b9745de61d0490) | [runtime_env.yaml](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/runtime_env.yaml) | [run_dapo_early_qwen2.5_32b.sh](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/run_dapo_early_qwen2.5_32b.sh) | [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=wmb4qxfht0n) | + +> [!IMPORTANT] +> +> **📢 Call for Contribution!** +> +> Welcome to submit your reproduction runs and setups! + +## Configuration + +### Separated Clip Epsilons (-> Clip-Higher) + +An example configuration: + +```yaml +actor_rollout_ref: + actor: + clip_ratio_low: 0.2 + clip_ratio_high: 0.28 +``` + +`clip_ratio_low` and `clip_ratio_high` specify the $\varepsilon_{\text {low }}$ and $\varepsilon_{\text {high }}$ in the DAPO objective. + +Core relevant code: + +```python +pg_losses1 = -advantages * ratio +pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high) +pg_losses = torch.maximum(pg_losses1, pg_losses2) +``` + +### Dynamic Sampling (with Group Filtering) + +An example configuration: + +```yaml +data: + gen_batch_size: 1536 + train_batch_size: 512 +algorithm: + filter_groups: + enable: True + metric: acc # score / seq_reward / seq_final_reward / ... + max_num_gen_batches: 10 # Non-positive values mean no upper limit +``` + +Setting `filter_groups.enable` to `True` will filter out groups whose outputs' `metric` are all the same, e.g., for `acc`, groups whose outputs' accuracies are all 1 or 0. + +The trainer will repeat sampling with `gen_batch_size` until there are enough qualified groups for `train_batch_size` or reaching the upper limit specified by `max_num_gen_batches`. + +Core relevant code: + +```python +prompt_bsz = self.config.data.train_batch_size +if num_prompt_in_batch < prompt_bsz: + print(f'{num_prompt_in_batch=} < {prompt_bsz=}') + num_gen_batches += 1 + max_num_gen_batches = self.config.algorithm.filter_groups.max_num_gen_batches + if max_num_gen_batches <= 0 or num_gen_batches < max_num_gen_batches: + print(f'{num_gen_batches=} < {max_num_gen_batches=}. Keep generating...') + continue + else: + raise ValueError( + f'{num_gen_batches=} >= {max_num_gen_batches=}. Generated too many. Please check your data.' + ) +else: + # Align the batch + traj_bsz = self.config.data.train_batch_size * self.config.actor_rollout_ref.rollout.n + batch = batch[:traj_bsz] +``` + +### Flexible Loss Aggregation Mode (-> Token-level Loss) + +An example configuration: + +```yaml +actor_rollout_ref: + actor: + loss_agg_mode: "token-mean" # / "seq-mean-token-sum" / "seq-mean-token-mean" + # NOTE: "token-mean" is the default behavior +``` + +Setting `loss_agg_mode` to `token-mean` will mean the (policy gradient) loss across all the tokens in all the sequences in a mini-batch. + +Core relevant code: + +```python +if loss_agg_mode == "token-mean": + loss = verl_F.masked_mean(loss_mat, loss_mask) +elif loss_agg_mode == "seq-mean-token-sum": + seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) # token-sum + loss = torch.mean(seq_losses) # seq-mean +elif loss_agg_mode == "seq-mean-token-mean": + seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) / torch.sum(loss_mask, dim=-1) # token-mean + loss = torch.mean(seq_losses) # seq-mean +else: + raise ValueError(f"Invalid loss_agg_mode: {loss_agg_mode}") +``` + +### Overlong Reward Shaping + +An example configuration: + +```yaml +data: + max_response_length: 20480 # 16384 + 4096 +reward_model: + overlong_buffer: + enable: True + len: 4096 + penalty_factor: 1.0 +``` + +Setting `overlong_buffer.enable` to `True` will penalize the outputs whose lengths are overlong but still within the hard context limit. + +Specifically, the penalty increases linearly from `0` to `overlong_buffer.penalty_factor` when the length of the output exceeds the `max_response_length` by `0` to `overlong_buffer.len` tokens. + +Core relevant code: + +```python +if self.overlong_buffer_cfg.enable: + overlong_buffer_len = self.overlong_buffer_cfg.len + expected_len = self.max_resp_len - overlong_buffer_len + exceed_len = valid_response_length - expected_len + overlong_penalty_factor = self.overlong_buffer_cfg.penalty_factor + overlong_reward = min(-exceed_len / overlong_buffer_len * overlong_penalty_factor, 0) + reward += overlong_reward +``` + +## FAQ + +### Where is the "Overlong Filtering" in the paper? + +Most experiments in the paper, including the best-performant one, are run without Overlong Filtering because it's somehow overlapping with Overlong Reward Shaping in terms of properly learning from the longest outputs. So we don't implement it here. + +### What's the difference between [the `recipe/dapo` directory in the `main` branch](https://github.com/volcengine/verl/tree/main/recipe/dapo) and the [`recipe/dapo` branch](https://github.com/volcengine/verl/tree/recipe/dapo/recipe/dapo)? + +[The `recipe/dapo` branch](https://github.com/volcengine/verl/tree/recipe/dapo/recipe/dapo) is for **as-is reproduction** and thus won't be updated with new features. + +[The `recipe/dapo` directory in the `main` branch](https://github.com/volcengine/verl/tree/main/recipe/dapo) works as an example of how to extend the latest `verl` to implement an algorithm recipe, which will be maintained with new features. + +### Why can't I produce similar results after modifications? + +RL infrastructures nowadays still have inherent unrobustness, on which we are still working hard to improve. + +We strongly recommend to only modify one thing at a time. + +We also list some known problems here: + +1. Enabling CUDA graph (`enforce_eager=False`) might cause model performance degradation, whose cause is still under investigation. diff --git a/e2c/verl/docs/algo/entropy.md b/e2c/verl/docs/algo/entropy.md new file mode 100644 index 0000000000000000000000000000000000000000..46153b7e8558583c9d4a0201a1317f09c6c1ecb1 --- /dev/null +++ b/e2c/verl/docs/algo/entropy.md @@ -0,0 +1,115 @@ +# Recipe: Entropy Mechanism + +Last updated: 06/27/2025. + + +
+ + The Entropy Mechanism of Reinforcement Learning for Large Language Model Reasoning. + +[![Paper](https://img.shields.io/badge/paper-A42C25?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/pdf/2505.22617) [![Github](https://img.shields.io/badge/PRIME-000000?style=for-the-badge&logo=github&logoColor=000&logoColor=white)](https://github.com/PRIME-RL/Entropy-Mechanism-of-RL) [![alphaXiv](https://img.shields.io/badge/discussion-A42C25?style=for-the-badge&logo=arxiv&logoColor=white&color=blue +)](https://www.alphaxiv.org/abs/2505.22617) [![Twitter](https://img.shields.io/badge/Twitter-%23000000.svg?style=for-the-badge&logo=twitter&logoColor=white)](https://x.com/stingning/status/1928088554166505667) [![Twitter](https://img.shields.io/badge/Twitter-%23000000.svg?style=for-the-badge&logo=twitter&logoColor=white)](https://x.com/charlesfornlp/status/1928089451080585283) [![Twitter-ak](https://img.shields.io/badge/Twitter-%23000000.svg?style=for-the-badge&logo=twitter&logoColor=white)](https://x.com/_akhaliq/status/1928077929105268861) + + + + +
+ + +## 🎉News + +- **[2025/05/29]** 🎉 Ranked **#1** of the day on [Huggingface Daily Papers](https://huggingface.co/papers?date=2025-05-29). +- **[2025/05/29]** Released our Paper on arXiv. See [here](https://arxiv.org/pdf/2505.22617). We provide insights into the entropy mechanism of RL for LLMs and propose two simple yet effective strategies to alleviate the entropy collapse. + + + +## ✨Getting started + +After preparing the training data, for training Qwen2.5-7B on a single node, taking the KL-Cov approach as an example, you can simply run: + +``` +cd verl +conda activate your_env +bash recipe/dapo/7b_kl_cov.sh +``` + +While for training Qwen2.5-32B on multi nodes, you can run the following commands: + +``` +cd verl +conda activate your_env +bash recipe/dapo/32b_kl_cov.sh +``` + +## 📖Introduction + +
+ issue +
+ +This paper addresses the entropy collapse issue in scaling reinforcement learning (RL) for large language models (LLMs), where policy entropy drops sharply during training, leading to overconfidence and performance saturation. We empirically establish a relationship between entropy ($H$) and performance ($R$): $R=−aexp(H)+b$, showing performance is bottlenecked by entropy exhaustion. + +
+ issue +
+ +Theoretically, we find entropy changes are driven by the covariance between action probability and logit updates, which correlates with advantage in Policy Gradient methods. High-probability, high-advantage actions reduce entropy, while rare, high-advantage actions increase it. Empirically, the covariance term remains positive, explaining entropy’s monotonic decline. To mitigate this, we propose ​​Clip-Cov​​ and ​​KL-Cov​​, which restrict updates for high-covariance tokens. These methods effectively prevent entropy collapse, and improve performance. + +## 📃Evaluation + +
+ issue +
+ + +Our method is able to maintain a considerably higher level of entropy throughout training. For example, when the baseline's entropy reaches a plateau and can no longer be consumed, the KL-Cov method still sustains an entropy level over 10 times higher. Meanwhile, the response length of the policy model steadily increases, and its performance on the test set consistently surpasses that of the baseline. This indicates that our model is able to explore more freely during training, learning better policy through RL. +| **Method** | **AIME24** | **AIME25** | **AMC** | **MATH-500** | **OMNI-MATH** | **OlympiadBench** | **Minerva** | **Avg.** | +| ----------------- | ---------: | ---------: | -------: | -----------: | ------------: | ----------------: | ----------: | -------: | +| *Qwen2.5-7B* | | | | | | | | | +| GRPO | 21.2 | 9.6 | 58.7 | 78.8 | 27.9 | 40.7 | 36.7 | 38.6 | +| w. Clip-higher | 18.1 | 11.5 | 56.6 | 79.2 | 29.8 | 43.3 | 40.4 | 38.8 | +| w. **`CLIP-Cov`** | 22.1 | **15.8** | 58.2 | 80.4 | **30.5** | **44.1** | **41.1** | 40.4 | +| w. **`KL-Cov`** | **22.6** | 12.9 | **61.4** | **80.8** | 29.1 | 42.6 | 38.2 | **40.6** | +| *Qwen2.5-32B* | | | | | | | | | +| GRPO | 21.8 | 16.2 | 69.7 | 84.2 | 35.2 | 43.6 | 45.5 | 45.8 | +| w. Clip-higher | 35.6 | 22.3 | 69.5 | 77.2 | 35.1 | 42.5 | 43.0 | 47.2 | +| w. **`CLIP-Cov`** | 32.3 | 22.7 | 67.2 | **87.0** | **42.0** | **57.2** | 46.0 | 50.3 | +| w. **`KL-Cov`** | **36.8** | **30.8** | **74.5** | 84.6 | 39.1 | 49.0 | **46.3** | **52.2** | + +Our two approaches both achieve non-trivial improvements across all benchmarks. Compared to GRPO, our method outperforms it by 2.0% on average for the 7B model and by 6.4% for the 32B model. Moreover, we observe that our method yields more substantial gains on the larger Qwen2.5-32B. Specifically, our method achieves improvements of 15.0% and 14.6% compared to GRPO on the most challenging benchmarks, AIME24 and AIME25, respectively. + + +## 🎈Citation +If you find this paper or repo helpful, please cite us. + +```bibtex +@article{cui2025entropy, + title={The Entropy Mechanism of Reinforcement Learning for Reasoning Language Models}, + author={Cui, Ganqu and Zhang, Yuchen and Chen, Jiacheng and Yuan, Lifan and Wang, Zhi and Zuo, Yuxin and Li, Haozhan and Fan, Yuchen and Chen, Huayu and Chen, Weize and others}, + journal={arXiv preprint arXiv:2505.22617}, + year={2025} +} +``` +## 🌻Acknowledgement +We implement our reinforcement learning algorithm extending from [verl](https://github.com/volcengine/verl). We utilize [vLLM](https://github.com/vllm-project/vllm) for inference. Our models are trained primarily on [Qwen2.5 family](https://github.com/QwenLM/Qwen2.5). Our training data is built from [DAPO-MATH](https://huggingface.co/datasets/BytedTsinghua-SIA/DAPO-Math-17k). Thanks for their great contributions! + +## 📬 Contact + +For questions, discussion, or collaboration opportunities, feel free to contact: +- Ganqu Cui: cuiganqu@pjlab.org.cn +- Yuchen Zhang: yuchen.zhang2003@gmail.com +- Jiacheng Chen: jackchan9345@gmail.com +- Ning Ding: ningding.cs@gmail.com + diff --git a/e2c/verl/docs/algo/gpg.md b/e2c/verl/docs/algo/gpg.md new file mode 100644 index 0000000000000000000000000000000000000000..36bede8c319040ae713ef335372f2caa40ce44a3 --- /dev/null +++ b/e2c/verl/docs/algo/gpg.md @@ -0,0 +1,36 @@ +# GPG: Group Policy Gradient + +Last updated: 07/03/2025. + +Group Policy Gradient (GPG) is a minimalist reinforcement learning (RL) method that enhances the reasoning ability of large language models without relying on supervised fine-tuning or complex tricks. GPG revisits traditional policy gradients and directly optimizes the RL objective—no surrogate losses, no KL penalties, no critic, and no reference model. Compared to GRPO, GPG is simpler, more efficient, and achieves better results on many tasks. For more details, please refer to the original paper [GPG: A Simple and Strong Reinforcement Learning Baseline for Model Reasoning +](https://arxiv.org/abs/2504.02546). + +## Key Components +- Use a corrected advantage function to improve policy gradient accuracy and training efficiency. +- By eliminating the critic and reference models, avoiding KL divergence constraints, significantly simplifies the training process compared to Group Relative Policy Optimization (GRPO) + +## Configuration +To configure GPG within the framework, use the following YAML settings. + +```yaml +algorithm: + adv_estimator: gpg +actor_rollout_ref: + actor: + policy_loss: + loss_mode: "gpg" +``` + +## Advanced Extensions +GPG is a simple and strong baseline for model reasoning. Although it avoids using KL loss in its original form, you can still use KL loss to further improve the performance. + +```yaml +algorithm: + adv_estimator: gpg +actor_rollout_ref: + actor: + use_kl_loss: True # enable kl regularization + kl_loss_coef: 0.01 + policy_loss: + loss_mode: "gpg" +``` \ No newline at end of file diff --git a/e2c/verl/docs/algo/grpo.md b/e2c/verl/docs/algo/grpo.md new file mode 100644 index 0000000000000000000000000000000000000000..192ccd8403a5d188c9e24ebd603164f9cf339151 --- /dev/null +++ b/e2c/verl/docs/algo/grpo.md @@ -0,0 +1,71 @@ +# Group Relative Policy Optimization (GRPO) + +Last updated: 05/31/2025. + +In reinforcement learning, classic algorithms like PPO rely on a "critic" model to estimate the value of actions, guiding the learning process. However, training this critic model can be resource-intensive. + +GRPO simplifies this process by eliminating the need for a separate critic model. Instead, it operates as follows: +- Group Sampling: For a given problem, the model generates multiple possible solutions, forming a "group" of outputs. +- Reward Assignment: Each solution is evaluated and assigned a reward based on its correctness or quality. +- Baseline Calculation: The average reward of the group serves as a baseline. +- Policy Update: The model updates its parameters by comparing each solution's reward to the group baseline, reinforcing better-than-average solutions and discouraging worse-than-average ones. + +This approach reduces computational overhead by avoiding the training of a separate value estimation model, making the learning process more efficient. For more details, refer to the original paper [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://arxiv.org/pdf/2402.03300) + +## Key Components + +- No Value Function (Critic-less): unlike PPO, GRPO does not train a separate value network (critic) +- Group Sampling (Grouped Rollouts): instead of evaluating one rollout per input, GRPO generates multiple completions (responses) from the current policy for each prompt. This set of completions is referred to as a group. +- Relative Rewards: within each group, completions are scored (e.g., based on correctness), and rewards are normalized relative to the group. + +## Configuration + +Note that all configs containing `micro_batch_size` are used to configure the maximum sample or token count per forward or backward pass to avoid GPU OOMs, whose value should not change algorithmic/convergence behavior. + +Despite that many configurations start with the `ppo_` prefix, they work across different RL algorithms in verl, as the GRPO training loop is similar to that of PPO (without critic). + +![image](https://github.com/user-attachments/assets/16aebad1-0da6-4eb3-806d-54a74e712c2d) + +- `actor_rollout.ref.rollout.n`: For each prompt, sample n times. Default to 1. For GRPO, please set it to a value larger than 1 for group sampling. + +- `data.train_batch_size`: The global batch size of prompts used to generate a set of sampled trajectories/rollouts. The number of responses/trajectories is `data.train_batch_size * actor_rollout.ref.rollout.n` + +- `actor_rollout_ref.actor.ppo_mini_batch_size`: The set of sampled trajectories is split into multiple mini-batches with batch_size=ppo_mini_batch_size for PPO actor updates. The ppo_mini_batch_size is a global size across all workers. + +- `actor_rollout_ref.actor.ppo_epochs`: Number of epochs for GRPO updates on one set of sampled trajectories for actor + +- `actor_rollout_ref.actor.clip_ratio`: The GRPO clip range. Default to 0.2 + +- `algorithm.adv_estimator`: Default is gae. Please set it to grpo instead + +- `actor_rollout_ref.actor.loss_agg_mode`: Default is "token-mean". Options include "token-mean", "seq-mean-token-sum", "seq-mean-token-mean". The original GRPO paper takes the sample-level loss (seq-mean-token-mean), which may be unstable in long-CoT scenarios. All GRPO example scripts provided in verl uses the default configuration "token-mean" for loss aggregation instead. + +Instead of adding KL penalty in the reward, GRPO regularizes by directly adding the KL divergence between the trained policy and the reference policy to the loss: + +- `actor_rollout_ref.actor.use_kl_loss`: To use kl loss in the actor. When used, we are not applying KL in the reward function. Default is False. Please set it to True for GRPO. + +- `actor_rollout_ref.actor.kl_loss_coef`: The coefficient of kl loss. Default is 0.001. + +- `actor_rollout_ref.actor.kl_loss_type`: Support kl(k1), abs, mse(k2), low_var_kl(k3) and full. Appending "+" in the end (e.g., 'k1+' and 'k3+') would apply straight through to employ k2 for unbiased gradient estimation, regardless of the kl value estimation (see https://github.com/volcengine/verl/pull/2953#issuecomment-3162113848 for more details). How to calculate the kl divergence between actor and reference policy. See this blog post for detailed analysis: http://joschu.net/blog/kl-approx.html + +## Advanced Extensions + +### DrGRPO + +[Understanding R1-Zero-Like Training: A Critical Perspective](https://arxiv.org/pdf/2503.20783) claims there's optimization bias in GRPO, which leads to artificially longer responses, especially for incorrect outputs. This inefficiency stems from the way GRPO calculates advantages using group-based reward normalization. Instead, DrGRPO aggregates token-level losses by normalizing with a global constant to eliminate length bias. + +Configure the following to enable DrGRPO, with all other parameters the same as GRPO's: + +- `actor_rollout_ref.actor.loss_agg_mode`: "seq-mean-token-sum-norm", which turns off seq-dim averaging +- `actor_rollout_ref.actor.use_kl_loss`: Please set it to False for DrGRPO +- `algorithm.norm_adv_by_std_in_grpo`: False, which turns off standard deviation norm + +## Reference Example + +Qwen2.5 GRPO training log and commands: [link](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/qwen2-7b-fsdp2.log) + +```bash +bash examples/grpo_trainer/run_qwen3-8b.sh +``` + +For more reference performance, please see https://verl.readthedocs.io/en/latest/algo/baseline.html diff --git a/e2c/verl/docs/algo/opo.md b/e2c/verl/docs/algo/opo.md new file mode 100644 index 0000000000000000000000000000000000000000..338f3a762d9585c608af28cdf4e75837dbfe11e4 --- /dev/null +++ b/e2c/verl/docs/algo/opo.md @@ -0,0 +1,33 @@ +# On-Policy RL with Optimal Reward Baseline (OPO) + +Last updated: 06/02/2025. + +Loose on-policy constraints and suboptimal baselines in reinforcement learning often lead to training instability such as large policy shifts and entropy collapse. OPO addresses these challenges by using exact on-policy training with the theretically optimal reward baseline for advantage estimation. It achieves lower policy shifts and higher output entropy, encouraging more diverse and less repetitive responses. + +OPO uses group sampling to generate multiple outputs for each input like GRPO. Unlike group-based algorithms which typically use the mean reward of a group as its baseline, OPO employs a theoretically optimal baseline: the length-weighted reward of the group. It also omits the standard deviation normalization. By adopting these two key components, OPO enables the training of a single policy model with the objective of maximizing only the expected reward. For more detailes, refer to the original paper [On-Policy RL with Optimal Reward Baseline](https://arxiv.org/pdf/2505.23585). + +## Key Components + +- Exact On-Policy Training: always generates responses from the current policy, without using any pre-generated data or off-policy data. +- Optimal Reward Baseline: uses a length-weighted reward of the group as the baseline for normalizing the rewards. + +## Configuration + +To configure OPO within the framework, use the following YAML settings. These parameters are crucial for enabling exact on-policy training and activating the optimal reward baseline. + +```yaml +algorithm: + adv_estimator: opo # Use OPO for optimal reward baseline +data: + train_batch_size: 1024 +actor_rollout_ref: + actor: + ppo_mini_batch_size: 1024 # ppo_mini_batch_size should equal to train_batch_size to enable exact on-policy training + entropy_coeff: 0 # disable entropy regularization + use_kl_loss: False # disable kl regularization + kl_loss_coef: 0 +``` + +## Advanced Extensions + +OPO can also be extended to other algorithms like RLOO and Reinforce++. It just needs to adjust their configurations to enable exact on-policy training and incorporate the optimal length-weighted reward baseline with minimal modifications to their advantage estimation functions. diff --git a/e2c/verl/docs/algo/ppo.md b/e2c/verl/docs/algo/ppo.md new file mode 100644 index 0000000000000000000000000000000000000000..19302fcdde45bbc165e540667aa5b4eabf85844f --- /dev/null +++ b/e2c/verl/docs/algo/ppo.md @@ -0,0 +1,105 @@ +# Proximal Policy Optimization (PPO) + +Last updated: 06/19/2025. + +Proximal Policy Optimization (PPO) is a family of policy gradient methods for reinforcement learning, proposed by OpenAI in 2017. PPO strikes a balance between simplicity, stability, and performance, making it one of the most widely used algorithms in modern RL applications, including large-scale language model fine-tuning. + +Traditional policy gradient methods like REINFORCE or Vanilla Policy Gradient suffer from: + +- High variance and sample inefficiency. +- Instability due to large policy updates. + +PPO addresses this problem using a clipped surrogate objective that avoids overly large updates without requiring second-order derivatives. + +For more technical details regarding PPO, we suggest reading the introduction in the [OpenAI spinning up tutorial](https://spinningup.openai.com/en/latest/algorithms/ppo.html), and the paper [Proximal Policy Optimization Algorithms](https://arxiv.org/abs/1707.06347). + +## Key Components + +- Actor-Critic Architecture: PPO requires both an actor model (policy) and a critic model (value function). This differs from other algorithms like GRPO and RLOO that don't require a critic model. + +- Generalized Advantage Estimation (GAE): PPO uses GAE for computing advantage values, which helps reduce variance in policy gradient estimates while maintaining low bias. + +- Clipped Surrogate Objective: The core of PPO is implemented through the clipped surrogate objective function that limits policy updates. + +## Configuration + +Note that all configs containing `micro_batch_size` are used to configure the maximum sample or token count per forward or backward pass to avoid GPU OOMs, whose value should not change algorithmic/convergence behavior. + +Most critic configs are similar to those of actors. Note that the critic model is omitted from the figure below. + +![image](https://github.com/user-attachments/assets/16aebad1-0da6-4eb3-806d-54a74e712c2d) + +- `data.train_batch_size`: The global batch size of prompts used to generate a set of sampled trajectories/rollouts. The number of responses/trajectories is `data.train_batch_size * actor_rollout.ref.rollout.n` + +- `actor_rollout_ref.actor.ppo_mini_batch_size`: The set of sampled trajectories is split into multiple mini-batches with batch_size=ppo_mini_batch_size for PPO actor updates. The ppo_mini_batch_size is a global size across all workers + +- `actor_rollout_ref.critic.ppo_mini_batch_size`: The set of sampled trajectories is split into multiple mini-batches with batch_size=ppo_mini_batch_size for PPO critic updates. The ppo_mini_batch_size is a global size across all workers + +- `actor_rollout_ref.actor.clip_ratio`: The PPO clip range. Default to 0.2 + +- `actor_rollout_ref.actor.ppo_epochs`: Number of epochs for PPO updates on one set of sampled trajectories for actor + +- `critic.ppo_epochs`: Number of epochs for PPO updates on one set of sampled trajectories for critic. Defaults to `actor_rollout_ref.actor.ppo_epochs` + +- `algorithm.gemma`: discount factor + +- `algorithm.lam`: The lambda term that trades off between bias and variance in the GAE estimator + +- `algorithm.adv_estimator`: Support gae, grpo, reinforce_plus_plus, reinforce_plus_plus_baseline, rloo + +## Advanced Extensions + +### KL Divergence Control + +Options to prevent the policy from diverging too far from a reference policy. Two mechanisms are available: KL reward penalty and KL loss. For more technical details, see [Training language models to follow instructions with human feedback](https://arxiv.org/abs/2203.02155) + +Options to use KL loss for KL divergence control: + +- `actor_rollout_ref.actor.use_kl_loss`: to use kl loss in the actor. When used, we are not applying KL in the reward function. Default is False + +- `actor_rollout_ref.actor.kl_loss_coef`: The coefficient of kl loss. Default is 0.001. + +- `actor_rollout_ref.actor.kl_loss_type`: Support kl(k1), abs, mse(k2), low_var_kl(k3) and full. Appending "+" in the end (e.g., 'k1+' and 'k3+') would apply straight through to employ k2 for unbiased gradient estimation, regardless of the kl value estimation (see https://github.com/volcengine/verl/pull/2953#issuecomment-3162113848 for more details). How to calculate the kl divergence between actor and reference policy. See this blog post for detailed analysis: http://joschu.net/blog/kl-approx.html + +Options to use KL penalty in the reward: + +- `algorithm.use_kl_in_reward`: Whether to enable in-reward kl penalty. Default is False. + +- `algorithm.kl_penalty`: Support kl(k1), abs, mse(k2), low_var_kl(k3) and full. This defines the way to calculate the kl divergence between actor and reference policy. For specific options, refer to `kl_penalty` in core_algos.py. See this blog post for detailed analysis: http://joschu.net/blog/kl-approx.html + +- `algorithm.kl_ctrl.kl_coef`: The (initial) coefficient of in-reward kl_penalty. Default is 0.001. +- `algorithm.kl_ctrl.type`: 'fixed' for FixedKLController and 'adaptive' for AdaptiveKLController. +- `algorithm.kl_ctrl.horizon`: See source code of AdaptiveKLController for details. +- `algorithm.kl_ctrl.target_kl`: See source code of AdaptiveKLController for details. + +### Dual-clip PPO + +The Dual-Clip PPO introduces a approach by applying a lower bound to the policy ratio when the advantage is less than zero, when multiplied by a large raito, does not exceed a specified lower bound. + +![image](https://github.com/user-attachments/assets/fc232181-d8b0-4307-8dd2-4dc0a4c1c139) + +- `actor_rollout_ref.actor.clip_ratio_c`: lower bound of the value for Dual-clip PPO, defaults to 3.0 + +## Reference Example + +Qwen2.5 training log and commands: [link](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-0.5B-bsz256_2-prompt1024-resp512-0.567.log) + +```bash +bash run_gemma.sh + trainer.n_gpus_per_node=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + trainer.logger=console \ + critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + data.train_batch_size=256 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size=2 \ + critic.ppo_micro_batch_size=2 +``` + +Reference performance with verl v0.2: + +| Model | Method | Score | Link | +|-------------------------------|------------------|-------|------------------------------------------------------------------------------------------------| +| Qwen/Qwen2.5-0.5B-Instruct | pretrained model | 36.4 | [Qwen Blog](https://qwenlm.github.io/blog/qwen2.5-llm/) | +| Qwen/Qwen2.5-0.5B-Instruct | PPO | 56.7 | [PPO Command and Logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-0.5B-bsz256_2-prompt1024-resp512-0.567.log) | diff --git a/e2c/verl/docs/algo/spin.md b/e2c/verl/docs/algo/spin.md new file mode 100644 index 0000000000000000000000000000000000000000..bf48717935b518e283ee2db645cc350fc8228c6a --- /dev/null +++ b/e2c/verl/docs/algo/spin.md @@ -0,0 +1,179 @@ +# Recipe: Self-Play Fine-Tuning (SPIN) + +Last updated: 05/31/2025. + +`verl` provides a recipe inspired by the paper **"Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models"** (SPIN). SPIN is a language model finetuning algorithm that enables iterative self-improvement through a self-play mechanism inspired by game theory. + +**Core Idea:** Models learn by playing against themselves, reducing reliance on external preference datasets or stronger teacher models: + +1. **Synthetic Data Generation:** The current model generates responses, creating its own training data from previous iterations. +2. **Two-Player Game Setup:** A game involving two players acted by a single LLM. +3. **Iterative Training:** The model progressively improves by refining its policy, with each iteration's model becoming the opponent for the next iteration. + +Paper Authors: [Zixiang Chen](https://github.com/uclaml/SPIN)\*, [Yihe Deng](https://github.com/uclaml/SPIN)\*, [Huizhuo Yuan](https://scholar.google.com/citations?user=8foZzX4AAAAJ)\*, [Kaixuan Ji](https://scholar.google.com/citations?user=FOoKDukAAAAJ), [Quanquan Gu](https://web.cs.ucla.edu/~qgu/) + +[[Webpage](https://uclaml.github.io/SPIN/)] [[Huggingface](https://huggingface.co/papers/2401.01335)] [[Paper](https://arxiv.org/abs/2401.01335)] [[Original Implementation](https://github.com/uclaml/SPIN)] + +verl Implementation Authors: [Chendong Wang](https://cdwang96.github.io/), [Chenyang Zhao](https://github.com/zhaochenyang20) + +--- + +## Key Function (compute_online_dpo_loss) and Related works +SPIN (Chen et al., 2024) proposes an iterative self-play mechanism to fine-tune language models. In each iteration, SPIN's training objective, when using a logistic loss function, is equivalent to Direct Preference Optimization (DPO) loss (Rafailov et al., 2023). + +This `verl` recipe realizes SPIN's core concept by using DPO loss iteratively (Xu et al., 2023; Xiong et al., 2023; Snorkel AI, 2024). This means that in each iteration, we fine-tune the LLM using DPO loss for preference optimization. Notably, Xu et al. (2023) explored iterative preference optimization with pairwise cringe loss, while Xiong et al. (2023) discussed how to bridge theory and practice for RLHF under KL constraints using iterative training. The concept of iterative preference learning was also explored in online DPO (Guo et al., 2024), which focuses on direct alignment from online AI feedback. In online DPO, preference data is dynamically updated during training, allowing the model to learn from its own generated data. + +Specifically, we developed the **`compute_online_dpo_loss`** function and built this SPIN recipe on top of it. By incorporating online preference generation, this approach enables continuously refining language models without relying on fixed external preference datasets. + +**Reference Papers:** +* [Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models](https://arxiv.org/abs/2401.01335) (Chen et al., 2024) +* [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://arxiv.org/abs/2305.18290) (Rafailov et al., 2023) +* [Somethings are more cringe than others: Preference optimization with the pairwise cringe loss](https://arxiv.org/abs/2312.16682) (Xu et al., 2023) +* [Iterative preference learning from human feedback: Bridging theory and practice for rlhf under kl-constraint](https://arxiv.org/abs/2312.11456) (Xiong et al., 2023) +* [Snorkel-Mistral-PairRM-DPO](https://huggingface.co/snorkelai/Snorkel-Mistral-PairRM-DPO) (Snorkel AI, 2024) +* [Direct language model alignment from online ai feedback](https://arxiv.org/abs/2402.04792) (Guo et al., 2024) + + +## Our Online DPO Implementation + +Our `compute_online_dpo_loss` function adapts `verl`'s existing PPO infrastructure (based on `verl` v0.3.0.post1) for this iterative online DPO. Key aspects of our implementation include: + +* **No Critic:** Unlike PPO, we omit the value function critic. +* **Dynamic Reference Model:** An explicit reference policy (`ref_policy_wg`) is used for DPO loss. This reference model's weights can be periodically updated from the actor (`ref_update_freq`), providing a dynamic baseline. +* **Online Preference Generation:** The `compute_onlineDPO_pref` function (in `core_algos.py`) dynamically creates chosen/rejected pairs based on a reward source (e.g., rule-based ranking for math problems). +* **DPO Loss Integration:** We replace PPO's policy loss with our `compute_online_dpo_loss` (in `core_algos.py`) within the actor update (`dp_actor.py`), directly optimizing the policy using the generated preferences. +* **Iterative Training Orchestration:** The `SpinTrainer` (in `spin_trainer.py`) manages the entire self-play loop: generation, preference labeling, optional reference model updates, and policy updates, enabling continuous self-improvement aligned with SPIN's principles. + +--- +## Algorithm + +This recipe implements an Online algorithm adapted to the `verl` Reinforcement Learning framework, which provides an alternative to PPO for fine-tuning language models. + +**Online Loop:** Instead of maximizing a scalar reward signal in PPO, this approach directly optimizes the policy model to align with preference data generated *online* during training: + +1. **Generation:** The current model generates multiple responses for each prompt in a batch. +2. **Preference Labeling:** A function evaluates these generated responses to determine which one is preferred (chosen) and which is dispreferred (rejected). This can be done using a reward function or implicit ranking based on specific rules. (In this recipe, we use rule-based ranking on the math problem). +3. **Update:** This preference tuple (`prompt`, `chosen_response`, `rejected_response`) is used to update the actor model using `compute_online_dpo_loss`, comparing against a reference model. + +**Connection with SPIN:** +Instead of only using a fixed target data distribution, the online generation loop in step 2 will dynamically change the target data distribution by using a certain Preference Labeling method (rule-based ranking on the math problem by selecting the better one in this recipe). This explores the direction mentioned in SPIN's paper Section 7 about "dynamically changing target data distribution" to potentially elevate LLM performance beyond the fixed human-annotated data ceiling. + +--- + +## Reproduce the Experiment (Example Setup) + +The following steps outline how to set up the environment and run the SPIN recipe, based on the provided test log using GSM8K and Qwen2.5-3B-Instruct. + +1. **Setup Environment (Example using Docker):** + ```bash + # Start a container with GPU access and shared memory + docker run -it --name spin_test --gpus all \ + --shm-size=32g \ + --ipc=host \ + -v /path/to/host/.cache:/root/.cache \ + -e HF_TOKEN= \ + lmsysorg/sglang:latest \ + /bin/bash + + # Inside the container or on your host machine: + # Ensure /tmp is writable + mkdir -p /tmp + chmod 1777 /tmp + + # Install Python 3.10 (if not present) and venv + sudo apt update + sudo apt install -y python3.10 python3.10-venv tmux + python3 -m ensurepip --upgrade + + # Create and activate a virtual environment + python3 -m venv ~/.python/spin_env + source ~/.python/spin_env/bin/activate + + # Install uv (fast package installer) + python3 -m pip install uv + ``` + +2. **Install verl and Dependencies:** + ```bash + # Clone the verl repository and checkout the spin branch + cd ~ + git clone git@github.com:volcengine/verl.git && cd verl + + # Install flash-attn (handle potential build issues) + python3 -m uv pip install wheel packaging + python3 -m uv pip install flash-attn --no-build-isolation --no-deps + + # Install verl with sglang extras + python3 -m uv pip install -e ".[sglang]" + ``` + *Note: If `flash-attn` installation fails, try the manual steps again or consult its documentation.* + +3. **Login & Download Data/Model:** + ```bash + # Login to Weights & Biases (optional, for logging) + export WANDB_API_KEY= + # wandb login + + # Download the GSM8K dataset + python3 examples/data_preprocess/gsm8k.py --local_save_dir ~/data/gsm8k # Adjusted path + + # Download the base model (Example: Qwen2.5-3B-Instruct) + huggingface-cli download Qwen/Qwen2.5-3B-Instruct --local-dir $HOME/models/Qwen2.5-3B-Instruct + ``` + +4. **Configure:** + * Modify the configuration file (e.g., `config/spin_trainer.yaml` or the one specified in the run script) with correct paths to your downloaded model, data, desired hyperparameters (`dpo_beta`, learning rate, etc.), and distributed training settings (nodes, GPUs per node). + * Pay attention to `actor_rollout_ref.model_path`, `data` paths, `reward_model` config (if using one), and `trainer.ref_update_freq`. + +5. **Run Training:** + ```bash + # Set CUDA visible devices (adjust based on your hardware and config) + export CUDA_VISIBLE_DEVICES=0,1,2,3 + + # Launch the training script (e.g., test.sh or a custom script) + # Ensure test.sh points to the correct config and main script + bash recipe/spin/run_spin.sh + ``` + +--- + +## Configuration + +* The primary configuration is typically managed through a YAML file specified in the launch script (e.g., `config/spin_trainer.yaml`). +* Key configuration sections: + * `data`: Paths to training/validation prompt files, batch sizes, sequence lengths. + * `actor_rollout_ref`: Paths to the base model (used for actor and initial reference), FSDP settings, optimization parameters (learning rate, scheduler). + * `reward_model`: Configuration for the reward model used for online preference labeling (path, batch size, etc.). Can be omitted if using a simpler reward function. + * `algorithm`: DPO-specific hyperparameters like `dpo_beta`, `dpo_loss_type`. + * `trainer`: Distributed training settings (nodes, GPUs per node), logging (WandB), checkpointing frequency, and `ref_update_freq` (set > 0 to enable periodic reference model updates from the actor). + +--- + +## Key Files + +* `main_spin.py`: Main entry point using Hydra to load the config and launch the `SpinTrainer`. +* `spin_trainer.py`: Defines the `SpinTrainer` class, orchestrating the Online DPO training loop. +* `fsdp_workers.py`: Implements Ray workers (Actor, Reference) potentially using FSDP. +* `dp_actor.py`: Contains the actor class, including the DPO policy update logic. +* `core_algos.py`: Includes helper functions for `compute_online_dpo_loss` and `compute_onlineDPO_pref`. +* `config/spin_trainer.yaml` (or similar): Main Hydra configuration file for the recipe. +* `run_spin.sh` (or similar): Example bash script for launching a training run. +* `README.md`: This file. + +--- + +## Acknowledgement + +We sincerely thank the contribution and guidance from the `verl` community and advisors, including (adapted from SPPO): + +* [Zixiang Chen](https://sites.google.com/view/zxchen) +* [Yuhao Yang](https://github.com/yhyang201) +* [Yifan Zhang](https://github.com/yifanzhang-pro) +* [Yongan Xiang](https://github.com/BearBiscuit05) +* [Junrong Lin](https://github.com/ocss884) +* [Yuxuan Tong](https://github.com/tongyx361) +* [Guangming Shen](https://github.com/PeterSH6) +* [Biao He](https://www.linkedin.com/in/biao-he/) +* [Qingquan Song](https://qingquansong.github.io/) +* [Chenyang Zhao](https://zhaochenyang20.github.io/Chayenne/) +* [Quanquan Gu](https://web.cs.ucla.edu/~qgu/) diff --git a/e2c/verl/docs/algo/sppo.md b/e2c/verl/docs/algo/sppo.md new file mode 100644 index 0000000000000000000000000000000000000000..bf7c4e9e669329f7e8022da6b5b7f0901f578460 --- /dev/null +++ b/e2c/verl/docs/algo/sppo.md @@ -0,0 +1,52 @@ +# Recipe: Self-Play Preference Optimization (SPPO) + +Last updated: 05/28/2025. + +verl provides a community recipe implementation for the paper [Self-Play Preference Optimization for Language Model Alignment](https://arxiv.org/abs/2405.00675). SPPO can significantly enhance the performance of an LLM without strong external signals such as responses or preferences from GPT-4. It can outperform the model trained with iterative direct preference optimization (DPO), among other methods. SPPO is theoretically grounded, ensuring that the LLM can converge to the von Neumann winner (i.e., Nash equilibrium) under general, potentially intransitive preference, and empirically validated through extensive evaluations on multiple datasets. + +Paper Authors: [Yue Wu](https://yuewu.us/)\*, [Zhiqing Sun](https://www.cs.cmu.edu/~zhiqings/)\*, [Huizhuo Yuan](https://scholar.google.com/citations?user=8foZzX4AAAAJ)\*, [Kaixuan Ji](https://scholar.google.com/citations?user=FOoKDukAAAAJ), [Yiming Yang](https://www.cs.cmu.edu/~yiming/), [Quanquan Gu](https://web.cs.ucla.edu/~qgu/) + +verl Implementation Authors: [Yuhao Yang](https://github.com/yhyang201), [Chenyang Zhao](https://github.com/zhaochenyang20) + +[[Webpage](https://uclaml.github.io/SPPO/)] [[Huggingface](https://huggingface.co/papers/2405.00675)] [[Paper](https://arxiv.org/abs/2405.00675)][[Original Implementation](https://github.com/uclaml/SPPO)] + +## Reproduce the Experiment + +We evaluate the performance of SPPO on the MATH dataset. Starting from an initial score of 46.6 with Qwen2.5-7B-Instruct, we achieve a score of 65.6 after 20 epochs of training, placing our model approximately in the top 20 on the [MATH leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math). It's important to note that verl's internal evaluation metrics may not perfectly align with the official evaluation methodology for Qwen2.5-7B-Instruct. Therefore, for consistency and fair comparison, we report only the results based on verl's evaluation framework. + +``` +git clone git@github.com:volcengine/verl.git +cd verl +python3 -m uv pip install -e ".[sglang]" + +export WANDB_API_KEY= + +python3 examples/data_preprocess/math_dataset.py --local_dir ~/data/math +huggingface-cli download Qwen/Qwen2.5-7B-Instruct --local-dir $HOME/models/Qwen2.5-7B-Instruct + +export CUDA_VISIBLE_DEVICES=0,1,2,3 +bash recipe/sppo/run_qwen2.5-7b_rm.sh +``` + +Note that the installation would occasionally fail to install flash-attn. If this happens, you can install it manually by running: + +```bash +python3 -m uv pip install wheel +python3 -m uv pip install packaging +python3 -m uv pip install flash-attn --no-build-isolation --no-deps +``` + +## Acknowledgement + +We sincerely thank the contribution and guidance from: + +- [Yue Wu](https://yuewu.us/) +- [Chendong Wang](https://cdwang96.github.io/) +- [Yifan Zhang](https://github.com/yifanzhang-pro) +- [Yongan Xiang](https://github.com/BearBiscuit05) +- [Junrong Lin](https://github.com/ocss884) +- [Yuxuan Tong](https://github.com/tongyx361) +- [Guangming Shen](https://github.com/PeterSH6) +- [Biao He](https://www.linkedin.com/in/biao-he/) +- [Qingquan Song](https://qingquansong.github.io/) +- [Quanquan Gu](https://web.cs.ucla.edu/~qgu/) diff --git a/e2c/verl/docs/amd_tutorial/amd_build_dockerfile_page.rst b/e2c/verl/docs/amd_tutorial/amd_build_dockerfile_page.rst new file mode 100644 index 0000000000000000000000000000000000000000..fc462c17fbd8aab8aa57456b73bcf35e5aec5394 --- /dev/null +++ b/e2c/verl/docs/amd_tutorial/amd_build_dockerfile_page.rst @@ -0,0 +1,796 @@ +Getting started with AMD (ROCM Kernel) +===================================================== + +Last updated: 07/06/2025. + +Author: `Yusheng Su `_ + +Setup +----- + +If you run on AMD GPUs (MI300) with ROCM platform, you cannot use the previous quickstart to run verl. You should follow the following steps to build a docker and set ``RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES`` or ``RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES`` when starting ray in verl's RLHF training. + + +docker/Dockerfile.rocm +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + FROM "rlsys/rocm-6.3.4-patch:rocm6.3.4-numa-patch_ubuntu-22.04" + + SHELL ["/bin/bash", "-ceuxo", "pipefail"] + + ENV MAX_JOBS=512 + + ENV PATH="/usr/local/python3.12/bin:$PATH" + RUN ln -sf /usr/bin/python3.12 /usr/bin/python && \ + ln -sf /usr/bin/pip3.12 /usr/bin/pip + + ############################################ + RUN apt-get update + RUN apt-get install -y pkg-config liblzma-dev + ############################################ + + ########################################### + ##########Install TransformerEngine######## + ########################################### + WORKDIR /workspace/ + # transformer-engine install + # https://github.com/ROCm/TransformerEngine + RUN rm -rf TransformerEngine + RUN git clone --recursive https://github.com/ROCm/TransformerEngine.git + WORKDIR /workspace/TransformerEngine + git checkout 236178e5 + # git checkout bb061ade + # git checkout 864405c + ENV NVTE_FRAMEWORK=pytorch + ENV NVTE_ROCM_ARCH=gfx942 + ENV NVTE_USE_HIPBLASLT=1 + ENV NVTE_USE_ROCM=1 + # export CMAKE_PREFIX_PATH="/opt/rocm:/opt/rocm/hip:/usr/local:/usr:${CMAKE_PREFIX_PATH:-}" + ENV CMAKE_PREFIX_PATH="/opt/rocm:/opt/rocm/hip:/usr/local:/usr" + RUN MAX_JOBS=$(MAX_JOBS) pip install . -vvv + WORKDIR /workspace/ + ########################################### + ########################################### + ########################################### + + + + + + #################################################################################### + ################Install vllm - sglang require vllm 0.6.7 dependency################# + #################################################################################### + #### Require vllm 0.6.7 - checkout 113274a0 + WORKDIR /workspace/ + RUN rm -rf vllm + RUN pip uninstall -y vllm + # Refer to here (down-grade vllm to 0.6.3): https://docs.vllm.ai/en/v0.6.3/getting_started/amd-installation.html + RUN git clone https://github.com/ROCm/vllm.git + # git clone https://github.com/vllm-project/vllm.git + WORKDIR /workspace/vllm + RUN git checkout 113274a0 + ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" + #ENV MAX_JOBS=512 + ENV MAX_JOBS=${MAX_JOBS} + RUN pip install "boto3>=1.26.0" + RUN pip install setuptools_scm + # will add src into py. You can delete the repo + RUN python3 setup.py install + WORKDIR /workspace/ + #################################################################################### + #################################################################################### + #################################################################################### + + + + ########################################### + ############For hack docker################ + ########################################### + RUN pip install setuptools==75.8.0 + ########################################### + ########################################### + ########################################### + + + + ########################################### + ############build sgalng################### + ########################################### + # Set environment variables + ENV BASE_DIR=/sgl-workspace + ENV BUILD_TYPE=all + ENV SGL_REPO=https://github.com/sgl-project/sglang + ENV SGL_BRANCH=v0.4.6.post5 + ENV TRITON_REPO=https://github.com/ROCm/triton.git + ENV TRITON_COMMIT=improve_fa_decode_3.0.0 + ENV AITER_REPO=https://github.com/ROCm/aiter.git + ENV AITER_COMMIT=v0.1.2 + # v0.1.2 version - commit id: 9d11f47 + # ENV AITER_COMMIT=9d11f47 + ENV HIP_FORCE_DEV_KERNARG=1 + ENV HSA_NO_SCRATCH_RECLAIM=1 + ENV SGLANG_SET_CPU_AFFINITY=1 + ENV SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 + ENV NCCL_MIN_NCHANNELS=112 + ENV MOE_PADDING=1 + ENV VLLM_FP8_PADDING=1 + ENV VLLM_FP8_ACT_PADDING=1 + ENV VLLM_FP8_WEIGHT_PADDING=1 + ENV VLLM_FP8_REDUCE_CONV=1 + ENV TORCHINDUCTOR_MAX_AUTOTUNE=1 + ENV TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=1 + ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942" + ENV AMDGPU_TARGETS=gfx942 + ENV ROCM_ARCH=gfx942 + ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" + # Switch to working directory + WORKDIR /sgl-workspace + # Clean and create directory + RUN rm -rf /sgl-workspace && mkdir -p /sgl-workspace + + # Clone and build sglang + RUN git clone ${SGL_REPO} \ + && cd sglang \ + && git checkout ${SGL_BRANCH} || echo "Using default branch" \ + && cd sgl-kernel \ + && rm -f pyproject.toml \ + && mv pyproject_rocm.toml pyproject.toml \ + && python setup_rocm.py install \ + && cd .. \ + && if [ "$BUILD_TYPE" = "srt" ]; then \ + python -m pip --no-cache-dir install -e "python[srt_hip]"; \ + else \ + python -m pip --no-cache-dir install -e "python[all_hip]"; \ + fi \ + && cd /sgl-workspace \ + && cp -r /sgl-workspace/sglang /sglang \ + && python -m pip cache purge + + # Install common Python packages + RUN pip install IPython orjson python-multipart torchao pybind11 + # Rebuild Triton + RUN pip uninstall -y triton || true \ + && git clone ${TRITON_REPO} \ + && cd triton \ + && git checkout ${TRITON_COMMIT} \ + && cd python \ + && python3 setup.py install \ + && cd /sgl-workspace + # ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942 --amdgpu-lower-module-lds-strategy=1" + # ENV HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=gfx942" + + # Build aiter + #version: Commit 9d11f47 + # && git checkout ${AITER_COMMIT} \ + RUN pip uninstall -y aiter || true + RUN git clone ${AITER_REPO} \ + && cd aiter \ + && git checkout ${AITER_COMMIT} \ + && git submodule sync \ + && git submodule update --init --recursive \ + && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py install \ + && cd /sgl-workspace + + # Copy MI300X config + RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ + /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ + -type f -name '*MI300X*' | \ + xargs -I {} sh -c 'vf_config=$(echo "$1" | sed "s/MI300X/MI300X_VF/"); cp "$1" "$vf_config"' -- {} + + # Environment setup complete. + RUN echo "Environment setup complete." + + WORKDIR /workspace/ + ########################################### + ########################################### + ########################################### + + + + + + + ########################################### + ###############vllm v0.8.5################# + ########################################### + WORKDIR /workspace/ + + ENV VLLM_TARGET_DEVICE=rocm + ENV ROCM_PATH=/opt/rocm + ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev + # Find the repo path in: DockerFile/Dockerfile.rocm_yang + # RUN git clone https://github.com/RLFoundation/vllm-patch.git + RUN pip uninstall -y vllm || true + RUN rm -rf vllm-patch + RUN git clone https://github.com/RLFoundation/vllm-patch.git \ + && cd vllm-patch \ + && git checkout v0.8.5-sleep-numa \ + && rm -rf build/ dist/ *.egg-info \ + && ln -sf /opt/rocm/lib/libamdhip64.so /usr/lib/libamdhip64.so \ + && SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev PYTORCH_ROCM_ARCH="gfx90a;gfx942" MAX_JOBS=${MAX_JOBS} python3 setup.py install + # RUN SETUPTOOLS_SCM_PRETEND_VERSION=0.8.5.dev PYTORCH_ROCM_ARCH="gfx90a;gfx942" MAX_JOBS=${MAX_JOBS} python3 setup.py develop + WORKDIR /workspace/ + ########################################### + ########################################### + ########################################### + + + + + ######################################### + #### Install megatron-core############### + ######################################### + RUN pip uninstall -y megatron-core && \ + git clone https://github.com/yushengsu-thu/Megatron-LM-amd_version.git && \ + cd Megatron-LM-amd_version && \ + pip install -vvv -e . && \ + cd /workspace/ + ######################################### + ######################################### + ######################################### + + + + + ####################################### + ################apex################### + ####################################### + WORKDIR /workspace/ + RUN pip uninstall -y apex && \ + git clone git@github.com:ROCm/apex.git && \ + cd apex && \ + python setup.py install && \ + cd /workspace/ + ####################################### + ####################################### + ####################################### + + + ################################################################################ + ###########################Add torch_memory_saver############################### + ################################################################################ + # Set environment variables + ENV HIPCC_COMPILE_FLAGS_APPEND="--amdgpu-target=gfx90a;gfx942 -D__HIP_PLATFORM_AMD__" + ENV CFLAGS="-D__HIP_PLATFORM_AMD__" + ENV CXXFLAGS="-D__HIP_PLATFORM_AMD__" + RUN pip install "git+https://github.com/YangWang92/torch_memory_saver_numa.git@numa" + ################################################################################ + ################################################################################ + ################################################################################ + + + + ######################################## + ######Install ray####################### + ######################################## + # need to add this patch: https://github.com/ray-project/ray/pull/53531/files + RUN pip uninstall ray -y + RUN pip install "ray[data,train,tune,serve]>=2.47.0" + ######################################## + ######################################## + ######################################## + + + ########################################## + #######Install other dependencies######### + ########################################## + RUN pip install "tensordict==0.6.2" --no-deps && \ + pip install accelerate \ + codetiming \ + datasets \ + dill \ + hydra-core \ + liger-kernel \ + numpy \ + pandas \ + peft \ + "pyarrow>=15.0.0" \ + pylatexenc \ + torchdata \ + wandb \ + orjson \ + pybind11 + + WORKDIR /workspace/ + RUN git clone https://github.com/volcengine/verl.git && \ + cd verl && \ + pip install -e . + ########################################## + ########################################## + ########################################## + + WORKDIR /workspace/ + CMD ["/usr/bin/bash"] + + +Build the image: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + docker docker/build -t verl-rocm . + +Run the container +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Note: You can pull the docker from this DockerHub: [RLSys Foundation](https://hub.docker.com/u/yushengsuthu) +Pull the image: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + docker pull rlsys/verl:verl-0.4.1_ubuntu-22.04_rocm6.3.4-numa-patch_vllm0.8.5_sglang0.4.6.post4 + + docker tag rlsys/verl:verl-0.4.1_ubuntu-22.04_rocm6.3.4-numa-patch_vllm0.8.5_sglang0.4.6.post4 verl-rocm:latest + +Run the container +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +Optional: Running without root and with user permissions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + docker run --rm -it \ + --device /dev/dri \ + --device /dev/kfd \ + -p 8265:8265 \ + --group-add video \ + --cap-add SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --privileged \ + -v $HOME/.ssh:/root/.ssh \ + -v $HOME:$HOME \ + --shm-size 128G \ + -w $PWD \ + verl-rocm \ + /bin/bash + +(Optional): If you do not want to root mode and require assign yourself as the user +Please add ``-e HOST_UID=$(id -u)`` and ``-e HOST_GID=$(id -g)`` into the above docker launch script. + +Example +------- + +Due to to special setting in AMD (ROCM) torch, +1. If your ``ray>=2.45.0`` (default), you need to set ``RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES`` when starting ray in verl's RLHF training and add this [patch](https://github.com/ray-project/ray/pull/53531/files). +2. If your ``ray<2.45.0``, you need to set ``RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES`` when starting ray in verl's RLHF training. +Inference ``$ENGINE`` can be ``vllm`` or ``sglang``. We choose ``vllm`` as default in the following examples. + + + +PPO +~~~ + +.. code-block:: bash + + YOUR_PROJECT_NAME=r1-verl-ppo-upstream + YOUR_RUN_NAME=r1-training_ppo-upstream + # export HYDRA_FULL_ERROR=1 + + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + + # [ray] < 2.45.0 + #export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1 + + # [ray] >= 2.45.0 + export RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES=1 # Patch with https://github.com/ray-project/ray/pull/52794 + + GPUS_PER_NODE=8 + MODEL_PATH=Qwen/Qwen2.5-0.5B-Instruct + python3 examples/data_preprocess/gsm8k.py --local_save_dir data/gsm8k + python3 -c "import transformers; transformers.pipeline('text-generation', model='$MODEL_PATH')" + ENGINE=vllm #sglang + + PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=256 \ + data.val_batch_size=1312 \ + data.max_prompt_length=512 \ + data.max_response_length=256 \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=$ENGINE \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + critic.optim.lr=1e-5 \ + critic.model.path=$MODEL_PATH \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=console \ + trainer.project_name=$YOUR_PROJECT_NAME \ + trainer.experiment_name=$YOUR_RUN_NAME \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=$GPUS_PER_NODE \ + trainer.nnodes=1 \ + trainer.save_freq=10 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 #2>&1 | tee verl_demo.log + +GRPO +~~~~ + +.. code-block:: bash + + YOUR_PROJECT_NAME=r1-verl-grpo-upstream + YOUR_RUN_NAME=r1-training_grpo-upstream + # export HYDRA_FULL_ERROR=1 + # export FSDP_VERBOSE=1 + + #export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + + # [ray] < 2.45.0 + #export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1 + + # [ray] >= 2.45.0 + export RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES=1 # Patch with https://github.com/ray-project/ray/pull/52794 + + GPUS_PER_NODE=8 + MODEL_PATH=Qwen/Qwen2.5-0.5B-Instruct + # MODEL_PATH=Qwen/Qwen2-7B-Instruct + python3 examples/data_preprocess/gsm8k.py --local_save_dir data/gsm8k + python3 -c "import transformers; transformers.pipeline('text-generation', model='$MODEL_PATH')" + ENGINE=vllm #sglang + + python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.val_batch_size=1312 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.model.enable_gradient_checkpointing=Flase \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$ENGINE \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name=$YOUR_PROJECT_NAME \ + trainer.experiment_name=$YOUR_RUN_NAME \ + trainer.n_gpus_per_node=$GPUS_PER_NODE \ + trainer.val_before_train=False \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 + + + +Multi-node training: slurm with Docker/Podman container +--------------------------------------------------------------------------------------- + +If you want to run multi-node training with slurm, you can use the following script. + +.. note:: + 1. You need to use ``podman`` or ``docker`` in the following script. We will release the apptainer script later. + 2. If you want to use ``podman``, you just replace ``docker`` with ``podman`` in the following script. + +The script includes the following steps: + +1. SLURM Configuration +2. Environment Setup +3. Docker/Podman Container Setup +4. Ray Cluster Initialization +5. Data Preprocessing +6. Model Setup +7. Training Launch + + +slurm_script.sh +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + #!/bin/bash + + #SBATCH --job-name=verl-ray-on-slurm + #SBATCH --nodes=2 + #SBATCH --ntasks-per-node=2 + #SBATCH --mem=200G + #SBATCH --time=30-00:00:00 + #SBATCH --gpus-per-node=8 + #SBATCH --cpus-per-task=28 + #SBATCH --output=../verl_log/slurm-%j.out + #SBATCH --error=../verl_log/slurm-%j.err + #SBATCH --nodelist=gpu-[0,1] + + + # load necessary modules + ### Run this setup + # [Cluster]: Use docker + # docker pull docker.io/rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 + + + ########################################################################## + ###The following setting should be set in different project and cluster### + ########################################################################## + + ### Project + CONTAINER_NAME="multinode_verl_training" + IMG="verl.rocm" + DOCKERFILE="docker/Dockerfile.rocm" + # echo $PWD + verl_workdir="${HOME}/projects/verl_upstream" + export TRANSFORMERS_CACHE="${HOME}/.cache/huggingface" + export HF_HOME=$TRANSFORMERS_CACHE + + ### Cluster Network Setting + export NCCL_DEBUG=TRACE + export GPU_MAX_HW_QUEUES=2 + export TORCH_NCCL_HIGH_PRIORITY=1 + export NCCL_CHECKS_DISABLE=1 + # export NCCL_IB_HCA=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9 + export NCCL_IB_GID_INDEX=3 + export NCCL_CROSS_NIC=0 + export CUDA_DEVICE_MAX_CONNECTIONS=1 + export NCCL_PROTO=Simple + export RCCL_MSCCL_ENABLE=0 + export TOKENIZERS_PARALLELISM=false + export HSA_NO_SCRATCH_RECLAIM=1 + ########################################################################## + + ## Assign using GPUs + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + + ### For rocm and training script + # [ray] < 2.45.0 + #export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1 + + # [ray] >= 2.45.0 + export RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES=1 # Patch with https://github.com/ray-project/ray/pull/52794 + + + # Build and launch the Docker container + srun bash -c " + # Exit on any error + set -e + + # Clean up dangling images (images with tag) + docker image prune -f + + # Need to pull the docker first + docker pull rlsys/verl:verl-0.4.1_ubuntu-22.04_rocm6.3.4-numa-patch_vllm0.8.5_sglang0.4.6.post4 + + if ! docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "${IMG}"; then + echo \"Building ${IMG} image...\" + docker build -f \"${DOCKERFILE}\" -t \"${IMG}\" . + else + echo \"${IMG} image already exists, skipping build\" + fi + + # Removing old container if exists + docker rm \"${CONTAINER_NAME}\" 2>/dev/null || true + + # Checking network devices + ibdev2netdev + + # Launch the docker + docker run --rm -d \ + -e HYDRA_FULL_ERROR=1 \ + -e RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1 \ + -e RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES=1 \ + -e NCCL_DEBUG=${NCCL_DEBUG} \ + -e GPU_MAX_HW_QUEUES=${GPU_MAX_HW_QUEUES} \ + -e TORCH_NCCL_HIGH_PRIORITY=${TORCH_NCCL_HIGH_PRIORITY} \ + -e NCCL_CHECKS_DISABLE=${NCCL_CHECKS_DISABLE} \ + -e NCCL_IB_HCA=${NCCL_IB_HCA} \ + -e NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} \ + -e NCCL_CROSS_NIC=${NCCL_CROSS_NIC} \ + -e CUDA_DEVICE_MAX_CONNECTIONS=${CUDA_DEVICE_MAX_CONNECTIONS} \ + -e NCCL_PROTO=${NCCL_PROTO} \ + -e RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE} \ + -e TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM} \ + -e HSA_NO_SCRATCH_RECLAIM=${HSA_NO_SCRATCH_RECLAIM} \ + -e TRANSFORMERS_CACHE=${TRANSFORMERS_CACHE} \ + -e HF_HOME=${HF_HOME} \ + --network host \ + --device /dev/dri \ + --device /dev/kfd \ + --device /dev/infiniband \ + --group-add video \ + --cap-add SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --privileged \ + -v \${HOME}:\${HOME} \ + -v \${HOME}/.ssh:/root/.ssh \ + -w "${verl_workdir}" \ + --shm-size 128G \ + --name \"${CONTAINER_NAME}\" \ + \"${IMG}\" \ + tail -f /dev/null + + echo \"Container setup completed\" + " + # (Optional): If you do not want to root mode and require assign yuorself as the user + # Please add `-e HOST_UID=$(id -u)` and `-e HOST_GID=$(id -g)` into the above docker launch script. + + + + + + ### Ray launch the nodes before training + + # Getting the node names + nodes_array=($(scontrol show hostnames "$SLURM_JOB_NODELIST" | tr '\n' ' ')) + + head_node=${nodes_array[0]} + head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) + + # if we detect a space character in the head node IP, we'll + # convert it to an ipv4 address. This step is optional. + if [[ "$head_node_ip" == *" "* ]]; then + IFS=' ' read -ra ADDR <<<"$head_node_ip" + if [[ ${#ADDR[0]} -gt 16 ]]; then + head_node_ip=${ADDR[1]} + else + head_node_ip=${ADDR[0]} + fi + echo "IPV6 address detected. We split the IPV4 address as $head_node_ip" + fi + + port=6379 + ip_head=$head_node_ip:$port + export ip_head + echo "IP Head: $ip_head" + + # make sure we set environment variables before Ray initialization + + # Print out all env variables + printenv + + echo "Starting HEAD at $head_node" + srun --nodes=1 --ntasks=1 -w "$head_node" \ + docker exec "${CONTAINER_NAME}" \ + ray start --head --node-ip-address="$head_node_ip" --port=$port \ + --dashboard-port=8266 \ + --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & + # optional, though may be useful in certain versions of Ray < 1.0. + sleep 10 + + # number of nodes other than the head node + worker_num=$((SLURM_JOB_NUM_NODES - 1)) + + for ((i = 1; i <= worker_num; i++)); do + node_i=${nodes_array[$i]} + echo "Debug: Starting worker on node_i = ${node_i}" + if [ -z "$node_i" ]; then + echo "Error: Empty node name for worker $i" + continue + fi + echo "Starting WORKER $i at $node_i" + srun --nodes=1 --ntasks=1 -w "$node_i" \ + docker exec "${CONTAINER_NAME}" \ + ray start --address "$ip_head" --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & + sleep 5 + done + + + + + # Ray initlization test (See whether any error in the above execution) + echo "Testing Ray initialization in the slurm nodes..." + docker exec "${CONTAINER_NAME}" python3 -c ' + import ray + try: + ray.init(address="auto") + print("\n=== Ray Cluster Status ===") + print(f"Number of nodes: {len(ray.nodes())}") + for node in ray.nodes(): + print("Node: {}, Status: {}".format(node["NodeManagerHostname"], node["Alive"])) + # print(f"Node: {node}") + ray.shutdown() + print("Ray initialization successful!") + except Exception as e: + print(f"Ray initialization failed: {str(e)}") + ' + echo "=== Ray test completed ===" + ###### + + + + # Run data preprocessing + + echo "Starting data preprocessing..." + docker exec "${CONTAINER_NAME}" \ + python3 "examples/data_preprocess/gsm8k.py" "--local_save_dir" "../data/gsm8k" + + echo "Starting data preprocessing..." + docker exec "${CONTAINER_NAME}" \ + python3 "examples/data_preprocess/math_dataset.py" "--local_dir" "../data/math" + + train_files="../data/gsm8k/train.parquet" + val_files="../data/gsm8k/test.parquet" + + # Download and test model + echo "Loading model..." + docker exec "${CONTAINER_NAME}" \ + python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2-7B-Instruct')" + MODEL_PATH="Qwen/Qwen2-7B-Instruct" + + # Set model path after pipeline test + MODEL_PATH="Qwen/Qwen2.5-0.5B-Instruct" + + echo "== Data and model loading Done ==" + + echo "Start to train..." + + docker exec "${CONTAINER_NAME}" \ + python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2-7B-Instruct')" + MODEL_PATH="Qwen/Qwen2-7B-Instruct" + + + PYTHONUNBUFFERED=1 srun --overlap --nodes=${SLURM_NNODES} --ntasks=1 -w "$head_node" \ + docker exec "${CONTAINER_NAME}" \ + python3 -m verl.trainer.main_ppo \ + data.train_files=$train_files \ + data.val_files=$val_files \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.enable_gradient_checkpointing=False \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=$MODEL_PATH \ + critic.model.enable_gradient_checkpointing=False \ + critic.ppo_micro_batch_size_per_gpu=8 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.kl_ctrl.kl_coef=0.0001 \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example' \ + trainer.experiment_name='Qwen2.5-32B-Instruct_function_rm' \ + trainer.n_gpus_per_node=${SLURM_GPUS_PER_NODE} \ + trainer.val_before_train=False \ + trainer.nnodes=${SLURM_NNODES} \ + trainer.save_freq=-1 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 + + +Run slurm_script.sh +~~~~~~~~~~~~~~~~~~~~ +Just sbatch your slurm_script.sh + +.. code-block:: bash + + sbatch slurm_script.sh + diff --git a/e2c/verl/docs/amd_tutorial/amd_vllm_page.rst b/e2c/verl/docs/amd_tutorial/amd_vllm_page.rst new file mode 100644 index 0000000000000000000000000000000000000000..9c64755cb521ad531ad7e7fab2509b6d469dcbde --- /dev/null +++ b/e2c/verl/docs/amd_tutorial/amd_vllm_page.rst @@ -0,0 +1,105 @@ +verl performance tuning for AMD (ROCm Kernel) +===================================================== + +Last updated: 04/25/2025. + +Author: `Yang Wang `_ + +Patch vLLM to Enable Sleep Mode for AMD GPUs +-------------------------------------------------------------- + +By default, verl requires vLLM to enable sleep mode, which allows vLLM to offload GPU memory to CPU memory after rollout. However, this feature is still under review by the vLLM community. + +To enable vLLM's sleep mode, you can first use community patched code (from `this pull request `_) to build vLLM from the source code in the corresponding pull request. After the patch merged in vLLM main branch, you can directly install vLLM from the latest version. + +1. Clone the vLLM repository and build it with the following commands: + +.. code-block:: bash + + git clone -b sleep_amd https://github.com/HollowMan6/vllm.git + cd vllm + sudo ln -sf /opt/rocm/lib/libamdhip64.so /usr/lib/libamdhip64.so + VLLM_TARGET_DEVICE=rocm ROCM_PATH=/opt/rocm/ VLLM_GPU_LANG=HIP SETUPTOOLS_SCM_PRETEND_VERSION=0.8.4.dev python3 setup.py develop + +2. Additionally, make sure to use the ROCm version in your Docker image lager than or equal to ROCm 6.3.4, and we recommend to use ROCm 6.4.0 for better performance (see `this comment `_). + +After the upgrade, you can verify whether sleep mode is enabled by running the following test code (from `this comment `_). + +.. code-block:: python + + import torch + from vllm import LLM + + llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct", enable_sleep_mode=True) + + def run_inference(prompt): + outputs = llm.generate(prompt) + for output in outputs: + prompt = output.prompt + generated_text = output.outputs[0].text + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") + + + print("CUDA Memory Usage (after inference):") + torch.cuda.empty_cache() + print(f"{torch.cuda.memory_allocated()=}") + + run_inference("San Francisco is") + llm.sleep() + + print("CUDA Memory Usage (after sleep):") + torch.cuda.empty_cache() + print(f"{torch.cuda.memory_allocated()=}") + + llm.wake_up() + + print("CUDA Memory Usage (after wakeup):") + torch.cuda.empty_cache() + print(f"{torch.cuda.memory_allocated()=}") + + run_inference("Paris is") + +If sleep mode is enabled, you should see the memory usage reduce after sleep. + +After applying the vLLM patch and completing the installation, you can enable sleep mode in verl to reduce memory overhead. This allows verl to offload unused GPU memory during rollout, significantly lowering the memory footprint during long-context training or multi-node reinforcement learning. + + +Enable CUDA Graph and Bypass ROCm-related issues +-------------------------------------------------------------- + +Due to potential issues with CUDA graph capture in ROCm, we’ve found that vLLM’s CUDA graph feature cannot be enabled on multiple nodes in verl on AMD platforms with vLLM V1 mode. This leads to significantly slower rollout performance. + +Our investigation shows that ROCm may trigger an unexpected crash when attempting to capture large batches with CUDA graph. One workaround is to patch the LLM configuration (from `this commit `_). + +.. code-block:: python + + self.inference_engine = LLM( + model=model_path, + enable_sleep_mode=True, + tensor_parallel_size=tensor_parallel_size, + distributed_executor_backend="external_launcher", + dtype=config.dtype, + enforce_eager=config.enforce_eager, + gpu_memory_utilization=config.gpu_memory_utilization, + disable_custom_all_reduce=True, + disable_mm_preprocessor_cache=True, + limit_mm_per_prompt=limit_mm_per_prompt, + skip_tokenizer_init=False, + max_model_len=max_model_len, + load_format=load_format, + disable_log_stats=config.disable_log_stats, + max_num_batched_tokens=max_num_batched_tokens, + enable_chunked_prefill=config.enable_chunked_prefill, + enable_prefix_caching=True, + trust_remote_code=trust_remote_code, + # enable compilation config to bypass oom on rocm + # change depends on your GPU memory size + compilation_config={"cudagraph_capture_sizes": [1, 2, 4, 8, 16, 32, 64]}, + seed=config.get('seed', 0), + ) + +Then, you can choose to enable CUDA graph by setting the following environment variables (see `this page `_): + +.. code-block:: bash + + actor_rollout_ref.rollout.enforce_eager=False \ diff --git a/e2c/verl/docs/api/data.rst b/e2c/verl/docs/api/data.rst new file mode 100644 index 0000000000000000000000000000000000000000..1f6018bc9d809d4df026030c39dfdf6ae52d6e02 --- /dev/null +++ b/e2c/verl/docs/api/data.rst @@ -0,0 +1,61 @@ +Data interface +========================= + +Last updated: 05/19/2025 (API docstrings are auto-generated). + +DataProto is the interface for data exchange. + +The :class:`verl.DataProto` class contains two key members: + +- batch: a :class:`tensordict.TensorDict` object for the actual data +- meta_info: a :class:`Dict` with additional meta information + +TensorDict +~~~~~~~~~~~~ + +:attr:`DataProto.batch` is built on top of :class:`tensordict`, a project in the PyTorch ecosystem. +A TensorDict is a dict-like container for tensors. To instantiate a TensorDict, you must specify key-value pairs as well as the batch size. + +.. code-block:: python + + >>> import torch + >>> from tensordict import TensorDict + >>> tensordict = TensorDict({"zeros": torch.zeros(2, 3, 4), "ones": torch.ones(2, 3, 5)}, batch_size=[2,]) + >>> tensordict["twos"] = 2 * torch.ones(2, 5, 6) + >>> zeros = tensordict["zeros"] + >>> tensordict + TensorDict( + fields={ + ones: Tensor(shape=torch.Size([2, 3, 5]), device=cpu, dtype=torch.float32, is_shared=False), + twos: Tensor(shape=torch.Size([2, 5, 6]), device=cpu, dtype=torch.float32, is_shared=False), + zeros: Tensor(shape=torch.Size([2, 3, 4]), device=cpu, dtype=torch.float32, is_shared=False)}, + batch_size=torch.Size([2]), + device=None, + is_shared=False) + +One can also index a tensordict along its batch_size. The contents of the TensorDict can be manipulated collectively as well. + +.. code-block:: python + + >>> tensordict[..., :1] + TensorDict( + fields={ + ones: Tensor(shape=torch.Size([1, 3, 5]), device=cpu, dtype=torch.float32, is_shared=False), + twos: Tensor(shape=torch.Size([1, 5, 6]), device=cpu, dtype=torch.float32, is_shared=False), + zeros: Tensor(shape=torch.Size([1, 3, 4]), device=cpu, dtype=torch.float32, is_shared=False)}, + batch_size=torch.Size([1]), + device=None, + is_shared=False) + >>> tensordict = tensordict.to("cuda:0") + >>> tensordict = tensordict.reshape(6) + +For more about :class:`tensordict.TensorDict` usage, see the official tensordict_ documentation. + +.. _tensordict: https://pytorch.org/tensordict/overview.html + + +Core APIs +~~~~~~~~~~~~~~~~~ + +.. autoclass:: verl.DataProto + :members: to, select, union, make_iterator, concat diff --git a/e2c/verl/docs/api/single_controller.rst b/e2c/verl/docs/api/single_controller.rst new file mode 100644 index 0000000000000000000000000000000000000000..44ea366ffe4b12ce5293821877ce70a0073f2152 --- /dev/null +++ b/e2c/verl/docs/api/single_controller.rst @@ -0,0 +1,30 @@ +Single Controller interface +============================ + +Last updated: 05/27/2025 (API docstrings are auto-generated). + +The Single Controller provides a unified interface for managing distributed workers +using Ray or other backends and executing functions across them. +It simplifies the process of dispatching tasks and collecting results, particularly +when dealing with data parallelism or model parallelism. + + +Core APIs +~~~~~~~~~~~~~~~~~ + +.. autoclass:: verl.single_controller.Worker + :members: __init__, __new__, get_master_addr_port, get_cuda_visible_devices, world_size, rank + +.. autoclass:: verl.single_controller.WorkerGroup + :members: __init__, world_size + +.. autoclass:: verl.single_controller.ClassWithInitArgs + :members: __init__, __call__ + +.. autoclass:: verl.single_controller.ResourcePool + :members: __init__, world_size, local_world_size_list, local_rank_list + +.. autoclass:: verl.single_controller.ray.RayWorkerGroup + :members: __init__ + +.. autofunction:: verl.single_controller.ray.create_colocated_worker_cls \ No newline at end of file diff --git a/e2c/verl/docs/api/trainer.rst b/e2c/verl/docs/api/trainer.rst new file mode 100644 index 0000000000000000000000000000000000000000..abfa51f01a31606f436a95fde13770577b9ab540 --- /dev/null +++ b/e2c/verl/docs/api/trainer.rst @@ -0,0 +1,31 @@ +Trainer Interface +================================ + +Last updated: 06/08/2025 (API docstrings are auto-generated). + +Trainers drive the training loop. Introducing new trainer classes in case of new training paradiam is encouraged. + +.. autosummary:: + :nosignatures: + + verl.trainer.ppo.ray_trainer.RayPPOTrainer + + +Core APIs +~~~~~~~~~~~~~~~~~ + +.. autoclass:: verl.trainer.ppo.ray_trainer.RayPPOTrainer + :members: __init__, init_workers, fit + +.. automodule:: verl.utils.tokenizer + :members: hf_tokenizer + +.. automodule:: verl.trainer.ppo.core_algos + :members: agg_loss, kl_penalty, compute_policy_loss, kl_penalty + +.. automodule:: verl.trainer.ppo.reward + :members: load_reward_manager, compute_reward, compute_reward_async + +.. autoclass:: verl.workers.reward_manager.NaiveRewardManager + +.. autoclass:: verl.workers.reward_manager.DAPORewardManager diff --git a/e2c/verl/docs/api/utils.rst b/e2c/verl/docs/api/utils.rst new file mode 100644 index 0000000000000000000000000000000000000000..e15e3a5a32bdbb129a25d93b12e751385caa30b5 --- /dev/null +++ b/e2c/verl/docs/api/utils.rst @@ -0,0 +1,76 @@ +Utilities +============ + +Last updated: 05/19/2025 (API docstrings are auto-generated). + +This section documents the utility functions and classes in the VERL library. + +Python Functional Utilities +------------------------------ + +.. automodule:: verl.utils.py_functional + :members: append_to_dict + +File System Utilities +------------------------ + +.. automodule:: verl.utils.fs + :members: copy_to_local + +Tracking Utilities +--------------------- + +.. automodule:: verl.utils.tracking + :members: Tracking + +Metrics Utilities +--------------------- + +.. automodule:: verl.utils.metric + :members: reduce_metrics + +Checkpoint Management +------------------------ + +.. automodule:: verl.utils.checkpoint.checkpoint_manager + :members: find_latest_ckpt_path + +.. automodule:: verl.utils.checkpoint.fsdp_checkpoint_manager + :members: FSDPCheckpointManager + +Dataset Utilities +--------------------- + +.. automodule:: verl.utils.dataset.rl_dataset + :members: RLHFDataset, collate_fn + +Torch Functional Utilities +----------------------------- + +.. automodule:: verl.utils.torch_functional + :members: get_constant_schedule_with_warmup, masked_whiten, masked_mean, logprobs_from_logits + +Sequence Length Balancing +---------------------------- + +.. automodule:: verl.utils.seqlen_balancing + :members: get_reverse_idx, rearrange_micro_batches + +Ulysses Utilities +-------------------- + +.. automodule:: verl.utils.ulysses + :members: gather_outputs_and_unpad, ulysses_pad_and_slice_inputs + +FSDP Utilities +------------------ + +.. automodule:: verl.utils.fsdp_utils + :members: get_fsdp_wrap_policy, get_init_weight_context_manager, init_fn, load_fsdp_model_to_gpu, load_fsdp_optimizer, offload_fsdp_model_to_cpu, offload_fsdp_optimizer, + +Debug Utilities +------------------- + +.. automodule:: verl.utils.profiler + :members: log_gpu_memory_usage, GPUMemoryLogger + diff --git a/e2c/verl/docs/ascend_tutorial/ascend_profiling_en.rst b/e2c/verl/docs/ascend_tutorial/ascend_profiling_en.rst new file mode 100644 index 0000000000000000000000000000000000000000..04a77e0cca4f79ddee15098232fac8c5544e0a4c --- /dev/null +++ b/e2c/verl/docs/ascend_tutorial/ascend_profiling_en.rst @@ -0,0 +1,132 @@ +Data collection based on FSDP backend on Ascend devices(en) +========================================================================================== + +Last updated: 08/14/2025. + +This is a tutorial for data collection using the GRPO or DAPO algorithm +based on FSDP on Ascend devices. + +Configuration +------------- + +Leverage two levels of configuration to control data collection: + +1. **Global profiler control**: Use parameters in ``ppo_trainer.yaml`` to control the collection mode and steps. +2. **Role profile control**: Use parameters in each role's ``profile`` field to control the collection mode for each role. + +Global collection control +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use parameters in ppo_trainer.yaml to control the collection mode +and steps. + +- global_profiler: Control the ranks and mode of profiling + + - tool: The profiling tool to use, options are nsys, npu, torch, + torch_memory. + - steps: This parameter can be set as a list that has + collection steps, such as [2, 4], which means it will collect steps 2 + and 4. If set to null, no collection occurs. + - save_path: The path to save the collected data. Default is + "outputs/profile". + + +Role collection control +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In each role's ``profiler`` field, you can control the collection mode for that role. + +- enable: Whether to enable profiling for this role. +- all_ranks: Whether to collect data from all ranks. +- ranks: A list of ranks to collect data from. If empty, no data is collected. +- tool_config: Configuration for the profiling tool used by this role. + +Use parameters in each role's ``profiler.tool_config.npu`` to control npu profiler behavior: + +- level: Collection level—options are level_none, level0, level1, and + level2 + + - level_none: Disables all level-based data collection (turns off + profiler_level). + - level0: Collect high-level application data, underlying NPU data, + and operator execution details on NPU. + - level1: Extends level0 by adding CANN-layer AscendCL data and AI + Core performance metrics on NPU. + - level2: Extends level1 by adding CANN-layer Runtime data and AI + CPU metrics. + +- contents: A list of options to control the collection content, such as + npu, cpu, memory, shapes, module, stack. + + - npu: Whether to collect device-side performance data. + - cpu: Whether to collect host-side performance data. + - memory: Whether to enable memory analysis. + - shapes: Whether to record tensor shapes. + - module: Whether to record framework-layer Python call stack + information. + - stack: Whether to record operator call stack information. + +- analysis: Enables automatic data parsing. +- discrete: Whether to enable discrete mode. + + +Examples +-------- + +Disabling collection +~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + global_profiler: + steps: null # disable profile + +End-to-End collection +~~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + global_profiler: + steps: [1, 2, 5] + actor_rollout_ref: + actor: + profiler: + enable: True + all_ranks: True + tool_config: + npu: + discrete: False + # rollout & ref follow actor settings + + +Discrete Mode Collection +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + global_profiler: + steps: [1, 2, 5] + actor_rollout_ref: + actor: + profiler: + enable: True + all_ranks: True + tool_config: + npu: + discrete: True + # rollout & ref follow actor settings + + +Visualization +------------- + +Collected data is stored in the user-defined save_path and can be +visualized by using the `MindStudio Insight `_ tool. + +If the analysis parameter is set to False, offline parsing is required after data collection: + +.. code:: python + + import torch_npu + # Set profiler_path to the parent directory of the "localhost.localdomain___ascend_pt" folder + torch_npu.profiler.profiler.analyse(profiler_path=profiler_path) \ No newline at end of file diff --git a/e2c/verl/docs/ascend_tutorial/ascend_profiling_zh.rst b/e2c/verl/docs/ascend_tutorial/ascend_profiling_zh.rst new file mode 100644 index 0000000000000000000000000000000000000000..b4b4896f8b09083cec2ef1b45290c0935cecdf06 --- /dev/null +++ b/e2c/verl/docs/ascend_tutorial/ascend_profiling_zh.rst @@ -0,0 +1,119 @@ +Data collection based on FSDP backend on Ascend devices(zh) +==================================== + +在昇腾设备上基于FSDP后端进行数据采集 + +Last updated: 08/14/2025. + +这是一份在昇腾设备上基于FSDP后端使用GRPO或DAPO算法进行数据采集的教程。 + +配置 +---- + +使用两级profile设置来控制数据采集 + +- 全局采集控制:使用verl/trainer/config/ppo_trainer.yaml中的配置项控制采集的模式和步数, +- 角色profile控制:通过每个角色中的配置项控制等参数。 + +全局采集控制 +~~~~~~~~~~~~ + +通过 ppo_trainer.yaml 中的参数控制采集步数和模式: + +- global_profiler: 控制采集的rank和模式 + + - tool: 使用的采集工具,选项有 nsys、npu、torch、torch_memory。 + - steps: 此参数可以设置为包含采集步数的列表,例如 [2, 4],表示将采集第2步和第4步。如果设置为 null,则不进行采集。 + - save_path: 保存采集数据的路径。默认值为 "outputs/profile"。 + +角色profiler控制 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +在每个角色的 ``profiler`` 字段中,您可以控制该角色的采集模式。 + +- enable: 是否为此角色启用性能分析。 +- all_ranks: 是否从所有rank收集数据。 +- ranks: 要收集数据的rank列表。如果为空,则不收集数据。 +- tool_config: 此角色使用的性能分析工具的配置。 + +通过每个角色的 ``profiler.tool_config.npu`` 中的参数控制具体采集行为: + +- level: 采集级别—选项有 level_none、level0、level1 和 level2 + + - level_none: 禁用所有基于级别的数据采集(关闭 profiler_level)。 + - level0: 采集高级应用数据、底层NPU数据和NPU上的算子执行详情。 + - level1: 在level0基础上增加CANN层AscendCL数据和NPU上的AI Core性能指标。 + - level2: 在level1基础上增加CANN层Runtime数据和AI CPU指标。 + +- contents: 控制采集内容的选项列表,例如 + npu、cpu、memory、shapes、module、stack。 + + - npu: 是否采集设备端性能数据。 + - cpu: 是否采集主机端性能数据。 + - memory: 是否启用内存分析。 + - shapes: 是否记录张量形状。 + - module: 是否记录框架层Python调用栈信息。 + - stack: 是否记录算子调用栈信息。 + +- analysis: 启用自动数据解析。 +- discrete: 使用离散模式。 + +示例 +---- + +禁用采集 +~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + global_profiler: + steps: null # disable profile + +端到端采集 +~~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + global_profiler: + steps: [1, 2, 5] + actor_rollout_ref: + actor: + profiler: + enable: True + all_ranks: True + tool_config: + npu: + discrete: False + # rollout & ref follow actor settings + + +离散模式采集 +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + global_profiler: + steps: [1, 2, 5] + actor_rollout_ref: + actor: + profiler: + enable: True + all_ranks: True + tool_config: + npu: + discrete: True + # rollout & ref follow actor settings + + +可视化 +------ + +采集后的数据存放在用户设置的save_path下,可通过 `MindStudio Insight `_ 工具进行可视化。 + +如果analysis参数设置为False,采集之后需要进行离线解析: + +.. code:: python + + import torch_npu + # profiler_path请设置为"localhost.localdomain___ascend_pt"目录的上一级目录 + torch_npu.profiler.profiler.analyse(profiler_path=profiler_path) \ No newline at end of file diff --git a/e2c/verl/docs/ascend_tutorial/ascend_quick_start.rst b/e2c/verl/docs/ascend_tutorial/ascend_quick_start.rst new file mode 100644 index 0000000000000000000000000000000000000000..1c1a500bc56952d5a6165d957ea6c44cf46a488d --- /dev/null +++ b/e2c/verl/docs/ascend_tutorial/ascend_quick_start.rst @@ -0,0 +1,224 @@ +verl x Ascend +=================================== + +Last updated: 08/15/2025. + +我们在 verl 上增加对华为昇腾设备的支持。 + +硬件支持 +----------------------------------- + +Atlas 200T A2 Box16 + +Atlas 900 A2 PODc + +Atlas 800T A3 + + +安装 +----------------------------------- + +基础环境准备 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ++-----------+-------------+ +| software | version | ++-----------+-------------+ +| Python | == 3.10 | ++-----------+-------------+ +| CANN | == 8.1.RC1 | ++-----------+-------------+ +| torch | == 2.5.1 | ++-----------+-------------+ +| torch_npu | == 2.5.1 | ++-----------+-------------+ + +基础环境准备请参照这份 `文档 `_ 。 + +vllm & vllm-ascend +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +为了能够在 verl 中正常使用 vllm,需使用以下命令编译安装 vllm 和 vllm-ascend。请注意根据机器类型区分安装方式。 + +.. code-block:: bash + + # vllm + git clone -b v0.7.3 --depth 1 https://github.com/vllm-project/vllm.git + cd vllm + pip install -r requirements-build.txt + + # for Atlas 200T A2 Box16 + VLLM_TARGET_DEVICE=empty pip install -e . --extra-index https://download.pytorch.org/whl/cpu/ + + # for Atlas 900 A2 PODc + VLLM_TARGET_DEVICE=empty pip install -e . + +.. code-block:: bash + + # vllm-ascend + git clone -b v0.7.3.post1 --depth 1 https://github.com/vllm-project/vllm-ascend.git + cd vllm-ascend + export COMPILE_CUSTOM_KERNELS=1 + python setup.py install + +安装verl +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + git clone https://github.com/volcengine/verl.git + cd verl + pip install -r requirements-npu.txt + pip install -e . + +其他三方库说明 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ++--------------+---------------+ +| software | description | ++--------------+---------------+ +| transformers | v4.52.4 | ++--------------+---------------+ +| flash_attn | not supported | ++--------------+---------------+ +| liger-kernel | not supported | ++--------------+---------------+ + +1. 支持通过 transformers 使能 --flash_attention_2, transformers 需等于 4.52.4版本。 +2. 不支持通过 flash_attn 使能 flash attention 加速。 +3. 不支持 liger-kernel 使能。 +4. 针对 x86 服务器,需要安装 cpu 版本的 torchvision。 + +.. code-block:: bash + + pip install torchvision==0.20.1+cpu --index-url https://download.pytorch.org/whl/cpu + + +快速开始 +----------------------------------- +正式使用前,建议您通过对Qwen2.5-0.5B GRPO的训练尝试以检验环境准备和安装的正确性。 + +1.下载数据集并将数据集预处理为parquet格式,以便包含计算RL奖励所需的必要字段 + +.. code-block:: bash + + python3 examples/data_preprocess/gsm8k.py --local_save_dir ~/data/gsm8k + +2.执行训练 + +.. code-block:: bash + + set -x + + export VLLM_ATTENTION_BACKEND=XFORMERS + + python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=512 \ + data.max_response_length=128 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=5e-7 \ + actor_rollout_ref.model.use_remove_padding=False \ + actor_rollout_ref.actor.entropy_coeff=0.001 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 \ + trainer.device=npu $@ + +(可选) 设置MindSpeed训练后端指导 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +1. 参考 `MindSpeed README `_ 说明安装 MindSpeed 加速库。 + +2. 使能 verl worker 模型 ``strategy`` 配置为 ``megatron`` ,例如 ``actor_rollout_ref.actor.strategy=megatron``。 + +3. MindSpeed 自定义入参可通过 ``override_transformer_config`` 参数传入,例如对 actor 模型开启 FA 特性可使用 ``+actor_rollout_ref.actor.megatron.override_transformer_config.use_flash_attn=True``。 + +4. 更多特性信息可参考 `MindSpeed+verl 文档 `_ 。 + +支持现状 +----------------------------------- + +**表1** RL类算法 + ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| algorithm | model | actor.strategy | rollout.name | hardware | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| GRPO | Qwen2.5-7B-instruct | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| GRPO | Qwen2.5-32B-instruct | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| GRPO | Qwen2.5-VL-3B-instruct | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| GRPO | Qwen2.5-VL-7B-instruct | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| GRPO | Qwen2.5-VL-32B-instruct | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| GRPO | Qwen3-8B | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| GRPO | Qwen3-32B | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| DAPO | Qwen2.5-7B-instruct | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| DAPO | Qwen2.5-32B | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| DAPO | Qwen3-8B-base | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| DAPO | Qwen3-14B-base | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| DAPO | Qwen3-30B-A3B-base | FSDP | vllm-ascend | Atlas 200T A2 Box16 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| DAPO | Qwen3-30B-A3B | megatron | vllm-ascend | Atlas 800T A3 | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ +| PPO | Qwen3-8B | FSDP | vllm-ascend | Atlas 900 A2 PODc | ++-----------+-------------------------+-------------------+-------------------+--------------------------+ + +**表2** SFT类算法 + ++-----------+-------------------------+-------------------+----------------------+ +| algorithm | model | actor.strategy | hardware | ++-----------+-------------------------+-------------------+----------------------+ +| SFT-PEFT | Qwen3-8B | FSDP | Atlas 900 A2 PODc | ++-----------+-------------------------+-------------------+----------------------+ +| ReTool-SFT| Qwen2.5-7B-instruct | FSDP | Atlas 900 A2 PODc | ++-----------+-------------------------+-------------------+----------------------+ + + + +计划 +----------------------------------- + +查看 `roadmap `_ 获取更多特性的支持进度。 + + + +声明 +----------------------------------- +verl中提供的ascend支持代码皆为参考样例,如在生产环境中使用请通过官方正式途径沟通,谢谢。 diff --git a/e2c/verl/docs/conf.py b/e2c/verl/docs/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..1c780182c04f4ae9cc2f0c31286cbb5b8e6b37ae --- /dev/null +++ b/e2c/verl/docs/conf.py @@ -0,0 +1,106 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = "verl" +copyright = "2024 ByteDance Seed Foundation MLSys Team" +author = "Guangming Sheng, Chi Zhang, Yanghua Peng, Haibin Lin" + + +# -- General configuration --------------------------------------------------- +# The master toctree document. +master_doc = "index" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.autosectionlabel", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", +] +# Use Google style docstrings instead of NumPy docstrings. +napoleon_google_docstring = True +napoleon_numpy_docstring = False + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Add the JavaScript file +html_js_files = [ + "js/runllm-widget.js", + "js/resizable-sidebar.js", +] + +# Add custom CSS file for full-width layout +html_css_files = [ + "custom.css", +] + +exclude_patterns += ["README.md", "README_vllm0.7.md"] + +suppress_warnings = ["ref.duplicate", "ref.myst"] diff --git a/e2c/verl/docs/examples/config.rst b/e2c/verl/docs/examples/config.rst new file mode 100644 index 0000000000000000000000000000000000000000..534a49616f2cb6eabe0e780a6f624a039dd42889 --- /dev/null +++ b/e2c/verl/docs/examples/config.rst @@ -0,0 +1,673 @@ +.. _config-explain-page: + +Config Explanation +=================== + +Last updated: 06/18/2025. + +ppo_trainer.yaml for RL FSDP Backend +------------------------------------- + +Data +~~~~ + +.. code:: yaml + + data: + tokenizer: null + train_files: ~/data/rlhf/gsm8k/train.parquet + val_files: ~/data/rlhf/gsm8k/test.parquet + prompt_key: prompt + max_prompt_length: 512 + max_response_length: 512 + train_batch_size: 1024 + return_raw_input_ids: False # This should be set to true when the tokenizer between policy and rm differs + return_raw_chat: False + return_full_prompt: False + shuffle: True + filter_overlong_prompts: False + filter_overlong_prompts_workers: 1 + truncation: error + image_key: images + trust_remote_code: True + custom_cls: + path: null + name: null + +- ``data.train_files``: Training set parquet. Can be a list or a single + file. The program will read all files into memory, so it can't be too + large (< 100GB). The path can be either local path or HDFS path. For + HDFS path, we provide utils to download it to DRAM and convert the + HDFS path to local path. +- ``data.val_files``: Validation parquet. Can be a list or a single + file. +- ``data.prompt_key``: The field in the dataset where the prompt is + located. Default is 'prompt'. +- ``data.max_prompt_length``: Maximum prompt length. All prompts will be + left-padded to this length. An error will be reported if the length is + too long +- ``data.max_response_length``: Maximum response length. Rollout in RL + algorithms (e.g. PPO) generates up to this length +- ``data.train_batch_size``: Batch size sampled for one training + iteration of different RL algorithms. +- ``data.return_raw_input_ids``: Whether to return the original + input_ids without adding chat template. This is mainly used to + accommodate situations where the reward model's chat template differs + from the policy. It needs to be decoded first, then apply the RM's + chat template. If using a model-based RM, and the policy and RM + chat_templates are different, this flag needs to be set +- ``data.return_raw_chat``: Whether to return the original chat (prompt) + without applying chat template. +- ``data.return_full_prompt``: Whether to return the full prompt with chat template +- ``data.shuffle``: Whether to shuffle the data in the dataloader. +- ``data.filter_overlong_prompts``: Default don't filter. +- ``data.filter_overlong_prompts_workers``: For large-scale dataset, filtering + overlong prompts could be timeconsuming. You cat set the ``filter_overlong_prompts_workers`` + to use multiprocessing for speed up. Default to 1. +- ``data.truncation``: Truncate the input_ids or prompt length if they + exceed max_prompt_length. Default is 'error', not allow exceed the + max_prompt_length. The users should increase the max_prompt_length if + throwing the error. You can also set ``left``, ``right`` and ``middle``. + When ``middle`` is selected, the logic splits the allowed max length roughly in half + and keeps the head and tail of the sequence, effectively discarding the middle section. +- ``data.image_key``: The field in the multi-modal dataset where the image is + located. Default is 'images'. +- ``data.trust_remote_code``: If the remote tokenizer has python file, we can use this field to allow + using remote tokenizer. For example: moonshotai/Moonlight-16B-A3B-Instruct + +Customized Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Customized dataset extension is implemented for the SFT trainer and can be extended to other trainers with similar changes. + +.. code:: yaml + + custom_cls: + path: null + name: null + +- ``data.custom_cls.path``: The path to the file containing your customized dataset class. If not specified, pre-implemented dataset will be used. +- ``data.custom_cls.name``: The name of the dataset class within the specified file. + +Actor/Rollout/Reference Policy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + actor_rollout_ref: + hybrid_engine: True + model: + path: ~/models/deepseek-llm-7b-chat + external_lib: null + override_config: + model_config: {} + moe_config: # Megatron only, can adjust moe configuration + freeze_moe_router: False # Megatron only, can freeze moe router (no grad) + enable_gradient_checkpointing: False + enable_activation_offload: False + trust_remote_code: False + use_remove_padding: False + actor: + strategy: fsdp # This is for backward-compatibility + ppo_mini_batch_size: 256 + ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu + ppo_micro_batch_size_per_gpu: 8 + use_dynamic_bsz: False + ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length} + grad_clip: 1.0 + clip_ratio: 0.2 + entropy_coeff: 0.0 + use_kl_loss: False # True for GRPO + tis_imp_ratio_cap: -1 # set to positive values for Truncated Importance Sampling (requires setting `rollout.calculate_log_probs` as True) + use_torch_compile: True # False to disable torch compile + kl_loss_coef: 0.001 # for grpo + kl_loss_type: low_var_kl # for grpo + ppo_epochs: 1 + data_loader_seed: null + shuffle: False + ulysses_sequence_parallel_size: 1 # sp size + optim: + lr: 1e-6 + lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime + min_lr_ratio: 0.0 # only used with cosine lr scheduler, default to 0.0 + num_cycles: 0.5 # only used with cosine lr scheduler, default to 0.5 + warmup_style: constant # select from constant/cosine + total_training_steps: -1 # must be override by program + fsdp_config: + wrap_policy: + # transformer_layer_cls_to_wrap: None + min_num_params: 0 + param_offload: False + optimizer_offload: False + fsdp_size: -1 + checkpoint: + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + # For more flexibility, you can specify the contents to load from the checkpoint. + load_contents: ${actor_rollout_ref.actor.checkpoint.save_contents} + ref: + fsdp_config: + param_offload: False + wrap_policy: + # transformer_layer_cls_to_wrap: None + min_num_params: 0 + log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu + log_prob_micro_batch_size_per_gpu: 16 + log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} + ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size} # sp size + rollout: + name: vllm + temperature: 1.0 + top_k: -1 # 0 for hf rollout, -1 for vllm rollout + top_p: 1 + prompt_length: ${data.max_prompt_length} # not use for opensource + response_length: ${data.max_response_length} + # for vllm rollout + dtype: bfloat16 # should align with FSDP + gpu_memory_utilization: 0.5 + ignore_eos: False + enforce_eager: True + free_cache_engine: True + load_format: dummy_dtensor + tensor_model_parallel_size: 2 + max_num_batched_tokens: 8192 + max_num_seqs: 1024 + log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu + log_prob_micro_batch_size_per_gpu: 16 + log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} + # for hf rollout + do_sample: True + engine_kwargs: # inference engine parameters, please refer vllm/sglang official doc for detail + vllm: {} + sglang: {} + + n: 1 # for each prompt, sample n responses (i.e. num sample times). set it to values > 1 for grpo, rloo + calculate_log_probs: False # set to True for computing log probs via rollouts + val_kwargs: + # sampling parameters for validation + top_k: -1 # 0 for hf rollout, -1 for vllm rollout + top_p: 1.0 + temperature: 0 + n: 1 + do_sample: False # default eager for validation + + agent: + custom_async_server: # Use custom async server implementation for rollout + path: null + name: null + +**Common config for actor, rollout and reference model** + +- ``actor_rollout_ref.hybrid_engine``: Whether it's a hybrid engine, + currently only supports hybrid engine +- ``actor_rollout_ref.model.path``: Huggingface model path. This can be + either local path or HDFS path. For HDFS path, we provide utils to + download it to DRAM and convert the HDFS path to local path. +- ``actor_rollout_ref.model.external_libs``: Additional Python packages + that need to be imported. Used to register models or tokenizers into + the Huggingface system. +- ``actor_rollout_ref.model.override_config``: Used to override some of + the model's original configurations, mainly dropout +- ``actor_rollout_ref.model.enable_gradient_checkpointing``: FSDP only, decide + Whether to enable gradient checkpointing for the actor, + Megatron uses recompute options in ``override_transformer_config`` to set this +- ``actor_rollout_ref.model.enable_activation_offload``: Whether to enable + activation offloading for the actor +- ``actor_rollout_ref.model.trust_remote_code``: Whether to enable loading + a remote code model +- ``actor_rollout_ref.model.use_fused_kernels``: Whether to use fused + kernels in the model. If set to True, the following parameters will be + used. + - ``actor_rollout_ref.model.fused_kernel_options.impl_backend``: The + implementation backend for fused kernels. Options: "triton" or + "torch". Default is "torch". + While in megatron, we only support "triton" as the + implementation backend, so there is no need for this option. +- ``actor_rollout_ref.model.use_remove_padding``: Whether to use remove + padding in the model. If set to True, the model will remove padding + tokens in the input_ids and response_ids. This helps a lot in improving model running efficiency. + +**Actor model** + +- ``actor_rollout_ref.actor.strategy``: fsdp or megatron. In this + example, we use fsdp backend. + +- ``actor_rollout_ref.actor.ppo_mini_batch_size``: One sample is split + into multiple sub-batches with batch_size=ppo_mini_batch_size for PPO + updates. The ppo_mini_batch_size is a global num across all workers/gpus + +- ``actor_rollout_ref.actor.ppo_micro_batch_size``: [Will be deprecated, use ppo_micro_batch_size_per_gpu] + Similar to gradient accumulation, the micro_batch_size_per_gpu for one forward pass, + trading speed for GPU memory. The value represent the global view. + +- ``actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu``: Similar to gradient + accumulation, the micro_batch_size_per_gpu for one forward pass, trading speed + for GPU memory. The value represent the local num per gpu. + +- ``actor_rollout_ref.actor.grad_clip``: Gradient clipping for actor + updates +- ``actor_rollout_ref.actor.use_kl_loss``: to use kl loss in actor. When used, we are not applying KL in the reward function. + +- ``actor_rollout_ref.actor.clip_ratio``: PPO clip ratio + +- ``actor_rollout_ref.actor.use_torch_compile``: Whether to use torch compile in actor + +- ``actor_rollout_ref.actor.entropy_coeff``: The weight of entropy when + calculating PPO loss. The default value is changed to 0.0 since v0.3.x + +- ``actor_rollout_ref.actor.ppo_epochs``: Number of epochs for PPO + updates on one set of sampled data + +- ``actor_rollout_ref.actor.data_loader_seed``: From torch 2.6.0 Megatron backend can get wrong seed generated by pytorch + between cp ranks and cause misalignment between data on these ranks, so we shall manually set the seed to avoid hanging + issue. if ``actor_rollout_ref.actor.shuffle`` is not null, this must be set. + +- ``actor_rollout_ref.actor.shuffle``: Whether to shuffle data when + there are multiple epochs + +- ``actor_rollout_ref.actor.optim``: Actor's optimizer parameters + +- ``actor_rollout_ref.actor.fsdp_config``: FSDP config for actor + training + + - ``wrap_policy``: FSDP wrap policy. By default, it uses Huggingface's + wrap policy, i.e., wrapping by DecoderLayer + + - No need to set transformer_layer_cls_to_wrap, so we comment it. + + - ``*_offload``: Whether to enable parameter, gradient and optimizer + offload + + - Trading speed for GPU memory. + +- ``actor_rollout_ref.actor.use_kl_loss``: Whether to enable kl loss. Default is False. + +- ``actor_rollout_ref.actor.kl_loss_coef``: The coefficient of kl loss. Default is 0.001. + +- ``actor_rollout_ref.actor.kl_loss_type``: Support ``kl`` (``k1``), ``abs``, ``mse`` (``k2``), ``low_var_kl`` (``k3``) and ``full``. Appending ``+`` in the end (e.g., ``k1+`` and ``k3+``) would use straight-through to employ ``k2`` for unbiased gradient estimation, regardless of the kl value estimation (see https://github.com/volcengine/verl/pull/2953#issuecomment-3162113848 for more details). How to calculate the kl divergence between actor and reference policy. For specific options, refer to `kl_penalty()` in `core_algos.py `_ . See this blog post for detailed analysis: http://joschu.net/blog/kl-approx.html + +- ``actor_rollout_ref.actor.checkpoint``: The configurations of checkpoint function in actor + + - ``save_contents``: The contents to save in the checkpoint. By default, we save model, optimizer and extra information in the checkpoint. + The extra information includes Rng states currently, FSDP supported lr_scheduler, and Megatron opt_param_scheduler will coming soon. + We do not store hf_model in checkpoint by default, but we provide a tool in ``scripts/model_merge.py`` to convert checkpoint format to hf format. + + - ``load_contents``: The contents to load in the checkpoint, you can specify different checkpoint loading contents. By default, it is the same with ``save_checkpoint``. + +**Reference Model** + +Reference model will be enabled when ``actor.use_kl_loss`` or/and ``algorithm.use_kl_in_reward`` is/are True. + +- ``actor_rollout_ref.ref``: FSDP config same as actor. **For models + larger than 7B, it's recommended to turn on offload for ref by + default** + +- ``actor_rollout_ref.ref.log_prob_micro_batch_size``: [Will be deprecate, use log_prob_micro_batch_size_per_gpu] + The batch size for one forward pass in the computation of ``ref_log_prob``. The value represent the global num. + +- ``actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu``: The batch size + for one forward pass in the computation of ``ref_log_prob``. The value represent the local num per gpu. + +**Rollout Model** + +- ``actor_rollout_ref.rollout.name``: hf/vllm/sglang. + +- Rollout (Auto-regressive) parameters. The key should be equal to the + property name in vLLM's ``SamplingParams``. + + - ``temperature``, ``top_k``, ``top_p`` and others: Sampling + parameters in ``SamplingParams``. + +- ``actor_rollout_ref.rollout.dtype``: Rollout model parameters type. This should be align with + the actor model parameter type in FSDP/Megatron backend. + +- ``actor_rollout_ref.rollout.gpu_memory_utilization``: + + - For vLLM v0.7.0 and later: The fraction of **total** GPU memory to be used for the vLLM instance. + - For SGLang: Corresponding to ``mem_fraction_static``, the fraction of the free GPU memory used for **static** memory like model weights and KV cache. + +- ``actor_rollout_ref.rollout.tensor_model_parallel_size``: TP size for rollout. Only effective + for vllm. + +- ``actor_rollout_ref.rollout.log_prob_micro_batch_size``: [Will be deprecate, use log_prob_micro_batch_size_per_gpu] + The batch size for one forward pass in the computation of ``log_prob``. The value represent the global num. + +- ``actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu``: Micro batch size per gpu (The batch size for + one forward pass) for recalculating ``log_prob``. The value represent the local num per gpu. + +- ``actor_rollout_ref.rollout.do_sample``: Whether to sample during training rollout. If set to False, the rollout model + will perform greedy sampling. + +- ``actor_rollout_ref.rollout.val_kwargs```: Sampling parameters used specifically during validation. + + - ``top_k``: Top-k sampling parameter. Default to -1 for vLLM rollout or 0 for HF rollout. + - ``top_p``: Top-p sampling parameter. Default is 1.0 (disabled). + - ``temperature``: Sampling temperature. Default is 0 (deterministic greedy). + - ``n``: Number of responses to generate during validation. Default is 1. + - ``do_sample``: Whether to use sampling during validation. Default is False for + deterministic outputs. When set to True, the rollout will use the ``actor_rollout_ref.rollout.val_kwargs`` parameters + (top_k, top_p, temperature) to control the sampling behavior. + +- ``actor_rollout_ref.rollout.engine_kwargs.vllm``: extra vllm engine args, please refer vllm official doc for detail + +- ``actor_rollout_ref.rollout.engine_kwargs.sglang``: extra sglang engine args, please refer sglang official doc for detail + +- ``actor_rollout_ref.rollout.ignore_eos``: Whether to ignore the EOS + token and continue generating tokens after the EOS token is generated. + +- ``actor_rollout_ref.rollout.free_cache_engine``: Offload the KVCache + after rollout generation stage. Default is True. When set to True, + for vllm v0.5.4 and v0.6.3, we need to disable the usage of CUDAGraph + (set ``enforce_eager`` to True.) + +- ``actor_rollout_ref.rollout.enforce_eager``: Whether to use CUDAGraph + in vLLM generation. Default set to True to disable CUDAGraph. + +- ``actor_rollout_ref.rollout.load_format``: Which weight loader to use + to load the actor model weights to the rollout model. + + - ``auto``: Use Megatron weight loader. + - ``megatron``: Use Megatron weight loader. Deployed with Megatron + backend. The input model ``state_dict()`` is already partitioned + along TP dimension and already gathered along PP dimension. This + weight loader requires that the Rollout model and Actor model's + parameters shape and name should be identical. + - ``dtensor``: Default solution when using Huggingface weight loader. + Deployed with FSDP backend and the state_dict_type is + ``StateDictType.SHARDED_STATE_DICT``. Recommend to use this weight + loader + - ``hf``: Use Huggingface weight loader. Deployed with FSDP backend + and the state_dict_type is ``StateDictType.FULL_STATE_DICT``. This + solution doesn't need to rewrite the weight loader for each model + implemented in vLLM but it results in larger peak memory usage. + - ``dummy_hf``, ``dummy_megatron``, ``dummy_dtensor``: Random + initialization. + +.. note:: **NOTED**: In this config field, users only need to select from ``dummy_megatron``, ``dummy_dtensor``, ``dummy_hf`` for rollout initialization and our hybrid engine will select the corresponding weight loader (i.e., ``megatron``, ``dtensor``, ``hf``) during actor/rollout weight synchronization. + + +Megatron Optimizer and Optimizer Parameter Scheduler +____________________________________________________ + +.. code:: yaml + + optim: + optimizer: adam + lr: 1e-6 + clip_grad: 1.0 + total_training_steps: -1 # must be override by program + lr_warmup_init: 0.0 # initial learning rate for warmup, default to 0.0 + lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime + lr_decay_steps: null + lr_decay_style: constant # select from constant/linear/cosine/inverse_square_root + min_lr: 0.0 # minimum learning rate, default to 0.0 + weight_decay: 0.01 + weight_decay_incr_style: constant # select from constant/linear/cosine + lr_wsd_decay_style: exponential # select from constant/exponential/cosine + lr_wsd_decay_steps: null + use_checkpoint_opt_param_scheduler: False # use checkpoint optimizer parameter scheduler + + +Notice that there are some differences in APIs between Megatron optimizer and FSDP optimizer. + +- Megatron optimizer scheduler names the period after lr_warmup as lr_decay_steps, so the ``warmup_style`` actually means the style of lr decay after warmup. +- Megatron optimizer also support weight decay decay mechanism +- ``use_checkpoint_opt_param_scheduler`` determines whether to use the checkpoint optimizer parameter scheduler. If set to True, the optimizer parameter scheduler will be saved in the checkpoint and loaded from the checkpoint during resuming training. + +For learning rate decay, original Megatron pretrain default option of ``lr_decay_style`` is ``linear``, +meaning that the learning rate will be linearly decayed from the initial learning rate to ``min_lr`` within the +``lr_decay_steps``. However, in verl, to align with FSDP's default behavior, we set the default +``lr_decay_style`` to ``constant``, meaning that the learning rate will be kept constant after the warmup stage. + + +Critic Model +~~~~~~~~~~~~ + +Most parameters for Critic are similar to Actor Model. + +Reward Model +~~~~~~~~~~~~ + +.. code:: yaml + + reward_model: + enable: False + model: + input_tokenizer: ${actor_rollout_ref.model.path} # set this to null if the chat template is identical + path: ~/models/Anomy-RM-v0.1 + external_lib: ${actor_rollout_ref.model.external_lib} + trust_remote_code: False + fsdp_config: + min_num_params: 0 + param_offload: False + micro_batch_size_per_gpu: 16 + max_length: null + reward_manager: naive + +- ``reward_model.enable``: Whether to enable reward model. If False, we + compute the reward only with the user-defined reward functions. In + GSM8K and Math examples, we disable reward model. For RLHF alignment + example using full_hh_rlhf, we utilize reward model to assess the + responses. If False, the following parameters are not effective. +- ``reward_model.model`` + + - ``input_tokenizer``: Input tokenizer. If the reward model's chat + template is inconsistent with the policy, we need to first decode to + plaintext, then apply the rm's chat_template. Then score with RM. If + chat_templates are consistent, it can be set to null. + - ``path``: RM's HDFS path or local path. Note that RM only supports + AutoModelForSequenceClassification. Other model types need to define + their own RewardModelWorker and pass it from the code. + - ``trust_remote_code``: Whether to enable loading a remote code model, + default to False. +- ``reward_model.reward_manager``: Reward Manager. This defines the mechanism + of computing rule-based reward and handling different reward sources. Default + is ``naive``. If all verification functions are multiprocessing-safe, the reward + manager can be set to ``prime`` for parallel verification. + +Customized Reward Function +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + custom_reward_function: + path: null + name: compute_score + +- ``custom_reward_function.path``: The path to the file containing your customized reward function. If not specified, pre-implemented reward functions will be used. +- ``custom_reward_function.name`` (Optional) : The name of the reward function within the specified file. Default is 'compute_score'. + +Algorithm +~~~~~~~~~ + +.. code:: yaml + + algorithm: + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + use_kl_in_reward: False + kl_penalty: kl # how to estimate kl divergence + kl_ctrl: + type: fixed + kl_coef: 0.005 + horizon: 10000 + target_kl: 0.1 + +- ``gamma``: discount factor +- ``lam``: Trade-off between bias and variance in the GAE estimator +- ``adv_estimator``: Support ``gae``, ``grpo``, ``reinforce_plus_plus``, ``reinforce_plus_plus_baseline``, ``rloo``, ``rloo_vectorized``, ``grpo_vectorized`` +- ``use_kl_in_reward``: Whether to enable in-reward kl penalty. Default is False. +- ``kl_penalty``: Support ``kl``, ``abs``, ``mse``, ``low_var_kl`` and ``full``. How to + calculate the kl divergence between actor and reference policy. For + specific options, refer to `kl_penalty()` in `core_algos.py `_ . +- ``kl_ctrl``: Config for in-reward kl_penalty controller + - ``kl_coef``: The (initial) coefficient of in-reward kl_penalty. Default is 0.001. + - ``type``: 'fixed' for FixedKLController and 'adaptive' for AdaptiveKLController. + - ``horizon`` and ``target_kl``: See source code of AdaptiveKLController for details. + +Trainer +~~~~~~~ + +.. code:: yaml + + trainer: + total_epochs: 30 + project_name: verl_examples + experiment_name: gsm8k + logger: ['console', 'wandb'] + log_val_generations: 0 + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + val_before_train: True + test_freq: 2 + critic_warmup: 0 + default_hdfs_dir: null # hdfs checkpoint path + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} # local checkpoint path + resume_mode: auto # or disable or resume_path if resume_from_path is set + resume_from_path: null + remove_previous_ckpt_in_save: False + del_local_ckpt_after_load: False + ray_wait_register_center_timeout: 300 + +- ``trainer.total_epochs``: Number of epochs in training. +- ``trainer.project_name``: For wandb, swanlab, mlflow +- ``trainer.experiment_name``: For wandb, swanlab, mlflow +- ``trainer.logger``: Support console and wandb, swanlab, mlflow, tensorboard, trackio +- ``trainer.log_val_generations``: The number of logged generation during validation (default ``0``) +- ``trainer.nnodes``: Number of nodes used in the training. +- ``trainer.n_gpus_per_node``: Number of GPUs per node. +- ``trainer.save_freq``: The frequency (by iteration) to save checkpoint + of the actor and critic model. +- ``trainer.val_before_train``: Whether to run validation before training. +- ``trainer.test_freq``: The validation frequency (by iteration). +- ``trainer.critic_warmup``: The number of iteration to train the critic + model before actual policy learning. +- ``trainer.resume_mode``: The mode of resuming training. Support + ``disable``, ``auto`` and ``resume_path``. If set to ``auto`` as default, the + program will automatically resume from the latest checkpoint in the + ``default_local_dir``. If set to ``resume_path``, the program will resume + from the path specified in ``resume_from_path``. +- ``trainer.resume_from_path``: The path to resume training from. Only + effective when ``resume_mode`` is set to ``resume_path``. +- ``trainer.remove_previous_ckpt_in_save``: Whether to remove previous + checkpoints in the save directory. Default is False. +- ``trainer.del_local_ckpt_after_load``: Whether to delete local + checkpoints after loading them. Default is False. +- ``trainer.ray_wait_register_center_timeout``: The timeout for waiting + for the ray register center to be ready. Default is 300 seconds. + + +This figure illustrates how the configurations affect the training. + +https://excalidraw.com/#json=pfhkRmiLm1jnnRli9VFhb,Ut4E8peALlgAUpr7E5pPCA + +.. image:: https://github.com/user-attachments/assets/16aebad1-0da6-4eb3-806d-54a74e712c2d + + +evaluation.yaml +--------------- + +Data +~~~~ + +.. code:: yaml + + data: + path: /tmp/math_Qwen2-7B-Instruct.parquet + prompt_key: prompt + response_key: responses + data_source_key: data_source + reward_model_key: reward_model + +- ``data.path``: Path to the dataset file (Parquet format). +- ``data.prompt_key``: The field in the dataset where the prompt is located. Default is 'prompt'. +- ``data.response_key``: The key holds the generated responses. This should be a list of strings representing the responses. Default is 'responses'. +- ``data.data_source_key``: This is used to separate metric calculations for different data sources, ensuring that metrics are calculated independently for each source. +- ``data.reward_model_key``: The key holds the reference answers. These reference answers typically serve as the ground truth or test cases for the task. + +Customized Reward Function +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: yaml + + custom_reward_function: + path: null + name: compute_score + +- ``custom_reward_function.path``: The path to the file containing your customized reward function. If not specified, pre-implemented reward functions will be used. +- ``custom_reward_function.name`` (Optional) : The name of the reward function within the specified file. Default is 'compute_score'. + +sft_trainer.yaml for SFT FSDP Backend +-------------------------------------- + + +Optim +~~~~~~~ + +.. code:: yaml + + optim: + lr: 1e-5 + weight_decay: 0.01 + warmup_steps_ratio: 0.1 + clip_grad: 1.0 + lr_scheduler: cosine + +- ``optim.lr``: Learning rate for the optimizer. +- ``optim.weight_decay``: Weight decay for the optimizer. +- ``optim.warmup_steps_ratio``: Ratio of warmup steps to total training steps. +- ``optim.clip_grad``: Gradient clipping value. +- ``optim.lr_scheduler``: Learning rate scheduler type. Options: + + - ``cosine``: Cosine learning rate scheduler with warmup (default). + - ``wsd``: Warmup-Stable-Decay scheduler that provides a stable learning rate phase between warmup and decay phases. + +Model +~~~~~~~~~~~~ + +Most parameters for Model are similar to Reward Model. + +.. code:: yaml + + model: + partial_pretrain: ~/models/gemma-1.1-7b-it + fsdp_config: + model_dtype: fp32 + wrap_policy: + min_num_params: 0 + cpu_offload: False + offload_params: False + external_lib: null + enable_gradient_checkpointing: False + trust_remote_code: False + lora_rank: 0 + lora_alpha: 16 + target_modules: all-linear + use_liger: False + +- ``partial_pretrain``: HDFS path or local path for the pretrained model. +- ``fsdp_config`` + + - ``model_dtype``: Model parameters type, default to ``fp32``. + Support: ``bf16``, ``fp16``, ``fp32``. + - ``cpu_offload``: Whether to enable CPU offloading for FSDP. If True, + the offload_params will be used as argument. + - ``offload_params``: Whether to offload parameters to CPU + when not involved in computation. If True, then this offloads gradients + to CPU as well, meaning that the optimizer step runs on CPU. + +- ``lora_rank``: The rank of the LoRA model, default to 0. If ``lora_rank``>0, + we will train LoRA modules instead of tuning the full model. +- ``lora_alpha``: The alpha parameter for LoRA scaling, default to 16. +- ``target_modules``: The names of the modules to apply the adapter to, + default to ``all-linear``. See `peft docs `_ for detail. + +- ``use_liger``: Whether to enable Liger kernel, default to False. If True, + we apply Liger kernel to the model (depends on `liger-kernel`). diff --git a/e2c/verl/docs/examples/gsm8k_example.rst b/e2c/verl/docs/examples/gsm8k_example.rst new file mode 100644 index 0000000000000000000000000000000000000000..a30ac47aad4720b3ab779646ac27fd4a298feafb --- /dev/null +++ b/e2c/verl/docs/examples/gsm8k_example.rst @@ -0,0 +1,190 @@ +GSM8K Example +============= + +Last updated: 03/25/2025. + +Introduction +------------ + +In this example, we train an LLM to tackle the GSM8k task. + +Paper: https://arxiv.org/pdf/2110.14168 + +Dataset: https://huggingface.co/datasets/gsm8k + +Note that the original paper mainly focuses on training a verifier (a +reward model) to solve math problems via Best-of-N sampling. In this +example, we train an RLHF agent using a rule-based reward model. + +Dataset Introduction +-------------------- + +GSM8k is a math problem dataset. The prompt is an elementary school +problem. The LLM model is required to answer the math problem. + +The training set contains 7473 samples and the test set contains 1319 +samples. + +**An example** + +Prompt + + Katy makes coffee using teaspoons of sugar and cups of water in the + ratio of 7:13. If she used a total of 120 teaspoons of sugar and cups + of water, calculate the number of teaspoonfuls of sugar she used. + +Solution + + The total ratio representing the ingredients she used to make the + coffee is 7+13 = <<7+13=20>>20 Since the fraction representing the + number of teaspoons she used is 7/20, she used 7/20\ *120 = + <<7/20*\ 120=42>>42 #### 42 + +Step 1: Prepare dataset +----------------------- + +.. code:: bash + + cd examples/data_preprocess + python3 gsm8k.py --local_save_dir ~/data/gsm8k + +Step 2: Download Model +---------------------- + +There're three ways to prepare the model checkpoints for post-training: + +- Download the required models from huggingface or modelscope + +.. code:: bash + + huggingface-cli download deepseek-ai/deepseek-math-7b-instruct --local-dir ~/models/deepseek-math-7b-instruct --local-dir-use-symlinks False + # or + modelscope download --model deepseek-ai/deepseek-math-7b-instruct --local_dir ~/models/deepseek-math-7b-instruct + +- Already store your store model in the local directory or HDFS path. +- Also, you can directly use the model name in huggingface (e.g., + deepseek-ai/deepseek-math-7b-instruct) in + ``actor_rollout_ref.model.path`` and ``critic.model.path`` field in + the run script. You can also download models from modelscope by setting environmental variable ``VERL_USE_MODELSCOPE=True``. + See examples/ppo_trainer/run_deepseek7b_llm_modelscope.sh for example. + +Noted that users should prepare checkpoints for actor, critic and reward +model. + +[Optional] Step 3: SFT your Model +--------------------------------- + +We provide a SFT Trainer using PyTorch FSDP in +`fsdp_sft_trainer.py `_. +Users can customize their own SFT +script using our FSDP SFT Trainer. + +We also provide various training scripts for SFT on GSM8K dataset in `gsm8k sft directory `_. + +.. code:: shell + + set -x + + torchrun -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=question \ + data.response_key=answer \ + data.micro_batch_size_per_gpu=8 \ + model.partial_pretrain=deepseek-ai/deepseek-coder-6.7b-instruct \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-deepseek-coder-6.7b-instruct \ + trainer.total_epochs=4 \ + trainer.logger='["console","wandb"]' + + +If you use AMD GPUs (ROCm kernel), you need to add the following environment variables into the run script: + + .. code-block:: bash + + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES + export CUDA_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES + + +Step 4: Perform PPO training with your model on GSM8K Dataset +------------------------------------------------------------- + +- Prepare your own run.sh script. Here's an example for GSM8k dataset + and deepseek-llm-7b-chat model. +- Users could replace the ``data.train_files`` ,\ ``data.val_files``, + ``actor_rollout_ref.model.path`` and ``critic.model.path`` based on + their environment. +- See :doc:`config` for detailed explanation of each config field. + +**Reward Model/Function** + +We use a rule-based reward model. We force the model to produce a final +answer following 4 “#” as shown in the solution. We extract the final +answer from both the solution and model's output using regular +expression matching. We compare them and assign a reward of 1 to correct +answer, 0.1 to incorrect answer and 0 to no answer. + +**Training Script** + +The training script example for FSDP and Megatron-LM backend are stored in examples/ppo_trainer directory. + +.. code:: bash + + cd ../ppo_trainer + bash run_deepseek7b_llm.sh + +The script of run_deepseek7b_llm.sh + +.. code:: bash + + set -x + + python3 -m verl.trainer.main_ppo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=32 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='deepseek_llm_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=1 \ + trainer.total_epochs=15 $@ + + +If you use AMD GPUs (ROCm kernel), you need to add the following environment variables into the run script: + + .. code-block:: bash + + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES + export CUDA_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES + +If you encounter any issues in using AMD GPUs running VeRL, feel free to contact me - `Yusheng Su `_. \ No newline at end of file diff --git a/e2c/verl/docs/examples/multi_modal_example.rst b/e2c/verl/docs/examples/multi_modal_example.rst new file mode 100644 index 0000000000000000000000000000000000000000..844005b66eac5a8b0543d3e67a722c0c11293c95 --- /dev/null +++ b/e2c/verl/docs/examples/multi_modal_example.rst @@ -0,0 +1,45 @@ +Multi-Modal Example Architecture +================================= + +Last updated: 04/28/2025. + +Introduction +------------ + +Now, verl has supported multi-modal training. You can use fsdp and +vllm/sglang to start a multi-modal RL task. Megatron supports is also +on the way. + +Follow the steps below to quickly start a multi-modal RL task. + +Step 1: Prepare dataset +----------------------- + +.. code:: python + + # it will be saved in the $HOME/data/geo3k folder + python examples/data_preprocess/geo3k.py + +Step 2: Download Model +---------------------- + +.. code:: bash + + # download the model from huggingface + python3 -c "import transformers; transformers.pipeline(model='Qwen/Qwen2.5-VL-7B-Instruct')" + +Step 3: Perform GRPO training with multi-modal model on Geo3K Dataset +--------------------------------------------------------------------- + +.. code:: bash + + # run the task + bash examples/grpo_trainer/run_qwen2_5_vl-7b.sh + + + + + + + + diff --git a/e2c/verl/docs/examples/ppo_code_architecture.rst b/e2c/verl/docs/examples/ppo_code_architecture.rst new file mode 100644 index 0000000000000000000000000000000000000000..94d62413a2a684385eae801281995d6a02f05b3a --- /dev/null +++ b/e2c/verl/docs/examples/ppo_code_architecture.rst @@ -0,0 +1,209 @@ +PPO Example Architecture +======================== + +Last updated: 02/17/2025. + +Let's start with the Proximal Policy Optimization algorithm, which is +most widely used algorithm in LLM post-training. + +The main entry point of the PPO algorithm example is: +`main_ppo.py `_. +In this tutorial, we will go through the code architecture in `main_ppo.py `_. + +Define the data +--------------- + +Users need to preprocess and store the dataset in parquet files. +And we implement `RLHFDataset` to load and tokenize the parquet files. + +For ``RLHFDataset`` (Default), at least 1 fields are required: + +- ``prompt``: Contains the string prompt + +We already provide some examples of processing the datasets to parquet +files in `data_preprocess directory `_. Currently, we support +preprocess of GSM8k, MATH, Hellasage, Full_hh_rlhf datasets. See :doc:`../preparation/prepare_data` for +more information. + +Define the reward functions for different datasets +-------------------------------------------------- + +In this main entry point, the users only need to define their own reward +function based on the datasets (or applications) utilized in PPO +training. + +For example, we already provide reward functions for `GSM8k `_ +and `MATH `_ +datasets in the ``_select_rm_score_fn``. In the ``RewardManager``, we +will compute the reward score based on the data_source to select +corresponding reward functions. For some RLHF datasets (e.g., +full_hh_rlhf), the reward model is utilized to assess the responses +without any reward functions. In this case, the ``RewardManager`` will +return the ``rm_score`` computed by the reward model directly. + +See `reward functions `_ for detailed implementation. + +Define worker classes +--------------------- + +.. code:: python + + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: # for FSDP backend + assert config.critic.strategy in {"fsdp", "fsdp2"} + from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker + from verl.single_controller.ray import RayWorkerGroup + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == 'megatron': # for Megatron backend + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker + from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup + ray_worker_group_cls = NVMegatronRayWorkerGroup # Ray worker class for Megatron-LM + + else: + raise NotImplementedError + + from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role + + role_worker_mapping = { + Role.ActorRollout: ActorRolloutRefWorker, + Role.Critic: CriticWorker, + Role.RefPolicy: ActorRolloutRefWorker + } + + global_pool_id = 'global_pool' + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + Role.Critic: global_pool_id, + Role.RefPolicy: global_pool_id, + } + +Step 1: Construct the mapping between roles and workers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A role represents a group of workers in the same process. We have +pre-defined several roles in `ray_trainer.py `_. + +.. code:: python + + class Role(Enum): + """ + To create more roles dynamically, you can subclass Role and add new members + """ + Actor = 0 # This worker only has Actor + Rollout = 1 # This worker only has Rollout + ActorRollout = 2 # This worker has both actor and rollout, it's a HybridEngine + Critic = 3 # This worker only has critic + RefPolicy = 4 # This worker only has reference policy + RewardModel = 5 # This worker only has reward model + ActorRolloutRef = 6 # This worker contains actor, rollout and reference policy simultaneously + +Step 2: Define the worker class corresponding to this role +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- We have pre-implemented the ``ActorRolloutRefWorker``. Through + different configs, it can be a standalone actor, a standalone rollout, + an ActorRollout HybridEngine, or an ActorRolloutRef HybridEngine +- We also pre-implemented workers for ``Actor``, ``Rollout``, + ``Critic``, ``Reward Model`` and ``Reference model`` on two different + backend: PyTorch FSDP + and Megatron-LM. + See `FSDP Workers `_ + and `Megatron-LM Workers `_ + for more information. + +Step 3: Define resource pool id and resource pool spec +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Resource pool is a division of global GPU resources, + ``resource_pool_spec`` is a dict, mapping from id to # of GPUs + + - In the above example, we defined a global resource pool: + global_pool_id, and then put all roles on this one resource pool + with all the GPUs in this post-training task. This refers to + *co-locate* placement where all the models share the same set of + GPUs. + +- See resource pool and placement for advance usage. + +Defining reward model/function +------------------------------ + +.. code:: python + + # we should adopt a multi-source reward function here + # - for rule-based rm, we directly call a reward score + # - for model-based rm, we call a model + # - for code related prompt, we send to a sandbox if there are test cases + # - finally, we combine all the rewards together + # - The reward type depends on the tag of the data + if config.reward_model.enable: + from verl.workers.fsdp_workers import RewardModelWorker + role_worker_mapping[Role.RewardModel] = RewardModelWorker + mapping[Role.RewardModel] = global_pool_id + + reward_fn = RewardManager(tokenizer=tokenizer, num_examine=0) + + # Note that we always use function-based RM for validation + val_reward_fn = RewardManager(tokenizer=tokenizer, num_examine=1) + + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + +Since not all tasks use model-based RM, users need to define here +whether it's a model-based RM or a function-based RM + +- If it's a model-based RM, directly add the ``RewardModel`` role in the + resource mapping and add it to the resource pool mapping. + + - Note that the pre-defined ``RewardModelWorker`` only supports models + with the structure of huggingface + ``AutoModelForSequenceClassification``. If it's not this model, you + need to define your own RewardModelWorker in `FSDP Workers `_ + and `Megatron-LM Workers `_. + +- If it's a function-based RM, the users are required to classified the + reward function for each datasets. + +.. code:: python + + def _select_rm_score_fn(data_source): + if data_source == 'openai/gsm8k': + return gsm8k.compute_score + elif data_source == 'lighteval/MATH': + return math.compute_score + else: + raise NotImplementedError + +See reward functions implemented in `directory `_ +for more information. + +Define, init and run the PPO Trainer +------------------------------------ + +.. code:: python + + trainer = RayPPOTrainer(config=config, + tokenizer=tokenizer, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn) + trainer.init_workers() + trainer.fit() + +- We first initialize the ``RayPPOTrainer`` with user config, tokenizer + and all the above worker mapping, resource pool, worker group and + reward functions +- We first call the ``trainer.init_workers()`` to initialize the models + on the allocated GPUs (in the resource pool) +- The actual PPO training will be executed in ``trainer.fit()`` + +verl can be easily extended to other RL algorithms by reusing the Ray +model workers, resource pool and reward functions. See :doc:`extension<../advance/dpo_extension>` for +more information. + +Details of the ``RayPPOTrainer`` is discussed in :doc:`Ray Trainer<../workers/ray_trainer>`. diff --git a/e2c/verl/docs/examples/sandbox_fusion_example.rst b/e2c/verl/docs/examples/sandbox_fusion_example.rst new file mode 100644 index 0000000000000000000000000000000000000000..f3359efda2e14fa6d869b9af21060d6053ac112e --- /dev/null +++ b/e2c/verl/docs/examples/sandbox_fusion_example.rst @@ -0,0 +1,54 @@ +Sandbox Fusion Example +============================ + +Last updated: 06/27/2025. + +Introduction +------------ + +Sandbox Fusion is a remote code sandbox service that provides a secure environment for running and evaluating code generated by Large Language Models (LLMs). This example demonstrates how to train an LLM and use Sandbox Fusion to verify generated code, enhancing both security and performance. + +By leveraging a remote code sandbox service with greater CPU resources for concurrent code verification, you can reduce the reward stage time by 10-30%, depending on the quality of the generated code. + +Step 1: Prepare the Dataset +--------------------------- + +We use the Eurus-2-RL-Data dataset for training. This dataset combines math and code questions, making it suitable for LLM training tasks. You can download it from HuggingFace: `Eurus-2-RL-Data Dataset `_. + +Step 2: Set Up the Sandbox Fusion Service +----------------------------------------- + +Sandbox Fusion is a remote code sandbox service designed to securely run and evaluate LLM-generated code. To use it: + +1. **Access Full Documentation**: For detailed setup instructions, refer to the `Sandbox Fusion Documentation `_. +2. **Deploy the Service**: Choose one of the following deployment methods: + + - **Local Deployment**: Follow the guide `here `_. + - **FaaS Instance (Volcengine)**: Create an instance using the `Volcengine Documentation `_. + +After deployment, you will receive an API endpoint in the format: ``https:///run_code``. + +Step 3: Configure the Training Script +------------------------------------- + +To integrate Sandbox Fusion into your training script, configure the following parameters: + +**Key Settings for Sandbox Fusion** + +- ``reward_model.sandbox_fusion.url=''``: Enable Sandbox Fusion by specifying the API endpoint (must end with ``/run_code``). +- ``reward_model.sandbox_fusion.max_concurrent=256``: Set the maximum number of concurrent API requests to the Sandbox Fusion service. +- ``reward_model.sandbox_fusion.memory_limit_mb=1024``: Set the memory limit (in MB) for each sandbox instance. Defaults to 1024MB if not specified. + +**Additional Optimization** + +To further reduce code verification time, enable parallel processing with: + +- ``reward_model.reward_manager=prime``: The Prime reward manager verifies code across multiple subprocesses concurrently. + +**Example Script** + +For a practical implementation, refer to the example script: + +``examples/ppo_trainer/run_deepseek7b_llm_sandbox_fusion.sh`` + +Once you’ve set your API endpoint in the script, you can start the training job. \ No newline at end of file diff --git a/e2c/verl/docs/examples/skypilot_examples.rst b/e2c/verl/docs/examples/skypilot_examples.rst new file mode 100644 index 0000000000000000000000000000000000000000..de91781be63290be6da5bf4b62624addb6446a2d --- /dev/null +++ b/e2c/verl/docs/examples/skypilot_examples.rst @@ -0,0 +1,146 @@ +SkyPilot Examples +================= + +Last updated: 09/04/2025. + +This guide provides examples of running VERL reinforcement learning training on Kubernetes clusters or cloud platforms with GPU nodes using `SkyPilot `_. + +Installation and Configuration +------------------------------- + +Step 1: Install SkyPilot +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Choose the installation based on your target platform: + +.. code-block:: bash + + # For Kubernetes only + pip install "skypilot[kubernetes]" + + # For AWS + pip install "skypilot[aws]" + + # For Google Cloud Platform + pip install "skypilot[gcp]" + + # For Azure + pip install "skypilot[azure]" + + # For multiple platforms + pip install "skypilot[kubernetes,aws,gcp,azure]" + +Step 2: Configure Your Platform +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See https://docs.skypilot.co/en/latest/getting-started/installation.html + +Step 3: Set Up Environment Variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Export necessary API keys for experiment tracking: + +.. code-block:: bash + + # For Weights & Biases tracking + export WANDB_API_KEY="your-wandb-api-key" + + # For HuggingFace gated models (if needed) + export HF_TOKEN="your-huggingface-token" + +Examples +-------- + +All example configurations are available in the `examples/skypilot/ `_ directory on GitHub. See the `README `_ for additional details. + +PPO Training +~~~~~~~~~~~~ + +.. code-block:: bash + + sky launch -c verl-ppo verl-ppo.yaml --secret WANDB_API_KEY -y + +Runs PPO training on GSM8K dataset using Qwen2.5-0.5B-Instruct model across 2 nodes with H100 GPUs. Based on examples in ``examples/ppo_trainer/``. + +`View verl-ppo.yaml on GitHub `_ + +GRPO Training +~~~~~~~~~~~~~ + +.. code-block:: bash + + sky launch -c verl-grpo verl-grpo.yaml --secret WANDB_API_KEY -y + +Runs GRPO (Group Relative Policy Optimization) training on MATH dataset using Qwen2.5-7B-Instruct model. Memory-optimized configuration for 2 nodes. Based on examples in ``examples/grpo_trainer/``. + +`View verl-grpo.yaml on GitHub `_ + +Multi-turn Tool Usage Training +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + sky launch -c verl-multiturn verl-multiturn-tools.yaml \ + --secret WANDB_API_KEY --secret HF_TOKEN -y + +Single-node training with 8xH100 GPUs for multi-turn tool usage with Qwen2.5-3B-Instruct. Includes tool and interaction configurations for GSM8K. Based on examples in ``examples/sglang_multiturn/`` but uses vLLM instead of sglang. + +`View verl-multiturn-tools.yaml on GitHub `_ + +Configuration +------------- + +The example YAML files are pre-configured with: + +- **Infrastructure**: Kubernetes clusters (``infra: k8s``) - can be changed to ``infra: aws`` or ``infra: gcp``, etc. +- **Docker Image**: VERL's official Docker image with CUDA 12.6 support +- **Setup**: Automatically clones and installs VERL from source +- **Datasets**: Downloads required datasets during setup phase +- **Ray Cluster**: Configures distributed training across nodes +- **Logging**: Supports Weights & Biases via ``--secret WANDB_API_KEY`` +- **Models**: Supports gated HuggingFace models via ``--secret HF_TOKEN`` + +Launch Command Options +---------------------- + +- ``-c ``: Cluster name for managing the job +- ``--secret KEY``: Pass secrets for API keys (can be used multiple times) +- ``-y``: Skip confirmation prompt + +Monitoring Your Jobs +-------------------- + +Check Cluster Status +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + sky status + +View Logs +~~~~~~~~~ + +.. code-block:: bash + + sky logs verl-ppo # View logs for the PPO job + +SSH into Head Node +~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + ssh verl-ppo + +Access Ray Dashboard +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + sky status --endpoint 8265 verl-ppo # Get dashboard URL + +Stop a Cluster +~~~~~~~~~~~~~~ + +.. code-block:: bash + + sky down verl-ppo diff --git a/e2c/verl/docs/faq/faq.rst b/e2c/verl/docs/faq/faq.rst new file mode 100644 index 0000000000000000000000000000000000000000..50a90508ba12cc5095f0310961d047383e8dbd60 --- /dev/null +++ b/e2c/verl/docs/faq/faq.rst @@ -0,0 +1,209 @@ +Frequently Asked Questions +==================================== + +Last updated: 09/24/2025. + +Ray related +------------ + +How to add breakpoint for debugging with distributed Ray? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Please checkout the official debugging guide from Ray: https://docs.ray.io/en/latest/ray-observability/ray-distributed-debugger.html + + +"Unable to register worker with raylet" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The cause of this issue is due to some system setting, e.g., SLURM added some constraints on how the CPUs are shared on a node. +While `ray.init()` tries to launch as many worker processes as the number of CPU cores of the machine, +some constraints of SLURM restricts the `core-workers` seeing the `raylet` process, leading to the problem. + +To fix this issue, you can set the config term ``ray_init.num_cpus`` to a number allowed by your system. + +Distributed training +------------------------ + +How to run multi-node post-training with Ray? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can start a ray cluster and submit a ray job, following the official guide from Ray: https://docs.ray.io/en/latest/ray-core/starting-ray.html + +Then in the configuration, set the ``trainer.nnode`` config to the number of machines for your job. + +How to use verl on a Slurm-managed cluster? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Ray provides users with `this `_ official +tutorial to start a Ray cluster on top of Slurm. We have verified the :doc:`GSM8K example<../examples/gsm8k_example>` +on a Slurm cluster under a multi-node setting with the following steps. + +1. [Optional] If your cluster support `Apptainer or Singularity `_ and you wish +to use it, convert verl's Docker image to an Apptainer image. Alternatively, set up the environment with the package +manager available on your cluster or use other container runtimes (e.g. through `Slurm's OCI support `_) available to you. + +.. code:: bash + + apptainer pull /your/dest/dir/vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3.sif docker://verlai/verl:vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3 + +2. Follow :doc:`GSM8K example<../examples/gsm8k_example>` to prepare the dataset and model checkpoints. + +3. Modify `examples/slurm/ray_on_slurm.slurm `_ with your cluster's own information. + +4. Submit the job script to the Slurm cluster with `sbatch`. + +Please note that Slurm cluster setup may vary. If you encounter any issues, please refer to Ray's +`Slurm user guide `_ for common caveats. + +If you changed Slurm resource specifications, please make sure to update the environment variables in the job script if necessary. + + +Install related +------------------------ + +NotImplementedError: TensorDict does not support membership checks with the `in` keyword. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Detail error information: + +.. code:: bash + + NotImplementedError: TensorDict does not support membership checks with the `in` keyword. If you want to check if a particular key is in your TensorDict, please use `key in tensordict.keys()` instead. + +Cause of the problem: There is no suitable version of tensordict package for the linux-arm64 platform. The confirmation method is as follows: + +.. code:: bash + + pip install tensordict==0.6.2 + +Output example: + +.. code:: bash + + ERROR: Could not find a version that satisfies the requirement tensordict==0.6.2 (from versions: 0.0.1a0, 0.0.1b0, 0.0.1rc0, 0.0.2a0, 0.0.2b0, 0.0.3, 0.1.0, 0.1.1, 0.1.2, 0.8.0, 0.8.1, 0.8.2, 0.8.3) + ERROR: No matching distribution found for tensordict==0.6.2 + +Solution 1st: + Install tensordict from source code: + +.. code:: bash + + pip uninstall tensordict + git clone https://github.com/pytorch/tensordict.git + cd tensordict/ + git checkout v0.6.2 + python setup.py develop + pip install -v -e . + +Solution 2nd: + Temperally modify the error takeplace codes: tensordict_var -> tensordict_var.keys() + + +Illegal memory access +--------------------------------- + +If you encounter the error message like ``CUDA error: an illegal memory access was encountered`` during rollout, please check the vLLM documentation for troubleshooting steps specific to your vLLM version. + +Checkpoints +------------------------ + +If you want to convert the model checkpoint into huggingface safetensor format, please refer to ``verl/model_merger``. + + +Triton ``compile_module_from_src`` error +------------------------------------------------ + +If you encounter triton compilation error similar to the stacktrace below, please set the ``use_torch_compile`` flag according to +https://verl.readthedocs.io/en/latest/examples/config.html to disable just-in-time compilation for fused kernels. + +.. code:: bash + + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/runtime/jit.py", line 345, in + return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs) + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 338, in run + return self.fn.run(*args, **kwargs) + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/runtime/jit.py", line 607, in run + device = driver.active.get_current_device() + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/runtime/driver.py", line 23, in __getattr__ + self._initialize_obj() + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/runtime/driver.py", line 20, in _initialize_obj + self._obj = self._init_fn() + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/runtime/driver.py", line 9, in _create_driver + return actives[0]() + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 371, in __init__ + self.utils = CudaUtils() # TODO: make static + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 80, in __init__ + mod = compile_module_from_src(Path(os.path.join(dirname, "driver.c")).read_text(), "cuda_utils") + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 57, in compile_module_from_src + so = _build(name, src_path, tmpdir, library_dirs(), include_dir, libraries) + File "/data/lbh/conda_envs/verl/lib/python3.10/site-packages/triton/runtime/build.py", line 48, in _build + ret = subprocess.check_call(cc_cmd) + File "/data/lbh/conda_envs/verl/lib/python3.10/subprocess.py", line 369, in check_call + raise CalledProcessError(retcode, cmd) + +What is the meaning of train batch size, mini batch size, and micro batch size? +------------------------------------------------------------------------------------------ + +This figure illustrates the relationship between different batch size configurations. + +https://excalidraw.com/#json=pfhkRmiLm1jnnRli9VFhb,Ut4E8peALlgAUpr7E5pPCA + +.. image:: https://github.com/user-attachments/assets/16aebad1-0da6-4eb3-806d-54a74e712c2d + +How to generate ray timeline to analyse performance of a training job? +------------------------------------------------------------------------------------------ + +To generate the ray timeline file, you can set the config term ``ray_init.timeline_file`` to a json file path. +For example: + +.. code:: bash + + ray_init.timeline_file=/tmp/ray_timeline.json + +The file will be generated in the specified path at the end of a training job. +You can use tools like chrome://tracing or the Perfetto UI and view the ray timeline file. + +This figure shows the ray timeline file generated by from a training job on 1 node with 4 GPUs + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray_timeline.png?raw=true + +How to set proxy only for wandb? +------------------------------------------------------------------------------------------ + +If you need a proxy to access wandb, you can add below config in your training job script. +Comparing to using global https_proxy env variable, this approach won't mess up other http requests, such as ChatCompletionScheduler. + +.. code:: bash + + +trainer.wandb_proxy=http:// + +Missmatch between inference and training sequence (high actor/grad_norm) +------------------------------------------------------------------------------------------ + +If you encounter the issue of actor/grad_norm metric continuously increasing during training, it might be caused by a significant precision mismatching between the inference engine and training. You can use the following parameter to confirm this: + +.. code:: bash + + actor_rollout_ref.rollout.calculate_log_probs=True + +This parameter will add metrics like training/rollout_probs_diff_mean , which can be used to verify if there is a precision difference between inference and training. + +Under normal circumstances, the value of training/rollout_probs_diff_mean should be below 0.005. If you observe this value to be higher than 0.01, it indicates a precision issue from the inference engine. +The precision issue is known to occur under the following conditions: + +1. Using non-Hopper architecture GPUs, such as A100, L20, B200, etc. + +2. Using vLLM `with issue 22103 `_ as the inference engine. + +3. The input and output texts are long, for example, in multi-turn scenarios using reasioning models like Qwen3 for RL training. + +If all three conditions above are met and you observe that rollout_probs_diff_mean is too high, it is recommended to add the following parameter to resolve the precision issue: + +.. code:: bash + + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_cascade_attn=True + +The root cause of this issue is a bug in the flash attention used by vLLM. Although it has been fixed, the fix has not yet been released in the latest version of vLLM (v0.10.2). +For a more detailed explanation of this issue, please refer to `Fix LSE output error in FA2 kv-split `_. + +Until vLLM releases a new version with this fix, it is recommended to use the configuration above to disable cascade attention as a workaround. diff --git a/e2c/verl/docs/hybrid_flow.rst b/e2c/verl/docs/hybrid_flow.rst new file mode 100644 index 0000000000000000000000000000000000000000..3aa5a4a97cb88e564babc11392899149338a5b49 --- /dev/null +++ b/e2c/verl/docs/hybrid_flow.rst @@ -0,0 +1,266 @@ +========================================================= +HybridFlow Programming Guide +========================================================= + +Last updated: 06/02/2025. + +.. _vermouth: https://github.com/vermouth1992 + +Author: `Chi Zhang `_ + +verl is an open source implementation of the paper `HybridFlow `_ [1]_. In this section, we will introduce the basic concepts of HybridFlow, the motivation and how to program with verl APIs. + +Motivation and Design +------------------------ +We use dataflow to represent RL systems. [4]_. + +DataFlow +~~~~~~~~~~~~~~~~~~~~ + +Dataflow is an abstraction of computations. Neural Network training is a typical dataflow. It can be represented by computational graph. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/dataflow.jpeg?raw=true + :alt: The dataflow graph from CS231n 2024 lecture 4 + +This figure [2]_ represents the computation graph of a polynomial function followed by a sigmoid function. In the data flow of neural network computation, each node represents an operator, and each edge represents the direction of forward/backward propagation. The computation graph determines the architecture of the neural network. + +RL as a dataflow problem +++++++++++++++++++++++++++++++++++++++++++++++ + +Reinforcement learning (RL) training can also be represented as a dataflow. Below is the dataflow graph that represents the PPO algorithm used in RLHF [3]_: + +.. image:: https://picx.zhimg.com/70/v2-cb8ab5ee946a105aab6a563e92682ffa_1440w.avis?source=172ae18b&biz_tag=Post + :alt: PPO dataflow graph, credit to Zhihu 低级炼丹师 + +However, the dataflow of RL has fundamental differences compared with dataflow of neural network training as follows: + ++--------------------------+--------------------------------------------------+---------------------+ +| Workload | Node | Edge | ++--------------------------+--------------------------------------------------+---------------------+ +| Neural Network Training | Operator (+/-/matmul/softmax) | Tensor movement | ++--------------------------+--------------------------------------------------+---------------------+ +| Reinforcement Learning | High-level operators (rollout/model forward) | Data Movement | ++--------------------------+--------------------------------------------------+---------------------+ + +In the case of tabular reinforcement learning, each operator is a simple scalar math operation (e.g., bellman update). In deep reinforcement learning(DRL), each operator is a high-level neural network computation such as model inference/update. This makes RL a two-level dataflow problem: + +- Control flow: defines how the high-level operators are executed (e.g., In PPO, we first perform rollout. Then, we perform advantage computation. Finally, we perform training). It expresses the **core logics of RL algorithms**. +- Computation flow: defines the dataflow of **neural network computation** (e.g., model forward/backward/optimizer). + + +Design Choices +~~~~~~~~~~~~~~~~~~~~ +The model size used in DRL before the LLM era is typically small. Thus, the high-level neural network computation can be done in a single process. This enables embedding the computation flow inside the control flow as a single process. + +However, in the LLM era, the computation flow (e.g., training neural network) becomes a multi-process program. This naturally leads to two design choices: + +1. Convert the control flow into a multi-process program as well. Then colocate with computation flow (unified multi-controller) + +- Advantages: + + - Achieves the **optimal performance** under fixed computation flow and control flow as the communication overhead in both training and data transfer is minimized. + +- Disadvantages: + + - The computation and/or control flow is **hard to reuse** from software perspective as computation code is coupled with specific controller code. For example, the training loop of PPO is generic. Say we have an PPO training flow implemented with a specific computation flow such as FSDP. Neither the control flow or computation flow can be reused if we want to switch the computation flow from FSDP to Megatron, due to the coupling of control and computation flows. + - Requires more efforts from the user under flexible and dynamic control flows, due to the multi-process nature of the program. + +2. Separate the flows: single process for the control flow and multi-process for computation flow + +- Advantages: + + - The computation flow defined elsewhere can be **easily reused** after the decoupling. + - The controller runs on a single process. Implementing a new RL algorithm with a **different control flow is simple and easy**. + +- Disadvantages: + + - Additional **data communication overhead** each time the controller process and computatation processes interact. The data has to be sent back and forth. + +In verl, the latter strategy with separate control flow and computation flow is adopted. verl is designed to decouple the control flow of RL algorithms, and the implementation of computation engines. + +Overall Execution Diagram +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Below is a simplified diagram denoting the execution of a reinforcement learning job. In the diagram, the controller runs on a single process, while the generator/actor workers, critic workers run on multiple processes, placed with specific resource groups. For rollout, the controller passes the data to the generator to perform sample generation. When the rollout is done, the data is passed back to controller for the next step of the algorithm. Similar execution is done for other workers. With the hybrid controller design, the data flow and computation is decoupled to provide both efficiency in computation and flexibility in defining algorithm training loops. + +.. figure:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/driver_worker.png?raw=true + :alt: The execution diagram + +Codebase walkthrough (PPO) +------------------------------------------------ + +Entry function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Code: https://github.com/volcengine/verl/blob/main/verl/trainer/main_ppo.py + +In this file, we define a remote function `main_task` that serves as the controller (driver) process as shown in the above figure. We also define a ``RewardManager``, where users can customize their reward function based on the data source in the dataset. Note that `RewardManager` should return the final token-level reward that is optimized by RL algorithms. Note that users can combine model-based rewards and rule-based rewards. +The ``main_task`` constructs a RayPPOTrainer instance and launch the fit. Note that ``main_task`` **runs as a single process**. + +We highly recommend that the ``main_task`` is NOT scheduled on the head of the ray cluster because ``main_task`` will consume a lot of memory but the head usually contains very few resources. + +Ray trainer +~~~~~~~~~~~~~~~~~~~~ +Code: https://github.com/volcengine/verl/blob/main/verl/trainer/ppo/ray_trainer.py + +The RayPPOTrainer manages + +- Worker and WorkerGroup construction +- Runs the main loop of PPO algorithm + +Note that, the fit function of RayPPOTrainer **runs as a single process**. + +Worker and WorkerGroup construction +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Each workerGroup manages a list of workers that runs remotely. Note that the worker group runs in the process of its constructor. +Each worker inside the WorkerGroup runs on a GPU. The worker group serves as a proxy for the controller process to interact with a list of workers, in order to perform certain computations. **In order to do so, we have to bind the methods of the worker into the method of the WorkerGroup and define the data dispatch and data collection**. This is done via simple decoration that will be introduced in the Worker definition section. + +For example, in PPO, we define 3 worker groups: + +- ActorRolloutRef: manages actor, rollout and reference policy. ActorRolloutRefWorker can be instantiated as a single actor, a single rollout, a single reference policy, a combined actor/rollout or a combined actor/rollout/ref. This design is aimed for the maximum code reuse in various scenarios. The reason for colocating actor and rollout is for fast weight transfer using nccl. The reason for coloating actor and reference is to implement an efficient lora PPO as the reference policy is simply the base model of PPO in lora. The colocation is done via ``verl.single_controller.ray.base.create_colocated_worker_cls``, where it creates a single ray remote class exposing all class methods from these roles. +- Critic: manages the critic model +- Reward: manages the reward model + +The worker group will be constructed on the resource pool it designates. The resource pool is a set of GPUs in the ray cluster. + +Worker definition +~~~~~~~~~~~~~~~~~~~~ + +.. _ActorRolloutRefWorker: https://github.com/volcengine/verl/blob/main/verl/workers/fsdp_workers.py + +We take `ActorRolloutRefWorker `_ for an example. +The APIs it should expose to the controller process are: + +- init_model: build the underlying model +- generate_sequences: given prompts, generate responses +- compute_log_prob: compute the log-probability of a generated sequence using actor +- compute_ref_log_prob: compute the log-probability of a generated sequence using reference policy +- save_checkpoint: save the checkpoint + +Note that these methods are defined in the worker that can only be invoked via remote calls. For example, if the controller process wants to initialize the model, it has to call + +.. code-block:: python + + for worker in actor_rollout_ref_wg: + worker.init_model.remote() + +If the controller process wants to generate sequences, it has to call + +.. code-block:: python + + data = xxx + # split the data into dp chunks + data_dp_lst = data.split(dp_size) + output_dp_lst = [] + for i, worker in enumerate(actor_rollout_ref_wg): + output_future = worker.generate_sequences.remote(data_dp_lst[i]) + output_dp_lst.append(output_future) + output = torch.cat(ray.get(output_dp_lst), dim=0) + +We observe that controller process calling worker group methods in general can be divided into 3 parts: + +- Split the data into data parallel sizes +- Dispatch the corresponding data into each worker +- Collect and concatenate the data when the computation finishes + +In verl, we design a syntax sugar to encapsulate the 3 processes into a single call from the controller process. + +.. code-block:: python + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def generate_sequences(data): + ... + + # on the driver + output = actor_rollout_ref_wg.generate_sequences(data) + +We decorate the method of the worker with a ``register`` that explicitly defines how the input data should be split and dispatched to each worker, and how the output data should be collected and concatenated by the controller. For example, ``Dispatch.DP_COMPUTE_PROTO`` splits the input data into dp chunks, dispatch each data to each worker, collect the output and concatenate the results. Note that this function requires the input and output to be a DataProto defined here (https://github.com/volcengine/verl/blob/main/verl/protocol.py). + + +PPO main loop +~~~~~~~~~~~~~~~~~~~~ +With the aforementioned APIs, we can implement the main loop of PPO as if it is a single process program + +.. code-block:: python + + for prompt in dataloader: + output = actor_rollout_ref_wg.generate_sequences(prompt) + old_log_prob = actor_rollout_ref_wg.compute_log_prob(output) + ref_log_prob = actor_rollout_ref_wg.compute_ref_log_prob(output) + values = critic_wg.compute_values(output) + rewards = reward_wg.compute_scores(output) + # compute_advantages is running directly on the control process + advantages = compute_advantages(values, rewards) + output = output.union(old_log_prob) + output = output.union(ref_log_prob) + output = output.union(values) + output = output.union(rewards) + output = output.union(advantages) + # update actor + actor_rollout_ref_wg.update_actor(output) + critic.update_critic(output) + +Takeaways +~~~~~~~~~~~~~~~~~~~~ +- This programming paradigm enables users to use different computation backend without modification of the control process. +- This programming paradigm enables flexible placement (by changing the mapping of WorkerGroup and ResourcePool) without modification of the control process. + +Repository organization +------------------------------------------------ + +Important code files in the repository are organized as below: + +.. code-block:: bash + + verl # the verl package + trainer + main_ppo.py # the entrypoint for RL training + ppo + ray_trainer.py # the training loop for RL algorithms such as PPO + fsdp_sft_trainer.py # the SFT trainer with FSDP backend + config + generation.yaml # configuration template for rollout + ppo_trainer.yaml # configuration template for the RL trainer + workers + protocol.py # the interface of DataProto + fsdp_workers.py # the FSDP worker interfaces: ActorRolloutRefWorker, CriticWorker, RewardModelWorker + megatron_workers.py # the Megatron worker interfaces: ActorRolloutRefWorker, CriticWorker, RewardModelWorker + actor + dp_actor.py # data parallel actor with FSDP backend + megatron_actor.py # nD parallel actor with Megatron backend + critic + dp_critic.py # data parallel critic with FSDP backend + megatron_critic.py # nD parallel critic with FSDP backend + reward_model + megatron + reward_model.py # reward model with Megatron backend + rollout + vllm + vllm_rollout.py # rollout with vllm backend + hf_rollout.py # rollout with huggingface TGI backend + sharding_manager + fsdp_ulysses.py # data and model resharding when using FSDP + ulysses + fsdp_vllm.py # data and model resharding when using FSDP + ulysses + vllm + megatron_vllm.py # data and model resharding when using Megatron + vllm + utils + dataset # datasets for SFT/RM/RL + reward_score # function based reward + gsm8k.py # reward function for gsm8k dataset + math.py # reward function for math dataset + seqlen_balancing.py # the sequence balance optimization + models + llama # Megatron implementation for llama, deepseek, mistral, etc + transformers # ulysses integration with transformer models such as llama, qwen, etc + weight_loader_registery.py # registry of weight loaders for loading hf ckpt into Megatron + third_party + vllm # adaptor for vllm's usage in RL + vllm_spmd # vllm >= v0.7 adaptor + examples # example scripts + tests # integration and unit tests + .github # the configuration of continuous integration tests + + +.. [1] HybridFlow: A Flexible and Efficient RLHF Framework: https://arxiv.org/abs/2409.19256v2 +.. [2] Data flow graph credit to CS231n 2024 lecture 4: https://cs231n.stanford.edu/slides/2024/lecture_4.pdf +.. [3] PPO dataflow graph credit to 低级炼丹师 from Zhihu​: https://zhuanlan.zhihu.com/p/635757674 +.. [4] RLFlow diff --git a/e2c/verl/docs/index.rst b/e2c/verl/docs/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..1c6680fe0673c89fd572f09b96c682fc9944e8b3 --- /dev/null +++ b/e2c/verl/docs/index.rst @@ -0,0 +1,193 @@ +Welcome to verl's documentation! +================================================ + +verl is a flexible, efficient and production-ready RL training framework designed for large language models (LLMs) post-training. It is an open source implementation of the `HybridFlow `_ paper. + +verl is flexible and easy to use with: + +- **Easy extension of diverse RL algorithms**: The hybrid programming model combines the strengths of single-controller and multi-controller paradigms to enable flexible representation and efficient execution of complex Post-Training dataflows. Allowing users to build RL dataflows in a few lines of code. + +- **Seamless integration of existing LLM infra with modular APIs**: Decouples computation and data dependencies, enabling seamless integration with existing LLM frameworks, such as PyTorch FSDP, Megatron-LM, vLLM and SGLang. Moreover, users can easily extend to other LLM training and inference frameworks. + +- **Flexible device mapping and parallelism**: Supports various placement of models onto different sets of GPUs for efficient resource utilization and scalability across different cluster sizes. + +- Ready integration with popular HuggingFace models + + +verl is fast with: + +- **State-of-the-art throughput**: By seamlessly integrating existing SOTA LLM training and inference frameworks, verl achieves high generation and training throughput. + +- **Efficient actor model resharding with 3D-HybridEngine**: Eliminates memory redundancy and significantly reduces communication overhead during transitions between training and generation phases. + +-------------------------------------------- + +.. _Contents: + +.. toctree:: + :maxdepth: 2 + :caption: Quickstart + + start/install + start/quickstart + start/multinode + start/ray_debug_tutorial + start/more_resources + start/agentic_rl + +.. toctree:: + :maxdepth: 2 + :caption: Programming guide + + hybrid_flow + single_controller + +.. toctree:: + :maxdepth: 1 + :caption: Data Preparation + + preparation/prepare_data + preparation/reward_function + +.. toctree:: + :maxdepth: 2 + :caption: Configurations + + examples/config + +.. toctree:: + :maxdepth: 1 + :caption: PPO Example + + examples/ppo_code_architecture + examples/gsm8k_example + examples/multi_modal_example + examples/skypilot_examples + +.. toctree:: + :maxdepth: 1 + :caption: Algorithms + + algo/ppo.md + algo/grpo.md + algo/collabllm.md + algo/dapo.md + algo/spin.md + algo/sppo.md + algo/entropy.md + algo/opo.md + algo/baseline.md + algo/gpg.md + +.. toctree:: + :maxdepth: 1 + :caption: PPO Trainer and Workers + + workers/ray_trainer + workers/fsdp_workers + workers/megatron_workers + workers/sglang_worker + workers/model_engine + +.. toctree:: + :maxdepth: 1 + :caption: Performance Tuning Guide + + perf/dpsk.md + perf/perf_tuning + README_vllm0.8.md + perf/device_tuning + perf/verl_profiler_system.md + perf/nsight_profiling.md + +.. toctree:: + :maxdepth: 1 + :caption: Adding new models + + advance/fsdp_extension + advance/megatron_extension + +.. toctree:: + :maxdepth: 1 + :caption: Advanced Features + + advance/checkpoint + advance/rope + advance/ppo_lora.rst + sglang_multiturn/multiturn.rst + sglang_multiturn/interaction_system.rst + advance/placement + advance/dpo_extension + examples/sandbox_fusion_example + advance/rollout_trace.rst + advance/rollout_skip.rst + advance/one_step_off + advance/agent_loop + +.. toctree:: + :maxdepth: 1 + :caption: Hardware Support + + amd_tutorial/amd_build_dockerfile_page.rst + amd_tutorial/amd_vllm_page.rst + ascend_tutorial/ascend_quick_start.rst + ascend_tutorial/ascend_profiling_zh.rst + ascend_tutorial/ascend_profiling_en.rst + +.. toctree:: + :maxdepth: 1 + :caption: API References + + api/data + api/single_controller.rst + api/trainer.rst + api/utils.rst + + +.. toctree:: + :maxdepth: 2 + :caption: FAQ + + faq/faq + +.. toctree:: + :maxdepth: 1 + :caption: Development Notes + + sglang_multiturn/sandbox_fusion.rst + +Contribution +------------- + +verl is free software; you can redistribute it and/or modify it under the terms +of the Apache License 2.0. We welcome contributions. +Join us on `GitHub `_, `Slack `_ and `Wechat `_ for discussions. + +Contributions from the community are welcome! Please check out our `project roadmap `_ and `good first issues `_ to see where you can contribute. + +Code Linting and Formatting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We use pre-commit to help improve code quality. To initialize pre-commit, run: + +.. code-block:: bash + + pip install pre-commit + pre-commit install + +To resolve CI errors locally, you can also manually run pre-commit by: + +.. code-block:: bash + + pre-commit run + +Adding CI tests +^^^^^^^^^^^^^^^^^^^^^^^^ + +If possible, please add CI test(s) for your new feature: + +1. Find the most relevant workflow yml file, which usually corresponds to a ``hydra`` default config (e.g. ``ppo_trainer``, ``ppo_megatron_trainer``, ``sft_trainer``, etc). +2. Add related path patterns to the ``paths`` section if not already included. +3. Minimize the workload of the test script(s) (see existing scripts for examples). + +We are HIRING! Send us an `email `_ if you are interested in internship/FTE opportunities in MLSys/LLM reasoning/multimodal alignment. diff --git a/e2c/verl/docs/perf/device_tuning.rst b/e2c/verl/docs/perf/device_tuning.rst new file mode 100644 index 0000000000000000000000000000000000000000..567683b3b449f0c6a675a58fb168657b2a5e3734 --- /dev/null +++ b/e2c/verl/docs/perf/device_tuning.rst @@ -0,0 +1,281 @@ +Hardware Resource Needed for RL +=============================== + +Last updated: 06/25/2025. + +Since RL requires more resources compared to regular training, +determining how much resources are needed to successfully run it before training +is a relatively difficult task. To provide more people with reference points for +resource selection when dealing with different models and tasks, this section is +mainly dedicated to introducing the environmental requirements based on experiments +we have conducted. + +However, due to limited staff and equipment resources, we also hope for more +contributions from the open-source community. When submitting a PR, it is necessary +to provide a script to be added to the example/tuning scripts. + +We need two types of scripts: one is the configuration that can run with the **minimum +resources(min)**, and the other is the configuration that runs with **recommended resources(recommended)**. For the former, +it can be understood as a script that can run after applying all memory optimization techniques +(e.g., offload, gradient checkpointing). For the latter, it can be understood as a script that +can run while avoiding operations that incur additional time overhead as much as possible (targetting best throughput). + +When defining script names, please follow this format: +``[model]_[task]_[gpunums]_[device]_[train]_[infer].sh``. This will effectively improve +the script's recognizability. You can place the script under the ``examples/tuning/`` directory. + +If you happen to have a configuration that has already been tested, we welcome you to submit +a PR and include a screenshot from Wandb or other verifiable evidence. + +---------------------------------------- + +0.5B +~~~ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Tag + - Model + - Task + - Resource + - MaxBatch + - Train + - Infer + - Link + - Contributor + * - MIN + - Qwen2.5-0.5B + - GRPO-LoRA + - 1*H100 + - 116 + - fsdp + - vllm0.8.3 + - `qwen2-0.5b_grpo-lora_1_h100_fsdp_vllm.sh `_ + - `SimonHuang `_ + +1.5B +~~~ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Tag + - Model + - Task + - Resource + - MaxBatch + - Train + - Infer + - Link + - Contributor + * - MIN + - Qwen2.5-1.5B + - GRPO-LoRA + - 1*H100 + - 128 + - fsdp + - vllm0.8.3 + - `qwen2-1.5b_grpo-lora_1_h100_fsdp_vllm.sh `_ + - `SimonHuang `_ + +3B +~~~ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Tag + - Model + - Task + - Resource + - MaxBatch + - Train + - Infer + - Link + - Contributor + * - MIN + - Qwen2.5-3B + - GRPO-LoRA + - 1*H100 + - 62 + - fsdp + - vllm0.8.3 + - `qwen2-3b_grpo-lora_1_h100_fsdp_vllm.sh `_ + - `SimonHuang `_ + +7B +~~~ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Tag + - Model + - Task + - Resource + - MaxBatch + - Train + - Infer + - Link + - Contributor + * - MIN + - Qwen2-7B + - GRPO + - 2*H800 + - \ + - fsdp + - vllm0.8.2 + - `qwen2-7b_grpo_2_h800_fsdp_vllm `_ + - `Xiangyongan `_ + * - MIN + - Qwen2.5-7B + - GRPO-LoRA + - 1*H100 + - 16 + - fsdp + - vllm0.8.3 + - `qwen2-7b_grpo-lora_1_h100_fsdp_vllm.sh `_ + - `SimonHuang `_ + +14B +~~~ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Tag + - Model + - Task + - Resource + - MaxBatch + - Train + - Infer + - Link + - Contributor + * - MIN + - Qwen2-14B + - GRPO + - 4*H800 + - \ + - fsdp + - vllm0.8.2 + - `qwen2-14b_grpo_4_h800_fsdp_vllm `_ + - `Xiangyongan `_ + * - MIN + - Qwen2.5-14B + - GRPO-LoRA + - 2*H100 + - 116 + - fsdp + - vllm0.8.3 + - `qwen2-14b_grpo-lora_2_h100_fsdp_vllm.sh `_ + - `SimonHuang `_ + +32B +~~~ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Tag + - Model + - Task + - Resource + - MaxBatch + - Train + - Infer + - Link + - Contributor + * - MIN + - Qwen2-32B + - GRPO + - 8*H20 + - \ + - megatron + - vllm0.8.2 + - `qwen2-32b_grpo_8_h20_megatron_vllm `_ + - `Xiangyongan `_ + * - MIN + - Qwen2.5-32B + - GRPO-LoRA + - 4*H100 + - 180 + - fsdp + - vllm0.8.3 + - `qwen2-32b_grpo-lora_4_h100_fsdp_vllm.sh `_ + - `SimonHuang `_ + +70B +~~~ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Tag + - Model + - Task + - Resource + - MaxBatch + - Train + - Infer + - Link + - Contributor + * - MIN + - Qwen2-70B + - GRPO + - 32*H20 + - \ + - fsdp + - vllm0.8.2 + - `qwen2-70b_grpo_32_h20_fsdp_vllm `_ + - `Xiangyongan `_ + * - MIN + - Qwen2-70B + - GRPO + - 32*H800 + - \ + - fsdp + - vllm0.8.3 + - `qwen2-70b_grpo_32_h800_fsdp_vllm `_ + - `Xiangyongan `_ + * - MIN + - Qwen2.5-72B + - GRPO-LoRA + - 8*H100 + - 176 + - fsdp + - vllm0.8.3 + - `qwen2-72b_grpo-lora_8_h100_fsdp_vllm.sh `_ + - `SimonHuang `_ + +405B +~~~~ + +.. table:: + :widths: auto + + ====== ====== ====== ======== ======== ====== ====== ====== + tag model task resource MaxBatch train infer link + ====== ====== ====== ======== ======== ====== ====== ====== + \ \ \ \ \ \ \ + ====== ====== ====== ======== ======== ====== ====== ====== + +671B +~~~~ + +.. table:: + :widths: auto + + ====== ====== ====== ======== ======== ====== ====== ====== + tag model task resource MaxBatch train infer link + ====== ====== ====== ======== ======== ====== ====== ====== + \ \ \ \ \ \ \ + ====== ====== ====== ======== ======== ====== ====== ====== diff --git a/e2c/verl/docs/perf/dpsk.md b/e2c/verl/docs/perf/dpsk.md new file mode 100644 index 0000000000000000000000000000000000000000..7ea5bd196c3a63cc8d5e06189eb8dc92400136ce --- /dev/null +++ b/e2c/verl/docs/perf/dpsk.md @@ -0,0 +1,88 @@ +# Training DeepSeek 671b + +Last updated: 08/20/2025. + +verl integrates Megatron to support large MoE models such as `Qwen3-235B-A22B` and `deepseek-ai/DeepSeek-V3`. This is an ongoing community effort. + +In the journey the community added the following features and optimizations that enable verl with larger models: +- per tensor weight resharding between rollout and training +- context parallelism and expert parallelism enabled via megatron +- dynamic batch size (sequence balance) for megatron +- reduced ray-related serialization overhead +- optimizer offloading, recomputation, and efficient kernels +- various debugging metrics and utils +- hybrid optimizer + +and the megatron backend now has a wider list of models supported: +- DeepSeek-V3 +- Moonlight +- Qwen3 +- Qwen2.5-VL (to be merged soon) +- Qwen2 +- Mixtral + +## Getting Started + +### preparation +The recommended image with pre-built Megatron dependency is `verlai/verl:app-verl0.4-vllm0.8.5-mcore0.13.0-preview`, which is built using the Dockerfile at [docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.13.preview](https://github.com/volcengine/verl/blob/main/docker/verl0.4-cu124-torch2.6-fa2.7.4/Dockerfile.app.vllm.mcore0.13.preview). + +The image is build in Hopper GPUs with DeepEP. It does not support None-Hopper GPUs, such as A100. You may need to reinstall DeepEP to work with A100. + +With `OFFLOAD_FRACTION=1`, the system's minimum requirements are lowered. It can run on as few as 96 H20 (96GB) GPUs for DeepSeek-V3, and on as few as 32 H20 (96GB) GPUs for Qwen3-235B-A22B. However, this configuration will use 1.6TB CPU memory per node. If you run out of CPU memory or require faster training speed, you can add more nodes. + +### DeepSeek 671b + +For DeepSeek-V3 671b, please refer to [examples/grpo_trainer/run_deepseek671b_math_megatron_96gb.sh](https://github.com/volcengine/verl/blob/main/examples/grpo_trainer/run_deepseek671b_math_megatron_96gb.sh). + +MTP and quantilization is disabled during RL training. + +To train your project, configure the following environment variables based on the number of available GPUs. These are recommended settings and can be adjusted based on your specific hardware. +| num gpus | NNODES | TP | PP | EP | OFFLOAD_FRACTION | OFFLOAD_OPTIM | LAST_LAYER | +| -- | -- | -- | -- | -- | -- | -- | -- | +| 96 | 12 | 8 | 12 | 8 | 1. | False | 6 | +| 128 | 16 | 8 | 16 | 8 | 0.5 | True | 1 | +| 256 | 32 | 8 | 16 | 8 | 0. | True | 1 | +| 512 | 64 | 1 | 16 | 32 | 0 | True | 1 | + +### Qwen3 235b + +For Qwen3-235b, please refer to [examples/grpo_trainer/run_qwen3-235b_megatron_96gb.sh](https://github.com/volcengine/verl/blob/main/examples/grpo_trainer/run_qwen3-235b_megatron_96gb.sh). + +To train your project, configure the following environment variables based on the number of available GPUs. These are recommended settings and can be adjusted based on your specific hardware. +| num gpus | NNODES | TP | PP | EP | OFFLOAD_FRACTION | OFFLOAD_OPTIM | LAST_LAYER | +| -- | -- | -- | -- | -- | -- | -- | -- | +| 32 | 4 | 4 | 8 | 4 | 1. | False | 6 | +| 64 | 8 | 4 | 8 | 4 | 0.5 | True | 6 | +| 128 | 16 | 4 | 8 | 4 | 0 | True | 6 | +| 256 | 32 | 4 | 8 | 4 | 0 | True | 6 | + +### Benchmark +Here are some benchmark results for DeepSeek / Qwen3-235B. All configurations match the recommended settings based on the number of GPUs. + +| model | num gpus | mean response length | rollout time(s) | GPU memory(GB) | CPU memory(GB) | MFU | step time(s) | +| -- | -- | -- | -- | -- | -- | -- | -- | +| DeepSeek 671b | 96 | 1960 | 1050 | 66 | 1500 | 0.19 | 1700 | + +### Qwen3-30B-A3B MOE + +For Qwen3-30b, please refer to [examples/grpo_trainer/run_qwen3moe-30b_megatron_96gb.sh](https://github.com/volcengine/verl/blob/main/examples/grpo_trainer/run_qwen3moe-30b_megatron_96gb.sh). + +To train your project, configure the following environment variables based on the number of available GPUs. These are recommended settings and can be adjusted based on your specific hardware. +| num gpus | NNODES | TP | PP | EP | OFFLOAD_FRACTION | OFFLOAD_OPTIM | MFU | +| -- | -- | -- | -- | -- | -- | -- | -- | +| 8 | 1 | 1 | 1 | 8 | 1. | True | 0.4 | +| 16 | 2 | 1 | 1 | 8 | 1. | True | 0.37 | +| 32 | 4 | 1 | 1 | 8 | 1. | True | 0.31 | + + +## Upcoming Optimizations + +The community continue to optimize large MoE models further, ongoing efforts include: +- further optimizing memory consumption, and provide recommended/tuned configurations with various machine types +- optimizing long context RL training performance +- performance improvement with SGLang x Megatron + +We invite the community to try and improve verl together. Get connected with us on [slack](https://join.slack.com/t/verlgroup/shared_invite/zt-2w5p9o4c3-yy0x2Q56s_VlGLsJ93A6vA)/[wechat](https://raw.githubusercontent.com/eric-haibin-lin/verl-community/refs/heads/main/WeChat.JPG)/[Github issues](https://github.com/volcengine/verl/issues/708)! + +## Acknowledgement +@vermouth1992 @ISEEKYAN @ETOgaosion @yzlnew @ShareLer @BearBiscuit05 @ccclyu @ann-qin-lu @SwordFaith @zzong2006 @zhaochenyang20 @ocss884 @eric-haibin-lin @chenhaiq @techkang diff --git a/e2c/verl/docs/perf/nsight_profiling.md b/e2c/verl/docs/perf/nsight_profiling.md new file mode 100644 index 0000000000000000000000000000000000000000..490de5e7e4f7b6ba6c0e372eb7c0c3bfce2a77b9 --- /dev/null +++ b/e2c/verl/docs/perf/nsight_profiling.md @@ -0,0 +1,94 @@ +# NVIDIA Nsight Systems profiling in verl + +Last updated: 06/20/2025. + +This guide explains how to use NVIDIA Nsight Systems for profiling verl training runs. + +## Configuration + +Profiling in verl can be configured through several parameters in the trainer configuration file (ppo_trainer.yaml or other files like dapo_trainer.yaml): + +### Prerequisites + +Nsight Systems version is important, please reference `docker/Dockerfile.vllm.sglang.megatron` for the version we used. + +### Global profiling control + +verl has one single controller process and multiple worker processes. Both controller and worker processes can be profiled. Since the controller process can be executed in any nodes in the cluster, there is a message printed in the logging to indicate the controller process node hostname and process id. + +In `global_profiler`, three new config entries control the profiler behaviors: + +* **`global_profiler.steps`**. List of step numbers at which profiling should be performed. For example: [1, 2, 5] will profile steps 1, 2, and 5. And ``null`` means no profiling. + +* **`global_profiler.profile_continuous_steps`**. If true, and the following `global_profiler.discrete==False`, then the continuous steps in `global_profiler.steps` will be combined into one database. For example the above step 1 and 2 are in one database, and 5 in another. If false, every step occupies at least one database. The reason for this config is to observe the program behaviors between steps. + +Nsys options in controller nodes and worker nodes are configured in `global_profiler.global_tool_config.nsys`: + +* **`global_profiler.global_tool_config.nsys.controller_nsight_options`**. This config group is for the single controller. All fields in this config group will be just sent to Nsight Systems when Ray starts the controller process. `ppo_trainer.yaml` provides a workable example. Users can reference [Nsight Systems manual](https://docs.nvidia.com/nsight-systems/UserGuide/index.html) and [Ray user guide](https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html) for more details. +* **`global_profiler.global_tool_config.nsys.worker_nsight_options`**. This config group is for the worker processes. Similarly all fields in this config group will be just sent to Nsight Systems when Ray starts the controller process. Capture range is used to control the profiler when to start and stop. So `capture-range: "cudaProfilerApi"` is fixed and does not change it. Users can change `capture-range-end` with some accurate calculation or just leave it `null`. + +### Worker process profiling + +Verl manages mulitiple RL roles, _Actor_, _Ref_, _Rollout_, _Critic_, _Reward_, which are implemented in different Worker classes. And these workers can be combined into one Ray Actor, running in a process group. Each RL role has its own profiling config group, `profiler`, which consists of three fields: + +* **`all_ranks` and `ranks`**. When `all_ranks` is set `True` then all ranks will be profiled; when set `False`, `ranks` will be profiled. By default, verl profiles the whole training process in a series ` worker_process_..nsys-rep` files for each process rank. PID is the process ID; RID is the capture range ID. +* **`discrete`**. When set `False`, all the roles actions in one training step will be dumped in one database. When set `True`, the actions annotated by `DistProfiler.annotate` will be dumped into a discrete database. In this case, each role's action occupies one ``. +* **Verl collocate mode**. Verl can combine two Worker sub classes to one Worker Actor. In this case, the user should take care that the combined Workers have consistent `discrete`. The Nsight Systems profiler uses a `torch.cuda.profiler.start()` and `stop()` pair to dump a `` database anyway. + +### where to find the profiling data + +By default the `*.nsys-rep` files are saved in the directory `/tmp/ray/session_latest/logs/nsight/` at each node. According to the Ray manual, this default directory is not changeable. ["however, Ray preserves the `--output` option of the default config"](https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html). + +Some users may think it is not convenient, but it is understandable that Ray may start hundreds of processes and it would be a big network file system pressure if we save the files in one central place. + +## Usage Example + +To enable profiling for specific components and steps, modify your ppo_trainer.yaml like this: + +### Disable profiler + +```yaml + profiler: + steps: null # disable profile +``` + +### Enable profiler and one database for one training step + +```yaml + global_profiler: + steps: [1, 2, 5] + discrete: False + actor_rollout_ref: + actor: + profiler: + enable: True + all_ranks: True + # rollout & ref follow actor settings + critic: + profiler: + enable: True + all_ranks: True + reward_model: + profiler: + enable: True + all_ranks: True +``` + +### Enable profiler and multiple databases for one training step + +```yaml + profiler: + steps: [1, 2, 5] + discrete: True +``` + +## Profiling Output + +When profiling is enabled, verl will generate Nsight Systems profiles for the specified components and steps. The profiles will include: + +- CUDA kernel execution +- Memory operations +- CPU-GPU synchronization +- NVTX markers for key operations + +Nsight Systems supports multi-report view, to open multiple databases together. In this mode, different processes and steps can be aligned in one time line for better analysis. diff --git a/e2c/verl/docs/perf/perf_tuning.rst b/e2c/verl/docs/perf/perf_tuning.rst new file mode 100644 index 0000000000000000000000000000000000000000..b5edd50c4dfc88afdf18f2525c44fb882dc96eaf --- /dev/null +++ b/e2c/verl/docs/perf/perf_tuning.rst @@ -0,0 +1,224 @@ +Performance Tuning Guide +============================== + +Last updated: 07/17/2025. + +Author: `Guangming Sheng `_, `Jiali Zheng `_ + +In this section, we will discuss how to tune the performance of all the stages in verl, including: + +1. Rollout generation throughput. + +2. Enable ``use_remove_padding=True`` for sequence packing (i.e., data packing and remove padding). + +3. Batch size tuning for forward and backward computation + +4. Enable ``use_dynamic_bsz=True`` for higher throughput. + +5. Utilize Ulysses Sequence Parallel for Long Context Training + +6. LigerKernel for SFT performance optimization + +7. Forward prefetch in FSDP training backend + +8. Memory optimization for entropy calculation from logits + +Rollout Generation Tuning +-------------------------- + +verl currently supports two rollout backends: vLLM and TGI (with SGLang support coming soon). + +Below are key factors for tuning vLLM-based rollout. Before tuning, we recommend setting ``actor_rollout_ref.rollout.disable_log_stats=False`` so that rollout statistics are logged. + +- Increase ``gpu_memory_utilization``. + + - For vLLM v0.7.0 and later, the vLLM instance will only use gpu_memory_utilization of the **total** memory. + - For SGLang, it's the fraction of the free GPU memory used for **static** memory like model weights and KV cache. However, the remaining (1-gpu_memory_utilization) will also be used during inference. + + However, if model parameters and optimizer states are not offloaded, using too high a fraction can lead to OOM. + A value between 0.5 and 0.7 often strikes a good balance between high throughput and avoiding OOM. + + Note: since the definition of ``gpu_memory_utilization`` varies across inference engines, a value that works well for one engine may cause OOM for another. + +- Adjust ``max_num_seqs`` or ``max_num_batched_tokens``. + If the GPU cache utilization is relatively low in the log, increase ``max_num_seqs`` or ``max_num_batched_tokens`` + can enlarge the effective batch size in the decoding stage, allowing more concurrent requests per batch. + We recommend setting ``max_num_batched_tokens > 2048`` for higher throughput. + +- Use a smaller ``tensor_parallel_size``. + When GPU resources allow, a smaller tensor parallel size spawns more vLLM replicas. + Data parallelism (DP) can yield higher throughput than tensor parallelism (TP), but also increases KVCache consumption. + Carefully balance the trade-off between more replicas and higher memory usage. + Our experiment in Sec. 8.4 of `HybridFlow paper `_ evaluate this trade-off. + +- Balance performance and memory using ``cudagraph_capture_sizes``. + If ``cudagraph_capture_sizes`` is set, vLLM will try to capture the model execution graph for different batch sizes. + Since cudagraph memory can not be offloaded to cpu, The memory stay in gpu when update actor is running. + Using smaller batch sizes can avoid OOM but slightly reduce throughput. + Must to set ``enforce_eager=False`` to use ``cudagraph_capture_sizes``. + +More tuning details such as dealing with Preemption and Chunked-prefill +can be found in `vLLM official tuning guide `_ + +For optimal performance, we recommend using vLLM v0.8.3 or later. See https://github.com/volcengine/verl/blob/main/docs/README_vllm0.8.md for details. + +Enable remove padding (sequence packing) +----------------------------------------- + +Currently, for llama, mistral, gemma1 and qwen based models, users can enable `use_remove_padding=True` to utilize the +sequence packing implementation provided by transformers library. + +For other models, transformers library may also support it but we haven't tested it yet. +Users can add the desired model config to the `test_transformer.py `_ file. +And test its functionality by running the following command: + +.. code-block:: bash + + pytest -s tests/models/test_transformer.py + +If the test passes, you can add your desired model into the model `registry.py `_ file. +Then, you can enjoy the performance boost of sequence packing +and welcome to PR your tested model to verl! + + +Batch Size Tuning +----------------- + +To achieve higher throughput in experience preparation (i.e., model fwd) and model update (i.e., actor/critic fwd/bwd), +users may need to tune the ``*micro_batch_size_per_gpu`` for different computation. + +In verl, the core principle for setting batch sizes is: + +- **Algorithmic metrics** (train batch size, PPO mini-batch size) are *global* (from a single-controller perspective), + normalized in each worker. See the `normalization code `_. + +- **Performance-related parameters** (micro batch size, max token length for dynamic batch size) are *local* parameters that define the per-GPU data allocations. + See the `normalization code `_. + +.. note:: In your training script, please use ``*micro_batch_size_per_gpu`` instead of ``*micro_batch_size``. + So that you don't need to consider the normalization of the ``micro_batch_size`` and ``micro_batch_size`` will be deprecated. + +Batch Size Tuning tips +"""""""""""""""""""""" + +Therefore, users may need to tune the ``*micro_batch_size_per_gpu`` to accelerate training. Here're some tips: + +1. **Enable gradient checkpointing**: + Set ``actor_rollout_ref.model.enable_gradient_checkpointing=True`` and ``critic.model.enable_gradient_checkpointing=True``. + This often allows for larger micro-batch sizes and will be beneficial for large mini-batch training. + +2. Increase the ``*micro_batch_size_per_gpu`` as much as possible till equals to normalized ``mini_batch_size``. + +3. **Use larger forward-only parameters**: + Forward only parameter, such as ``actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu``, + ``actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu``, ``critic.forward_micro_batch_size_per_gpu`` could be larger (e.g., 2x) than training related micro batch sizes, + such as ``actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu``, ``critic.ppo_micro_batch_size_per_gpu``. + +4. **Allow larger micro-batch sizes for Critic and Reward models**: + micro batch size of Critic and Reward model could be larger than Actor model. This is because the actor model has much larger vocab size in the final layer. + +5. **Enable activation offloading**: + Set ``actor_rollout_ref.model.enable_activation_offload=True`` and ``critic.model.enable_activation_offload=True``. + This often works together with gradient checkpointing to get larger micro-batch sizes and it's only available in FSDP backend now. + +Tuning for Dynamic Batch Size +----------------------------- + +Dynamic batch size is a technique that allows the model to process similar number of tokens in a single forward pass (with different actual batch sizes). +This can significantly improve the training efficiency and reduce the memory usage. + +To utilize this technique, users can set ``use_dynamic_bsz=True`` in actor, ref, critic and reward models. +With ``use_dynamic_bsz=True``, users don't need to tune ``*micro_batch_size_per_gpu``. +Instead, users should tune the following parameters: + +- ``actor_rollout_ref.actor.ppo_max_token_len_per_gpu``, ``critic.ppo_max_token_len_per_gpu``: + The maximum number of tokens to be processed in fwd and bwd of ``update_policy`` and ``update_critic``. + +- ``actor_rollout_ref.ref.log_prob_max_token_len_per_gpu`` and ``actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu``: + The maximum number of tokens to be processed in a the fwd computation of ``compute_log_prob`` and ``compute_ref_log_prob``. + +- ``critic.forward_micro_batch_size_per_gpu``, ``reward_model.forward_micro_batch_size_per_gpu``: + The maximum number of tokens to be processed in a the fwd computation of ``compute_values``, ``compute_rm_score``. + +Dynamic Batch Size Tuning tips +"""""""""""""""""""""""""""""" + +Here're some tips to tune the above parameters: + +1. **Increase** ``actor_rollout_ref.actor.ppo_max_token_len_per_gpu`` + Make it at least 2 x (max_prompt_length + max_response_length). We set it to 3x in `run_qwen2-7b_rm_seq_balance.sh `_. + Try to increase it to get higher throughput. + +2. **Forward-only parameters can be larger**: + Similar to the non-dynamic-batch scenario, forward-only token limits can exceed those used in forward/backward operations. + +3. **Use larger limits for Critic and Reward models**: + Critic and Reward parameters can be set at least 2× the Actor’s limits. For instance, we set them to 4× here: + `run_qwen2-7b_rm_seq_balance.sh `_ + +.. :math:`\text{critic.ppo_max_token_len_per_gpu} = 2 \times \text{actor.ppo_max_token_len_per_gpu})`. + +Ulysses Sequence Parallel for Long Context Training +---------------------------------------------------- + +To utilize this technique, users can set ``ulysses_sequence_parallel_size>1`` in actor, ref, critic and reward models. + +We support different model utilize different ulysses_sequence_parallel_size sizes. + +To train long sequence (>32k), users may need to decrease the ``*micro_batch_size_per_gpu`` and ``*max_token_len_per_gpu`` to avoid OOM. + +LigerKernel for SFT +---------------------- + +LigerKernel is a high-performance kernel for Supervised Fine-Tuning (SFT) that can improve training efficiency. To enable LigerKernel in your SFT training: + +1. Install liger-kernel via ``pip3 install liger-kernel``. In your SFT configuration file (e.g., ``verl/trainer/config/sft_trainer.yaml``), set the ``use_liger`` parameter: + + .. code-block:: yaml + + model: + use_liger: True # Enable LigerKernel for SFT + +2. The default value is ``False``. Enable it only when you want to use LigerKernel's optimizations. + +3. LigerKernel is particularly useful for improving training performance in SFT scenarios. + +Forward prefetch in FSDP training backend +---------------------- + +During the training phase, users can enable forward prefetching in FSDP by setting ``fsdp_config.forward_prefetch=True``. For example, ``actor_rollout_ref.actor.fsdp_config.forward_prefetch=True``. This configuration prefetches the next forward-pass all-gather operation before completing the current forward computation, overlapping communication with computation and improving efficiency. For further details, refer to the `FSDP forward_prefetch `_ documentation. + +.. note:: + Backward prefetch is unsupported because the ``BACKWARD_POST`` policy may prefetch incorrectly in nested-module cases. For details, see the `FSDP documentation `_ + +Migrating to FSDP2 +---------------------- + +FSDP2 offers notable improvements over FSDP1. According to `PyTorch TorchTitan benchmarks `_: + +- 7% lower GPU memory usage on average +- 1.5% throughput improvement with BF16 training +- Better composability with DTensor and per-parameter sharding + +**Enabling FSDP2 in VERL:** + + .. code-block:: python + + # Enable FSDP2 in actor configuration + actor_rollout_ref.actor.strategy="fsdp2" + +.. note:: + FSDP2 requires PyTorch 2.1+ and is recommended for models with transformer architecture. + +Memory optimization for entropy calculation from logits +---------------------- + +The ``logits`` tensor (typically of shape ``[bsz*seq_len, voc]``) can consume significant memory. When using ``compute_entropy_from_logits``, memory usage reaches approximately ``[bsz*seq_len, voc] × (4 bytes (float32) + 2 bytes (autocast for softmax+logsumexp) + 1 byte (softmax output))``. + +To reduce this memory peak, enable chunked computation by setting: +``actor_rollout_ref.ref.entropy_from_logits_with_chunking = True`` +This processes the tensor in chunks of shape ``[chunk_size, voc]`` (e.g., 2048) rather than the full sequence length, exclusively during the model's forward pass. + +Additionally, during training, standard gradient checkpointing (``enable_gradient_checkpointing=True``) does not apply to entropy calculations. To reduce memory peaks in this context, set: +``actor_rollout_ref.actor.entropy_checkpointing = True`` +This enables entropy recomputation specifically for the entropy calculation, lowering memory usage during training. diff --git a/e2c/verl/docs/perf/verl_profiler_system.md b/e2c/verl/docs/perf/verl_profiler_system.md new file mode 100644 index 0000000000000000000000000000000000000000..fc7ecc38eed92ca5e05274e23f40b6f1ce7033b0 --- /dev/null +++ b/e2c/verl/docs/perf/verl_profiler_system.md @@ -0,0 +1,36 @@ +# verl Profiler System + +Last updated: 08/18/2025. + +## Architecture + +The architecture of verl profiler system is like below: + +![verl-profiler-arch](https://raw.githubusercontent.com/eric-haibin-lin/verl-community/2bc7ed0ba2f37f21707bfac3b241eca4b86d1bc6/docs/verl_profiler_arch.png) + +There is a global profiler and tool configuration to set some common config in single controller level, deciding + +- `tool`: which tool to use +- `steps`: which steps to profile +- `save_path`: results saving path + +When some tool need to profile behavior of each role, configurations in role-level is needed: + +- `tool`: which tool to use +- `enable`: whether enable profiling on this role +- rank info: `all_ranks` and `rank` to decide which rank to profile or log output + +For tool config in role-level, there are some detailed behavior needed to control, like the `discrete` mode in nsys profiler. + +Every role has a profiler config, and by default, rollout/ref/reward models follow the Actor's behavior. + +## To Add a new profiling tool + +New added profiling tool shall reuse the current APIs as much as possible. + +1. The logic of **whether to use the tool**: `tool == [new tool]`. +2. Add the global and local tool config to `ppo_trainer.yaml`/`ppo_megatron_trainer.yaml` and each `[role].yaml`, under `global_tool_config.[new tool]` and `tool_config.[new tool]` +3. The tool config should be implemented in `verl/utils/profiler/config.py`, inherit the `BaseConfig` class. +4. Implement profiling tool initialization logic using configurations in `global_profiler.global_tool_config.[new tool]` and the results saving logics (can also save in role-level profile) +5. For role function-level profiling, please follow the nsys profiler way in `nvtx_profiler.py`, implement a profiler class inherit `DistProfiler` and import new profiler in `verl/utils/profiler/__init__.py` +6. Add unit test and examples for others to use in convinience. \ No newline at end of file diff --git a/e2c/verl/docs/preparation/prepare_data.rst b/e2c/verl/docs/preparation/prepare_data.rst new file mode 100644 index 0000000000000000000000000000000000000000..c429e4b167967652a0c3fb52d9e0029f1b9899d4 --- /dev/null +++ b/e2c/verl/docs/preparation/prepare_data.rst @@ -0,0 +1,128 @@ +Prepare Data for Post-Training +======================================== + +Last updated: 02/09/2025. + +Before starting the post-training job, we need to prepare the data for +the policy training. The data should be stored in the parquet format. + +We provide several data preprocess scripts for different datasets, +including GSM8K, MATH, HelloSwag, Full_hh_rlhf. To prepare other datasets, we need +to follow the following steps: The data preprocess script can be divided +into two parts: + +1. The first part is the common part, which loads the dataset from + huggingface's ``datasets`` package. Then preprocess the datasets with + the ``make_map_fn`` and then store in the parquet format. + +.. code:: python + + import re + import os + import datasets + + from verl.utils.hdfs_io import copy, makedirs + import argparse + + # To extract the solution for each prompts in the dataset + # def extract_solution(solution_str): + # ... + + + if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--local_dir', default='/opt/tiger/gsm8k') + parser.add_argument('--hdfs_dir', default=None) + + args = parser.parse_args() + + num_few_shot = 5 + data_source = 'openai/gsm8k' + + dataset = datasets.load_dataset(data_source, 'main') + + train_dataset = dataset['train'] + test_dataset = dataset['test'] + + # Construct a `def make_map_fn(split)` for the corresponding datasets. + # ... + + train_dataset = train_dataset.map(function=make_map_fn('train'), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn('test'), with_indices=True) + + local_dir = args.local_dir + hdfs_dir = args.hdfs_dir + + train_dataset.to_parquet(os.path.join(local_dir, 'train.parquet')) + test_dataset.to_parquet(os.path.join(local_dir, 'test.parquet')) + + makedirs(hdfs_dir) + + copy(src=local_dir, dst=hdfs_dir) + +2. The users are required to implement the ``make_map_fn()`` function + (as well as the ``extract_solution``) on their own to support + different datasets or tasks. + +We already implemented the data preprocess of GSM8k, MATH, Hellaswag and Full_hh_rlhf +datasets. And we take the GSM8k dataset as an example: + +**GSM8K** + +In the ``make_map_fn``, each data field should consist of the following +5 fields: + +1. ``data_source``: The name of the dataset. To index the corresponding + reward function in the ``RewardModel`` +2. ``prompt``: This field should be constructed in the format of + huggingface chat_template. The tokenizer in ``RLHFDataset`` will + apply chat template and tokenize the prompt. +3. ``ability``: Define the task category. +4. ``reward_model``: Currently, we only utilize the ``ground_truth`` + field during evaluation. The ``ground_truth`` is computed by the + ``extract_solution`` function. **NOTED** that the implementation of + the corresponding reward function should align with this extracted + ``ground_truth``. +5. ``extra_info``: Record some information of the current prompt. Not + use for now. + +.. code:: python + + def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) # extract the solution after #### + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split('#### ')[1].replace(',', '') + return final_solution + + instruction_following = "Let's think step by step and output the final answer after \"####\"." + + # add a row to each data item that represents a unique id + def make_map_fn(split): + + def process_fn(example, idx): + question = example.pop('question') + + question = question + ' ' + instruction_following + + answer = example.pop('answer') + solution = extract_solution(answer) + data = { + "data_source": data_source, + "prompt": [{ + "role": "user", + "content": question + }], + "ability": "math", + "reward_model": { + "style": "rule", + "ground_truth": solution + }, + "extra_info": { + 'split': split, + 'index': idx + } + } + return data + + return process_fn diff --git a/e2c/verl/docs/preparation/reward_function.rst b/e2c/verl/docs/preparation/reward_function.rst new file mode 100644 index 0000000000000000000000000000000000000000..286e2aff49fea71e34ac706d509725cc94aece13 --- /dev/null +++ b/e2c/verl/docs/preparation/reward_function.rst @@ -0,0 +1,71 @@ +Implement Reward Function for Dataset +====================================== + +Last updated: 06/02/2025. + +For each dataset, we need to implement a reward function or utilize a reward model to compute the rewards for the generated responses. +We already pre-implemented some reward functions in `reward_score directory `_. +You can also use customized reward functions. + +Currently, we support reward functions for GSM8k and MATH datasets. For RLHF datasets (e.g., +full_hh_rlhf) and Code Generation (e.g., APPS), we utilize reward model +and SandBox (will opensource soon) for evaluation respectively. + +RewardManager +------------- + +In the entrypoint of the PPO Post-Training script `main_ppo.py `_, +we implement a ``RewardManager`` that utilize pre-implemented reward functions to compute the scores for each response. + +In the ``RewardManager``, we implemented a ``__call__`` function to +compute the score for each response. +All the reward functions are executed by ``compute_score_fn``. +The input is a ``DataProto``, which includes: + +- ``input_ids``, ``attention_mask``: ``input_ids`` and ``attention_mask`` after applying + chat_template, including prompt and response +- ``responses``: response tokens +- ``ground_truth``: The ground truth string of the current prompt. + Stored in ``non_tensor_batch`` in the ``DataProto``, which should be + preprocessed in the parquet files. +- ``data_source``: The dataset name of the current prompt. Stored in + ``non_tensor_batch`` in the ``DataProto``, which should be + preprocessed in the parquet files. + +After detokenize the responses, the responses string and the ground +truth string will be input to the ``compute_score_fn`` to compute the +score for each response. + +Reward Functions +---------------- + +Pre-implemented +~~~~~~~~~~~~~~~ + +We already pre-implemented some reward functions in `reward_score directory `_. + +- In the `GSM8k example `_, we + force the response to output the final answer after four ####, then + use string matching to compare with the ground truth. If completely + correct, score 1 point; if the format is correct, score 0.1 points; if + the format is incorrect, score 0 points. +- In the `MATH example `_, we follow + the implementation in `lm-evaluation-harness repository `_. + +Customized +~~~~~~~~~~ + +You can implement customized reward functions in a separate file and specify them using ``custom_reward_function.path`` and ``custom_reward_function.name``. For the set of them, please refer to :ref:`config-explain-page`. + +The parameters of your reward function should be ``data_source``, ``solution_str``, ``ground_truth``, and ``extra_info``. +For example: + +.. code:: python + + def my_reward_fn(data_source, solution_str, ground_truth, extra_info=None): + return len(solution_str)/100 + +If you are testing only a single customized reward function, you can simply name it 'compute_score' and leave ``custom_reward_function.name`` unset. + +To run multiple tests with different customized reward functions, you can modify both ``custom_reward_function.path`` and ``custom_reward_function.name`` for each trial. +For instance, you might create a single `my_reward.py` file and implement multiple reward functions within it. This way, for different trials, you only need to adjust ``custom_reward_function.name``, making it more convenient to conduct multiple tests within scripts. diff --git a/e2c/verl/docs/requirements-docs.txt b/e2c/verl/docs/requirements-docs.txt new file mode 100644 index 0000000000000000000000000000000000000000..55ccdb8f7149bd6b774b592dca068e63e87256db --- /dev/null +++ b/e2c/verl/docs/requirements-docs.txt @@ -0,0 +1,13 @@ +# markdown support +recommonmark +myst_parser +# markdown table support +sphinx-markdown-tables + +# theme default rtd + +# crate-docs-theme +sphinx-rtd-theme + +# pin tokenizers version to avoid env_logger version req +tokenizers==0.21 diff --git a/e2c/verl/docs/sglang_multiturn/interaction_system.rst b/e2c/verl/docs/sglang_multiturn/interaction_system.rst new file mode 100644 index 0000000000000000000000000000000000000000..812a9484eb264d79500bd0aba9607d43146bd01c --- /dev/null +++ b/e2c/verl/docs/sglang_multiturn/interaction_system.rst @@ -0,0 +1,417 @@ +Interaction System for Multi-turn RL Training +============================================= + +Last updated: 06/25/2025. + +Overview +-------- + +The verl interaction system enables dynamic, multi-turn conversational feedback during reinforcement learning training. This system allows models to engage in iterative problem-solving scenarios where interaction agents can provide corrective feedback, guidance, or evaluation based on the model's responses. + +**New in Multi-Interaction Support**: The system now supports multiple named interactions within a single training session, enabling sophisticated training scenarios where different samples can use different interaction strategies. This allows for curriculum learning, domain-specific feedback, and flexible agent switching at the sample level. + +Key features: + +- **Async-based Architecture**: Non-blocking interaction processing for distributed training +- **Instance Management**: Stateful session handling with unique instance IDs for concurrent interactions +- **SGLang Integration**: Seamless integration with SGLang rollout system for multi-turn conversations +- **Configuration-driven**: Dynamic agent loading via YAML configuration files +- **Multi-Interaction Support**: Registry system enabling multiple named interactions per rollout +- **Sample-Level Selection**: Each sample can specify which interaction to use via configuration +- **Reward Integration**: Turn-level scoring mechanism integrated with verl's reward system + +Architecture +------------ + +The interaction system follows a plugin-based architecture with clear separation of concerns: + +.. code-block:: + + Interaction Registry System + ↓ + BaseInteraction (Abstract Interface) + ↓ + Multiple Named Interactions (e.g., Gsm8kInteraction, CustomInteraction) + ↓ + SGLang Rollout Integration (interaction_map) + ↓ + Sample-Level Interaction Selection + ↓ + Async Request Lifecycle Management + +Core Components +~~~~~~~~~~~~~~~ + +**Interaction Registry System** + +The interaction registry system allows loading and managing multiple named interactions: + +.. code-block:: python + + from verl.interactions.utils.interaction_registry import initialize_interactions_from_config + + # Load multiple interactions from config + interaction_map = initialize_interactions_from_config("config.yaml") + + # Access specific interaction by name + gsm8k_interaction = interaction_map["gsm8k"] + custom_interaction = interaction_map["custom_solver"] + +**BaseInteraction Interface** + +All interaction agents must implement the ``BaseInteraction`` abstract class: + +.. code-block:: python + + from verl.interactions.base import BaseInteraction + from typing import Dict, Any, List, Tuple, Optional + + class BaseInteraction: + def __init__(self, config: Dict[str, Any]): + self.config = config + self.name: str = config.get("name", "interaction_agent") + + async def start_interaction(self, instance_id: Optional[str] = None, **kwargs) -> str: + """Initialize interaction session, return instance_id""" + + async def generate_response(self, instance_id: str, messages: List[Dict[str, Any]], **kwargs) -> Tuple[bool, str, float, Dict[str, Any]]: + """Generate response, return (should_terminate, response, score, metadata)""" + + async def calculate_score(self, instance_id: str, **kwargs) -> float: + """Calculate turn-level score for RL training""" + + async def finalize_interaction(self, instance_id: str, **kwargs) -> None: + """Clean up resources""" + +**Request Lifecycle** + +The interaction system integrates with SGLang's async rollout via state management: + +1. ``PENDING`` → Initialize interaction via ``start_interaction()`` +2. ``GENERATING`` → Model generates response +3. ``INTERACTING`` → Process response via ``generate_response()`` +4. ``GENERATING`` → Continue if not terminated, otherwise ``COMPLETED`` + +Configuration +------------- + +**Basic Setup** + +Enable interaction in your rollout configuration: + +.. code-block:: yaml + + actor_rollout_ref: + rollout: + multi_turn: + enable: true + interaction_config_path: "path/to/interaction_config.yaml" + max_user_turns: 10 + max_assistant_turns: 10 + +**Interaction Configuration File** + +Create an interaction configuration file (e.g., ``interaction_config.yaml``): + +**Single Interaction (Legacy Format)** + +.. code-block:: yaml + + interaction: + - name: "gsm8k" + class_name: "verl.interactions.gsm8k_interaction.Gsm8kInteraction" + config: {} + +**Multiple Interactions (New Format)** + +.. code-block:: yaml + + interaction: + - name: "gsm8k" + class_name: "verl.interactions.gsm8k_interaction.Gsm8kInteraction" + config: {} + - name: "custom_solver" + class_name: "custom.interactions.CustomInteraction" + config: + solver_type: "advanced" + timeout: 30 + - name: "code_verifier" + class_name: "verl.interactions.base.BaseInteraction" + config: + verification_mode: "strict" + +**Automatic Name Generation** + +If no ``name`` field is provided, the system will automatically generate one from the class name: + +.. code-block:: yaml + + interaction: + - class_name: "verl.interactions.gsm8k_interaction.Gsm8kInteraction" + config: {} + # Automatically generates name: "gsm8k" + +The system will dynamically load all specified interaction classes and make them available by name. + +Implementation Example: GSM8K +----------------------------- + +The GSM8K interaction demonstrates a complete implementation for math problem-solving scenarios: + +.. code-block:: python + + from verl.interactions.base import BaseInteraction + from verl.utils.reward_score import gsm8k + from uuid import uuid4 + + class Gsm8kInteraction(BaseInteraction): + def __init__(self, config: dict): + super().__init__(config) + self._instance_dict = {} + + async def start_interaction(self, instance_id=None, ground_truth=None, **kwargs): + if instance_id is None: + instance_id = str(uuid4()) + self._instance_dict[instance_id] = { + "response": "", + "ground_truth": ground_truth, + "reward": 0.0, + } + return instance_id + + async def generate_response(self, instance_id, messages, **kwargs): + # Extract last assistant message content + content = "" + for item in reversed(messages): + if item.get("role") == "assistant": + content = item.get("content", "") + break + + # Ensure GSM8K format (#### prefix) + self._instance_dict[instance_id]["response"] = content + + reward = await self.calculate_score(instance_id) + if reward == 1.0: + return True, "Your response is correct!", 1.0, {} + else: + return False, "Your response is incorrect! You need to reflect on your answer and try again.", 0.0, {} + + async def calculate_score(self, instance_id, **kwargs): + return gsm8k.compute_score( + self._instance_dict[instance_id]["response"], + self._instance_dict[instance_id]["ground_truth"], + method="strict", format_score=0.0, score=1.0, + ) + + async def finalize_interaction(self, instance_id, **kwargs): + del self._instance_dict[instance_id] + +Training Integration +-------------------- + +**Training Script Configuration** + +Include interaction configuration in your training command: + +.. code-block:: bash + + python3 -m verl.trainer.main_ppo \\ + --config-path="$CONFIG_PATH" \\ + --config-name='gsm8k_multiturn_grpo_w_interaction' \\ + algorithm.adv_estimator=grpo \\ + data.train_batch_size=512 \\ + data.return_raw_chat=True \\ + actor_rollout_ref.rollout.name=sglang \\ + actor_rollout_ref.rollout.multi_turn.interaction_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/interaction_config/gsm8k_interaction_config.yaml" \\ + trainer.total_epochs=15 + +**Data Requirements** + +Ensure your dataset includes interaction parameters with the ``name`` field for interaction selection: + +.. code-block:: python + + # Dataset should include interaction_kwargs in non_tensor_batch + interaction_kwargs = [ + {"name": "gsm8k", "query": "What is 2+2?", "ground_truth": "4"}, + {"name": "custom_solver", "query": "Solve: x^2 + 5x + 6 = 0", "ground_truth": "x = -2, -3"}, + {"name": "gsm8k", "query": "What is 3+3?", "ground_truth": "6"}, + ] + +**Sample-Level Interaction Selection** + +Each sample can specify which interaction to use via the ``name`` field. This enables flexible training scenarios where different samples use different interaction strategies: + +.. code-block:: python + + # Example: Math problems use GSM8K interaction, code problems use code verifier + data_samples = [ + { + "prompt": "What is 15% of 200?", + "interaction_kwargs": { + "name": "gsm8k", + "query": "What is 15% of 200?", + "ground_truth": "30" + } + }, + { + "prompt": "Write a function to check if a number is prime", + "interaction_kwargs": { + "name": "code_verifier", + "code_type": "python", + "expected_behavior": "return True for prime numbers" + } + } + ] + +**Backward Compatibility** + +If no ``name`` field is provided in ``interaction_kwargs``, the system defaults to ``"gsm8k"`` for backward compatibility. + +Best Practices +-------------- + +**Resource Management** + +- Always implement proper cleanup in ``finalize_interaction()`` +- Use unique instance IDs to avoid conflicts in concurrent training +- Handle edge cases like empty messages or malformed content + +**Performance Optimization** + +- Keep interaction logic lightweight to avoid blocking training +- Use async/await properly to maintain non-blocking behavior +- Consider caching expensive computations within interaction instances + +**Testing** + +Comprehensive testing is essential for interaction systems: + +.. code-block:: python + + import pytest + from unittest.mock import patch + + @pytest.mark.asyncio + async def test_interaction_workflow(): + interaction = YourInteraction({}) + + # Test complete workflow + instance_id = await interaction.start_interaction(ground_truth="expected_answer") + + + messages = [{"role": "user", "content": "user_content"}, {"role": "assistant", "content": "assistant_content"}] + should_terminate, response, reward, metadata = await interaction.generate_response(instance_id, messages) + + assert should_terminate in [True, False] + assert isinstance(reward, float) + + await interaction.finalize_interaction(instance_id) + +Advanced Usage +-------------- + +**Multi-Interaction Training Strategies** + +You can design sophisticated training scenarios using multiple interactions: + +.. code-block:: python + + # Example: Progressive difficulty with different interaction agents + class MathTrainingPipeline: + def create_interaction_config(self): + return { + "interaction": [ + { + "name": "basic_math", + "class_name": "verl.interactions.gsm8k_interaction.Gsm8kInteraction", + "config": {"difficulty": "easy"} + }, + { + "name": "advanced_math", + "class_name": "custom.interactions.AdvancedMathInteraction", + "config": {"difficulty": "hard", "allow_hints": True} + }, + { + "name": "competition_math", + "class_name": "custom.interactions.CompetitionMathInteraction", + "config": {"time_limit": 300, "show_steps": False} + } + ] + } + + def create_curriculum_data(self, epoch): + if epoch < 5: + return [{"name": "basic_math", ...} for _ in samples] + elif epoch < 10: + return [{"name": "advanced_math", ...} for _ in samples] + else: + return [{"name": "competition_math", ...} for _ in samples] + +**Custom Scoring Functions** + +You can integrate custom reward functions: + +.. code-block:: python + + async def calculate_score(self, instance_id, **kwargs): + response = self._instance_dict[instance_id]["response"] + ground_truth = self._instance_dict[instance_id]["ground_truth"] + + # Custom evaluation logic + if custom_evaluation_function(response, ground_truth): + return 1.0 + else: + return 0.0 + +**Multi-step Interactions** + +For complex scenarios requiring multiple feedback rounds: + +.. code-block:: python + + async def generate_response(self, instance_id, messages, **kwargs): + instance = self._instance_dict[instance_id] + instance["attempts"] += 1 + + # Evaluate current response + reward = await self.calculate_score(instance_id) + + if reward > 0.8: + return True, "Excellent work!", reward, {} + elif instance["attempts"] < 3: + return False, "Good attempt, but try to improve...", reward, {} + else: + return True, "Maximum attempts reached.", reward, {} + +Troubleshooting +--------------- + +**Common Issues** + +1. **Instance ID Conflicts**: Ensure unique instance IDs across concurrent sessions +2. **Memory Leaks**: Always call ``finalize_interaction()`` to clean up resources +3. **Blocking Operations**: Keep interaction logic async and non-blocking +4. **Configuration Errors**: Verify interaction config path and class name are correct +5. **Interaction Name Conflicts**: Ensure all interactions have unique names in the configuration +6. **Missing Interaction**: Verify the ``name`` field in ``interaction_kwargs`` matches available interactions +7. **Backward Compatibility**: When migrating from single to multi-interaction, add ``name`` fields to existing data + +**Debugging** + +Enable debug logging to trace interaction flow: + +.. code-block:: bash + + export VERL_LOGGING_LEVEL=DEBUG + +**Performance Monitoring** + +Monitor interaction performance impact on training throughput and adjust accordingly. + +Related Documentation +-------------------- + +- :doc:`multiturn`: Basic multi-turn rollout configuration +- :doc:`sandbox_fusion`: Tool integration with SGLang +- :doc:`search_tool_example`: Search tool implementation example \ No newline at end of file diff --git a/e2c/verl/docs/sglang_multiturn/multiturn.rst b/e2c/verl/docs/sglang_multiturn/multiturn.rst new file mode 100644 index 0000000000000000000000000000000000000000..fb3c29693cc5aab7a7a50a0b274a2ea653daaa4b --- /dev/null +++ b/e2c/verl/docs/sglang_multiturn/multiturn.rst @@ -0,0 +1,354 @@ +Multi-turn Rollout Support +========================== + +Last updated: 06/27/2025. + +Basic Configuration +~~~~~~~~~~~~~~~~~~~ + +To enable multi-turn rollout, make sure to configure the following fields in your rollout configuration: + +.. code-block:: yaml + + actor_rollout_ref: + rollout: + multi_turn: True + name: "sglang" + +These configuration activates the sglang engine for multi-turn interaction during rollout. + +Custom Tool Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~ + +For custom environment interaction tools, you can implement your own tools based on ``verl.tools.base_tool.BaseTool``. Then, specify your tool configurations in a YAML file: + +.. code-block:: yaml + + tools: + - class_name: "" + config: + type: native + tool_schema: + +You may refer to GSM8KTool_example_configuration_, which is one example of the tool configurations. Its implementation can be found in gsm8k_tool.py_. + +Finally, set the ``tools_config_file`` in your rollout config: + +.. code-block:: yaml + + actor_rollout_ref: + rollout: + tool_kwargs: + tools_config_file: + +This allows integration of customized tool behaviors during actor rollout steps. + +If you want rollout with simulated interaction, you can set the ``interaction_config_file`` in your rollout config: + +.. code-block:: yaml + + interaction: + - class_name: "" + config: {} + +.. code-block:: yaml + + actor_rollout_ref: + rollout: + interaction_config_file: + +If your tool creates multi-modal inputs, you should return a list of multi-modal inputs in your tool.execute() implementation. + +Image and video should be processed before returning. For example, if you are using Qwen2.5-VL, you can use the following code to get the representations: + +.. code-block:: python + + async def create(self, ...) -> tuple[str, ToolResponse]: + ... + from verl.utils.dataset.vision_utils import process_image, process_video + + img1 = process_image(img1) + video1 = process_video(video1) + + # due to the (image | video) key is ("image" | "video") instead of ("images" | "videos") in vllm, we need to use ("image" | "video") to specify list of images/videos + # link: https://github.com/vllm-project/vllm/blob/3c545c0c3b98ee642373a308197d750d0e449403/vllm/multimodal/parse.py#L205 + return instance_id, ToolResponse(image=[img1, ...], video=[video1, ...], text="...") + + async def execute(self, ...) -> Tuple[str | Dict[str, Any], float, dict]: + ... + from verl.utils.dataset.vision_utils import process_image, process_video + + img1 = process_image(img1) + video1 = process_video(video1) + + # due to the (image | video) key is ("image" | "video") instead of ("images" | "videos") in vllm, we need to use ("image" | "video") to specify list of images/videos + # link: https://github.com/vllm-project/vllm/blob/3c545c0c3b98ee642373a308197d750d0e449403/vllm/multimodal/parse.py#L205 + return ToolResponse(image=[img1, ...], video=[video1, ...], text="..."), 0, {} + +remeber to set ``return_multi_modal_inputs: False`` in your dataset config in order to process the multi-modal inputs in the rollout correctly. +Refer to the `Handling Multi-Modal Inputs in Datasets`_ section for more details. + +MCP Tool Configuration +~~~~~~~~~~~~~~~~~~~~~~ + +For MCP interaction tools, you can flexibly configure them using a YAML file. The typical setup is as follows: + +.. code-block:: yaml + + tools: + - class_name: "" + config: + type: mcp + mcp: + mcp_servers_config_path: ./mcp_server.json + tool_selected_list: {} + +The ``tool_selected_list`` field is optional and specifies which tools to use from the servers. If you want to enable all available tools, simply omit this attribute. Besides, ``mcp_servers_config_path`` points to a JSON file containing the MCP server configurations. For example: + +.. code-block:: json + + { + "mcpServers": { + "SSE Server": { + "url": "your_server_url", + "auth_token": "your_server_api_token" + }, + "STDIO Server": { + "command": "npx", + "args": ["-y", "server-mcp@0.2.1"], + "env": { + "SERVER_API_KEY": "your_server_api_token" + } + } + } + } + +Since the content formats returned by the MCP server may vary, users can inherit from ``MCPBaseTool`` and override the ``_parse_tool_result`` method to implement custom parsing logic. + +.. code-block:: python + + class MCPYourTool(MCPBaseTool): + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + super().__init__(config, tool_schema) + + def _parse_tool_result(self, content: list) -> Tuple[str, dict]: + ... + +Overall, you may refer to mcp_search_tool.py_ and mcp_tool_config.yaml_ for custom implementation and configuration. + +Multi-turn Tokenization +~~~~~~~~~~~~~~~~~~~~~~~ + +Tokenizing multi-turn rollouts poses a challenge: after applying the chat template and tokenizing the full message list, it's hard to identify which tokens belong to assistant messages. Since the token list is flat, it lacks direct alignment with the message roles. + +To address this, we adopt a **delta-based tokenization** strategy. Each time the LLM generates a new message, we: + +1. Apply the chat template to all prior messages (`messages[:i]`). +2. Apply the chat template again including the latest message (`messages[:i+1]`). +3. Tokenize only the *delta* between these two serialized message strings. + +This ensures that only tokens generated by the assistant are included in the loss mask. + +.. code-block:: python + + # When using tokenizer + # Exclude the assistant prompt (e.g., "<|im_start|>assistant") from the loss by setting add_generation_prompt=True + prev = tokenizer.apply_chat_template(messages[:i], add_generation_prompt=True, tokenize=False) + curr = tokenizer.apply_chat_template(messages[:i+1], add_generation_prompt=False, tokenize=False) + token_ids += tokenizer.encode(curr[len(prev):], add_special_tokens=False) + loss_mask += [1] * len(token_ids) # Mask only the new assistant tokens + +.. code-block:: python + + # When using processor + # Exclude the assistant prompt (e.g., "<|im_start|>assistant") from the loss by setting add_generation_prompt=True + prev = processor.apply_chat_template(messages[:i], add_generation_prompt=True, tokenize=False) + prev_model_inputs = processor(text=prev, images=images, videos=videos, return_tensors="pt")[0].tolist() + curr = processor.apply_chat_template(messages[:i+1], add_generation_prompt=False, tokenize=False) + curr_model_inputs = processor(text=curr, images=images, videos=videos, return_tensors="pt")[0].tolist() + token_ids += curr_model_inputs["input_ids"][len(prev_model_inputs["input_ids"]):] + loss_mask += [1] * len(token_ids) # Mask only the new assistant tokens + +While we've validated this produces consistent results with full message tokenization, future models' chat template could break compatibility. To guard against silent inconsistencies, we compare the delta-based tokenization with full-tokenization results by default at the end of each rollout. + +If you see the following warning, you can check the mismatched substring in the log: + +.. code-block:: + + Inconsistent training and inference tokenization detected. This may lead to unexpected behavior during training. Please review your chat template to determine if this is intentional. For more information, refer to the multiturn README.md. + +The tokenization sanity check mode can be configured using the ``actor_rollout_ref.rollout.multi_turn.tokenization_sanity_check_mode`` parameter, which accepts the following values: + +- ``strict`` (default): Performs strict comparison between delta-based and full tokenization results, raising warnings for any differences. + +- ``ignore_strippable``: Ignores differences in whitespace characters (``\n``, ``\t``, ``\r``, spaces) while still checking for meaningful text mismatches. This is useful when debugging chat template issues where whitespace variations are expected and acceptable. + +- ``disable``: Completely disables the tokenization sanity check. Only use this if you have thoroughly validated that tokenization discrepancies are expected and won't impact training. + +Example configuration: + +.. code-block:: yaml + + actor_rollout_ref: + rollout: + multi_turn: + tokenization_sanity_check_mode: "ignore_strippable" # Choose from: "disable", "ignore_strippable", "strict" + +Handling Multi-Modal Inputs in Datasets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If your dataset includes multi-modal inputs (such as images or videos), you can control whether these are pre-processed and included in each sample by setting the return_multi_modal_inputs flag in your dataset config (used by RLHFDataset). + +- ``return_multi_modal_inputs: True`` (default): The dataset will pre-process and include a multi_modal_inputs dictionary for each sample. This dict contains the model-ready representations (e.g., image tensors, video tensors, etc.) as produced by your processor. This is useful for single-turn or SFT-style training, where the model expects all modalities to be present in the batch. + +- ``return_multi_modal_inputs: False``: The dataset will not include the multi_modal_inputs field. This is recommended for multi-turn RL or tool-augmented rollouts, where the model may generate new multi-modal inputs dynamically during rollout, and you want to avoid conflicts or redundant data in the batch. + + +Special Cases +^^^^^^^^^^^^^ + +Some models (e.g., Qwen/QwQ-32B and Qwen3 series) remove internal reasoning content during chat template rendering. As a result, the message content can vary across turns, making the delta-based tokenization inaccurate. + +For example, for the following conversation: + +.. code-block:: python + + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is 2 + 2?"}, + {"role": "assistant", "content": "user asked about a simple math question. 2 + 2 = 4."}, + {"role": "user", "content": "Explain why."}, + {"role": "assistant", "content": "user wants to know the reasoning behind the answer. Search for a good explanation", + "tool_calls": [{"id": "tool1", "type": "search", "arguments": {"query": "Why is 2 + 2 = 4?"}}]}, + {"role": "tool", "content": "The sum of two and two is four because it is a basic arithmetic operation."}, + {"role": "assistant", "content": "The tool provided a good explanation.The sum of two and two is four because it is a basic arithmetic operation."} + ] + +1. Qwen/QwQ-32B will remove all reasoning content except the last assistant message after applying the chat template. + +.. code-block:: text + + <|im_start|>system + You are a helpful assistant.<|im_end|> + <|im_start|>user + What is 2 + 2?<|im_end|> + <|im_start|>assistant + 2 + 2 = 4.<|im_end|> + <|im_start|>user + Explain why.<|im_end|> + <|im_start|>assistant + + {"name": "", "arguments": {"query": "Why is 2 + 2 = 4?"}} + <|im_end|> + <|im_start|>user + + The sum of two and two is four because it is a basic arithmetic operation. + <|im_end|> + <|im_start|>assistant + The tool provided a good explanation. The sum of two and two is four because it is a basic arithmetic operation.<|im_end|> + +2. Qwen3 series will remove all reasoning content before the last user message. + +.. code-block:: text + + <|im_start|>system + You are a helpful assistant.<|im_end|> + <|im_start|>user + What is 2 + 2?<|im_end|> + <|im_start|>assistant + 2 + 2 = 4.<|im_end|> + <|im_start|>user + Explain why.<|im_end|> + <|im_start|>assistant + + user wants to know the reasoning behind the answer. Search for a good explanation + + + + {"name": "", "arguments": {"query": "Why is 2 + 2 = 4?"}} + <|im_end|> + <|im_start|>user + + The sum of two and two is four because it is a basic arithmetic operation. + <|im_end|> + <|im_start|>assistant + + The tool provided a good explanation. + + + The sum of two and two is four because it is a basic arithmetic operation.<|im_end|> + +To handle this, we fall back to a **fixed base conversation** containing only a single system and user message. Since this base doesn't include assistant messages or reasoning content, it remains consistent across turns. + +.. code-block:: python + + BASE_CHAT_HISTORY = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "I am a user."} + ] + prev = tokenizer.apply_chat_template(BASE_CHAT_HISTORY, add_generation_prompt=True, tokenize=False) + curr = tokenizer.apply_chat_template([*BASE_CHAT_HISTORY, messages[i]], add_generation_prompt=False, tokenize=False) + token_ids += tokenizer.encode(curr[len(prev):], add_special_tokens=False) + loss_mask += [1] * len(token_ids) + +This method works well for Qwen3 series. However, Qwen/QwQ-32B currently has a bug in its chat template. A fix_ has been proposed but not yet adopted. Until then, use the following command to download the fixed model revision: + +.. code-block:: bash + + pip install huggingface_hub + huggingface-cli download Qwen/QwQ-32B --revision refs/pr/81 + +.. _fix: https://huggingface.co/Qwen/QwQ-32B/discussions/81 + +Discrepancy Between Training and Inference Templates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Although the above approach fixes the delta mismatch issue, the removal of reasoning content in the inference-time chat template introduces a new discrepancy: training uses the full reasoning content, while inference does not. + +This mismatch can affect model performance in unpredictable ways. To avoid it, we default to using the full response (including reasoning) for both training and rollout. + +However, this approach comes with trade-offs: + +1. Long reasoning contents can easily exceed the model's context window, especially in multi-turn rollout. +2. There's a mismatch between rollout and production environment now—models will not have reasoning content from past turns if you use the default chat template in production. + +We are still evaluating the impact of these issues. If you experience context length problems or prefer rollouts that match production (i.e., exclude reasoning), you can enable: + +``actor_rollout_ref.rollout.multi_turn.use_inference_chat_template = True`` + +GSM8K Multi-turn Training Performance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See the training performance of multi-turn rollout on the GSM8K task HERE_. + +.. _HERE: https://wandb.ai/zhaochenyang20/gsm8k_async_rl/runs/1ro1r7om?nw=nwuserzhaochenyang20 + +.. _GSM8KTool_example_configuration: https://github.com/volcengine/verl/blob/main/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml + +.. _gsm8k_tool.py: https://github.com/volcengine/verl/blob/main/verl/tools/gsm8k_tool.py + +.. _mcp_search_tool.py: https://github.com/volcengine/verl/blob/main/verl/tools/mcp_search_tool.py + +.. _mcp_tool_config.yaml: https://github.com/volcengine/verl/blob/main/examples/sglang_multiturn/config/tool_config/mcp_tool_config.yaml + +Interaction System +~~~~~~~~~~~~~~~~~~ + +For dynamic conversational feedback during RL training, see: + +.. toctree:: + :maxdepth: 1 + + interaction_system + +Search Tool Integration +~~~~~~~~~~~~~~~~~~~~~~~ + +.. toctree:: + :maxdepth: 1 + + search_tool_example + +Code Walkthrough +~~~~~~~~~~~~~~~~~~~~~~~ +If you want to learn more in depth about the code execution flow, please read https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/rlhf/verl/multi-turn/code-walk-through diff --git a/e2c/verl/docs/sglang_multiturn/sandbox_fusion.rst b/e2c/verl/docs/sglang_multiturn/sandbox_fusion.rst new file mode 100644 index 0000000000000000000000000000000000000000..94adb8a356cbe98309b9287b7b255767c2bcd860 --- /dev/null +++ b/e2c/verl/docs/sglang_multiturn/sandbox_fusion.rst @@ -0,0 +1,304 @@ +=============================== +Sandbox Fusion Tool Integration +=============================== + +Last updated: 06/10/2025. + +Motivations +=========== + +- As users of verl, we want to allow the model to call certain tools during Actor rollout, incorporating the results into the training process. +- A colleague from ByteDance proposed a paper aimed at enhancing model capability through code execution tools. +- We aim to support tool-calling capabilities of inference engines using `sandbox-fusion` as the code execution system, providing the community with a reimplementation of `retools`. + +Reward Compute with Sandbox Fusion + FaaS Integration +===================================================== + +- In current datasets and tasks, similar work already exists (e.g., Prime), which uses local processes as runners to execute model-generated code for reward computation. +- On this basis, #1429 has advanced the design by integrating FaaS as the runner for reward computation. + +Goals +===== + +- Adapt to the `sglang` tool-calling protocol and define tools for sandbox fusion. +- Integrate with the `async-rollout` process, ensuring sandbox fusion tools follow asyncIO conventions. +- Design and implement a basic rate limiter to prevent issues such as 429 errors. + +Non-Goals +========= + +- Training effectiveness is out of scope. +- Observability metrics are not considered. +- Distributed failover and component fault tolerance are not addressed. + +Design Details +============== + +Tool Schema Definition +---------------------- + +- Currently, only code execution is considered, requiring a `code` field in the JSON from the model. +- Only Python code is supported for now, so no `language` parameter is defined. + +.. code-block:: python + + OpenAIFunctionToolSchema( + type="function", + function=OpenAIFunctionSchema( + name="code_interpreter", + description="A tool for executing code.", + parameters=OpenAIFunctionParametersSchema( + type="object", + properties={ + "code": OpenAIFunctionPropertySchema( + type="string", + description="The code to execute.", + enum=None, + ) + }, + required=["code"], + ), + strict=False, + ) + ) + +Configuration Parameters +-------------------------- + ++----------------------------+--------------------------------------------------------------+ +| Parameter Name | Description | ++============================+==============================================================+ +| `num_workers` | Number of worker threads/processes per DP to request runner. | ++----------------------------+--------------------------------------------------------------+ +| `rate_limit` | Global limit of concurrent code executions. Default: 10 | ++----------------------------+--------------------------------------------------------------+ +| `default_timeout` | Timeout (in seconds) for each code execution. Default: 30 | ++----------------------------+--------------------------------------------------------------+ +| `default_language` | Default programming language. Default: "python" | ++----------------------------+--------------------------------------------------------------+ +| `enable_global_rate_limit` | Whether to enable global rate limiting. Default: True | ++----------------------------+--------------------------------------------------------------+ +| `sandbox_fusion_url` | URL for the veFaas sandbox execution service | ++----------------------------+--------------------------------------------------------------+ + +Rate Limiting Design +----------------------- + +Objective: + +- Limit the number of inflight requests using a token bucket model. + +- Ensure ordered submission to code runners to avoid starvation due to backoff. + +Design Highlights: + +- Use Ray Global Actor as a singleton distributed counter at cluster level. + +- Semaphore used for counting, with `acquire` and `release` in separate thread pools to preserve order. + +- Use Ray’s cloud-pickle to serialize functions for decoupled `ExecutionWorker`. + +.. code-block:: python + + @ray.remote(concurrency_groups={"acquire": 1,"release": 10}) + class TokenBucketWorker: + def __init__(self, rate_limit: int): + self.rate_limit = rate_limit + self.current_count = 0 + self._semaphore = threading.Semaphore(rate_limit) + + @ray.method(concurrency_group="acquire") + def acquire(self): + self._semaphore.acquire() + self.current_count += 1 + + @ray.method(concurrency_group="release") + def release(self): + self._semaphore.release() + self.current_count -= 1 + + def get_current_count(self): + return self.current_count + + class ExecutionWorker: + def __init__(self, enable_global_rate_limit=True, rate_limit=10): + self.rate_limit_worker = self._init_rate_limit(rate_limit) if enable_global_rate_limit else None + + def _init_rate_limit(self, rate_limit): + return TokenBucketWorker.options(name="rate-limiter", get_if_exists=True).remote(rate_limit) + + def execute(self, fn: Callable[..., T], *fn_args, **fn_kwargs) -> T: + with ExitStack() as stack: + stack.callback(self.rate_limit_worker.release.remote) + ray.get(self.rate_limit_worker.acquire.remote()) + try: + return fn(*fn_args, **fn_kwargs) + except Exception as e: + logger.warning(f"Error when executing code: {e}") + + def init_execution_pool(num_workers: int, enable_global_rate_limit=True, rate_limit=10, mode: PoolMode=PoolMode.ThreadMode): + if mode == PoolMode.ThreadMode: + return ray.remote(ExecutionWorker).options(max_concurrency=num_workers).remote( + enable_global_rate_limit=enable_global_rate_limit, + rate_limit=rate_limit + ) + else: + raise NotImplementedError("Process mode is not implemented yet") + +Tool Implementation +------------------- + +- Use `instance_id` to identify requests across multiple dialogue rounds. + +- Use `execution_pool` to implement async invocation. + +- Cleanup state after rollout completion. + +.. code-block:: python + + class SandboxFusionTool(BaseTool): + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + ... + self.execution_pool = init_execution_pool(...) + ... + + async def create(self, instance_id: Optional[str] = None, ...): + ... + + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> Tuple[str, float, dict]: + code = parameters.get("code", "") + timeout = parameters.get("timeout", self.default_timeout) + language = parameters.get("language", self.default_language) + if not isinstance(code, str): + code = str(code) + + result = await self.execution_pool.execute.remote(self.execute_code,instance_id,code,timeout,language) + self._instance_dict[instance_id]["reward"].append(result.strip()) + + return result, result, {} + + def execute_code(self,instance_id,code,timeout=30,language="python"): + result_status, metadata = _process_single_case(0, None, None,self.sandbox_fusion_url, code, timeout, language) + # we should always expect this since we don't have correct answer + if metadata["run_status"] == "Finished": + actual_output = metadata["stdout"] if metadata["stdout"] is not None else "" + return actual_output + else: + return "no stdout here" + + async def calc_reward(self, instance_id: str, ...): + ... + + async def release(self, instance_id: str, ...): + ... + +Test Plan +========= + +Unit Tests +---------- + +- **test_tools_registration**: Test tool registration and initialization. +- **test_rollout_req_creation**: Validate that `AsyncRolloutReq` is built correctly. +- **test_over_size_case**: Ensure rollout terminates early when exceeding `max_seq_len`. +- **test_tool_call_basic_case**: Mock `sglang` output, validate tool call and result. +- **test_tool_call_batch_case**: Test batch processing of tool calls. +- **test_basic_multi_process_init**: Validate Ray global actor behaves as singleton. +- **TestSingleNodeRateLimiterCase**: Verify rate limiter works in single-node mode. +- **test_rotten_execution**: Ensure rate limiter recovers from function errors. +- **TestMultiNodeRateLimiterCase**: Verify behavior in multi-node environments. + +e2e Tests +---------- +we provide e2e test scripts in `tests/special_e2e` folder, named `tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_sf_tool.sh` + +by setting 'trainer.rollout_data_dir' you can dump the rollout data to local disk. here is an sample taken from the rollout data: + +.. code-block:: python + + { + "input": " + + system\nYou are a math expert. You are given a question and you need to solve it step by step. Reasoning step by step before any tool call. You should use the `calc_gsm8k_reward` tool after step by step solving the question, before generate final answer at least once and refine your answer if necessary. Put your final answer in the format of `#### `.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"code_interpreter\", \"description\": \"A tool for executing code.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"code\": {\"type\": \"string\", \"description\": \"The code to execute.\", \"enum\": null}}, \"required\": [\"code\"]}, \"strict\": false}}\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n\n + + user\nJohn gets a bonus that's the same percentage every year. Last year he made $100,000 and got a $10,000 bonus. This year he makes $200,000. How much will John make this year when adding both his total pay and bonus together? Let's think step by step and output the final answer after `####`.\n + + assistant\n", + "output": "assistant\n\nOkay, let's see. John gets a bonus that's the same percentage every year. Last year, he made $100,000 and got a $10,000 bonus. This year, he makes $200,000. The question is asking how much he'll make this year when adding both his total pay and bonus together.\n\nFirst, I need to figure out the percentage bonus he gets. Since the bonus is the same percentage each year, if I can find that percentage based on last year's details, I can apply it to this year's earnings to find this year's bonus. Then, I'll add that bonus to his total pay this year.\n\nLast year, he earned $100,000 and got a $10,000 bonus. To find the percentage, I can divide the bonus by his total earnings. So, 10,000 divided by 100,000. Let me calculate that.\n\n10,000 / 100,000 equals 0.1, which is 10% as a percentage. So, his bonus is 10% of his pay each year.\n\nThis year, he makes $200,000. So, the bonus this year would be 10% of 200,000. Let me compute that using the code interpreter to ensure accuracy.\n + + \n{\"name\": \"code_interpreter\", \"arguments\": {\"code\": \"total_pay_this_year = 200000\\nbonus_percentage = 10 / 100\\nbonus_this_year = total_pay_this_year * bonus_percentage\\ntotal_income_this_year = total_pay_this_year + bonus_this_year\\nprint(total_income_this_year)\", \"executes\": \"True\"}}\n\n + + tool\n220000.0\n\n + + assistant\n\nSo the code shows that John's total income this year would be $220,000. This is consistent with the 10% bonus on last year's pay of $100,000, which was $10,000. Therefore, this year's bonus of $20,000 added to his total pay of $200,000 results in a total income of $220,000.\n\nI need to verify that there are no other factors at play. The problem states it's a constant percentage bonus each year, so there's no mention of changing percentages or other variables. Therefore, applying a straightforward percentage of 10% on this year's partial salary amount would result in $20,000. Adding that to his total pay gives $220,000. The calculation is correct and aligns with the question's parameters.\n\n\n#### 220000.0", + "score": 0, + "step": 1 + } + +here is the readable format version: + +.. code-block:: python + + [system] + + You are a math expert. You are given a question and you need to solve it step by step. Reasoning step by step before any tool call. You should use the `calc_gsm8k_reward` tool after step by step solving the question, before generate final answer at least once and refine your answer if necessary. Put your final answer in the format of `#### `. + + # Tools + + You may call one or more functions to assist with the user query. + + You are provided with function signatures within XML tags: + + {"type": "function", "function": {"name": "code_interpreter", "description": "A tool for executing code.", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": "The code to execute.", "enum": null}}, "required": ["code"]}, "strict": false}} + + + For each function call, return a json object with function name and arguments within XML tags: + + {"name": , "arguments": } + + + [user] + + John gets a bonus that's the same percentage every year. Last year he made $100,000 and got a $10,000 bonus. This year he makes $200,000. How much will John make this year when adding both his total pay and bonus together? Let's think step by step and output the final answer after `####`. + + [assistant] + + + Okay, let's see. John gets a bonus that's the same percentage every year. Last year, he made $100,000 and got a $10,000 bonus. This year, he makes $200,000. The question is asking how much he'll make this year when adding both his total pay and bonus together. + + First, I need to figure out the percentage bonus he gets. Since the bonus is the same percentage each year, if I can find that percentage based on last year's details, I can apply it to this year's earnings to find this year's bonus. Then, I'll add that bonus to his total pay this year. + + Last year, he earned $100,000 and got a $10,000 bonus. To find the percentage, I can divide the bonus by his total earnings. So, 10,000 divided by 100,000. Let me calculate that. + + 10,000 / 100,000 equals 0.1, which is 10% as a percentage. So, his bonus is 10% of his pay each year. + + This year, he makes $200,000. So, the bonus this year would be 10% of 200,000. Let me compute that using the code interpreter to ensure accuracy. + + {"name": "code_interpreter", "arguments": {"code": "total_pay_this_year = 200000\nbonus_percentage = 10 / 100\nbonus_this_year = total_pay_this_year * bonus_percentage\ntotal_income_this_year = total_pay_this_year + bonus_this_year\nprint(total_income_this_year)", "executes": "True"}} + + + [tool] + + 220000.0 + + [assistant] + + + So the code shows that John's total income this year would be $220,000. This is consistent with the 10% bonus on last year's pay of $100,000, which was $10,000. Therefore, this year's bonus of $20,000 added to his total pay of $200,000 results in a total income of $220,000. + + I need to verify that there are no other factors at play. The problem states it's a constant percentage bonus each year, so there's no mention of changing percentages or other variables. Therefore, applying a straightforward percentage of 10% on this year's partial salary amount would result in $20,000. Adding that to his total pay gives $220,000. The calculation is correct and aligns with the question's parameters. + + + #### 220000.0 + + +You can also use the `RolloutViewer` TUI tool to view the dumped rollout data: + + +.. code-block:: bash + + python scripts/rollout_viewer.py ${trainer.rollout_data_dir} + + +.. image:: https://github.com/user-attachments/assets/e34e5157-2880-4a21-afb2-73885d0dfb11 + :alt: RolloutViewer screenshot \ No newline at end of file diff --git a/e2c/verl/docs/sglang_multiturn/search_tool_example.rst b/e2c/verl/docs/sglang_multiturn/search_tool_example.rst new file mode 100644 index 0000000000000000000000000000000000000000..cbbdeb0d08e6102a00a85bd5544c345bb086969f --- /dev/null +++ b/e2c/verl/docs/sglang_multiturn/search_tool_example.rst @@ -0,0 +1,264 @@ +======================= +Search Tool Integration +======================= + +Last updated: 05/30/2025. + +Introduction +------------ +- We have added a search tool calling function to Multi-Turn RL, enabling the model to initiate retrieval requests during Actor rollout and directly use retrieval results for training. **We support using a local dense retriever as the retrieval tool, as well as integrating with your own local retrieval engine.** + + + +Quick Reproduction +------------------ + +Create a New Docker Container +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + docker run \ + -it \ + --shm-size 32g \ + --gpus all \ + -v {Huggingface-Cache-Path}:/root/.cache \ + --ipc=host \ + --network=host \ + --privileged \ + --name sglang_{your-name} \ + lmsysorg/sglang:dev \ + /bin/zsh + +If you need to restart after exiting the container: + +.. code:: bash + + docker start -i sglang_{your-name} + +Update Python and Configure the Virtual Environment using uv +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + apt update + apt install -y python3.10 python3.10-venv + + # Create a virtual environment + python3 -m venv ~/.python/verl-multiturn-rollout + + # Activate the virtual environment + source ~/.python/verl-multiturn-rollout/bin/activate + + # Install uv + python3 -m pip install uv + +Install verl Upstream +~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + cd ~ + git clone https://github.com/volcengine/verl.git + cd verl + + # Install verl + python3 -m uv pip install . + python3 -m uv pip install -r ./requirements_sglang.txt + + # Manually install flash-attn + python3 -m uv pip install wheel + python3 -m uv pip install packaging + python3 -m uv pip install flash-attn --no-build-isolation --no-deps + +Set Up a Local Retrieval Engine +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using your own local retrieval service, you can skip this +step. We chose the local dense retriever provided in the search-R1 +example; detailed instructions are in the `searchR1 +docs `__. +In brief: + +- The GPU version offers higher accuracy and speed; each GPU uses about + 5–7 GB of memory. +- The CPU version can be used for simple testing but has lower + retrieval precision, which will degrade training performance. See the + `retriever + documentation `__ + in search-R1 for details. +- Recommend using Conda to install faiss-gpu=1.8.0; venv may cause errors. + +**Note**: To start both the training process and the local retrieval +service, we launch two separate Python environments. The training uses +uv in the verl-multiturn-rollout environment, while the retriever uses +conda to install ``faiss-gpu``. + +.. code:: bash + + # Download the Miniconda installer script + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh + + # Install to $HOME/miniconda3 in batch mode + bash ~/miniconda.sh -b -p $HOME/miniconda3 + + # Activate conda (only in the current shell) + eval "$($HOME/miniconda3/bin/conda shell.bash hook)" + + # (Optional) Add conda to your default shell startup + conda init + + # Reload shell config + source ~/.bashrc + + # Create and activate the retriever environment with Python 3.10 + conda create -n retriever python=3.10 -y + conda activate retriever + + # Install PyTorch (with GPU support) and related libraries + conda install pytorch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 pytorch-cuda=12.1 -c pytorch -c nvidia -y + + # Install other Python packages + pip install transformers datasets pyserini huggingface_hub + + # Install the GPU version of faiss + conda install faiss-gpu=1.8.0 -c pytorch -c nvidia -y + + # Install the API service framework + pip install uvicorn fastapi + +Download the Indexing and Corpus +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The local retrieval files are large—prepare sufficient disk space. +Downloading is about 60–70 GB, and uncompressed takes about 132 GB: + +.. code:: bash + + conda activate retriever + + save_path=/the/path/to/save + python examples/sglang_multiturn/search_r1_like/local_dense_retriever/download.py --save_path $save_path + cat $save_path/part_* > $save_path/e5_Flat.index + gzip -d $save_path/wiki-18.jsonl.gz + +Start the Local flat e5 Retrieval Server +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. The first startup will download models and load the index. +2. Apart from the download, startup takes about 1–2 minutes. +3. After startup, each GPU uses about 5–7 GB of memory, leaving the rest + for multi-turn RL training. + +.. code:: bash + + conda activate retriever + + index_file=$save_path/e5_Flat.index + corpus_file=$save_path/wiki-18.jsonl + retriever_name=e5 + retriever_path=intfloat/e5-base-v2 + + python examples/sglang_multiturn/search_r1_like/local_dense_retriever/retrieval_server.py \ + --index_path $index_file \ + --corpus_path $corpus_file \ + --topk 3 \ + --retriever_name $retriever_name \ + --retriever_model $retriever_path \ + --faiss_gpu + +Set Up WANDB_API_KEY +~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + export WANDB_API_KEY={YOUR_WANDB_API_KEY} + + # Define a timestamp function + function now() { + date '+%Y-%m-%d-%H-%M' + } + +**Preprocess the Dataset** +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + **Note:** The following data processing and training commands must be + run in the verl-multiturn-rollout environment. + +.. code:: bash + + python3 examples/data_preprocess/preprocess_search_r1_dataset.py + +Testing on 8 x H20 +~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + # Ensure the now() function is defined + # Create a logs directory + mkdir -p logs + + # Set GPUs and run with a suitable log path + export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + + nohup bash examples/sglang_multiturn/search_r1_like/run_qwen2.5-3b_instruct_search_multiturn.sh \ + trainer.experiment_name=qwen2.5-3b-it_rm-searchR1-like-sgl-multiturn-$(now) \ + > logs/searchR1-like$(now).log 2>&1 & + +Custom Search Configuration +--------------------------- + +To enable multi-turn reasoning, set the following fields in your config: + +.. code:: yaml + + actor_rollout_ref: + rollout: + name: "sglang" + multi_turn: + enable: True + +You must specify ``retrieval_service_url`` in ``examples/sglang_multiturn/config/tool_config/search_tool_config.yaml``, and properly configure concurrency. For more details on concurrency, refer to the Sandbox Fusion example: + +.. code:: yaml + + tools: + - class_name: verl.tools.search_tool.SearchTool + config: + retrieval_service_url: http://127.0.0.1:8000/retrieve + num_workers: 120 + rate_limit: 120 + timeout: 30 + +The retriever input/output formats are as follows. If your service +parameters match, only modify ``retrieval_service_url``. You can also +customize in ``search_r1_like_utils.py``. + +.. code:: python + + Input format: + { + "queries": ["What is Python?", "Tell me about neural networks."], + "topk": 3, + "return_scores": true + } + + Output format (when return_scores=True, similarity scores are returned): + { + "result": [ + [ # Results for each query + { + "document": doc, "score": score + }, + # ... more documents + ], + # ... results for other queries + ] + } + +Notes +----- + +1. The total training time is about 27 hours; meanwhile, the validation + dataset is very large (51 k), and each validation takes about 6000 s. + (Therefore, ``val_before_train=False`` by default) diff --git a/e2c/verl/docs/single_controller.rst b/e2c/verl/docs/single_controller.rst new file mode 100644 index 0000000000000000000000000000000000000000..d12177854e0ad2f2060a4255a4cde9cd93fe8263 --- /dev/null +++ b/e2c/verl/docs/single_controller.rst @@ -0,0 +1,336 @@ +The Design of ``verl.single_controller`` +============================================== + +Last updated: 05/21/2025. + +**Author:**\ `Wang Zhang `__ + +Preface +------- + +We prepared this document for developers of ``verl``, particularly those +interested in understanding or contributing to the +``verl.single_controller`` module. It is not intended for end users, but +for contributors seeking to understand the architectural rationale and +internal mechanics. + +-------------- + +Origin +------ + +The ``single_controller`` module originated from a request I received — +to adapt a toy single-process RLHF script into a distributed system with +minimal changes, while maintaining ease of debugging. + +Common practice — such as using PyTorch’s Distributed Data Parallel +(DDP) — typically involves wrapping ``nn.Module`` and launching multiple +processes that execute the same function under different ranks. However, +this approach presents two main limitations in the context of +distributed RLHF: - Difficulty representing multiple DAGs as required by +PPO; - Difficulty inspecting intermediate tensors during training. + +To maintain debuggability, we opted for a different approach — breaking +the training loop into well-defined stages like ``generate_sequences``, +``compute_advantages``, and so on. + +We selected `Ray `__ as the initial backend for +``verl`` due to its ability to expose Python class methods as RPC +endpoints. However, Ray’s default model only supports **one method call, +one RPC**, while training LLMs typically requires coordination across +multiple processes. + +To hide this multi-Ray actors invocation for a single method from users, +we introduced the following components: + +- ``WorkerGroup`` – manages a group of remote workers and provides + a unified interface for multi-process distributed computation; +- ``ResourcePool`` – binds computational resources to worker + processes; +- ``ClassWithArgs`` – enables delayed remote instantiation with + specified initialization arguments. + +-------------- + +A Running Example: ``generate_sequences`` +----------------------------------------- + +To illustrate the design, we walk through how the ``generate_sequences`` +method in the ``ActorRolloutRefWorker`` class is registered and invoked +across distributed workers. + +-------------- + +Step 1: Register with a Decorator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The first step is to define the ``generate_sequences`` and decorate it +with ``@register`` as it will be called in driver script. + +**Source:** +`fsdp_workers.py `__ + +.. code:: python + + class ActorRolloutRefWorker(Worker): + ... + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def generate_sequences(self, prompts: DataProto): + prompts = prompts.to(torch.cuda.current_device()) + ... + +The ``@register`` decorator adds metadata to the ``generate_sequences`` +method. Currently, it doesn’t alter functionality, but attaches +attributes via a magic key (``MAGIC_ATTR``): + +**Source:** +`decorator.py `__ + +.. code:: python + + def register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.ALL, blocking=True, materialize_futures=True): + ... + def decorator(func): + @wraps(func) + def inner(*args, **kwargs): + if materialize_futures: + args, kwargs = _materialize_futures(*args, **kwargs) + return func(*args, **kwargs) + + attrs = {"dispatch_mode": dispatch_mode, "execute_mode": execute_mode, "blocking": blocking} + setattr(inner, MAGIC_ATTR, attrs) + return inner + + return decorator + +As the code shows, values of ``dispatch_mode``, ``execute_mode`` and +``blocking`` is attached the ``generate_sequences`` method. + +-------------- + +Step 2: Binding During Initialization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These attached attributes are extracted and utilized when +``ActorRolloutRefWorker``, wrapped in a ``RayClassWithArgs``, is passed +into a ``RayWorkerGroup``. + +**Source:** +`main_generation.py `__ + +.. code:: python + + ray_cls_with_init = RayClassWithInitArgs(cls=ray.remote(ActorRolloutRefWorker), config=config, role="rollout") + resource_pool = RayResourcePool(process_on_nodes=[config.trainer.n_gpus_per_node] * config.trainer.nnodes) + wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) + +During the +`initialization `__ +of ``RayWorkerGroup``, two key steps occur: + +1. Worker instances (Ray actors) are created: + `RayWorkerGroup._init_with_resource_pool `__ +2. Methods decorated with ``@register`` are bound to ``RayWorkerGroup``: + `RayWorkerGroup._bind_worker_method `__ + +.. figure:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/worker_group_init.png?raw=true + :alt: initialization_and_binding_of_worker_group + + initialization_and_binding_of_worker_group + +The binding procedure is the heart of ``verl.single_controller``. + +**Key function:** +`WorkerGroup._bind_worker_method `__ + +.. code:: python + + def _bind_worker_method(self, user_defined_cls, func_generator): + ... + for method_name in dir(user_defined_cls): + try: + method = getattr(user_defined_cls, method_name) + assert callable(method) + except Exception: + continue # Skip properties + <<>> + +When a method has the ``MAGIC_ATTR``, the attributes set by +``@register`` are extracted: + +.. code:: python + + <<>> + if hasattr(method, MAGIC_ATTR): + attribute = getattr(method, MAGIC_ATTR) + dispatch_mode = attribute["dispatch_mode"] + execute_mode = attribute["execute_mode"] + blocking = attribute["blocking"] + + <<>> + +As show in the flow chart above, these attributes are fed into +``func_generator``. However, ``func_generator`` takes ``method_name``, +``dispatch_fn``, ``collect_fn``, ``execute_fn``, ``blocking``. We need +to find the corresponding ``dispatch_fn`` and ``collect_fn`` associated +with the ``dispatch_mode`` (``DP_COMPUTE_PROTO``) from +`DISPATCH_MODE_FN_REGISTRY `__: + +.. code:: python3 + + DISPATCH_MODE_FN_REGISTRY = { + Dispatch.ONE_TO_ALL: { + "dispatch_fn": dispatch_one_to_all, + "collect_fn": collect_all_to_all, + }, + ... + Dispatch.DP_COMPUTE_PROTO: { + "dispatch_fn": dispatch_dp_compute_data_proto, + "collect_fn": collect_dp_compute_data_proto, + }, + ... + } + +Similarly, the ``execute_fn`` is selected by ``execute_mode`` and +extracted by: + +.. code:: python + + <<>> + # get execute_fn_name + execute_mode = get_predefined_execute_fn(execute_mode=execute_mode) + wg_execute_fn_name = execute_mode["execute_fn_name"] + + # get execute_fn from string + try: + execute_fn = getattr(self, wg_execute_fn_name) + assert callable(execute_fn), "execute_fn must be callable" + except Exception: + print(f"execute_fn {wg_execute_fn_name} is invalid") + raise + <<>> + +In this ``generate_sequences`` cases: - +``dispatch_mode = Dispatch.DP_COMPUTE_PROTO`` - +``dispatch_fn = dispatch_dp_compute_data_proto`` - +``collect_fn = collect_dp_compute_data_proto`` - +``execute_fn = RayWorkerGroup.execute_all`` + +ONE_TO_ALL v.s. DP_COMPUTE_PROTO +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``dispatch_mode`` is associated with a ``dispatch_fn`` and a +``collect_fn``. As the name implies, ``dispatch_fn`` processes the input +arguments in ``WorkerGroup`` and generate a batch (list) of input +arguments, each of which will be fed into a worker attached to the +``WorkerGroup``. + +``dispatch_fn`` of ``ONE_TO_ALL`` is +`dispatch_one_to_all `__, +which just duplicates all the input arguments into N replicas, where N +equals the number of Workers attached to the ``worker_group``: + +.. code:: python + + def dispatch_one_to_all(worker_group, *args, **kwargs): + args = tuple([arg] * worker_group.world_size for arg in args) + kwargs = {k: [v] * worker_group.world_size for k, v in kwargs.items()} + return args, kwargs + +``dispatch_fn`` of ``DP_COMPUTE_PROTO`` is +`dispatch_dp_compute_data_proto `__, +which uses ``DataProto.chunk`` to split a large ``DataProto`` into N +smaller ``DataProto``, where N equals the world_size (number of the +workers) of the ``worker_group``: + +.. code:: python + + def dispatch_dp_compute_data_proto(worker_group, *args, **kwargs): + from verl.single_controller.base.worker_group import WorkerGroup + + assert isinstance(worker_group, WorkerGroup) + # Note: enable auto padding for dp compute DatapProto + splitted_args, splitted_kwargs = _split_args_kwargs_data_proto_with_auto_padding( + worker_group.world_size, + *args, + **kwargs, + ) + return splitted_args, splitted_kwargs + +The ``collect_fn`` follows the same pattern and process a batch (list) +of returned value from all workers of a ``WorkerGroup`` and merge it +into a list as ``collect_all_to_all`` does or a large ``DataProto`` as +``collect_dp_compute_data_proto`` does. + +Finally, a new method is dynamically generated using ``func_generator`` +and added to the ``WorkerGroup`` instance: + +.. code:: python + + <<>> + # bind a new method to the RayWorkerGroup + func = func_generator( + self, + method_name, + dispatch_fn=dispatch_fn, + collect_fn=collect_fn, + execute_fn=execute_fn, + blocking=blocking, + ) + + try: + setattr(self, method_name, func) + method_names.append(method_name) + except Exception as e: + raise ValueError(f"Fail to set method_name {method_name}") from e + +This makes the method invocable via the ``WorkerGroup`` interface. + +-------------- + +Step 3: Call Chain +~~~~~~~~~~~~~~~~~~ + +All the machinery above ensures that distributed calls feel identical to +single-process ones. In the original single-process script, the code +looks like: + +.. code:: python + + rollout = Rollout() + rollout.generate_sequences(batch) + +With ``verl``, the multiprocess program becomes: + +.. code:: python + + rollout = RayWorkerGroup(resource_pool=[4], RayClassWithArgs(Rollout)) + rollout.generate_sequences(batch) + +.. figure:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/call_generate_sequences.png?raw=true + :alt: call_chain_of_generate_sequences + + call_chain_of_generate_sequences + +Behind this simple call: - ``dispatch_fn`` splits input across workers - +``execute_fn`` performs the actual remote invocation - ``collect_fn`` +gathers the results + +All of this is abstracted away, enabling developers to write distributed +code with minimal changes to their existing logic. + +-------------- + +Beyond RL Post-Training: Generalizing ``verl.single_controller`` +---------------------------------------------------------------- + +The ``verl.single_controller`` module generalizes well beyond +reinforcement learning. It provides a clean abstraction to batch-process +remote method calls, with automatic input/output handling. + +By minimizing the gap between single-process and multi-process scripts, +``verl.single_controller`` opens the door to distributed computing in +broader domains — not limited to RL post-training. + +We hope this design inspires more examples and extensions from the +community. diff --git a/e2c/verl/docs/start/agentic_rl.rst b/e2c/verl/docs/start/agentic_rl.rst new file mode 100644 index 0000000000000000000000000000000000000000..73c0a7ce1e1d8a43f9811b571b634fa94f162a10 --- /dev/null +++ b/e2c/verl/docs/start/agentic_rl.rst @@ -0,0 +1,133 @@ +Agentic RL Training +=================== + +Last updated: 07/15/2025. + +Overview +---------- +The goal of Agentic RL is to improve the performance of backend models from reinforcement learning to the Agent. During the training process, a series of features are developed: + +1. Server-based asynchronous rollout +2. Multi-turn conversations and tool calls +3. LangGraph-based Agent + + +This document explains the system principles and usage involved to help users implement Agentic RL. + + +Server-based Asynchronous Rollout +--------------------------------- + +Since Agents need to interact with the environment through various tool calls, in order to avoid GPU idling while waiting for tool call return results, an asyncio based co-routing mechanism is utilized to execute each rollout requests asynchronously, thereby improving training performance. To support asynchronous rollout, the inference engine (server) and the agent (client) are architecturally separated, implementing a server-based system with the following objectives: + +1. Enabling load balancing mechanisms to balance loads across multiple GPUs and reduce the impact of long-tail requests on performance. For this purpose, scheduling capabilities in stream mode (recipe\stream_mode) are implemented as a recipe. +2. Preventing agent specific features such as tracing from affecting the inference engine. + +System Architecture +~~~~~~~~~~~~~~~~~~~ + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/agent_loop.png?raw=true + +For more detail on internal design, please refer to :doc:`Agent Loop<../advance/agent_loop>`. + +System Components +~~~~~~~~~~~~~~~~~ + ++--------------------------+----------------------------------------------------------------------------+ +| Component | Role | ++==========================+============================================================================+ +| AgentLoop | Client, implements Agent functions | ++--------------------------+----------------------------------------------------------------------------+ +| AsyncLLMServerManager | Inference gateway, provides generate interface for AgentLoop | ++--------------------------+----------------------------------------------------------------------------+ +| AsyncServer | Server, each instance is connected to one DP group of the inference engine | ++--------------------------+----------------------------------------------------------------------------+ + +**"generate" Interface** + +The "generate" function based on ray actor is used between the Client and Server instead of the standard chat completion API. This is because the conversion between tokens and text can be irreversible. For example, the token converted from "" will be different from that generated by the LLM. During the training phase, it is necessary to strictly use the tokens generated by LLM inference to avoid inaccurate in computing advantage, which may affect model performance. Having the Server provide a token-based API helps the Client maintain the relationship between the text generated by tool calls and the tokens returned by the LLM, so as to output correct tokens for training. + + +**Inference Engine Adaptation** +AsyncServer uniformly provides a generate function to the upper layer, with separate implementations for SGLang and vLLM to hide underlying differences: + +1. The SGLang AsyncServer uses the async_generate interface of the SGLang engine, which is located on the first GPU of each TP group. Therefore, AsyncServer needs to remotely call async_generate through ray actor. +2. The vLLM AsyncServer uses the generate interface of the vLLM engine, which can communicate with the GPUs in the TP group through ZMQ and can be directly called in AsyncServer. + + +Usage Example +~~~~~~~~~~~~~ + +Follow :doc:`GSM8K example<../examples/gsm8k_example>` to prepare the dataset and model checkpoints. + +There are two options required to use agent loop: + +- `data.return_raw_chat=True` +- `actor_rollout_ref.rollout.mode=async` + +This example uses the sglang inference engine by default, and you can also modify rollout_name to use vllm. + +.. code-block:: bash + + bash examples/grpo_trainer/run_qwen2-7b_seq_balance.sh + + +Multi-turn Conversations and Tool Calls +--------------------------------------- + +Follow :doc:`Multi-turn Rollout Support<../sglang_multiturn/multiturn>` to prepare tool and configuration files. + +The Tool Agent Loop has an additional requirement: adding an "agent_name" field to the dataset. During rollout, it will choose to use tool_agent_loop or single_turn_agent (default) based on this field. + +Usage Example +~~~~~~~~~~~~~ + +.. code-block:: bash + + # install mlflow to view toolcall and llm trace + pip install mlflow + + # This will download and preprocess the GSM8K dataset into ~/data/gsm8k/ and add the "agent_name" field. + python examples/data_preprocess/gsm8k_tool_agent_loop.py + + # Start training with tool calls and enabled mlflow based trace helping to debug the rollout details + bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_tool_agent_mlflow.sh + + # When training is done, start a mlflow server to view trace + mlflow ui -h 0.0.0.0 -p 5000 --backend-store-uri sqlite:////tmp/mlruns.db + + # then you can open http://:5000 from browser to view trace + + +Note: During training, because the model may sometimes fail to generate correct toolcall tags, an error message "Failed to decode tool call" will be output to the console, which does not indicate an abnormality in training. + + +Follow :doc:`Rollout trace<../advance/rollout_trace>` to known more about trace feature. + + + +Agent Framework +--------------- + +System Architecture +~~~~~~~~~~~~~~~~~~~ + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/langgraph_agent.png?raw=true + +System Components +~~~~~~~~~~~~~~~~~ + ++--------------------------+-----------------------------------------------------------------------------------------------+ +| Component | Role | ++==========================+===============================================================================================+ +| ChatModel | LLM object of LangChain, used to adapt to the “generate” api provided by AsyncLLMServerManager| ++--------------------------+-----------------------------------------------------------------------------------------------+ +| RectAgentLoop | Agent adaptation layer, which by default supports a naive LangGraph Agentic. | +| | New classes can be derived to support user-defined Agents, and the run function needs to be | +| | implemented to complete Agent calls. | ++--------------------------+-----------------------------------------------------------------------------------------------+ +| AsyncServer | Server, each instance is connected to one DP group of the inference engine. | ++--------------------------+-----------------------------------------------------------------------------------------------+ + + +Follow doc "recipe/langgraph_agent/example/README.md" for more details. \ No newline at end of file diff --git a/e2c/verl/docs/start/install.rst b/e2c/verl/docs/start/install.rst new file mode 100644 index 0000000000000000000000000000000000000000..01fb45986d9d40319b99fda8ecd70735c188987c --- /dev/null +++ b/e2c/verl/docs/start/install.rst @@ -0,0 +1,337 @@ +Installation +============ + +Requirements +------------ + +- **Python**: Version >= 3.10 +- **CUDA**: Version >= 12.1 + +verl supports various backends. Currently, the following configurations are available: + +- **FSDP** and **Megatron-LM** (optional) for training. +- **SGLang**, **vLLM** and **TGI** for rollout generation. + +Choices of Backend Engines +---------------------------- + +1. Training: + +We recommend using **FSDP** backend to investigate, research and prototype different models, datasets and RL algorithms. The guide for using FSDP backend can be found in :doc:`FSDP Workers<../workers/fsdp_workers>`. + +For users who pursue better scalability, we recommend using **Megatron-LM** backend. Currently, we support `Megatron-LM v0.12.2 `_. The guide for using Megatron-LM backend can be found in :doc:`Megatron-LM Workers<../workers/megatron_workers>`. + + +2. Inference: + +For inference, vllm 0.8.3 and later versions have been tested for stability. We recommend turning on env var `VLLM_USE_V1=1` for optimal performance. + +For SGLang, refer to the :doc:`SGLang Backend<../workers/sglang_worker>` for detailed installation and usage instructions. SGLang rollout is under extensive development and offers many advanced features and optimizations. We encourage users to report any issues or provide feedback via the `SGLang Issue Tracker `_. + +For huggingface TGI integration, it is usually used for debugging and single GPU exploration. + +Install from docker image +------------------------- + +We provide pre-built Docker images for quick setup. And from this version, +we utilize a new image release hierarchy for productivity and stability. + +The image types are divided into three large categories: + +- **Base Image**: Without inference and training frameworks, only basic dependencies are installed. + Can directly install vllm or SGLang on top of it, without need of reinstall torch or CUDA. +- **Application Image**: Stable version with inference and training frameworks installed. +- **Community Image**: Unstable version with the latest frameworks and features. + +The first two types of images are hosted on dockerhub `verlai/verl `_ repository, while the preview images are hosted on community repository. + +.. note:: + + The image versions are mapped with verl releases, for example, image with tag ``verl0.4`` is built for verl release ``v0.4.x``. + +Base Image +:::::::::: + +The stable base image is ``verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.1-fa2.7.4`` for vLLM and sglang. The installed package versions can be found from tags, and the Dockerfile can be found in ``docker/verl[version]-[packages]/Dockerfile.base``. + +The update of base image is not frequent, and the app image can be built on top of it without reinstalling base packages. + +Application Image +::::::::::::::::: + +From this version, we divide images built for vLLM and SGLang as the divergence of dependent packages like Pytorch and FlashInfer. + +There are 2 types of application images available: + +- **vLLM with FSDP and Megatron**: ``verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2`` +- **SGLang with FSDP and Megatron**: ``verlai/verl:app-verl0.5-transformers4.55.4-sglang0.4.10.post2-mcore0.13.0-te2.2`` + +Docker images with Megatron backends are runnable with large language model like ``Qwen/Qwen3-235B-A22B``, ``deepseek-ai/DeepSeek-V3-0324`` post-training. Refer to the :doc:`Large Language Model Post-Training documentation<../perf/dpsk>` for more details. + +Application images can be updated frequently, and the Dockerfile can be found in ``docker/verl[version]-[packages]/Dockerfile.app.[frameworks]``. Based on the base image, it is easy to build your own application image with the desired inference and training frameworks. + +Community Image +::::::::::::::: + +Community images are provided by the community, including the latest versions of vLLM and SGLang, and may include experimental features or configurations. And also works for other hardwares or platforms like AMD GPUs with ROCM or AWS EFA and Sagemaker. + +For latest vLLM with FSDP, please refer to `hiyouga/verl `_ repository and the latest version is ``hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.4-flashinfer0.2.2-cxx11abi0``. + +For latest SGLang with FSDP, please refer to `hebiaobuaa/verl `_ repository and the latest version is ``hebiaobuaa/verl:app-verl0.5-sglang0.4.9.post6-mcore0.12.2-te2.2`` which is provided by SGLang RL Group. + +For latest vLLM with Megatron, please refer to `iseekyan/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.15.0-te2.7` + +See files under ``docker/`` for NGC-based image or if you want to build your own. + +Note that For aws instances with EFA net interface (Sagemaker AI Pod), +you need to install EFA driver as shown in ``docker/Dockerfile.extenstion.awsefa`` + +Installation from Docker +:::::::::::::::::::::::: + +After pulling the desired Docker image and installing desired inference and training frameworks, you can run it with the following steps: + +1. Launch the desired Docker image and attach into it: + +.. code:: bash + + docker create --runtime=nvidia --gpus all --net=host --shm-size="10g" --cap-add=SYS_ADMIN -v .:/workspace/verl --name verl sleep infinity + docker start verl + docker exec -it verl bash + + +2. If you use the images provided, you only need to install verl itself without dependencies: + +.. code:: bash + + # install the nightly version (recommended) + git clone https://github.com/volcengine/verl && cd verl + pip3 install --no-deps -e . + +[Optional] If you hope to switch between different frameworks, you can install verl with the following command: + +.. code:: bash + + # install the nightly version (recommended) + git clone https://github.com/volcengine/verl && cd verl + pip3 install -e .[vllm] + pip3 install -e .[sglang] + + +Install from custom environment +--------------------------------------------- + +We recommend to use docker images for convenience. However, if your environment is not compatible with the docker image, you can also install verl in a python environment. + + +Pre-requisites +:::::::::::::: + +For training and inference engines to utilize better and faster hardware support, CUDA/cuDNN and other dependencies are required, +and some of the dependencies are easy to be overridden when installing other packages, +so we put them in the :ref:`Post-installation` step. + +.. note:: + + The installation steps below are recommended configurations for the latest version of verl. + If you are trying to customize your own environment, please ignore the strict constraints. + +We need to install the following pre-requisites: + +- **CUDA**: Version >= 12.4 +- **cuDNN**: Version >= 9.8.0 +- **Apex** + +CUDA above 12.4 is recommended to use as the docker image, +please refer to `NVIDIA's official website `_ for other version of CUDA. + +.. code:: bash + + # change directory to anywher you like, in verl source code directory is not recommended + wget https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda-repo-ubuntu2204-12-4-local_12.4.1-550.54.15-1_amd64.deb + dpkg -i cuda-repo-ubuntu2204-12-4-local_12.4.1-550.54.15-1_amd64.deb + cp /var/cuda-repo-ubuntu2204-12-4-local/cuda-*-keyring.gpg /usr/share/keyrings/ + apt-get update + apt-get -y install cuda-toolkit-12-4 + update-alternatives --set cuda /usr/local/cuda-12.4 + + +cuDNN can be installed via the following command, +please refer to `NVIDIA's official website `_ for other version of cuDNN. + +.. code:: bash + + # change directory to anywher you like, in verl source code directory is not recommended + wget https://developer.download.nvidia.com/compute/cudnn/9.8.0/local_installers/cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb + dpkg -i cudnn-local-repo-ubuntu2204-9.8.0_1.0-1_amd64.deb + cp /var/cudnn-local-repo-ubuntu2204-9.8.0/cudnn-*-keyring.gpg /usr/share/keyrings/ + apt-get update + apt-get -y install cudnn-cuda-12 + +NVIDIA Apex is required for Megatron-LM and FSDP training. +You can install it via the following command, but notice that this steps can take a very long time. +It is recommended to set the ``MAX_JOBS`` environment variable to accelerate the installation process, +but do not set it too large, otherwise the memory will be overloaded and your machines may hang. + +.. code:: bash + + # change directory to anywher you like, in verl source code directory is not recommended + git clone https://github.com/NVIDIA/apex.git && \ + cd apex && \ + MAX_JOB=32 pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ + + +Install dependencies +:::::::::::::::::::: + +.. note:: + + We recommend to use a fresh new conda environment to install verl and its dependencies. + + **Notice that the inference frameworks often strictly limit your pytorch version and will directly override your installed pytorch if not paying enough attention.** + + As a countermeasure, it is recommended to install inference frameworks first with the pytorch they needed. For vLLM, if you hope to use your existing pytorch, + please follow their official instructions + `Use an existing PyTorch installation `_ . + + +1. First of all, to manage environment, we recommend using conda: + +.. code:: bash + + conda create -n verl python==3.10 + conda activate verl + + +2. Then, execute the ``install.sh`` script that we provided in verl: + +.. code:: bash + + # Make sure you have activated verl conda env + # If you need to run with megatron + bash scripts/install_vllm_sglang_mcore.sh + # Or if you simply need to run with FSDP + USE_MEGATRON=0 bash scripts/install_vllm_sglang_mcore.sh + + +If you encounter errors in this step, please check the script and manually follow the steps in the script. + + +Install verl +:::::::::::: + +For installing the latest version of verl, the best way is to clone and +install it from source. Then you can modify our code to customize your +own post-training jobs. + +.. code:: bash + + git clone https://github.com/volcengine/verl.git + cd verl + pip install --no-deps -e . + + +Post-installation +::::::::::::::::: + +Please make sure that the installed packages are not overridden during the installation of other packages. + +The packages worth checking are: + +- **torch** and torch series +- **vLLM** +- **SGLang** +- **pyarrow** +- **tensordict** +- **nvidia-cudnn-cu12**: For Magetron backend + +If you encounter issues about package versions during running verl, please update the outdated ones. + + +Install with AMD GPUs - ROCM kernel support +------------------------------------------------------------------ + +When you run on AMD GPUs (MI300) with ROCM platform, you cannot use the previous quickstart to run verl. You should follow the following steps to build a docker and run it. +If you encounter any issues in using AMD GPUs running verl, feel free to contact me - `Yusheng Su `_. + +Find the docker for AMD ROCm: `docker/Dockerfile.rocm `_ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + +.. code-block:: bash + + # Build the docker in the repo dir: + # docker build -f docker/Dockerfile.rocm -t verl-rocm:03.04.2015 . + # docker images # you can find your built docker + FROM rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 + + # Set working directory + # WORKDIR $PWD/app + + # Set environment variables + ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" + + # Install vllm + RUN pip uninstall -y vllm && \ + rm -rf vllm && \ + git clone -b v0.6.3 https://github.com/vllm-project/vllm.git && \ + cd vllm && \ + MAX_JOBS=$(nproc) python3 setup.py install && \ + cd .. && \ + rm -rf vllm + + # Copy the entire project directory + COPY . . + + # Install dependencies + RUN pip install "tensordict<0.6" --no-deps && \ + pip install accelerate \ + codetiming \ + datasets \ + dill \ + hydra-core \ + liger-kernel \ + numpy \ + pandas \ + datasets \ + peft \ + "pyarrow>=15.0.0" \ + pylatexenc \ + "ray[data,train,tune,serve]" \ + torchdata \ + transformers \ + wandb \ + orjson \ + pybind11 && \ + pip install -e . --no-deps + +Build the image +:::::::::::::::::::::::: + +.. code-block:: bash + + docker build -t verl-rocm . + +Launch the container +:::::::::::::::::::::::::::: + +.. code-block:: bash + + docker run --rm -it \ + --device /dev/dri \ + --device /dev/kfd \ + -p 8265:8265 \ + --group-add video \ + --cap-add SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --privileged \ + -v $HOME/.ssh:/root/.ssh \ + -v $HOME:$HOME \ + --shm-size 128G \ + -w $PWD \ + verl-rocm \ + /bin/bash + +If you do not want to root mode and require assign yourself as the user, +Please add ``-e HOST_UID=$(id -u)`` and ``-e HOST_GID=$(id -g)`` into the above docker launch script. + +verl with AMD GPUs currently supports FSDP as the training engine, vLLM and SGLang as the inference engine. We will support Megatron in the future. diff --git a/e2c/verl/docs/start/more_resources.rst b/e2c/verl/docs/start/more_resources.rst new file mode 100644 index 0000000000000000000000000000000000000000..aa8cb2a62b46579ee4bef2880d7f62485175495e --- /dev/null +++ b/e2c/verl/docs/start/more_resources.rst @@ -0,0 +1,7 @@ +More Resources +============== + +Last updated: 06/30/2025. + +- Introduction to verl (`Slides `_) +- verl Code Walkthrough (`Slides `_, `Talk in Chinese `_) diff --git a/e2c/verl/docs/start/multinode.rst b/e2c/verl/docs/start/multinode.rst new file mode 100644 index 0000000000000000000000000000000000000000..3c54b8526356c32879bb1e4b21d68698b1a82d54 --- /dev/null +++ b/e2c/verl/docs/start/multinode.rst @@ -0,0 +1,821 @@ +Multinode Training +================== + +Last updated: 06/10/2025. + +.. _wuxibin89: https://github.com/wuxibin89 + +Author: `Xibin Wu `_, `Yusheng Su `_. + +Option 1: Launch Manually +------------------------------ + +Set up multinode ray cluster +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1. Start head node with ``ray start --head --dashboard-host=0.0.0.0``, there're 2 address you should care about: + +- GCS address: ``ray start --address=
``, where worker node should connect to. +- Dashboard address: ``
:8265``, where you should submit job to the cluster. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/head.png?raw=true + +2. Start worker node with ``ray start --address=
`` you get above. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/worker.png?raw=true + +3. Now you should see the cluster have 2 nodes with ``ray status``. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/status.png?raw=true + +4. Additionally, you can access dashboard in the browser with the address you get above. + +*Firewall rules maybe need configure to access the dashboard, if there's any trouble, please contact your network administrator.* + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/overview.png?raw=true + +Submit job to ray cluster +~~~~~~~~~~~~~~~~~~~~~~~~~ +1. Submit ray job to cluster with the dashboard address you get above. + +.. code-block:: bash + + ray job submit --address="http://127.0.0.1:8265" \ + --runtime-env=verl/trainer/runtime_env.yaml \ + --no-wait \ + -- \ + python3 -m verl.trainer.main_ppo \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=2 \ + ... + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/submit.png?raw=true + +2. Then you can check the job status with the following commands: + +- ray job list: list all jobs submitted to the cluster. +- ray job logs : query the logs of the job. +- ray job status : query the status of the job. +- ray job stop : request the job to be stopped. +- ray job list | grep submission_id | grep JobStatus | grep RUNNING | grep -oP 'raysubmit_[^'\''"]+' | head -n 1: get the latest job submission ID of the running job. +- ray job logs --follow: added ``--follow`` parameter to ray job logs command to enable continuous log streaming. + +3. You can also access driver/task/actor logs in ``/tmp/ray/session_latest/logs/``, driver log is ``job-driver-raysubmit_.log``. + +4. We strongly recommend you to view job detail from dashboard in multinode training, because it provide more structure way to view the job information. + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/job.png?raw=true +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/job_detail.png?raw=true + +Option 2: Launch via SkyPilot on Kubernetes or clouds +------------------------------------------------------ + +.. note:: + Ready-to-use SkyPilot example configurations are available in the `examples/skypilot/ `_ directory: + + - ``verl-ppo.yaml`` - PPO training with GSM8K dataset + - ``verl-grpo.yaml`` - GRPO training with MATH dataset + - ``verl-multiturn-tools.yaml`` - Multi-turn tool usage training + + See the `SkyPilot examples README `_ for detailed usage instructions. + +Step 1: Setup SkyPilot +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +SkyPilot can support different clouds, here we use GCP as example. `install skypilot `_ + +.. code-block:: bash + + conda create -y -n sky python=3.10 + conda activate sky + pip install "skypilot[gcp]" + + conda install -c conda-forge google-cloud-sdk + gcloud init + + # Run this if you don't have a credential file. + # This will generate ~/.config/gcloud/application_default_credentials.json. + gcloud auth application-default login + + # Check if the GCP credential is correctly setup. + sky check gcp + +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/setup_skypilot.png?raw=true + +Step 2: Prepare dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + git clone https://github.com/volcengine/verl.git + cd examples/data_preprocess + python3 gsm8k.py --local_save_dir ~/data/gsm8k + + +Step 3: Submit a job with SkyPilot +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1. Create a SkyPilot YAML ``verl-cluster.yml`` with the following content: + +.. parsed-literal:: workdir: . will sync all the data in the current dir to the remote cluster. + +.. code-block:: yaml + + resources: + accelerators: L4:1 # every node has 1 L4 GPU + image_id: docker:verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.0-fa2.7.4 + memory: 64+ # every node has 64 GB memory + ports: 8265 # expose port for ray dashboard + + num_nodes: 2 # cluster size + + # --------------- Work Directory Synchronization (workdir) --------------- + # Defines the local working directory to be synchronized to the remote cluster. + # Here, '.' means synchronizing the directory where the sky submit command is currently run. + workdir: . + + # --------------- (secrets) --------------- + secrets: + ## your wandb api key ## + WANDB_API_KEY: null + + # --------------- File Mounts/Data Upload (file_mounts) --------------- + # If your dataset (gsm8k folder) is local, it needs to be uploaded to the remote cluster. + file_mounts: + # Remote path (relative to remote user's home directory): Local path + # /remote/dir1/file: /local/dir1/file + data/gsm8k: ~/data/gsm8k + + # --------------- Environment Setup (setup) --------------- + # Commands run on each node of the remote cluster to set up the environment (e.g., install dependencies). These are run directly inside Docker. + setup: | + rm -rf verl + git clone https://github.com/volcengine/verl.git + cd verl + pip3 install -v -e .[vllm] + + # --------------- Run Command (run) --------------- + # The actual task commands to be executed on the remote cluster. + # This script will first start the Ray cluster (different ray start commands are executed on Head and Worker nodes). + # Then, your training script will only be run on the Head node (SKYPILOT_NODE_RANK == 0). + run: | + # Get the Head node's IP and total number of nodes (environment variables injected by SkyPilot). + head_ip=`echo "$SKYPILOT_NODE_IPS" | head -n1` + num_nodes=`echo "$SKYPILOT_NODE_IPS" | wc -l` # Here num_nodes should be equal to 2. + + # login wandb + python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + # Start Ray based on node role (Head=0, Worker>0). + # This logic is a standard Ray cluster startup script. + if [ "$SKYPILOT_NODE_RANK" == "0" ]; then + # Head node starts Ray Head. + echo "Starting Ray head node..." + # Check if a Ray Head is already running to avoid duplicate starts. + ps aux | grep ray | grep 6379 &> /dev/null || ray start --head --disable-usage-stats \ + --port=6379 \ + --dashboard-host=0.0.0.0 \ + --dashboard-port=8265 + + # Wait for all worker nodes to join the cluster. + while [ $(ray nodes | grep NODE_ID | wc -l) -lt $num_nodes ]; do + echo "Waiting for all nodes to join... ($(ray nodes | grep NODE_ID | wc -l)/$num_nodes)" + sleep 5 + done + + # Head node executes the training script. + echo "Executing training script on head node..." + + python3 -m verl.trainer.main_ppo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=512 \ + data.max_response_length=256 \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=['console','wandb'] \ + trainer.val_before_train=False \ + trainer.default_hdfs_dir=null \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=2 \ + trainer.save_freq=20 \ + trainer.test_freq=20 \ + trainer.total_epochs=2 \ + trainer.project_name=verl_examples \ + trainer.experiment_name=experiment_name_gsm8k + + else + # Wait for Ray Head to start. + sleep 10 # Increase waiting time to ensure Head finishes starting. + # Worker node starts Ray Worker. + echo "Starting Ray worker node..." + + # Check if a Ray Worker is already running to avoid duplicate starts. + ps aux | grep ray | grep $head_ip:6379 &> /dev/null || ray start --address $head_ip:6379 --disable-usage-stats + + # Add sleep to after `ray start` to give ray enough time to daemonize + sleep 5 # Ensure Worker successfully connects to Head. + fi + + # No commands are added to the Worker node here; the Worker's main task is to start Ray and wait for the Head node to assign tasks. + echo "Node setup and Ray start script finished for rank $SKYPILOT_NODE_RANK." + + +.. code-block:: bash + + export WANDB_API_KEY= + sky launch -c verl --secret WANDB_API_KEY verl-cluster.yml + +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/running_job.png?raw=true +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/running_job_1.png?raw=true +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/finished.png?raw=true + +**Check the cluster on GCP** + +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/gcp_instances.png?raw=true + +**Check Ray Dashboard** + +We can see the cluster on the RAY Dashboard with the GCP head node: + +```console +$ sky status --endpoint 8265 verl +1.2.3.4:8265 +``` + +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/ray_dashboard_overview.png?raw=true +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/ray_dashboard_jobs.png?raw=true +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/ray_dashboard_cluster.png?raw=true + + +**Check the checkpoint of model** + +.. code-block:: bash + + # login the head node + ssh verl + # The global step will vary. Find the correct path from the training logs. + cd ~/sky_workdir/checkpoints/verl_examples/gsm8k/ + # Then list contents to find the checkpoint, e.g.: + ls -R . + +.. image:: https://github.com/yottalabsai/open-source/blob/main/static/verl/saved_model.png?raw=true + + +Option 3: Launch via Slurm +------------------------------ + +Ray provides users with `this `_ official +tutorial to start a Ray cluster on top of Slurm. We have verified the :doc:`GSM8K example<../examples/gsm8k_example>` +on a Slurm cluster under a multi-node setting with the following steps. + +1. [Optional] If your cluster support `Apptainer or Singularity `_ and you wish +to use it, convert verl's Docker image to an Apptainer image. Alternatively, set up the environment with the package +manager available on your cluster or use other container runtimes (e.g. through `Slurm's OCI support `_) available to you. + +.. code:: bash + + apptainer pull /your/dest/dir/vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3.sif docker://verlai/verl:vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3 + +2. Follow :doc:`GSM8K example<../examples/gsm8k_example>` to prepare the dataset and model checkpoints. + +3. Modify `examples/slurm/ray_on_slurm.slurm `_ with your cluster's own information. + +4. Submit the job script to the Slurm cluster with `sbatch`. + +Please note that Slurm cluster setup may vary. If you encounter any issues, please refer to Ray's +`Slurm user guide `_ for common caveats. + +If you changed Slurm resource specifications, please make sure to update the environment variables in the job script if necessary. + + +Option 4: Launch via dstack +------------------------------ + +`dstackai/dstack `_ is an open-source container orchestrator that simplifies distributed training across cloud providers and on-premises environments +without the need to use K8S or Slurm. + +Prerequisite +~~~~~~~~~~~~ +Once dstack is `installed `_, initialize the directory as a repo with ``dstack init``. + +.. code-block:: bash + + mkdir myproject && cd myproject + dstack init + +**Create a fleet** + +Before submitting distributed training jobs, create a `dstack` `fleet `_. + +Run a Ray cluster task +~~~~~~~~~~~~~~~~~~~~~~ + +Once the fleet is created, define a Ray cluster task, e.g. in ``ray-cluster.dstack.yml``: + +.. code-block:: yaml + + type: task + name: ray-verl-cluster + + nodes: 2 + + env: + - WANDB_API_KEY + - PYTHONUNBUFFERED=1 + - CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + + image: whatcanyousee/verl:ngc-cu124-vllm0.8.5-sglang0.4.6-mcore0.12.0-te2.2 + commands: + - git clone https://github.com/volcengine/verl + - cd verl + - pip install --no-deps -e . + - pip install hf_transfer hf_xet + - | + if [ $DSTACK_NODE_RANK = 0 ]; then + python3 examples/data_preprocess/gsm8k.py --local_save_dir ~/data/gsm8k + python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2.5-7B-Instruct')" + ray start --head --port=6379; + else + ray start --address=$DSTACK_MASTER_NODE_IP:6379 + fi + + # Expose Ray dashboard port + ports: + - 8265 + + resources: + gpu: 80GB:8 + shm_size: 128GB + + # Save checkpoints on the instance + volumes: + - /checkpoints:/checkpoints + +Now, if you run this task via `dstack apply`, it will automatically forward the Ray's dashboard port to `localhost:8265`. + +.. code-block:: bash + + dstack apply -f ray-cluster.dstack.yml + +As long as the `dstack apply` is attached, you can use `localhost:8265` to submit Ray jobs for execution + +Submit Ray jobs +~~~~~~~~~~~~~~~ + +Before you can submit Ray jobs, ensure to install `ray` locally: + +.. code-block:: shell + + pip install ray + +Now you can submit the training job to the Ray cluster which is available at ``localhost:8265``: + +.. code-block:: shell + + $ RAY_ADDRESS=http://localhost:8265 + $ ray job submit \ + -- python3 -m verl.trainer.main_ppo \ + data.train_files=/root/data/gsm8k/train.parquet \ + data.val_files=/root/data/gsm8k/test.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=512 \ + data.max_response_length=256 \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2.5-7B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.project_name=ppo_training \ + trainer.experiment_name=qwen-2.5-7B \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=2 \ + trainer.default_local_dir=/checkpoints \ + trainer.save_freq=10 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 2>&1 | tee verl_demo.log \ + trainer.resume_mode=disable + + +For more details on how `dstack` works, check out its `documentation `_. + +How to debug? +--------------------- + + +Ray Distributed Debugger VSCode Extension (Recommended) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. Starting with Ray 2.39, Anyscale has introduced the `Ray Distributed Debugger `_ VSCode extension. Follow the extension’s installation instructions, then add your cluster using the dashboard URL you obtained earlier. + + .. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/debugger.png?raw=true + :alt: Ray Distributed Debugger VSCode extension screenshot + +2. Prerequisites. + + Ensure the following are installed (see the extension README for more detail): + + - Visual Studio Code + - `ray[default]` >= 2.9.1 + - `debugpy` >= 1.8.0 + + .. image:: https://github.com/aoshen524/verl/blob/main/docs/start/c7098b755ff689859837773a916c857.png?raw=true + :alt: VSCode with Ray prerequisites + +3. Environment Variables. + + To enable post‑mortem debugging, set: + + .. code-block:: bash + + export RAY_DEBUG_POST_MORTEM=1 + + .. admonition:: Note + :class: important + + Be sure to remove any legacy flags before starting Ray: + + - `RAY_DEBUG=legacy` + - `--ray-debugger-external` + +4. Configuring BreakpointsSet up breakpoint() in your code, and submit job to cluster. Then the extension will show the breakpoint information. + + + 1. Insert `breakpoint()` calls into your remote functions. + 2. Submit your job to the cluster. + + The extension will detect active breakpoints and display them in VSCode. + + .. image:: https://github.com/aoshen524/verl/blob/main/docs/start/4ddad74395c79a1402331c0ce73316f.png?raw=true + :alt: Detected breakpoint in VSCode + + **Note:** Breakpoints are only supported inside functions decorated with `@ray.remote`. + +5. Launching the Debugger. + + Run your job directly from the command line (do not use a `launch.json`): + + .. code-block:: bash + + python job.py + +6. Attaching to a Breakpoint. + + Once the process hits the first `breakpoint()`, click the Ray Distributed Debugger icon in the VSCode sidebar to attach the debugger. + + .. image:: https://github.com/aoshen524/verl/blob/main/docs/start/4ddad74395c79a1402331c0ce73316f.png?raw=true + :alt: Attaching VSCode debugger to Ray process + +7. Debugging With Multiple breakpoint(). + + For each subsequent task, first disconnect the current debugger session, then click the extension icon again to attach to the next breakpoint. + + .. image:: https://github.com/aoshen524/verl/blob/main/docs/start/6e83c910a62c82fecb89c6619e001cd.png?raw=true + :alt: Disconnecting and reconnecting the debugger + +Legacy Ray Debugger +~~~~~~~~~~~~~~~~~~~ +1. Ray has a builtin legacy `debugger `_ that allows you to debug your distributed applications. To enable debugger, start ray cluster with ``RAY_DEBUG=legacy`` and ``--ray-debugger-external``. + +.. code-block:: bash + + # start head node + RAY_DEBUG=legacy ray start --head --dashboard-host=0.0.0.0 --ray-debugger-external + # start worker node + RAY_DEBUG=legacy ray start --address='10.124.46.192:6379' --ray-debugger-external + +2. Set up breakpoint in your code, and submit job to cluster. Then run ``ray debug`` to wait breakpoint: + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/legacy.png?raw=true + + +Multi-node training on AMD clusters +--------------------------------------------------------------------------------------- + +If you want to run multi-node training with slurm with Docker/Podman container on AMD Cluster, you can use the following script. + +If you encounter any issues in using AMD GPUs running verl, please contact `Yusheng Su `_. + +.. note:: + 1. You need to use ``podman`` or ``docker`` in the following script. We will release the apptainer script later. + 2. If you want to use ``podman``, you just replace ``docker`` with ``podman`` in the following script. + +The script includes the following steps: + +1. SLURM Configuration +2. Environment Setup +3. Docker/Podman Container Setup +4. Ray Cluster Initialization +5. Data Preprocessing +6. Model Setup +7. Training Launch + + +slurm_script.sh +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + #!/bin/bash + + #SBATCH --job-name=verl-ray-on-slurm + #SBATCH --nodes=2 + #SBATCH --ntasks-per-node=2 + #SBATCH --mem=200G + #SBATCH --time=30-00:00:00 + #SBATCH --gpus-per-node=8 + #SBATCH --cpus-per-task=28 + #SBATCH --output=../verl_log/slurm-%j.out + #SBATCH --error=../verl_log/slurm-%j.err + #SBATCH --nodelist=gpu-[0,1] + + + # load necessary modules + ### Run this setup + # [Cluster]: Use docker + # docker pull docker.io/rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 + + + ########################################################################## + ###The following setting should be set in different project and cluster### + ########################################################################## + + ### Project + CONTAINER_NAME="multinode_verl_training" + IMG="verl.rocm" + DOCKERFILE="docker/Dockerfile.rocm" + # echo $PWD + verl_workdir="${HOME}/projects/verl_upstream" + export TRANSFORMERS_CACHE="${HOME}/.cache/huggingface" + export HF_HOME=$TRANSFORMERS_CACHE + + ### Cluster Network Setting + export NCCL_DEBUG=TRACE + export GPU_MAX_HW_QUEUES=2 + export TORCH_NCCL_HIGH_PRIORITY=1 + export NCCL_CHECKS_DISABLE=1 + # export NCCL_IB_HCA=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9 + export NCCL_IB_GID_INDEX=3 + export NCCL_CROSS_NIC=0 + export CUDA_DEVICE_MAX_CONNECTIONS=1 + export NCCL_PROTO=Simple + export RCCL_MSCCL_ENABLE=0 + export TOKENIZERS_PARALLELISM=false + export HSA_NO_SCRATCH_RECLAIM=1 + ########################################################################## + + ### For rocm and training script + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES + export CUDA_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES + + + # Build and launch the Docker container + srun bash -c " + # Exit on any error + set -e + + # Clean up dangling images (images with tag) + docker image prune -f + + # Need to pull the docker first + docker pull docker.io/rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 + + if ! docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "${IMG}"; then + echo \"Building ${IMG} image...\" + docker build -f \"${DOCKERFILE}\" -t \"${IMG}\" . + else + echo \"${IMG} image already exists, skipping build\" + fi + + # Removing old container if exists + docker rm \"${CONTAINER_NAME}\" 2>/dev/null || true + + # Checking network devices + ibdev2netdev + + # Launch the docker + docker run --rm -d \ + -e HYDRA_FULL_ERROR=1 \ + -e HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES} \ + -e ROCR_VISIBLE_DEVICES=${ROCR_VISIBLE_DEVICES} \ + -e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \ + -e NCCL_DEBUG=${NCCL_DEBUG} \ + -e GPU_MAX_HW_QUEUES=${GPU_MAX_HW_QUEUES} \ + -e TORCH_NCCL_HIGH_PRIORITY=${TORCH_NCCL_HIGH_PRIORITY} \ + -e NCCL_CHECKS_DISABLE=${NCCL_CHECKS_DISABLE} \ + -e NCCL_IB_HCA=${NCCL_IB_HCA} \ + -e NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} \ + -e NCCL_CROSS_NIC=${NCCL_CROSS_NIC} \ + -e CUDA_DEVICE_MAX_CONNECTIONS=${CUDA_DEVICE_MAX_CONNECTIONS} \ + -e NCCL_PROTO=${NCCL_PROTO} \ + -e RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE} \ + -e TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM} \ + -e HSA_NO_SCRATCH_RECLAIM=${HSA_NO_SCRATCH_RECLAIM} \ + -e TRANSFORMERS_CACHE=${TRANSFORMERS_CACHE} \ + -e HF_HOME=${HF_HOME} \ + --network host \ + --device /dev/dri \ + --device /dev/kfd \ + --device /dev/infiniband \ + --group-add video \ + --cap-add SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --privileged \ + -v \${HOME}:\${HOME} \ + -v \${HOME}/.ssh:/root/.ssh \ + -w "${verl_workdir}" \ + --shm-size 128G \ + --name \"${CONTAINER_NAME}\" \ + \"${IMG}\" \ + tail -f /dev/null + + echo \"Container setup completed\" + " + # (Optional): If you do not want to root mode and require assign yuorself as the user + # Please add `-e HOST_UID=$(id -u)` and `-e HOST_GID=$(id -g)` into the above docker launch script. + + + + + + ### Ray launch the nodes before training + + # Getting the node names + nodes_array=($(scontrol show hostnames "$SLURM_JOB_NODELIST" | tr '\n' ' ')) + + head_node=${nodes_array[0]} + head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) + + # if we detect a space character in the head node IP, we'll + # convert it to an ipv4 address. This step is optional. + if [[ "$head_node_ip" == *" "* ]]; then + IFS=' ' read -ra ADDR <<<"$head_node_ip" + if [[ ${#ADDR[0]} -gt 16 ]]; then + head_node_ip=${ADDR[1]} + else + head_node_ip=${ADDR[0]} + fi + echo "IPV6 address detected. We split the IPV4 address as $head_node_ip" + fi + + port=6379 + ip_head=$head_node_ip:$port + export ip_head + echo "IP Head: $ip_head" + + # make sure we set environment variables before Ray initialization + + # Print out all env variables + printenv + + echo "Starting HEAD at $head_node" + srun --nodes=1 --ntasks=1 -w "$head_node" \ + docker exec "${CONTAINER_NAME}" \ + ray start --head --node-ip-address="$head_node_ip" --port=$port \ + --dashboard-port=8266 \ + --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & + # optional, though may be useful in certain versions of Ray < 1.0. + sleep 10 + + # number of nodes other than the head node + worker_num=$((SLURM_JOB_NUM_NODES - 1)) + + for ((i = 1; i <= worker_num; i++)); do + node_i=${nodes_array[$i]} + echo "Debug: Starting worker on node_i = ${node_i}" + if [ -z "$node_i" ]; then + echo "Error: Empty node name for worker $i" + continue + fi + echo "Starting WORKER $i at $node_i" + srun --nodes=1 --ntasks=1 -w "$node_i" \ + docker exec "${CONTAINER_NAME}" \ + ray start --address "$ip_head" --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & + sleep 5 + done + + + + + # Ray initlization test (See whether any error in the above execution) + echo "Testing Ray initialization in the slurm nodes..." + docker exec "${CONTAINER_NAME}" python3 -c ' + import ray + try: + ray.init(address="auto") + print("\n=== Ray Cluster Status ===") + print(f"Number of nodes: {len(ray.nodes())}") + for node in ray.nodes(): + print("Node: {}, Status: {}".format(node["NodeManagerHostname"], node["Alive"])) + # print(f"Node: {node}") + ray.shutdown() + print("Ray initialization successful!") + except Exception as e: + print(f"Ray initialization failed: {str(e)}") + ' + echo "=== Ray test completed ===" + ###### + + + + # Run data preprocessing + + echo "Starting data preprocessing..." + docker exec "${CONTAINER_NAME}" \ + python3 "examples/data_preprocess/gsm8k.py" "--local_save_dir" "../data/gsm8k" + + echo "Starting data preprocessing..." + docker exec "${CONTAINER_NAME}" \ + python3 "examples/data_preprocess/math_dataset.py" "--local_dir" "../data/math" + + train_files="../data/gsm8k/train.parquet" + val_files="../data/gsm8k/test.parquet" + + # Download and test model + echo "Loading model..." + docker exec "${CONTAINER_NAME}" \ + python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2-7B-Instruct')" + MODEL_PATH="Qwen/Qwen2-7B-Instruct" + + # Set model path after pipeline test + MODEL_PATH="Qwen/Qwen2.5-0.5B-Instruct" + + echo "== Data and model loading Done ==" + + echo "Start to train..." + + docker exec "${CONTAINER_NAME}" \ + python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2-7B-Instruct')" + MODEL_PATH="Qwen/Qwen2-7B-Instruct" + + + PYTHONUNBUFFERED=1 srun --overlap --nodes=${SLURM_NNODES} --ntasks=1 -w "$head_node" \ + docker exec "${CONTAINER_NAME}" \ + python3 -m verl.trainer.main_ppo \ + data.train_files=$train_files \ + data.val_files=$val_files \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.enable_gradient_checkpointing=False \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=$MODEL_PATH \ + critic.model.enable_gradient_checkpointing=False \ + critic.ppo_micro_batch_size_per_gpu=8 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.kl_ctrl.kl_coef=0.0001 \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example' \ + trainer.experiment_name='Qwen2.5-32B-Instruct_function_rm' \ + trainer.n_gpus_per_node=${SLURM_GPUS_PER_NODE} \ + trainer.val_before_train=False \ + trainer.nnodes=${SLURM_NNODES} \ + trainer.save_freq=-1 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 + + +Run multi-node training with above slurm_script.sh +~~~~~~~~~~~~~~~~~~~~ +Just sbatch your slurm_script.sh + +.. code-block:: bash + + sbatch slurm_script.sh + diff --git a/e2c/verl/docs/start/quickstart.rst b/e2c/verl/docs/start/quickstart.rst new file mode 100644 index 0000000000000000000000000000000000000000..02ea6c20a0fc55c42af08f8084f9b4e1608eb2f1 --- /dev/null +++ b/e2c/verl/docs/start/quickstart.rst @@ -0,0 +1,151 @@ +.. _quickstart: + +========================================================= +Quickstart: PPO training on GSM8K dataset +========================================================= + +Post-train a LLM using GSM8K dataset. + +Introduction +------------ + +.. _hf_dataset_gsm8k: https://huggingface.co/datasets/gsm8k + +In this example, we train an LLM to tackle the `GSM8k `_ task with function-based rewards. [1]_ + +Prerequisite: + +- the latest version of ``verl`` and its dependencies installed following the installation guide. Using the docker image is recommended. + +- a GPU with at least 24 GB HBM + + +Dataset Introduction +-------------------- + +GSM8k is a math problem dataset. The prompt is an elementary school +problem. The LLM model is asked to solve the math problem. Below is an example: + +Prompt + + Katy makes coffee using teaspoons of sugar and cups of water in the + ratio of 7:13. If she used a total of 120 teaspoons of sugar and cups + of water, calculate the number of teaspoonfuls of sugar she used. + +Solution + + The total ratio representing the ingredients she used to make the + coffee is 7+13 = <<7+13=20>>20 Since the fraction representing the + number of teaspoons she used is 7/20, she used 7/20\ *120 = + <<7/20*\ 120=42>>42 #### 42 + +Step 1: Prepare the dataset +---------------------------- + +We preprocess the dataset in parquet format so that (1) it contains necessary fields for computing RL rewards and (2) is faster to read. + +.. code-block:: bash + + python3 examples/data_preprocess/gsm8k.py --local_save_dir ~/data/gsm8k + +Step 2: Download a model for post-training +------------------------------------------- + +In this example, we start with the ``Qwen2.5-0.5B-Instruct`` model. + +If you want to perform SFT before RL, refer to the :doc:`Complete GSM8K Example<../examples/gsm8k_example>`, the `sft directory `_ and `SFT Trainer `_ for further details. + +.. code-block:: bash + + python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2.5-0.5B-Instruct')" + +Step 3: Perform PPO training with the instruct model +---------------------------------------------------------------------- + +**Reward Model/Function** + +We use a pre-defined rule-based reward model. We force the model to produce a final +answer following 4 “#” as shown in the solution. We extract the final +answer from both the solution and model's output using regular +expression matching. We assign a reward of 1 to correct +answer, 0.0 to incorrect answer and 0 to no answer. + +For more details, please refer to `verl/utils/reward_score/gsm8k.py `_. + +**Training Script** + +Now let's run PPO training with the dataset and model above. [2]_ + + +Set the ``data.train_files`` ,\ ``data.val_files``, ``actor_rollout_ref.model.path`` and ``critic.model.path`` based on your dataset and model names or paths. +You may set ``VERL_USE_MODELSCOPE=True`` to download models from `modelscope `_ instead of `huggingface `_. + +.. code-block:: bash + + PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=512 \ + data.max_response_length=256 \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=console \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + trainer.save_freq=10 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 2>&1 | tee verl_demo.log + +You are expected to see the following logs, indicating training in progress. The key metric ``val/test_score/openai/gsm8k`` is computed every ``trainer.test_freq`` steps: + +.. code-block:: bash + + step:0 - timing/gen:21.470 - timing/ref:4.360 - timing/values:5.800 - actor/reward_kl_penalty:0.000 - actor/reward_kl_penalty_coeff:0.001 - timing/adv:0.109 - timing/update_critic:15.664 - critic/vf_loss:14.947 - critic/vf_clipfrac:0.000 - critic/vpred_mean:-2.056 - critic/grad_norm:1023.278 - critic/lr(1e-4):0.100 - timing/update_actor:20.314 - actor/entropy_loss:0.433 - actor/pg_loss:-0.005 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/grad_norm:1.992 - actor/lr(1e-4):0.010 - critic/score/mean:0.004 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.004 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:-0.000 - critic/advantages/max:2.360 - critic/advantages/min:-2.280 - critic/returns/mean:0.003 - critic/returns/max:0.000 - critic/returns/min:0.000 - critic/values/mean:-2.045 - critic/values/max:9.500 - critic/values/min:-14.000 - response_length/mean:239.133 - response_length/max:256.000 - response_length/min:77.000 - prompt_length/mean:104.883 - prompt_length/max:175.000 - prompt_length/min:68.000 + step:1 - timing/gen:23.020 - timing/ref:4.322 - timing/values:5.953 - actor/reward_kl_penalty:0.000 - actor/reward_kl_penalty:0.001 - timing/adv:0.118 - timing/update_critic:15.646 - critic/vf_loss:18.472 - critic/vf_clipfrac:0.384 - critic/vpred_mean:1.038 - critic/grad_norm:942.924 - critic/lr(1e-4):0.100 - timing/update_actor:20.526 - actor/entropy_loss:0.440 - actor/pg_loss:0.000 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.060 - actor/lr(1e-4):0.010 - critic/score/mean:0.000 - critic/score/max:0.000 - critic/score/min:0.000 - critic/rewards/mean:0.000 - critic/rewards/max:0.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.000 - critic/advantages/max:2.702 - critic/advantages/min:-2.616 - critic/returns/mean:0.000 - critic/returns/max:0.000 - critic/returns/min:0.000 - critic/values/mean:-2.280 - critic/values/max:11.000 - critic/values/min:-16.000 - response_length/mean:232.242 - response_length/max:256.000 - response_length/min:91.000 - prompt_length/mean:102.398 - prompt_length/max:185.000 - prompt_length/min:70.000 + +Checkout ``Algorithm Baselines`` page for full training and validation logs for reference. + +The checkpoint is saved at the following dir by default: ``checkpoints/${trainer.project_name}/${trainer.experiment_name}``. You can merge the saved checkpoints to huggingface model using ``verl.model_merger`` module, for example: + +.. code-block:: bash + + python3 -m verl.model_merger merge \ + --backend fsdp \ + --local_dir checkpoints/${trainer.project_name}/${trainer.experiment_name}/global_step_1/actor \ + --target_dir checkpoints/${trainer.project_name}/${trainer.experiment_name}/global_step_1/actor/huggingface + +For more details about checkpoint and model merging, please refer to :ref:`checkpoint-page`. + +To enable ``wandb`` for experiment tracking, set the following configs: + +.. code-block:: bash + + trainer.logger='["console","wandb"]' \ + trainer.project_name=$YOUR_PROJECT_NAME \ + trainer.experiment_name=$YOUR_RUN_NAME \ + +If you encounter out of memory issues with HBM less than 32GB, enable the following configs would help: + +.. code-block:: bash + + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + critic.ppo_micro_batch_size_per_gpu=1 \ + +For the full set of configs, please refer to :ref:`config-explain-page` for detailed explanation and performance tuning. + + +.. [1] The original paper (https://arxiv.org/pdf/2110.14168) mainly focuses on training a verifier (a reward model) to solve math problems via Best-of-N sampling. In this example, we train an RL agent using a rule-based reward model. +.. [2] More training script examples for FSDP and Megatron-LM backend are stored in `examples/ppo_trainer `_ directory. diff --git a/e2c/verl/docs/start/ray_debug_tutorial.rst b/e2c/verl/docs/start/ray_debug_tutorial.rst new file mode 100644 index 0000000000000000000000000000000000000000..9e7c87dfaee0c04f24bdb6921717b8068d1ee6a2 --- /dev/null +++ b/e2c/verl/docs/start/ray_debug_tutorial.rst @@ -0,0 +1,96 @@ +Ray Debug Tutorial +================== + +Last updated: 04/23/2025 + + +.. _wuxibin89: https://github.com/wuxibin89 + +Author: `Ao Shen `_. + +How to debug? +--------------------- + + +Ray Distributed Debugger VSCode Extension (Recommended) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. Starting with Ray 2.39, Anyscale has introduced the `Ray Distributed Debugger `_ VSCode extension. Follow the extension’s installation instructions, then add your cluster using the dashboard URL you obtained earlier. + + .. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/debugger.png?raw=true + :alt: Ray Distributed Debugger VSCode extension screenshot + +2. Prerequisites. + + Ensure the following are installed (see the extension README for more detail): + + - Visual Studio Code + - `ray[default]` >= 2.9.1 + - `debugpy` >= 1.8.0 + + .. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/readme.png?raw=true + :alt: VSCode with Ray prerequisites + +3. Environment Variables. + + To enable post‑mortem debugging, set: + + .. code-block:: bash + + export RAY_DEBUG_POST_MORTEM=1 + + .. admonition:: Note + :class: important + + Be sure to remove any legacy flags before starting Ray: + + - `RAY_DEBUG=legacy` + - `--ray-debugger-external` + +4. Configuring BreakpointsSet up breakpoint() in your code, and submit job to cluster. Then the extension will show the breakpoint information. + + + 1. Insert `breakpoint()` calls into your remote functions. + 2. Submit your job to the cluster. + + The extension will detect active breakpoints and display them in VSCode. + + **Note:** Breakpoints are only supported inside functions decorated with `@ray.remote`. + +5. Launching the Debugger. + + Run your job directly from the command line (do not use a `launch.json`): + + .. code-block:: bash + + python job.py + +6. Attaching to a Breakpoint. + + Once the process hits the first `breakpoint()`, click the Ray Distributed Debugger icon in the VSCode sidebar to attach the debugger. + + .. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/launch.png?raw=true + :alt: Attaching VSCode debugger to Ray process + +7. Debugging With Multiple breakpoint(). + + For each subsequent task, first disconnect the current debugger session, then click the extension icon again to attach to the next breakpoint. + + .. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/disconnect.png?raw=true + :alt: Disconnecting and reconnecting the debugger + +Legacy Ray Debugger +~~~~~~~~~~~~~~~~~~~ +1. Ray has a builtin legacy `debugger `_ that allows you to debug your distributed applications. To enable debugger, start ray cluster with ``RAY_DEBUG=legacy`` and ``--ray-debugger-external``. + +.. code-block:: bash + + # start head node + RAY_DEBUG=legacy ray start --head --dashboard-host=0.0.0.0 --ray-debugger-external + # start worker node + RAY_DEBUG=legacy ray start --address='10.124.46.192:6379' --ray-debugger-external + +2. Set up breakpoint in your code, and submit job to cluster. Then run ``ray debug`` to wait breakpoint: + +.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/legacy.png?raw=true + diff --git a/e2c/verl/docs/workers/fsdp_workers.rst b/e2c/verl/docs/workers/fsdp_workers.rst new file mode 100644 index 0000000000000000000000000000000000000000..b158fb265dff21c81d665e95655727fbf74ac44e --- /dev/null +++ b/e2c/verl/docs/workers/fsdp_workers.rst @@ -0,0 +1,144 @@ +PyTorch FSDP Backend +====================== + +Last updated: 02/12/2025. + +We support PyTorch FSDP Backend by implementing various workers for +actor, critic, reference, rollout and reward models. We also implement +the ``FSDPVLLMShardingManager`` that reshard weight between FSDP and +vLLM in `fsdp_vllm.py `_. + +**Pros** + +- Readily support various models. + + - Users only need to implement the corresponding + ``dtensor_weight_loader`` for weight synchronization between FSDP + and vLLM. While for ``hf_weight_loader``, users can directly apply + any models supported both in HF and vLLM without any code change. + +- Easy to organize the forward and backward computation for each model. + +**Cons** + +- Poor scalability when it comes to large-scale models (e.g. Llama 70B + and 405B) +- The resharding overhead between actor and rollout could be larger than + Megatron-LM backend. + +Due to the simplicity, we recommend using FSDP backend for algorithm +research and prototyping. + +FSDP Workers +-------------- + +ActorRolloutRefWorker +^^^^^^^^^^^^^^^^^^^^^ + +Actor/Rollout HybridEngine +'''''''''''''''''''''''''' + +1. HybridEngine, Actor and Rollout initialization API. + +.. code:: python + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + +``ONE_TO_ALL``: when calling the ``init_model`` function from the driver +process, each worker (on a GPU) will execute the following model +initialization process. + +The initialization details of HybridEngine, Actor and Rollout are +highlighted below: + +1. ``DataParallelPPOActor`` implements the simple PPO computation logics + when the model is built with FSDP, including compute log prob, model + update. +2. ``vLLMRollout`` support generation with vLLM. We modify the vLLM + Engine and make it executed under SPMD to fit into our + ``WorkerGroup`` design. +3. ``FSDPVLLMShardingManager`` a context manager to perform actual + resharding between actor and rollout. + +See `source code `_. for more information. + +1. Generate sequence and recompute log prob + +.. code:: python + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def generate_sequences(self, prompts: DataProto): + +- ``Dispatch.DP_COMPUTE_PROTO``: The data will be dispatched and + collected along the DP dimension + +- In this function, the rollout model will perform auto-regressive + generation and the actor model will recompute the old log prob for the + generated response. + +3. Update actor model + +.. code:: python + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def update_actor(self, data: DataProto): + +- Update the actor model weight using PPO & entropy loss. + +ReferenceModel +'''''''''''''' + +1. Reference model initialization + +The reference model is initialized using the same function as the actor +model without initializing the HybridEngine and Optimizer. Then the +actor model is also wrapped by the ``DataParallelPPOActor``. + +2. Compute reference log prob + +.. code:: python + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def compute_ref_log_prob(self, data: DataProto): + +- In this function, the reference model will call the compute log prob + function in ``DataParallelPPOActor`` to compute the reference log + prob. + +CriticWorker and RewardWorker +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Model initialization + +Quite similar to reference model. The CriticWorker will perform +additional initialization for the Optimizer. + +2. Compute Values for CriticWorker + +.. code:: python + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def compute_values(self, data: DataProto): + +3. Update Critic + +.. code:: python + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def update_critic(self, data: DataProto): + +4. Compute Reward + +.. code:: python + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def compute_rm_score(self, data: DataProto): + + +HybridShard +------------ + +We didn't support FSDP `HybridShard`. To support this, we may need to +construct a 2D device mesh and test the corresponding +``dtensor_weight_loader`` and ``hf_weight_loader`` for each model. diff --git a/e2c/verl/docs/workers/megatron_workers.rst b/e2c/verl/docs/workers/megatron_workers.rst new file mode 100644 index 0000000000000000000000000000000000000000..bd02836a6240beec576e85e64f08050e5f517fa9 --- /dev/null +++ b/e2c/verl/docs/workers/megatron_workers.rst @@ -0,0 +1,283 @@ +Megatron-LM Backend +=================== + +Last updated: 06/24/2025. + +We support Megatron Backend by implementing various workers for actor, +critic, reference, rollout and reward models. We also implement the +``3DHybridEngine`` using Megatron-LM and vLLM/SGLang in +`megatron_vllm.py `_ +and `megatron_sglang.py `_. + +**Pros** + +- Support 5D parallelism (TP, EP, CP, DP, PP) and sequence parallelism + for best scalablility and throughput. +- 3D HybridEngine can significantly reduce peak memory usage and reduce + weight synchronize overhead between actor and rollout. + +**Cons** + +- Huggingface Models and Megatron checkpoints need tools for conversion. + + +Development Progress +-------------------- + + +Note that [Deprecated] means that the feature is not supported in the latest +version of verl. +[To-Optimize] means that the feature is implemented but not optimized yet. +[WIP] means that the feature is working in progress. +[In-Release] means that the feature is ready and in review process, +coming at any time. + + ++---------------+-----------------------------------------------------------+ +| [Deprecated] | Megatron 3D Parallelism with custom models | ++---------------+-----------------------------------------------------------+ +| [Done] | Megatron 0.11.0 ``GPTModel`` support | ++---------------+-----------------------------------------------------------+ +| [Done] | Megatron GRPO support | ++---------------+-----------------------------------------------------------+ +| [Done] | Megatron with vLLM 0.8.2, with per-tensor weights loading | ++---------------+-----------------------------------------------------------+ +| [Done] | Megatron with Context Parallel | ++---------------+-----------------------------------------------------------+ +| [Done] | Qwen2MoE model support | ++---------------+-----------------------------------------------------------+ +| [To-Optimize] | Megatron dist Checkpoint | ++---------------+-----------------------------------------------------------+ +| [To-Optimize] | Huggingface and Megatron Checkpoint Converter | ++---------------+-----------------------------------------------------------+ +| [To-Optimize] | Efficient fused linear, entropy and cross entropy | ++---------------+-----------------------------------------------------------+ +| [Done] | Megatron offload(param, grad, optimizer) | ++---------------+-----------------------------------------------------------+ +| [Done] | Megatron Profiler | ++---------------+-----------------------------------------------------------+ +| [In-Release] | Megatron 0.12.0, TE 2.2 with vLLM 0.8.3 and Fused Attn | ++---------------+-----------------------------------------------------------+ +| [WIP] | Moonlight/DeepSeek-V3 model support | ++---------------+-----------------------------------------------------------+ +| [WIP] | Expert Parallel support | ++---------------+-----------------------------------------------------------+ +| [WIP] | Megatron support dynamic batch size | ++---------------+-----------------------------------------------------------+ +| [To-Do] | Performance tuning | ++---------------+-----------------------------------------------------------+ +| [MileStone] | Runnable with DeepSeek-V3 671B post-training | ++---------------+-----------------------------------------------------------+ + + + +Utils of Megatron Workers +------------------------- + +MegatronWorker +^^^^^^^^^^^^^^ + +``MegatronWorker`` is the base class of different megatron worker +classes. In this class, ``get_megatron_global_info`` and +``get_megatron_rank_info`` function to retrieve the 3D parallel world +size and rank of each ``Worker`` running on specific GPU. These information +will be used in transfer protocol for Megatron Backend. + +The following ``Worker`` class for different models will be utilized to +construct the ``WorkerGroup`` . + +We implement various of APIs for each ``Worker`` class decorated by the +``@register(dispatch_mode=)`` . These APIs can be called by the ray +driver process. The data can be correctly collect and dispatch following +the ``dispatch_mode`` on each function. The supported dispatch_model +(i.e., transfer protocols) can be found in `decorator.py `_. + +ActorRolloutRefWorker +^^^^^^^^^^^^^^^^^^^^^ + +This class is implemented for Actor/Rollout HybridEngine or for the +reference model to initialize their model and perform computation. + +Actor/Rollout HybridEngine +'''''''''''''''''''''''''' + +1. HybridEngine, Actor and Rollout initialization API. + +.. code:: python + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + +``ONE_TO_ALL``: when calling the ``init_model`` function from the driver +process, each worker (on a GPU) will execute the following model +initialization process. + +The initialization details of HybridEngine, Actor and Rollout are +highlighted below: + +1. ``MegatronPPOActor`` implements the simple PPO computation logics + when the model is built with Megatron, including compute log prob, + model update. +2. ``vLLMRollout`` support generation with vLLM. We modify the vLLM + Engine and make it executed under SPMD to fit into our + ``WorkerGroup`` design. +3. ``MegatronVLLMShardingManager`` a context manager to perform actual + resharding between actor and rollout. + +See `source code `_ for more information. + +.. code:: python + + # build actor model + self.actor = MegatronPPOActor(config=self.config.actor, + model_config=self.actor_model_config, + megatron_config=megatron_config, + actor_module=self.actor_module, + actor_optimizer=self.actor_optimizer, + actor_optimizer_config=self.actor_optim_config) + + # build rollout + # rollout initialization + rollout = vLLMRollout(actor_module=params, + config=self.config.rollout, + tokenizer=self.tokenizer, + model_hf_config=self.actor_model_config, + train_tp=mpu.get_tensor_model_parallel_world_size()) + # perform weight resharding between actor and rollout + sharding_manager = MegatronVLLMShardingManager(module=self.hybrid_engine, + inference_engine=rollout.inference_engine, + model_config=self.actor_model_config, + layer_name_mapping=layer_name_mapping) + ... + +1. Generate sequence and recompute log prob + +.. code:: python + + @register(dispatch_mode=Dispatch.MEGATRON_PP_AS_DP_PROTO) + def generate_sequences(self, prompts: DataProto): + +- ``Dispatch.MEGATRON_PP_AS_DP_PROTO``: The PP dimension of the actor + model will be regarded as DP dimension. Then the driver process will + dispatch and collect the data according to this reorganization. This + is because, in HybridEngine, the actor weight, which usually applied + larger 3D parallel sizes, will be gathered along the PP dimension and + TP dimension. Therefore, the corresponding data should be dispatched + and collected through the 3D parallel group of the rollout model, + rather than the actor model. However, the world_size and rank + information can only be retrieved from ``get_megatron_global_info`` and + ``get_megatron_rank_info``, which records the 3D information for the + actor model. Moreover, the data resharding inside TP dimension will be + processed within the HybridEngine. + +- In this function, the rollout model will perform auto-regressive + generation and the actor model will recompute the old log prob for the + generated response. + +3. Update actor model + +.. code:: python + + @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) + def update_actor(self, data: DataProto): + +- ``Dispatch.MEGATRON_COMPUTE_PROTO``: User passes the data partitioned + by DP dimension. The data is dispatched to all tp/pp ranks within the + same dp group, and ultimately only collects output data from tp=0 and + the last pp. +- Update the actor model weight using PPO & entropy loss. + + +..note:: + + Currently, training Tensor Parallel Size can be different from inference + Tensor Parallel Size. + + +ReferenceModel +'''''''''''''' + +1. Reference model initialization + +The reference model is initialized using the same function as the actor +model without initializing the HybridEngine and Optimizer. Then the +actor model is also wrapped by the ``MegatronPPOActor``. + +2. Compute reference log prob + +.. code:: python + + @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) + def compute_ref_log_prob(self, data: DataProto): + +- In this function, the reference model will call the compute log prob + function in ``MegatronPPOActor`` to compute the reference log prob. + +CriticWorker and RewardWorker +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Model initialization + +Quite similar to reference model. The CriticWorker will perform +additional initialization for the Optimizer. + +2. Compute Values for CriticWorker + +.. code:: python + + @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) + def compute_values(self, data: DataProto): + +3. Update Critic + +.. code:: python + + @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) + def update_critic(self, data: DataProto): + +4. Compute Reward + +.. code:: python + + @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) + def compute_rm_score(self, data: DataProto): + + +Utils of Train Optimization +--------------------------- + +Offload +^^^^^^^ +When resources are tight, the offload method can lower GPU memory +usage, helping training and inference frameworks work well under verl. +It moves parameters, gradients, and optimizers to CPU memory and only +loads them back to the GPU when needed. + +If you want to use the offload, you can add the following parameters +for the actor and ref separately. + +.. code:: python + + # For the actor + actor_rollout_ref.actor.megatron.param_offload=True \ + actor_rollout_ref.actor.megatron.grad_offload=True \ + actor_rollout_ref.actor.megatron.optimizer_offload=True \ + # For the ref w/o grad and optimizer + actor_rollout_ref.ref.megatron.param_offload=True \ + + +For the critic, you can include these parameters. + +.. code:: python + + # For the critic + critic.megatron.param_offload=True \ + critic.megatron.grad_offload=True \ + critic.megatron.optimizer_offload=True \ + + +Related MCore Document +---------------------- + +There is also a detailed document of using MCore to train different +kinds of models, please refer to `MCore Document `_. diff --git a/e2c/verl/docs/workers/model_engine.rst b/e2c/verl/docs/workers/model_engine.rst new file mode 100644 index 0000000000000000000000000000000000000000..6642242bc3cde037ace437927fcf5da1dadb7b3e --- /dev/null +++ b/e2c/verl/docs/workers/model_engine.rst @@ -0,0 +1,125 @@ +Model Engine +============ + +.. _vermouth: https://github.com/vermouth1992 + +Author: `Chi Zhang `_ + +Last updated: 09/25/2025. + +Current Support Matrix +---------------------- + ++----------+-----------+--------------+-------------+--------------------------+ +| Backends | Model | Scalability | Model | Pain points | +| | Supported | | Definition | | +| | | | | | ++==========+===========+==============+=============+==========================+ +| FSDP | Day 1 | - Dense is OK| Huggingface | Monkey patch can be | +| + | support | | + monkey | easily impacted by | +| ulysses | HF model | - MoE is bad | patch | transformers version | ++----------+-----------+--------------+-------------+--------------------------+ +| MCore | Limited | Best | GPTModel | Supporting new models is | +| | | | (One model | difficult | +| | | | for all) | | ++----------+-----------+--------------+-------------+--------------------------+ + +- We monkey patch attention function to support ulysses +- We monkey patch VLM models to support FSDP with mixed data with and + without images + +Class Hierarchy +--------------- + +Note that all the workers and trainers run in **SPMD** mode. SFT/DPO/RM +trainer is directly invoked by ``torchrun``. The Actor/Critic worker can +also be invoked by a RayWorkerGroup and provides APIs to a single +controller. + +- Base Engine level: implement model init, optimizer init, lr scheduler + init, sharding, checkpoint manager. +- Full Engine level: subclass base engine and implement + ``forward_step``. +- Worker/SPMD trainer level: **engine agnostic**, implement training + logics using abstract engine APIs + +RL trainer utilizes workers to construct HybridFlow program. This is out +of the scope of model engine. + +Existing Model Types +-------------------- + +========== ====================== ====================== +Model type Language model Value model +========== ====================== ====================== +Input text/image/video/audio text/image/video/audio +Output logits for next token logits as value +========== ====================== ====================== + +Currently, we have two model types: language model and value model. We +expect to expand the category to include Qwen-Omni family (output both +text and audio) and VLA models. + +Data Format +----------- + +Currently, verl adopts left-right padding data format in RL trainer. +This creates massive padding when the discrepancy between response +length is large. We will start to implement no-padding format throughout +the whole system. + +.. image:: https://github.com/vermouth1992/verl-data/blob/master/images/data_format.png?raw=true + :alt: Data Format + +Here is the migration plan: +- Implement no-padding format in engine +- Add a transformation layer in Actor/Critic worker. +- Replace Actor/Critic Worker in RL trainer +- Implement no-padding throughput system + +Checkpoint System +----------------- + +.. image:: https://github.com/vermouth1992/verl-data/blob/master/images/verl-ckpt.png?raw=true + :alt: Model Engine Checkpoint System + +The engine constructs the model using huggingface config, then load +weights from huggingface checkpoint. If the engine directly uses +huggingface model definition, it can use function provided by +``transformers``. Otherwise, each engine has to write their own +checkpoint load logic (e.g., +`mbridge `__). During model +training, each engine has to implement save_checkpoint and +load_checkpoint that save/load intermediate sharded checkpoint including +model, optimizer and lr scheduler states. Each engine has to implement a +checkpoint merge script, that merges the intermediate sharded checkpoint +back to huggingface format. + +API +--- + +A tentative model engine API can be found: +https://github.com/volcengine/verl/blob/main/verl/workers/engine/base.py#L24 + +Extension +--------- + +Add a new backend +~~~~~~~~~~~~~~~~~ + +- Start a new folder under ``verl/workers/engine``. Then, implement + ``transformer_impl.py``. If you want to implement a non-transformer + model, please contact us in advance. +- Add the engine config to the GSM8k SFT trainer script: + https://github.com/volcengine/verl/blob/main/tests/special_e2e/sft/run_sft_engine_gsm8k.sh +- Invoke the tests with your backend: + https://github.com/volcengine/verl/blob/main/tests/special_e2e/sft/test_sft_engine_all.sh. + This test script will run various backends and various + configurations, and compare the loss and grad norm of the first step + to make sure they are close. + +Add a new model type +~~~~~~~~~~~~~~~~~~~~ + +- This is mainly reserved for models whose the output is not just text + (e.g., Qwen3-Omni). Please discuss with us before you proceed. diff --git a/e2c/verl/docs/workers/ray_trainer.rst b/e2c/verl/docs/workers/ray_trainer.rst new file mode 100644 index 0000000000000000000000000000000000000000..9c482d39a4223ca292029325db3d064a417c9ba1 --- /dev/null +++ b/e2c/verl/docs/workers/ray_trainer.rst @@ -0,0 +1,241 @@ +PPO Ray Trainer +=============== + +Last updated: 02/12/2025. + +We implement the RayPPOTrainer, which is a trainer runs on the driver +process on a single CPU/GPU node (default is CPU). + +The PPORayTrainer include 3 core functions for data preparation, +WorkerGroup initialization and PPO training loop. + +Data Preparation +---------------- + +The ``PPORayTrainer``, as a single process, is responsible for loading a +complete batch of samples (prompts) from the dataset and then dispatch +to different worker_groups running on different GPUs. + +To generalize the data loading, we implement the ``RLHFDataset`` class +to load the preprocessed parquet files, apply chat templates to the +prompts, add padding, truncate prompts that exceed max prompt length and +then tokenize. + +.. code:: python + + self.train_dataset = RLHFDataset(data_files=self.config.data.train_files, + tokenizer=self.tokenizer, + config=self.config.data) + +Then, the dataloader will iterate the dataset under PPO mini batch size. + +WorkerGroup Initialization +-------------------------- + +We first introduce a basic implementation of initializing the +``WorkerGroup`` of the actor model on a given set of GPUs. + +.. code:: python + + # max_colocate_count means the number of WorkerGroups (i.e. processes) in each RayResourcePool + # For FSDP backend, we recommend using max_colocate_count=1 that merge all WorkerGroups into one. + # For Megatron backend, we recommend using max_colocate_count>1 that can utilize different WorkerGroup for differnt models + resource_pool = RayResourcePool(process_on_nodes=[config.trainer.n_gpus_per_node] * config.trainer.nnodes, + use_gpu=True, + max_colocate_count=1) + # define actor rollout cls to be init on remote + actor_rollout_cls = RayClassWithInitArgs(cls=ActorRolloutWorker) + # define actor_rollout worker group + actor_rollout_worker_group = MegatronRayWorkerGroup(resource_pool=resource_pool, + ray_cls_with_init=actor_rollout_cls, + default_megatron_kwargs=config.actor_rollout.megatron) + +Different WorkerGroups, like ``actor_rollout_worker_group`` , +``critic_worker_group`` and ``ref_worker_group`` lies on a separate +process in the above implementation. + +The driver process can then call the distributed compute function within +the ``actor_rollout_worker_group`` and other roles to construct the RL +training loop. + +For models colocated in the same set of GPUs, we further provide a +fine-grain optimization, which merge the ``worker_group`` of different roles +in the same process. This optimization can save the redundant +CUDA/distributed context in different processes. + +.. code:: python + + # initialize WorkerGroup + # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, + # you should not use `create_colocated_worker_cls`. Instead, directly pass different resource pool to different worker groups. + # See TODO(url) for more information. + all_wg = {} + for resource_pool, class_dict in self.resource_pool_to_cls.items(): + worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + wg_dict = self.ray_worker_group_cls(resource_pool=resource_pool, ray_cls_with_init=worker_dict_cls) + spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + all_wg.update(spawn_wg) + + if self.use_critic: + self.critic_wg = all_wg['critic'] + self.critic_wg.init_model() + + if self.use_reference_policy: + self.ref_policy_wg = all_wg['ref'] + self.ref_policy_wg.init_model() + + if self.use_rm: + self.rm_wg = all_wg['rm'] + self.rm_wg.init_model() + + # we should create rollout at the end so that vllm can have a better estimation of kv cache memory + self.actor_rollout_wg = all_wg['actor_rollout'] + self.actor_rollout_wg.init_model() + +.. note:: For megatron backend, if we merge the ``worker_groups`` into the same processes, all the roles will utilize the same 3D parallel size. To optimize this, we may need to maintain several 3D process groups for each role in the same distributed context. If you want to use different 3D parallel size for different roles, please follow the similar architecture of the first code block to initialize each role's ``worker_group`` + + +PPO Training Loop +----------------- + +We implement the PPO training loop by calling the functions in +worker_group of each role. The input and output data of each function is +a ``DataProto`` object implemented in `protocol.py `_. In the training +loop, trainer will dispatch/collect the data to/from different GPUs +following the transfer protocols wrapped in the workers' functions. The +computation of PPO micro batches is processed in ``update_actor`` and +``update_critic`` functions. + +To extend to other RLHF algorithms, such as DPO, GRPO, please refer to +:doc:`../advance/dpo_extension`. + +.. code:: python + + def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the worker group through RPC to construct the PPO dataflow. + The light-weight advantage computation is done on the driver process. + """ + from verl.utils.tracking import Tracking + from omegaconf import OmegaConf + + logger = Tracking(project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True)) + + global_steps = 0 + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None: + val_metrics = self._validate() + pprint(f'Initial validation metrics: {val_metrics}') + + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + metrics = {} + + batch: DataProto = DataProto.from_single_dict(batch_dict) + # batch = batch.to('cuda') + + # pop those keys for generation + gen_batch = batch.pop(batch_keys=['input_ids', 'attention_mask', 'position_ids']) + + # generate a batch + with Timer(name='gen', logger=None) as timer: + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + metrics['timing/gen'] = timer.last + + batch = batch.union(gen_batch_output) + + if self.use_reference_policy: + # compute reference log_prob + with Timer(name='ref', logger=None) as timer: + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + metrics['timing/ref'] = timer.last + + # compute values + with Timer(name='values', logger=None) as timer: + values = self.critic_wg.compute_values(batch) + batch = batch.union(values) + metrics['timing/values'] = timer.last + + with Timer(name='adv', logger=None) as timer: + # compute scores. Support both model and function-based. + # We first compute the scores using reward model. Then, we call reward_fn to combine + # the results from reward model and rule-based results. + if self.use_rm: + # we first compute reward model score + reward_tensor = self.rm_wg.compute_rm_score(batch) + batch = batch.union(reward_tensor) + + # we combine with rule-based rm + reward_tensor = self.reward_fn(batch) + batch.batch['token_level_scores'] = reward_tensor + + # compute rewards. apply_kl_penalty if available + batch, kl_metrics = apply_kl_penalty(batch, + kl_ctrl=self.kl_ctrl_in_reward, + kl_penalty=self.config.algorithm.kl_penalty) + metrics.update(kl_metrics) + + # compute advantages, executed on the driver process + batch = compute_advantage(batch, + self.config.algorithm.gamma, + self.config.algorithm.lam, + adv_estimator=self.config.algorithm.adv_estimator) + metrics['timing/adv'] = timer.last + + # update critic + if self.use_critic: + with Timer(name='update_critic', logger=None) as timer: + critic_output = self.critic_wg.update_critic(batch) + metrics['timing/update_critic'] = timer.last + critic_output_metrics = reduce_metrics(critic_output.meta_info['metrics']) + metrics.update(critic_output_metrics) + + # implement critic warmup + if self.config.trainer.critic_warmup <= global_steps: + # update actor + with Timer(name='update_actor', logger=None) as timer: + actor_output = self.actor_rollout_wg.update_actor(batch) + metrics['timing/update_actor'] = timer.last + actor_output_metrics = reduce_metrics(actor_output.meta_info['metrics']) + metrics.update(actor_output_metrics) + + # validate + if self.val_reward_fn is not None and (global_steps + 1) % self.config.trainer.test_freq == 0: + with Timer(name='testing', logger=None) as timer: + val_metrics: dict = self._validate() + val_metrics = {f'val/{key}': val for key, val in val_metrics.items()} + metrics['timing/testing'] = timer.last + metrics.update(val_metrics) + + # collect metrics + data_metrics = compute_data_metrics(batch=batch) + metrics.update(data_metrics) + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=global_steps) + + if self.config.trainer.save_freq > 0 and (global_steps + 1) % self.config.trainer.save_freq == 0: + actor_local_path = os.path.join(self.config.trainer.default_local_dir, 'actor', + f'global_step_{global_steps}') + actor_remote_path = os.path.join(self.config.trainer.default_hdfs_dir, 'actor') + self.actor_rollout_wg.save_checkpoint(actor_local_path, actor_remote_path) + + if self.use_critic: + critic_local_path = os.path.join(self.config.trainer.default_local_dir, 'critic', + f'global_step_{global_steps}') + critic_remote_path = os.path.join(self.config.trainer.default_hdfs_dir, 'critic') + self.critic_wg.save_checkpoint(critic_local_path, critic_remote_path) + + global_steps += 1 + + # perform validation after training + if self.val_reward_fn is not None: + val_metrics = self._validate() + pprint(f'Final validation metrics: {val_metrics}') diff --git a/e2c/verl/docs/workers/sglang_worker.rst b/e2c/verl/docs/workers/sglang_worker.rst new file mode 100644 index 0000000000000000000000000000000000000000..08cc48a075d3f3a2abc131e881f186c0f0df8fed --- /dev/null +++ b/e2c/verl/docs/workers/sglang_worker.rst @@ -0,0 +1,237 @@ +SGLang Backend +============== + +Last updated: 05/31/2025. + +**Authored By SGLang RL Team and listed alphabetically by last name** + +`Jingyi Chen `_, `Yitong Guan `_, `Zhuobin Huang `_, `Jiajun Li `_, `Ji Li `_, `Shenggui Li `_, `Junrong Lin `_, `Xiang Long `_, `Rui Lu `_, `Jin Pan `_, `Shuai Shi `_, `Yushen Su `_, `Xinyuan Tong `_, `Chendong Wang `_, `Hanchen Zhang `_, `Haoran Wang `_, `Yongan Xiang `_, `Chengxing Xie `_, `Yuhao Yang `_, `Jinwei Yao `_, `Qiaolin Yu `_, `Yuzhen Zhou `_, `Chenyang Zhao `_ + + + +Introduction +------------ +`SGLang `_ is an open-source state-of-the-art inference service engine, fully adopted by xAI to support all inference needs of Grok during research and serving processes. + +Currently, verl fully supports using SGLang as the inference engine during the rollout phase. As a rollout engine, SGLang provides the same feature coverage as vLLM., including memory saving and multi-node rollout features. After installing verl and SGLang, simply add ``actor_rollout_ref.rollout.name=sglang`` at startup script to seamlessly switch between the two inference frameworks. + +In addition, the SGLang team is actively working on supporting features such as Multi-Turn Agentic RL, VLM RLHF, Server-Based RLHF, and Partial Rollout. You can track the related development progress in the `Tracking Roadmap `_. + +Installation +------------ +Please always follow the following command to install SGLang with verl. + +.. code-block:: bash + + pip install --upgrade pip + # Currently 0.4.8, subject to updates at any time, please refer to the latest version specified in `setup.py` + pip install -e ".[sglang]" + +You can check the following dependencies are in your environment: + +.. note:: + + - **PyTorch**: 2.6.0+cu124 + - **CUDA**: 12.4 + - **flashinfer-python**: 0.2.5+cu124torch2.6 + - **SGLang**: 0.4.6.post5 + - **sgl-kernel**: 0.1.4 + +Using SGLang as the Inference Backend for PPO Training on a Single Machine +------------------------------------------------------------------------- +We use Qwen/Qwen2-7B-Instruct on the gsm8k dataset for a simple test. + +1. Run the following command to prepare the gsm8k dataset: + +.. code-block:: bash + + python3 examples/data_preprocess/gsm8k.py + +2. Run the following script to conduct a PPO experiment on a single machine with 4 GPUs: + +.. code-block:: bash + + export SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK=True + PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=4096 \ + data.max_prompt_length=4096 \ + data.max_response_length=4096 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2-7B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + critic.model.fsdp_config.param_offload=True \ + critic.model.fsdp_config.optimizer_offload=True \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=console \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 2>&1 | tee verl_demo.log + +Why export SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. ``verl`` initializes a ``SGLangRollout`` module during rollout, which is used to evaluate/generate samples. + +2. ``SGLangRollout`` will initialize ``Engine``, and further initialize a ``torch.distributed.DeviceMesh``, used to support Tensor Parallel (TP). + +3. ``DeviceMesh.init()`` internally checks the free GPU memory of all participating devices. If the difference is too large (more than ~10%), it directly reports an error to avoid initialization failures or deadlocks. + +Why might there be inconsistent GPU memory? +""""""""""""""""""""""""""""""""""""""""""" + +**1. Ray Distributed Actor loads the model at different times** + +``verl`` uses Ray-based multi-process, multi-GPU concurrent training. Each ``WorkerDict`` may be called at different times: + +.. code-block:: python + + self.rollout = SGLangRollout(...) + +Different workers initialize the model at different times → different memory usage. + +**2. Delayed initialization causes memory bias** + +Some workers start model loading/inference (e.g., ``generate_sequences()``, ``compute_log_prob()``) earlier than others. +Early workers already use up GPU memory → late workers still have empty memory → memory difference appears. + +**3. SGLang's TP init uses "all-device broadcast", but there's no uniform release timing** + +Although ``SGLangRollout`` may only involve subset of GPUs, its ``Engine`` initialization calls ``torch.distributed.init_process_group()`` and broadcasts weights, so: + +- Non-rollout GPUs also join the communication. +- Later on, ``DeviceMesh`` init will fail due to "inconsistent memory". + +**4. Different FSDP/TP loading behaviors also lead to mismatch** + +If using: + +.. code-block:: bash + + actor.fsdp_config.param_offload=True + ref.fsdp_config.param_offload=True + +Then some workers keep params on CPU while others already sharded to GPU → leads to asymmetric memory layout. + +Using SGLang as the Inference Backend for PPO Training Across Multiple Machines +------------------------------------------------------------------------------ +SGLang also supports running verl's RAY-based cross-machine inference in IPv4 and IPv6 scenarios. In the script below, we use TP=16 for cross-machine inference. Suppose we have two interconnected machines: node0 with IP 10.94.16.4 and node1 with IP 10.94.16.5. + +1. Start Ray on node0: + +.. code-block:: bash + + ray start --head --dashboard-host=0.0.0.0 + +You will see the following prompt: + +.. code-block:: bash + + Usage stats collection is enabled. To disable this, add `--disable-usage-stats` to the command that starts the cluster, or run the following command: `ray disable-usage-stats` before starting the cluster. See https://docs.ray.io/en/master/cluster/usage-stats.html for more details. + + Local node IP: 10.94.16.4 + + -------------------- + Ray runtime started. + -------------------- + + Next steps + To add another node to this Ray cluster, run + ray start --address='10.94.16.4:6379' + +2. Have node1 join the Ray cluster: + +Run the following command on node1: + +.. code-block:: bash + + ray start --address='10.94.16.4:6379' + +Run the following command to confirm that the Ray cluster now has two nodes: + +.. code-block:: bash + + ray status + +You can see that the cluster has two nodes with 16 GPUs: + +.. code-block:: bash + + ======== Autoscaler status: 2025-04-09 09:25:37.694016 ======== + Node status + --------------------------------------------------------------- + Active: + 1 node_ef382ffd687d8f6b060c1b68e63ada7341b936fe5b1901dd04de1027 + 1 node_1eb4d7d07e793114c23a89d1a41f1f76acf6ef5b35af844a4ee8e4ba + Pending: + (no pending nodes) + Recent failures: + (no failures) + + Resources + --------------------------------------------------------------- + Usage: + 0.0/360.0 CPU + 0.0/16.0 GPU + 0B/3.39TiB memory + 0B/372.53GiB object_store_memory + +3. Run the following script to train meta-llama/Llama-3.1-8B-Instruct with TP=16 across 2 machines using 16 GPUs: + +.. code-block:: bash + + DATA_DIR=$HOME/data/gsm8k + + python3 -m verl.trainer.main_ppo \ + actor_rollout_ref.rollout.name=sglang \ + data.train_files=$DATA_DIR/train.parquet \ + data.val_files=$DATA_DIR/test.parquet \ + data.train_batch_size=4096 \ + data.max_prompt_length=4096 \ + data.max_response_length=4096 \ + actor_rollout_ref.model.path=meta-llama/Llama-3.1-8B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=16 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=meta-llama/Llama-3.1-8B-Instruct \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size=16 \ + critic.model.fsdp_config.param_offload=True \ + critic.model.fsdp_config.optimizer_offload=True \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.val_before_train=True \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=2 \ + trainer.save_freq=-1 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 2>&1 | tee verl_demo.log diff --git a/e2c/verl/examples/data_preprocess/aime2024_multiturn_w_tool.py b/e2c/verl/examples/data_preprocess/aime2024_multiturn_w_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..76cdd0576d3801118b160b850bcbd8d2fe6723b1 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/aime2024_multiturn_w_tool.py @@ -0,0 +1,79 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the DAPO-Math-17k dataset to multiturn format +""" + +import argparse +import os + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/retool_aime2024", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_path = "BytedTsinghua-SIA/AIME-2024" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path, "default") + else: + dataset = datasets.load_dataset(data_path, "default") + + train_dataset = dataset["train"] + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + orig_extra_info = example.pop("extra_info") + extra_info = orig_extra_info.copy() + extra_info["need_tools_kwargs"] = True + extra_info["tools_kwargs"] = { + "code_interpreter": { + "create_kwargs": { + "ground_truth": example["reward_model"]["ground_truth"], + }, + }, + } + example["extra_info"] = extra_info + return example + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/dapo_multiturn_w_tool.py b/e2c/verl/examples/data_preprocess/dapo_multiturn_w_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..aab356f41bf38e789a31f1ee879ce9beb8b0aa40 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/dapo_multiturn_w_tool.py @@ -0,0 +1,79 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the DAPO-Math-17k dataset to multiturn format +""" + +import argparse +import os + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/retool_dapo", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_path = "BytedTsinghua-SIA/DAPO-Math-17k" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path, "default") + else: + dataset = datasets.load_dataset(data_path, "default") + + train_dataset = dataset["train"] + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + orig_extra_info = example.pop("extra_info") + extra_info = orig_extra_info.copy() + extra_info["need_tools_kwargs"] = True + extra_info["tools_kwargs"] = { + "code_interpreter": { + "create_kwargs": { + "ground_truth": example["reward_model"]["ground_truth"], + }, + }, + } + example["extra_info"] = extra_info + return example + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/full_hh_rlhf.py b/e2c/verl/examples/data_preprocess/full_hh_rlhf.py new file mode 100644 index 0000000000000000000000000000000000000000..4e8a148df1e322f476cedffe4eadc5ae6ee9b6f1 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/full_hh_rlhf.py @@ -0,0 +1,161 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +- Preprocess data and split the training set into 75% for training RM and 25% for validting RM. +- All the training data is used to train SFT and RL. +- Both chosen and rejected is used to train SFT +""" + +import argparse +import os + +import pandas as pd +from datasets import load_dataset +from tqdm.auto import tqdm + +from verl.utils.fs import copy, makedirs + + +def generate_sft_dataset(target_hdfs_path_dir, local_dir="~/data/full_hh_rlh/sft", local_dataset_path=None): + if local_dataset_path is not None: + dataset = load_dataset(local_dataset_path) + else: + dataset = load_dataset("Dahoas/full-hh-rlhf") + output = {"prompt": [], "response": []} + for data in tqdm(dataset["train"]): + # add chosen + output["prompt"].append(data["prompt"]) + output["response"].append(data["chosen"]) + + # add rejection + output["prompt"].append(data["prompt"]) + output["response"].append(data["rejected"]) + + df = pd.DataFrame(output) + + local_dir = os.path.expanduser(local_dir) + os.makedirs(local_dir, exist_ok=True) + + local_path = os.path.join(local_dir, "train.parquet") + + df.to_parquet(path=local_path) + + if target_hdfs_path_dir is not None: + hdfs_dir = target_hdfs_path_dir + "/" + "train.parquet" + makedirs(hdfs_dir) + + copy(local_path, hdfs_dir) + + +def generate_rm_dataset(target_hdfs_path_dir, local_dir="~/data/full_hh_rlh/rm", local_dataset_path=None): + if local_dataset_path is not None: + train_dataset = load_dataset(local_dataset_path, split="train[:75%]") + test_dataset = load_dataset(local_dataset_path, split="train[-25%:]") + else: + train_dataset = load_dataset("Dahoas/full-hh-rlhf", split="train[:75%]") + test_dataset = load_dataset("Dahoas/full-hh-rlhf", split="train[-25%:]") + + local_dir = os.path.expanduser(local_dir) + os.makedirs(local_dir, exist_ok=True) + + for dataset, name in zip([train_dataset, test_dataset], ["train", "test"], strict=True): + output = {"prompt": [], "chosen": [], "rejected": []} + for data in tqdm(dataset): + # add chosen + output["prompt"].append(data["prompt"]) + output["chosen"].append(data["chosen"]) + output["rejected"].append(data["rejected"]) + + df = pd.DataFrame(output) + + local_path = os.path.join(local_dir, name + ".parquet") + + df.to_parquet(path=local_path) + + if target_hdfs_path_dir is not None: + hdfs_dir = target_hdfs_path_dir + "/" + name + ".parquet" + makedirs(hdfs_dir) + + copy(local_path, hdfs_dir) + + +def generate_rl_dataset(target_hdfs_path_dir, local_dir="~/data/full_hh_rlhf/rl", local_dataset_path=None): + if local_dataset_path is not None: + dataset = load_dataset(local_dataset_path) + else: + dataset = load_dataset("Dahoas/full-hh-rlhf") + train_dataset = dataset["train"] + + data_source = "Dahoas/full-hh-rlhf" + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + prompt = example.pop("prompt") + response = example.pop("response") + + data = { + "data_source": data_source, + "prompt": [{"role": "user", "content": prompt}], + "ability": "alignment", + "reward_model": { + "style": "model", + "ground_truth": response, # should not be used + }, + "extra_info": {"split": split, "index": idx}, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + local_dir = os.path.expanduser(local_dir) + local_path = os.path.join(local_dir, "train.parquet") + train_dataset.to_parquet(local_path) + + if target_hdfs_path_dir is not None: + hdfs_dir = target_hdfs_path_dir + "/" + "train.parquet" + makedirs(hdfs_dir) + + copy(local_path, hdfs_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--split", type=str, choices=["sft", "rm", "rl"], required=True) + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", type=str, required=False, default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", + type=str, + default="~/data/full_hh_rlhf", + help="The save directory for the preprocessed dataset.", + ) + + args = parser.parse_args() + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + if args.split == "sft": + generate_sft_dataset(args.hdfs_dir, os.path.join(local_save_dir, args.split), args.local_dataset_path) + elif args.split == "rm": + generate_rm_dataset(args.hdfs_dir, os.path.join(local_save_dir, args.split), args.local_dataset_path) + elif args.split == "rl": + generate_rl_dataset(args.hdfs_dir, os.path.join(local_save_dir, args.split), args.local_dataset_path) + else: + raise NotImplementedError diff --git a/e2c/verl/examples/data_preprocess/geo3k.py b/e2c/verl/examples/data_preprocess/geo3k.py new file mode 100644 index 0000000000000000000000000000000000000000..ba84fd3fc440761a200d0fbdea1535bfe9889b45 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/geo3k.py @@ -0,0 +1,102 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the Geometry3k dataset to parquet format +""" + +import argparse +import os + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None) + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/geo3k", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_source = "hiyouga/geometry3k" + + if local_dataset_path is not None: + dataset = datasets.load_dataset( + local_dataset_path, + ) + else: + dataset = datasets.load_dataset( + data_source, + ) + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + instruction_following = ( + r"You FIRST think about the reasoning process as an internal monologue and then provide the final answer. " + r"The reasoning process MUST BE enclosed within tags. " + r"The final answer MUST BE put in \boxed{}." + ) + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + problem = example.pop("problem") + prompt = problem + " " + instruction_following + answer = example.pop("answer") + images = example.pop("images") + + data = { + "data_source": data_source, + "prompt": [ + { + "role": "user", + "content": prompt, + } + ], + "images": images, + "ability": "math", + "reward_model": {"style": "rule", "ground_truth": answer}, + "extra_info": { + "split": split, + "index": idx, + "answer": answer, + "question": problem, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True, num_proc=8) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True, num_proc=8) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_save_dir, "test.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/geo3k_multiturn_w_tool.py b/e2c/verl/examples/data_preprocess/geo3k_multiturn_w_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..53c7197f9d2d00ccf256aee57e2de1847a926725 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/geo3k_multiturn_w_tool.py @@ -0,0 +1,120 @@ +# Copyright 2023-2025 SGLang Team +# Copyright Amazon.com, Inc. or its affiliates. +# Copyright 2025 Reallm Labs Ltd. or its affiliates +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Preprocess the Geometry3k dataset to parquet format +""" + +import argparse +import os + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", + default="~/data/geo3k_multiturn_w_tool", + help="The save directory for the preprocessed dataset.", + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_source = "hiyouga/geometry3k" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path) + else: + dataset = datasets.load_dataset(data_source) + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + instruction_following = ( + r"You FIRST think about the reasoning process as an internal monologue and then provide the final answer. " + r"The reasoning process MUST BE enclosed within tags. " + r"The final answer MUST BE put in \boxed{}." + ) + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + problem = example.pop("problem") + prompt = problem + " " + instruction_following + answer = example.pop("answer") + images = example.pop("images") + data = { + "data_source": data_source, + "prompt": [ + { + "role": "system", + "content": ( + "You are a math expert. You are given a question and you need to solve it step by step. " + "Reasoning step by step before any tool call. " + "You should use the `calc_geo3k_reward` tool after step by step solving the question, " + "before generate final answer at least once and refine your answer if necessary. " + ), + }, + { + "role": "user", + "content": prompt, + }, + ], + "images": images, + "ability": "math", + "reward_model": {"style": "rule", "ground_truth": answer}, + "extra_info": { + "split": split, + "index": idx, + "answer": answer, + "question": problem, + "need_tools_kwargs": True, + "tools_kwargs": { + "calc_geo3k_reward": { + "create_kwargs": {"ground_truth": answer}, + # "execute_kwargs": {}, + # "calc_reward_kwargs": {}, + # "release_kwargs": {}, + }, + }, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True, num_proc=8) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True, num_proc=8) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_save_dir, "test.parquet")) + if hdfs_dir is not None: + makedirs(hdfs_dir) + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/gsm8k.py b/e2c/verl/examples/data_preprocess/gsm8k.py new file mode 100644 index 0000000000000000000000000000000000000000..1656cdbc896a8f14fc7e09705d36335f52165533 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/gsm8k.py @@ -0,0 +1,105 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the GSM8k dataset to parquet format +""" + +import argparse +import os +import re + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + + +def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split("#### ")[1].replace(",", "") + return final_solution + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/gsm8k", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_source = "openai/gsm8k" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path, "main") + else: + dataset = datasets.load_dataset(data_source, "main") + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + instruction_following = 'Let\'s think step by step and output the final answer after "####".' + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question_raw = example.pop("question") + + question = question_raw + " " + instruction_following + + answer_raw = example.pop("answer") + solution = extract_solution(answer_raw) + data = { + "data_source": data_source, + "prompt": [ + { + "role": "user", + "content": question, + } + ], + "ability": "math", + "reward_model": {"style": "rule", "ground_truth": solution}, + "extra_info": { + "split": split, + "index": idx, + "answer": answer_raw, + "question": question_raw, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_save_dir, "test.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/gsm8k_multiturn_sft.py b/e2c/verl/examples/data_preprocess/gsm8k_multiturn_sft.py new file mode 100644 index 0000000000000000000000000000000000000000..4589362f933aa95493fdd98ce965eb810180c98a --- /dev/null +++ b/e2c/verl/examples/data_preprocess/gsm8k_multiturn_sft.py @@ -0,0 +1,102 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the GSM8k dataset to parquet format +""" + +import argparse +import os +import re + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + + +def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split("#### ")[1].replace(",", "") + return final_solution + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/gsm8k_sft", help="The save directory for the preprocessed dataset." + ) + parser.add_argument("--hdfs_dir", default=None) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_source = "openai/gsm8k" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path, "main") + else: + dataset = datasets.load_dataset(data_source, "main") + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + instruction_following = 'Let\'s think step by step and output the final answer after "####".' + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question_raw = example.pop("question") + + question = question_raw + " " + instruction_following + + answer_raw = example.pop("answer") + data = { + "messages": [ + { + "role": "user", + "content": question, + }, + { + "role": "assistant", + "content": answer_raw, + }, + ], + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) + + hdfs_dir = args.hdfs_dir + + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + local_save_dir = os.path.expanduser(local_save_dir) + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_save_dir, "test.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/gsm8k_multiturn_w_interaction.py b/e2c/verl/examples/data_preprocess/gsm8k_multiturn_w_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..c06b325c3e8076c0caff1360920f86cdd2f33bd2 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/gsm8k_multiturn_w_interaction.py @@ -0,0 +1,119 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the GSM8k dataset to parquet format +""" + +import argparse +import os +import re + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + + +def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split("#### ")[1].replace(",", "") + return final_solution + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/gsm8k", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_source = "openai/gsm8k" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path, "main") + else: + dataset = datasets.load_dataset(data_source, "main") + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + instruction_following = "Let's think step by step and output the final answer after `####`." + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question_raw = example.pop("question") + + question = question_raw + " " + instruction_following + + answer_raw = example.pop("answer") + solution = extract_solution(answer_raw) + data = { + "data_source": data_source, + "prompt": [ + { + "role": "system", + "content": ( + "You are a math expert. You are given a question and you need to solve it step by step. " + "You should rethinking carefully if user point out your answer is wrong. " + "Put your final answer in the format of `#### `." + ), + }, + { + "role": "user", + "content": question, + }, + ], + "ability": "math", + "reward_model": {"style": "rule", "ground_truth": solution}, + "extra_info": { + "split": split, + "index": idx, + "answer": answer_raw, + "question": question_raw, + "interaction_kwargs": { + "name": "gsm8k", + "query": question, + "ground_truth": solution, + }, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_save_dir, "test.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/gsm8k_multiturn_w_tool.py b/e2c/verl/examples/data_preprocess/gsm8k_multiturn_w_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..083550ad7f160a5caac97d85ee33164b0437119d --- /dev/null +++ b/e2c/verl/examples/data_preprocess/gsm8k_multiturn_w_tool.py @@ -0,0 +1,129 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the GSM8k dataset to parquet format +""" + +import argparse +import os +import re + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + + +def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split("#### ")[1].replace(",", "") + return final_solution + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/gsm8k", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_source = "openai/gsm8k" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path, "main") + else: + dataset = datasets.load_dataset(data_source, "main") + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + instruction_following = "Let's think step by step and output the final answer after `####`." + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question_raw = example.pop("question") + + question = question_raw + " " + instruction_following + + answer_raw = example.pop("answer") + solution = extract_solution(answer_raw) + data = { + "data_source": data_source, + "prompt": [ + { + "role": "system", + "content": ( + "You are a math expert. You are given a question and you need to solve it step by step. " + "Reasoning step by step before any tool call. " + "You should use the `calc_gsm8k_reward` tool after step by step solving the question, " + "before generate final answer at least once and refine your answer if necessary. " + "Put your final answer in the format of `#### `." + ), + }, + { + "role": "user", + "content": question, + }, + ], + "ability": "math", + "reward_model": {"style": "rule", "ground_truth": solution}, + "extra_info": { + "split": split, + "index": idx, + "answer": answer_raw, + "question": question_raw, + "need_tools_kwargs": True, + "tools_kwargs": { + "calc_gsm8k_reward": { + "create_kwargs": {"ground_truth": solution}, + # "execute_kwargs": {}, + # "calc_reward_kwargs": {}, + # "release_kwargs": {}, + }, + }, + "interaction_kwargs": { + "query": question, + "ground_truth": solution, + }, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_save_dir, "test.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/gsm8k_tool_agent_loop.py b/e2c/verl/examples/data_preprocess/gsm8k_tool_agent_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..743d7c5f154b2fa4c5fcc0103a9311578b8298b9 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/gsm8k_tool_agent_loop.py @@ -0,0 +1,130 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the GSM8k dataset to parquet format +""" + +import argparse +import os +import re + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + + +def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split("#### ")[1].replace(",", "") + return final_solution + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/gsm8k", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_source = "openai/gsm8k" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path, "main") + else: + dataset = datasets.load_dataset(data_source, "main") + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + instruction_following = "Let's think step by step and output the final answer after `####`." + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question_raw = example.pop("question") + + question = question_raw + " " + instruction_following + + answer_raw = example.pop("answer") + solution = extract_solution(answer_raw) + data = { + "data_source": data_source, + "agent_name": "tool_agent", + "prompt": [ + { + "role": "system", + "content": ( + "You are a math expert. You are given a question and you need to solve it step by step. " + "Reasoning step by step before any tool call. " + "You should use the `calc_gsm8k_reward` tool after step by step solving the question, " + "before generate final answer at least once and refine your answer if necessary. " + "Put your final answer in the format of `#### `." + ), + }, + { + "role": "user", + "content": question, + }, + ], + "ability": "math", + "reward_model": {"style": "rule", "ground_truth": solution}, + "extra_info": { + "split": split, + "index": idx, + "answer": answer_raw, + "question": question_raw, + "need_tools_kwargs": True, + "tools_kwargs": { + "calc_gsm8k_reward": { + "create_kwargs": {"ground_truth": solution}, + # "execute_kwargs": {}, + # "calc_reward_kwargs": {}, + # "release_kwargs": {}, + }, + }, + "interaction_kwargs": { + "query": question, + "ground_truth": solution, + }, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_save_dir, "test.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/hellaswag.py b/e2c/verl/examples/data_preprocess/hellaswag.py new file mode 100644 index 0000000000000000000000000000000000000000..dc73a810a80570d406bb727099f5524037be2370 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/hellaswag.py @@ -0,0 +1,108 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess Hellaswag dataset. + +""" + +import argparse +import os +import re + +import datasets + +from verl.utils.hdfs_io import copy, makedirs + + +def preprocess(text): + text = text.strip() + # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag. + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None, help="The save directory for the preprocessed dataset.") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/hellaswag", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + data_source = "Rowan/hellaswag" + + if local_dataset_path is not None: + dataset = datasets.load_dataset(local_dataset_path) + else: + dataset = datasets.load_dataset(data_source, trust_remote_code=True) + + train_dataset = dataset["train"] + val_dataset = dataset["validation"] + test_dataset = dataset["test"] + + instruction = "Please complete the following sentence.\n" + + def make_map_fn(split): + def process_fn(doc, idx): + ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize() + query = preprocess(doc["activity_label"] + ": " + ctx) + choices = [preprocess(ending) for ending in doc["endings"]] + gold = int(doc["label"]) + + data = { + "data_source": data_source, + "prompt": [{"role": "user", "content": query}], + "ability": "nlp", + "reward_model": { + "style": "model", + "eval": "multiple_choice", # using loglikelihood + "ground_truth": gold, + "choices": choices, + }, + "extra_info": {"split": split, "index": idx}, + } + return data + + return process_fn + + # filter data that doesn't have a label + train_dataset = train_dataset.filter(lambda x: len(x["label"]) > 0) + val_dataset = val_dataset.filter(lambda x: len(x["label"]) > 0) + test_dataset = test_dataset.filter(lambda x: len(x["label"]) > 0) + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + val_dataset = val_dataset.map(function=make_map_fn("validation"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) + + hdfs_dir = args.hdfs_dir + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + train_dataset.to_parquet(os.path.join(local_save_dir, "train.parquet")) + val_dataset.to_parquet(os.path.join(local_save_dir, "validation.parquet")) + test_dataset.to_parquet(os.path.join(local_save_dir, "test.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + + copy(src=local_save_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/math_dataset.py b/e2c/verl/examples/data_preprocess/math_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..b23a032fb1207a47dcd1bc77194a7c1a124aad55 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/math_dataset.py @@ -0,0 +1,106 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the MATH-lighteval dataset to parquet format +""" + +import argparse +import json +import os + +import datasets + +from verl.utils.hdfs_io import copy, makedirs +from verl.utils.reward_score.math_reward import last_boxed_only_string, remove_boxed + + +def extract_solution(solution_str): + return remove_boxed(last_boxed_only_string(solution_str)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default=None) + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--local_dataset_path", default=None, help="The local path to the raw dataset, if it exists.") + parser.add_argument( + "--local_save_dir", default="~/data/math", help="The save directory for the preprocessed dataset." + ) + + args = parser.parse_args() + local_dataset_path = args.local_dataset_path + + # 'lighteval/MATH' is no longer available on huggingface. + # Use mirror repo: DigitalLearningGmbH/MATH-lighteval + data_source = "DigitalLearningGmbH/MATH-lighteval" + print(f"Loading the {data_source} dataset from huggingface...", flush=True) + if local_dataset_path is not None: + dataset = datasets.load_dataset( + local_dataset_path, + ) + else: + dataset = datasets.load_dataset( + data_source, + ) + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + instruction_following = "Let's think step by step and output the final answer within \\boxed{}." + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question = example.pop("problem") + + question = question + " " + instruction_following + + answer = example.pop("solution") + solution = extract_solution(answer) + data = { + "data_source": data_source, + "prompt": [{"role": "user", "content": question}], + "ability": "math", + "reward_model": {"style": "rule", "ground_truth": solution}, + "extra_info": {"split": split, "index": idx}, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) + + local_save_dir = args.local_dir + if local_save_dir is not None: + print("Warning: Argument 'local_dir' is deprecated. Please use 'local_save_dir' instead.") + else: + local_save_dir = args.local_save_dir + + local_dir = os.path.expanduser(local_save_dir) + hdfs_dir = args.hdfs_dir + + train_dataset.to_parquet(os.path.join(local_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_dir, "test.parquet")) + # Save one example as JSON for reference + example = train_dataset[0] + with open(os.path.join(local_dir, "train_example.json"), "w") as f: + json.dump(example, f, indent=2) + example = test_dataset[0] + with open(os.path.join(local_dir, "test_example.json"), "w") as f: + json.dump(example, f, indent=2) + if hdfs_dir is not None: + makedirs(hdfs_dir) + + copy(src=local_dir, dst=hdfs_dir) diff --git a/e2c/verl/examples/data_preprocess/multiturn.py b/e2c/verl/examples/data_preprocess/multiturn.py new file mode 100644 index 0000000000000000000000000000000000000000..4bf0192b817d3acfc41f7e4a74a4a891d3ae61d6 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/multiturn.py @@ -0,0 +1,125 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Create a simple multi-turn dataset for testing +""" + +import argparse +import os + +import pandas as pd + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default="~/data/multiturn") + parser.add_argument("--hdfs_dir", default=None) + args = parser.parse_args() + + # Create example conversations + conversations = [] + + # Conversation 1 + conversations.append( + { + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + {"role": "assistant", "content": "The capital of France is Paris."}, + {"role": "user", "content": "And what about Germany?"}, + {"role": "assistant", "content": "The capital of Germany is Berlin."}, + ] + } + ) + + # Conversation 2 + conversations.append( + { + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Can you explain quantum computing?"}, + { + "role": "assistant", + "content": "Quantum computing is a type of computing that uses quantum-mechanical phenomena, " + "such as superposition and entanglement, to perform operations on data.", + }, + {"role": "user", "content": "How is it different from classical computing?"}, + { + "role": "assistant", + "content": "Classical computing uses bits that are either 0 or 1, while quantum computing uses " + "quantum bits or qubits that can exist in multiple states simultaneously due to superposition.", + }, + ] + } + ) + + # Conversation 3 + conversations.append( + { + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Write a simple Python function to calculate factorial."}, + { + "role": "assistant", + "content": ( + "```python\ndef factorial(n):\n if n == 0 or n == 1:\n return 1\n else:\n " + "return n * factorial(n-1)\n```\n\nThis is a recursive function to calculate the " + "factorial of a number." + ), + }, + {"role": "user", "content": "Can you make it iterative instead?"}, + { + "role": "assistant", + "content": ( + "```python\ndef factorial(n):\n result = 1\n for i in range(1, n+1):\n " + "result *= i\n return result\n```\n\nThis is an iterative version of the factorial function." + ), + }, + ] + } + ) + + # Create train and test datasets + train_data = conversations[:2] # First 2 conversations for training + test_data = conversations[2:] # Last conversation for testing + + # Create output directory + local_dir = os.path.expanduser(args.local_dir) + os.makedirs(local_dir, exist_ok=True) + + # Save to parquet files + train_df = pd.DataFrame(train_data) + test_df = pd.DataFrame(test_data) + + train_df.to_parquet(os.path.join(local_dir, "train.parquet")) + test_df.to_parquet(os.path.join(local_dir, "test.parquet")) + + # Handle HDFS if specified + if args.hdfs_dir is not None: + try: + from verl.utils.hdfs_io import copy, makedirs + + makedirs(args.hdfs_dir) + copy(src=local_dir, dst=args.hdfs_dir) + except ImportError: + print("Warning: HDFS support not available. Skipping HDFS copy.") + + # Print statistics + print(f"Train dataset size: {len(train_df)}") + print(f"Test dataset size: {len(test_df)}") + print(f"Data saved to {local_dir}") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/examples/data_preprocess/preprocess_search_r1_dataset.py b/e2c/verl/examples/data_preprocess/preprocess_search_r1_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c10d59b9c006ae7234ce21f7bdb25562259b23 --- /dev/null +++ b/e2c/verl/examples/data_preprocess/preprocess_search_r1_dataset.py @@ -0,0 +1,178 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import tempfile + +import pandas as pd +from huggingface_hub import hf_hub_download +from huggingface_hub.utils import EntryNotFoundError + +from verl.utils.hdfs_io import copy, makedirs + +# Setup logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + +# Configuration constants +DEFAULT_SYSTEM_CONTENT = "You are a helpful and harmless assistant." +DEFAULT_USER_CONTENT_PREFIX = ( + "Answer the given question. You must conduct reasoning inside and " + "first every time you get new information. After reasoning, if you find you lack " + "some knowledge, you can call a search engine by query " + "and it will return the top searched results between and " + ". You can search as many times as your want. If you find no " + "further external knowledge needed, you can directly provide the answer inside " + " and , without detailed illustrations. For example, " + " Beijing . Question: " +) + + +def process_single_row(row, current_split_name, row_index): + """ + Process a single row of data for SearchR1-like format. + + Args: + row: DataFrame row containing the original data + current_split_name: Name of the current split (train/test) + row_index: Index of the row in the DataFrame + + Returns: + pd.Series: Processed row data in the required format + """ + question = row.get("question", "") + + # Build prompt structure + user_content = user_content_prefix.rstrip("\n") + question + prompt = [{"role": "system", "content": system_content}, {"role": "user", "content": user_content}] + + # Extract ground truth from reward_model or fallback to golden_answers + reward_model_data = row.get("reward_model") + if isinstance(reward_model_data, dict) and "ground_truth" in reward_model_data: + ground_truth = reward_model_data.get("ground_truth") + else: + ground_truth = row.get("golden_answers", []) + + # Process data source + data_source_tagged = "searchR1_" + str(row.get("data_source", "")) + + # Build tools kwargs structure + tools_kwargs = { + "search": { + "create_kwargs": {"ground_truth": ground_truth, "question": question, "data_source": data_source_tagged} + } + } + + # Build complete extra_info structure + extra_info = { + "index": row_index, + "need_tools_kwargs": True, + "question": question, + "split": current_split_name, + "tools_kwargs": tools_kwargs, + } + + return pd.Series( + { + "data_source": data_source_tagged, + "prompt": prompt, + "ability": row.get("ability"), + "reward_model": reward_model_data, + "extra_info": extra_info, + "metadata": row.get("metadata"), + } + ) + + +def main(): + local_save_dir = os.path.expanduser(args.local_dir) + os.makedirs(local_save_dir, exist_ok=True) + + processed_files = [] + + # Download and process files using temporary directory + with tempfile.TemporaryDirectory() as tmp_download_dir: + for split in ["train", "test"]: + parquet_filename = f"{split}.parquet" + logger.info(f"Processing {split} split...") + + try: + # Download Parquet file from HuggingFace + logger.info(f"Downloading {parquet_filename} from {args.hf_repo_id}") + local_parquet_filepath = hf_hub_download( + repo_id=args.hf_repo_id, + filename=parquet_filename, + repo_type="dataset", + local_dir=tmp_download_dir, + local_dir_use_symlinks=False, + ) + + # Load and process Parquet file + df_raw = pd.read_parquet(local_parquet_filepath) + logger.info(f"Loaded {len(df_raw)} rows from {parquet_filename}") + + def apply_process_row(row, split_name=split): + return process_single_row(row, current_split_name=split_name, row_index=row.name) + + df_processed = df_raw.apply(apply_process_row, axis=1) + + # Save processed DataFrame + output_file_path = os.path.join(local_save_dir, f"{split}.parquet") + df_processed.to_parquet(output_file_path, index=False) + logger.info(f"Saved {len(df_processed)} processed rows to {output_file_path}") + processed_files.append(output_file_path) + + except EntryNotFoundError: + logger.warning(f"{parquet_filename} not found in repository {args.hf_repo_id}") + except Exception as e: + logger.error(f"Error processing {split} split: {e}") + + if not processed_files: + logger.warning("No data was processed or saved") + return + + logger.info(f"Successfully processed {len(processed_files)} files to {local_save_dir}") + + # Copy to HDFS if specified + if args.hdfs_dir: + try: + makedirs(args.hdfs_dir) + copy(src=local_save_dir, dst=args.hdfs_dir) + logger.info(f"Successfully copied files to HDFS: {args.hdfs_dir}") + except Exception as e: + logger.error(f"Error copying files to HDFS: {e}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Download Search-R1 from HuggingFace, process, and save to Parquet.") + parser.add_argument( + "--hf_repo_id", default="PeterJinGo/nq_hotpotqa_train", help="HuggingFace dataset repository ID." + ) + parser.add_argument( + "--local_dir", + default="~/data/searchR1_processed_direct", + help="Local directory to save the processed Parquet files.", + ) + parser.add_argument("--hdfs_dir", default=None, help="Optional HDFS directory to copy the Parquet files to.") + + args = parser.parse_args() + + # System and user content configuration + system_content = DEFAULT_SYSTEM_CONTENT + user_content_prefix = DEFAULT_USER_CONTENT_PREFIX + + main() diff --git a/e2c/verl/examples/generation/run_deepseek7b_mutli_node.sh b/e2c/verl/examples/generation/run_deepseek7b_mutli_node.sh new file mode 100644 index 0000000000000000000000000000000000000000..e939268ff8d960193f06b4770bb0f43631263135 --- /dev/null +++ b/e2c/verl/examples/generation/run_deepseek7b_mutli_node.sh @@ -0,0 +1,22 @@ +set -x + +data_path=$HOME/data/rlhf/gsm8k/test.parquet +save_path=$HOME/data/rlhf/math/deepseek_v2_lite_gen_test.parquet +model_path=deepseek-ai/deepseek-llm-7b-chat + +python3 -m verl.trainer.main_generation \ + trainer.nnodes=2 \ + trainer.n_gpus_per_node=8 \ + data.path=$data_path \ + data.prompt_key=prompt \ + data.n_samples=1 \ + data.output_path=$save_path \ + model.path=$model_path\ + +model.trust_remote_code=True \ + rollout.temperature=1.0 \ + rollout.top_k=50 \ + rollout.top_p=0.7 \ + rollout.prompt_length=2048 \ + rollout.response_length=1024 \ + rollout.tensor_model_parallel_size=16 \ + rollout.gpu_memory_utilization=0.8 diff --git a/e2c/verl/examples/generation/run_deepseek_v2_lite_math.sh b/e2c/verl/examples/generation/run_deepseek_v2_lite_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..0c5a74b1f489f5aa38da8273f73f8b4e65a24b9a --- /dev/null +++ b/e2c/verl/examples/generation/run_deepseek_v2_lite_math.sh @@ -0,0 +1,22 @@ +set -x + +data_path=$HOME/data/gsm8k/test.parquet +save_path=$HOME/data/gsm8k/deepseek_v2_lite_gen_test.parquet +model_path=deepseek-ai/deepseek-llm-7b-chat + +python3 -m verl.trainer.main_generation \ + trainer.nnodes=1 \ + trainer.n_gpus_per_node=8 \ + data.path=$data_path \ + data.prompt_key=prompt \ + data.n_samples=1 \ + data.output_path=$save_path \ + model.path=$model_path \ + +model.trust_remote_code=True \ + rollout.temperature=1.0 \ + rollout.top_k=50 \ + rollout.top_p=0.7 \ + rollout.prompt_length=2048 \ + rollout.response_length=1024 \ + rollout.tensor_model_parallel_size=2 \ + rollout.gpu_memory_utilization=0.8 diff --git a/e2c/verl/examples/gmpo_trainer/README.md b/e2c/verl/examples/gmpo_trainer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3d2b6335bbb52e43fe34489fbe9f76fa32d6db44 --- /dev/null +++ b/e2c/verl/examples/gmpo_trainer/README.md @@ -0,0 +1,63 @@ +
+ +# Geometric-Mean Policy Optimization +
+ +This is the official implementaion of paper [***Geometric-Mean Policy Optimization***](https://arxiv.org/abs/2507.20673). + +
+image +
+ + +## 1. Contents +- Geometric-Mean Policy Optimization + - [1. Contents](#1-contents) + - [2. Introduction](#2-introduction) + - [3. Code Usage](#4-code-usage) + - [4. Contacts](#5-contacts) + - [5. Citation](#7-citation) + +## 2. Introduction + +Recent advancements, such as Group Relative Policy Optimization (GRPO), have enhanced the reasoning capabilities of large language models by optimizing the arithmetic mean of token-level rewards. However, GRPO suffers from unstable policy updates when processing tokens with outlier importance-weighted rewards, which manifests as extreme importance sampling ratios during training, i.e., the ratio between the sampling probabilities assigned to a token by the current and old policies. In this work, we propose Geometric-Mean Policy Optimization (GMPO), a stabilized variant of GRPO. Instead of optimizing the arithmetic mean, GMPO maximizes the geometric mean of token-level rewards, which is inherently less sensitive to outliers and maintains a more stable range of importance sampling ratio. In addition, we provide comprehensive theoretical and experimental analysis to justify the design and stability benefits of GMPO. Beyond improved stability, GMPO-7B outperforms GRPO by an average of 4.1% on multiple mathematical benchmarks and 1.4% on multimodal reasoning benchmark, including AIME24, AMC, MATH500, OlympiadBench, Minerva, and Geometry3K. + +## 3. Code Usage + +The key configurations are: +``` +clip_ratio_low=0.4 +clip_ratio_high=0.4 +loss_mode=geo_mean +``` + +To get started quickly, run: +``` +bash examples/gmpo_trainer/run_qwen2_5-7b_math.sh +``` + +GMPO can be combined with other methods such as DAPO (experimental - not fully tested): +``` +bash examples/gmpo_trainer/test_dapo_7b_math.sh +bash examples/gmpo_trainer/test_dapo_qwen3_30b_math.sh +``` + +## 4. Contacts +If you have any question about our work or this repository, please don't hesitate to contact us by emails or open an issue under this project. +- [zhaoyuzhong20@mails.ucas.ac.cn](zhaoyuzhong20@mails.ucas.ac.cn) +- [liuyue171@mails.ucas.ac.cn](liuyue171@mails.ucas.ac.cn) +- [lecu@microsoft.com](lecu@microsoft.com) +- [wanfang@ucas.ac.cn](wanfang@ucas.ac.cn) + +## 5. Citation +``` +@misc{zhao2025geometricmeanpolicyoptimization, + title={Geometric-Mean Policy Optimization}, + author={Yuzhong Zhao and Yue Liu and Junpeng Liu and Jingye Chen and Xun Wu and Yaru Hao and Tengchao Lv and Shaohan Huang and Lei Cui and Qixiang Ye and Fang Wan and Furu Wei}, + year={2025}, + eprint={2507.20673}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2507.20673}, +} +``` diff --git a/e2c/verl/examples/gmpo_trainer/run_qwen2_5-7b_math.sh b/e2c/verl/examples/gmpo_trainer/run_qwen2_5-7b_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..06ad91c9fa47cf685425335a37af7eeb3eab15b6 --- /dev/null +++ b/e2c/verl/examples/gmpo_trainer/run_qwen2_5-7b_math.sh @@ -0,0 +1,60 @@ +set -x + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +use_kl_loss=False +loss_mode=geo_mean +clip_ratio=0.4 +save_contents="['model', 'optimizer', 'extra']" + +export WANDB_MODE=offline +save_contents="['hf_model']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-Math-7B \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.checkpoint.save_contents=${save_contents} \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_gmpo_example_gsm8k_math' \ + trainer.experiment_name='qwen2_5_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/gmpo_trainer/test_dapo_7b_math.sh b/e2c/verl/examples/gmpo_trainer/test_dapo_7b_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..a355c859b80d05754836fa87314289986ebfef67 --- /dev/null +++ b/e2c/verl/examples/gmpo_trainer/test_dapo_7b_math.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7b-MATH-0527a1' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.4 +clip_ratio_high=0.4 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-8} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=4 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=4 +fsdp_size=32 + +loss_mode=geo_mean + +# export WANDB_MODE=offline +save_contents="['model', 'optimizer', 'extra']" +# save_contents="['hf_model']" + +# reference run wandb: https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/runs/ow47vvon?nw=nwusertongyuxuan361 + +python3 -m verl.trainer.main_ppo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.model.use_remove_padding=True \ + +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \ + actor_rollout_ref.actor.checkpoint.save_contents="${save_contents}" \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=10 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=200 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/examples/gmpo_trainer/test_dapo_qwen3_30b_math.sh b/e2c/verl/examples/gmpo_trainer/test_dapo_qwen3_30b_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..c63805a3baa17b4d75e0c675f9a4f0be24cd1976 --- /dev/null +++ b/e2c/verl/examples/gmpo_trainer/test_dapo_qwen3_30b_math.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen3-30B-A3B-Base-MATH-0527a1' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.4 +clip_ratio_high=0.4 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +loss_mode=geo_mean + +# export WANDB_MODE=offline +save_contents="['model', 'optimizer', 'extra']" +# save_contents="['hf_model']" + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-8} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B-Base"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=4 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=4 +fsdp_size=32 + +python3 -m verl.trainer.main_ppo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \ + actor_rollout_ref.actor.checkpoint.save_contents="${save_contents}" \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=10 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=300 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/examples/gpg_trainer/gpg.md b/e2c/verl/examples/gpg_trainer/gpg.md new file mode 100644 index 0000000000000000000000000000000000000000..b40cc83bcd7aeaaef43622df7659fc03b394138d --- /dev/null +++ b/e2c/verl/examples/gpg_trainer/gpg.md @@ -0,0 +1,34 @@ +# GPG: Group Policy Gradient + +Group Policy Gradient (GPG) is a minimalist reinforcement learning (RL) method that enhances the reasoning ability of large language models without relying on supervised fine-tuning or complex tricks. GPG revisits traditional policy gradients and directly optimizes the RL objective—no surrogate losses, no KL penalties, no critic, and no reference model. Compared to GRPO, GPG is simpler, more efficient, and achieves better results on many tasks. For more details, please refer to the original paper [GPG: A Simple and Strong Reinforcement Learning Baseline for Model Reasoning +](https://arxiv.org/abs/2504.02546). + +## Key Components +- Use a corrected advantage function to improve policy gradient accuracy and training efficiency. +- By eliminating the critic and reference models, avoiding KL divergence constraints, significantly simplifies the training process compared to Group Relative Policy Optimization (GRPO) + +## Configuration +To configure GPG within the framework, use the following YAML settings. + +```yaml +algorithm: + adv_estimator: gpg +actor_rollout_ref: + actor: + policy_loss: + loss_mode: "gpg" +``` + +## Advanced Extensions +GPG is a simple and strong baseline for model reasoning. Although it avoids using KL loss in its original form, you can still use KL loss to further improve the performance. + +```yaml +algorithm: + adv_estimator: gpg +actor_rollout_ref: + actor: + use_kl_loss: True # enable kl regularization + kl_loss_coef: 0.01 + policy_loss: + loss_mode: "gpg" +``` \ No newline at end of file diff --git a/e2c/verl/examples/gpg_trainer/run_qwen2-7b_math.sh b/e2c/verl/examples/gpg_trainer/run_qwen2-7b_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..1454bf2947bb49d6f61d0e8fe26f375c093d405c --- /dev/null +++ b/e2c/verl/examples/gpg_trainer/run_qwen2-7b_math.sh @@ -0,0 +1,52 @@ +set -x + +# If you are using vllm<=0.6.3, you might need to set the following environment variable to avoid bugs: +# export VLLM_ATTENTION_BACKEND=XFORMERS + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gpg \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.policy_loss.loss_mode=gpg \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_gpg_example_gsm8k_math' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/gpg_trainer/run_qwen2-7b_math_megatron.sh b/e2c/verl/examples/gpg_trainer/run_qwen2-7b_math_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c48b44132a38619b619d55e9dca1c450e3b88b5 --- /dev/null +++ b/e2c/verl/examples/gpg_trainer/run_qwen2-7b_math_megatron.sh @@ -0,0 +1,53 @@ +set -x + +# If you are using vllm<=0.6.3, you might need to set the following environment variable to avoid bugs: +# export VLLM_ATTENTION_BACKEND=XFORMERS +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=gpg \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.policy_loss.loss_mode=gpg \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_gpg_example_gsm8k_math' \ + trainer.experiment_name='qwen2_7b_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/README.md b/e2c/verl/examples/grpo_trainer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..28338348b2d5bbabc69fd83b31afdae045a2b29b --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/README.md @@ -0,0 +1,69 @@ +# Group Relative Policy Optimization (GRPO) + +In reinforcement learning, classic algorithms like PPO rely on a "critic" model to estimate the value of actions, guiding the learning process. However, training this critic model can be resource-intensive. + +GRPO simplifies this process by eliminating the need for a separate critic model. Instead, it operates as follows: +- Group Sampling: For a given problem, the model generates multiple possible solutions, forming a "group" of outputs. +- Reward Assignment: Each solution is evaluated and assigned a reward based on its correctness or quality. +- Baseline Calculation: The average reward of the group serves as a baseline. +- Policy Update: The model updates its parameters by comparing each solution's reward to the group baseline, reinforcing better-than-average solutions and discouraging worse-than-average ones. + +This approach reduces computational overhead by avoiding the training of a separate value estimation model, making the learning process more efficient. For more details, refer to the original paper [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://arxiv.org/pdf/2402.03300) + +## Key Components + +- No Value Function (Critic-less): unlike PPO, GRPO does not train a separate value network (critic) +- Group Sampling (Grouped Rollouts): instead of evaluating one rollout per input, GRPO generates multiple completions (responses) from the current policy for each prompt. This set of completions is referred to as a group. +- Relative Rewards: within each group, completions are scored (e.g., based on correctness), and rewards are normalized relative to the group. + +## Configuration + +Note that all configs containing `micro_batch_size` are used to configure the maximum sample or token count per forward or backward pass to avoid GPU OOMs, whose value should not change algorithmic/convergence behavior. + +Despite that many configurations start with the `ppo_` prefix, they work across different RL algorithms in verl, as the GRPO training loop is similar to that of PPO (without critic). + +![image](https://github.com/user-attachments/assets/16aebad1-0da6-4eb3-806d-54a74e712c2d) + +- `actor_rollout.ref.rollout.n`: For each prompt, sample n times. Default to 1. For GRPO, please set it to a value larger than 1 for group sampling. + +- `data.train_batch_size`: The global batch size of prompts used to generate a set of sampled trajectories/rollouts. The number of responses/trajectories is `data.train_batch_size * actor_rollout.ref.rollout.n` + +- `actor_rollout_ref.actor.ppo_mini_batch_size`: The set of sampled trajectories is split into multiple mini-batches with batch_size=ppo_mini_batch_size for PPO actor updates. The ppo_mini_batch_size is a global size across all workers. + +- `actor_rollout_ref.actor.ppo_epochs`: Number of epochs for GRPO updates on one set of sampled trajectories for actor + +- `actor_rollout_ref.actor.clip_ratio`: The GRPO clip range. Default to 0.2 + +- `algorithm.adv_estimator`: Default is gae. Please set it to grpo instead + +- `actor_rollout_ref.actor.loss_agg_mode`: Default is "token-mean". Options include "token-mean", "seq-mean-token-sum", "seq-mean-token-mean". The original GRPO paper takes the sample-level loss (seq-mean-token-mean), which may be unstable in long-CoT scenarios. All GRPO example scripts provided in verl uses the default configuration "token-mean" for loss aggregation instead. + +Instead of adding KL penalty in the reward, GRPO regularizes by directly adding the KL divergence between the trained policy and the reference policy to the loss: + +- `actor_rollout_ref.actor.use_kl_loss`: To use kl loss in the actor. When used, we are not applying KL in the reward function. Default is False. Please set it to True for GRPO. + +- `actor_rollout_ref.actor.kl_loss_coef`: The coefficient of kl loss. Default is 0.001. + +- `actor_rollout_ref.actor.kl_loss_type`: Support kl(k1), abs, mse(k2), low_var_kl(k3) and full. Appending "+" in the end (e.g., 'k1+' and 'k3+') would apply straight through to employ k2 for unbiased gradient estimation, regardless of the kl value estimation (see https://github.com/volcengine/verl/pull/2953#issuecomment-3162113848 for more details). How to calculate the kl divergence between actor and reference policy. See this blog post for detailed analysis: http://joschu.net/blog/kl-approx.html + +## Advanced Extensions + +### DrGRPO + +The work [Understanding R1-Zero-Like Training: A Critical Perspective](https://arxiv.org/pdf/2503.20783) claims there's optimization bias in GRPO, that leads to artificially longer responses, especially for incorrect outputs. This inefficiency stems from the way GRPO calculates advantages using group-based reward normalization, which can inadvertently favor longer, less accurate responses. Instead, DrGRPO aggregates token-level losses by normalizing with a global constant to eliminate length bias. + +Configure the following to enable DrGRPO, with all other parameters the same as GRPO's: + +- `actor_rollout_ref.actor.loss_agg_mode`: "seq-mean-token-sum-norm", which turns off seq-dim averaging +- `actor_rollout_ref.actor.use_kl_loss`: Please set it to False for DrGRPO +- `algorithm.norm_adv_by_std_in_grpo`: False, which turns off standard deviation norm + +## Reference Example + +Qwen2.5 GRPO training log and commands: [link](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/qwen2-7b-fsdp2.log) + +```bash +bash examples/grpo_trainer/run_qwen3-8b.sh +``` + +For more reference performance, please see https://verl.readthedocs.io/en/latest/algo/baseline.html diff --git a/e2c/verl/examples/grpo_trainer/run_deepseek671b_math_megatron_80gb.sh b/e2c/verl/examples/grpo_trainer/run_deepseek671b_math_megatron_80gb.sh new file mode 100644 index 0000000000000000000000000000000000000000..dd81acb7fd7f616377643cf42d8bfe2b93050546 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_deepseek671b_math_megatron_80gb.sh @@ -0,0 +1,117 @@ +set -x + +# # 0. download HF checkpoint +# # remove the `quantization_config` in the `config.json` +# # set `num_nextn_predict_layers=0` to disable MTP, which is not currently supported +# huggingface-cli download deepseek-ai/DeepSeek-V3-0324 + +# no offline dist checkpoint needed, now with mbridge>=0.13.0, we can directly init model from huggingface downloaded fp8 weights +# tested on docker://verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2 +LLM="" + + +# 2. run the script +gsm8k_train_path=/root/data/gsm8k/train.parquet +gsm8k_test_path=/root/data/gsm8k/test.parquet +train_files=$gsm8k_train_path +test_files=$gsm8k_test_path + +ALL_OFFLOAD=${ALL_OFFLOAD:-True} +COMMON_PARAM_OFFLOAD=${COMMON_PARAM_OFFLOAD:-$ALL_OFFLOAD} +COMMON_GRAD_OFFLOAD=${COMMON_GRAD_OFFLOAD:-$ALL_OFFLOAD} +COMMON_OPTIMIZER_OFFLOAD=${COMMON_OPTIMIZER_OFFLOAD:-$ALL_OFFLOAD} + +ACTOR_PARAM_OFFLOAD=${ACTOR_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +ACTOR_GRAD_OFFLOAD=${ACTOR_GRAD_OFFLOAD:-$COMMON_GRAD_OFFLOAD} +ACTOR_OPTIMIZER_OFFLOAD=${ACTOR_OPTIMIZER_OFFLOAD:-$COMMON_OPTIMIZER_OFFLOAD} +REF_PARAM_OFFLOAD=${REF_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +CRITIC_PARAM_OFFLOAD=${CRITIC_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +CRITIC_GRAD_OFFLOAD=${CRITIC_GRAD_OFFLOAD:-$COMMON_GRAD_OFFLOAD} +CRITIC_OPTIMIZER_OFFLOAD=${CRITIC_OPTIMIZER_OFFLOAD:-$COMMON_OPTIMIZER_OFFLOAD} +RM_PARAM_OFFLOAD=${RM_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} + +# 256 H100(80GB) +NODES=32 +PP=16 +TP=1 +EP=16 +ETP=1 +INFER_TP=32 +# consider TP/ETP, and enable recompute if short of memory + +# full recompute + +n_resp_per_prompt=4 +max_prompt_length=2048 +max_response_length=4096 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 1)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=True +kl_loss_coef=0.001 + +# RAY_ADDRESS='auto' ray job submit --working-dir . -- +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + algorithm.adv_estimator=grpo \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=512 \ + data.max_prompt_length=$max_prompt_length \ + data.max_response_length=$max_response_length \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=$LLM \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.top_p=1.0 \ + actor_rollout_ref.rollout.top_k=-1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$INFER_TP \ + trainer.logger='["console","tensorboard"]' \ + trainer.project_name='verl_megatron_gsm8k_examples' \ + trainer.experiment_name='dsv3-32nodes' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=$NODES \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_first_pipeline_stage=4 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_last_pipeline_stage=1 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=$PP \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=$PP \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=$TP \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=$TP \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.actor.megatron.param_offload=${ACTOR_PARAM_OFFLOAD} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${ACTOR_OPTIMIZER_OFFLOAD} \ + actor_rollout_ref.actor.megatron.grad_offload=${ACTOR_GRAD_OFFLOAD} \ + actor_rollout_ref.ref.megatron.param_offload=${REF_PARAM_OFFLOAD} \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 \ + actor_rollout_ref.actor.megatron.use_mbridge=True \ + trainer.default_local_dir=$CKPT_DIR \ + trainer.val_before_train=False \ + trainer.total_epochs=100 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_deepseek671b_math_megatron_96gb.sh b/e2c/verl/examples/grpo_trainer/run_deepseek671b_math_megatron_96gb.sh new file mode 100644 index 0000000000000000000000000000000000000000..ede8eeda79ff27be7c58c4bd74fd1055366b7fb2 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_deepseek671b_math_megatron_96gb.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +## !!!!!!!important!!!!!! +# 1. set the following environment variables on all your nodes +# env_vars: +# CUDA_DEVICE_MAX_CONNECTIONS: "1" +# NCCL_NVLS_ENABLE: "0" +# VLLM_USE_V1: 1 +# 2. install mbridge=0.1.13 on all your node with the following command: +# pip3 install git+https://github.com/ISEEKYAN/mbridge +# 3. remove the `quantization_config` in the DeepSeek-V3's `config.json` and +# set `num_nextn_predict_layers=0` to disable MTP, which is not currently supported + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +[ -f "${SCRIPT_DIR}/env.sh" ] && source "${SCRIPT_DIR}/env.sh" + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=True +kl_loss_coef=0.001 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1204 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=96 +n_resp_per_prompt=8 +train_prompt_mini_bsz=32 + + +# minimum nodes for DeepSeek-V3: 12 nodes +NNODES=${NNODES:-12} + +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} + +MODEL_PATH=$RAY_DATA_HOME/models/DeepSeek-V3-config-verl + +TRAIN_FILE=$RAY_DATA_HOME/dataset/dapo-math-17k.parquet +TEST_FILE=$RAY_DATA_HOME/dataset/aime-2024.parquet + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 10 / 10)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 1)) +offload=True +optim_offload=${OFFLOAD_OPTIM:-True} +gen_tp=32 +train_tp=${TP:-8} +train_pp=${PP:-12} + +EP=${EP:-8} +ETP=1 +CP=1 +optimizer_offload_fraction=${OFFLOAD_FRACTION:-1.} +LAST_LAYER=${LAST_LAYER:-6} + + +project_name='verl-deepseek-v3' +exp_name="671B-${NNODES}-pp${train_pp}-tp${train_tp}-ep${EP}-actor-length${actor_ppo_max_token_len}" +CKPTS_DIR=$RAY_DATA_HOME/ckpt/${project_name}/${exp_name} + +python3 -m verl.trainer.main_ppo \ + --config-path=config \ + --config-name='ppo_megatron_trainer.yaml' \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.rollout.name=vllm \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.megatron.use_mbridge=True \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \ + +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${optim_offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.actor.megatron.context_parallel_size=${CP} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.clip_grad=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.nccl_timeout=1200 \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.ref.megatron.context_parallel_size=${CP} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=False \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_shared_expert_overlap=False \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_enable_deepep=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_token_dispatcher_type=flex \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.account_for_embedding_in_pipeline_split=False \ + +actor_rollout_ref.actor.megatron.override_transformer_config.account_for_loss_in_pipeline_split=False \ + +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_last_pipeline_stage=${LAST_LAYER} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','wandb'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=10 \ + trainer.save_freq=100 \ + trainer.total_epochs=10 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm.sh b/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm.sh new file mode 100644 index 0000000000000000000000000000000000000000..af9204ab1ccc4c6784eab178f849d7a2882a27e5 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm.sh @@ -0,0 +1,40 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=80 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=160 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=160 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='deepseek_llm_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_math.sh b/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..198e6f4ae71e89fa1559facdabe3e3f8dd7ac4d7 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_math.sh @@ -0,0 +1,49 @@ +set -x + + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k_math' \ + trainer.experiment_name='deepseek_llm_7b_function_rm_math' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_math_megatron.sh b/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_math_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..46788e16f5bcf31c53ac3ee489743ee4bd985a8d --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_math_megatron.sh @@ -0,0 +1,50 @@ +set -x + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k_math' \ + trainer.experiment_name='deepseek_llm_7b_math_megatron' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_seq_balance.sh b/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_seq_balance.sh new file mode 100644 index 0000000000000000000000000000000000000000..72cd4445a8edc7a70686cea8b96c7b3066b88f36 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_deepseek7b_llm_seq_balance.sh @@ -0,0 +1,39 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='deepseek_llm_7b_function_rm_seq_packing' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_glm41v_9b.sh b/e2c/verl/examples/grpo_trainer/run_glm41v_9b.sh new file mode 100644 index 0000000000000000000000000000000000000000..a845bcc244f79ae7301a04c0e010a2586d528166 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_glm41v_9b.sh @@ -0,0 +1,46 @@ +set -x +ENGINE=${1:-vllm} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=zai-org/GLM-4.1V-9B-Thinking \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=10 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$ENGINE \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='glm41v_9b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_gptoss_20b.sh b/e2c/verl/examples/grpo_trainer/run_gptoss_20b.sh new file mode 100644 index 0000000000000000000000000000000000000000..4de21659d8cac021a20dd7ffd4520598d2b6d31c --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_gptoss_20b.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# install flashinfer +cd $HOME +git clone https://github.com/flashinfer-ai/flashinfer.git --recursive +cd flashinfer +python -m pip install -v . + +# install sglang +cd $HOME +git fetch origin pull/9379/head:fix_weight_loading +cd $HOME/sglang +git checkout fix_weight_loading +pip install --upgrade pip +pip install -e "python[all]" + +pip install peft +pip install transformers -U +pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.8cxx11abiTRUE-cp311-cp311-linux_x86_64.whl +pip install numpy==1.26.4 + + +cat > get_model.py << EOF +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, Mxfp4Config + +model_id = "openai/gpt-oss-20b" +output_dir = "$HOME/models/gpt-oss-20b-bf16" + +quantization_config = Mxfp4Config(dequantize=True) +model_kwargs = dict( + attn_implementation="eager", + torch_dtype=torch.bfloat16, + quantization_config=quantization_config, + use_cache=False, + device_map="auto", +) + +model = AutoModelForCausalLM.from_pretrained(model_id, **model_kwargs) + +# Patch config with custom attribute before saving +model.config.attn_implementation = "eager" + +model.save_pretrained(output_dir) +tokenizer = AutoTokenizer.from_pretrained(model_id) +tokenizer.save_pretrained(output_dir) +EOF + +python get_model.py + + + +model_dir=$HOME/models/gpt-oss-20b-bf16 +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="$gsm8k_train_path" \ + data.val_files="$gsm8k_test_path" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=${model_dir} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + +actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.engine_kwargs.sglang.attention_backend=triton \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k_math' \ + trainer.experiment_name='oai_oss_20b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=50 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_minicpmo2_6.sh b/e2c/verl/examples/grpo_trainer/run_minicpmo2_6.sh new file mode 100644 index 0000000000000000000000000000000000000000..d1daab99a9fb16e6698ad8a7a22d7ea64e091281 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_minicpmo2_6.sh @@ -0,0 +1,49 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=False \ + data.truncation='error' \ + data.image_key=images \ + data.trust_remote_code=True \ + data.custom_cls.path=recipe/minicpmo/rl_dataset.py \ + data.custom_cls.name=RLHFDataset \ + actor_rollout_ref.model.path=openbmb/MiniCPM-o-2_6 \ + actor_rollout_ref.model.trust_remote_code=True \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.use_dynamic_bsz=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.fsdp_config.use_orig_params=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=False \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='minicpmo2_6_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_mistral13b_skyworkrm_hhrlhf.sh b/e2c/verl/examples/grpo_trainer/run_mistral13b_skyworkrm_hhrlhf.sh new file mode 100644 index 0000000000000000000000000000000000000000..dd2fd793bd9cca9871ed5c8b2036f2a87d2e7137 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_mistral13b_skyworkrm_hhrlhf.sh @@ -0,0 +1,50 @@ +train_files=data/full_hh_rlhf/rl/train.parquet +test_files=data/full_hh_rlhf/rl/train.parquet # no use + +max_prompt_length=4096 +max_response_length=2048 + +gen_tp=4 +n_per_prompt=5 +adv_estimator="grpo" + +project_name=verl_full_hh_rlhf_examples +exp_name="grpo_mistral13B-skyworkLlama8b-hhrlhf" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=$adv_estimator \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=512 \ + data.prompt_key="prompt" \ + data.return_raw_chat=True \ + data.max_prompt_length=$max_prompt_length \ + data.max_response_length=$max_response_length \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=mistralai/Mistral-Nemo-Instruct-2407 \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=10 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=10 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.n=$n_per_prompt \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + reward_model.enable=True \ + reward_model.model.fsdp_config.param_offload=True \ + reward_model.model.path=Skywork/Skywork-Reward-Llama-3.1-8B \ + reward_model.model.input_tokenizer=mistralai/Mistral-Nemo-Instruct-2407 \ + reward_model.micro_batch_size_per_gpu=4 \ + algorithm.use_kl_in_reward=False \ + trainer.logger='["console","wandb"]' \ + trainer.val_before_train=False \ + trainer.project_name=$project_name \ + trainer.experiment_name=$exp_name \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=10 \ + trainer.test_freq=-1 \ + trainer.total_epochs=5 $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_moonlight16b_math_megatron.sh b/e2c/verl/examples/grpo_trainer/run_moonlight16b_math_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..61a2beb19e9d189ff356f26e0948d4aa52f2b8d2 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_moonlight16b_math_megatron.sh @@ -0,0 +1,58 @@ +set -x + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +HF_MODEL_PATH=moonshotai/Moonlight-16B-A3B +DIST_CKPT_PATH=${DIST_CKPT_PATH} + +train_path=$HOME/data/gsm8k/train.parquet +test_path=$HOME/data/gsm8k/test.parquet + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=grpo \ + data.train_files="$train_path" \ + data.val_files="$test_path" \ + data.train_batch_size=192 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.trust_remote_code=True \ + actor_rollout_ref.model.path=$HF_MODEL_PATH \ + actor_rollout_ref.model.trust_remote_code=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=3 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=4 \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=1 \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.actor.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=3 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=4 \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=4 \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=1 \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.ref.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k_math' \ + trainer.experiment_name='moonlight_megatron_ep' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=3 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2-7b.sh b/e2c/verl/examples/grpo_trainer/run_qwen2-7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..ba3c64a6ad5202e5ac7734e94dbeaba7a8ae2aff --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2-7b.sh @@ -0,0 +1,41 @@ +set -x + + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2-7b_math.sh b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..f4e6ec408ff3518ee1a41240a9ea1bb2e92e5179 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_math.sh @@ -0,0 +1,49 @@ +set -x + + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k_math' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2-7b_math_megatron.sh b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_math_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..2a29ccd077a423d7ba71023de1c47a5b6956fbae --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_math_megatron.sh @@ -0,0 +1,61 @@ +set -x + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +rollout_mode="sync" +if [ "$rollout_mode" = "async" ]; then + export VLLM_USE_V1=1 + return_raw_chat="True" +fi + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +USE_FUSED_KERNELS=True + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.return_raw_chat=$return_raw_chat \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.model.use_fused_kernels=$USE_FUSED_KERNELS \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=$rollout_mode \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k_math' \ + trainer.experiment_name='qwen2_7b_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2-7b_seq_balance.sh b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_seq_balance.sh new file mode 100644 index 0000000000000000000000000000000000000000..fdc1ef606d7ee1a96fa14da2940afa3366b36029 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_seq_balance.sh @@ -0,0 +1,52 @@ +set -x + + +# For async rollout mode, dataset should return raw chat. +rollout_mode="async" +rollout_name="sglang" # sglang or vllm +if [ "$rollout_mode" = "async" ]; then + export VLLM_USE_V1=1 + return_raw_chat="True" +fi + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.return_raw_chat=$return_raw_chat \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$rollout_name \ + actor_rollout_ref.rollout.mode=$rollout_mode \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm_kl1e-3' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2-7b_seq_balance_math_megatron.sh b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_seq_balance_math_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..fbcb83ffb8aa160b6f89e1ead725248fb951ed0f --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_seq_balance_math_megatron.sh @@ -0,0 +1,57 @@ +set -x + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +offload=True + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=12000 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k_math' \ + trainer.experiment_name='qwen2_7b_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2-7b_sgl_megatron.sh b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_sgl_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..5dc4ec87fa75512d24f76e2875b60efc3ffb9090 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2-7b_sgl_megatron.sh @@ -0,0 +1,47 @@ +set -x + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5-3b_gsm8k_grpo_lora.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5-3b_gsm8k_grpo_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..d321b65d43fdce3b8a9b706feed02287baeb7193 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5-3b_gsm8k_grpo_lora.sh @@ -0,0 +1,51 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + trainer.val_before_train=False \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=16 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.model.lora_rank=64 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.actor.optim.lr=3e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=16 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2.5_3b_grpo_lora' \ + trainer.n_gpus_per_node=2 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ + + # actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + # data.train_batch_size=1024 \ + # trainer.n_gpus_per_node=8 \ + # actor_rollout_ref.model.use_shm=True \ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5-7b_math_megatron_diff_tp.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5-7b_math_megatron_diff_tp.sh new file mode 100644 index 0000000000000000000000000000000000000000..e7053d1dd73f8eb7b0f1410408a37ec18cb198e8 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5-7b_math_megatron_diff_tp.sh @@ -0,0 +1,50 @@ +set -x + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k_math' \ + trainer.experiment_name='qwen2_7b_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_32b_grpo_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_32b_grpo_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..6d0d4fe4e2ed921f7c326cba012533106d048cb9 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_32b_grpo_npu.sh @@ -0,0 +1,41 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-32B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6\ + actor_rollout_ref.model.use_remove_padding=False \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=8 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_5_32b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=2 \ + trainer.save_freq=-1 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 \ + trainer.device=npu $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_discrete_prof_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_discrete_prof_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..27ab478da2868b249c7d146d6895340b86647bf0 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_discrete_prof_npu.sh @@ -0,0 +1,72 @@ +set -x + +# profiling configuration +PROFILE_STEPS="[2,4]" +PROFILE_RANKS_ALL=False +DISCRETE=True +PROFILE_RANKS="[1,2]" + +# profiling NPU options +SAVE_PATH="$HOME/profile_data" +LEVEL="level1" +CONTENTS=['npu','cpu'] +ANALYSIS=True + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=32 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-7B-Instruct \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.use_remove_padding=False \ + actor_rollout_ref.actor.optim.lr=5e-8 \ + actor_rollout_ref.actor.ppo_mini_batch_size=2 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.profiler.enable=True \ + actor_rollout_ref.actor.profiler.ranks=$PROFILE_RANKS \ + actor_rollout_ref.actor.profiler.all_ranks=$PROFILE_RANKS_ALL \ + actor_rollout_ref.actor.profiler.tool_config.npu.discrete=$DISCRETE \ + actor_rollout_ref.actor.profiler.tool_config.npu.contents=$CONTENTS \ + actor_rollout_ref.actor.profiler.tool_config.npu.level=$LEVEL \ + actor_rollout_ref.actor.profiler.tool_config.npu.analysis=$ANALYSIS \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.ref.profiler.enable=True \ + actor_rollout_ref.ref.profiler.ranks=$PROFILE_RANKS \ + actor_rollout_ref.ref.profiler.all_ranks=$PROFILE_RANKS_ALL \ + actor_rollout_ref.ref.profiler.tool_config.npu.discrete=$DISCRETE \ + actor_rollout_ref.ref.profiler.tool_config.npu.contents=$CONTENTS \ + actor_rollout_ref.ref.profiler.tool_config.npu.level=$LEVEL \ + actor_rollout_ref.ref.profiler.tool_config.npu.analysis=$ANALYSIS \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_5_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=5 \ + trainer.device=npu \ + global_profiler.tool=npu \ + global_profiler.steps=$PROFILE_STEPS \ + global_profiler.save_path=$SAVE_PATH + $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_e2e_prof_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_e2e_prof_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..1ac6dfe94452b6d9235de672ca7afbb6805ad0d5 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_e2e_prof_npu.sh @@ -0,0 +1,69 @@ +set -x + +# profiling configuration +PROFILE_STEPS="[2,4]" +PROFILE_RANKS_ALL=True +DISCRETE=False + +# profiling NPU options +SAVE_PATH="$HOME/profile_data" +LEVEL="level1" +CONTENTS=['npu','cpu'] +ANALYSIS=True + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=32 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=5e-8 \ + actor_rollout_ref.model.use_remove_padding=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=2 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.profiler.enable=True \ + actor_rollout_ref.actor.profiler.all_ranks=$PROFILE_RANKS_ALL \ + actor_rollout_ref.actor.profiler.tool_config.npu.discrete=$DISCRETE \ + actor_rollout_ref.actor.profiler.tool_config.npu.contents=$CONTENTS \ + actor_rollout_ref.actor.profiler.tool_config.npu.level=$LEVEL \ + actor_rollout_ref.actor.profiler.tool_config.npu.analysis=$ANALYSIS \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.ref.profiler.enable=True \ + actor_rollout_ref.ref.profiler.all_ranks=$PROFILE_RANKS_ALL \ + actor_rollout_ref.ref.profiler.tool_config.npu.discrete=$DISCRETE \ + actor_rollout_ref.ref.profiler.tool_config.npu.contents=$CONTENTS \ + actor_rollout_ref.ref.profiler.tool_config.npu.level=$LEVEL \ + actor_rollout_ref.ref.profiler.tool_config.npu.analysis=$ANALYSIS \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_5_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=5 \ + trainer.device=npu \ + global_profiler.tool=npu \ + global_profiler.steps=$PROFILE_STEPS \ + global_profiler.save_path=$SAVE_PATH + $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..07dda340c39627146c0f213bc1bbf27f8e1aa1cc --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_7b_grpo_npu.sh @@ -0,0 +1,42 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=5e-8 \ + actor_rollout_ref.model.use_remove_padding=False \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_5_7b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=5 \ + trainer.device=npu $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b-megatron.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b-megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..632bdc8fa1e097092416338a81426fcb661d8946 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b-megatron.sh @@ -0,0 +1,88 @@ +set -x +ENGINE=${1:-vllm} +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +HF_MODEL_PATH=Qwen/Qwen2.5-VL-7B-Instruct +DIST_CKPT_PATH=${DIST_CKPT_PATH} + +# convert HF model to meagatron format offlinely +# python scripts/converter_hf_to_mcore.py --hf_model_path $HF_MODEL_PATH --output_path $DIST_CKPT_PATH + + +# megatron tuning guide: +# 1. recommend to offload all states by setting ALL_OFFLOAD=True +# 2. enable dynamic batch size by setting actor_rollout_ref.actor.use_dynamic_bsz=True ref.log_prob_use_dynamic_bsz=True rollout.log_prob_use_dynamic_bsz=True +# 3. set ppo_max_token_len_per_gpu and log_prob_max_token_len_per_gpu as large as possible for better MFU (limited by GPU memory). assure ppo_max_token_len_per_gpu > max_prompt_length+max_response_length, if sequence length is too long, you can increase the TP/PP size +# 4. if memory is very limited, enable full recompute, but the mfu will be 30% lower +# full recompute settings: +# +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ +# +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ +# +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 \ + +ALL_OFFLOAD=${ALL_OFFLOAD:-True} +COMMON_PARAM_OFFLOAD=${COMMON_PARAM_OFFLOAD:-$ALL_OFFLOAD} +COMMON_GRAD_OFFLOAD=${COMMON_GRAD_OFFLOAD:-$ALL_OFFLOAD} +COMMON_OPTIMIZER_OFFLOAD=${COMMON_OPTIMIZER_OFFLOAD:-$ALL_OFFLOAD} + +ACTOR_PARAM_OFFLOAD=${ACTOR_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +ACTOR_GRAD_OFFLOAD=${ACTOR_GRAD_OFFLOAD:-$COMMON_GRAD_OFFLOAD} +ACTOR_OPTIMIZER_OFFLOAD=${ACTOR_OPTIMIZER_OFFLOAD:-$COMMON_OPTIMIZER_OFFLOAD} +REF_PARAM_OFFLOAD=${REF_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} + + +train_path=$HOME/data/geo3k/train.parquet +test_path=$HOME/data/geo3k/test.parquet + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=grpo \ + data.train_files="$train_path" \ + data.val_files="$test_path" \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=$HF_MODEL_PATH \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=1 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=5120 \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=20480 \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=20480 \ + actor_rollout_ref.rollout.name=$ENGINE \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=1 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.actor.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + actor_rollout_ref.ref.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + actor_rollout_ref.actor.megatron.param_offload=${ACTOR_PARAM_OFFLOAD} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${ACTOR_OPTIMIZER_OFFLOAD} \ + actor_rollout_ref.actor.megatron.grad_offload=${ACTOR_GRAD_OFFLOAD} \ + actor_rollout_ref.ref.megatron.param_offload=${REF_PARAM_OFFLOAD} \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_7b_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b-sglang.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b-sglang.sh new file mode 100644 index 0000000000000000000000000000000000000000..86267a5602a9f87eac1965eeaf1a0020b7d7b565 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b-sglang.sh @@ -0,0 +1,53 @@ +set -x + +# python examples/data_preprocess/geo3k.py --local_dir ~/data/geo3k + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=10 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.multi_stage_wake_up=True \ + global_profiler.tool=torch_memory \ + global_profiler.save_path=./mem_snapshots \ + global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries=100000 \ + global_profiler.global_tool_config.torch_memory.stack_depth=32 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.mode=sync \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..b64ec094118bfece1ee081326f82bd0813b835c6 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b.sh @@ -0,0 +1,47 @@ +set -x +ENGINE=${1:-vllm} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=10 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$ENGINE \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_freeze_vision.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_freeze_vision.sh new file mode 100644 index 0000000000000000000000000000000000000000..8f51d568744e0a7bb240b0ae2eaa6bf703493110 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_freeze_vision.sh @@ -0,0 +1,47 @@ +set -x +ENGINE=${1:-vllm} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.freeze_vision_tower=True \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=10 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$ENGINE \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_lora.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..cb1af5b0847c9f31db837c183caab93754d2d057 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_lora.sh @@ -0,0 +1,52 @@ +set -x +ENGINE=${1:-vllm} +# If you are using vllm<=0.6.3, you might need to set the following environment variable to avoid bugs: +# export VLLM_ATTENTION_BACKEND=XFORMERS + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=3e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=10 \ + actor_rollout_ref.model.lora_rank=64 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.model.exclude_modules='.*visual.*' \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$ENGINE \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=False \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_seq_balance.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_seq_balance.sh new file mode 100644 index 0000000000000000000000000000000000000000..e9933b106a44ec14234f86ac19da06557c7af92f --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl-7b_seq_balance.sh @@ -0,0 +1,45 @@ +set -x +ENGINE=${1:-vllm} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=6144 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$ENGINE \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=False \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=6144 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_32b_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_32b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..c29838a336b290c9478544389316d28bc70a9ca2 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_32b_npu.sh @@ -0,0 +1,52 @@ +set -x +ENGINE=${1:-vllm} + +# Some models are optimized by vllm ascend. While in some case, e.g. rlhf training, +# the optimized model may not be suitable. In this case, set this value to 0 to disable the optimized model. +export USE_OPTIMIZED_MODEL=0 + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-32B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=8 \ + actor_rollout_ref.rollout.name=$ENGINE \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_32b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=2 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=15 \ + trainer.device=npu $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_3b_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_3b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..07ab65ee2a1a726f28198aa2b048918b7b077c59 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_3b_npu.sh @@ -0,0 +1,52 @@ +set -x +ENGINE=${1:-vllm} + +# Some models are optimized by vllm ascend. While in some case, e.g. rlhf training, +# the optimized model may not be suitable. In this case, set this value to 0 to disable the optimized model. +export USE_OPTIMIZED_MODEL=0 + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=16 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$ENGINE \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_3b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=15 \ + trainer.device=npu $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_7b_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_7b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..6d8f959817b035e61c7f6a430c60a6f360527a83 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen2_5_vl_7b_npu.sh @@ -0,0 +1,52 @@ +set -x +ENGINE=${1:-vllm} + +# Some models are optimized by vllm ascend. While in some case, e.g. rlhf training, +# the optimized model may not be suitable. In this case, set this value to 0 to disable the optimized model. +export USE_OPTIMIZED_MODEL=0 + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=$ENGINE \ + +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_7b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=15 \ + trainer.device=npu $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen3-235b_megatron_96gb.sh b/e2c/verl/examples/grpo_trainer/run_qwen3-235b_megatron_96gb.sh new file mode 100644 index 0000000000000000000000000000000000000000..0d3b855b6a998d95af5ca41ea97dedd453013183 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen3-235b_megatron_96gb.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +## !!!!!!!important!!!!!! +## set the following environment variables on all your nodes +# env_vars: +# CUDA_DEVICE_MAX_CONNECTIONS: "1" +# NCCL_NVLS_ENABLE: "0" +# VLLM_USE_V1: 1 +# install mbridge=0.1.13 on all your node with the following command: +# pip3 install git+https://github.com/ISEEKYAN/mbridge + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +[ -f "${SCRIPT_DIR}/env.sh" ] && source "${SCRIPT_DIR}/env.sh" + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=True +kl_loss_coef=0.001 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1204 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 1)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=${TRAIN_BS:-32} +n_resp_per_prompt=8 +train_prompt_mini_bsz=16 + +# minimum nodes need for qwen3-235B-A22B +NNODES=${NNODES:-4} +# Paths + +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} + +MODEL_PATH=$RAY_DATA_HOME/models/Qwen3-235B-A22B + +TRAIN_FILE=$RAY_DATA_HOME/dataset/dapo-math-17k.parquet +TEST_FILE=$RAY_DATA_HOME/dataset/aime-2024.parquet + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 10 / 10)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 1)) +offload=True +OPTIM_OFFLOAD=${OPTIM_OFFLOAD:-True} +gen_tp=8 +train_tp=${TP:-4} +train_pp=${PP:-8} + +EP=${EP:-4} +ETP=1 +CP=1 +optimizer_offload_fraction=${OFFLOAD_FRACTION:-1.} +last_layer=${LAST_LAYER:-10} + +project_name='verl-qwen3' +exp_name="235B-${NNODES}-pp${train_pp}-tp${train_tp}-ep${EP}-actor-length${actor_ppo_max_token_len}" +CKPTS_DIR=$RAY_DATA_HOME/ckpt/${project_name}/${exp_name} + +# TODO: support cuda graph for rollout by setting the following config + # actor_rollout_ref.rollout.cudagraph_capture_sizes=[1,2,4,8,16,32] + # actor_rollout_ref.rollout.enforce_eager=False + +python3 -m verl.trainer.main_ppo \ + --config-path=config \ + --config-name='ppo_megatron_trainer.yaml' \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.free_cache_engine=True \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.megatron.use_mbridge=True \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \ + +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${OPTIM_OFFLOAD} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.actor.megatron.context_parallel_size=${CP} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.clip_grad=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.nccl_timeout=1200 \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.ref.megatron.context_parallel_size=${CP} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_token_dispatcher_type="flex" \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_enable_deepep=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.account_for_loss_in_pipeline_split=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.account_for_embedding_in_pipeline_split=True \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','wandb'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=10 \ + trainer.save_freq=100 \ + trainer.total_epochs=10 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/examples/grpo_trainer/run_qwen3-32b_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen3-32b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..0ee01c43d1aa4529584569185403d9ad26c49277 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen3-32b_npu.sh @@ -0,0 +1,59 @@ +set -x + +project_name='GRPO-Qwen3' +exp_name='GRPO-Qwen3-32b-npu' +gen_tp=4 +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-32B"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/gsm8k/train.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/gsm8k/test.parquet"} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.train_batch_size=1024 \ + data.max_prompt_length=2048 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=${MODEL_PATH} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=4 \ + +actor_rollout_ref.actor.fsdp_config.mixed_precision.param_dtype=bf16 \ + +actor_rollout_ref.actor.fsdp_config.mixed_precision.reduce_dtype=bf16 \ + +actor_rollout_ref.actor.fsdp_config.mixed_precision.buffer_dtype=fp32 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=['console','tensorboard'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=4 \ + trainer.resume_from_path=checkpoints/ \ + trainer.save_freq=500 \ + trainer.test_freq=50 \ + trainer.total_epochs=50 \ + trainer.device=npu $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen3-8b.sh b/e2c/verl/examples/grpo_trainer/run_qwen3-8b.sh new file mode 100644 index 0000000000000000000000000000000000000000..a99b432d6abe46a7c62f69e47398ef99b10aa5c2 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen3-8b.sh @@ -0,0 +1,43 @@ +# Tested successfully on the hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.4-flashinfer0.2.2-cxx11abi0 image. +# It outperforms the Qwen2 7B base model by two percentage points on the test set of GSM8K. + +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen3-8B \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen3_8b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen3-8b_npu.sh b/e2c/verl/examples/grpo_trainer/run_qwen3-8b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..1d3a190b3f2cdd5d5a982e1b6d2bbeb6406bcf32 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen3-8b_npu.sh @@ -0,0 +1,59 @@ +set -x + +project_name='GRPO-Qwen3' +exp_name='GRPO-Qwen3-8B-npu' +gen_tp=2 +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-8B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.train_batch_size=256 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=${MODEL_PATH} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=10 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.default_local_dir=${CKPTS_DIR} \ + trainer.device=npu \ + trainer.resume_mode=auto \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.ref.fsdp_config.forward_prefetch=True \ + ++actor_rollout_ref.actor.entropy_from_logits_with_chunking=True \ + ++actor_rollout_ref.ref.entropy_from_logits_with_chunking=True \ + trainer.val_before_train=True \ + trainer.save_freq=5 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 \ No newline at end of file diff --git a/e2c/verl/examples/grpo_trainer/run_qwen3moe-30b_megatron_96gb.sh b/e2c/verl/examples/grpo_trainer/run_qwen3moe-30b_megatron_96gb.sh new file mode 100644 index 0000000000000000000000000000000000000000..6937db5fcfa2d270591499e137832f523d6f3fec --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_qwen3moe-30b_megatron_96gb.sh @@ -0,0 +1,195 @@ +set -x + +# tested in NNODES=1~4 * 96G H20 GPU +NNODES=${NNODES:-1} +NGPUS_PER_NODES=${NGPUS_PER_NODES:-8} + +project_name='DAPO-Qwen3-30b-MATH' +exp_name='DAPO-Qwen3-30b-MATH-megatron' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=128 +train_ppo_micro_batch_size_per_gpu=2 +infer_ppo_micro_batch_size_per_gpu=2 +# Paths +MODEL_PATH=Qwen/Qwen3-30B-A3B + +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +TRAIN_FILE=$RAY_DATA_HOME/dataset/dapo-math-17k.parquet +TEST_FILE=$RAY_DATA_HOME/dataset/aime-2024.parquet +TEST_FILE="['$aime24_test_path']" + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length))) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length))) +offload=True + +optimizer_offload_fraction=${OFFLOAD_FRACTION:-1.} + +COMMON_PP=${COMMON_PP:-1} +COMMON_VPP=${COMMON_VPP:-null} +COMMON_CP=${COMMON_CP:-1} +COMMON_TP=${COMMON_TP:-1} +COMMON_EP=${COMMON_EP:-8} +COMMON_ETP=${COMMON_ETP:-1} + +TRAIN_TP=${TRAIN_TP:-$COMMON_TP} +INFER_TP=${INFER_TP:-4} + +ACTOR_PP=${ACTOR_PP:-$COMMON_PP} +ACTOR_VPP=${ACTOR_VPP:-$COMMON_VPP} +ACTOR_CP=${ACTOR_CP:-$COMMON_CP} +ACTOR_TP=${ACTOR_TP:-$TRAIN_TP} +ACTOR_EP=${ACTOR_EP:-$COMMON_EP} +ACTOR_ETP=${ACTOR_ETP:-$COMMON_ETP} +ROLLOUT_TP=${ROLLOUT_TP:-$INFER_TP} +REF_PP=${REF_PP:-$COMMON_PP} +REF_VPP=${REF_VPP:-$COMMON_VPP} +REF_CP=${REF_CP:-$COMMON_CP} +REF_TP=${REF_TP:-$TRAIN_TP} +REF_EP=${REF_EP:-$COMMON_EP} +REF_ETP=${REF_ETP:-$COMMON_ETP} +CRITIC_PP=${CRITIC_PP:-$COMMON_PP} +CRITIC_VPP=${CRITIC_VPP:-$COMMON_VPP} +CRITIC_CP=${CRITIC_CP:-$COMMON_CP} +CRITIC_TP=${CRITIC_TP:-$TRAIN_TP} +CRITIC_EP=${CRITIC_EP:-$COMMON_EP} +CRITIC_ETP=${CRITIC_ETP:-$COMMON_ETP} +RM_PP=${RM_PP:-$COMMON_PP} +RM_VPP=${RM_VPP:-$COMMON_VPP} +RM_CP=${RM_CP:-$COMMON_CP} +RM_TP=${RM_TP:-$TRAIN_TP} +RM_EP=${RM_EP:-$COMMON_EP} +RM_ETP=${RM_ETP:-$COMMON_ETP} + +# install mbridge +# pip3 install git+https://github.com/ISEEKYAN/mbridge +USE_MBRIDGE=True +USE_DIST_CKPT=False + +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.model.use_fused_kernels=False \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_ppo_micro_batch_size_per_gpu} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.lr_decay_style='constant' \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \ + +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \ + actor_rollout_ref.actor.megatron.use_mbridge=$USE_MBRIDGE \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=$USE_DIST_CKPT \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${ACTOR_TP} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${ACTOR_PP} \ + actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=${ACTOR_VPP} \ + actor_rollout_ref.actor.megatron.context_parallel_size=${ACTOR_CP} \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=${ACTOR_EP} \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=${ACTOR_ETP} \ + +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_token_dispatcher_type="flex" \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_enable_deepep=True \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${infer_ppo_micro_batch_size_per_gpu} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${INFER_TP} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${infer_ppo_micro_batch_size_per_gpu} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${REF_TP} \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${REF_PP} \ + actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=${REF_VPP} \ + actor_rollout_ref.ref.megatron.context_parallel_size=${REF_CP} \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=${REF_EP} \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=${REF_ETP} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','wandb'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODES}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=10 \ + trainer.save_freq=100 \ + trainer.total_epochs=10 \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/examples/grpo_trainer/run_seed_oss_36b.sh b/e2c/verl/examples/grpo_trainer/run_seed_oss_36b.sh new file mode 100644 index 0000000000000000000000000000000000000000..37c4afb34312c4d77cb268b3c1f32592ad8a8ff7 --- /dev/null +++ b/e2c/verl/examples/grpo_trainer/run_seed_oss_36b.sh @@ -0,0 +1,48 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=64 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=ByteDance-Seed/Seed-OSS-36B-Base \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=8 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=2 \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.ref.strategy=fsdp2 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console"]' \ + trainer.project_name='verl_grpo_seed_oss_36b' \ + trainer.experiment_name='seed_oss_36b' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/e2c/verl/examples/ppo_trainer/README.md b/e2c/verl/examples/ppo_trainer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7b7261c5dd1a372f9883eeae7c540bec096d09b4 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/README.md @@ -0,0 +1,103 @@ +# Proximal Policy Optimization (PPO) + +Proximal Policy Optimization (PPO) is a family of policy gradient methods for reinforcement learning, proposed by OpenAI in 2017. PPO strikes a balance between simplicity, stability, and performance, making it one of the most widely used algorithms in modern RL applications, including large-scale language model fine-tuning. + +Traditional policy gradient methods like REINFORCE or Vanilla Policy Gradient suffer from: + +- High variance and sample inefficiency. +- Instability due to large policy updates. + +PPO addresses this problem using a clipped surrogate objective that avoids overly large updates without requiring second-order derivatives. + +For more technical details regarding PPO, we suggest reading the introduction in the [OpenAI spinning up tutorial](https://spinningup.openai.com/en/latest/algorithms/ppo.html), and the paper [Proximal Policy Optimization Algorithms](https://arxiv.org/abs/1707.06347). + +## Key Components + +- Actor-Critic Architecture: PPO requires both an actor model (policy) and a critic model (value function). This differs from other algorithms like GRPO and RLOO that don't require a critic model. + +- Generalized Advantage Estimation (GAE): PPO uses GAE for computing advantage values, which helps reduce variance in policy gradient estimates while maintaining low bias. + +- Clipped Surrogate Objective: The core of PPO is implemented through the clipped surrogate objective function that limits policy updates. + +## Configuration + +Note that all configs containing `micro_batch_size` are used to configure the maximum sample or token count per forward or backward pass to avoid GPU OOMs, whose value should not change algorithmic/convergence behavior. + +Most critic configs are similar to those of actors. Note that the critic model is omitted from the figure below. + +![image](https://github.com/user-attachments/assets/16aebad1-0da6-4eb3-806d-54a74e712c2d) + +- `data.train_batch_size`: The global batch size of prompts used to generate a set of sampled trajectories/rollouts. The number of responses/trajectories is `data.train_batch_size * actor_rollout.ref.rollout.n` + +- `actor_rollout_ref.actor.ppo_mini_batch_size`: The set of sampled trajectories is split into multiple mini-batches with batch_size=ppo_mini_batch_size for PPO actor updates. The ppo_mini_batch_size is a global size across all workers + +- `actor_rollout_ref.critic.ppo_mini_batch_size`: The set of sampled trajectories is split into multiple mini-batches with batch_size=ppo_mini_batch_size for PPO critic updates. The ppo_mini_batch_size is a global size across all workers + +- `actor_rollout_ref.actor.clip_ratio`: The PPO clip range. Default to 0.2 + +- `actor_rollout_ref.actor.ppo_epochs`: Number of epochs for PPO updates on one set of sampled trajectories for actor + +- `critic.ppo_epochs`: Number of epochs for PPO updates on one set of sampled trajectories for critic. Defaults to `actor_rollout_ref.actor.ppo_epochs` + +- `algorithm.gamma`: discount factor + +- `algorithm.lam`: The lambda term that trades off between bias and variance in the GAE estimator + +- `algorithm.adv_estimator`: Support gae, grpo, reinforce_plus_plus, reinforce_plus_plus_baseline, rloo, rloo_vectorized + +## Advanced Extensions + +### KL Divergence Control + +Options to prevent the policy from diverging too far from a reference policy. Two mechanisms are available: KL reward penalty and KL loss. For more technical details, see [Training language models to follow instructions with human feedback](https://arxiv.org/abs/2203.02155) + +Options to use KL loss for KL divergence control: + +- `actor_rollout_ref.actor.use_kl_loss`: to use kl loss in the actor. When used, we are not applying KL in the reward function. Default is False + +- `actor_rollout_ref.actor.kl_loss_coef`: The coefficient of kl loss. Default is 0.001. + +- `actor_rollout_ref.actor.kl_loss_type`: Support kl(k1), abs, mse(k2), low_var_kl(k3) and full. Appending "+" in the end (e.g., 'k1+' and 'k3+') would apply straight through to employ k2 for unbiased gradient estimation, regardless of the kl value estimation (see https://github.com/volcengine/verl/pull/2953#issuecomment-3162113848 for more details). How to calculate the kl divergence between actor and reference policy. See this blog post for detailed analysis: http://joschu.net/blog/kl-approx.html + +Options to use KL penalty in the reward: + +- `algorithm.use_kl_in_reward`: Whether to enable in-reward kl penalty. Default is False. + +- `algorithm.kl_penalty`: Support kl(k1), abs, mse(k2), low_var_kl(k3) and full. This defines the way to calculate the kl divergence between actor and reference policy. For specific options, refer to `kl_penalty` in core_algos.py. See this blog post for detailed analysis: http://joschu.net/blog/kl-approx.html + +- `algorithm.kl_ctrl.kl_coef`: The (initial) coefficient of in-reward kl_penalty. Default is 0.001. +- `algorithm.kl_ctrl.type`: 'fixed' for FixedKLController and 'adaptive' for AdaptiveKLController. +- `algorithm.kl_ctrl.horizon`: See source code of AdaptiveKLController for details. +- `algorithm.kl_ctrl.target_kl`: See source code of AdaptiveKLController for details. + +### Dual-clip PPO + +The Dual-Clip PPO introduces a approach by applying a lower bound to the policy ratio when the advantage is less than zero, when multiplied by a large raito, does not exceed a specified lower bound. + +![image](https://github.com/user-attachments/assets/fc232181-d8b0-4307-8dd2-4dc0a4c1c139) + +- `actor_rollout_ref.actor.clip_ratio_c`: lower bound of the value for Dual-clip PPO, defaults to 3.0 + +## Reference Example + +Qwen2.5 training log and commands: [link](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-0.5B-bsz256_2-prompt1024-resp512-0.567.log) + +```bash +bash run_gemma.sh + trainer.n_gpus_per_node=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + trainer.logger=console \ + critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + data.train_batch_size=256 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size=2 \ + critic.ppo_micro_batch_size=2 +``` + +Reference performance with verl v0.2: + +| Model | Method | Score | Link | +|-------------------------------|------------------|-------|------------------------------------------------------------------------------------------------| +| Qwen/Qwen2.5-0.5B-Instruct | pretrained model | 36.4 | [Qwen Blog](https://qwenlm.github.io/blog/qwen2.5-llm/) | +| Qwen/Qwen2.5-0.5B-Instruct | PPO | 56.7 | [PPO Command and Logs](https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-0.5B-bsz256_2-prompt1024-resp512-0.567.log) | diff --git a/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm.sh b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm.sh new file mode 100644 index 0000000000000000000000000000000000000000..6a93a75b4035cd21caa8c8b123ec1397b649de62 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm.sh @@ -0,0 +1,42 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=32 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='deepseek_llm_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=1 \ + trainer.use_legacy_worker_impl=auto \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_modelscope.sh b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_modelscope.sh new file mode 100644 index 0000000000000000000000000000000000000000..eb6dc79234a14152eb8583e58096e4d4fd8f0d04 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_modelscope.sh @@ -0,0 +1,42 @@ +set -x + +VERL_USE_MODELSCOPE=True \ +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=32 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='deepseek_llm_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=1 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_pfppo.sh b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_pfppo.sh new file mode 100644 index 0000000000000000000000000000000000000000..312c6b50b78272e1b0af06fa1b49fcf88f00639b --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_pfppo.sh @@ -0,0 +1,45 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + algorithm.use_pf_ppo=True \ + algorithm.pf_ppo.reweight_method=pow \ # ["pow", "max_min", "max_random"] + algorithm.pf_ppo.weight_pow=2.0 \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.rollout.n=5 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=32 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='deepseek_llm_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=1 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_sandbox_fusion.sh b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_sandbox_fusion.sh new file mode 100644 index 0000000000000000000000000000000000000000..69ee7b8bd76518dcb19aaca7d798d4a99a77e784 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_sandbox_fusion.sh @@ -0,0 +1,44 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + reward_model.sandbox_fusion.url='https://xxxxxxxxx.apigateway-cn-beijing.volceapi.com/run_code' \ + reward_model.sandbox_fusion.max_concurrent=128 \ + reward_model.reward_manager=prime \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/Eurus-2-RL-Data/train.parquet \ + data.val_files=$HOME/data/Eurus-2-RL-Data/validation.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=32 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_sandbox_fusion' \ + trainer.experiment_name='deepseek_llm_7b_function_sandbox_fusion' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=1 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_sp2.sh b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_sp2.sh new file mode 100644 index 0000000000000000000000000000000000000000..3cb8a852b5ffd3eea40781b421157d699434408b --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_deepseek7b_llm_sp2.sh @@ -0,0 +1,43 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=64 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + critic.optim.lr=1e-5 \ + critic.ulysses_sequence_parallel_size=2 \ + critic.model.use_remove_padding=True \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=64 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='deepseek_llm_7b_function_rm_sp2' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_deepseek_full_hh_rlhf.sh b/e2c/verl/examples/ppo_trainer/run_deepseek_full_hh_rlhf.sh new file mode 100644 index 0000000000000000000000000000000000000000..2944de647c47e2ce6d74d0da09cb613ffabfbf6c --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_deepseek_full_hh_rlhf.sh @@ -0,0 +1,41 @@ +set -x + +train_files=$HOME/data/full_hh_rlhf/rl/train.parquet +test_files=$HOME/data/full_hh_rlhf/rl/train.parquet # no use + +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=512 \ + data.max_prompt_length=128 \ + data.max_response_length=128 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + critic.optim.lr=1e-5 \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.ppo_micro_batch_size_per_gpu=4 \ + reward_model.enable=True \ + reward_model.megatron.tensor_model_parallel_size=4 \ + reward_model.model.path=deepseek-ai/deepseek-llm-7b-chat \ + reward_model.micro_batch_size_per_gpu=4 \ + reward_model.param_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_megatron_full_hh_rlhf_examples' \ + trainer.experiment_name='deepseek_llm_7b_model_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=100 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron.sh b/e2c/verl/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..a128aabf30abb87553b31e217c09d8f4166acb43 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron.sh @@ -0,0 +1,49 @@ +set -x + +# Example runnable on H20 * 8 + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + critic.optim.lr=1e-5 \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_ppo_gsm8k_math_examples' \ + trainer.experiment_name='deepseek_llm_7b_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=100 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron_nsys.sh b/e2c/verl/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron_nsys.sh new file mode 100644 index 0000000000000000000000000000000000000000..e467c3a5c3f97dad99e2345870239e99970f8a70 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron_nsys.sh @@ -0,0 +1,65 @@ +set -x + +# Example runnable on H20 * 8 + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files=${train_files:-"$gsm8k_train_path"} +test_files=${test_files:-"$gsm8k_test_path"} + +# Nsight profiling configuration +PROFILE_STEPS="[1]" # or [] or null +PROFILE_RANKS_ALL=False # or True +PROFILE_RANKS=[0,4] +DISCRETE=True # or True + +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.profiler.enable=True \ + actor_rollout_ref.actor.profiler.ranks=$PROFILE_RANKS \ + actor_rollout_ref.actor.profiler.all_ranks=$PROFILE_RANKS_ALL \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + critic.optim.lr=1e-5 \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.ppo_micro_batch_size_per_gpu=4 \ + critic.profiler.enable=True \ + critic.profiler.ranks=$PROFILE_RANKS \ + critic.profiler.all_ranks=$PROFILE_RANKS_ALL \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_ppo_gsm8k_math_examples' \ + trainer.experiment_name='deepseek_llm_7b_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=100 \ + trainer.total_training_steps=1 \ + global_profiler.tool=nsys \ + global_profiler.steps=$PROFILE_STEPS \ + global_profiler.global_tool_config.nsys.discrete=$DISCRETE $@ diff --git a/e2c/verl/examples/ppo_trainer/run_gemma.sh b/e2c/verl/examples/ppo_trainer/run_gemma.sh new file mode 100644 index 0000000000000000000000000000000000000000..b015275c13496ae2514db6c756114d76897c7f71 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_gemma.sh @@ -0,0 +1,40 @@ +set -x + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=google/gemma-2-2b-it \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=False \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=False \ + critic.model.path=google/gemma-2-2b-it \ + critic.model.enable_gradient_checkpointing=False \ + critic.ppo_micro_batch_size_per_gpu=4 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example' \ + trainer.experiment_name='gemma2b_function_rm' \ + trainer.n_gpus_per_node=2 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_moonlight16b_a3b_gsm8k_megatron.sh b/e2c/verl/examples/ppo_trainer/run_moonlight16b_a3b_gsm8k_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..9e1d40576f0a4b8fb97cb5e23260c5c3020c902b --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_moonlight16b_a3b_gsm8k_megatron.sh @@ -0,0 +1,106 @@ +set -x + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + + +# 0. download the model +huggingface-cli download moonshotai/Moonlight-16B-A3B-Instruct + +# 1. convert the model to mcore format +# change the HF_MODEL_PATH and DIST_CKPT_PATH to your own path +HF_MODEL_PATH=/data/models/moonshotai/Moonlight-16B-A3B-Instruct +DIST_CKPT_PATH=/data/mcore_ckpt/Moonlight-16B-A3B-Instruct +python scripts/converter_hf_to_mcore.py --hf_model_path $HF_MODEL_PATH --output_path $DIST_CKPT_PATH + + +# 2. run the script +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +train_files=$gsm8k_train_path +test_files=$gsm8k_test_path + +ALL_OFFLOAD=${ALL_OFFLOAD:-False} +COMMON_PARAM_OFFLOAD=${COMMON_PARAM_OFFLOAD:-$ALL_OFFLOAD} +COMMON_GRAD_OFFLOAD=${COMMON_GRAD_OFFLOAD:-$ALL_OFFLOAD} +COMMON_OPTIMIZER_OFFLOAD=${COMMON_OPTIMIZER_OFFLOAD:-$ALL_OFFLOAD} + +ACTOR_PARAM_OFFLOAD=${ACTOR_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +ACTOR_GRAD_OFFLOAD=${ACTOR_GRAD_OFFLOAD:-$COMMON_GRAD_OFFLOAD} +ACTOR_OPTIMIZER_OFFLOAD=${ACTOR_OPTIMIZER_OFFLOAD:-$COMMON_OPTIMIZER_OFFLOAD} +REF_PARAM_OFFLOAD=${REF_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +CRITIC_PARAM_OFFLOAD=${CRITIC_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +CRITIC_GRAD_OFFLOAD=${CRITIC_GRAD_OFFLOAD:-$COMMON_GRAD_OFFLOAD} +CRITIC_OPTIMIZER_OFFLOAD=${CRITIC_OPTIMIZER_OFFLOAD:-$COMMON_OPTIMIZER_OFFLOAD} +RM_PARAM_OFFLOAD=${RM_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} + + +NODES=4 +PP=2 +TP=8 +EP=8 +ETP=1 +VLLM_TP=4 + +# RAY_ADDRESS='auto' ray job submit --working-dir . -- +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.trust_remote_code=True \ + actor_rollout_ref.model.path=$LLM \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + critic.optim.lr=1e-5 \ + critic.model.path=$LLM \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_megatron_gsm8k_examples' \ + trainer.experiment_name='moonlight_16b_a3b_instruct_1node' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=$NODES \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + actor_rollout_ref.model.trust_remote_code=True \ + critic.model.trust_remote_code=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_last_pipeline_stage=13 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$VLLM_TP \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=$PP \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=$PP \ + critic.megatron.pipeline_model_parallel_size=$PP \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=$TP \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=$TP \ + critic.megatron.tensor_model_parallel_size=$TP \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=$EP \ + critic.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=$ETP \ + critic.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.actor.megatron.param_offload=${ACTOR_PARAM_OFFLOAD} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${ACTOR_OPTIMIZER_OFFLOAD} \ + actor_rollout_ref.actor.megatron.grad_offload=${ACTOR_GRAD_OFFLOAD} \ + actor_rollout_ref.ref.megatron.param_offload=${REF_PARAM_OFFLOAD} \ + critic.megatron.param_offload=${CRITIC_PARAM_OFFLOAD} \ + critic.megatron.optimizer_offload=${CRITIC_OPTIMIZER_OFFLOAD} \ + critic.megatron.grad_offload=${CRITIC_GRAD_OFFLOAD} \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \ + critic.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.actor.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + actor_rollout_ref.ref.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + critic.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + trainer.val_before_train=False \ + trainer.total_epochs=100 $@ + \ No newline at end of file diff --git a/e2c/verl/examples/ppo_trainer/run_qwen1.5_moe_a2.7b-gsm8k_megatron.sh b/e2c/verl/examples/ppo_trainer/run_qwen1.5_moe_a2.7b-gsm8k_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..b82ea1d4373d33df10e604d92392f6b16780f3db --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen1.5_moe_a2.7b-gsm8k_megatron.sh @@ -0,0 +1,73 @@ +set -x + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +# 0. download the model +#huggingface-cli download Qwen/Qwen1.5-MoE-A2.7B-Chat + +# 1. convert the model to mcore format +# change the HF_MODEL_PATH and DIST_CKPT_PATH to your own path +HF_MODEL_PATH=/data/models/Qwen/Qwen1.5-MoE-A2.7B-Chat +DIST_CKPT_PATH=/data/mcore_ckpt/Qwen1.5-MoE-A2.7B-Chat +python scripts/converter_hf_to_mcore.py --hf_model_path $HF_MODEL_PATH --output_path $DIST_CKPT_PATH + +# 2. run the script +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +train_files=$gsm8k_train_path +test_files=$gsm8k_test_path + +NODES=4 +PP=2 +TP=4 +CP=1 +VLLM_TP=4 + +# RAY_ADDRESS='auto' ray job submit --working-dir . -- +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=$HF_MODEL_PATH \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=$TP \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=$PP \ + actor_rollout_ref.actor.megatron.context_parallel_size=$CP \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.actor.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=$TP \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=$PP \ + actor_rollout_ref.ref.megatron.context_parallel_size=$CP \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.ref.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$VLLM_TP \ + critic.optim.lr=1e-5 \ + critic.model.path=$HF_MODEL_PATH \ + critic.ppo_micro_batch_size_per_gpu=4 \ + critic.megatron.tensor_model_parallel_size=$TP \ + critic.megatron.pipeline_model_parallel_size=$PP \ + critic.megatron.context_parallel_size=$CP \ + critic.megatron.use_dist_checkpointing=True \ + critic.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_megatron_gsm8k_examples' \ + trainer.experiment_name='qwen1.5_moe_nochat' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=$NODES \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=100 $@ + \ No newline at end of file diff --git a/e2c/verl/examples/ppo_trainer/run_qwen2-7b_math_gsm8k_megatron.sh b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_math_gsm8k_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..934d6e19b4edd9b4001a7a6afcff59d99646eccf --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_math_gsm8k_megatron.sh @@ -0,0 +1,47 @@ +set -x + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2-7B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_ppo_gsm8k_math_examples' \ + trainer.experiment_name='qwen2_7b_megatron' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=100 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm.sh b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm.sh new file mode 100644 index 0000000000000000000000000000000000000000..57b7bd7524b17114233f7ed1b82939f79366dbcd --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm.sh @@ -0,0 +1,71 @@ +# Discliamer: the model used in the script is only for academic purpose. +set -x + +# Data preparation scripts are available in ``examples/data_preprocess``. +# Example usage: +# +# python3 examples/data_preprocess/math_dataset.py --local_dir ~/data/math +# python3 examples/data_preprocess/gsm8k.py --local_save_dir ~/data/gsm8k + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + + +# prepare model ckpt +huggingface-cli download Qwen/Qwen2-7B-Instruct --local-dir $HOME/models/Qwen2-7B-Instruct & +huggingface-cli download sfairXC/FsfairX-LLaMA3-RM-v0.1 --local-dir $HOME/models/FsfairX-LLaMA3-RM-v0.1 & +wait + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path="$HOME/models/Qwen2-7B-Instruct" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.optim.lr_warmup_steps_ratio=0.05 \ + critic.model.path="$HOME/models/Qwen2-7B-Instruct" \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=32 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + reward_model.enable=True \ + reward_model.model.path="$HOME/models/FsfairX-LLaMA3-RM-v0.1" \ + reward_model.model.use_remove_padding=True \ + reward_model.model.fsdp_config.param_offload=True \ + reward_model.micro_batch_size_per_gpu=32 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example' \ + trainer.val_before_train=False \ + trainer.experiment_name='Qwen2-7B-Instruct_hybrid_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance.sh b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance.sh new file mode 100644 index 0000000000000000000000000000000000000000..e0ddc01e75eafa1c9003a6a415622d44688f79d9 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance.sh @@ -0,0 +1,60 @@ +set -x + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=4096 \ + data.max_prompt_length=4096 \ + data.max_response_length=4096 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=512 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen2-7B-Instruct \ + critic.model.enable_gradient_checkpointing=True \ + critic.use_dynamic_bsz=True \ + critic.ppo_max_token_len_per_gpu=98304 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + reward_model.enable=True \ + reward_model.model.path=sfairXC/FsfairX-LLaMA3-RM-v0.1\ + reward_model.model.use_remove_padding=True \ + reward_model.model.fsdp_config.param_offload=True \ + reward_model.micro_batch_size_per_gpu=32 \ + reward_model.use_dynamic_bsz=True \ + reward_model.forward_max_token_len_per_gpu=98304 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='qwen2-7b_hybrid_rm_bsz8k_p4k_r4k_seq_packing' \ + trainer.n_gpus_per_node=8 \ + trainer.val_before_train=False \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance_fused_kernels.sh b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance_fused_kernels.sh new file mode 100644 index 0000000000000000000000000000000000000000..7e0a335efe20465fe19b9c1784d0e1e360af405c --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance_fused_kernels.sh @@ -0,0 +1,64 @@ +set -x + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +FUSED_KERNEL_BACKEND=triton # or 'torch' for torch backend + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=4096 \ + data.max_prompt_length=4096 \ + data.max_response_length=4096 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.model.fused_kernel_options.impl_backend=$FUSED_KERNEL_BACKEND \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=512 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen2-7B-Instruct \ + critic.model.enable_gradient_checkpointing=True \ + critic.use_dynamic_bsz=True \ + critic.ppo_max_token_len_per_gpu=98304 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + reward_model.enable=True \ + reward_model.model.path=sfairXC/FsfairX-LLaMA3-RM-v0.1\ + reward_model.model.use_remove_padding=True \ + reward_model.model.fsdp_config.param_offload=True \ + reward_model.micro_batch_size_per_gpu=32 \ + reward_model.use_dynamic_bsz=True \ + reward_model.forward_max_token_len_per_gpu=98304 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='qwen2-7b_hybrid_rm_bsz8k_p4k_r4k_seq_packing_fused_kernel' \ + trainer.n_gpus_per_node=8 \ + trainer.val_before_train=False \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance_nsys.sh b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance_nsys.sh new file mode 100644 index 0000000000000000000000000000000000000000..0acfe43e8628d9b86b3c1e6b45ae6c91684a6bc2 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance_nsys.sh @@ -0,0 +1,81 @@ +set -x + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files=${train_files:-"$gsm8k_train_path"} +test_files=${test_files:-"$gsm8k_test_path"} + +PROFILE_STEPS="[1,2,5]" # or [] or null +PROFILE_RANKS_ALL=False # or True +PROFILE_RANKS=[0,4] +DISCRETE=True # or True + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=4096 \ + data.max_prompt_length=4096 \ + data.max_response_length=4096 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=512 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=12000 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.profiler.enable=True \ + actor_rollout_ref.actor.profiler.ranks=$PROFILE_RANKS \ + actor_rollout_ref.actor.profiler.all_ranks=$PROFILE_RANKS_ALL \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen2-7B-Instruct \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=2 \ + critic.use_dynamic_bsz=True \ + critic.ppo_max_token_len_per_gpu=98304 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + critic.profiler.enable=True \ + critic.profiler.ranks=$PROFILE_RANKS \ + critic.profiler.all_ranks=$PROFILE_RANKS_ALL \ + reward_model.enable=True \ + reward_model.model.path=sfairXC/FsfairX-LLaMA3-RM-v0.1\ + reward_model.model.use_remove_padding=True \ + reward_model.model.fsdp_config.param_offload=True \ + reward_model.micro_batch_size_per_gpu=32 \ + reward_model.use_dynamic_bsz=True \ + reward_model.forward_max_token_len_per_gpu=98304 \ + reward_model.profiler.enable=True \ + reward_model.profiler.ranks=$PROFILE_RANKS \ + reward_model.profiler.all_ranks=$PROFILE_RANKS_ALL \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='qwen2-7b_hybrid_rm_bsz8k_p4k_r4k_seq_packing' \ + trainer.n_gpus_per_node=8 \ + trainer.val_before_train=False \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=15 \ + trainer.total_training_steps=6 \ + global_profiler.profile_continuous_steps=True \ + global_profiler.tool=nsys \ + global_profiler.steps=$PROFILE_STEPS \ + global_profiler.global_tool_config.nsys.discrete=$DISCRETE $@ diff --git a/e2c/verl/examples/ppo_trainer/run_qwen2-7b_seq_balance.sh b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_seq_balance.sh new file mode 100644 index 0000000000000000000000000000000000000000..9717e5f942ba3d15d532ef5c14dd5b2105dd1007 --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_seq_balance.sh @@ -0,0 +1,60 @@ +set -x + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +# For async rollout mode, dataset should return raw chat. +rollout_mode="sync" +if [ "$rollout_mode" = "async" ]; then + return_raw_chat="True" +fi + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.return_raw_chat=$return_raw_chat \ + data.train_batch_size=4096 \ + data.max_prompt_length=4096 \ + data.max_response_length=4096 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=512 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=$rollout_mode \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen2-7B-Instruct \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_max_token_len_per_gpu=98304 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='qwen2-7b_function_rm_bsz8k_p4k_r4k_seq_packing' \ + trainer.n_gpus_per_node=8 \ + trainer.val_before_train=False \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_qwen2-7b_sglang_seq_balance.sh b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_sglang_seq_balance.sh new file mode 100644 index 0000000000000000000000000000000000000000..5108e8b5dd92f53d6c822528d3be50983c6044ff --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen2-7b_sglang_seq_balance.sh @@ -0,0 +1,51 @@ +set -x + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=4096 \ + data.max_prompt_length=4096 \ + data.max_response_length=4096 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=512 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen2-7B-Instruct \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_max_token_len_per_gpu=98304 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='qwen2-7b_function_rm_bsz8k_p4k_r4k_seq_packing' \ + trainer.n_gpus_per_node=8 \ + trainer.val_before_train=False \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_qwen2.5-32b.sh b/e2c/verl/examples/ppo_trainer/run_qwen2.5-32b.sh new file mode 100644 index 0000000000000000000000000000000000000000..58037658500a443a35424158af3d40fc9b87512c --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen2.5-32b.sh @@ -0,0 +1,50 @@ +set -x + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-32B-Instruct \ + actor_rollout_ref.model.enable_gradient_checkpointing=False \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen2.5-32B-Instruct \ + critic.model.enable_gradient_checkpointing=False \ + critic.ppo_micro_batch_size_per_gpu=8 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example' \ + trainer.experiment_name='Qwen2.5-32B-Instruct_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=4 \ + trainer.save_freq=20 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/ppo_trainer/run_qwen3-8b_npu.sh b/e2c/verl/examples/ppo_trainer/run_qwen3-8b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..40fb751e62ca0f8e2cf66250cb0a4c9ef03d58ed --- /dev/null +++ b/e2c/verl/examples/ppo_trainer/run_qwen3-8b_npu.sh @@ -0,0 +1,55 @@ +set -x + +export VLLM_USE_V1=1 + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/dapo-math-17k.parquet \ + data.val_files=$HOME/data/dapo-math-17k.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=2000 \ + data.max_response_length=12000 \ + data.shuffle=False \ + actor_rollout_ref.model.path=Qwen/Qwen3-8B \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.rollout.max_num_batched_tokens=14000 \ + actor_rollout_ref.rollout.max_num_seqs=64 \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.enforce_eager=False \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen3-8B \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=1 \ + critic.ulysses_sequence_parallel_size=2 \ + critic.model.fsdp_config.param_offload=True \ + critic.model.fsdp_config.optimizer_offload=True \ + critic.use_dynamic_bsz=True \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_example_dapo_math_17k' \ + trainer.experiment_name='qwen3_8b_fsdp' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=-1 \ + trainer.val_before_train=False \ + trainer.device=npu \ + trainer.max_actor_ckpt_to_keep=1 \ + trainer.max_critic_ckpt_to_keep=1 \ + trainer.total_training_steps=100 $@ \ No newline at end of file diff --git a/e2c/verl/examples/ray/tutorial.ipynb b/e2c/verl/examples/ray/tutorial.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ca176af0f7940f705281de7ce707d1fa27238c02 --- /dev/null +++ b/e2c/verl/examples/ray/tutorial.ipynb @@ -0,0 +1,963 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0ddc582b", + "metadata": {}, + "source": [ + "# VeRL Ray API Tutorial" + ] + }, + { + "cell_type": "markdown", + "id": "71fe3b94", + "metadata": {}, + "source": [ + "## Chapter 1: Ray Basics" + ] + }, + { + "cell_type": "code", + "execution_count": 144, + "id": "1347d381", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 145, + "id": "e75b9d44", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "import ray\n", + "import torch\n", + "\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 146, + "id": "2e90ae00", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-11-01 17:27:19,132\tINFO worker.py:1752 -- Started a local Ray instance.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9cc9d2ccbdfb48918c8fd6cd13a0807a", + "version_major": 2, + "version_minor": 0 + }, + "text/html": [ + "
\n", + "
\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Python version:3.9.2
Ray version:2.10.0
\n", + "\n", + "
\n", + "
\n" + ], + "text/plain": [ + "RayContext(dashboard_url='', python_version='3.9.2', ray_version='2.10.0', ray_commit='09abba26b5bf2707639bb637c208d062a47b46f6')" + ] + }, + "execution_count": 146, + "metadata": {}, + "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(GPUAccumulator pid=224400)\u001b[0m rank 0, value: tensor([1.], device='cuda:0')\n", + "\u001b[36m(GPUAccumulator pid=225234)\u001b[0m rank 2, value: tensor([3.], device='cuda:0')\n", + "\u001b[36m(GPUAccumulator pid=225607)\u001b[0m rank 0, value: tensor([2.], device='cuda:0')\n", + "\u001b[36m(GPUAccumulator pid=226423)\u001b[0m rank 1, value: tensor([3.], device='cuda:0')\n", + "\u001b[36m(GPUAccumulator pid=226857)\u001b[0m rank 3, value: tensor([6.], device='cuda:0')\n", + "\u001b[36m(GPUAccumulatorDecorator pid=227475)\u001b[0m 10\n", + "\u001b[36m(GPUAccumulatorDecorator pid=227475)\u001b[0m rank 0, value: tensor([10.], device='cuda:0')\n", + "\u001b[36m(GPUAccumulatorDecorator pid=227655)\u001b[0m rank 1, value: tensor([11.], device='cuda:0')\n" + ] + } + ], + "source": [ + "# Build a local ray cluster. The head node and worker node are on this machine\n", + "ray.init()" + ] + }, + { + "cell_type": "markdown", + "id": "a127e4e4", + "metadata": {}, + "source": [ + "Implement an Accumulator class." + ] + }, + { + "cell_type": "code", + "execution_count": 147, + "id": "20e7b9a3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "@ray.remote\n", + "class Accumulator:\n", + " def __init__(self):\n", + " self.value = 0\n", + "\n", + " def add(self, x):\n", + " self.value += x\n", + "\n", + " def get_value(self):\n", + " return self.value" + ] + }, + { + "cell_type": "code", + "execution_count": 148, + "id": "3b80098c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Instantiate an accumulator. Accumulator can be viewed as a process, acting as an RPC service.\n", + "accumulator = Accumulator.remote()" + ] + }, + { + "cell_type": "code", + "execution_count": 149, + "id": "b14b1009", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\n" + ] + } + ], + "source": [ + "value_ref = accumulator.get_value.remote() # Check the current value. Note that this function returns immediately and does not actually wait for the remote execution to complete.\n", + "# Get the value\n", + "value = ray.get(value_ref)\n", + "print(value)" + ] + }, + { + "cell_type": "code", + "execution_count": 150, + "id": "513a84b3", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10\n" + ] + } + ], + "source": [ + "# Accumulate, then check the result.\n", + "accumulator.add.remote(10) # Similarly, the 'add' here will return immediately.\n", + "new_value = ray.get(accumulator.get_value.remote())\n", + "print(new_value)" + ] + }, + { + "cell_type": "markdown", + "id": "3c332fe0", + "metadata": {}, + "source": [ + "## Chapter 2: Resource Pool and RayWorkerGroup\n", + "In the previous example, it was a simple single-process worker. \n", + "In this example, we implement a worker with a GPU and form a RayWorkerGroup. Within this RayWorkerGroup, we implement a simple operation of an accumulator." + ] + }, + { + "cell_type": "code", + "execution_count": 151, + "id": "04229afb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from verl.single_controller.base import Worker\n", + "from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup, merge_resource_pool" + ] + }, + { + "cell_type": "code", + "execution_count": 152, + "id": "0d0dbd58", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "resource_pool = RayResourcePool([4], use_gpu=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 153, + "id": "68f6838a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "@ray.remote\n", + "class GPUAccumulator(Worker):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + " # The initial value of each rank is the same as the rank\n", + " self.value = torch.zeros(size=(1,), device=\"cuda\") + self.rank\n", + "\n", + " def add(self, x):\n", + " self.value += x\n", + " print(f\"rank {self.rank}, value: {self.value}\")\n", + " return self.value.cpu()" + ] + }, + { + "cell_type": "code", + "execution_count": 154, + "id": "23aad8fe", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tensor([1.]), tensor([2.]), tensor([3.]), tensor([4.])]\n" + ] + } + ], + "source": [ + "# Each worker's initial value is its rank, and then each rank's value is incremented by 1, so the values obtained on each rank are [1, 2, 3, 4]\n", + "class_with_args = RayClassWithInitArgs(cls=GPUAccumulator)\n", + "worker_group = RayWorkerGroup(resource_pool, class_with_args)\n", + "print(worker_group.execute_all_sync(\"add\", x=[1, 1, 1, 1]))" + ] + }, + { + "cell_type": "markdown", + "id": "e6705284", + "metadata": {}, + "source": [ + "The principle of parameter passing: The input parameter is a list of length world_size, where each element in the list is dispatched respectively to each worker in the RayWorkerGroup. \n", + "The return parameter is also a list, corresponding to the return value of each worker." + ] + }, + { + "cell_type": "markdown", + "id": "d25c2412", + "metadata": {}, + "source": [ + "### GPU Resource Sharing" + ] + }, + { + "cell_type": "markdown", + "id": "f74f6d24", + "metadata": {}, + "source": [ + "RayWorkerGroups mapped to the same resource pool share the GPU. In this example, we implement three resource pools: the first occupies 4 GPUs, the second also occupies 4 GPUs, and the last occupies all 8 GPUs. Among them, the first resource pool reuses the resource pool mentioned above." + ] + }, + { + "cell_type": "code", + "execution_count": 155, + "id": "49f9c06f", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Create a new resource pool and then merge the newly created resource pool with the previous one.\n", + "resource_pool_1 = RayResourcePool([4], use_gpu=True, name_prefix=\"a\")\n", + "resource_pool_merge = merge_resource_pool(resource_pool, resource_pool_1)" + ] + }, + { + "cell_type": "code", + "execution_count": 156, + "id": "05c2e305", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Establish a RayWorkerGroup on the newly created resource pool.\n", + "worker_group_1 = RayWorkerGroup(resource_pool_1, class_with_args)\n", + "worker_group_merge = RayWorkerGroup(resource_pool_merge, class_with_args)" + ] + }, + { + "cell_type": "code", + "execution_count": 157, + "id": "6b9b13f4", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tensor([2.]), tensor([3.]), tensor([4.]), tensor([5.])]\n" + ] + } + ], + "source": [ + "# Run 'add' on the second set of 4 GPUs; the result should be [2, 3, 4, 5].\n", + "output_1 = worker_group_1.execute_all_sync(\"add\", x=[2, 2, 2, 2])\n", + "print(output_1)" + ] + }, + { + "cell_type": "code", + "execution_count": 158, + "id": "d856d030", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tensor([3.]), tensor([4.]), tensor([5.]), tensor([6.]), tensor([7.]), tensor([8.]), tensor([9.]), tensor([10.])]\n" + ] + } + ], + "source": [ + "# Run 'add' on the merged set of 8 GPUs; the result should be [3, 4, 5, 6, 7, 8, 9, 10].\n", + "output_merge = worker_group_merge.execute_all_sync(\"add\", x=[3, 3, 3, 3, 3, 3, 3, 3])\n", + "print(output_merge)" + ] + }, + { + "cell_type": "code", + "execution_count": 159, + "id": "33a4628c", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4 4 8\n" + ] + } + ], + "source": [ + "print(worker_group.world_size, worker_group_1.world_size, worker_group_merge.world_size)" + ] + }, + { + "cell_type": "markdown", + "id": "3df19d13", + "metadata": {}, + "source": [ + "## Chapter 3: Data Dispatch, Execution and Collection" + ] + }, + { + "cell_type": "markdown", + "id": "acb22d9d", + "metadata": {}, + "source": [ + "In the above example, we used the `execute_all_sync` function in the RayWorkerGroup to dispatch data from the driver to each worker. This is very inconvenient for coding. \n", + "In this chapter, we use the form of function decorators to allow RayWorkerGroup to directly call functions written in the Worker, and to greatly simplify parameter passing." + ] + }, + { + "cell_type": "code", + "execution_count": 160, + "id": "35237432", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from verl.single_controller.base.decorator import Dispatch, Execute, register" + ] + }, + { + "cell_type": "code", + "execution_count": 161, + "id": "88b8ba3b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "@ray.remote\n", + "class GPUAccumulatorDecorator(Worker):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + " # The initial value of each rank is the same as the rank\n", + " self.value = torch.zeros(size=(1,), device=\"cuda\") + self.rank\n", + "\n", + " # map from a single input to all the worker\n", + " @register(Dispatch.ONE_TO_ALL)\n", + " def add(self, x):\n", + " print(x)\n", + " self.value = self.value + x\n", + " print(f\"rank {self.rank}, value: {self.value}\")\n", + " return self.value.cpu()" + ] + }, + { + "cell_type": "code", + "execution_count": 162, + "id": "eddaa043", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "class_with_args = RayClassWithInitArgs(cls=GPUAccumulatorDecorator)\n", + "gpu_accumulator_decorator = RayWorkerGroup(resource_pool_merge, class_with_args)" + ] + }, + { + "cell_type": "code", + "execution_count": 163, + "id": "10087c91", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tensor([10.]), tensor([11.]), tensor([12.]), tensor([13.]), tensor([14.]), tensor([15.]), tensor([16.]), tensor([17.])]\n" + ] + } + ], + "source": [ + "# As we can see, 10 is automatically dispatched to each Worker in this RayWorkerGroup.\n", + "print(gpu_accumulator_decorator.add(x=10))" + ] + }, + { + "cell_type": "markdown", + "id": "540ee6ad", + "metadata": {}, + "source": [ + "### Custom Dispatch, Collection\n", + "Users can customize `dispatch` and `collection` function. You only need to write the `dispatch_fn` and `collect_fn` functions yourself. We also support executing RPC only on rank_zero, with specific examples provided below." + ] + }, + { + "cell_type": "code", + "execution_count": 164, + "id": "8e041270", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from verl.single_controller.base.decorator import Dispatch, collect_all_to_all, register" + ] + }, + { + "cell_type": "code", + "execution_count": 165, + "id": "43b5be31", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def two_to_all_dispatch_fn(worker_group, *args, **kwargs):\n", + " \"\"\"\n", + " Assume the input is a list of 2. Duplicate the input interleaved and pass to each worker.\n", + " \"\"\"\n", + " for arg in args:\n", + " assert len(arg) == 2\n", + " for i in range(worker_group.world_size - 2):\n", + " arg.append(arg[i % 2])\n", + " for k, v in kwargs.items():\n", + " assert len(v) == 2\n", + " for i in range(worker_group.world_size - 2):\n", + " v.append(v[i % 2])\n", + " return args, kwargs\n", + "\n", + "\n", + "@ray.remote\n", + "class TestActor(Worker):\n", + " # TODO: pass *args and **kwargs is bug prone and not very convincing\n", + " def __init__(self, x) -> None:\n", + " super().__init__()\n", + " self._x = x\n", + "\n", + " def foo(self, y):\n", + " return self._x + y\n", + "\n", + " @register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.RANK_ZERO)\n", + " def foo_rank_zero(self, x, y):\n", + " return self._x + y + x\n", + "\n", + " @register(dispatch_mode={\"dispatch_fn\": two_to_all_dispatch_fn, \"collect_fn\": collect_all_to_all})\n", + " def foo_custom(self, x, y):\n", + " return self._x + y + x" + ] + }, + { + "cell_type": "code", + "execution_count": 166, + "id": "83ec6609", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "class_with_args = RayClassWithInitArgs(cls=TestActor, x=2)\n", + "worker_group = RayWorkerGroup(resource_pool, class_with_args)" + ] + }, + { + "cell_type": "code", + "execution_count": 167, + "id": "62c58d8a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "output_ref = worker_group.foo_custom(x=[1, 2], y=[5, 6])\n", + "assert output_ref == [8, 10, 8, 10]\n", + "\n", + "output_ref = worker_group.foo_rank_zero(x=1, y=2)\n", + "assert output_ref == 5" + ] + }, + { + "cell_type": "code", + "execution_count": 168, + "id": "14689353", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8\n" + ] + } + ], + "source": [ + "print(gpu_accumulator_decorator.world_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 169, + "id": "2c80bbf4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Shutdown ray cluster\n", + "ray.shutdown()" + ] + }, + { + "cell_type": "markdown", + "id": "a5c8151c", + "metadata": {}, + "source": [ + "## Chapter 4: NVMegatronRayWorkerGroup" + ] + }, + { + "cell_type": "markdown", + "id": "cd5680e9", + "metadata": {}, + "source": [ + "Due to the Ray issue, we can only support max_colocate_count=1 in RayResourcePool for now. \n", + "This means that each GPU can only have one process.\n", + "We can support max_colocate > 1 when applying this pull request: https://github.com/ray-project/ray/pull/44385" + ] + }, + { + "cell_type": "markdown", + "id": "92724419", + "metadata": {}, + "source": [ + "Therefore, we need to restart the ray and initialize a new resource_pool to demonstrate the **NVMegatronRayWorkerGroup**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b038538", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Build a local ray cluster. The head node and worker node are on this machine\n", + "ray.init()" + ] + }, + { + "cell_type": "markdown", + "id": "ebfd8798", + "metadata": {}, + "source": [ + "Finally, we implement a `NVMegatronRayWorkerGroup`, within which we create a Megatron and then run a tensor parallel (tp) split Llama mlp layer. Here, we use a complex dispatch mode, `Megatron_COMPUTE`. This dispatch mode assumes that user passes the data partitioned by DP dimension. The data is dispatched to all tp/pp ranks within the same dp group, and ultimately only collects output data from tp=0 and the last pp. In this way, for users that only write code on the driver, the Megatron behind the RPC becomes transparent." + ] + }, + { + "cell_type": "code", + "execution_count": 171, + "id": "5a032154", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/tiger/Megatron-LM\n", + "/opt/tiger/Megatron-LM/megatron/__init__.py\n" + ] + } + ], + "source": [ + "import sys\n", + "\n", + "current_pythonpath = os.environ.get(\"PYTHONPATH\", \"\")\n", + "\n", + "new_path = \"/opt/tiger/Megatron-LM\"\n", + "\n", + "new_pythonpath = f\"{new_path}:{current_pythonpath}\" if current_pythonpath else new_path\n", + "\n", + "os.environ[\"PYTHONPATH\"] = new_pythonpath\n", + "\n", + "print(new_path)\n", + "sys.path.append(new_path)\n", + "\n", + "import megatron\n", + "\n", + "print(megatron.__file__)" + ] + }, + { + "cell_type": "code", + "execution_count": 172, + "id": "8c84cd5a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from megatron.core import parallel_state as mpu\n", + "from omegaconf import OmegaConf\n", + "\n", + "from verl.single_controller.base.decorator import Dispatch, Execute, register\n", + "from verl.single_controller.base.megatron.worker import MegatronWorker\n", + "from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup\n", + "from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup" + ] + }, + { + "cell_type": "code", + "execution_count": 173, + "id": "1b1debcc", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "resource_pool = RayResourcePool([4], use_gpu=True, max_colocate_count=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 174, + "id": "bccbe081", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "@ray.remote\n", + "class MLPLayerWorker(MegatronWorker):\n", + " def __init__(self):\n", + " super().__init__()\n", + " rank = int(os.environ[\"LOCAL_RANK\"])\n", + " torch.distributed.init_process_group(backend=\"nccl\")\n", + " torch.cuda.set_device(rank)\n", + "\n", + " mpu.initialize_model_parallel(\n", + " tensor_model_parallel_size=4,\n", + " pipeline_model_parallel_size=1,\n", + " virtual_pipeline_model_parallel_size=None,\n", + " pipeline_model_parallel_split_rank=None,\n", + " use_sharp=False,\n", + " context_parallel_size=1,\n", + " expert_model_parallel_size=1,\n", + " nccl_communicator_config_path=None,\n", + " )\n", + " from megatron.core import tensor_parallel\n", + "\n", + " tensor_parallel.model_parallel_cuda_manual_seed(10)\n", + "\n", + " @register(Dispatch.ONE_TO_ALL)\n", + " def init_model(self, config):\n", + " from omegaconf import OmegaConf\n", + "\n", + " from verl.models.llama.megatron.layers import ParallelLlamaMLP\n", + " from verl.utils.megatron_utils import init_model_parallel_config\n", + "\n", + " megatron_config = OmegaConf.create(\n", + " {\n", + " \"sequence_parallel\": False,\n", + " \"param_dtype\": \"fp32\",\n", + " \"tensor_model_parallel_size\": mpu.get_tensor_model_parallel_world_size(),\n", + " \"pipeline_model_parallel_rank\": mpu.get_pipeline_model_parallel_rank(),\n", + " \"pipeline_model_parallel_size\": mpu.get_pipeline_model_parallel_world_size(),\n", + " \"virtual_pipeline_model_parallel_rank\": mpu.get_virtual_pipeline_model_parallel_rank(),\n", + " \"virtual_pipeline_model_parallel_size\": mpu.get_virtual_pipeline_model_parallel_world_size(),\n", + " }\n", + " )\n", + "\n", + " megatron_config = init_model_parallel_config(megatron_config)\n", + " self.parallel_layer = ParallelLlamaMLP(config=config, megatron_config=megatron_config)\n", + "\n", + " @register(Dispatch.ONE_TO_ALL)\n", + " def get_weights(self):\n", + " output = {}\n", + " for key, val in self.parallel_layer.named_parameters():\n", + " output[key] = val\n", + " return output\n", + "\n", + " @register(Dispatch.MEGATRON_COMPUTE)\n", + " def run_layer(self, x):\n", + " x = x.to(\"cuda\")\n", + " y = self.parallel_layer(x)\n", + " return y" + ] + }, + { + "cell_type": "code", + "execution_count": 175, + "id": "a655271d", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "layer_cls = RayClassWithInitArgs(cls=MLPLayerWorker)\n", + "layer_worker_group = NVMegatronRayWorkerGroup(\n", + " resource_pool=resource_pool,\n", + " ray_cls_with_init=layer_cls,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 176, + "id": "f105ebee", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4 4 1 1\n" + ] + } + ], + "source": [ + "print(layer_worker_group.world_size, layer_worker_group.tp_size, layer_worker_group.pp_size, layer_worker_group.dp_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 177, + "id": "38655091", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "ffn_hidden_size = 11008\n", + "batch_size = 16\n", + "seq_len = 2048\n", + "hidden_size = 4096\n", + "\n", + "config = OmegaConf.create(\n", + " {\n", + " \"hidden_size\": hidden_size,\n", + " \"intermediate_size\": ffn_hidden_size,\n", + " \"hidden_act\": \"silu\",\n", + " \"pretraining_tp\": 1,\n", + " \"tp\": layer_worker_group.tp_size,\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 178, + "id": "a026efca", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "x = torch.rand(size=(seq_len, batch_size, hidden_size), dtype=torch.float32)" + ] + }, + { + "cell_type": "code", + "execution_count": 179, + "id": "f5fcaf13", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[None, None, None, None]" + ] + }, + "execution_count": 179, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "layer_worker_group.init_model(config)" + ] + }, + { + "cell_type": "code", + "execution_count": 180, + "id": "3f5cc9b4", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([2048, 16, 4096])\n" + ] + } + ], + "source": [ + "output = layer_worker_group.run_layer(\n", + " [x]\n", + ") # This must be a list of size 1, ensuring that the input equals the data parallel (dp).\n", + "print(output[0].shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 181, + "id": "49792210", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Shutdown ray cluster\n", + "ray.shutdown()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/e2c/verl/examples/reinforce_plus_plus_trainer/run_qwen2-7b_math_rf.sh b/e2c/verl/examples/reinforce_plus_plus_trainer/run_qwen2-7b_math_rf.sh new file mode 100644 index 0000000000000000000000000000000000000000..3e1de4af113eeb25013f396a8fd78cca56081231 --- /dev/null +++ b/e2c/verl/examples/reinforce_plus_plus_trainer/run_qwen2-7b_math_rf.sh @@ -0,0 +1,49 @@ +set -x + + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=reinforce_plus_plus \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=3e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=1024 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=mse \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=True \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/reinforce_plus_plus_trainer/run_qwen2-7b_math_rf_baseline.sh b/e2c/verl/examples/reinforce_plus_plus_trainer/run_qwen2-7b_math_rf_baseline.sh new file mode 100644 index 0000000000000000000000000000000000000000..fb827168a19aa2e929fc3af7b2e3c87b22c52295 --- /dev/null +++ b/e2c/verl/examples/reinforce_plus_plus_trainer/run_qwen2-7b_math_rf_baseline.sh @@ -0,0 +1,49 @@ +set -x + + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=reinforce_plus_plus_baseline \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=3e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=1024 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=mse \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=True \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/remax_trainer/run_qwen2.5-3b_seq_balance.sh b/e2c/verl/examples/remax_trainer/run_qwen2.5-3b_seq_balance.sh new file mode 100644 index 0000000000000000000000000000000000000000..feebe8a847594671fe7c8a9d2468c52eaaf33cac --- /dev/null +++ b/e2c/verl/examples/remax_trainer/run_qwen2.5-3b_seq_balance.sh @@ -0,0 +1,43 @@ +set -x + +export HF_DATASETS_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 + + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=remax \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=30000 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=True \ + algorithm.kl_penalty=kl \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_remax_example_gsm8k' \ + trainer.experiment_name='qwen2.5_3b_function_rm_kl1e-3' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=5 $@ diff --git a/e2c/verl/examples/remax_trainer/run_qwen2.5-7b_seq_balance.sh b/e2c/verl/examples/remax_trainer/run_qwen2.5-7b_seq_balance.sh new file mode 100644 index 0000000000000000000000000000000000000000..8734eb351319f88417c767aad670052ee4b113a4 --- /dev/null +++ b/e2c/verl/examples/remax_trainer/run_qwen2.5-7b_seq_balance.sh @@ -0,0 +1,43 @@ +set -x + +export HF_DATASETS_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 + + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=remax \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=True \ + algorithm.kl_penalty=kl \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_remax_example_gsm8k' \ + trainer.experiment_name='qwen2.5_7b_function_rm_kl1e-3' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=10 $@ diff --git a/e2c/verl/examples/rloo_trainer/run_qwen2-7b.sh b/e2c/verl/examples/rloo_trainer/run_qwen2-7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..fc9b6e29fdebd0245f7ecf6cf42d9b369e8fa1db --- /dev/null +++ b/e2c/verl/examples/rloo_trainer/run_qwen2-7b.sh @@ -0,0 +1,40 @@ +set -x + + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=rloo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=80 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=160 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=160 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=True \ + algorithm.kl_penalty=kl \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_rloo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/sft/gsm8k/run_deepseek_6b7.sh b/e2c/verl/examples/sft/gsm8k/run_deepseek_6b7.sh new file mode 100644 index 0000000000000000000000000000000000000000..8a067f05d50b5a4bf86c444be09a610e9afc35cd --- /dev/null +++ b/e2c/verl/examples/sft/gsm8k/run_deepseek_6b7.sh @@ -0,0 +1,28 @@ +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_deepseek_6b7.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + data.prompt_dict_keys=['question'] \ + +data.response_dict_keys=['answer'] \ + data.micro_batch_size_per_gpu=4 \ + model.partial_pretrain=deepseek-ai/deepseek-coder-6.7b-instruct \ + trainer.default_local_dir=$save_path \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-deepseek-coder-6.7b-instruct \ + trainer.total_epochs=4 \ + trainer.logger='["console","wandb"]' $@ \ No newline at end of file diff --git a/e2c/verl/examples/sft/gsm8k/run_gemma_2b.sh b/e2c/verl/examples/sft/gsm8k/run_gemma_2b.sh new file mode 100644 index 0000000000000000000000000000000000000000..5b59893d258ba5723746676156aa0bcf67b7cfb3 --- /dev/null +++ b/e2c/verl/examples/sft/gsm8k/run_gemma_2b.sh @@ -0,0 +1,30 @@ +# Tested with 2 & 4 GPUs + +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_gemma_2b.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + data.prompt_dict_keys=['question'] \ + +data.response_dict_keys=['answer'] \ + data.micro_batch_size_per_gpu=4 \ + model.partial_pretrain=google/gemma-2b-it \ + trainer.default_local_dir=$save_path \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-gemma-2b-it \ + trainer.total_epochs=2 \ + trainer.logger='["console","wandb"]' $@ \ No newline at end of file diff --git a/e2c/verl/examples/sft/gsm8k/run_gemma_7b.sh b/e2c/verl/examples/sft/gsm8k/run_gemma_7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..fe2bc3a6f39ba7a1534bb9052d739b1ca01ced15 --- /dev/null +++ b/e2c/verl/examples/sft/gsm8k/run_gemma_7b.sh @@ -0,0 +1,28 @@ +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_gemma_7b.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + data.prompt_dict_keys=['question'] \ + data.response_dict_keys=['answer'] \ + data.micro_batch_size_per_gpu=4 \ + model.partial_pretrain=google/gemma-1.1-7b-it \ + trainer.default_local_dir=$save_path \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-gemma-1.1-7b-it \ + trainer.total_epochs=4 \ + trainer.logger='["console","wandb"]' $@ diff --git a/e2c/verl/examples/sft/gsm8k/run_qwen3_8b_sft_peft_sp2_npu.sh b/e2c/verl/examples/sft/gsm8k/run_qwen3_8b_sft_peft_sp2_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..720e2340838d13418326821c7339d488b54bd805 --- /dev/null +++ b/e2c/verl/examples/sft/gsm8k/run_qwen3_8b_sft_peft_sp2_npu.sh @@ -0,0 +1,36 @@ +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_qwen3_8b_sft_peft_sp2_npu.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + optim.lr=1e-4 \ + data.prompt_dict_keys=['question'] \ + +data.response_dict_keys=['answer'] \ + data.micro_batch_size_per_gpu=64 \ + model.partial_pretrain=Qwen/Qwen3-8B \ + trainer.default_local_dir=$save_path \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-qwen3-8b-instruct \ + trainer.logger=console \ + trainer.total_epochs=2 $@ \ + model.lora_rank=32 \ + model.lora_alpha=16 \ + model.target_modules=all-linear \ + model.strategy=fsdp \ + ulysses_sequence_parallel_size=2 \ + use_remove_padding=true \ + trainer.device=npu diff --git a/e2c/verl/examples/sft/gsm8k/run_qwen_05_peft.sh b/e2c/verl/examples/sft/gsm8k/run_qwen_05_peft.sh new file mode 100644 index 0000000000000000000000000000000000000000..3a7d445580780135c4a1a9c6c045181cce9f21ac --- /dev/null +++ b/e2c/verl/examples/sft/gsm8k/run_qwen_05_peft.sh @@ -0,0 +1,37 @@ +# Tested with 2 & 4 GPUs + +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_qwen_05_peft.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + optim.lr=1e-4 \ + data.prompt_dict_keys=['question'] \ + +data.response_dict_keys=['answer'] \ + data.micro_batch_size_per_gpu=4 \ + model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ + trainer.default_local_dir=$save_path \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct \ + trainer.logger=console \ + trainer.total_epochs=1 $@ \ + model.lora_rank=32\ + model.lora_alpha=16 \ + model.target_modules=all-linear + + # Or you can do this: + # model.target_modules=[q_proj,v_proj] \ diff --git a/e2c/verl/examples/sft/gsm8k/run_qwen_05_sp2.sh b/e2c/verl/examples/sft/gsm8k/run_qwen_05_sp2.sh new file mode 100644 index 0000000000000000000000000000000000000000..7210a5a403822d6b6e4ea724004f295fde5aeb6b --- /dev/null +++ b/e2c/verl/examples/sft/gsm8k/run_qwen_05_sp2.sh @@ -0,0 +1,31 @@ +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_qwen_05_sp2.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + optim.lr=1e-4 \ + data.prompt_dict_keys=['question'] \ + +data.response_dict_keys=['answer'] \ + data.micro_batch_size=4 \ + model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ + trainer.default_local_dir=$save_path \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct-sp2 \ + trainer.logger=console \ + trainer.total_training_steps=1 $@ \ + ulysses_sequence_parallel_size=2 \ + use_remove_padding=true diff --git a/e2c/verl/examples/sft/gsm8k/run_qwen_05_sp2_liger.sh b/e2c/verl/examples/sft/gsm8k/run_qwen_05_sp2_liger.sh new file mode 100644 index 0000000000000000000000000000000000000000..1c5cd591f14fc9ab94d7abf0f8bf033ae7214414 --- /dev/null +++ b/e2c/verl/examples/sft/gsm8k/run_qwen_05_sp2_liger.sh @@ -0,0 +1,31 @@ +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_qwen_05_sp2.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + optim.lr=1e-4 \ + data.prompt_dict_keys=['question'] \ + +data.response_dict_keys=['answer'] \ + data.micro_batch_size=4 \ + model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ + model.use_liger=True \ + trainer.default_local_dir=$save_path \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct-sp2-liger \ + trainer.logger=console $@ \ + ulysses_sequence_parallel_size=2 \ + use_remove_padding=true diff --git a/e2c/verl/examples/sft/gsm8k/run_seed_oss_36b_sft.sh b/e2c/verl/examples/sft/gsm8k/run_seed_oss_36b_sft.sh new file mode 100644 index 0000000000000000000000000000000000000000..35c1d6c6d34f8a070691a1ba5155ff2e4fee7dea --- /dev/null +++ b/e2c/verl/examples/sft/gsm8k/run_seed_oss_36b_sft.sh @@ -0,0 +1,31 @@ +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_seed_oss_36b_sft.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + optim.lr=1e-4 \ + data.prompt_dict_keys=['question'] \ + +data.response_dict_keys=['answer'] \ + data.micro_batch_size=4 \ + model.partial_pretrain=ByteDance-Seed/Seed-OSS-36B-Base \ + trainer.default_local_dir=$save_path \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-seed-oss-36b \ + trainer.logger=console \ + trainer.total_training_steps=1 \ + ulysses_sequence_parallel_size=2 \ + use_remove_padding=true $@ diff --git a/e2c/verl/examples/sft/multiturn/run_qwen_05_sp2.sh b/e2c/verl/examples/sft/multiturn/run_qwen_05_sp2.sh new file mode 100644 index 0000000000000000000000000000000000000000..5e1fc47e9c54eedadc74120ec1fb51ccf85669bc --- /dev/null +++ b/e2c/verl/examples/sft/multiturn/run_qwen_05_sp2.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -x + +if [ "$#" -lt 2 ]; then + echo "Usage: run_qwen_05_sp2.sh [other_configs...]" + exit 1 +fi + +nproc_per_node=$1 +save_path=$2 + +# Shift the arguments so $@ refers to the rest +shift 2 + +torchrun --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/multiturn/train.parquet \ + data.val_files=$HOME/data/multiturn/test.parquet \ + data.multiturn.enable=true \ + data.multiturn.messages_key=messages \ + data.micro_batch_size=4 \ + model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ + trainer.default_local_dir=$save_path \ + trainer.project_name=multiturn-sft \ + trainer.experiment_name=multiturn-sft-qwen-2.5-0.5b-instruct-sp2 \ + trainer.logger=console \ + trainer.total_training_steps=1 $@ \ + ulysses_sequence_parallel_size=2 \ + use_remove_padding=true \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/README.md b/e2c/verl/examples/sglang_multiturn/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0c97c7e7507f3b5b108128c7068ea9ae6dae95ee --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/README.md @@ -0,0 +1,38 @@ +# Multi-Turn Rollout Example (GSM8K) + +This example demonstrates how to perform **multi-turn rollout** using SGLang with a tool-calling capable model (e.g., Qwen2.5-3B) on the GSM8K dataset. + +## Usage + +### Step 1: Download GSM8K Dataset + +```bash +cd examples/data_preprocess +python3 gsm8k_multiturn_w_tool.py +``` + +This will download and preprocess the GSM8K dataset into ~/data/gsm8k/. + +### Step 2: Run Multi-Turn Rollout + +If you have 8 GPUs +Use the standard 8-GPU script: + +```bash +cd your_verl_root_dir +bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +``` + +If you have only 4 GPUs +Use the fallback 4-GPU script: + +```bash +cd your_verl_root_dir +bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_4xgpu.sh +``` + +## Notes + +- The rollout supports multi-turn conversations with tool-calling capabilities. +- Current tools are used for GSM8K answer evaluation. +- Future versions may extend to search and code interpreter tools. diff --git a/e2c/verl/examples/sglang_multiturn/config/geo3k_multiturn_grpo.yaml b/e2c/verl/examples/sglang_multiturn/config/geo3k_multiturn_grpo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9523f196855c2e41572ef626e42330960506635 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/geo3k_multiturn_grpo.yaml @@ -0,0 +1,25 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + max_prompt_length: 2048 + max_response_length: 2048 + train_batch_size: 256 + return_raw_chat: True + return_multi_modal_inputs: False + +actor_rollout_ref: + hybrid_engine: True + model: + custom_chat_template: "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{%- if tools %}{{- '<|im_start|>system\\n' }}{%- if messages[0]['role'] == 'system' %}{{- messages[0]['content'] }}{%- else %}{{- 'You are a helpful assistant.' }}{%- endif %}{{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}{%- for tool in tools %}{{- \"\\n\" }}{{- tool | tojson }}{%- endfor %}{{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}{% for message in messages %}{% if message['role'] != 'system' or loop.first == false %}{%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{%- elif message.role == \"assistant\" %}{{- '<|im_start|>' + message.role }}{%- if message.content %}{{- '\\n' + message.content }}{%- endif %}{%- for tool_call in message.tool_calls %}{%- if tool_call.function is defined %}{%- set tool_call = tool_call.function %}{%- endif %}{{- '\\n\\n{\"name\": \"' }}{{- tool_call.name }}{{- '\", \"arguments\": ' }}{{- tool_call.arguments | tojson }}{{- '}\\n' }}{%- endfor %}{{- '<|im_end|>\\n' }}{%- elif message.role == \"tool\" %}{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}{{- '<|im_start|>user' }}{%- endif %}{{- '\\n\\n' }}{% if message['content'] is string %}{{ message.content }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif content['type'] == 'text' or 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{- '\\n' }}{%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}{{- '<|im_end|>\\n' }}{%- endif %}{%- endif %}{% endif %}{% endfor %}{%- else %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}{%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{%- elif message.role == \"assistant\" %}{{- '<|im_start|>' + message.role }}{%- if message.content %}{{- '\\n' + message.content }}{%- endif %}{%- for tool_call in message.tool_calls %}{%- if tool_call.function is defined %}{%- set tool_call = tool_call.function %}{%- endif %}{{- '\\n\\n{\"name\": \"' }}{{- tool_call.name }}{{- '\", \"arguments\": ' }}{{- tool_call.arguments | tojson }}{{- '}\\n' }}{%- endfor %}{{- '<|im_end|>\\n' }}{%- elif message.role == \"tool\" %}{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}{{- '<|im_start|>user' }}{%- endif %}{{- '\\n\\n' }}{% if message['content'] is string %}{{ message.content }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif content['type'] == 'text' or 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{- '\\n' }}{%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}{{- '<|im_end|>\\n' }}{%- endif %}{%- endif %}{% endfor %}{%- endif %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" + rollout: + name: sglang + multi_turn: + enable: True + max_assistant_turns: 5 + # tool_config_path: "./config/tool_config/gsm8k_tool_config.yaml" diff --git a/e2c/verl/examples/sglang_multiturn/config/geo3k_multiturn_megatron_grpo.yaml b/e2c/verl/examples/sglang_multiturn/config/geo3k_multiturn_megatron_grpo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e208f3336eeba29793f2c81a86762167eaf6f53 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/geo3k_multiturn_megatron_grpo.yaml @@ -0,0 +1,25 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_megatron_trainer + - _self_ + +data: + max_prompt_length: 2048 + max_response_length: 2048 + train_batch_size: 256 + return_raw_chat: True + return_multi_modal_inputs: False + +actor_rollout_ref: + hybrid_engine: True + model: + custom_chat_template: "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{%- if tools %}{{- '<|im_start|>system\\n' }}{%- if messages[0]['role'] == 'system' %}{{- messages[0]['content'] }}{%- else %}{{- 'You are a helpful assistant.' }}{%- endif %}{{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}{%- for tool in tools %}{{- \"\\n\" }}{{- tool | tojson }}{%- endfor %}{{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}{% for message in messages %}{% if message['role'] != 'system' or loop.first == false %}{%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{%- elif message.role == \"assistant\" %}{{- '<|im_start|>' + message.role }}{%- if message.content %}{{- '\\n' + message.content }}{%- endif %}{%- for tool_call in message.tool_calls %}{%- if tool_call.function is defined %}{%- set tool_call = tool_call.function %}{%- endif %}{{- '\\n\\n{\"name\": \"' }}{{- tool_call.name }}{{- '\", \"arguments\": ' }}{{- tool_call.arguments | tojson }}{{- '}\\n' }}{%- endfor %}{{- '<|im_end|>\\n' }}{%- elif message.role == \"tool\" %}{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}{{- '<|im_start|>user' }}{%- endif %}{{- '\\n\\n' }}{% if message['content'] is string %}{{ message.content }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif content['type'] == 'text' or 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{- '\\n' }}{%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}{{- '<|im_end|>\\n' }}{%- endif %}{%- endif %}{% endif %}{% endfor %}{%- else %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}{%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{%- elif message.role == \"assistant\" %}{{- '<|im_start|>' + message.role }}{%- if message.content %}{{- '\\n' + message.content }}{%- endif %}{%- for tool_call in message.tool_calls %}{%- if tool_call.function is defined %}{%- set tool_call = tool_call.function %}{%- endif %}{{- '\\n\\n{\"name\": \"' }}{{- tool_call.name }}{{- '\", \"arguments\": ' }}{{- tool_call.arguments | tojson }}{{- '}\\n' }}{%- endfor %}{{- '<|im_end|>\\n' }}{%- elif message.role == \"tool\" %}{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}{{- '<|im_start|>user' }}{%- endif %}{{- '\\n\\n' }}{% if message['content'] is string %}{{ message.content }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif content['type'] == 'text' or 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{- '\\n' }}{%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}{{- '<|im_end|>\\n' }}{%- endif %}{%- endif %}{% endfor %}{%- endif %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" + rollout: + name: sglang + multi_turn: + enable: True + max_assistant_turns: 5 + # tool_config_path: "./config/tool_config/gsm8k_tool_config.yaml" diff --git a/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo.yaml b/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9109232a4fa7e2c46a4d66faa57b146f5ff8131 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo.yaml @@ -0,0 +1,21 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + max_prompt_length: 1024 + max_response_length: 1024 + train_batch_size: 256 + return_raw_chat: True + +actor_rollout_ref: + hybrid_engine: True + rollout: + name: sglang + multi_turn: + enable: True + max_assistant_turns: 5 diff --git a/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo_server.yaml b/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo_server.yaml new file mode 100644 index 0000000000000000000000000000000000000000..502210dbec824e7ecdc9544d42f3b64b7a4b42b9 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo_server.yaml @@ -0,0 +1,28 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + max_prompt_length: 1024 + max_response_length: 1024 + train_batch_size: 256 + return_raw_chat: True + +actor_rollout_ref: + hybrid_engine: True + rollout: + name: sglang + multi_turn: + enable: True + max_assistant_turns: 5 + sglang_rollout_mode: server + server: + timeout: 60 + max_attempts: 3 + retry_delay: 2 + max_connections: 1000 + max_start_wait_time: 300.0 \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo_w_interaction.yaml b/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo_w_interaction.yaml new file mode 100644 index 0000000000000000000000000000000000000000..122f7e50f1ee9f41723047e8fc40aedf52d44d9a --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_grpo_w_interaction.yaml @@ -0,0 +1,21 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + max_prompt_length: 1024 + max_response_length: 1024 + train_batch_size: 256 + return_raw_chat: True + +actor_rollout_ref: + hybrid_engine: True + rollout: + name: sglang + multi_turn: + enable: True + max_user_turns: 5 diff --git a/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_megatron_grpo.yaml b/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_megatron_grpo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8aff859cc331a454014a051f885260517089d659 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/gsm8k_multiturn_megatron_grpo.yaml @@ -0,0 +1,22 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_megatron_trainer + - _self_ + +data: + max_prompt_length: 1024 + max_response_length: 1024 + train_batch_size: 256 + return_raw_chat: True + +actor_rollout_ref: + hybrid_engine: True + rollout: + name: sglang + multi_turn: + enable: True + max_assistant_turns: 5 + diff --git a/e2c/verl/examples/sglang_multiturn/config/interaction_config/gsm8k_interaction_config.yaml b/e2c/verl/examples/sglang_multiturn/config/interaction_config/gsm8k_interaction_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78faf386ef8a3a68de7dcd51c3c1281a403d5422 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/interaction_config/gsm8k_interaction_config.yaml @@ -0,0 +1,4 @@ +interaction: + - name: "gsm8k" + class_name: "verl.interactions.gsm8k_interaction.Gsm8kInteraction" + config: {} \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/config/retool_multiturn_grpo.yaml b/e2c/verl/examples/sglang_multiturn/config/retool_multiturn_grpo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1cfaccce28f848a171405bd228384c7e0e62be9 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/retool_multiturn_grpo.yaml @@ -0,0 +1,22 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + max_prompt_length: 1024 + max_response_length: 1024 + train_batch_size: 256 + return_raw_chat: True + +actor_rollout_ref: + hybrid_engine: True + rollout: + name: sglang + multi_turn: + enable: True + max_assistant_turns: 5 + tool_config_path: "./config/tool_config/sandbox_fusion_tool_config.yaml" diff --git a/e2c/verl/examples/sglang_multiturn/config/search_multiturn_grpo.yaml b/e2c/verl/examples/sglang_multiturn/config/search_multiturn_grpo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e24f62b788135aa8d8bdc718d1aef989f841bda --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/search_multiturn_grpo.yaml @@ -0,0 +1,23 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + max_prompt_length: 1024 + max_response_length: 1024 + train_batch_size: 256 + return_raw_chat: True + shuffle: False + +actor_rollout_ref: + hybrid_engine: True + rollout: + name: sglang + multi_turn: + enable: True + max_assistant_turns: 2 + format: qwen diff --git a/e2c/verl/examples/sglang_multiturn/config/tool_config/geo3k_tool_config.yaml b/e2c/verl/examples/sglang_multiturn/config/tool_config/geo3k_tool_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..675a342e67cf0699d575b5a7db27c72a4c8e8f12 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/tool_config/geo3k_tool_config.yaml @@ -0,0 +1,16 @@ +tools: + - class_name: "verl.tools.geo3k_tool.Geo3kTool" + config: + type: native + tool_schema: + type: "function" + function: + name: "calc_geo3k_reward" + description: "A tool for calculating the reward of geo3k. (1.0 if parsed answer is correct, 0.0 if parsed answer is incorrect or not correctly parsed)" + parameters: + type: "object" + properties: + answer: + type: "string" + description: "The model's answer to the geo3k problem, must be a digits" + required: ["answer"] \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml b/e2c/verl/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4197baabf08e1ac076357db8286c8641fc02f54 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml @@ -0,0 +1,16 @@ +tools: + - class_name: "verl.tools.gsm8k_tool.Gsm8kTool" + config: + type: native + tool_schema: + type: "function" + function: + name: "calc_gsm8k_reward" + description: "A tool for calculating the reward of gsm8k. (1.0 if parsed answer is correct, 0.0 if parsed answer is incorrect or not correctly parsed)" + parameters: + type: "object" + properties: + answer: + type: "string" + description: "The model's answer to the GSM8K math problem, must be a digits" + required: ["answer"] diff --git a/e2c/verl/examples/sglang_multiturn/config/tool_config/mcp_server.json b/e2c/verl/examples/sglang_multiturn/config/tool_config/mcp_server.json new file mode 100644 index 0000000000000000000000000000000000000000..29424f71e0b17814a3242fefb5bc2e149c3e9c64 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/tool_config/mcp_server.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "Tavily Expert": { + "url": "your_tavily_expert_url", + "auth_token": "your_tavily_api_token" + } + } +} \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/config/tool_config/mcp_tool_config.yaml b/e2c/verl/examples/sglang_multiturn/config/tool_config/mcp_tool_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40abf7c67126061db364147b4ae626574d7e0a77 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/tool_config/mcp_tool_config.yaml @@ -0,0 +1,11 @@ +tools: + - class_name: verl.tools.mcp_search_tool.MCPSearchTool + config: + rate_limit: 120 + timeout: 120 + type: mcp + mcp: + mcp_servers_config_path: ./mcp_server.json + # optional + tool_selected_list: + - tavily_search_tool \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/config/tool_config/sandbox_fusion_tool_config.yaml b/e2c/verl/examples/sglang_multiturn/config/tool_config/sandbox_fusion_tool_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..516acf56946b8de6fa40e07cc53042e8a2fcdd18 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/tool_config/sandbox_fusion_tool_config.yaml @@ -0,0 +1,24 @@ +tools: + - class_name: "verl.tools.sandbox_fusion_tools.SandboxFusionTool" + config: + sandbox_fusion_url: "https://xxx.apigateway-cn-beijing.volceapi.com/run_code" + num_workers: 10 + enable_global_rate_limit: true + rate_limit: 10 + default_timeout: 30 + default_language: "python" + memory_limit_mb: 1024 + type: native + + tool_schema: + type: "function" + function: + name: "code_interpreter" + description: "A tool for executing code." + parameters: + type: "object" + properties: + code: + type: "string" + description: "The code to execute." + required: ["code"] \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/config/tool_config/search_tool_config.yaml b/e2c/verl/examples/sglang_multiturn/config/tool_config/search_tool_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..926b6b832f283175f92cc86b6cc4a1964096a8d3 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/config/tool_config/search_tool_config.yaml @@ -0,0 +1,23 @@ +tools: + - class_name: verl.tools.search_tool.SearchTool + config: + retrieval_service_url: http://127.0.0.1:8000/retrieve + num_workers: 120 + rate_limit: 120 + timeout: 30 + type: native + tool_schema: + type: function + function: + name: search + description: Searches the web for relevant information based on the given query. + parameters: + type: object + properties: + query_list: + type: array + item: + type: string + description: A list of fully-formed semantic queries. The tool will return search results for each query. + required: + - query_list \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_geo3k_multiturn.sh b/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_geo3k_multiturn.sh new file mode 100644 index 0000000000000000000000000000000000000000..d9306e9df71b4921d9056dd2aa0505b8eaa86b12 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_geo3k_multiturn.sh @@ -0,0 +1,54 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='geo3k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=2048 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='geo3k_async_rl' \ + trainer.experiment_name='qwen2.5-3b_function_rm-geo3k-sgl-multi-w-tool-verify-n16' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + data.train_files=$HOME/data/geo3k_multiturn_w_tool/train.parquet \ + data.val_files=$HOME/data/geo3k_multiturn_w_tool/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/geo3k_tool_config.yaml" \ + trainer.total_epochs=15 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_geo3k_multiturn_4xgpu.sh b/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_geo3k_multiturn_4xgpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..66f12a5e515ecae9d80d57404441e8e4bcaf671d --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_geo3k_multiturn_4xgpu.sh @@ -0,0 +1,58 @@ +# run on 4xH100 +# make sure your current working directory is the root of the project + +set -x +export HYDRA_FULL_ERROR=1 +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='geo3k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=2048 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='geo3k_async_rl' \ + trainer.experiment_name='qwen2.5-3b_function_rm-geo3k-async-sgl-multi-w-tool-verify-n16-4cards' \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.total_epochs=15 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8192 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=8192 \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192 \ + critic.ppo_max_token_len_per_gpu=8192 \ + critic.forward_max_token_len_per_gpu=8192 \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/geo3k_tool_config.yaml" \ + $@ \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_megatron_geo3k_multiturn.sh b/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_megatron_geo3k_multiturn.sh new file mode 100644 index 0000000000000000000000000000000000000000..784594a7bfb610b5aa4a02e71f63775f76ee262e --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/geo3k/run_qwen2.5-3b_megatron_geo3k_multiturn.sh @@ -0,0 +1,64 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project +# this is a verification training script, the parallel setting should be tuned to your model + +set -x + +export PYTHONUNBUFFERED=1 +export RAY_DEDUP_LOGS=0 +export RUST_BACKTRACE=1 +export HYDRA_FULL_ERROR=1 +export CUDA_DEVICE_MAX_CONNECTIONS=1 + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='geo3k_multiturn_megatron_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=2048 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.context_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.megatron.seed=42 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.context_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='geo3k_async_rl' \ + trainer.experiment_name='qwen2.5-3b_function_rm-geo3k-sgl-multi-w-tool-n8-mcore-v2505201745_seed42' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + data.train_files=$HOME/data/geo3k_multiturn_w_tool/train.parquet \ + data.val_files=$HOME/data/geo3k_multiturn_w_tool/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/geo3k_tool_config.yaml" \ + trainer.total_epochs=15 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen0.5b_gsm8k_multiturn_curriculum.sh b/e2c/verl/examples/sglang_multiturn/run_qwen0.5b_gsm8k_multiturn_curriculum.sh new file mode 100644 index 0000000000000000000000000000000000000000..d67a76e48fe12f3463cbc0c870c3fec3511ab7c8 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen0.5b_gsm8k_multiturn_curriculum.sh @@ -0,0 +1,56 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.sampler.class_name="RandomCurriculumSampler" \ + data.sampler.class_path="pkg://tests.utils.dataset.test_create_rl_sampler_on_cpu" \ + data.dataloader_num_workers=0 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.train_batch_size=256 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='gsm8k_async_rl' \ + trainer.experiment_name='qwen3-4b_function_rm-gsm8k-sgl-multi-w-tool-verify-n16' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + trainer.total_epochs=15 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen2.5-0.5b_gsm8k_multiturn_w_interaction.sh b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-0.5b_gsm8k_multiturn_w_interaction.sh new file mode 100644 index 0000000000000000000000000000000000000000..b94f094174a8afc19702ea5365c8f61186b27346 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-0.5b_gsm8k_multiturn_w_interaction.sh @@ -0,0 +1,58 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" +TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-512} +MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-8} +OFFLOAD=${OFFLOAD:-False} + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo_w_interaction' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=$TRAIN_BATCH_SIZE \ + data.max_prompt_length=1024 \ + data.max_response_length=$((1024 * 3)) \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.enable_activation_offloading=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=$TRAIN_BATCH_SIZE \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$MICRO_BATCH_SIZE \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.fsdp_config.param_offload=$OFFLOAD \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=$OFFLOAD \ + actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=$MICRO_BATCH_SIZE \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=$MICRO_BATCH_SIZE \ + actor_rollout_ref.ref.fsdp_config.param_offload=$OFFLOAD \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='gsm8k_async_rl' \ + trainer.experiment_name='qwen2.5-0.5b_function_rm-gsm8k-sgl-multi-w-interaction-n8' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + data.train_files=$HOME/data/gsm8k_verl_sgl_multi_turn_w_interaction/train.parquet \ + data.val_files=$HOME/data/gsm8k_verl_sgl_multi_turn_w_interaction/test.parquet \ + actor_rollout_ref.rollout.multi_turn.interaction_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/interaction_config/gsm8k_interaction_config.yaml" \ + trainer.total_epochs=15 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c3dd6a451510b7bd3dab1fea608fe067022f44c --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -0,0 +1,68 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +function now() { + date '+%d-%H-%M' +} + +EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + global_profiler.tool=torch_memory \ + global_profiler.save_path=./mem_snapshots \ + global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries=100000 \ + global_profiler.global_tool_config.torch_memory.stack_depth=32 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.multi_stage_wake_up=True \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.over_sample_rate=0.1 \ + actor_rollout_ref.rollout.mode=sync \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='multi-turn-grpo-qwen2.5-3b-sglang' \ + trainer.experiment_name=$EXPERIMENT_NAME \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.val_before_train=True \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + trainer.total_epochs=15 \ + actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_4xgpu.sh b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_4xgpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..9e61893b05393c28f314416b9250703883df34f3 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_4xgpu.sh @@ -0,0 +1,60 @@ +# run on 4xH100 +# make sure your current working directory is the root of the project + +set -x +export HYDRA_FULL_ERROR=1 +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='gsm8k_async_rl' \ + trainer.experiment_name='qwen2.5-3b_function_rm-gsm8k-async-sgl-multi-w-tool-verify-n16-4cards' \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.total_epochs=15 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8192 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=8192 \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192 \ + critic.ppo_max_token_len_per_gpu=8192 \ + critic.forward_max_token_len_per_gpu=8192 \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + actor_rollout_ref.rollout.multi_turn.interaction_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/interaction_config/gsm8k_interaction_config.yaml" \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=1 \ + $@ \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_4xgpu_server.sh b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_4xgpu_server.sh new file mode 100644 index 0000000000000000000000000000000000000000..79e5e568e76f923595847bb1048323e9f382b654 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_4xgpu_server.sh @@ -0,0 +1,60 @@ +# run on 4xH100 +# make sure your current working directory is the root of the project + +set -x +export HYDRA_FULL_ERROR=1 +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo_server' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console", "wandb"]' \ + trainer.project_name='gsm8k_async_rl_server' \ + trainer.experiment_name='qwen2.5-3b_function_rm-gsm8k-async-sgl-multi-w-tool-verify-n16-4cards' \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.total_epochs=15 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8192 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=8192 \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192 \ + critic.ppo_max_token_len_per_gpu=8192 \ + critic.forward_max_token_len_per_gpu=8192 \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + actor_rollout_ref.rollout.multi_turn.interaction_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/interaction_config/gsm8k_interaction_config.yaml" \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=1 \ + $@ \ No newline at end of file diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_server.sh b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_server.sh new file mode 100644 index 0000000000000000000000000000000000000000..47ba0f12db14cb89483cc1b53da1df55b517e419 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_server.sh @@ -0,0 +1,63 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +function now() { + date '+%d-%H-%M' +} + +EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo_server' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.multi_stage_wake_up=True \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.over_sample_rate=0 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='multi-turn-grpo-qwen2.5-3b-sglang' \ + trainer.experiment_name=$EXPERIMENT_NAME \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.val_before_train=True \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + trainer.total_epochs=15 \ + actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_tool_agent_mlflow.sh b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_tool_agent_mlflow.sh new file mode 100644 index 0000000000000000000000000000000000000000..11c104fa94f4b19657149e2018da0a1321831083 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_tool_agent_mlflow.sh @@ -0,0 +1,57 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.trace.backend=mlflow \ + actor_rollout_ref.rollout.trace.token2text=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","mlflow"]' \ + trainer.project_name='gsm8k_tool-agent' \ + trainer.experiment_name='qwen2.5-3b_function_rm-gsm8k-sgl-tool-agent-verify-n16' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.total_training_steps=2 \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + trainer.total_epochs=15 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_megatron_gsm8k_multiturn.sh b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_megatron_gsm8k_multiturn.sh new file mode 100644 index 0000000000000000000000000000000000000000..5522ee9250986ca0058e86c8438c03d81c3bac90 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen2.5-3b_megatron_gsm8k_multiturn.sh @@ -0,0 +1,64 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project +# this is a verification training script, the parallel setting should be tuned to your model + +set -x + +export PYTHONUNBUFFERED=1 +export RAY_DEDUP_LOGS=0 +export RUST_BACKTRACE=1 +export HYDRA_FULL_ERROR=1 +export CUDA_DEVICE_MAX_CONNECTIONS=1 + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_megatron_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=/user/longxiang1/models/Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.context_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.megatron.seed=42 \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.context_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='gsm8k_async_rl' \ + trainer.experiment_name='qwen2.5-3b_function_rm-gsm8k-sgl-multi-w-tool-n8-mcore-v2505201745_seed42' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + data.train_files=/user/longxiang1/data/gsm8k_verl_sgl_multi_turn_preprocessed_v2/train.parquet \ + data.val_files=/user/longxiang1/data/gsm8k_verl_sgl_multi_turn_preprocessed_v2/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + trainer.total_epochs=15 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen3-4b_gsm8k_multiturn.sh b/e2c/verl/examples/sglang_multiturn/run_qwen3-4b_gsm8k_multiturn.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f1f99e4bd2f678d392c93d1ee88277099bd997d --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen3-4b_gsm8k_multiturn.sh @@ -0,0 +1,55 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen3-4B \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.rollout.over_sample_rate=0.1 \ + actor_rollout_ref.rollout.mode=sync \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='gsm8k_async_rl' \ + trainer.experiment_name='qwen3-4b_function_rm-gsm8k-sgl-multi-w-tool-verify-n16' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + trainer.total_epochs=15 $@ + diff --git a/e2c/verl/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh b/e2c/verl/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh new file mode 100644 index 0000000000000000000000000000000000000000..399486932648d558c86ea4cef530d6e083472e44 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh @@ -0,0 +1,101 @@ +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +pip install --upgrade "huggingface-hub>=0.34.0" +hf download \ + BytedTsinghua-SIA/DAPO-Math-17k \ + --repo-type dataset \ + --local-dir $HOME/data/BytedTsinghua-SIA/DAPO-Math-17k + + +hf download \ + Maxwell-Jia/AIME_2024 \ + --repo-type dataset \ + --local-dir $HOME/data/Maxwell-Jia/AIME_2024 + + +# Note: +# 1. +# a sandbox fusion server is needed to run the code interpreter tool. +# docker run -it -p 8080:8080 volcengine/sandbox-fusion:server-20250609 + +# 2. +# The model located at font-info/qwen3-4b-sft-SGLang-RL (https://huggingface.co/font-info/qwen3-4b-sft-SGLang-RL) +# is a fine-tuned version provided by the SGLang RL team. Without supervised fine-tuning (SFT) +# on the Retool dataset, Dapo training will not converge. + +# If you still wish to perform SFT from scratch, follow the steps below: + +# Step 1: Download the SFT dataset +#huggingface-cli download JoeYing/ReTool-SFT --repo-type dataset --local-dir ./ReTool-SFT + +# Step 2: Preprocess the data for SFT +#python3 recipe/retool/retool_sft_preprocess.py + +# Step 3: Run SFT training +#bash recipe/retool/run_qwen2-32b_sft.sh + +# having trouble setup? see https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/multi-turn/release_log/latest_sglang.md for more details. + + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_ctrl.kl_coef=0.0 \ + data.train_files=$HOME/data/BytedTsinghua-SIA/DAPO-Math-17k \ + data.val_files=$HOME/data/Maxwell-Jia/AIME_2024 \ + data.return_raw_chat=True \ + data.train_batch_size=32 \ + data.max_prompt_length=2048 \ + data.max_response_length=16384 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.custom_cls.path=$PROJECT_DIR/recipe/retool/retool.py \ + data.custom_cls.name=CustomRLHFDataset \ + custom_reward_function.path=$PROJECT_DIR/recipe/retool/retool.py \ + custom_reward_function.name=compute_score \ + actor_rollout_ref.model.path=font-info/qwen3-4b-sft-SGLang-RL \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.clip_ratio_low=0.2 \ + actor_rollout_ref.actor.clip_ratio_high=0.28 \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.use_dynamic_bsz=False \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.multi_stage_wake_up=True \ + actor_rollout_ref.rollout.multi_turn.enable=True \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=16 \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=16 \ + actor_rollout_ref.rollout.multi_turn.tool_config_path=$PROJECT_DIR/recipe/retool/sandbox_fusion_tool_config.yaml \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.val_kwargs.top_p=0.6 \ + actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \ + actor_rollout_ref.rollout.val_kwargs.n=30 \ + trainer.logger=['console','wandb'] \ + trainer.project_name=sglang-dapo-multiturn \ + trainer.experiment_name=qwen3_4b_sft_dapo_multiturn \ + trainer.n_gpus_per_node=8 \ + trainer.log_val_generations=20 \ + trainer.val_before_train=True \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.total_epochs=15 \ + $@ diff --git a/e2c/verl/examples/sglang_multiturn/search_r1_like/local_dense_retriever/download.py b/e2c/verl/examples/sglang_multiturn/search_r1_like/local_dense_retriever/download.py new file mode 100644 index 0000000000000000000000000000000000000000..6fe554936fafc57ada63198fadd4f30af0de8b8a --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/search_r1_like/local_dense_retriever/download.py @@ -0,0 +1,44 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 Search-R1 Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/PeterGriffinJin/Search-R1/blob/main/scripts/download.py + + +import argparse + +from huggingface_hub import hf_hub_download + +parser = argparse.ArgumentParser(description="Download files from a Hugging Face dataset repository.") +parser.add_argument("--repo_id", type=str, default="PeterJinGo/wiki-18-e5-index", help="Hugging Face repository ID") +parser.add_argument("--save_path", type=str, required=True, help="Local directory to save files") + +args = parser.parse_args() + +repo_id = "PeterJinGo/wiki-18-e5-index" +for file in ["part_aa", "part_ab"]: + hf_hub_download( + repo_id=repo_id, + filename=file, # e.g., "e5_Flat.index" + repo_type="dataset", + local_dir=args.save_path, + ) + +repo_id = "PeterJinGo/wiki-18-corpus" +hf_hub_download( + repo_id=repo_id, + filename="wiki-18.jsonl.gz", + repo_type="dataset", + local_dir=args.save_path, +) diff --git a/e2c/verl/examples/sglang_multiturn/search_r1_like/local_dense_retriever/retrieval_server.py b/e2c/verl/examples/sglang_multiturn/search_r1_like/local_dense_retriever/retrieval_server.py new file mode 100644 index 0000000000000000000000000000000000000000..2f67c1439d27b1db5aefdec5bb141fb0456ac6d3 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/search_r1_like/local_dense_retriever/retrieval_server.py @@ -0,0 +1,415 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 Search-R1 Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/PeterGriffinJin/Search-R1/blob/main/search_r1/search/retrieval_server.py + +import argparse +import json +import warnings +from typing import Optional + +import datasets +import faiss +import numpy as np +import torch +import uvicorn +from fastapi import FastAPI +from pydantic import BaseModel +from tqdm import tqdm +from transformers import AutoModel, AutoTokenizer + + +def load_corpus(corpus_path: str): + corpus = datasets.load_dataset("json", data_files=corpus_path, split="train", num_proc=4) + return corpus + + +def load_docs(corpus, doc_idxs): + results = [corpus[int(idx)] for idx in doc_idxs] + return results + + +def load_model(model_path: str, use_fp16: bool = False): + model = AutoModel.from_pretrained(model_path, trust_remote_code=True) + model.eval() + model.cuda() + if use_fp16: + model = model.half() + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + return model, tokenizer + + +def pooling(pooler_output, last_hidden_state, attention_mask=None, pooling_method="mean"): + if pooling_method == "mean": + last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0) + return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] + elif pooling_method == "cls": + return last_hidden_state[:, 0] + elif pooling_method == "pooler": + return pooler_output + else: + raise NotImplementedError("Pooling method not implemented!") + + +class Encoder: + def __init__(self, model_name, model_path, pooling_method, max_length, use_fp16): + self.model_name = model_name + self.model_path = model_path + self.pooling_method = pooling_method + self.max_length = max_length + self.use_fp16 = use_fp16 + + self.model, self.tokenizer = load_model(model_path=model_path, use_fp16=use_fp16) + self.model.eval() + + @torch.no_grad() + def encode(self, query_list: list[str], is_query=True) -> np.ndarray: + # processing query for different encoders + if isinstance(query_list, str): + query_list = [query_list] + + if "e5" in self.model_name.lower(): + if is_query: + query_list = [f"query: {query}" for query in query_list] + else: + query_list = [f"passage: {query}" for query in query_list] + + if "bge" in self.model_name.lower(): + if is_query: + query_list = [ + f"Represent this sentence for searching relevant passages: {query}" for query in query_list + ] + + inputs = self.tokenizer( + query_list, max_length=self.max_length, padding=True, truncation=True, return_tensors="pt" + ) + inputs = {k: v.cuda() for k, v in inputs.items()} + + if "T5" in type(self.model).__name__: + # T5-based retrieval model + decoder_input_ids = torch.zeros((inputs["input_ids"].shape[0], 1), dtype=torch.long).to( + inputs["input_ids"].device + ) + output = self.model(**inputs, decoder_input_ids=decoder_input_ids, return_dict=True) + query_emb = output.last_hidden_state[:, 0, :] + else: + output = self.model(**inputs, return_dict=True) + query_emb = pooling( + output.pooler_output, output.last_hidden_state, inputs["attention_mask"], self.pooling_method + ) + if "dpr" not in self.model_name.lower(): + query_emb = torch.nn.functional.normalize(query_emb, dim=-1) + + query_emb = query_emb.detach().cpu().numpy() + query_emb = query_emb.astype(np.float32, order="C") + + del inputs, output + torch.cuda.empty_cache() + + return query_emb + + +class BaseRetriever: + def __init__(self, config): + self.config = config + self.retrieval_method = config.retrieval_method + self.topk = config.retrieval_topk + + self.index_path = config.index_path + self.corpus_path = config.corpus_path + + def _search(self, query: str, num: int, return_score: bool): + raise NotImplementedError + + def _batch_search(self, query_list: list[str], num: int, return_score: bool): + raise NotImplementedError + + def search(self, query: str, num: int = None, return_score: bool = False): + return self._search(query, num, return_score) + + def batch_search(self, query_list: list[str], num: int = None, return_score: bool = False): + return self._batch_search(query_list, num, return_score) + + +class BM25Retriever(BaseRetriever): + def __init__(self, config): + super().__init__(config) + from pyserini.search.lucene import LuceneSearcher + + self.searcher = LuceneSearcher(self.index_path) + self.contain_doc = self._check_contain_doc() + if not self.contain_doc: + self.corpus = load_corpus(self.corpus_path) + self.max_process_num = 8 + + def _check_contain_doc(self): + return self.searcher.doc(0).raw() is not None + + def _search(self, query: str, num: int = None, return_score: bool = False): + if num is None: + num = self.topk + hits = self.searcher.search(query, num) + if len(hits) < 1: + if return_score: + return [], [] + else: + return [] + scores = [hit.score for hit in hits] + if len(hits) < num: + warnings.warn("Not enough documents retrieved!", stacklevel=2) + else: + hits = hits[:num] + + if self.contain_doc: + all_contents = [json.loads(self.searcher.doc(hit.docid).raw())["contents"] for hit in hits] + results = [ + { + "title": content.split("\n")[0].strip('"'), + "text": "\n".join(content.split("\n")[1:]), + "contents": content, + } + for content in all_contents + ] + else: + results = load_docs(self.corpus, [hit.docid for hit in hits]) + + if return_score: + return results, scores + else: + return results + + def _batch_search(self, query_list: list[str], num: int = None, return_score: bool = False): + results = [] + scores = [] + for query in query_list: + item_result, item_score = self._search(query, num, True) + results.append(item_result) + scores.append(item_score) + if return_score: + return results, scores + else: + return results + + +class DenseRetriever(BaseRetriever): + def __init__(self, config): + super().__init__(config) + self.index = faiss.read_index(self.index_path) + if config.faiss_gpu: + co = faiss.GpuMultipleClonerOptions() + co.useFloat16 = True + co.shard = True + self.index = faiss.index_cpu_to_all_gpus(self.index, co=co) + + self.corpus = load_corpus(self.corpus_path) + self.encoder = Encoder( + model_name=self.retrieval_method, + model_path=config.retrieval_model_path, + pooling_method=config.retrieval_pooling_method, + max_length=config.retrieval_query_max_length, + use_fp16=config.retrieval_use_fp16, + ) + self.topk = config.retrieval_topk + self.batch_size = config.retrieval_batch_size + + def _search(self, query: str, num: int = None, return_score: bool = False): + if num is None: + num = self.topk + query_emb = self.encoder.encode(query) + scores, idxs = self.index.search(query_emb, k=num) + idxs = idxs[0] + scores = scores[0] + results = load_docs(self.corpus, idxs) + if return_score: + return results, scores.tolist() + else: + return results + + def _batch_search(self, query_list: list[str], num: int = None, return_score: bool = False): + if isinstance(query_list, str): + query_list = [query_list] + if num is None: + num = self.topk + + results = [] + scores = [] + for start_idx in tqdm(range(0, len(query_list), self.batch_size), desc="Retrieval process: "): + query_batch = query_list[start_idx : start_idx + self.batch_size] + batch_emb = self.encoder.encode(query_batch) + batch_scores, batch_idxs = self.index.search(batch_emb, k=num) + batch_scores = batch_scores.tolist() + batch_idxs = batch_idxs.tolist() + + # load_docs is not vectorized, but is a python list approach + flat_idxs = sum(batch_idxs, []) + batch_results = load_docs(self.corpus, flat_idxs) + # chunk them back + batch_results = [batch_results[i * num : (i + 1) * num] for i in range(len(batch_idxs))] + + results.extend(batch_results) + scores.extend(batch_scores) + + del batch_emb, batch_scores, batch_idxs, query_batch, flat_idxs, batch_results + torch.cuda.empty_cache() + + if return_score: + return results, scores + else: + return results + + +def get_retriever(config): + if config.retrieval_method == "bm25": + return BM25Retriever(config) + else: + return DenseRetriever(config) + + +##################################### +# FastAPI server below +##################################### + + +class Config: + """ + Minimal config class (simulating your argparse) + Replace this with your real arguments or load them dynamically. + """ + + def __init__( + self, + retrieval_method: str = "bm25", + retrieval_topk: int = 10, + index_path: str = "./index/bm25", + corpus_path: str = "./data/corpus.jsonl", + dataset_path: str = "./data", + data_split: str = "train", + faiss_gpu: bool = True, + retrieval_model_path: str = "./model", + retrieval_pooling_method: str = "mean", + retrieval_query_max_length: int = 256, + retrieval_use_fp16: bool = False, + retrieval_batch_size: int = 128, + ): + self.retrieval_method = retrieval_method + self.retrieval_topk = retrieval_topk + self.index_path = index_path + self.corpus_path = corpus_path + self.dataset_path = dataset_path + self.data_split = data_split + self.faiss_gpu = faiss_gpu + self.retrieval_model_path = retrieval_model_path + self.retrieval_pooling_method = retrieval_pooling_method + self.retrieval_query_max_length = retrieval_query_max_length + self.retrieval_use_fp16 = retrieval_use_fp16 + self.retrieval_batch_size = retrieval_batch_size + + +class QueryRequest(BaseModel): + queries: list[str] + topk: Optional[int] = None + return_scores: bool = False + + +app = FastAPI() + + +@app.post("/retrieve") +def retrieve_endpoint(request: QueryRequest): + """ + Endpoint that accepts queries and performs retrieval. + + Input format: + { + "queries": ["What is Python?", "Tell me about neural networks."], + "topk": 3, + "return_scores": true + } + + Output format (when return_scores=True,similarity scores are returned): + { + "result": [ + [ # Results for each query + { + {"document": doc, "score": score} + }, + # ... more documents + ], + # ... results for other queries + ] + } + """ + if not request.topk: + request.topk = config.retrieval_topk # fallback to default + + # Perform batch retrieval + results, scores = retriever.batch_search( + query_list=request.queries, num=request.topk, return_score=request.return_scores + ) + + # Format response + resp = [] + for i, single_result in enumerate(results): + if request.return_scores: + # If scores are returned, combine them with results + combined = [] + for doc, score in zip(single_result, scores[i], strict=True): + combined.append({"document": doc, "score": score}) + resp.append(combined) + else: + resp.append(single_result) + return {"result": resp} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Launch the local faiss retriever.") + parser.add_argument( + "--index_path", type=str, default="/home/peterjin/mnt/index/wiki-18/e5_Flat.index", help="Corpus indexing file." + ) + parser.add_argument( + "--corpus_path", + type=str, + default="/home/peterjin/mnt/data/retrieval-corpus/wiki-18.jsonl", + help="Local corpus file.", + ) + parser.add_argument("--topk", type=int, default=3, help="Number of retrieved passages for one query.") + parser.add_argument("--retriever_name", type=str, default="e5", help="Name of the retriever model.") + parser.add_argument( + "--retriever_model", type=str, default="intfloat/e5-base-v2", help="Path of the retriever model." + ) + parser.add_argument("--faiss_gpu", action="store_true", help="Use GPU for computation") + + args = parser.parse_args() + + # 1) Build a config (could also parse from arguments). + # In real usage, you'd parse your CLI arguments or environment variables. + config = Config( + retrieval_method=args.retriever_name, # or "dense" + index_path=args.index_path, + corpus_path=args.corpus_path, + retrieval_topk=args.topk, + faiss_gpu=args.faiss_gpu, + retrieval_model_path=args.retriever_model, + retrieval_pooling_method="mean", + retrieval_query_max_length=256, + retrieval_use_fp16=True, + retrieval_batch_size=512, + ) + + # 2) Instantiate a global retriever so it is loaded once and reused. + retriever = get_retriever(config) + + # 3) Launch the server. By default, it listens on http://127.0.0.1:8000 + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/e2c/verl/examples/sglang_multiturn/search_r1_like/run_qwen2.5-3b_instruct_search_multiturn.sh b/e2c/verl/examples/sglang_multiturn/search_r1_like/run_qwen2.5-3b_instruct_search_multiturn.sh new file mode 100644 index 0000000000000000000000000000000000000000..4415e47a95316790202fed8a5f326dbecc22e466 --- /dev/null +++ b/e2c/verl/examples/sglang_multiturn/search_r1_like/run_qwen2.5-3b_instruct_search_multiturn.sh @@ -0,0 +1,66 @@ +# run on 8xH20 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + + +TRAIN_DATA="$HOME/data/searchR1_processed_direct/train.parquet" +VAL_DATA="$HOME/data/searchR1_processed_direct/test.parquet" + +TOOL_CONFIG="$CONFIG_PATH/tool_config/search_tool_config.yaml" + + + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='search_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=512 \ + data.val_batch_size=256 \ + data.max_prompt_length=4096 \ + data.max_response_length=3000 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.285 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.max_model_len=15000 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.val_before_train=False \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='search_r1_like_async_rl' \ + trainer.experiment_name='qwen2.5-3b-instruct_function_rm-search-async-sgl-multi-w-searchtool-verify-n16' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=100 \ + trainer.test_freq=50 \ + data.train_files="$TRAIN_DATA" \ + data.val_files="$VAL_DATA" \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$TOOL_CONFIG" \ + trainer.total_epochs=1 $@ + diff --git a/e2c/verl/examples/skypilot/README.md b/e2c/verl/examples/skypilot/README.md new file mode 100644 index 0000000000000000000000000000000000000000..78bd8458a83914a75c096dda8ef6e81e519981f1 --- /dev/null +++ b/e2c/verl/examples/skypilot/README.md @@ -0,0 +1,107 @@ +# verl with SkyPilot + +Run verl reinforcement learning training jobs on Kubernetes clusters or cloud platforms with GPU nodes using [SkyPilot](https://github.com/skypilot-org/skypilot). + +## Installation and Configuration + +### Step 1: Install SkyPilot + +Choose the installation based on your target platform: + +```bash +# For Kubernetes only +pip install "skypilot[kubernetes]" + +# For AWS +pip install "skypilot[aws]" + +# For Google Cloud Platform +pip install "skypilot[gcp]" + +# For Azure +pip install "skypilot[azure]" + +# For multiple platforms +pip install "skypilot[kubernetes,aws,gcp,azure]" +``` + +### Step 2: Configure Your Platform + +See https://docs.skypilot.co/en/latest/getting-started/installation.html + +### Step 3: Set Up Environment Variables + +Export necessary API keys for experiment tracking: + +```bash +# For Weights & Biases tracking +export WANDB_API_KEY="your-wandb-api-key" + +# For HuggingFace gated models (if needed) +export HF_TOKEN="your-huggingface-token" +``` + +## Examples + +### PPO Training +```bash +sky launch -c verl-ppo verl-ppo.yaml --secret WANDB_API_KEY -y +``` +Runs PPO training on GSM8K dataset using Qwen2.5-0.5B-Instruct model across 2 nodes with H100 GPUs. Based on examples in [`../ppo_trainer/`](../ppo_trainer/). + +### GRPO Training +```bash +sky launch -c verl-grpo verl-grpo.yaml --secret WANDB_API_KEY -y +``` +Runs GRPO (Group Relative Policy Optimization) training on MATH dataset using Qwen2.5-7B-Instruct model. Memory-optimized configuration for 2 nodes. Based on examples in [`../grpo_trainer/`](../grpo_trainer/). + +### Multi-turn Tool Usage Training +```bash +sky launch -c verl-multiturn verl-multiturn-tools.yaml --secret WANDB_API_KEY --secret HF_TOKEN -y +``` +Single-node training with 8xH100 GPUs for multi-turn tool usage with Qwen2.5-3B-Instruct. Includes tool and interaction configurations for GSM8K. Based on examples in [`../sglang_multiturn/`](../sglang_multiturn/) but uses vLLM instead of sglang. + +## Configuration + +The example YAML files are pre-configured with: + +- **Infrastructure**: Kubernetes clusters (`infra: k8s`) - can be changed to `infra: aws` or `infra: gcp`, etc. +- **Docker Image**: verl's official Docker image with CUDA 12.6 support +- **Setup**: Automatically clones and installs verl from source +- **Datasets**: Downloads required datasets during setup phase +- **Ray Cluster**: Configures distributed training across nodes +- **Logging**: Supports Weights & Biases via `--secret WANDB_API_KEY` +- **Models**: Supports gated HuggingFace models via `--secret HF_TOKEN` + +## Launch Command Options + +- `-c `: Cluster name for managing the job +- `--secret KEY`: Pass secrets for API keys (can be used multiple times) +- `-y`: Skip confirmation prompt + +## Monitoring Your Jobs + +### Check cluster status +```bash +sky status +``` + +### View logs +```bash +sky logs verl-ppo # View logs for the PPO job +``` + +### SSH into head node +```bash +ssh verl-ppo +``` + +### Access Ray dashboard +```bash +sky status --endpoint 8265 verl-ppo # Get dashboard URL +``` + +### Stop a cluster +```bash +sky down verl-ppo +``` diff --git a/e2c/verl/examples/skypilot/verl-grpo.yaml b/e2c/verl/examples/skypilot/verl-grpo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3d51855d1fd05befbffc7298bca8b6619d66d79 --- /dev/null +++ b/e2c/verl/examples/skypilot/verl-grpo.yaml @@ -0,0 +1,99 @@ +resources: + infra: k8s + accelerators: H100:1 + memory: 128+ + image_id: docker:verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.0-fa2.7.4 + ports: 8265 + +num_nodes: 2 + +secrets: + WANDB_API_KEY: + +setup: | + rm -rf verl + git clone https://github.com/volcengine/verl.git + cd verl + pip3 install -v -e .[vllm] + pip3 install flashinfer-python + echo "Downloading Math dataset..." + mkdir -p ~/data/math + python3 "$(pwd)/examples/data_preprocess/math_dataset.py" --local_dir ~/data/math + echo "Math dataset download completed" + +run: | + HEAD_IP=$(echo "$SKYPILOT_NODE_IPS" | head -n1) + NUM_NODES=$SKYPILOT_NUM_NODES + NUM_GPUS_PER_NODE=$SKYPILOT_NUM_GPUS_PER_NODE + + if [ "$SKYPILOT_NODE_RANK" == "0" ]; then + echo "Starting Ray head node..." + ps aux | grep ray | grep 6379 &> /dev/null || ray start --head --disable-usage-stats \ + --port=6379 \ + --dashboard-host=0.0.0.0 \ + --dashboard-port=8265 + + # Wait for all worker nodes to join + retry_count=0 + max_retries=30 + while [ $retry_count -lt $max_retries ]; do + connected_nodes=$(ray status 2>/dev/null | grep -c "node_" || echo "0") + echo "Connected nodes: $connected_nodes/$NUM_NODES (attempt $((retry_count+1))/$max_retries)" + + if [ "$connected_nodes" -ge "$NUM_NODES" ]; then + echo "All nodes connected to Ray cluster" + break + fi + + retry_count=$((retry_count+1)) + sleep 10 + done + + python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/math/train.parquet \ + data.val_files=$HOME/data/math/test.parquet \ + data.train_batch_size=32 \ + data.max_prompt_length=256 \ + data.max_response_length=256 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-7B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=16 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.ppo_epochs=1 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.rollout.n=1 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=2048 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=[console,wandb] \ + trainer.project_name=verl_math_grpo_demo \ + trainer.experiment_name=qwen25_7b_grpo \ + trainer.n_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.nnodes=$NUM_NODES \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=1 + + else + sleep 15 + echo "Starting Ray worker node..." + ps aux | grep ray | grep $HEAD_IP:6379 &> /dev/null || ray start --address $HEAD_IP:6379 --disable-usage-stats + sleep 10 + fi + + echo "Node setup and Ray start script finished for rank $SKYPILOT_NODE_RANK." \ No newline at end of file diff --git a/e2c/verl/examples/skypilot/verl-multiturn-tools.yaml b/e2c/verl/examples/skypilot/verl-multiturn-tools.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7496ad83061ab572e9d405a668276ba0004b0864 --- /dev/null +++ b/e2c/verl/examples/skypilot/verl-multiturn-tools.yaml @@ -0,0 +1,91 @@ +resources: + infra: k8s + accelerators: H100:8 + memory: 128+ + image_id: docker:verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.0-fa2.7.4 + ports: 8265 + +num_nodes: 1 + +secrets: + WANDB_API_KEY: + HF_TOKEN: # in case you're using gated models from the HF hub + +setup: | + rm -rf verl + git clone https://github.com/volcengine/verl.git + cd verl + pip3 install -v -e .[vllm] + pip3 install flashinfer-python + pip install "transformers<4.54.0" # https://github.com/vllm-project/vllm-ascend/issues/2046 + # Download GSM8K dataset for multiturn tool training + echo "Downloading GSM8K dataset..." + mkdir -p ~/data/gsm8k + python3 "$(pwd)/examples/data_preprocess/gsm8k.py" --local_dir ~/data/gsm8k + echo "GSM8K dataset download completed" + +run: | + NUM_GPUS_PER_NODE=$SKYPILOT_NUM_GPUS_PER_NODE + PROJECT_DIR="$(pwd)/verl" + CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + + # Single node setup - no worker coordination needed + echo "Starting Ray head node..." + ps aux | grep ray | grep 6379 &> /dev/null || ray start --head --disable-usage-stats \ + --port=6379 \ + --dashboard-host=0.0.0.0 \ + --dashboard-port=8265 + + cd verl + + python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=512 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=64 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=64 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=[console,wandb] \ + trainer.project_name=verl_multiturn_tools \ + trainer.experiment_name=qwen25_7b_gsm8k_multiturn_tools \ + trainer.n_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.nnodes=1 \ + trainer.save_freq=10 \ + trainer.test_freq=5 \ + trainer.total_epochs=10 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8192 \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=8192 \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192 \ + critic.ppo_max_token_len_per_gpu=8192 \ + critic.forward_max_token_len_per_gpu=8192 \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + actor_rollout_ref.rollout.multi_turn.interaction_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/interaction_config/gsm8k_interaction_config.yaml" \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=1 + + echo "Node setup and Ray start script finished for rank $SKYPILOT_NODE_RANK." \ No newline at end of file diff --git a/e2c/verl/examples/skypilot/verl-ppo.yaml b/e2c/verl/examples/skypilot/verl-ppo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1ba8de45aec6fcb19803b0c20c35f7c81f433d3 --- /dev/null +++ b/e2c/verl/examples/skypilot/verl-ppo.yaml @@ -0,0 +1,109 @@ +resources: + infra: k8s + accelerators: H100:1 + memory: 128+ + image_id: docker:verlai/verl:base-verl0.5-cu126-cudnn9.8-torch2.7.0-fa2.7.4 + ports: 8265 + +num_nodes: 2 + +secrets: + WANDB_API_KEY: + +setup: | + rm -rf verl + git clone https://github.com/volcengine/verl.git + cd verl + pip3 install -v -e .[vllm] + pip3 install flashinfer-python + # Download GSM8K dataset - alternative approach + echo "Downloading GSM8K dataset..." + mkdir -p ~/data/gsm8k + # Check if the script exists and use absolute path + if [ -f "$(pwd)/examples/data_preprocess/gsm8k.py" ]; then + python3 "$(pwd)/examples/data_preprocess/gsm8k.py" --local_dir ~/data/gsm8k + else + echo "Warning: gsm8k.py script not found, skipping dataset download" + # You might want to download the dataset manually or use a different approach + fi + echo "GSM8K dataset download completed" + +run: | + # Get the Head node's IP and total number of nodes + HEAD_IP=$(echo "$SKYPILOT_NODE_IPS" | head -n1) + NUM_NODES=$SKYPILOT_NUM_NODES + + # login wandb + # python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + if [ "$SKYPILOT_NODE_RANK" == "0" ]; then + # Head node starts Ray Head + echo "Starting Ray head node..." + ps aux | grep ray | grep 6379 &> /dev/null || ray start --head --disable-usage-stats \ + --port=6379 \ + --dashboard-host=0.0.0.0 \ + --dashboard-port=8265 + + # Wait for all worker nodes to join the cluster with better checking + echo "Waiting for all nodes to join Ray cluster..." + retry_count=0 + max_retries=30 + while [ $retry_count -lt $max_retries ]; do + connected_nodes=$(ray status 2>/dev/null | grep -c "node_" || echo "0") + echo "Connected nodes: $connected_nodes/$NUM_NODES (attempt $((retry_count+1))/$max_retries)" + + if [ "$connected_nodes" -ge "$NUM_NODES" ]; then + echo "All nodes connected to Ray cluster" + break + fi + + retry_count=$((retry_count+1)) + sleep 10 + done + + if [ $retry_count -eq $max_retries ]; then + echo "WARNING: Not all nodes connected to Ray cluster after $max_retries attempts" + echo "Current Ray status:" + ray status + fi + + python3 -m verl.trainer.main_ppo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=512 \ + data.max_response_length=256 \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=[console,wandb] \ + trainer.val_before_train=False \ + trainer.default_hdfs_dir=null \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=2 \ + trainer.save_freq=20 \ + trainer.test_freq=20 \ + trainer.total_epochs=2 \ + trainer.project_name=verl_examples \ + trainer.experiment_name=experiment_name_gsm8k + + else + # Wait for Ray Head to start + sleep 15 + # Worker node starts Ray Worker + echo "Starting Ray worker node..." + ps aux | grep ray | grep $HEAD_IP:6379 &> /dev/null || ray start --address $HEAD_IP:6379 --disable-usage-stats + sleep 10 + fi + + echo "Node setup and Ray start script finished for rank $SKYPILOT_NODE_RANK." \ No newline at end of file diff --git a/e2c/verl/examples/slurm/ray_on_slurm.slurm b/e2c/verl/examples/slurm/ray_on_slurm.slurm new file mode 100644 index 0000000000000000000000000000000000000000..86567d811be50e583dd715a3a60cf0053451e891 --- /dev/null +++ b/e2c/verl/examples/slurm/ray_on_slurm.slurm @@ -0,0 +1,98 @@ +#!/bin/bash +#SBATCH --job-name=verl-ray-on-slurm +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=1 +#SBATCH --mem=200G +#SBATCH --partition=your-partition +#SBATCH --time=01:00:00 +#SBATCH --account=your-account +#SBATCH --gpus-per-node=4 +#SBATCH --cpus-per-task=64 +#SBATCH --output=slurm-%j.out +#SBATCH --error=slurm-%j.err + +# load necessary modules + +# replace these information with your own +verl_workdir=/path/to/verl +train_files=/path/to/gsm8k/train.parquet +val_files=/path/to/gsm8k/test.parquet +apptainer_image_path=/path/to/verl-ngc.sif +# replace these information with your own + +# Getting the node names +nodes=$(scontrol show hostnames "$SLURM_JOB_NODELIST") +nodes_array=("$nodes") + +head_node=${nodes_array[0]} +head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) + +# if we detect a space character in the head node IP, we'll +# convert it to an ipv4 address. This step is optional. +if [[ "$head_node_ip" == *" "* ]]; then +IFS=' ' read -ra ADDR <<<"$head_node_ip" +if [[ ${#ADDR[0]} -gt 16 ]]; then + head_node_ip=${ADDR[1]} +else + head_node_ip=${ADDR[0]} +fi +echo "IPV6 address detected. We split the IPV4 address as $head_node_ip" +fi + +port=6379 +ip_head=$head_node_ip:$port +export ip_head +echo "IP Head: $ip_head" + +# make sure we set environment variables before Ray initialization + +printenv + +echo "Starting HEAD at $head_node" +srun --nodes=1 --ntasks=1 -w "$head_node" \ + apptainer run --nv --bind $verl_workdir $apptainer_image_path \ + ray start --head --node-ip-address="$head_node_ip" --port=$port \ + --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & +# optional, though may be useful in certain versions of Ray < 1.0. +sleep 10 + +# number of nodes other than the head node +worker_num=$((SLURM_JOB_NUM_NODES - 1)) + +for ((i = 1; i <= worker_num; i++)); do + node_i=${nodes_array[$i]} + echo "Starting WORKER $i at $node_i" + srun --nodes=1 --ntasks=1 -w "$node_i" \ + apptainer run --nv --bind $verl_workdir $apptainer_image_path \ + ray start --address "$ip_head" --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & + sleep 5 +done + +PYTHONUNBUFFERED=1 srun --overlap --nodes=1 --ntasks=1 -w "$head_node" \ + apptainer run --nv --bind $verl_workdir $apptainer_image_path \ + python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$train_files \ + data.val_files=$val_files \ + data.train_batch_size=256 \ + data.max_prompt_length=512 \ + data.max_response_length=256 \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.use_kl_in_reward=False \ + trainer.logger=console \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node="${SLURM_GPUS_PER_NODE}" \ + trainer.nnodes="${SLURM_NNODES}" \ + trainer.save_freq=10 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 2>&1 | tee verl_demo_slurm.log diff --git a/e2c/verl/examples/split_placement/README.md b/e2c/verl/examples/split_placement/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a552972594f9ddd142d6889cdee1a5def55c2939 --- /dev/null +++ b/e2c/verl/examples/split_placement/README.md @@ -0,0 +1,61 @@ +# Split Placement Example +Here we introduce how to run the naive implementation of the split placement of PPO algorithm. +We will release the complete version of flexible placement in the near future. + + For quickstart, you can only follow Step 2 to modify the code and then follow Step 4 to execute the split placement example. + +### Step 1: Placing the models to different GPUs +Specify the placement and resource allocation. In the example, we place the actor and reference in the first half of the GPUs while map the critic and reward model (if any) to the second half of the GPUs. +```python +actor_rollout_ref_pool_id = 'actor_rollout_ref_pool' +critic_pool_id = 'critic_pool' +if config.trainer.nnodes // 2 == 0 and config.trainer.n_gpus_per_node // 2 > 0: + resource_pool_spec = { + actor_rollout_ref_pool_id: [config.trainer.n_gpus_per_node // 2] * config.trainer.nnodes, + critic_pool_id: [config.trainer.n_gpus_per_node // 2] * config.trainer.nnodes, + } +else: + resource_pool_spec = { + actor_rollout_ref_pool_id: [config.trainer.n_gpus_per_node] * (config.trainer.nnodes // 2), + critic_pool_id: [config.trainer.n_gpus_per_node] * (config.trainer.nnodes // 2), + } +print(f'resource_pool_spec: {resource_pool_spec}') +mapping = { + Role.ActorRollout: actor_rollout_ref_pool_id, + Role.Critic: critic_pool_id, + Role.RefPolicy: actor_rollout_ref_pool_id, +} +mapping[Role.RewardModel] = critic_pool_id +``` + +### Step 2: Make the models executed asynchronously +Based on the model placement, we need to make the models executed asynchronously. + +To do so, you need to turn off the `blocking` flag (i.e., `blocking=False`) in our decorator of some model operations. +For example, we hope the actor update and critic update can be executed in parallel, then we need to make the following modification in `fsdp_workers.py` + +``` +@register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO, blocking=False) +def update_actor(self, data: DataProto): + ... + +@register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO, blocking=False) +def update_critic(self, data: DataProto): + ... +``` + +We can also parallelize the computation of `ref_log_prob` and `values` and `rewards` in the split placement. For simplicity of the tutorial, we don't do this in this example. + +### Step 3: Execute these operation in parallel in the single controller process +To implement the parallel execution of the actor and critic update, the only thing we need to modify in the `ray_trainer.py` is to `get` the concurrent `futures` on the single controller process. + +```python +critic_output = critic_output.get() +actor_output = actor_output.get() +``` + +### Step 4: Run the split placement example + +``` +bash run_deepseek7b_llm.sh +``` diff --git a/e2c/verl/examples/split_placement/config/ppo_trainer_split.yaml b/e2c/verl/examples/split_placement/config/ppo_trainer_split.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a27afb1508351c000ae5c515214dd31ee0f2755c --- /dev/null +++ b/e2c/verl/examples/split_placement/config/ppo_trainer_split.yaml @@ -0,0 +1,188 @@ +# the ppo trainer split config will override default ppo_trainer.yaml + +hydra: + searchpath: + - file://../../verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + tokenizer: null + train_files: ~/data/rlhf/gsm8k/train.parquet + val_files: ~/data/rlhf/gsm8k/test.parquet + prompt_key: prompt + max_prompt_length: 512 + max_response_length: 512 + train_batch_size: 1024 + val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves + return_raw_input_ids: False # This should be set to true when the tokenizer between policy and rm differs + return_raw_chat: False + return_full_prompt: False + shuffle: True + +actor_rollout_ref: + hybrid_engine: True + model: + path: ~/models/deepseek-llm-7b-chat + external_lib: null + override_config: { } + enable_gradient_checkpointing: True + use_remove_padding: False + actor: + strategy: fsdp # This is for backward-compatibility + ppo_mini_batch_size: 256 + ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu + ppo_micro_batch_size_per_gpu: null + use_dynamic_bsz: False + ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length} + grad_clip: 1.0 + clip_ratio: 0.2 + entropy_coeff: 0.0 + use_kl_loss: False # True for GRPO + kl_loss_coef: 0.001 # for grpo + kl_loss_type: low_var_kl # for grpo + ppo_epochs: 1 + shuffle: False + ulysses_sequence_parallel_size: 1 # sp size + optim: + lr: 1e-6 + lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime + min_lr_ratio: null # only useful for warmup with cosine + warmup_style: constant # select from constant/cosine + total_training_steps: -1 # must be override by program + fsdp_config: + wrap_policy: + # transformer_layer_cls_to_wrap: None + min_num_params: 0 + param_offload: False + optimizer_offload: False + fsdp_size: -1 + ref: + fsdp_config: + param_offload: False + wrap_policy: + # transformer_layer_cls_to_wrap: None + min_num_params: 0 + log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} + ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size} # sp size + rollout: + name: vllm + temperature: 1.0 + top_k: -1 # 0 for hf rollout, -1 for vllm rollout + top_p: 1 + prompt_length: ${data.max_prompt_length} # not use for opensource + response_length: ${data.max_response_length} + # for vllm rollout + dtype: bfloat16 # should align with FSDP + gpu_memory_utilization: 0.5 + ignore_eos: False + enforce_eager: True + free_cache_engine: True + load_format: dummy_dtensor + tensor_model_parallel_size: 2 + max_num_batched_tokens: 8192 + max_num_seqs: 1024 + log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} + disable_log_stats: True + enable_chunked_prefill: True # could get higher throughput + # for hf rollout + do_sample: True + # number of responses (i.e. num sample times) + n: 1 # > 1 for grpo + +critic: + strategy: fsdp + optim: + lr: 1e-5 + lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime + min_lr_ratio: null # only useful for warmup with cosine + warmup_style: constant # select from constant/cosine + total_training_steps: -1 # must be override by program + model: + path: ~/models/deepseek-llm-7b-chat + tokenizer_path: ${actor_rollout_ref.model.path} + override_config: { } + external_lib: ${actor_rollout_ref.model.external_lib} + enable_gradient_checkpointing: True + use_remove_padding: False + fsdp_config: + param_offload: False + optimizer_offload: False + wrap_policy: + # transformer_layer_cls_to_wrap: None + min_num_params: 0 + fsdp_size: -1 + ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} + ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu + ppo_micro_batch_size_per_gpu: null + forward_micro_batch_size: ${critic.ppo_micro_batch_size} + forward_micro_batch_size_per_gpu: ${critic.ppo_micro_batch_size_per_gpu} + use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + ppo_max_token_len_per_gpu: 32768 # (${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}) * 2 + forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu} + ulysses_sequence_parallel_size: 1 # sp size + ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs} + shuffle: ${actor_rollout_ref.actor.shuffle} + grad_clip: 1.0 + cliprange_value: 0.5 + +reward_model: + enable: False + strategy: fsdp + model: + input_tokenizer: ${actor_rollout_ref.model.path} # set this to null if the chat template is identical + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + external_lib: ${actor_rollout_ref.model.external_lib} + use_remove_padding: False + fsdp_config: + min_num_params: 0 + param_offload: False + fsdp_size: -1 + micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu + micro_batch_size_per_gpu: null # set a number + max_length: null + ulysses_sequence_parallel_size: 1 # sp size + use_dynamic_bsz: ${critic.use_dynamic_bsz} + forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + reward_manager: naive + +algorithm: + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + use_kl_in_reward: False + kl_penalty: kl # how to estimate kl divergence + kl_ctrl: + type: fixed + kl_coef: 0.001 + +trainer: + total_epochs: 30 + total_training_steps: null + project_name: verl_examples + experiment_name: gsm8k + logger: [ 'console', 'wandb' ] + log_val_generations: 0 + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + # auto: find the last ckpt to resume. If can't find, start from scratch + resume_mode: auto # or disable or resume_path if resume_from_path is set + resume_from_path: null + test_freq: -1 + critic_warmup: 0 + default_hdfs_dir: null + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. diff --git a/e2c/verl/examples/split_placement/main_ppo_split.py b/e2c/verl/examples/split_placement/main_ppo_split.py new file mode 100644 index 0000000000000000000000000000000000000000..3e610abd2b06bd8f1587b9138109597974f386bc --- /dev/null +++ b/e2c/verl/examples/split_placement/main_ppo_split.py @@ -0,0 +1,216 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. +""" + +import hydra +import ray +import torch +from omegaconf import OmegaConf +from split_monkey_patch import fit + +from verl import DataProto +from verl.trainer.ppo.ray_trainer import RayPPOTrainer +from verl.utils.reward_score import gsm8k, math_reward + + +def _select_rm_score_fn(data_source): + if data_source == "openai/gsm8k": + return gsm8k.compute_score + elif data_source == "lighteval/MATH": + return math_reward.compute_score + else: + raise NotImplementedError + + +class RewardManager: + def __init__(self, tokenizer, num_examine) -> None: + self.tokenizer = tokenizer + self.num_examine = num_examine # the number of batches of decoded responses to print to the console + + def __call__(self, data: DataProto, return_dict: bool = False): + """We will expand this function gradually based on the available datasets""" + + # If there is rm score, we directly return rm score. Otherwise, we compute via rm_score_fn + if "rm_scores" in data.batch.keys(): + return data.batch["rm_scores"] + + reward_tensor = torch.zeros_like(data.batch["responses"], dtype=torch.float32) + + already_print_data_sources = {} + + for i in range(len(data)): + data_item = data[i] # DataProtoItem + + prompt_ids = data_item.batch["prompts"] + + prompt_length = prompt_ids.shape[-1] + + valid_prompt_length = data_item.batch["attention_mask"][:prompt_length].sum() + valid_prompt_ids = prompt_ids[-valid_prompt_length:] + + response_ids = data_item.batch["responses"] + valid_response_length = data_item.batch["attention_mask"][prompt_length:].sum() + valid_response_ids = response_ids[:valid_response_length] + + # decode + sequences = torch.cat((valid_prompt_ids, valid_response_ids)) + sequences_str = self.tokenizer.decode(sequences) + + ground_truth = data_item.non_tensor_batch["reward_model"]["ground_truth"] + + # select rm_score + data_source = data_item.non_tensor_batch["data_source"] + compute_score_fn = _select_rm_score_fn(data_source) + + score = compute_score_fn(solution_str=sequences_str, ground_truth=ground_truth) + reward_tensor[i, valid_response_length - 1] = score + + if data_source not in already_print_data_sources: + already_print_data_sources[data_source] = 0 + + if already_print_data_sources[data_source] < self.num_examine: + already_print_data_sources[data_source] += 1 + print(sequences_str) + + if return_dict: + return {"reward_tensor": reward_tensor} + else: + return reward_tensor + + +@hydra.main(config_path="config", config_name="ppo_trainer_split", version_base=None) +def main(config): + if not ray.is_initialized(): + # this is for local ray cluster + default_runtime_env = {"env_vars": {"TOKENIZERS_PARALLELISM": "true", "NCCL_DEBUG": "WARN"}} + ray_init_kwargs = config.ray_kwargs.get("ray_init", {}) + runtime_env_kwargs = ray_init_kwargs.get("runtime_env", {}) + runtime_env = OmegaConf.merge(default_runtime_env, runtime_env_kwargs) + ray_init_kwargs = OmegaConf.create({**ray_init_kwargs, "runtime_env": runtime_env}) + print(f"ray init kwargs: {ray_init_kwargs}") + ray.init(**OmegaConf.to_container(ray_init_kwargs)) + + ray.get(main_task.remote(config)) + + +@ray.remote +def main_task(config): + # print initial config + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_to_local + + pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values + OmegaConf.resolve(config) + + # download the checkpoint from hdfs + local_path = copy_to_local(config.actor_rollout_ref.model.path) + + # instantiate tokenizer + from verl.utils import hf_tokenizer + + tokenizer = hf_tokenizer(local_path) + + # define worker classes + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + assert config.critic.strategy in {"fsdp", "fsdp2"} + from verl.single_controller.ray import RayWorkerGroup + from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker + + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.single_controller.ray import RayWorkerGroup + from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker + + ray_worker_group_cls = RayWorkerGroup + + else: + raise NotImplementedError + + from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role + + role_worker_mapping = { + Role.ActorRollout: ray.remote(ActorRolloutRefWorker), + Role.Critic: ray.remote(CriticWorker), + } + + # NOTE: initialze two resource pool + actor_rollout_ref_pool_id = "actor_rollout_ref_pool" + critic_pool_id = "critic_pool" + if config.trainer.nnodes // 2 == 0 and config.trainer.n_gpus_per_node // 2 > 0: + resource_pool_spec = { + actor_rollout_ref_pool_id: [config.trainer.n_gpus_per_node // 2] * config.trainer.nnodes, + critic_pool_id: [config.trainer.n_gpus_per_node // 2] * config.trainer.nnodes, + } + else: + resource_pool_spec = { + actor_rollout_ref_pool_id: [config.trainer.n_gpus_per_node] * (config.trainer.nnodes // 2), + critic_pool_id: [config.trainer.n_gpus_per_node] * (config.trainer.nnodes // 2), + } + print(f"resource_pool_spec: {resource_pool_spec}") + mapping = { + Role.ActorRollout: actor_rollout_ref_pool_id, + Role.Critic: critic_pool_id, + } + + # use reference model + if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) + mapping[Role.RefPolicy] = actor_rollout_ref_pool_id + + # we should adopt a multi-source reward function here + # - for rule-based rm, we directly call a reward score + # - for model-based rm, we call a model + # - for code related prompt, we send to a sandbox if there are test cases + # - finally, we combine all the rewards together + # - The reward type depends on the tag of the data + if config.reward_model.enable: + if config.reward_model.strategy in {"fsdp", "fsdp2"}: + from verl.workers.fsdp_workers import RewardModelWorker + elif config.reward_model.strategy == "megatron": + from verl.workers.megatron_workers import RewardModelWorker + else: + raise NotImplementedError + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + mapping[Role.RewardModel] = critic_pool_id + + reward_fn = RewardManager(tokenizer=tokenizer, num_examine=0) + + # Note that we always use function-based RM for validation + val_reward_fn = RewardManager(tokenizer=tokenizer, num_examine=1) + + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + + RayPPOTrainer.fit = fit + trainer = RayPPOTrainer( + config=config, + tokenizer=tokenizer, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + ) + trainer.init_workers() + trainer.fit() + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/examples/split_placement/run_deepseek7b_llm.sh b/e2c/verl/examples/split_placement/run_deepseek7b_llm.sh new file mode 100644 index 0000000000000000000000000000000000000000..473dcccdd9bb355b43c93700bc0ccbe3de379b57 --- /dev/null +++ b/e2c/verl/examples/split_placement/run_deepseek7b_llm.sh @@ -0,0 +1,37 @@ +set -x + +python3 main_ppo_split.py \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + critic.optim.lr=1e-5 \ + critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ + critic.model.enable_gradient_checkpointing=False \ + critic.ppo_micro_batch_size_per_gpu=8 \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_example_gsm8k' \ + trainer.experiment_name='deepseek_llm_7b_function_rm' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/split_placement/split_monkey_patch.py b/e2c/verl/examples/split_placement/split_monkey_patch.py new file mode 100644 index 0000000000000000000000000000000000000000..a1a0e43196fafd1b1b2d0ae8639b775a045c31ce --- /dev/null +++ b/e2c/verl/examples/split_placement/split_monkey_patch.py @@ -0,0 +1,228 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +An naive implementation of split placment example +""" + +import uuid +from copy import deepcopy +from pprint import pprint + +import numpy as np +import torch + +from verl import DataProto +from verl.trainer.ppo.ray_trainer import ( + AdvantageEstimator, + apply_kl_penalty, + compute_advantage, + compute_data_metrics, + compute_timing_metrics, + marked_timer, +) +from verl.utils.metric import reduce_metrics + + +def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the worker group through RPC + to construct the PPO dataflow. + The light-weight advantage computation is done on the driver process. + """ + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + self.global_steps = 0 + + # load checkpoint before doing anything + self._load_checkpoint() + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + val_metrics = self._validate() + pprint(f"Initial validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if self.config.trainer.get("val_only", False): + return + + # we start from step 1 + self.global_steps += 1 + last_val_metrics = None + + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + metrics = {} + timing_raw = {} + + batch: DataProto = DataProto.from_single_dict(batch_dict) + + # pop those keys for generation + gen_batch = batch.pop(batch_keys=["input_ids", "attention_mask", "position_ids"]) + is_last_step = self.global_steps >= self.total_training_steps + + with marked_timer("step", timing_raw): + # generate a batch + with marked_timer("gen", timing_raw): + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + timing_raw.update(gen_batch_output.meta_info["timing"]) + gen_batch_output.meta_info.pop("timing", None) + + if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: + with marked_timer("gen_max", timing_raw): + gen_baseline_batch = deepcopy(gen_batch) + gen_baseline_batch.meta_info["do_sample"] = False + gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) + + batch = batch.union(gen_baseline_output) + reward_baseline_tensor = self.reward_fn(batch) + reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) + + batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) + + batch.batch["reward_baselines"] = reward_baseline_tensor + + del gen_baseline_batch, gen_baseline_output + + batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(batch.batch))], dtype=object + ) + # repeat to align with repeated responses in rollout + batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + batch = batch.union(gen_batch_output) + + # Balance the number of valid tokens across DP ranks. + # NOTE: This usually changes the order of data in the `batch`, + # which won't affect the advantage calculation (since it's based on uid), + # but might affect the loss calculation (due to the change of mini-batching). + # TODO: Decouple the DP balancing and mini-batching. + self._balance_batch(batch, metrics=metrics) + + # compute global_valid tokens + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + # recompute old_log_probs + with marked_timer("old_log_prob", timing_raw): + old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) + batch = batch.union(old_log_prob) + + if self.use_reference_policy: + # compute reference log_prob + with marked_timer("ref", timing_raw): + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + + # compute values + if self.use_critic: + with marked_timer("values", timing_raw): + values = self.critic_wg.compute_values(batch) + batch = batch.union(values) + + with marked_timer("adv", timing_raw): + # compute scores. Support both model and function-based. + # We first compute the scores using reward model. Then, we call reward_fn to combine + # the results from reward model and rule-based results. + if self.use_rm: + # we first compute reward model score + reward_tensor = self.rm_wg.compute_rm_score(batch) + batch = batch.union(reward_tensor) + + # we combine with rule-based rm + reward_tensor = self.reward_fn(batch) + batch.batch["token_level_scores"] = reward_tensor + + # compute rewards. apply_kl_penalty if available + if self.config.algorithm.use_kl_in_reward: + batch, kl_metrics = apply_kl_penalty( + batch, kl_ctrl=self.kl_ctrl_in_reward, kl_penalty=self.config.algorithm.kl_penalty + ) + metrics.update(kl_metrics) + else: + batch.batch["token_level_rewards"] = batch.batch["token_level_scores"] + + # compute advantages, executed on the driver process + norm_adv_by_std_in_grpo = self.config.algorithm.get("norm_adv_by_std_in_grpo", True) + batch = compute_advantage( + batch, + adv_estimator=self.config.algorithm.adv_estimator, + gamma=self.config.algorithm.gamma, + lam=self.config.algorithm.lam, + num_repeat=self.config.actor_rollout_ref.rollout.n, + norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo, + config=self.config.algorithm, + ) + + # implement critic warmup + if self.config.trainer.critic_warmup <= self.global_steps: + # update actor + with marked_timer("update_actor_call", timing_raw): + actor_output = self.actor_rollout_wg.update_actor(batch) + else: + actor_output = None + + # update critic + if self.use_critic: + with marked_timer("update_critic_call", timing_raw): + critic_output = self.critic_wg.update_critic(batch) + + # NOTE: make sure you set blocking=False in update_actor and update_crtic in the worker class + with marked_timer("update_actor_critic", timing_raw): + critic_output = critic_output.get() + critic_output_metrics = reduce_metrics(critic_output.meta_info["metrics"]) + metrics.update(critic_output_metrics) + + if actor_output is not None: + actor_output = actor_output.get() + actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"]) + metrics.update(actor_output_metrics) + + # validate + if ( + self.val_reward_fn is not None + and self.config.trainer.test_freq > 0 + and (is_last_step or self.global_steps % self.config.trainer.test_freq == 0) + ): + with marked_timer("testing", timing_raw): + val_metrics: dict = self._validate() + if is_last_step: + last_val_metrics = val_metrics + metrics.update(val_metrics) + + if self.config.trainer.save_freq > 0 and ( + is_last_step or self.global_steps % self.config.trainer.save_freq == 0 + ): + with marked_timer("save_checkpoint", timing_raw): + self._save_checkpoint() + + # collect metrics + metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) + metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=self.global_steps) + + if self.global_steps >= self.total_training_steps: + pprint(f"Final validation metrics: {last_val_metrics}") + return + + self.global_steps += 1 diff --git a/e2c/verl/examples/tuning/0.5b/qwen2-0.5b_grpo-lora_1_h100_fsdp_vllm.sh b/e2c/verl/examples/tuning/0.5b/qwen2-0.5b_grpo-lora_1_h100_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..6105bd1623ebf85201571b68ebe6f9073075aa68 --- /dev/null +++ b/e2c/verl/examples/tuning/0.5b/qwen2-0.5b_grpo-lora_1_h100_fsdp_vllm.sh @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +export CUDA_VISIBLE_DEVICES=4 +NOW=$(date +%Y%m%d) +export WANDB_DIR=gsm8k-grpo-lora-qwen2.5-0.5b-${NOW} +export WANDB_PROJECT=${WANDB_DIR} +export WANDB_EXP=0.5b-${NOW} +MODEL_PATH=Qwen/Qwen2.5-0.5B-Instruct + +set -x +nproc_per_gpu=1 +nnodes=1 +ngpu_per_node=1 +total_procs=$(( nproc_per_gpu * nnodes * ngpu_per_node )) +mini_batch_size=$(( total_procs )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + trainer.val_before_train=False \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=${total_procs} \ + data.val_batch_size=${total_procs} \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=32 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.actor.optim.lr=3e-5 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${mini_batch_size} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.1 \ + actor_rollout_ref.rollout.n=1 \ + actor_rollout_ref.rollout.max_num_seqs=512 \ + actor_rollout_ref.rollout.max_model_len=1536 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1536 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.entropy_coeff=0.001 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=${WANDB_PROJECT} \ + trainer.experiment_name=${WANDB_EXP} \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ 2>&1 | tee ${WANDB_PROJECT}.log diff --git a/e2c/verl/examples/tuning/1.5b/qwen2-1.5b_grpo-lora_1_h100_fsdp_vllm.sh b/e2c/verl/examples/tuning/1.5b/qwen2-1.5b_grpo-lora_1_h100_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..6b6ede29bcb3652e4dab7a3497c4d9a50270526b --- /dev/null +++ b/e2c/verl/examples/tuning/1.5b/qwen2-1.5b_grpo-lora_1_h100_fsdp_vllm.sh @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +NOW=$(date +%Y%m%d) +export WANDB_DIR=gsm8k-grpo-lora-qwen2.5-1.5b-${NOW} +export WANDB_PROJECT=${WANDB_DIR} +export WANDB_EXP=1.5b-${NOW} +MODEL_PATH=Qwen/Qwen2.5-1.5B-Instruct + +set -x +nproc_per_gpu=128 +nnodes=1 +ngpu_per_node=1 +total_procs=$(( nproc_per_gpu * nnodes * ngpu_per_node )) +mini_batch_size=$(( total_procs )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=${total_procs} \ + data.val_batch_size=${total_procs} \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.use_shm=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=32 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.actor.optim.lr=3e-5 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.1 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.max_num_seqs=512 \ + actor_rollout_ref.rollout.max_model_len=1536 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1536 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.entropy_coeff=0.001 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=${WANDB_PROJECT} \ + trainer.experiment_name=${WANDB_EXP} \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ 2>&1 | tee ${WANDB_PROJECT}.log diff --git a/e2c/verl/examples/tuning/14b/qwen2-14b_grpo-lora_2_h100_fsdp_vllm.sh b/e2c/verl/examples/tuning/14b/qwen2-14b_grpo-lora_2_h100_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..247945ffc41c922d40e75351ade95d266baa90cf --- /dev/null +++ b/e2c/verl/examples/tuning/14b/qwen2-14b_grpo-lora_2_h100_fsdp_vllm.sh @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +NOW=$(date +%Y%m%d) +export WANDB_DIR=gsm8k-grpo-lora-qwen2.5-14b-${NOW} +export WANDB_PROJECT=${WANDB_DIR} +export WANDB_EXP=14b-${NOW} +MODEL_PATH=Qwen/Qwen2.5-14B-Instruct + +set -x +nproc_per_gpu=58 # 32√ → 64× → 48√ → 56√ → 60× → 58√ → 59× +nnodes=1 +ngpu_per_node=2 +total_procs=$(( nproc_per_gpu * nnodes * ngpu_per_node )) +mini_batch_size=$(( total_procs )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=${total_procs} \ + data.val_batch_size=${total_procs} \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.use_shm=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=32 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.actor.optim.lr=3e-5 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.25 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.max_num_seqs=512 \ + actor_rollout_ref.rollout.max_model_len=1536 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1536 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ + actor_rollout_ref.actor.entropy_coeff=0.001 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=${WANDB_PROJECT} \ + trainer.experiment_name=${WANDB_EXP} \ + trainer.n_gpus_per_node=2 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ 2>&1 | tee ${WANDB_PROJECT}.log diff --git a/e2c/verl/examples/tuning/14b/qwen2_14b_grpo_4_h800_fsdp_vllm.sh b/e2c/verl/examples/tuning/14b/qwen2_14b_grpo_4_h800_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..2df21533c5b94684feed43c44383493086fae3dd --- /dev/null +++ b/e2c/verl/examples/tuning/14b/qwen2_14b_grpo_4_h800_fsdp_vllm.sh @@ -0,0 +1,47 @@ +set -x + +gsm8k_train_path=$HOME/data/rlhf/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/rlhf/math/test.parquet +model_path=Qwen/Qwen2.5-Coder-14B-Instruct + +train_files="['$gsm8k_train_path']" +test_files="['$gsm8k_test_path']" + +PYTHONPATH=/opt/tiger/open_verl python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=$model_path \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_14b_function_rm' \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ diff --git a/e2c/verl/examples/tuning/32b/qwen2-32b_grpo-lora_4_h100_fsdp_vllm.sh b/e2c/verl/examples/tuning/32b/qwen2-32b_grpo-lora_4_h100_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..d707a4adcc0941daa1d620944a584c619003345d --- /dev/null +++ b/e2c/verl/examples/tuning/32b/qwen2-32b_grpo-lora_4_h100_fsdp_vllm.sh @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +NOW=$(date +%Y%m%d) +export WANDB_DIR=gsm8k-grpo-lora-qwen2.5-32b-${NOW} +export WANDB_PROJECT=${WANDB_DIR} +export WANDB_EXP=32b-${NOW} +MODEL_PATH=Qwen/Qwen2.5-32B-Instruct + +set -x +nproc_per_gpu=45 # 32√ → 64× → 48× → 40√ → 44√ → 46× → 45× +nnodes=1 +ngpu_per_node=4 +total_procs=$(( nproc_per_gpu * nnodes * ngpu_per_node )) +mini_batch_size=$(( total_procs )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=${total_procs} \ + data.val_batch_size=${total_procs} \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.use_shm=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=32 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.actor.optim.lr=3e-5 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.max_num_seqs=512 \ + actor_rollout_ref.rollout.max_model_len=1536 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1536 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ + actor_rollout_ref.actor.entropy_coeff=0.001 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=${WANDB_PROJECT} \ + trainer.experiment_name=${WANDB_EXP} \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ 2>&1 | tee ${WANDB_PROJECT}.log diff --git a/e2c/verl/examples/tuning/32b/qwen2_32B_grpo_8_h20_megatron_vllm.sh b/e2c/verl/examples/tuning/32b/qwen2_32B_grpo_8_h20_megatron_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..93a90665d6d0a8de36796d5474827cb30405f027 --- /dev/null +++ b/e2c/verl/examples/tuning/32b/qwen2_32B_grpo_8_h20_megatron_vllm.sh @@ -0,0 +1,51 @@ +set -x + +# we need this to avoid fragmentation of GPU memory +export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:256 + +gsm8k_train_path=$HOME/data/rlhf/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/rlhf/math/test.parquet +train_files="['$gsm8k_train_path']" +test_files="['$gsm8k_test_path']" + +model_path=Qwen/Qwen2.5-32B + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=512 \ + data.max_prompt_length=2048 \ + data.max_response_length=6144 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=$model_path \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=8 \ + actor_rollout_ref.actor.megatron.param_offload=True \ + actor_rollout_ref.actor.megatron.grad_offload=True \ + actor_rollout_ref.actor.megatron.optimizer_offload=True \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=8 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.megatron.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='megatron_vllm_qwen2_32b' \ + trainer.experiment_name='qwen2_32b_grpo_8_h20' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/examples/tuning/3b/qwen2-3b_grpo-lora_1_h100_fsdp_vllm.sh b/e2c/verl/examples/tuning/3b/qwen2-3b_grpo-lora_1_h100_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..fac34a5d537861f3c0a928fc3cb4730c0b190414 --- /dev/null +++ b/e2c/verl/examples/tuning/3b/qwen2-3b_grpo-lora_1_h100_fsdp_vllm.sh @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +NOW=$(date +%Y%m%d) +export WANDB_DIR=gsm8k-grpo-lora-qwen2.5-3b-${NOW} +export WANDB_PROJECT=${WANDB_DIR} +export WANDB_EXP=3b-${NOW} +MODEL_PATH=Qwen/Qwen2.5-3B-Instruct + +set -x +nproc_per_gpu=62 +nnodes=1 +ngpu_per_node=1 +total_procs=$(( nproc_per_gpu * nnodes * ngpu_per_node )) +mini_batch_size=$(( total_procs )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=${total_procs} \ + data.val_batch_size=${total_procs} \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.use_shm=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=32 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.actor.optim.lr=3e-5 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.1 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.max_num_seqs=512 \ + actor_rollout_ref.rollout.max_model_len=1536 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1536 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.entropy_coeff=0.001 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=${WANDB_PROJECT} \ + trainer.experiment_name=${WANDB_EXP} \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ 2>&1 | tee ${WANDB_PROJECT}.log diff --git a/e2c/verl/examples/tuning/70b/qwen2-70b_grpo_32_h20_fsdp_vllm.sh b/e2c/verl/examples/tuning/70b/qwen2-70b_grpo_32_h20_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..9a1d50ad1a8e3cc2843a7dce9aaf32398120e95b --- /dev/null +++ b/e2c/verl/examples/tuning/70b/qwen2-70b_grpo_32_h20_fsdp_vllm.sh @@ -0,0 +1,43 @@ +set -x + +gsm8k_train_path=$HOME/data/rlhf/gsm8k/train.parquet +gsm8k_val_path=$HOME/data/rlhf/math/test.parquet +model_path=Qwen/Qwen2-72B-Instruct + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$data_path \ + data.val_files=$gsm8k_val_path \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=model_path \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=16 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='Qwen2_72B_Instruct' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=4 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ \ No newline at end of file diff --git a/e2c/verl/examples/tuning/70b/qwen2-70b_grpo_32_h800_fsdp_vllm.sh b/e2c/verl/examples/tuning/70b/qwen2-70b_grpo_32_h800_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..b15f406b18813377b0152adf15315db865328b9e --- /dev/null +++ b/e2c/verl/examples/tuning/70b/qwen2-70b_grpo_32_h800_fsdp_vllm.sh @@ -0,0 +1,45 @@ +set -x + +#### important: vllm version must be >= 0.8.3 + +gsm8k_train_path=$HOME/data/rlhf/gsm8k/train.parquet +gsm8k_val_path=$HOME/data/rlhf/math/test.parquet +model_path=Qwen/Qwen2-72B-Instruct + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$gsm8k_train_path \ + data.val_files=$gsm8k_val_path \ + data.train_batch_size=1024 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=$model_path \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=16 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='Qwen2_72B_Instruct' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=4 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ \ No newline at end of file diff --git a/e2c/verl/examples/tuning/70b/qwen2-72b_grpo-lora_8_h100_fsdp_vllm.sh b/e2c/verl/examples/tuning/70b/qwen2-72b_grpo-lora_8_h100_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..7f93ed32faad0fd1f5004877a7bbee0d73702a69 --- /dev/null +++ b/e2c/verl/examples/tuning/70b/qwen2-72b_grpo-lora_8_h100_fsdp_vllm.sh @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +NOW=$(date +%Y%m%d) +export WANDB_DIR=gsm8k-grpo-lora-qwen2.5-72b-${NOW} +export WANDB_PROJECT=${WANDB_DIR} +export WANDB_EXP=72b-${NOW} +MODEL_PATH=Qwen/Qwen2.5-72B-Instruct + +set -x +nproc_per_gpu=22 # 16√ → 32× → 24× → 20√ → 22√ → 23× +nnodes=1 +ngpu_per_node=8 +total_procs=$(( nproc_per_gpu * nnodes * ngpu_per_node )) +mini_batch_size=$(( total_procs )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=${total_procs} \ + data.val_batch_size=${total_procs} \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.use_shm=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=32 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.actor.optim.lr=3e-5 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=8 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.max_num_seqs=512 \ + actor_rollout_ref.rollout.max_model_len=1536 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1536 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ + actor_rollout_ref.actor.entropy_coeff=0.001 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=${WANDB_PROJECT} \ + trainer.experiment_name=${WANDB_EXP} \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ 2>&1 | tee ${WANDB_PROJECT}.log diff --git a/e2c/verl/examples/tuning/7b/qwen2-7b_grpo-lora_1_h100_fsdp_vllm.sh b/e2c/verl/examples/tuning/7b/qwen2-7b_grpo-lora_1_h100_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..a663a90d63feca6e40080868cfdb012edb0600bf --- /dev/null +++ b/e2c/verl/examples/tuning/7b/qwen2-7b_grpo-lora_1_h100_fsdp_vllm.sh @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +NOW=$(date +%Y%m%d) +export WANDB_DIR=gsm8k-grpo-lora-qwen2.5-7b-${NOW} +export WANDB_PROJECT=${WANDB_DIR} +export WANDB_EXP=7b-${NOW} +MODEL_PATH=Qwen/Qwen2.5-7B-Instruct + +set -x +nproc_per_gpu=16 # 64√ → 128× → 96√ → 112× → 104× → 100√ → 102× → 101× +nnodes=1 +ngpu_per_node=1 +total_procs=$(( nproc_per_gpu * nnodes * ngpu_per_node )) +mini_batch_size=$(( total_procs )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=data/gsm8k/train.parquet \ + data.val_files=data/gsm8k/test.parquet \ + data.train_batch_size=${total_procs} \ + data.val_batch_size=${total_procs} \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.use_shm=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=32 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.model.target_modules=all-linear \ + actor_rollout_ref.actor.optim.lr=3e-5 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.2 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.max_num_seqs=512 \ + actor_rollout_ref.rollout.max_model_len=1536 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1536 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${mini_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.entropy_coeff=0.001 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=${WANDB_PROJECT} \ + trainer.experiment_name=${WANDB_EXP} \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ 2>&1 | tee ${WANDB_PROJECT}.log diff --git a/e2c/verl/examples/tuning/7b/qwen2-7b_grpo_2_h800_fsdp_vllm.sh b/e2c/verl/examples/tuning/7b/qwen2-7b_grpo_2_h800_fsdp_vllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..598e82b4192a3c2801db1092f3204212d5b64af4 --- /dev/null +++ b/e2c/verl/examples/tuning/7b/qwen2-7b_grpo_2_h800_fsdp_vllm.sh @@ -0,0 +1,48 @@ +set -x + + +gsm8k_train_path=$HOME/data/rlhf/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/rlhf/math/test.parquet +model_path=Qwen/Qwen2-7B-Instruct + +train_files="['$gsm8k_train_path']" +test_files="['$gsm8k_test_path']" + +PYTHONPATH=/opt/tiger/open_verl python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=$model_path \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=2 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/pyproject.toml b/e2c/verl/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..1426b507b0e06d6025f004248848cef730b53325 --- /dev/null +++ b/e2c/verl/pyproject.toml @@ -0,0 +1,113 @@ +# ------------------------------- +# build-system +# ------------------------------- +[build-system] +requires = [ + "setuptools>=61.0", + "wheel" +] +build-backend = "setuptools.build_meta" + +# ------------------------------- +# project (PEP 621 metadata) +# ------------------------------- +[project] +name = "verl" +# We'll mark the version as "dynamic" because it's read from the file "verl/version/version" +# (PEP 621 calls this "dynamic version"). +# The actual version is specified in the [tool.setuptools.dynamic] section below. +dynamic = ["version", "dependencies", "optional-dependencies", "authors", "urls"] + +description = "verl: Volcano Engine Reinforcement Learning for LLM" +license = {text = "Apache-2.0"} # Changed from file to text format +readme = {file = "README.md", content-type = "text/markdown"} +requires-python = ">=3.10" + +# ------------------------------- +# tool.ruff - Linting configuration +# ------------------------------- +[tool.ruff] +# Note: While the formatter will attempt to format lines such that they remain within the line-length, +# it isn't a hard upper bound, and formatted lines may exceed the line-length. +line-length = 120 +exclude = ["tests/workers/rollout/test_sglang_async_rollout_sf_tools.py", "scripts/legacy_model_merger.py"] + +[tool.ruff.lint] +isort = {known-first-party = ["verl"]} +# c.f. https://github.com/vllm-project/vllm/blob/ce8d6b75fc0586045df75ee1568a5b5f9957251b/pyproject.toml +select = [ + # pycodestyle + "E", + # Pyflakes + "F", + # pyupgrade + "UP", + # flake8-bugbear + "B", + # isort + "I", + "G", +] +ignore = [ + # star imports + "F405", "F403", + # lambda expression assignment + "E731", + # Loop control variable not used within loop body + "B007", + # f-string format + "UP032", + # `.log()` statement uses f-string + "G004", + # X | None for type annotations + "UP045", + # deprecated import + "UP035", +] + +# ------------------------------- +# tool.mypy - typechecking config +# ------------------------------- +[tool.mypy] +pretty = true +ignore_missing_imports = true +explicit_package_bases = true +follow_imports = "skip" + +# Blanket silence +ignore_errors = true + +[[tool.mypy.overrides]] +module = [ +"verl.trainer.config.algorithm", +"verl.trainer.ppo.core_algos", +"verl.trainer.ppo.reward", +"verl.workers.reward_manager", +"verl.workers.reward_manager.*", +] +ignore_errors = false + +# ------------------------------- +# tool.setuptools - Additional config +# ------------------------------- +[tool.setuptools] +# True means `setuptools` will attempt to include all relevant files in package_data automatically. +# This corresponds to `include_package_data=True` in setup.py. +include-package-data = true + +# We read the version from a file in 'verl/version/version' +[tool.setuptools.dynamic] +version = {file = "verl/version/version"} + +# If you need to mimic `package_dir={'': '.'}`: +[tool.setuptools.package-dir] +"" = "." + +# If you need to include specific non-Python data (like YAML files or version file): +# This is the rough equivalent of package_data={'': ['version/*'], 'verl': ['trainer/config/*.yaml']} +[tool.setuptools.package-data] +verl = [ + "version/*", + "trainer/config/*.yaml", + "trainer/config/*/*.yaml", +] diff --git a/e2c/verl/recipe/README.md b/e2c/verl/recipe/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6ea845bc0404c2bd0e6b4e85401907c7ea913462 --- /dev/null +++ b/e2c/verl/recipe/README.md @@ -0,0 +1,27 @@ +# Recipe +The examples under `recipes/` are representative extensions to verl for specific end-to-end RL training recipes. +The help the community reproduce experiments, verl team provides a snapshot of the codebase when each recipe is initially PR'ed to verl main. You can find them via [github branches](https://github.com/volcengine/verl/branches/all?query=recipe) + +# Awesome work using verl + +- [Logic-RL](https://github.com/Unakar/Logic-RL): a reproduction of DeepSeek R1 Zero on 2K Tiny Logic Puzzle Dataset. ![GitHub Repo stars](https://img.shields.io/github/stars/Unakar/Logic-RL) +- [Seed-Coder](https://github.com/ByteDance-Seed/Seed-Coder): RL training of Seed-Coder boosts performance on competitive programming ![GitHub Repo stars](https://img.shields.io/github/stars/ByteDance-Seed/Seed-Coder) +- [all-hands/openhands-lm-32b-v0.1](https://www.all-hands.dev/blog/introducing-openhands-lm-32b----a-strong-open-coding-agent-model): A strong, open coding agent model, trained with [multi-turn fine-tuning](https://github.com/volcengine/verl/pull/195) +- [s3](https://github.com/pat-jj/s3) **Efficient Yet Effective** Search Agent Training via RL ![GitHub Repo stars](https://img.shields.io/github/stars/pat-jj/s3) +- [Rec-R1](https://arxiv.org/pdf/2503.24289): Bridging Generative Large Language Models and Recommendation Systems via Reinforcement Learning +- [Explore RL Data Scaling](https://arxiv.org/abs/2503.22230): Exploring Data Scaling Trends and Effects in Reinforcement Learning from Human Feedback +- [FIRE](https://arxiv.org/abs/2410.21236): Flaming-hot initiation with regular execution sampling for large language models +- [DQO](https://arxiv.org/abs/2410.09302): Enhancing multi-Step reasoning abilities of language models through direct Q-function optimization +- [ProRL](https://arxiv.org/abs/2505.24864): Prolonged Reinforcement Learning Expands Reasoning Boundaries in Large Language Models +- [cognition-engineering](https://github.com/gair-nlp/cognition-engineering): Test time scaling drives cognition engineering. ![GitHub Repo stars](https://img.shields.io/github/stars/gair-nlp/cognition-engineering) +- [Trust Region Preference Approximation](https://github.com/XueruiSu/Trust-Region-Preference-Approximation): A simple and stable **reinforcement learning algorithm** for LLM reasoning. ![GitHub Repo stars](https://img.shields.io/github/stars/XueruiSu/Trust-Region-Preference-Approximation) +- [AdaRFT](https://github.com/uscnlp-lime/verl): Efficient Reinforcement Finetuning via **Adaptive Curriculum Learning** ![GitHub Repo stars](https://img.shields.io/github/stars/uscnlp-lime/verl) +- [critic-rl](https://github.com/HKUNLP/critic-rl): LLM critics for code generation ![GitHub Repo stars](https://img.shields.io/github/stars/HKUNLP/critic-rl) +- [self-rewarding-reasoning-LLM](https://arxiv.org/pdf/2502.19613): self-rewarding and correction with **generative reward models** ![GitHub Repo stars](https://img.shields.io/github/stars/RLHFlow/Self-rewarding-reasoning-LLM) +- [DeepEnlighten](https://github.com/DolbyUUU/DeepEnlighten): Reproduce R1 with **social reasoning** tasks and analyze key findings ![GitHub Repo stars](https://img.shields.io/github/stars/DolbyUUU/DeepEnlighten) +- [MetaSpatial](https://github.com/PzySeere/MetaSpatial): Reinforcing **3D Spatial Reasoning** in **VLMs** for the **Metaverse** ![GitHub Repo stars](https://img.shields.io/github/stars/PzySeere/MetaSpatial) +- [PURE](https://github.com/CJReinforce/PURE): **Credit assignment** is the key to successful reinforcement fine-tuning using **process reward model** ![GitHub Repo stars](https://img.shields.io/github/stars/CJReinforce/PURE) +- [cognitive-behaviors](https://github.com/kanishkg/cognitive-behaviors): Cognitive Behaviors that Enable Self-Improving Reasoners, or, Four Habits of Highly Effective STaRs ![GitHub Repo stars](https://img.shields.io/github/stars/kanishkg/cognitive-behaviors) +- [deepscaler](https://github.com/agentica-project/rllm/tree/deepscaler): iterative context scaling with GRPO ![GitHub Repo stars](https://img.shields.io/github/stars/agentica-project/deepscaler) +- [DAPO](https://dapo-sia.github.io/): the fully open source SOTA RL algorithm that beats DeepSeek-R1-zero-32B ![GitHub Repo stars](https://img.shields.io/github/stars/volcengine/verl) +- [NoisyRollout](https://github.com/NUS-TRAIL/NoisyRollout): Reinforcing Visual Reasoning with Data Augmentation ![GitHub Repo stars](https://img.shields.io/github/stars/NUS-TRAIL/NoisyRollout) diff --git a/e2c/verl/recipe/char_count/README.md b/e2c/verl/recipe/char_count/README.md new file mode 100644 index 0000000000000000000000000000000000000000..18f902d15ebc52f01f4de140e185156a8e4ecb7d --- /dev/null +++ b/e2c/verl/recipe/char_count/README.md @@ -0,0 +1,41 @@ +# Char Count +## Introduction +Char count is a simple NLP task. We create it for beginners to grasp the idea of RLVR. The task can be trained using a tiny model (e.g., https://huggingface.co/HuggingFaceTB/SmolLM2-135M) on a consumer GPU with only 8GB. + +## Problem formulation +The prompt is: "How many {char} are there in {word}?". In order for LLM to better answer this question, we create SFT dataset with intermediate steps. For example, + +```text +Question: How many n are there in n-i-n-e? +Answer: +n = n +i != n +n = n +e != n +\boxed{2} +``` + +Note that +- We add a dash between each individual char to make the task easier because each individual char will be tokenized to the same token by most tokenizer. +- In the SFT dataset, we create a CoT by listing all the individual chars and whether it equals to the target. In the end, it outputs the final answer inside the box. +- The task can be verified. +- The word is not always meaningful. Each char is sampled uniformly from a to z. We make the total length and the answer uniformly distributed within a range. + +## Scripts +To create the dataset, run +```bash +python3 create_dataset.py +``` +We create a train set and a val set. Both of them are used of SFT and RL. You can specify the total number of data, min/max length and data path. + +To run the SFT +```bash +bash train_sft.sh +``` +We train SFT for 3 epochs. After 3 epochs, the validation score is around 0.12. + +To run GRPO +```bash +bash train_grpo.sh +``` +We train GRPO for 2 epochs. After 2 epochs, the validation score is around 0.36. diff --git a/e2c/verl/recipe/char_count/create_dataset.py b/e2c/verl/recipe/char_count/create_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..985b1f03b904ec29e3c253f3147163d22b0bdfe9 --- /dev/null +++ b/e2c/verl/recipe/char_count/create_dataset.py @@ -0,0 +1,191 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Task description: +Given a random word and a random char, count the number of occurrence of char in the word. + +Create CoT dataset that split the word into separate char. Then list the char and count the occurrence. + +The word set comes from shakespeare +""" + +import os.path +import random + +prompt_template = "How many {} are there in word {}?" + + +def generate_random_char(): + return chr(97 + random.randint(0, 25)) + + +def create_prompt_response(min_length=3, max_length=5): + # randomly generate a length + word_length = random.randint(min_length, max_length) + # randomly generate a target count number. This makes the target number + target_count_number = random.randint(1, word_length) + + char_lst = [] + # generate the word + # step 1: generate the target word + target_char = generate_random_char() + + for _ in range(target_count_number): + char_lst.append(target_char) + + # step 2: generate other words + for _ in range(word_length - target_count_number): + while True: + char = generate_random_char() + if char != target_char: + char_lst.append(char) + break + + # step 3: random permute char_lst + random.shuffle(char_lst) + + word = "-".join(char_lst) + + prompt = prompt_template.format(target_char, word) + final_answer = [] + + # cot + number = 0 + for i, char in enumerate(char_lst): + cot = f"{char}" + if char != target_char: + cot += " != " + else: + cot += " = " + number += 1 + cot += f"{target_char}." + + final_answer.append(cot) + + conclusion = f"\\boxed{{{number}}} {target_char} in {word}." + + final_answer.append(conclusion) + + final_answer = "\n".join(final_answer) + + return prompt, final_answer + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--total_number", type=int, default=10000) + parser.add_argument("--min_length", type=int, default=5) + parser.add_argument("--max_length", type=int, default=20) + parser.add_argument("--data_path", type=str, default="~/data/char_count") + + args = vars(parser.parse_args()) + + total_number = args["total_number"] + min_length = args["min_length"] + max_length = args["max_length"] + data_path = args["data_path"] + data_path = os.path.expanduser(data_path) + + full_output = [] + for _ in range(total_number): + output = create_prompt_response(min_length=min_length, max_length=max_length) + full_output.append(output) + + # random reorder + random.shuffle(full_output) + + # split for train and test + train_split_len = int(0.9 * len(full_output)) + train_outputs = full_output[:train_split_len] + test_output = full_output[train_split_len:] + + sft_train_dataset = {"prompt": [], "response": []} + + for o in train_outputs: + sft_train_dataset["prompt"].append(o[0]) + sft_train_dataset["response"].append(o[1]) + + sft_test_dataset = {"prompt": [], "response": []} + + for o in test_output: + sft_test_dataset["prompt"].append(o[0]) + sft_test_dataset["response"].append(o[1]) + + import pandas as pd + + sft_train_dataset = pd.DataFrame(data=sft_train_dataset) + sft_test_dataset = pd.DataFrame(data=sft_test_dataset) + + folder = os.path.join(data_path, "sft") + + os.makedirs(folder, exist_ok=True) + + sft_train_dataset.to_parquet(os.path.join(folder, "train.parquet")) + sft_test_dataset.to_parquet(os.path.join(folder, "test.parquet")) + + # build RL dataset + rl_train_dataset = {"prompt": [], "data_source": [], "ability": [], "reward_model": [], "extra_info": []} + + rl_test_dataset = {"prompt": [], "data_source": [], "ability": [], "reward_model": [], "extra_info": []} + + from verl.utils.reward_score.math_reward import last_boxed_only_string, remove_boxed + + for o in train_outputs: + prompt = o[0] + response = o[1] + prompt_with_template = [ + { + "role": "user", + "content": prompt, + } + ] + + rl_train_dataset["prompt"].append(prompt_with_template) + rl_train_dataset["data_source"].append("char_count") + rl_train_dataset["ability"].append("other") + rl_train_dataset["reward_model"].append( + {"style": "rule", "ground_truth": remove_boxed(last_boxed_only_string(response))} + ) + rl_train_dataset["extra_info"].append({"response": response}) + + for o in test_output: + prompt = o[0] + response = o[1] + prompt_with_template = [ + { + "role": "user", + "content": prompt, + } + ] + + rl_test_dataset["prompt"].append(prompt_with_template) + rl_test_dataset["data_source"].append("char_count") + rl_test_dataset["ability"].append("other") + rl_test_dataset["reward_model"].append( + {"style": "rule", "ground_truth": remove_boxed(last_boxed_only_string(response))} + ) + rl_test_dataset["extra_info"].append({"response": response}) + + rl_train_dataset = pd.DataFrame(data=rl_train_dataset) + rl_test_dataset = pd.DataFrame(data=rl_test_dataset) + + folder = os.path.join(data_path, "rl") + + os.makedirs(folder, exist_ok=True) + + rl_train_dataset.to_parquet(os.path.join(folder, "train.parquet")) + rl_test_dataset.to_parquet(os.path.join(folder, "test.parquet")) diff --git a/e2c/verl/recipe/char_count/reward_function.py b/e2c/verl/recipe/char_count/reward_function.py new file mode 100644 index 0000000000000000000000000000000000000000..7c87ea49a1b105a4e1035f5ee07b9eb19384f38a --- /dev/null +++ b/e2c/verl/recipe/char_count/reward_function.py @@ -0,0 +1,34 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Reward function +""" + +from verl.utils.reward_score import math_reward + + +def char_count_reward_function(data_source, solution_str, ground_truth, extra_info=None): + try: + last_boxed_string = math_reward.last_boxed_only_string(solution_str) + if last_boxed_string is None: + return 0 + solution = math_reward.remove_boxed(last_boxed_string) + if solution == ground_truth: + return 1 + else: + return 0 + except Exception: + print(ground_truth, solution_str) + return 0 diff --git a/e2c/verl/recipe/char_count/train_grpo.sh b/e2c/verl/recipe/char_count/train_grpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..5de85422fc41c917aa8ea9106003d2cc20239dcc --- /dev/null +++ b/e2c/verl/recipe/char_count/train_grpo.sh @@ -0,0 +1,43 @@ +set -x + + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/char_count/rl/train.parquet \ + data.val_files=$HOME/data/char_count/rl/test.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=128 \ + data.max_response_length=128 \ + data.filter_overlong_prompts=False \ + data.truncation='error' \ + actor_rollout_ref.model.path=./models/sft/global_step_105 \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=16 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=5000 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","tensorboard"]' \ + trainer.project_name='verl_example' \ + trainer.experiment_name='smol135m_grpo' \ + trainer.val_before_train=True \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=2 \ + custom_reward_function.path=recipe/char_count/reward_function.py \ + custom_reward_function.name=char_count_reward_function diff --git a/e2c/verl/recipe/char_count/train_sft.sh b/e2c/verl/recipe/char_count/train_sft.sh new file mode 100644 index 0000000000000000000000000000000000000000..56f5cec5316d12e195d50ce1cd0b8e3ccf82288b --- /dev/null +++ b/e2c/verl/recipe/char_count/train_sft.sh @@ -0,0 +1,21 @@ +set -x + +nproc_per_node=1 +save_path=./models/sft + +torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/char_count/sft/train.parquet \ + data.val_files=$HOME/data/char_count/sft/test.parquet \ + data.prompt_key=prompt \ + data.response_key=response \ + data.micro_batch_size_per_gpu=8 \ + data.max_length=256 \ + data.train_batch_size=256 \ + use_remove_padding=True \ + model.partial_pretrain=HuggingFaceTB/SmolLM2-135M-Instruct \ + trainer.default_local_dir=$save_path \ + trainer.project_name=char_count-sft \ + trainer.experiment_name=char_count-sft-SmolLM2-135M-Instruct \ + trainer.total_epochs=3 \ + trainer.logger=console \ No newline at end of file diff --git a/e2c/verl/recipe/collabllm/README.md b/e2c/verl/recipe/collabllm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..953b08544cc0c4a83ef7a86e7520aab724fa0f92 --- /dev/null +++ b/e2c/verl/recipe/collabllm/README.md @@ -0,0 +1,74 @@ +# CollabLLM + +This repository implements [CollabLLM](https://arxiv.org/pdf/2502.00640) (ICML 2025) using the verl framework. For the original implementation, see the [CollabLLM repository](https://github.com/Wuyxin/collabllm). + + +CollabLLM is a method for training language models to collaborate effectively in multi-turn conversations. This implementation adapts the original imlpementation to work with the Verl training framework. + +## Quick start + +### 0. Environment +Make sure the required packages for `verl` are installed. Additionally, install `litellm` and export the required API keys. The API model will be used for user simulators and, optionally, LLM Judges (see the Configuration section below). + +### 1. Prepare Your Dataset + +First, process your dataset using the provided script: + +```bash +python process_dataset.py --dataset <> ... --dataset_type +``` + + +**Requirements:** +- Input: A Hugging Face multiturn dataset. Existing datasets: `collabllm/collabllm-multiturn-$DATASET`, with `DATASET` in one of [`math-hard(-large)`, `medium(-large)`, `bigcodebench(-large)`] (*-large are the datasets used in the CollabLLM paper) +- Example format: See [collabllm-multiturn-math-hard](https://huggingface.co/datasets/collabllm/collabllm-multiturn-math-hard) +- To generate your own dataset: Use [build_dataset.py](https://github.com/Wuyxin/collabllm/blob/main/scripts/engine/build_dataset.py) from the original CollabLLM repository + +*Note: Check `process_dataset.py` for example commands and usage.* + +### 2. Train Your Model + +**(Optional) For Supervised Fine-Tuning (SFT):** +```bash +bash train_sft_collabllm.sh +``` + +**For Reinforcement Learning (RL):** + +```bash +bash train_rl_collabllm.sh +``` + +The RL script shows an example to train CollabLLM on `math-hard-large`. + +- The config to sample future conversations are in `recipe/collabllm/config/collabllm_interaction_config.yaml`. +- The Multiturn-aware Reward is aggregated from these three conversational-level rewards: + + ``` + +reward_model.reward_kwargs.metric_weights.accuracy=1 \ + +reward_model.reward_kwargs.metric_weights.interactivity=1 \ + +reward_model.reward_kwargs.metric_weights.token_amount=-0.0001 \ + ``` + + You can remove, add, or modify the weights depending on your task. A list of implemented metrics you can already add are under `recipe/collabllm/metrics`. For example, on `medium-large`, you can replace `accuracy` with `bleu_score` via + ``` + +reward_model.reward_kwargs.metric_weights.bleu_score=1 + ``` + which will instead apply bleu score on the sampled future conversations. + +## Configuration +Read [doc](https://verl.readthedocs.io/en/latest/) for detailed configurations. + +## Citation +If you find CollabLLM useful in your research, please cite the following: + +```bibtex +@inproceedings{collabllm2025, + title={CollabLLM: From Passive Responders to Active Collaborators}, + author={Shirley Wu and Michel Galley and Baolin Peng and Hao Cheng and + Gavin Li and Yao Dou and Weixin Cai and James Zou and + Jure Leskovec and Jianfeng Gao}, + booktitle={International Conference on Machine Learning (ICML)}, + year={2025} +} +``` diff --git a/e2c/verl/recipe/collabllm/collabllm_agent_loop.py b/e2c/verl/recipe/collabllm/collabllm_agent_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..89435cbb4ffc023b9112aac95dcabd8826dd2398 --- /dev/null +++ b/e2c/verl/recipe/collabllm/collabllm_agent_loop.py @@ -0,0 +1,139 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from copy import deepcopy +from typing import Any +from uuid import uuid4 + +from recipe.collabllm.utils import is_valid_messages +from verl.experimental.agent_loop.agent_loop import AgentLoopOutput, register +from verl.experimental.agent_loop.tool_agent_loop import AgentData, AgentState, ToolAgentLoop +from verl.utils.rollout_trace import rollout_trace_op +from verl.workers.rollout.schemas import Message + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +@register("collabllm_agent") +class CollabLLMAgentLoop(ToolAgentLoop): + @rollout_trace_op + async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput: + messages = list(kwargs["raw_prompt"]) + image_data = deepcopy(kwargs.get("multi_modal_data", {}).get("image", None)) + metrics = {} + request_id = uuid4().hex + tools_kwargs = kwargs.get("tools_kwargs", {}) + + # Initialize interaction if needed + interaction = None + interaction_kwargs = {} + if self.interaction_config_file: + interaction_kwargs = kwargs["extra_info"]["interaction_kwargs"] + if "name" not in interaction_kwargs: + raise ValueError("'name' key is required in interaction_kwargs") + interaction_name = interaction_kwargs["name"] + if interaction_name not in self.interaction_map: + raise ValueError( + f"Interaction '{interaction_name}' not found in interaction_map. Available interactions: " + f"{list(self.interaction_map.keys())}" + ) + interaction = self.interaction_map[interaction_name] + await interaction.start_interaction(request_id, **interaction_kwargs) + # Create AgentData instance to encapsulate all state + agent_data = AgentData( + messages=messages, + image_data=image_data, + metrics=metrics, + request_id=request_id, + tools_kwargs=tools_kwargs, + interaction=interaction, + interaction_kwargs=interaction_kwargs, + ) + # for collabllm, firstly generate model reponses + await self._handle_pending_state(agent_data, sampling_params) + + status = await self._handle_generating_state(agent_data, sampling_params) + + if status == AgentState.TERMINATED: + # tell reward manager to score -1 and skip future interaction + # to avoid reward hacking with incompleted message + num_repeats = 0 + else: + # then, collect interaction rollouts + num_repeats = self.config.actor_rollout_ref.rollout.multi_turn.num_repeat_rollouts + + interaction_requests = [deepcopy(agent_data) for _ in range(num_repeats)] + + # messages are only used in collabllm reward manager + messages_lst = [] + for _agent_data in interaction_requests: + if not is_valid_messages(_agent_data.messages[-1]): + break + + prev_msg_len = len(_agent_data.messages) + await self.run_agent_data_loop(_agent_data, sampling_params, AgentState.INTERACTING) + messages_lst.append([Message(**msg) for msg in _agent_data.messages]) + + if interaction.config.get("enable_log"): + print(f"Assistant: ...{messages_lst[-1][prev_msg_len - 1].content[-100:]}") + print(f"User: {messages_lst[-1][prev_msg_len].content[:100]}...") + + # Finalize output + response_ids = agent_data.prompt_ids[-len(agent_data.response_mask) :] + prompt_ids = agent_data.prompt_ids[: len(agent_data.prompt_ids) - len(agent_data.response_mask)] + multi_modal_data = {"image": agent_data.image_data} if agent_data.image_data is not None else {} + + output = AgentLoopOutput( + prompt_ids=prompt_ids, + response_ids=response_ids[: self.response_length], + response_mask=agent_data.response_mask[: self.response_length], + multi_modal_data=multi_modal_data, + response_logprobs=agent_data.response_logprobs[: self.response_length] + if agent_data.response_logprobs + else None, + num_turns=agent_data.user_turns + agent_data.assistant_turns + 1, + metrics=agent_data.metrics, + extra_fields={ + "turn_scores": agent_data.turn_scores, + "messages": {"messages": messages_lst}, # compatiable with sglang interaction + }, + ) + return output + + async def run_agent_data_loop(self, agent_data: AgentData, sampling_params: dict[str, Any], state: AgentState): + """ + Run the agent data loop to process the agent data. + + Args: + agent_data (AgentData): The agent data to process. + sampling_params (dict[str, Any]): The sampling parameters. + state (AgentState, optional): The initial state of the agent. Defaults to None. + """ + + while state != AgentState.TERMINATED: + if state == AgentState.PENDING: + state = await self._handle_pending_state(agent_data, sampling_params) + elif state == AgentState.GENERATING: + state = await self._handle_generating_state(agent_data, sampling_params) + elif state == AgentState.PROCESSING_TOOLS: + state = await self._handle_processing_tools_state(agent_data) + elif state == AgentState.INTERACTING: + state = await self._handle_interacting_state(agent_data) + else: + logger.error(f"Invalid state: {state}") + state = AgentState.TERMINATED diff --git a/e2c/verl/recipe/collabllm/config/collabllm_interaction_config.yaml b/e2c/verl/recipe/collabllm/config/collabllm_interaction_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfc8257fce11ba47127d4845804ab22265624ad7 --- /dev/null +++ b/e2c/verl/recipe/collabllm/config/collabllm_interaction_config.yaml @@ -0,0 +1,10 @@ +interaction: + - name: "collabllm" + class_name: "verl.interactions.collabllm_interation.CollabLLMInteraction" + config: { + "user_model": "gpt-4o-mini", + "num_retries": 3, + "max_tokens": 512, + "temperature": 1.0, + "enable_log": True + } \ No newline at end of file diff --git a/e2c/verl/recipe/collabllm/metrics/accuracy.py b/e2c/verl/recipe/collabllm/metrics/accuracy.py new file mode 100644 index 0000000000000000000000000000000000000000..a81d1b8d18ce5ea1d815a361ca0912bdbc248e2b --- /dev/null +++ b/e2c/verl/recipe/collabllm/metrics/accuracy.py @@ -0,0 +1,104 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from recipe.collabllm.utils import extract_json, parse_messages + +ACCURACY_PROMPT = '''You are a helpful and meticulous evaluator. Your task is to \ +evaluate the *accuracy* of an AI model's answer to a target question. \ +You will be given the target question, the ground truth answer, and the conversation between the AI and the user. + +Provided Information: + +<|The Start of Target Question and Ground Truth Answer|> +Target Question: {single_turn_prompt} +Ground Truth Answer: {ground_truth} +<|The End of Target Question and Ground Truth Answer|> + +<|The Start of The Conversation|> +{chat_history} +<|The End of The Conversation|> + +You should determine whether the model's final response to the target question is \ +factually correct and consistent with the provided ground truth. + +Rating criteria (binary): + • 1 = Correct — the response matches the ground truth. + • 0 = Incorrect — the response contradicts or misses the ground truth. + +Output format (JSON): +{{ + "thought": "", + "accuracy": <0 or 1> +}} + +Double check if the JSON object is formatted correctly. Ensure that all fields are present and properly structured. \ +Use " or """ to wrap up the thought and use single quotes inside the "thought" field to avoid JSON escape issues. + +Your evaluation: +''' + + +async def compute_score(data_source, messages, ground_truth, extra_info, **kwargs): + # Check if litellm is available, fallback to openai if not + try: + import litellm + + use_litellm = True + except ImportError: + # litellm not found, falling back to openai + import openai + + use_litellm = False + + chat_history = parse_messages(messages, strip_sys_prompt=True) + prompt = ACCURACY_PROMPT.format( + single_turn_prompt=extra_info["interaction_kwargs"]["single_turn_prompt"], + ground_truth=ground_truth, + chat_history=chat_history, + ) + + if use_litellm: + full_response = ( + ( + await litellm.acompletion( + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + ) + .choices[0] + .message.content + ) + else: + client = openai.AsyncOpenAI() # Assumes API key is set in environment + full_response = ( + ( + await client.chat.completions.create( + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + ) + .choices[0] + .message.content + ) + + full_response = extract_json(full_response) + + assert isinstance(full_response, dict), f"Expected a dict, got {type(full_response)}" + assert {"accuracy", "thought"}.issubset(full_response.keys()), ( + f"Expected keys not found from {full_response.keys()}" + ) + + accuracy = full_response.pop("accuracy") + return float(accuracy) diff --git a/e2c/verl/recipe/collabllm/metrics/bleu_score.py b/e2c/verl/recipe/collabllm/metrics/bleu_score.py new file mode 100644 index 0000000000000000000000000000000000000000..e62a3b7f4e23991a4c8faee90b19a730063e374a --- /dev/null +++ b/e2c/verl/recipe/collabllm/metrics/bleu_score.py @@ -0,0 +1,116 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nltk.translate.bleu_score import sentence_bleu + +from recipe.collabllm.utils import extract_json, parse_messages + +EXTRACT_MULTITURN_COMPLETION_PROMPT = '''You are a thorough and diligent conversation analyzer. \ +Your task is to extract the final and complete version of a document that was generated during \ +a multiturn conversation between a user and a chat assistant. \ +The extracted content should reflect the final and comprehensive response provided by the assistant \ +based on the user’s request. + +You will be provided with the conversation: + +<|The Start of The Conversation|> +{chat_history} +<|The End of The Conversation|> + +Instructions for Extraction: + +1. Identify the Most Update-to-Date Contents: Review the entire conversation to identify the most updated parts \ +of the content provided by the assistant. This may include: + - Different sections of text (e.g., an essay, report, or article). + +2. Integrate Revisions: If the assistant made revisions, updates, or added sections throughout the conversation, \ +ensure that these changes are fully integrated into the final content. The goal is to extract a single, cohesive \ +output that incorporates all modifications and additions made during the conversation. For example, if the assistant \ +writes an introducation at the beginning and move on to the conclusion, the final output should include both the \ +introduction and the conclusion. + +3. Focus on Completeness: + - For text-based documents: Ensure that the extracted content is comprehensive and represents the full document \ + or section as discussed in the conversation. + +You should output a JSON object with two entries: +- "thought" (str): Output your thought process when extracting the final content. + 1. How do different parts of the conversation contribute to the final output? + 2. How do you make sure you included the most updated and complete information? + 3. How do you make sure you did not include any information that is not necessary? +- "final_completion" (str): The final and complete version of the document extracted from the conversation. + +Note: +1. If there are multiple lines, you should use triple quotes (""") to wrap the content. For example, \ + "final_completion": """first line. + second line.""" or "thought": """first line; + second line.""". +2. In the "final_completion" entry, replace all double quotes (") with single quotes (') to prevent JSON formatting \ +issues. For example, you can output "final_completion": "'Hello World' is a common phrase." + +Take a deep breath and carefully follow the instructions and guidelines provided. +''' + + +async def compute_score(data_source, messages, ground_truth, extra_info, **kwargs): + # Check if litellm is available, fallback to openai if not + try: + import litellm + + use_litellm = True + except ImportError: + # litellm not found, falling back to openai + import openai + + use_litellm = False + + chat_history = parse_messages(messages, strip_sys_prompt=True) + prompt = EXTRACT_MULTITURN_COMPLETION_PROMPT.format(chat_history=chat_history) + + if use_litellm: + full_response = ( + ( + await litellm.acompletion( + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + ) + .choices[0] + .message.content + ) + else: + client = openai.AsyncOpenAI() # Assumes API key is set in environment + full_response = ( + ( + await client.chat.completions.create( + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + ) + .choices[0] + .message.content + ) + + full_response = extract_json(full_response) + + assert isinstance(full_response, dict), f"Expected a dict, got {type(full_response)}" + assert {"final_completion", "thought"}.issubset(full_response.keys()), ( + f"Expected keys not found from {full_response.keys()}" + ) + + final_completion = full_response.pop("final_completion") + + bleu = sentence_bleu([ground_truth], final_completion) + return float(bleu) diff --git a/e2c/verl/recipe/collabllm/metrics/interactivity.py b/e2c/verl/recipe/collabllm/metrics/interactivity.py new file mode 100644 index 0000000000000000000000000000000000000000..a7ef69ef59660934853bda0a34335702d3f811f4 --- /dev/null +++ b/e2c/verl/recipe/collabllm/metrics/interactivity.py @@ -0,0 +1,108 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from recipe.collabllm.utils import extract_json, parse_messages + +INTERACTIVITY_PROMPT = '''You are a helpful and meticulous conversation evaluator. \ +Your task is to evaluate the interactivity of the responses provided by an AI assistant \ +to user questions in a given conversation: + +<|The Start of the Conversation to be Evaluated|> +{chat_history} +<|The End of the Conversation to be Evaluated|> + +You should assess the assistant's engagement, clarity, and ability to understand the user's needs. \ +Give a float number between 0 and 1. + +Scoring Criteria: +- Let U = user understanding & response clarity ∈ [0,1] + - 1.0 = Fully understands the user's intent and gives a clear answer. + - 0.7 = Mostly understands and the answer is generally clear. + - 0.3 = Partially misunderstands or the answer is hard to follow. + - 0.0 = Misunderstands the intent and gives an unclear or irrelevant answer. +- Let Q = clarification in [0,1] + - 1.0 = Asks precise, necessary clarifying questions when needed. + - 0.7 = Asks somewhat helpful but incomplete clarifications. + - 0.3 = Only asks generic questions (e.g., “Does that help?”). + - 0.0 = Asks no clarifying questions when needed. +- Let S = suggestion helpfulness in [0,1] + - 1.0 = Provides useful, actionable suggestions. + - 0.7 = Suggestions are somewhat helpful but limited. + - 0.3 = Suggestions are vague or generic. + - 0.0 = No suggestions when they would clearly help. +score = average([U, Q, S]) + +Output format (JSON): +{{ + "thought": "", + "interactivity": +}} + +Double check if the JSON object is formatted correctly. Ensure that all fields are present and properly structured. \ +Use " or """ to wrap up the thought. You should not use other triple quotes inside the "thought" field. \ +Instead you should use single quotes to avoid JSON escape issues. + +Your evaluation: +''' + + +async def compute_score(data_source, messages, ground_truth, extra_info, **kwargs): + # Check if litellm is available, fallback to openai if not + try: + import litellm + + use_litellm = True + except ImportError: + # litellm not found, falling back to openai + import openai + + use_litellm = False + + chat_history = parse_messages(messages, strip_sys_prompt=True) + prompt = INTERACTIVITY_PROMPT.format(chat_history=chat_history) + + if use_litellm: + full_response = ( + ( + await litellm.acompletion( + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + ) + .choices[0] + .message.content + ) + else: + client = openai.AsyncOpenAI() # Assumes API key is set in environment + full_response = ( + ( + await client.chat.completions.create( + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + ) + .choices[0] + .message.content + ) + + full_response = extract_json(full_response) + + assert isinstance(full_response, dict), f"Expected a dict, got {type(full_response)}" + assert {"interactivity", "thought"}.issubset(full_response.keys()), ( + f"Expected keys not found from {full_response.keys()}" + ) + + interactivity = full_response.pop("interactivity") + return float(interactivity) diff --git a/e2c/verl/recipe/collabllm/metrics/pass_rate.py b/e2c/verl/recipe/collabllm/metrics/pass_rate.py new file mode 100644 index 0000000000000000000000000000000000000000..5a07483a178044bd4e4fa6bf8667faac25826326 --- /dev/null +++ b/e2c/verl/recipe/collabllm/metrics/pass_rate.py @@ -0,0 +1,139 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from bigcodebench.eval import untrusted_check + +from recipe.collabllm.utils import extract_json, parse_messages + +EXTRACT_MULTITURN_COMPLETION_PROMPT = '''You are a thorough and diligent conversation analyzer. \ +Your task is to extract the final and complete version of a code function {entry_point} that was generated \ +during a multiturn conversation between a user and a chat assistant. \ +The extracted content should reflect the final and comprehensive response provided by the \ +assistant based on the user’s request. + +You will be provided with the task and the conversation: + +<|The Start of The Task|> +{single_turn_prompt} +<|The End of The Task|> + +<|The Start of The Conversation|> +{chat_history} +<|The End of The Conversation|> + +Instructions for Extraction: + +1. Identify the Most Update-to-Date Contents: Review the entire conversation to identify the most updated parts of \ +the content provided by the assistant. This may include: + - Different parts of the code snippet, function, class, or script. + +2. Integrate Revisions: If the assistant made revisions, updates, or added sections throughout the conversation, \ +ensure that these changes are fully integrated into the final content. The goal is to extract a single, cohesive \ +output that incorporates all modifications and additions made during the conversation. For example, if the assistant \ +writes a function at the beginning and changes a part, the final output should take the modification into account. + +3. Focus on Completeness: + - For code: Extract a complete and functional code snippet, including all necessary components such as imports, \ + functions, classes, and any other essential elements. The code should be runnable, but you do not need to \ + include any testing examples including the contents after `if __name__ == "__main__":`. Only the function code \ + is required. + +You should output a JSON object with two entries: +- "thought" (str): Output your thought process when extracting the final content. + 1. How do different parts of the conversation contribute to the final output? + 2. How do you make sure you included the most updated and complete information? + 3. How do you make sure you did not include any information that is not necessary? +- "final_completion" (str): The final and complete version of the code extracted from the conversation. \ +Rename main function name for the task to {entry_point} if needed. Remove any comments wrapped by """. + +Note: +1. If there are multiple lines, you should use triple quotes (""") to wrap the content. For example, \ + "final_completion": """first line. + second line.""" or "thought": """first line; + second line.""". You should not use other triple quotes inside. +2. In the "final_completion" entry, replace all double quotes (") with single quotes (') to prevent JSON formatting \ + issues. For example, you can output "final_completion": "'Hello World' is a common phrase." + +Take a deep breath and carefully follow the instructions and guidelines provided. +''' + + +async def compute_score(data_source, messages, ground_truth, extra_info, **kwargs): + # Check if litellm is available, fallback to openai if not + try: + import litellm + + use_litellm = True + except ImportError: + # litellm not found, falling back to openai + import openai + + use_litellm = False + + chat_history = parse_messages(messages, strip_sys_prompt=True) + + prompt = EXTRACT_MULTITURN_COMPLETION_PROMPT.format( + chat_history=chat_history, + single_turn_prompt=extra_info["interaction_kwargs"]["single_turn_prompt"], + entry_point=extra_info["single_turn_metadata"]["entry_point"], + ) + + if use_litellm: + full_response = ( + ( + await litellm.acompletion( + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + ) + .choices[0] + .message.content + ) + else: + client = openai.AsyncOpenAI() # Assumes API key is set in environment + full_response = ( + ( + await client.chat.completions.create( + messages=[{"role": "user", "content": prompt}], + **kwargs, + ) + ) + .choices[0] + .message.content + ) + + full_response = extract_json(full_response) + + assert isinstance(full_response, dict), f"Expected a dict, got {type(full_response)}" + assert {"final_completion", "thought"}.issubset(full_response.keys()), ( + f"Expected keys not found from {full_response.keys()}" + ) + + final_completion = full_response.pop("final_completion") + metadata = extra_info["single_turn_metadata"] + res = untrusted_check( + final_completion, + metadata["test"], + metadata["entry_point"], + max_as_limit=300 * 1024, + max_data_limit=300 * 1024, + max_stack_limit=300 * 1024, + min_time_limit=60, + gt_time_limit=60, + ) + passed = res[0] == "pass" + + # info = res[1] # for printing extra info + return float(passed) diff --git a/e2c/verl/recipe/collabllm/metrics/token_amount.py b/e2c/verl/recipe/collabllm/metrics/token_amount.py new file mode 100644 index 0000000000000000000000000000000000000000..8ffc5d5d8dc41c213205087b2be9dd8dca4ff9e6 --- /dev/null +++ b/e2c/verl/recipe/collabllm/metrics/token_amount.py @@ -0,0 +1,26 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def compute_score(data_source, messages, ground_truth, extra_info, **kwargs): + prompt = extra_info["prompt"] + + # Calculate the token penalty based on the length of the prompt + future_conv = messages[len(prompt) :] + + # simple length estimation + total_tokens = sum(len(m.content.split()) for m in future_conv) + + return total_tokens diff --git a/e2c/verl/recipe/collabllm/process_dataset.py b/e2c/verl/recipe/collabllm/process_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..cb04a2a9080b721101d2b2ab73eaedd4a2462e0b --- /dev/null +++ b/e2c/verl/recipe/collabllm/process_dataset.py @@ -0,0 +1,239 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 +""" +# available datasets: +# math-hard(-large), medium(-large), bigcodebench(-large) +# to create your own dataset, refer to https://github.com/Wuyxin/collabllm + +DATASET=math-hard-large + +python recipe/collabllm/process_dataset.py \ + --dataset collabllm/collabllm-multiturn-$DATASET \ + --local_dir $HOME/data/collabllm-$DATASET \ + --dataset_type sft + +python recipe/collabllm/process_dataset.py \ + --dataset collabllm/collabllm-multiturn-$DATASET \ + --local_dir $HOME/data/collabllm-$DATASET \ + --dataset_type rl + + +Preprocess collabllm/collabllm-multiturn-math-hard into (ground_truth, extra_info). + +- ground_truth: picked from --prefer_field (default: single_turn_completion), + falling back to --fallback_field (default: completion) +- extra_info: a shallow copy of the original example plus bookkeeping fields +- reward_model: {"style": "rule", "ground_truth": ground_truth} + +Saves one parquet per split into --local_dir and a small JSON preview. +""" + +import argparse +import json +import os +import uuid +from typing import Any, Optional + +from datasets import Dataset, concatenate_datasets, load_dataset + +SYSTEM_PROMPT = """The assistant is designed to be helpful, proactive, and highly interactive. + +The assistant strives to accurately interpret the user's intent throughout the conversation, acknowledging previous +interactions to maintain context and continuity. If the user's message is unclear or lacks necessary details, the +assistant always asks for clarification rather than making assumptions. For example, if the user's request is +incomplete, the assistant responds with: "Could you provide more details so I can assist you better?" + +The assistant asks specific follow-up questions and offers suggestions based on the user's needs, avoiding vague or +generic prompts. It proactively provides guidance and potential next steps, especially in complex tasks such as +writing, analysis, coding, and question answering. + +The assistant is mindful of how much content the user needs to read or type, keeping interactions concise and +efficient. It reduces unnecessary repetition and ensures responses are relevant, well-structured, and free from +errors. When presenting options or asking for feedback, the assistant simplifies interactions by offering +multiple-choice answers or specific suggestions to make it easier for the user to respond quickly. + +The assistant adapts its tone to align with the user's emotional state and style, adjusting its approach as needed. +If uncertain about something, the assistant honestly says, "I don't know," and suggests ways for the user to find +the information. + +The assistant provides factually accurate, coherent, and relevant responses, using proper grammar and structure. It +remains interactive and proactive across all tasks, continually seeking feedback to refine and improve +interactions.""" + + +# Required fields: "prompt", "ground_truth", "extra_info" +# In "extra_info" dict: +# (1) Rquired: "single_turn_prompt", which is the specific problem used to inform the user simulator, +# (2) Optional: "task_desc" (a short task description), +# (3) Optional: other fields for customized reward computation +def collapse_example(example: dict[str, Any]) -> dict[str, Any]: + if "prompt" not in example: + raise ValueError("Missing required 'prompt' field.") + + ground_truth = ( + example.get("ground_truth") or example.get("single_turn_completion") or example.get("completion") or "" + ) + + extra_info = {} + for k, v in example.items(): + if k in ("prompt", "ground_truth", "extra_info"): + continue + extra_info.setdefault(k, v) # keep extra_info values if keys overlap + + # make sure extra_info has the required fields + assert "single_turn_prompt" in extra_info, "Missing 'single_turn_prompt' in extra_info." + + # add system prompt as the beginning of the list + example["prompt"] = [{"role": "system", "content": SYSTEM_PROMPT}] + example["prompt"] + + extra_info.setdefault("prompt", example["prompt"]) # save the original prompt + extra_info.setdefault( + "interaction_kwargs", + { + "name": "collabllm", + "single_turn_prompt": extra_info.pop("single_turn_prompt"), + "task_desc": extra_info.pop("task_desc", "general ask-for-assistance task"), + }, + ) + return { + "prompt": example["prompt"], + "ground_truth": ground_truth, + "raw_prompt": example["prompt"], # save the original prompt + "extra_info": extra_info, + "reward_model": {"style": "rule", "ground_truth": ground_truth}, + "data_source": "collabllm", + "agent_name": "collabllm_agent", + "index": str(uuid.uuid4()), + } + + +# ---------- IO helpers ---------- +def save_parquet(ds_split: Dataset, filename: str, out_dir: str) -> None: + os.makedirs(out_dir, exist_ok=True) + path = os.path.join(out_dir, f"{filename}.parquet") + ds_split.to_parquet(path) + print(f"[OK] Wrote {filename}.parquet → {path} ({len(ds_split)} rows)") + + +def maybe_copy_to_hdfs(local_dir: str, hdfs_dir: Optional[str]) -> None: + if not hdfs_dir: + return + try: + from verl.utils.hdfs_io import copy, makedirs # type: ignore + except Exception as e: + print(f"[WARN] Skipping HDFS copy (verl not available): {e}") + return + makedirs(hdfs_dir) + copy(src=local_dir, dst=hdfs_dir) + print(f"[OK] Copied {local_dir} → {hdfs_dir}") + + +# ---------- Main ---------- +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--dataset", default="collabllm/collabllm-multiturn-math-hard", help="HF dataset path or local dir/file." + ) + ap.add_argument("--task_desc", default="solving math problems", help="Task description for the dataset.") + ap.add_argument("--local_dir", default="~/data/collabllm-math-hard", help="Output directory.") + ap.add_argument("--hdfs_dir", default=None, help="Optional HDFS destination (requires verl).") + ap.add_argument( + "--validation_size", type=float, default=0.1, help="Validation split size (fraction or absolute int)." + ) + ap.add_argument("--seed", type=int, default=42, help="Random seed for splitting.") + ap.add_argument("--num_proc", type=int, default=1, help="Parallel workers for map().") + ap.add_argument("--dataset_type", default="rl", choices=["rl", "sft"], help="Type of dataset (e.g., 'rl', 'sft').") + args = ap.parse_args() + + out_dir = os.path.expanduser(args.local_dir) + os.makedirs(out_dir, exist_ok=True) + + print(f"[INFO] Loading dataset: {args.dataset}") + ds_dict = load_dataset(args.dataset) + parts = list(ds_dict.values()) + ds_all: Dataset = parts[0] if len(parts) == 1 else concatenate_datasets(parts) + # Dataset({ + # features: ['prompt', 'completion', 'conv_id', 'score', 'single_turn_prompt', + # 'single_turn_completion', 'single_turn_metadata', 'turn_id', 'sessions', 'rewards'], + # num_rows: xxx + # }) + + if args.dataset_type == "rl": + # If multiple splits exist, merge them before collapsing/splitting. + ds_all = ds_all.map(lambda x: {"task_desc": args.task_desc}, num_proc=args.num_proc) + + print(f"[INFO] Collapsing to formatted fields on {len(ds_all)} rows…") + ds_all = ds_all.map( + function=collapse_example, + remove_columns=ds_all.column_names, + num_proc=args.num_proc, + ) + + def dedup_by_prompt(dataset): + seen = set() + unique_rows = [] + for ex in dataset: + prompt_key = json.dumps(ex["prompt"], sort_keys=True, ensure_ascii=False) + if prompt_key not in seen: + seen.add(prompt_key) + unique_rows.append(ex) + return Dataset.from_list(unique_rows) + + ds_all = dedup_by_prompt(ds_all) + + elif args.dataset_type == "sft": + df = ds_all.to_pandas() + + # Sort so that within each conv_id the highest turn_id is first, + # and if multiple rows share the same turn_id, the highest score comes first + df = df.sort_values(["conv_id", "turn_id", "score"], ascending=[True, False, False]) + + # Keep only the top row per conv_id + df = df.drop_duplicates(subset="conv_id", keep="first") + + # Back to HF Dataset + ds_all = Dataset.from_pandas(df, preserve_index=False) + + # Append assistant response into prompt list + def append_completion(example): + example["prompt"] = ( + [{"role": "system", "content": SYSTEM_PROMPT}] + + example["prompt"] + + [{"role": "assistant", "content": example["completion"]}] + ) + return example + + ds_all = ds_all.map(append_completion) + + # Keep only prompt column + cols_to_remove = [col for col in ds_all.column_names if col != "prompt"] + ds_all = ds_all.remove_columns(cols_to_remove) + + print(f"[INFO] Splitting with validation_size={args.validation_size}, seed={args.seed}") + split = ds_all.train_test_split(test_size=args.validation_size, seed=args.seed, shuffle=True) + train_ds, val_ds = split["train"], split["test"] + print(train_ds, val_ds) + + save_parquet(train_ds, f"{args.dataset_type}_train", out_dir) + save_parquet(val_ds, f"{args.dataset_type}_validation", out_dir) + + maybe_copy_to_hdfs(local_dir=out_dir, hdfs_dir=args.hdfs_dir) + print(f"[DONE] {args.dataset_type}_train.parquet and {args.dataset_type}_validation.parquet written.") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/recipe/collabllm/reward_function.py b/e2c/verl/recipe/collabllm/reward_function.py new file mode 100644 index 0000000000000000000000000000000000000000..457b32cd2ddfd0bee6c1a7b558787fd2d4ee8de0 --- /dev/null +++ b/e2c/verl/recipe/collabllm/reward_function.py @@ -0,0 +1,95 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import importlib.util +import os +import sys + +import litellm +import torch + + +async def conversation_level_reward_func( + data_source, messages, ground_truth, extra_info, metrics, **kwargs +) -> torch.Tensor: + """ + Async version of conversation-level reward function. + + Apply conversation-level reward function to the future interactions between the user simulator + and policy model, which are generated from `verl/interactions/collabllm_interation.py` + """ + num_retries = kwargs.get("num_retries", 6) + + rewards = {} + for metric in metrics: + current_dir = os.path.dirname(os.path.abspath(__file__)) + metric_file_path = os.path.join(current_dir, f"metrics/{metric}.py") + + if not os.path.exists(metric_file_path): + print(f"Error: Metric file '{metric_file_path}' not found. Assigning 0 to metric '{metric}'.") + rewards[metric] = 0.0 + continue + + spec = importlib.util.spec_from_file_location(f"metric_{metric}", metric_file_path) + if spec is None: + print(f"Error: Could not create spec for metric '{metric}'. Assigning 0 to metric '{metric}'.") + rewards[metric] = 0.0 + continue + + module = importlib.util.module_from_spec(spec) + + try: + sys.modules[f"metric_{metric}"] = module + assert spec.loader is not None + spec.loader.exec_module(module) + except Exception as e: + print(f"Error loading metric module from '{metric_file_path}': {e}. Assigning 0 to metric '{metric}'.") + rewards[metric] = 0.0 + continue + + # Assume each metric file has a compute_score function + if not hasattr(module, "compute_score"): + print( + f"Error: Function 'compute_score' not found in '{metric_file_path}'. Assigning 0 to metric '{metric}'." + ) + rewards[metric] = 0.0 + continue + + compute_score_fn = module.compute_score + + # Retry mechanism for calling the metric function + for attempt in range(num_retries): + try: + # Call the metric function (await if it's async) + if asyncio.iscoroutinefunction(compute_score_fn): + rewards[metric] = await compute_score_fn(data_source, messages, ground_truth, extra_info, **kwargs) + else: + rewards[metric] = compute_score_fn(data_source, messages, ground_truth, extra_info, **kwargs) + break # Success, exit retry loop + except Exception as e: + if attempt == num_retries - 1: # Last attempt + print( + f"Error: Failed to compute metric '{metric}' after {num_retries} attempts. " + f"Last error: {e}. Assigning 0 to metric '{metric}'." + ) + rewards[metric] = 0.0 + else: + print(f"Attempt {attempt + 1} failed for metric '{metric}': {e}. Retrying...") + if isinstance(e, litellm.RateLimitError): + await asyncio.sleep(max(2**attempt, 60)) # Exponential backoff + + # Return dict with metric names as keys + return {metric: torch.tensor(reward, dtype=torch.float32) for metric, reward in rewards.items()} diff --git a/e2c/verl/recipe/collabllm/train_rl_collabllm.sh b/e2c/verl/recipe/collabllm/train_rl_collabllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..6e506f8fd64c7fdae5381b4d86e31a6da4c1a30c --- /dev/null +++ b/e2c/verl/recipe/collabllm/train_rl_collabllm.sh @@ -0,0 +1,75 @@ +# Usage: sh recipe/collabllm/train_rl_collabllm.sh + +set -x + +PROJECT_DIR="$(pwd)" +export VLLM_USE_V1=1 + +RESUME_PATH="${1:-}" + +if [ -z "$RESUME_PATH" ]; then + RESUME_PATH=null +fi + +DATASET=math-hard-large +PROJECT_DIR="$(pwd)" + + +python3 -m verl.trainer.main_ppo \ + trainer.val_before_train=False \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/collabllm-$DATASET/rl_train.parquet \ + data.val_files=$HOME/data/collabllm-$DATASET/rl_validation.parquet \ + reward_model.reward_manager=collabllm \ + +reward_model.reward_kwargs.metric_weights.accuracy=1 \ + +reward_model.reward_kwargs.metric_weights.interactivity=1 \ + +reward_model.reward_kwargs.metric_weights.token_amount=-0.0001 \ + +reward_model.reward_kwargs.llm_judge_kwargs.model=gpt-4o-mini \ + +reward_model.reward_kwargs.llm_judge_kwargs.max_tokens=2048 \ + +reward_model.reward_kwargs.llm_judge_kwargs.temperature=0 \ + data.train_batch_size=16 \ + data.max_prompt_length=8196 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path="Qwen/Qwen2.5-7B-Instruct" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=8 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.multi_turn.enable=true \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=2 \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=3 \ + actor_rollout_ref.rollout.multi_turn.num_repeat_rollouts=3 \ + actor_rollout_ref.rollout.trace.token2text=True \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console", "wandb"]' \ + trainer.project_name=verlxcollabllm \ + trainer.experiment_name=collabllm-qwen2.5-7B-$DATASET \ + trainer.nnodes=1 \ + trainer.n_gpus_per_node=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + trainer.save_freq=100 \ + trainer.test_freq=10 \ + trainer.total_epochs=20 \ + custom_reward_function.path=recipe/collabllm/reward_function.py \ + custom_reward_function.name=conversation_level_reward_func \ + actor_rollout_ref.rollout.multi_turn.interaction_config_path="$PROJECT_DIR/recipe/collabllm/config/collabllm_interaction_config.yaml" \ + trainer.resume_from_path=$RESUME_PATH diff --git a/e2c/verl/recipe/collabllm/train_sft_collabllm.sh b/e2c/verl/recipe/collabllm/train_sft_collabllm.sh new file mode 100644 index 0000000000000000000000000000000000000000..f2328687a1185b5b7b69ccae13f2708f9eed9461 --- /dev/null +++ b/e2c/verl/recipe/collabllm/train_sft_collabllm.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -x + +if [ "$#" -lt 1 ]; then + echo "Usage: sft_train_collabllm.sh [ other_configs...]" + exit 1 +fi + +nproc_per_node=$1 + +# Shift the arguments so $@ refers to the rest +shift 1 + +DATASET=math-hard-large + +torchrun --nnodes=1 --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/collabllm-$DATASET/sft_train.parquet \ + data.val_files=$HOME/data/collabllm-$DATASET/sft_validation.parquet \ + data.multiturn.enable=true \ + data.multiturn.messages_key=prompt \ + optim.lr=1e-6 \ + data.train_batch_size=64 \ + data.micro_batch_size_per_gpu=2 \ + data.max_length=8196 \ + model.partial_pretrain=Qwen/Qwen2.5-7B-Instruct \ + trainer.project_name=collabllm-sft-$DATASET \ + trainer.experiment_name=collabllm-sft-qwen2.5-7B-$DATASET \ + trainer.logger=console \ + trainer.total_epochs=3 $@ \ + ulysses_sequence_parallel_size=1 \ + use_remove_padding=true $@ \ No newline at end of file diff --git a/e2c/verl/recipe/collabllm/utils.py b/e2c/verl/recipe/collabllm/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..588b2d006528dd117f98641cd18553b1f62b110e --- /dev/null +++ b/e2c/verl/recipe/collabllm/utils.py @@ -0,0 +1,280 @@ +# Copyright 2025 CollabLLM team and/or its affiliates +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +import re + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +def parse_messages(messages, strip_sys_prompt=True): + """ + Args: + messages: List[dict] + List of dictionaries with keys 'role' and 'content' + Example: messages = [{'role': 'user', 'content': 'Hello!'}, + {'role': 'assistant', 'content': 'Hi!'}, ...] + """ + if messages is None: + return "" + + if strip_sys_prompt: + messages = strip_system_prompt(messages) + + chat = "\n".join(f"**{m.role.capitalize()}**: {m.content}" for m in messages) + + return chat + + +def strip_system_prompt(messages): + """ + Args: + messages: List[dict] + List of dictionaries with keys 'role' and 'content' + Example: messages = [{'role': 'user', 'content': 'Hello!'}, + {'role': 'assistant', 'content': 'Hi!'}, ...] + """ + return [msg for msg in messages if msg.role != "system"] + + +def extract_json(s): + def convert_value(value): + true_values = {"true": True, "false": False, "null": None} + value_lower = value.lower() + if value_lower in true_values: + return true_values[value_lower] + try: + if "." in value or "e" in value.lower(): + return float(value) + else: + return int(value) + except ValueError: + return value # Return as string if not a number + + def parse_number(s, pos): + start = pos + while pos < len(s) and s[pos] in "-+0123456789.eE": + pos += 1 + num_str = s[start:pos] + try: + if "." in num_str or "e" in num_str.lower(): + return float(num_str), pos + else: + return int(num_str), pos + except ValueError: + logger.error(f"Invalid number at position {start}: {num_str}") + raise + + def skip_whitespace(s, pos): + while pos < len(s) and s[pos] in " \t\n\r": + pos += 1 + return pos + + def parse_string(s, pos): + quote_char = s[pos] + assert quote_char in ('"', "'") + pos += 1 + result = "" + while pos < len(s): + c = s[pos] + if c == "\\": + pos += 1 + if pos >= len(s): + raise ValueError("Invalid escape sequence") + c = s[pos] + escape_sequences = {"n": "\n", "t": "\t", "r": "\r", "\\": "\\", quote_char: quote_char} + result += escape_sequences.get(c, c) + elif c == quote_char: + pos += 1 + # Attempt to convert to a number if possible + converted_value = convert_value(result) + return converted_value, pos + else: + result += c + pos += 1 + raise ValueError("Unterminated string") + + def parse_key(s, pos): + pos = skip_whitespace(s, pos) + if s[pos] in ('"', "'"): + key, pos = parse_string(s, pos) + return key, pos + else: + raise ValueError(f"Expected string for key at position {pos}") + + def parse_object(s, pos): + obj = {} + assert s[pos] == "{" + pos += 1 + pos = skip_whitespace(s, pos) + while pos < len(s) and s[pos] != "}": + pos = skip_whitespace(s, pos) + key, pos = parse_key(s, pos) + pos = skip_whitespace(s, pos) + if pos >= len(s) or s[pos] != ":": + raise ValueError(f'Expected ":" at position {pos}') + pos += 1 + pos = skip_whitespace(s, pos) + value, pos = parse_value(s, pos) + obj[key] = value + pos = skip_whitespace(s, pos) + if pos < len(s) and s[pos] == ",": + pos += 1 + pos = skip_whitespace(s, pos) + elif pos < len(s) and s[pos] == "}": + break + elif pos < len(s) and s[pos] != "}": + raise ValueError(f'Expected "," or "}}" at position {pos}') + if pos >= len(s) or s[pos] != "}": + raise ValueError(f'Expected "}}" at position {pos}') + pos += 1 + return obj, pos + + def parse_array(s, pos): + lst = [] + assert s[pos] == "[" + pos += 1 + pos = skip_whitespace(s, pos) + while pos < len(s) and s[pos] != "]": + value, pos = parse_value(s, pos) + lst.append(value) + pos = skip_whitespace(s, pos) + if pos < len(s) and s[pos] == ",": + pos += 1 + pos = skip_whitespace(s, pos) + elif pos < len(s) and s[pos] == "]": + break + elif pos < len(s) and s[pos] != "]": + raise ValueError(f'Expected "," or "]" at position {pos}') + if pos >= len(s) or s[pos] != "]": + raise ValueError(f'Expected "]" at position {pos}') + pos += 1 + return lst, pos + + def parse_triple_quoted_string(s, pos): + if s[pos : pos + 3] == "'''": + quote_str = "'''" + elif s[pos : pos + 3] == '"""': + quote_str = '"""' + else: + raise ValueError(f"Expected triple quotes at position {pos}") + pos += 3 + result = "" + while pos < len(s): + if s[pos : pos + 3] == quote_str: + pos += 3 + # Attempt to convert to a number if possible + converted_value = convert_value(result) + return converted_value, pos + else: + result += s[pos] + pos += 1 + raise ValueError("Unterminated triple-quoted string") + + def parse_value(s, pos): + pos = skip_whitespace(s, pos) + if pos >= len(s): + raise ValueError("Unexpected end of input") + if s[pos] == "{": + return parse_object(s, pos) + elif s[pos] == "[": + return parse_array(s, pos) + elif s[pos : pos + 3] in ("'''", '"""'): + return parse_triple_quoted_string(s, pos) + elif s[pos] in ('"', "'"): + return parse_string(s, pos) + elif s[pos : pos + 4].lower() == "true": + return True, pos + 4 + elif s[pos : pos + 5].lower() == "false": + return False, pos + 5 + elif s[pos : pos + 4].lower() == "null": + return None, pos + 4 + elif s[pos] in "-+0123456789.": + return parse_number(s, pos) + else: + raise ValueError(f"Unexpected character at position {pos}: {s[pos]}") + + json_start = s.index("{") + json_end = s.rfind("}") + s = s[json_start : json_end + 1] + + s = s.strip() + result, pos = parse_value(s, 0) + pos = skip_whitespace(s, pos) + if pos != len(s): + raise ValueError(f"Unexpected content at position {pos}") + return result + + +def remove_think_block(msg: dict): + """ + remove .*? from content + """ + if "content" in msg and isinstance(msg["content"], str): + msg["content"] = re.sub(r".*?", "", msg["content"], flags=re.DOTALL).strip() + return msg + + +def is_valid_messages(msg: dict) -> bool: + """ + check if is valid messages, including: + 1. is paried with + 2. is not empty inside and outside + 3. is not nested, and at most one block is allowed. + 4. can not be empty if remove ending "<|im_end|>" + """ + content = msg.get("content") + if not isinstance(content, str): + return True + + # Base case: empty or whitespace-only content is invalid. + if not content.strip(): + return False + + num_think_open = content.count("") + num_think_close = content.count("") + + # Rule 1: Check for paired tags. + if num_think_open != num_think_close: + return False + + # Rule 3: Allow at most one think block. + if num_think_open > 1: + return False + + # Case 1: No blocks. + if num_think_open == 0: + visible_content = content + # Case 2: Exactly one block. + else: + # Rule 2: Check for empty content inside the think block. + match = re.search(r"(.*?)", content, re.DOTALL) + if not match or not match.group(1).strip(): + return False + + # The "visible" content is what's outside the think block. + visible_content = re.sub(r".*?", "", content, flags=re.DOTALL) + + visible_content = visible_content.strip() + + # Rule 4 & 2 (outside): Check if visible content is empty after handling <|im_end|>. + if visible_content.endswith("<|im_end|>"): + visible_content = visible_content[: -len("<|im_end|>")] + + if not visible_content.strip(): + return False + + return True diff --git a/e2c/verl/recipe/dapo/README.md b/e2c/verl/recipe/dapo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..75b80f1aa789495511cc35424eb70d3bc3e7599c --- /dev/null +++ b/e2c/verl/recipe/dapo/README.md @@ -0,0 +1,192 @@ +# Recipe: Decoupled Clip and Dynamic Sampling Policy Optimization (DAPO) + +> Open-Source Algorithm Implementation & Expriement Running: [Yuxuan Tong](https://tongyx361.github.io/), [Guangming Sheng](https://hk.linkedin.com/in/guangming-sheng-b50640211) + +> [!IMPORTANT] +> +> **🔥 News!!!** +> +> - [2025/04] We reproduced the results of two versions of DAPO ([Full](./run_dapo_qwen2.5_32b.sh) & [w/o Dynamic Sampling](./run_dapo_wo_ds_qwen2.5_32b.sh)), achieving 52% and 50% on AIME 2024 respectively, based on [the latest codebase on `recipe/dapo`](https://github.com/volcengine/verl/tree/recipe/dapo/recipe/dapo). Please check the details in [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=wmb4qxfht0n). +> - [2025/03] We published the training record of [an early version of DAPO (w/o Token-level PG Loss & Dynamic Sampling)](./run_dapo_early_qwen2.5_32b.sh), achieving 44% on AIME 2024, in [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=wmb4qxfht0n). + +🏠 [Homepage](https://dapo-sia.github.io/) | 📝 [Paper@arXiv](https://arxiv.org/abs/2503.14476) | 🤗 [Datasets&Models@HF](https://huggingface.co/collections/BytedTsinghua-SIA/dapo-67d7f1517ee33c8aed059da0) | 🐱 [Code@GitHub](https://github.com/volcengine/verl/tree/recipe/dapo/recipe/dapo) | 🐱 [Repo@GitHub](https://github.com/BytedTsinghua-SIA/DAPO) + +> We propose the **D**ecoupled Clip and Dynamic s**A**mpling **P**olicy **O**ptimization (DAPO) algorithm. By making our work publicly available, we provide the broader research community and society with practical access to scalable reinforcement learning, enabling all to benefit from these advancements. Our system is based on the awesome [verl](https://github.com/volcengine/verl) framework. Thanks for their great work! Applying DAPO training to Qwen2.5-32B base model proves to outperform the previous state-of-the-art DeepSeek-R1-Zero-Qwen-32B on AIME 2024, achieving **50%** accuracy with **50%** less training steps. +> +> ![dapo-main-result](https://dapo-sia.github.io/static/images/score.png) + +## Quickstart + +1. Prepare the datasets **on the Ray cluster**: + +```bash +bash prepare_dapo_data.sh # This downloads the datasets to ${HOME}/verl/data by default +``` + +2. Submit the job to the Ray cluster **from any machine**: + +```bash +cd verl # Repo root +export RAY_ADDRESS="http://${RAY_IP:-localhost}:8265" # The Ray cluster address to connect to +export WORKING_DIR="${PWD}" # The local directory to package to the Ray cluster +# Set the runtime environment like env vars and pip packages for the Ray cluster in yaml +export RUNTIME_ENV="./recipe/dapo/runtime_env.yaml" # This sets environment variables for the Ray cluster +bash recipe/dapo/run_dapo_qwen2.5_32b.sh # or other scripts +``` + +## Reproduction Runs + +| Setup | AIME 2024 Acc. | Hardware | Image | Commit | Environment Variables | Training Script | Training Record | +| -------------------------------------------- | -------------- | --------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| DAPO | 52% | 16x8xH800 | `hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.3-flashinfer0.2.2-cxx11abi0` | [`4f80e4`](https://github.com/volcengine/verl/tree/4f80e465c2ec79ab9c3c30ec74b9745de61d0490) | [runtime_env.yaml](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/runtime_env.yaml) | [run_dapo_qwen2.5_32b.sh](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/run_dapo_qwen2.5_32b.sh) | [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=wmb4qxfht0n) | +| DAPO w/o Dynamic Sampling | 50% | 16x8xH800 | `hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.3-flashinfer0.2.2-cxx11abi0` | [`4f80e4`](https://github.com/volcengine/verl/tree/4f80e465c2ec79ab9c3c30ec74b9745de61d0490) | [runtime_env.yaml](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/runtime_env.yaml) | [run_dapo_wo_ds_qwen2.5_32b.sh](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/run_dapo_wo_ds_qwen2.5_32b.sh) | [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=wmb4qxfht0n) | +| DAPO w/o Token-level Loss & Dynamic Sampling | 44% | 16x8xH20 | `hiyouga/verl:ngc-th2.5.1-cu120-vllm0.7.4-hotfix` | [`4f80e4`](https://github.com/volcengine/verl/tree/4f80e465c2ec79ab9c3c30ec74b9745de61d0490) | [runtime_env.yaml](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/runtime_env.yaml) | [run_dapo_early_qwen2.5_32b.sh](https://github.com/volcengine/verl/blob/4f80e465c2ec79ab9c3c30ec74b9745de61d0490/recipe/dapo/run_dapo_early_qwen2.5_32b.sh) | [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=wmb4qxfht0n) | + +> [!IMPORTANT] +> +> **📢 Call for Contribution!** +> +> Welcome to submit your reproduction runs and setups! + +## Configuration + +### Separated Clip Epsilons (-> Clip-Higher) + +An example configuration: + +```yaml +actor_rollout_ref: + actor: + clip_ratio_low: 0.2 + clip_ratio_high: 0.28 +``` + +`clip_ratio_low` and `clip_ratio_high` specify the $\varepsilon_{\text {low }}$ and $\varepsilon_{\text {high }}$ in the DAPO objective. + +Core relevant code: + +```python +pg_losses1 = -advantages * ratio +pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high) +pg_losses = torch.maximum(pg_losses1, pg_losses2) +``` + +### Dynamic Sampling (with Group Filtering) + +An example configuration: + +```yaml +data: + gen_batch_size: 1536 + train_batch_size: 512 +algorithm: + filter_groups: + enable: True + metric: acc # score / seq_reward / seq_final_reward / ... + max_num_gen_batches: 10 # Non-positive values mean no upper limit +``` + +Setting `filter_groups.enable` to `True` will filter out groups whose outputs' `metric` are all the same, e.g., for `acc`, groups whose outputs' accuracies are all 1 or 0. + +The trainer will repeat sampling with `gen_batch_size` until there are enough qualified groups for `train_batch_size` or reaching the upper limit specified by `max_num_gen_batches`. + +Core relevant code: + +```python +prompt_bsz = self.config.data.train_batch_size +if num_prompt_in_batch < prompt_bsz: + print(f'{num_prompt_in_batch=} < {prompt_bsz=}') + num_gen_batches += 1 + max_num_gen_batches = self.config.algorithm.filter_groups.max_num_gen_batches + if max_num_gen_batches <= 0 or num_gen_batches < max_num_gen_batches: + print(f'{num_gen_batches=} < {max_num_gen_batches=}. Keep generating...') + continue + else: + raise ValueError( + f'{num_gen_batches=} >= {max_num_gen_batches=}. Generated too many. Please check your data.' + ) +else: + # Align the batch + traj_bsz = self.config.data.train_batch_size * self.config.actor_rollout_ref.rollout.n + batch = batch[:traj_bsz] +``` + +### Flexible Loss Aggregation Mode (-> Token-level Loss) + +An example configuration: + +```yaml +actor_rollout_ref: + actor: + loss_agg_mode: "token-mean" # / "seq-mean-token-sum" / "seq-mean-token-mean" + # NOTE: "token-mean" is the default behavior +``` + +Setting `loss_agg_mode` to `token-mean` will mean the (policy gradient) loss across all the tokens in all the sequences in a mini-batch. + +Core relevant code: + +```python +if loss_agg_mode == "token-mean": + loss = verl_F.masked_mean(loss_mat, loss_mask) +elif loss_agg_mode == "seq-mean-token-sum": + seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) # token-sum + loss = torch.mean(seq_losses) # seq-mean +elif loss_agg_mode == "seq-mean-token-mean": + seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) / torch.sum(loss_mask, dim=-1) # token-mean + loss = torch.mean(seq_losses) # seq-mean +else: + raise ValueError(f"Invalid loss_agg_mode: {loss_agg_mode}") +``` + +### Overlong Reward Shaping + +An example configuration: + +```yaml +data: + max_response_length: 20480 # 16384 + 4096 +reward_model: + overlong_buffer: + enable: True + len: 4096 + penalty_factor: 1.0 +``` + +Setting `overlong_buffer.enable` to `True` will penalize the outputs whose lengths are overlong but still within the hard context limit. + +Specifically, the penalty increases linearly from `0` to `overlong_buffer.penalty_factor` when the length of the output exceeds the `max_response_length` by `0` to `overlong_buffer.len` tokens. + +Core relevant code: + +```python +if self.overlong_buffer_cfg.enable: + overlong_buffer_len = self.overlong_buffer_cfg.len + expected_len = self.max_resp_len - overlong_buffer_len + exceed_len = valid_response_length - expected_len + overlong_penalty_factor = self.overlong_buffer_cfg.penalty_factor + overlong_reward = min(-exceed_len / overlong_buffer_len * overlong_penalty_factor, 0) + reward += overlong_reward +``` + +## FAQ + +### Where is the "Overlong Filtering" in the paper? + +Most experiments in the paper, including the best-performant one, are run without Overlong Filtering because it's somehow overlapping with Overlong Reward Shaping in terms of properly learning from the longest outputs. So we don't implement it here. + +### What's the difference between [the `recipe/dapo` directory in the `main` branch](https://github.com/volcengine/verl/tree/main/recipe/dapo) and the [`recipe/dapo` branch](https://github.com/volcengine/verl/tree/recipe/dapo/recipe/dapo)? + +[The `recipe/dapo` branch](https://github.com/volcengine/verl/tree/recipe/dapo/recipe/dapo) is for **as-is reproduction** and thus won't be updated with new features. + +[The `recipe/dapo` directory in the `main` branch](https://github.com/volcengine/verl/tree/main/recipe/dapo) works as an example of how to extend the latest `verl` to implement an algorithm recipe, which will be maintained with new features. + +### Why can't I produce similar results after modifications? + +RL infrastructures nowadays still have inherent unrobustness, on which we are still working hard to improve. + +We strongly recommend to only modify one thing at a time. + +We also list some known problems here: + +1. Enabling CUDA graph (`enforce_eager=False`) might cause model performance degradation, whose cause is still under investigation. diff --git a/e2c/verl/recipe/dapo/config/dapo_megatron_trainer.yaml b/e2c/verl/recipe/dapo/config/dapo_megatron_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b846eaeb78ebaa29e5f4c250c2b9f25a906d90c8 --- /dev/null +++ b/e2c/verl/recipe/dapo/config/dapo_megatron_trainer.yaml @@ -0,0 +1,28 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_megatron_trainer + - _self_ + +data: + gen_batch_size: ${data.train_batch_size} + +reward_model: + reward_manager: dapo + overlong_buffer: + enable: False # We try to avoid forgetting to set enable + len: 0 + penalty_factor: 0.0 + log: False + +algorithm: + filter_groups: + _target_: verl.trainer.config.FilterGroupsConfig + enable: False # We try to avoid forgetting to set enable + metric: null # acc / score / seq_reward / seq_final_reward / ... + max_num_gen_batches: 0 # Non-positive values mean no upper limit + +trainer: + project_name: verl-dapo diff --git a/e2c/verl/recipe/dapo/config/dapo_trainer.yaml b/e2c/verl/recipe/dapo/config/dapo_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47ac00fd6a055d6c22e3facfa855844302345701 --- /dev/null +++ b/e2c/verl/recipe/dapo/config/dapo_trainer.yaml @@ -0,0 +1,28 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + gen_batch_size: ${data.train_batch_size} + +reward_model: + reward_manager: dapo + overlong_buffer: + enable: False # We try to avoid forgetting to set enable + len: 0 + penalty_factor: 0.0 + log: False + +algorithm: + filter_groups: + _target_: verl.trainer.config.FilterGroupsConfig + enable: False # We try to avoid forgetting to set enable + metric: null # acc / score / seq_reward / seq_final_reward / ... + max_num_gen_batches: 0 # Non-positive values mean no upper limit + +trainer: + project_name: verl-dapo diff --git a/e2c/verl/recipe/dapo/dapo_ray_trainer.py b/e2c/verl/recipe/dapo/dapo_ray_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..a4d5c098353c4fe2314af093a866bca37057fc42 --- /dev/null +++ b/e2c/verl/recipe/dapo/dapo_ray_trainer.py @@ -0,0 +1,404 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +FSDP PPO Trainer with Ray-based single controller. +This trainer supports model-agonistic model initialization with huggingface +""" + +import os +import uuid +from collections import defaultdict +from copy import deepcopy +from pprint import pprint + +import numpy as np +import torch +from tqdm import tqdm + +from verl import DataProto +from verl.trainer.ppo.core_algos import agg_loss +from verl.trainer.ppo.metric_utils import ( + compute_data_metrics, + compute_throughout_metrics, + compute_timing_metrics, + reduce_metrics, +) +from verl.trainer.ppo.ray_trainer import ( + AdvantageEstimator, + RayPPOTrainer, + apply_kl_penalty, + compute_advantage, + compute_response_mask, +) +from verl.utils.profiler import marked_timer +from verl.utils.rollout_skip import RolloutSkip + + +class RayDAPOTrainer(RayPPOTrainer): + """ + Note that this trainer runs on the driver process on a single CPU/GPU node. + """ + + def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the worker group through RPC + to construct the PPO dataflow. + The light-weight advantage computation is done on the driver process. + """ + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + self.global_steps = 0 + self.gen_steps = 0 + + # load checkpoint before doing anything + self._load_checkpoint() + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + val_metrics = self._validate() + assert val_metrics, f"{val_metrics=}" + pprint(f"Initial validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if self.config.trainer.get("val_only", False): + return + + if self.config.actor_rollout_ref.rollout.get("skip_rollout", False): + rollout_skip = RolloutSkip(self.config, self.actor_rollout_wg) + rollout_skip.wrap_generate_sequences() + + # add tqdm + progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress") + + # we start from step 1 + self.global_steps += 1 + self.gen_steps += 1 + last_val_metrics = None + + prev_step_profile = False + curr_step_profile = ( + self.global_steps in self.config.global_profiler.steps + if self.config.global_profiler.steps is not None + else False + ) + next_step_profile = False + + timing_raw = defaultdict(float) + batch = None + num_prompt_in_batch = 0 + num_gen_batches = 0 + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + metrics = {} + + with marked_timer("start_profile", timing_raw): + self._start_profiling( + not prev_step_profile and curr_step_profile + if self.config.global_profiler.profile_continuous_steps + else curr_step_profile + ) + + new_batch: DataProto = DataProto.from_single_dict(batch_dict) + num_gen_batches += 1 + # pop those keys for generation + if "multi_modal_data" in new_batch.non_tensor_batch.keys(): + gen_batch = new_batch.pop( + batch_keys=["input_ids", "attention_mask", "position_ids"], + non_tensor_batch_keys=["raw_prompt_ids", "multi_modal_data"], + ) + else: + gen_batch = new_batch.pop( + batch_keys=["input_ids", "attention_mask", "position_ids"], + non_tensor_batch_keys=["raw_prompt_ids"], + ) + gen_batch = gen_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + + is_last_step = self.global_steps >= self.total_training_steps + + with marked_timer("step", timing_raw): + # generate a batch + with marked_timer("gen", timing_raw, "red"): + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + timing_raw.update(gen_batch_output.meta_info["timing"]) + gen_batch_output.meta_info.pop("timing", None) + + if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: + with marked_timer("gen_max", timing_raw, "red"): + gen_baseline_batch = deepcopy(gen_batch) + gen_baseline_batch.meta_info["do_sample"] = False + gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) + + new_batch = new_batch.union(gen_baseline_output) + reward_baseline_tensor = self.reward_fn(new_batch) + reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) + + new_batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) + + new_batch.batch["reward_baselines"] = reward_baseline_tensor + + del gen_baseline_batch, gen_baseline_output + + new_batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(new_batch.batch))], dtype=object + ) + # repeat to align with repeated responses in rollout + new_batch = new_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + new_batch = new_batch.union(gen_batch_output) + + with marked_timer("reward", timing_raw, "yellow"): + # compute scores. Support both model and function-based. + # We first compute the scores using reward model. Then, we call reward_fn to combine + # the results from reward model and rule-based results. + if self.use_rm: + # we first compute reward model score + reward_tensor = self.rm_wg.compute_rm_score(new_batch) + new_batch = new_batch.union(reward_tensor) + + # we combine with rule-based rm + reward_extra_infos_dict: dict[str, list] + try: + reward_result = self.reward_fn(new_batch, return_dict=True) + reward_tensor = reward_result["reward_tensor"] + reward_extra_infos_dict = reward_result.get("reward_extra_info", {}) + except Exception as e: + print(f"Error in reward_fn: {e}") + reward_tensor = self.reward_fn(new_batch) + reward_extra_infos_dict = {} + + new_batch.batch["token_level_scores"] = reward_tensor + + if reward_extra_infos_dict: + new_batch.non_tensor_batch.update( + {k: np.array(v) for k, v in reward_extra_infos_dict.items()} + ) + + # compute rewards. apply_kl_penalty if available + if self.config.algorithm.use_kl_in_reward: + new_batch, kl_metrics = apply_kl_penalty( + new_batch, kl_ctrl=self.kl_ctrl_in_reward, kl_penalty=self.config.algorithm.kl_penalty + ) + metrics.update( + kl_metrics + ) # TODO: This will be cleared if we use multiple genenration batches + else: + new_batch.batch["token_level_rewards"] = new_batch.batch["token_level_scores"] + + if not self.config.algorithm.filter_groups.enable: + batch = new_batch + else: # NOTE: When prompts after filtering is less than train batch size, + # we skip to the next generation batch + metric_name = self.config.algorithm.filter_groups.metric + if metric_name == "seq_final_reward": + # Turn to numpy for easier filtering + new_batch.non_tensor_batch["seq_final_reward"] = ( + new_batch.batch["token_level_rewards"].sum(dim=-1).numpy() + ) + elif metric_name == "seq_reward": + new_batch.non_tensor_batch["seq_reward"] = ( + new_batch.batch["token_level_scores"].sum(dim=-1).numpy() + ) + + # Collect the sequence reward for each trajectory + prompt_uid2metric_vals = defaultdict(list) + for uid, metric_val in zip( + new_batch.non_tensor_batch["uid"], new_batch.non_tensor_batch[metric_name], strict=True + ): + prompt_uid2metric_vals[uid].append(metric_val) + + prompt_uid2metric_std = {} + for prompt_uid, metric_vals in prompt_uid2metric_vals.items(): + prompt_uid2metric_std[prompt_uid] = np.std(metric_vals) + + kept_prompt_uids = [ + uid + for uid, std in prompt_uid2metric_std.items() + if std > 0 or len(prompt_uid2metric_vals[uid]) == 1 + ] + num_prompt_in_batch += len(kept_prompt_uids) + + kept_traj_idxs = [] + for idx, traj_from_prompt_uid in enumerate(new_batch.non_tensor_batch["uid"]): + if traj_from_prompt_uid in kept_prompt_uids: + kept_traj_idxs.append(idx) + + new_batch = new_batch[kept_traj_idxs] + batch = new_batch if batch is None else DataProto.concat([batch, new_batch]) + + prompt_bsz = self.config.data.train_batch_size + if num_prompt_in_batch < prompt_bsz: + print(f"{num_prompt_in_batch=} < {prompt_bsz=}") + max_num_gen_batches = self.config.algorithm.filter_groups.max_num_gen_batches + if max_num_gen_batches <= 0 or num_gen_batches < max_num_gen_batches: + print(f"{num_gen_batches=}. Keep generating...") + progress_bar.update(1) + self.gen_steps += 1 + is_last_step = self.global_steps >= self.total_training_steps + continue + else: + raise ValueError( + f"{num_gen_batches=} >= {max_num_gen_batches=}." + + " Generated too many. Please check if your data are too difficult." + + " You could also try set max_num_gen_batches=0 to enable endless trials." + ) + else: + # Align the batch + traj_bsz = self.config.data.train_batch_size * self.config.actor_rollout_ref.rollout.n + batch = batch[:traj_bsz] + + # === Updating === + + batch.batch["response_mask"] = compute_response_mask(batch) + + # Balance the number of valid tokens across DP ranks. + # NOTE: This usually changes the order of data in the `batch`, + # which won't affect the advantage calculation (since it's based on uid), + # but might affect the loss calculation (due to the change of mini-batching). + # TODO: Decouple the DP balancing and mini-batching. + if self.config.trainer.balance_batch: + self._balance_batch(batch, metrics=metrics) + + # compute global_valid tokens + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + # recompute old_log_probs + with marked_timer("old_log_prob", timing_raw, "blue"): + old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) + entropys = old_log_prob.batch["entropys"] + response_masks = batch.batch["response_mask"] + loss_agg_mode = self.config.actor_rollout_ref.actor.loss_agg_mode + entropy_agg = agg_loss(loss_mat=entropys, loss_mask=response_masks, loss_agg_mode=loss_agg_mode) + old_log_prob_metrics = {"actor/entropy": entropy_agg.detach().item()} + metrics.update(old_log_prob_metrics) + old_log_prob.batch.pop("entropys") + batch = batch.union(old_log_prob) + + if self.use_reference_policy: + # compute reference log_prob + with marked_timer("ref", timing_raw, "olive"): + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + + # compute values + if self.use_critic: + with marked_timer("values", timing_raw, "cyan"): + values = self.critic_wg.compute_values(batch) + batch = batch.union(values) + + with marked_timer("adv", timing_raw, "brown"): + # compute advantages, executed on the driver process + norm_adv_by_std_in_grpo = self.config.algorithm.get("norm_adv_by_std_in_grpo", True) + batch = compute_advantage( + batch, + adv_estimator=self.config.algorithm.adv_estimator, + gamma=self.config.algorithm.gamma, + lam=self.config.algorithm.lam, + num_repeat=self.config.actor_rollout_ref.rollout.n, + norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo, + ) + + # update critic + if self.use_critic: + with marked_timer("update_critic", timing_raw, "pink"): + critic_output = self.critic_wg.update_critic(batch) + critic_output_metrics = reduce_metrics(critic_output.meta_info["metrics"]) + metrics.update(critic_output_metrics) + + # implement critic warmup + if self.config.trainer.critic_warmup <= self.global_steps: + # update actor + with marked_timer("update_actor", timing_raw, "red"): + actor_output = self.actor_rollout_wg.update_actor(batch) + actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"]) + metrics.update(actor_output_metrics) + + # Log rollout generations if enabled + rollout_data_dir = self.config.trainer.get("rollout_data_dir", None) + if rollout_data_dir: + self._log_rollout_data(batch, reward_extra_infos_dict, timing_raw, rollout_data_dir) + + # validate + if ( + self.val_reward_fn is not None + and self.config.trainer.test_freq > 0 + and (is_last_step or self.global_steps % self.config.trainer.test_freq == 0) + ): + with marked_timer("testing", timing_raw, "green"): + val_metrics: dict = self._validate() + if is_last_step: + last_val_metrics = val_metrics + metrics.update(val_metrics) + + if self.config.trainer.save_freq > 0 and ( + is_last_step or self.global_steps % self.config.trainer.save_freq == 0 + ): + with marked_timer("save_checkpoint", timing_raw, "green"): + self._save_checkpoint() + + with marked_timer("stop_profile", timing_raw): + next_step_profile = ( + self.global_steps + 1 in self.config.global_profiler.steps + if self.config.global_profiler.steps is not None + else False + ) + self._stop_profiling( + curr_step_profile and not next_step_profile + if self.config.global_profiler.profile_continuous_steps + else curr_step_profile + ) + prev_step_profile = curr_step_profile + curr_step_profile = next_step_profile + + # collect metrics + metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) + metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) + # TODO: implement actual tflpo and theoretical tflpo + n_gpus = self.resource_pool_manager.get_n_gpus() + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) + timing_raw = defaultdict(float) # clear timing + + metrics["train/num_gen_batches"] = num_gen_batches + batch = None + num_prompt_in_batch = 0 + num_gen_batches = 0 + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=self.global_steps) + + if is_last_step: + pprint(f"Final validation metrics: {last_val_metrics}") + progress_bar.close() + return + + progress_bar.update(1) + self.global_steps += 1 + self.gen_steps += 1 + # check if last step checkpint exists + checkpoint_dir = os.path.join(self.config.trainer.default_local_dir, f"global_step_{self.global_steps}") + if not os.path.exists(checkpoint_dir): + # save last step checkpoint + timing_raw = defaultdict(float) + with marked_timer("save_checkpoint", timing_raw, "green"): + self._save_checkpoint() + metrics = {f"timing/{k}": v for k, v in timing_raw.items()} + logger.log(data=metrics, step=self.global_steps) diff --git a/e2c/verl/recipe/dapo/main_dapo.py b/e2c/verl/recipe/dapo/main_dapo.py new file mode 100644 index 0000000000000000000000000000000000000000..e85e7d0dbf739c4637ade058813e6ce8ff2de648 --- /dev/null +++ b/e2c/verl/recipe/dapo/main_dapo.py @@ -0,0 +1,181 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. +""" + +import os +import socket + +import hydra +import ray +from omegaconf import OmegaConf + +from verl.trainer.ppo.reward import load_reward_manager +from verl.utils.device import is_cuda_available + +from .dapo_ray_trainer import RayDAPOTrainer + + +@hydra.main(config_path="config", config_name="dapo_trainer", version_base=None) +def main(config): + run_ppo(config) + + +def run_ppo(config) -> None: + if not ray.is_initialized(): + # this is for local ray cluster + default_runtime_env = { + "env_vars": {"TOKENIZERS_PARALLELISM": "true", "NCCL_DEBUG": "WARN", "VLLM_LOGGING_LEVEL": "WARN"} + } + ray_init_kwargs = config.ray_kwargs.get("ray_init", {}) + runtime_env_kwargs = ray_init_kwargs.get("runtime_env", {}) + runtime_env = OmegaConf.merge(default_runtime_env, runtime_env_kwargs) + ray_init_kwargs = OmegaConf.create({**ray_init_kwargs, "runtime_env": runtime_env}) + print(f"ray init kwargs: {ray_init_kwargs}") + ray.init(**OmegaConf.to_container(ray_init_kwargs)) + + try: + if ( + is_cuda_available + and config.global_profiler.tool == "nsys" + and OmegaConf.select(config.global_profiler, "steps") is not None + and len(OmegaConf.select(config.global_profiler, "steps")) > 0 + ): + nsight_options = OmegaConf.to_container( + config.global_profiler.global_tool_config.nsys.controller_nsight_options + ) + runner = TaskRunner.options(runtime_env={"nsight": nsight_options}).remote() + else: + runner = TaskRunner.remote() + ray.get(runner.run.remote(config)) + finally: + if ray.is_initialized(): + ray.shutdown() + + +@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head +class TaskRunner: + def run(self, config): + # print initial config + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_to_local + + print(f"TaskRunner hostname: {socket.gethostname()}, PID: {os.getpid()}") + + pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values + OmegaConf.resolve(config) + + # download the checkpoint from hdfs + local_path = copy_to_local(config.actor_rollout_ref.model.path) + + # instantiate tokenizer + from verl.utils import hf_processor, hf_tokenizer + + tokenizer = hf_tokenizer(local_path) + processor = hf_processor(local_path, use_fast=True) # used for multimodal LLM, could be none + + from verl.single_controller.ray import RayWorkerGroup + + # define worker classes + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + assert config.critic.strategy in {"fsdp", "fsdp2"} + + from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker + + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker + + ray_worker_group_cls = RayWorkerGroup + + else: + raise NotImplementedError + + from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role + + role_worker_mapping = { + Role.ActorRollout: ray.remote(ActorRolloutRefWorker), + Role.Critic: ray.remote(CriticWorker), + } + + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + Role.Critic: global_pool_id, + } + + # we should adopt a multi-source reward function here + # - for rule-based rm, we directly call a reward score + # - for model-based rm, we call a model + # - for code related prompt, we send to a sandbox if there are test cases + # - finally, we combine all the rewards together + # - The reward type depends on the tag of the data + if config.reward_model.enable: + if config.reward_model.strategy in {"fsdp", "fsdp2"}: + from verl.workers.fsdp_workers import RewardModelWorker + elif config.reward_model.strategy == "megatron": + from verl.workers.megatron_workers import RewardModelWorker + else: + raise NotImplementedError + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + mapping[Role.RewardModel] = global_pool_id + + # reference model + if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) + mapping[Role.RefPolicy] = global_pool_id + + reward_fn = load_reward_manager( + config, + tokenizer, + 0, + max_resp_len=config.data.max_response_length, + overlong_buffer_cfg=config.reward_model.overlong_buffer, + ) + + # Note that we always use function-based RM for validation + val_reward_fn = load_reward_manager( + config, + tokenizer, + 1, + max_resp_len=config.data.max_response_length, + overlong_buffer_cfg=config.reward_model.overlong_buffer, + ) + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + + trainer = RayDAPOTrainer( + config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + ) + trainer.init_workers() + trainer.fit() + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/recipe/dapo/prepare_dapo_data.sh b/e2c/verl/recipe/dapo/prepare_dapo_data.sh new file mode 100644 index 0000000000000000000000000000000000000000..b5dbb25a7dd3f0826eb435bb32ee317bff029322 --- /dev/null +++ b/e2c/verl/recipe/dapo/prepare_dapo_data.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -uxo pipefail + +export VERL_HOME=${VERL_HOME:-"${HOME}/verl"} +export TRAIN_FILE=${TRAIN_FILE:-"${VERL_HOME}/data/dapo-math-17k.parquet"} +export TEST_FILE=${TEST_FILE:-"${VERL_HOME}/data/aime-2024.parquet"} +export OVERWRITE=${OVERWRITE:-0} + +mkdir -p "${VERL_HOME}/data" + +if [ ! -f "${TRAIN_FILE}" ] || [ "${OVERWRITE}" -eq 1 ]; then + wget -O "${TRAIN_FILE}" "https://huggingface.co/datasets/BytedTsinghua-SIA/DAPO-Math-17k/resolve/main/data/dapo-math-17k.parquet?download=true" +fi + +if [ ! -f "${TEST_FILE}" ] || [ "${OVERWRITE}" -eq 1 ]; then + wget -O "${TEST_FILE}" "https://huggingface.co/datasets/BytedTsinghua-SIA/AIME-2024/resolve/main/data/aime-2024.parquet?download=true" +fi diff --git a/e2c/verl/recipe/dapo/run_dapo_early_qwen2.5_32b.sh b/e2c/verl/recipe/dapo/run_dapo_early_qwen2.5_32b.sh new file mode 100644 index 0000000000000000000000000000000000000000..517e5cefcacc993551fb3bdaabecfe8947e8366a --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_early_qwen2.5_32b.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Early-Qwen2.5-32B' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +# An early version for DAPO +loss_agg_mode="seq-mean-token-mean" + +enable_filter_groups=False +gen_prompt_bsz=512 # NOTE: no filtering here +train_prompt_bsz=512 +train_prompt_mini_bsz=32 +n_resp_per_prompt=16 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-16} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-32B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + + +# Performance Related Parameter +sp_size=8 +use_dynamic_bsz=True +actor_ppo_max_token_len=$((max_prompt_length + max_response_length)) +infer_ppo_max_token_len=$((max_prompt_length + max_response_length)) +offload=True +gen_tp=4 + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + --working-dir "${WORKING_DIR}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=5 \ + trainer.save_freq=5 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto diff --git a/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b.sh b/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b.sh new file mode 100644 index 0000000000000000000000000000000000000000..0ec1047a1712b7d09ab59618e8d3371e113513ac --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-32B' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=512 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-16} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-32B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=8 +use_dynamic_bsz=True +actor_ppo_max_token_len=$((max_prompt_length + max_response_length)) +infer_ppo_max_token_len=$((max_prompt_length + max_response_length)) +offload=True +gen_tp=4 + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + --working-dir "${WORKING_DIR}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=5 \ + trainer.save_freq=5 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto diff --git a/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b_npu.sh b/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..0e99b5fa6fdc1f3e546cc4c69f5e67a1260ad456 --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b_npu.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO-Qwen2.5-32B' +exp_name='Qwen2.5-32B-npu-32rank-gbs128' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 +clip_ratio_low=0.2 +clip_ratio_high=0.28 +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 +loss_agg_mode="token-mean" +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 + +NNODES=2 + +train_prompt_bsz=128 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +PWD=./ +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} + +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-32B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=8 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) +offload=True +gen_tp=4 +enable_chunked_prefill=True + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + --working-dir "${WORKING_DIR}" \ + --address "${RAY_ADDRESS}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + +actor_rollout_ref.model.override_config.attention_dropout=0. \ + +actor_rollout_ref.model.override_config.embd_pdrop=0. \ + +actor_rollout_ref.model.override_config.resid_pdrop=0. \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.90 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=${enable_chunked_prefill} \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger="['console','wandb']" \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=5 \ + trainer.save_freq=20 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.device=npu \ + trainer.resume_mode=auto \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.ref.fsdp_config.forward_prefetch=True \ diff --git a/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b_tis.sh b/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b_tis.sh new file mode 100644 index 0000000000000000000000000000000000000000..147628463905769d7c111d5980225fdfedb35402 --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_qwen2.5_32b_tis.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-32B-TIS' # Truncated Importance Sampling (TIS) -> https://fengyao.notion.site/off-policy-rl + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 +tis_imp_ratio_cap=2.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=512 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-16} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-32B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=8 +use_dynamic_bsz=True +actor_ppo_max_token_len=$((max_prompt_length + max_response_length)) +infer_ppo_max_token_len=$((max_prompt_length + max_response_length)) +offload=True +gen_tp=4 + + +# Truncated Importance Sampling (TIS) -> https://fengyao.notion.site/off-policy-rl + +# Please note that server mode(agent loop) hasn't return rollout_log_probs for now. +# so currently, server mode is not supported for TIS. + +# To turn on TIS, you need to set the following parameters. Note 2.0 is a hyper-parameter and can be tuned. +# actor_rollout_ref.actor.tis_imp_ratio_cap=2.0 +# actor_rollout_ref.rollout.calculate_log_probs=True + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + --working-dir "${WORKING_DIR}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.tis_imp_ratio_cap=${tis_imp_ratio_cap} \ + actor_rollout_ref.rollout.calculate_log_probs=True \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=5 \ + trainer.save_freq=5 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto diff --git a/e2c/verl/recipe/dapo/run_dapo_qwen2.5_7b_npu.sh b/e2c/verl/recipe/dapo/run_dapo_qwen2.5_7b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..bd6b3689b2335a504138b7d5c4dc27f5ae7fc0a8 --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_qwen2.5_7b_npu.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO-Qwen2.5-7B-Instruct' +exp_name='DAPO-Qwen2.5-7B-Instruct' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 +clip_ratio_low=0.2 +clip_ratio_high=0.28 +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 +loss_agg_mode="token-mean" +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 + +NNODES=1 + +train_prompt_bsz=16 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +n_resp_per_prompt=16 +train_prompt_mini_bsz=1 + +# Ray +PWD=./ +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} + +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-7B-Instruct"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout + +# Performance Related Parameter +sp_size=4 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) +offload=True +gen_tp=1 + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + --working-dir "${WORKING_DIR}" \ + --address "${RAY_ADDRESS}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + +actor_rollout_ref.model.override_config.attention_dropout=0. \ + +actor_rollout_ref.model.override_config.embd_pdrop=0. \ + +actor_rollout_ref.model.override_config.resid_pdrop=0. \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.50 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger="['console']" \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=5 \ + trainer.save_freq=20 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.device=npu \ + trainer.resume_mode=auto \ + actor_rollout_ref.actor.entropy_checkpointing=True \ + actor_rollout_ref.ref.entropy_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.ref.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.actor.entropy_from_logits_with_chunking=True \ + actor_rollout_ref.ref.entropy_from_logits_with_chunking=True \ No newline at end of file diff --git a/e2c/verl/recipe/dapo/run_dapo_qwen3_14b_base_npu.sh b/e2c/verl/recipe/dapo/run_dapo_qwen3_14b_base_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..48793f586241e51091570a35a798a210bc0f706c --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_qwen3_14b_base_npu.sh @@ -0,0 +1,139 @@ +#!/bin/bash +project_name='DAPO' +exp_name='DAPO-Qwen3-14B-Base' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=False +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=16 +gen_prompt_bsz=$((train_prompt_bsz * 2)) +n_resp_per_prompt=16 +train_prompt_mini_bsz=1 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-2} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-14B-Base"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout + +# Performance Related Parameter +sp_size=2 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) +offload=True +gen_tp=2 + +ray job submit --runtime-env="${RUNTIME_ENV}" \ + --address "${RAY_ADDRESS}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + +actor_rollout_ref.model.override_config.attention_dropout=0. \ + +actor_rollout_ref.model.override_config.embd_pdrop=0. \ + +actor_rollout_ref.model.override_config.resid_pdrop=0. \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=8 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger=['console'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=10 \ + trainer.save_freq=20 \ + trainer.total_epochs=1 \ + trainer.total_training_steps=100 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + data.shuffle=False \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + actor_rollout_ref.actor.entropy_checkpointing=True \ + actor_rollout_ref.ref.entropy_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.ref.fsdp_config.forward_prefetch=True \ + trainer.device=npu diff --git a/e2c/verl/recipe/dapo/run_dapo_qwen3_8b_base_npu.sh b/e2c/verl/recipe/dapo/run_dapo_qwen3_8b_base_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..70b474001fcc9e38885efb8cd4b7de5431725f76 --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_qwen3_8b_base_npu.sh @@ -0,0 +1,138 @@ +#!/bin/bash +project_name='DAPO' +exp_name='DAPO-Qwen3-8B-Base' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=False +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=16 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +n_resp_per_prompt=16 +train_prompt_mini_bsz=1 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-1} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-8B-Base"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout + +# Performance Related Parameter +sp_size=2 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / sp_size)) +offload=True +gen_tp=2 + +ray job submit --runtime-env="${RUNTIME_ENV}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + +actor_rollout_ref.model.override_config.attention_dropout=0. \ + +actor_rollout_ref.model.override_config.embd_pdrop=0. \ + +actor_rollout_ref.model.override_config.resid_pdrop=0. \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.90 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger=['console'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=10 \ + trainer.save_freq=20 \ + trainer.total_epochs=1 \ + trainer.total_training_steps=100 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + data.shuffle=False \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + actor_rollout_ref.actor.entropy_checkpointing=True \ + actor_rollout_ref.ref.entropy_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.ref.fsdp_config.forward_prefetch=True \ + trainer.device=npu diff --git a/e2c/verl/recipe/dapo/run_dapo_qwen3_moe_30b_base_fsdp_npu.sh b/e2c/verl/recipe/dapo/run_dapo_qwen3_moe_30b_base_fsdp_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..d399ddfe8b7eaed5f6835d1bf17caf4d19f1c248 --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_qwen3_moe_30b_base_fsdp_npu.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euxo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen3-MOE-30B-FSDP-128rank-gbs512' + +NNODES=8 +NPUS_PER_NODE=16 + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 +loss_agg_mode="token-mean" +ppo_mini_batch_size=32 + +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=512 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +n_resp_per_prompt=16 + +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} + +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B-Base"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=16 # For load-balance. For smaller cluster this can be set to as less as 2. +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / 2)) +offload=True +recompute=True +max_num_seqs=128 +gen_tp=2 + + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.rollout.max_num_seqs=${max_num_seqs} \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + +actor_rollout_ref.model.override_config.attention_dropout=0. \ + +actor_rollout_ref.model.override_config.embd_pdrop=0. \ + +actor_rollout_ref.model.override_config.resid_pdrop=0. \ + actor_rollout_ref.model.enable_gradient_checkpointing=${recompute} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${ppo_mini_batch_size} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=False \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.ref.fsdp_config.forward_prefetch=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger=['console','wandb'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=5 \ + trainer.save_freq=-1 \ + trainer.total_epochs=1 \ + trainer.device="npu" \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False + diff --git a/e2c/verl/recipe/dapo/run_dapo_qwen3_moe_30b_megatron_npu.sh b/e2c/verl/recipe/dapo/run_dapo_qwen3_moe_30b_megatron_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..81d5b150553de90f52aef97659e55643cb62fa6a --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_qwen3_moe_30b_megatron_npu.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +project_name='DAPO' +exp_name='DAPO-Qwen3-30B-megatron' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=16 +gen_prompt_bsz=$((train_prompt_bsz * 2)) +n_resp_per_prompt=16 +train_prompt_mini_bsz=2 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-1} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B"} +# MCORE_MODEL_PATH points to the converted checkpoint. +# To avoid loading these weights, set actor_rollout_ref.actor.megatron.use_dist_checkpointing=False. +MCORE_MODEL_PATH=${MCORE_MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B-dist_ckpt"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout + +# Performance Related Parameter +sp_size=8 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length))) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length))) +offload=True + +max_num_batched_tokens=$((max_prompt_length + max_response_length)) + +# Megatron backen +train_tp=4 +train_ep=2 +train_pp=2 +train_cp=1 + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + --address "${RAY_ADDRESS}" \ + -- python3 -m recipe.dapo.main_dapo \ + --config-name="dapo_megatron_trainer" \ + data.filter_overlong_prompts=False \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.shuffle=False \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.ppo_epochs=1 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + +actor_rollout_ref.model.override_config.attention_dropout=0. \ + +actor_rollout_ref.model.override_config.embd_pdrop=0. \ + +actor_rollout_ref.model.override_config.resid_pdrop=0. \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + +actor_rollout_ref.critic.optim.lr=5e-8 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=${train_ep} \ + actor_rollout_ref.actor.megatron.context_parallel_size=${train_cp} \ + actor_rollout_ref.actor.megatron.dist_checkpointing_path=${MCORE_MODEL_PATH} \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=${train_ep} \ + actor_rollout_ref.ref.megatron.context_parallel_size=${train_cp} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + actor_rollout_ref.ref.megatron.dist_checkpointing_path=${MCORE_MODEL_PATH} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.enable_prefix_caching=False \ + actor_rollout_ref.rollout.max_num_batched_tokens=${max_num_batched_tokens} \ + actor_rollout_ref.rollout.max_model_len=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger=['console'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=-1 \ + trainer.save_freq=-1 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.device="npu" \ + actor_rollout_ref.nccl_timeout=14400 \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + +actor_rollout_ref.actor.megatron.override_transformer_config.use_flash_attn=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 + diff --git a/e2c/verl/recipe/dapo/run_dapo_wo_ds_qwen2.5_32b.sh b/e2c/verl/recipe/dapo/run_dapo_wo_ds_qwen2.5_32b.sh new file mode 100644 index 0000000000000000000000000000000000000000..50c18eadb12ad24fdab9ad68efb50be2fa450341 --- /dev/null +++ b/e2c/verl/recipe/dapo/run_dapo_wo_ds_qwen2.5_32b.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euxo pipefail +# DAPO (w/o Dynamic Sampling) + +project_name='DAPO-verl' +exp_name='DAPO-wo-DS-Qwen2.5-32B' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 20)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=False +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-16} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-32B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=8 +use_dynamic_bsz=True +actor_ppo_max_token_len=$((max_prompt_length + max_response_length)) +infer_ppo_max_token_len=$((max_prompt_length + max_response_length)) +offload=True +gen_tp=4 + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + --working-dir "${WORKING_DIR}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=5 \ + trainer.save_freq=5 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto diff --git a/e2c/verl/recipe/dapo/runtime_env.yaml b/e2c/verl/recipe/dapo/runtime_env.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13f4b2ba230b892a277026d53a98cb42afc4ae4d --- /dev/null +++ b/e2c/verl/recipe/dapo/runtime_env.yaml @@ -0,0 +1,5 @@ +working_dir: ./ +excludes: ["/.git/"] +env_vars: + TORCH_NCCL_AVOID_RECORD_STREAMS: "1" + VLLM_USE_V1: "1" diff --git a/e2c/verl/recipe/dapo/test_dapo_7b.sh b/e2c/verl/recipe/dapo/test_dapo_7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..a1201dc32f7afeaa4d645ef083c6291440f54753 --- /dev/null +++ b/e2c/verl/recipe/dapo/test_dapo_7b.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7B-Math-Test' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 2)) +enable_overlong_buffer=True +overlong_buffer_len=512 +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=512 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +train_prompt_mini_bsz=32 +n_resp_per_prompt=16 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-4} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout + +# Mathematically equivalent +use_dynamic_bsz=True +infer_micro_batch_size=null +train_micro_batch_size=null +offload=False + +ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ + --working-dir "${WORKING_DIR}" \ + -- python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${train_micro_batch_size} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=2 \ + trainer.save_freq=2 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=disable diff --git a/e2c/verl/recipe/dapo/test_dapo_7b_math.sh b/e2c/verl/recipe/dapo/test_dapo_7b_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..e7fa99268689f636f7988127ae4627195116a8b0 --- /dev/null +++ b/e2c/verl/recipe/dapo/test_dapo_7b_math.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7b-MATH-0527a1' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-8} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=4 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=4 +fsdp_size=32 + +# reference run wandb: https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/runs/ow47vvon?nw=nwusertongyuxuan361 + +python3 -m verl.trainer.main_ppo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.model.use_remove_padding=True \ + +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=10 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=200 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/dapo/test_dapo_7b_math_lora.sh b/e2c/verl/recipe/dapo/test_dapo_7b_math_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..06c66baa42f005de27d43651632d08646a8df8e5 --- /dev/null +++ b/e2c/verl/recipe/dapo/test_dapo_7b_math_lora.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7b-MATH-0527a1' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-8} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=4 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=4 +fsdp_size=32 + +# remember to set VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 for this model + +python3 -m verl.trainer.main_ppo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.model.use_remove_padding=True \ + +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=8 \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=10 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=200 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/dapo/test_dapo_7b_math_megatron.sh b/e2c/verl/recipe/dapo/test_dapo_7b_math_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..a866e968508d3184150549228a1b6fc746f20728 --- /dev/null +++ b/e2c/verl/recipe/dapo/test_dapo_7b_math_megatron.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7b-MATH-megatron-0519a1' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-4} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=4 +train_tp=4 +train_pp=2 + +# TODO: support dynamic_bsz for megatron +# actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ +# actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ +# actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + +python3 -m verl.trainer.main_ppo \ + --config-path=config \ + --config-name='ppo_megatron_trainer.yaml' \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.clip_grad=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=10 \ + trainer.save_freq=10 \ + trainer.total_epochs=10 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/dapo/test_dapo_dspk_671b_megatron_96gb.sh b/e2c/verl/recipe/dapo/test_dapo_dspk_671b_megatron_96gb.sh new file mode 100644 index 0000000000000000000000000000000000000000..a62b68c66a539a5aa74ac7d6641368a728ebc2c2 --- /dev/null +++ b/e2c/verl/recipe/dapo/test_dapo_dspk_671b_megatron_96gb.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +# 0. download the config +# only need to download the configuration_deepseek.py and config.json +# remove the `quantization_config` in the `config.json` +# set `num_nextn_predict_layers=0` to disable MTP, which is not currently supported +huggingface-cli download deepseek-ai/DeepSeek-V3-0324 configuration_deepseek.py config.json + +project_name='DAPO' +exp_name='DAPO-DeepSeek-671b-megatron' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=False +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=0.1 + +loss_agg_mode="token-mean" + +train_prompt_bsz=256 # must be > n_gpus. need to fix +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 # mini_bsz * n >= micro_bsz * pp * dp + +NNODES=${NNODES:-64} + +# 1. download the dist_ckpt format model from https://huggingface.co/BearBiscuit05/dpsk-v3-671B-BF16-dist_ckpt/tree/main +# change the MODEL_PATH and MCORE_MODEL_PATH to your own path +# Paths +MODEL_PATH="" +MCORE_MODEL_PATH="" +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +aime24_test_path=${RAY_DATA_HOME}/data/aime-2024.parquet +# TEST_FILE="['$math500_test_path', '$aime24_test_path']" + +TEST_FILE="['$aime24_test_path']" + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=32 +train_tp=1 +train_ep=32 +train_pp=16 + +python3 -m verl.trainer.main_ppo \ + --config-path=config \ + --config-name='ppo_megatron_trainer.yaml' \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=${train_ep} \ + actor_rollout_ref.actor.megatron.dist_checkpointing_path=${MCORE_MODEL_PATH} \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_first_pipeline_stage=3 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.num_layers_in_last_pipeline_stage=2 \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.clip_grad=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=${train_ep} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + actor_rollout_ref.ref.megatron.dist_checkpointing_path=${MCORE_MODEL_PATH} \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=5 \ + trainer.save_freq=5 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=10 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/dapo/test_dapo_glm_air_megatron.sh b/e2c/verl/recipe/dapo/test_dapo_glm_air_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..2e7d91c07a5746d11c1be60f60c7c177d7af0c9b --- /dev/null +++ b/e2c/verl/recipe/dapo/test_dapo_glm_air_megatron.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +NNODES=${NNODES:-8} +NGPUS_PER_NODES=${NGPUS_PER_NODES:-8} + +project_name='DAPO' +exp_name='DAPO-GLM-AIR-MATH-megatron' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=128 +train_ppo_micro_batch_size_per_gpu=2 +infer_ppo_micro_batch_size_per_gpu=2 +# Paths +MODEL_PATH=/models/zai-org/GLM-4.5-Air-Base +# GLM Base model can use chat_template.jinja from instruct models +cp /models/zai-org/GLM-4.5-Air/chat_template.jinja ${MODEL_PATH}/chat_template.jinja + +TRAIN_FILE=/data/dapo/dapo-math-17k.parquet +aime24_test_path=/data/dapo/aime-2024.parquet +# math500_test_path=/data/rlhf/math500/test.parquet + +# TEST_FILE="['$math500_test_path', '$aime24_test_path']" + +TEST_FILE="['$aime24_test_path']" + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length))) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length))) +offload=True + +COMMON_PP=${COMMON_PP:-2} +COMMON_VPP=${COMMON_VPP:-null} +COMMON_CP=${COMMON_CP:-4} +COMMON_TP=${COMMON_TP:-2} +COMMON_EP=${COMMON_EP:-8} +COMMON_ETP=${COMMON_ETP:-1} + +TRAIN_TP=${TRAIN_TP:-$COMMON_TP} +INFER_TP=${INFER_TP:-8} + +ACTOR_PP=${ACTOR_PP:-$COMMON_PP} +ACTOR_VPP=${ACTOR_VPP:-$COMMON_VPP} +ACTOR_CP=${ACTOR_CP:-$COMMON_CP} +ACTOR_TP=${ACTOR_TP:-$TRAIN_TP} +ACTOR_EP=${ACTOR_EP:-$COMMON_EP} +ACTOR_ETP=${ACTOR_ETP:-$COMMON_ETP} +ROLLOUT_TP=${ROLLOUT_TP:-$INFER_TP} +REF_PP=${REF_PP:-$COMMON_PP} +REF_VPP=${REF_VPP:-$COMMON_VPP} +REF_CP=${REF_CP:-$COMMON_CP} +REF_TP=${REF_TP:-$TRAIN_TP} +REF_EP=${REF_EP:-$COMMON_EP} +REF_ETP=${REF_ETP:-$COMMON_ETP} +CRITIC_PP=${CRITIC_PP:-$COMMON_PP} +CRITIC_VPP=${CRITIC_VPP:-$COMMON_VPP} +CRITIC_CP=${CRITIC_CP:-$COMMON_CP} +CRITIC_TP=${CRITIC_TP:-$TRAIN_TP} +CRITIC_EP=${CRITIC_EP:-$COMMON_EP} +CRITIC_ETP=${CRITIC_ETP:-$COMMON_ETP} +RM_PP=${RM_PP:-$COMMON_PP} +RM_VPP=${RM_VPP:-$COMMON_VPP} +RM_CP=${RM_CP:-$COMMON_CP} +RM_TP=${RM_TP:-$TRAIN_TP} +RM_EP=${RM_EP:-$COMMON_EP} +RM_ETP=${RM_ETP:-$COMMON_ETP} + +USE_MBRIDGE=True +USE_DIST_CKPT=False + +# Install the latest mbridge +# pip install --no-cache-dir git+https://github.com/ISEEKYAN/mbridge.git + +python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_ppo_micro_batch_size_per_gpu} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.lr_decay_style='constant' \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.megatron.use_mbridge=$USE_MBRIDGE \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=$USE_DIST_CKPT \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${ACTOR_TP} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${ACTOR_PP} \ + actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=${ACTOR_VPP} \ + actor_rollout_ref.actor.megatron.context_parallel_size=${ACTOR_CP} \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=${ACTOR_EP} \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=${ACTOR_ETP} \ + actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity="selective" \ + actor_rollout_ref.actor.megatron.override_transformer_config.recompute_modules=["core_attn","moe_act","layernorm","mlp","moe"] \ + +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_shared_expert_overlap=False \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_token_dispatcher_type="flex" \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_enable_deepep=False \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.name='vllm' \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${infer_ppo_micro_batch_size_per_gpu} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${INFER_TP} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${infer_ppo_micro_batch_size_per_gpu} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${REF_TP} \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${REF_PP} \ + actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=${REF_VPP} \ + actor_rollout_ref.ref.megatron.context_parallel_size=${REF_CP} \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=${REF_EP} \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=${REF_ETP} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','wandb'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODES}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=10 \ + trainer.save_freq=100 \ + trainer.total_epochs=10 \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 \ No newline at end of file diff --git a/e2c/verl/recipe/dapo/test_dapo_qwen3_30b_math.sh b/e2c/verl/recipe/dapo/test_dapo_qwen3_30b_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..c6956635f2e9705744eb3c4918b86468f89e494d --- /dev/null +++ b/e2c/verl/recipe/dapo/test_dapo_qwen3_30b_math.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen3-30B-A3B-Base-MATH-0527a1' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-8} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B-Base"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=4 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=4 +fsdp_size=32 + +python3 -m verl.trainer.main_ppo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=10 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=300 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/dapo/test_dapo_qwen3_30b_math_single_node.sh b/e2c/verl/recipe/dapo/test_dapo_qwen3_30b_math_single_node.sh new file mode 100644 index 0000000000000000000000000000000000000000..5af2822ea267a601f2d7bde7dd6dd40e67dab876 --- /dev/null +++ b/e2c/verl/recipe/dapo/test_dapo_qwen3_30b_math_single_node.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen3-30B-A3B-Base-MATH-0719a1' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 4)) +enable_overlong_buffer=False +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=0.1 + +loss_agg_mode="token-mean" + +train_prompt_bsz=64 +n_resp_per_prompt=16 +train_prompt_mini_bsz=16 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-1} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B-Base"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=4 +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 1)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=4 +fsdp_size=8 + +python3 -m verl.trainer.main_ppo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=-1 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=300 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/deepeyes/README.md b/e2c/verl/recipe/deepeyes/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cde340914510306622873a08d82ffde50bb9c58b --- /dev/null +++ b/e2c/verl/recipe/deepeyes/README.md @@ -0,0 +1,55 @@ +# DeepEyes: Incentivizing "Thinking with Images" via Reinforcement Learning + +This directory contains the implementation for reproducing the DeepEyes paper within the verl framework, supporting multi-turn visual tool calls. This implementation is based on the original [DeepEyes paper](https://arxiv.org/abs/2505.14362) and its [official implementation](https://github.com/Visual-Agent/DeepEyes), integrated with the multi-modal and multi-turn capabilities of the verl framework. + +## Reproducing the Experiment + +> **Note on the 'Chart' Dataset:** +> +> The provided preprocessing script intentionally excludes `data_v0.8_visual_toolbox_v2.parquet`, which contains the 'Chart' data. This subset consists of very high-resolution images, often resembling large figures composed of multiple sub-plots, much like those found in academic papers. +> +> Consequently, even after using the zoom-in tool, the resulting cropped images remain large. This poses a significant risk of causing Out-of-Memory (OOM) errors, which can abruptly terminate the training process. +> +> **We strongly recommend against training on the 'Chart' dataset on a single node.** + +> **Note on the 'thinklite' Dataset:** +> Many images in the `thinklite` dataset have a very low resolution, with either a height or width below 28 pixels. This fails to meet the minimum input size required by the Qwen-2.5VL image processor and would cause errors during data loading. +> +> To mitigate this, we upscale these low-resolution images to satisfy the processor's requirements. However, please be aware that because the original resolution is low, subsequent `crop` operations by the zoom-in tool might frequently trigger exceptions, which could in turn affect the model's tool-use performance. + +First, launch an inference service to act as a judge for reward calculation. You can use the following script as a reference: + +```bash +python -m sglang.launch_server --model-path /path/to/Qwen2.5-72B-Instruct \ + --port 18901 \ + --tp-size 8 \ + --context-length 32768 \ + --trust-remote-code \ + --log-requests false +``` + +Next, you can start the training: + +```bash +bash recipe/deepeyes/run_deepeyes_grpo.sh +``` + +## Performance + +![score](https://private-user-images.githubusercontent.com/82520804/474784419-b13f4f72-bb3a-4281-a43b-1f34a9037c0c.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3NTQ0NTQxMTMsIm5iZiI6MTc1NDQ1MzgxMywicGF0aCI6Ii84MjUyMDgwNC80NzQ3ODQ0MTktYjEzZjRmNzItYmIzYS00MjgxLWE0M2ItMWYzNGE5MDM3YzBjLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNTA4MDYlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjUwODA2VDA0MTY1M1omWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTJjNGMxMjhiOGM4MTNhYTEzYTE2MTYzY2ZjYWRhNmEzMmVjNjUxOGI3MTgzOGQyM2ZmOWJlYTZlNDYzYzU0ZDkmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.qTDX-3fyLHWdeFh9o4b6nIAB57bT0XyLjKXhNV6k5nA) + +![entropy](https://private-user-images.githubusercontent.com/82520804/474785253-752106a9-e25d-4b44-aef9-1ac98015d05c.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3NTQ0NTQxMTMsIm5iZiI6MTc1NDQ1MzgxMywicGF0aCI6Ii84MjUyMDgwNC80NzQ3ODUyNTMtNzUyMTA2YTktZTI1ZC00YjQ0LWFlZjktMWFjOTgwMTVkMDVjLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNTA4MDYlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjUwODA2VDA0MTY1M1omWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTM4OGQ2ZGI3M2JlYWE4YTQyMzIxMWYxMzZhNDBmNmYxNzcwNDgxNThiZDRiMzQyYzUwZjc3OWE4YzdhYWEwMWUmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.PhimMTxXXEtMLPGzejPQuw-Ul0As8ey-hyy1qkeABIQ) + +![num_turns](https://private-user-images.githubusercontent.com/82520804/474785462-c99c7952-14db-485a-acd2-14e5956ecc34.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3NTQ0NTQxMTMsIm5iZiI6MTc1NDQ1MzgxMywicGF0aCI6Ii84MjUyMDgwNC80NzQ3ODU0NjItYzk5Yzc5NTItMTRkYi00ODVhLWFjZDItMTRlNTk1NmVjYzM0LnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNTA4MDYlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjUwODA2VDA0MTY1M1omWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTJkNWYwMGVjOWM4NDVhZTkzZWI5NWMzMGVjZTcyZGM2NDExY2FmYTBlYWJmZTk5YTU5MzM3NmNkYWI4Y2U4Y2YmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.Ieakk_ttMsNygVzpZZqGs1507j2GC-rqHSYH9iQQ71Q) + +See [Comment](https://github.com/volcengine/verl/pull/2398#issuecomment-3157142856) for more details. + +Note: AgentLoop does not directly record num_tool_calls, but records num_turns. In our scenario, you can calculate the number of tool calls by num_tool_calls = num_turns / 2 - 1. + +## References and Acknowledgements + +- [DeepEyes Paper](https://arxiv.org/abs/2505.14362) +- [DeepEyes Official Implementation](https://github.com/Visual-Agent/DeepEyes) + +--- +If you need further details for reproduction or encounter any issues, feel free to open an issue or contact the maintainers. \ No newline at end of file diff --git a/e2c/verl/recipe/deepeyes/configs/deepeyes_multiturn_grpo.yaml b/e2c/verl/recipe/deepeyes/configs/deepeyes_multiturn_grpo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5978f4dbd14290d9adbfbe4e6fd86887f46ce4d2 --- /dev/null +++ b/e2c/verl/recipe/deepeyes/configs/deepeyes_multiturn_grpo.yaml @@ -0,0 +1,32 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + max_prompt_length: 2048 + max_response_length: 2048 + train_batch_size: 256 + return_raw_chat: True + return_multi_modal_inputs: False + custom_cls: + path: "recipe/deepeyes/deepeyes.py" + name: CustomRLHFDataset + +actor_rollout_ref: + hybrid_engine: True + model: + custom_chat_template: "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{%- if tools %}{{- '<|im_start|>system\\n' }}{%- if messages[0]['role'] == 'system' %}{%- if messages[0]['content'] is string %}{{- messages[0]['content'] }}{%- else %}{{- messages[0]['content'][0]['text'] }}{%- endif %}{%- else %}{{- 'You are a helpful assistant.' }}{%- endif %}{{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}{%- for tool in tools %}{{- \"\\n\" }}{{- tool | tojson }}{%- endfor %}{{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}{% for message in messages %}{% if message['role'] != 'system' or loop.first == false %}{%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{%- elif message.role == \"assistant\" %}{{- '<|im_start|>' + message.role }}{%- if message.content %}{{- '\\n' + message.content }}{%- endif %}{%- for tool_call in message.tool_calls %}{%- if tool_call.function is defined %}{%- set tool_call = tool_call.function %}{%- endif %}{{- '\\n\\n{\"name\": \"' }}{{- tool_call.name }}{{- '\", \"arguments\": ' }}{{- tool_call.arguments | tojson }}{{- '}\\n' }}{%- endfor %}{{- '<|im_end|>\\n' }}{%- elif message.role == \"tool\" %}{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}{{- '<|im_start|>user' }}{%- endif %}{{- '\\n\\n' }}{% if message['content'] is string %}{{ message.content }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif content['type'] == 'text' or 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{- '\\n' }}{%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}{{- '<|im_end|>\\n' }}{%- endif %}{%- endif %}{% endif %}{% endfor %}{%- else %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}{%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{%- elif message.role == \"assistant\" %}{{- '<|im_start|>' + message.role }}{%- if message.content %}{{- '\\n' + message.content }}{%- endif %}{%- for tool_call in message.tool_calls %}{%- if tool_call.function is defined %}{%- set tool_call = tool_call.function %}{%- endif %}{{- '\\n\\n{\"name\": \"' }}{{- tool_call.name }}{{- '\", \"arguments\": ' }}{{- tool_call.arguments | tojson }}{{- '}\\n' }}{%- endfor %}{{- '<|im_end|>\\n' }}{%- elif message.role == \"tool\" %}{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}{{- '<|im_start|>user' }}{%- endif %}{{- '\\n\\n' }}{% if message['content'] is string %}{{ message.content }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif content['type'] == 'text' or 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{- '\\n' }}{%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}{{- '<|im_end|>\\n' }}{%- endif %}{%- endif %}{% endfor %}{%- endif %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" + rollout: + name: sglang + multi_turn: + enable: True + max_assistant_turns: 5 + tool_config_path: "recipe/deepeyes/config/image_zoom_in_tool_config.yaml" + +custom_reward_function: + path: "recipe/deepeyes/deepeyes.py" + name: compute_score \ No newline at end of file diff --git a/e2c/verl/recipe/deepeyes/configs/image_zoom_in_tool_config.yaml b/e2c/verl/recipe/deepeyes/configs/image_zoom_in_tool_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2802f094aefbebb736f5624c0ea5704e9cc36b7 --- /dev/null +++ b/e2c/verl/recipe/deepeyes/configs/image_zoom_in_tool_config.yaml @@ -0,0 +1,26 @@ +tools: + - class_name: "verl.tools.image_zoom_in_tool.ImageZoomInTool" + config: + num_workers: 256 + rate_limit: 256 + timeout: 60 + type: native + tool_schema: + type: "function" + function: + name: "image_zoom_in_tool" + description: "Zoom in on a specific region of an image by cropping it based on a bounding box (bbox) and an optional object label." + parameters: + type: "object" + properties: + bbox_2d: + type: "array" + items: + type: "number" + minItems: 4 + maxItems: 4 + description: "The bounding box of the region to zoom in, as [x1, y1, x2, y2], where (x1, y1) is the top-left corner and (x2, y2) is the bottom-right corner." + label: + type: "string" + description: "The name or label of the object in the specified bounding box (optional)." + required: ["bbox_2d"] \ No newline at end of file diff --git a/e2c/verl/recipe/deepeyes/deepeyes.py b/e2c/verl/recipe/deepeyes/deepeyes.py new file mode 100644 index 0000000000000000000000000000000000000000..94c5535a043722a545fad828e3b996493efeaf9b --- /dev/null +++ b/e2c/verl/recipe/deepeyes/deepeyes.py @@ -0,0 +1,426 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import logging +import os +import random +import re + +import requests +from openai import OpenAI +from PIL import Image + +import verl.utils.torch_functional as verl_F +from verl.utils.dataset.rl_dataset import RLHFDataset +from verl.utils.model import compute_position_id_with_mask + +logger = logging.getLogger(__name__) + +openai_api_key = "EMPTY" +openai_api_base = os.environ.get("LLM_AS_A_JUDGE_BASE", "http://10.1.100.71:18901/v1") + +client = OpenAI( + api_key=openai_api_key, + base_url=openai_api_base, +) + +model_name = "" +if openai_api_base: + try: + response = requests.get(f"{openai_api_base}/models") + response.raise_for_status() + models = response.json() + if models.get("data"): + model_name = models["data"][0]["id"] + else: + logger.warning("No models found at the specified API base for reward scoring.") + except (requests.exceptions.RequestException, KeyError, IndexError) as e: + logger.warning(f"Failed to get model from {openai_api_base}: {e}. Reward scoring will be disabled.") + + +class CustomRLHFDataset(RLHFDataset): + def __getitem__(self, item): + """ + Note that we also return the raw_input_ids so that it can be combined with other chat template + """ + row_dict: dict = self.dataframe[item] + row_dict[self.prompt_key] = [ + { + "role": "system", + # We don't need tool description, because custom_chat_template will add it. + "content": ( + "You are a helpful assistant. You can call functions to assist with the user query. " + "Important: You must call only one function at a time. After each function call, " + "wait for the execution result before making the next function call if needed." + ), + }, + { + "role": "user", + "content": row_dict[self.prompt_key][1]["content"], + }, + ] + messages = self._build_messages(row_dict) + model_inputs = {} + + if self.processor is not None: + raw_prompt = self.processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False) + multi_modal_data = {} + + images = None + row_dict_images = row_dict.pop(self.image_key, None) + if row_dict_images: + images = [Image.open(io.BytesIO(image["bytes"])) for image in row_dict_images] + + # due to the image key is "image" instead of "images" in vllm, we need to use "image" here + # link: https://github.com/vllm-project/vllm/blob/3c545c0c3b98ee642373a308197d750d0e449403/vllm/multimodal/parse.py#L205 # noqa: E501 + multi_modal_data["image"] = images + + model_inputs = self.processor(text=[raw_prompt], images=images, return_tensors="pt") + + input_ids = model_inputs.pop("input_ids") + attention_mask = model_inputs.pop("attention_mask") + + if "second_per_grid_ts" in model_inputs: + model_inputs.pop("second_per_grid_ts") + + # There's a trap here, multi_modal_inputs has to be a dict, not BatchFeature + row_dict["multi_modal_data"] = multi_modal_data + + # We will do batch.union() in the trainer, + # so we cannot have "multi_modal_inputs" in row_dict if rollout generates new multi_modal_inputs + if self.return_multi_modal_inputs: + row_dict["multi_modal_inputs"] = dict(model_inputs) + + # second_per_grid_ts isn't used for training, just for mrope + row_dict["multi_modal_inputs"].pop("second_per_grid_ts", None) + + else: + raw_prompt = self.tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False) + model_inputs = self.tokenizer(raw_prompt, return_tensors="pt", add_special_tokens=False) + input_ids = model_inputs.pop("input_ids") + attention_mask = model_inputs.pop("attention_mask") + + input_ids, attention_mask = verl_F.postprocess_data( + input_ids=input_ids, + attention_mask=attention_mask, + max_length=self.max_prompt_length, + pad_token_id=self.tokenizer.pad_token_id, + left_pad=True, + truncation=self.truncation, + ) + + if self.processor is not None and "Qwen2VLImageProcessor" in self.processor.image_processor.__class__.__name__: + from verl.models.transformers.qwen2_vl import get_rope_index + + position_ids = [ + get_rope_index( + self.processor, + input_ids=input_ids[0], + image_grid_thw=model_inputs.get("image_grid_thw"), + video_grid_thw=model_inputs.get("video_grid_thw"), + second_per_grid_ts=model_inputs.get("second_per_grid_ts"), + attention_mask=attention_mask[0], + ) + ] # (1, 3, seq_len) + + else: + position_ids = compute_position_id_with_mask(attention_mask) + + row_dict["input_ids"] = input_ids[0] + row_dict["attention_mask"] = attention_mask[0] + row_dict["position_ids"] = position_ids[0] + + raw_prompt_ids = self.tokenizer.encode(raw_prompt, add_special_tokens=False) + if len(raw_prompt_ids) > self.max_prompt_length: + if self.truncation == "left": + raw_prompt_ids = raw_prompt_ids[-self.max_prompt_length :] + elif self.truncation == "right": + raw_prompt_ids = raw_prompt_ids[: self.max_prompt_length] + elif self.truncation == "middle": + left_half = self.max_prompt_length // 2 + right_half = self.max_prompt_length - left_half + raw_prompt_ids = raw_prompt_ids[:left_half] + raw_prompt_ids[-right_half:] + elif self.truncation == "error": + raise RuntimeError(f"Prompt length {len(raw_prompt_ids)} is longer than {self.max_prompt_length}.") + + row_dict["raw_prompt_ids"] = raw_prompt_ids + # encode prompts without chat template + if self.return_raw_chat: + row_dict["raw_prompt"] = messages + + # get prompts with chat template + if self.return_full_prompt: + row_dict["full_prompts"] = raw_prompt # array of strings + + # add index for each prompt + index = row_dict.get("extra_info", {}).get("index", 0) + tools_kwargs = { + "image_zoom_in_tool": { + "create_kwargs": {"image": images[0]}, + # "execute_kwargs": {}, + # "calc_reward_kwargs": {}, + # "release_kwargs": {}, + } + } + row_dict["index"] = index + row_dict["tools_kwargs"] = tools_kwargs + row_dict["agent_name"] = "tool_agent" + return row_dict + + +def compute_score(data_source: str, solution_str: str, ground_truth: str, extra_info=None) -> float: + """ + Compute reward score for model solutions with robust handling of various formats. + + Returns a weighted combination of: + - Accuracy reward (0.8 weight): Whether the answer is semantically correct + - Format reward (0.2 weight): Whether the output follows expected format + - Tool reward (1.2 weight): Whether tools were used when answer is correct + """ + + # Initialize tracking variables + is_format_error = False + + # 1. Check tag format + count_think_1 = solution_str.count("") + count_think_2 = solution_str.count("") + if count_think_1 != count_think_2: + is_format_error = True + + # 2. Check vision tokens (skip this since tokenizer removes special tokens) + # We'll use and instead to detect tool usage + + # 3. Extract answer text with multiple fallback strategies + answer_text = "" + + # Strategy 1: Try to extract from tags first + predict_no_think = ( + solution_str.split("")[-1].strip() if "" in solution_str else solution_str.strip() + ) + + # Check tag format + count_answer_1 = predict_no_think.count("") + count_answer_2 = predict_no_think.count("") + if count_answer_1 != count_answer_2: + is_format_error = True + + # Try to extract from tags + answer_match = re.search(r"(.*?)", predict_no_think, re.DOTALL) + if answer_match: + answer_text = answer_match.group(1).strip() + else: + # No proper tags found - this is a format error + is_format_error = True + + # Strategy 2: If no tags, extract content after tool responses + # Look for pattern: ...assistant\n[actual_answer] + tool_response_match = re.search( + r"\s*assistant\s*\n(.*?)$", predict_no_think, re.DOTALL | re.MULTILINE + ) + if tool_response_match: + answer_text = tool_response_match.group(1).strip() + else: + # Strategy 3: If no tool responses, look for content after + if "" in solution_str: + # Remove any remaining tool-related tags and extract meaningful content + remaining_content = predict_no_think + # Remove tool calls and responses + remaining_content = re.sub(r".*?", "", remaining_content, flags=re.DOTALL) + remaining_content = re.sub( + r".*?", "", remaining_content, flags=re.DOTALL + ) + # Remove user/assistant markers + remaining_content = re.sub(r"\b(user|assistant)\b", "", remaining_content) + answer_text = remaining_content.strip() + else: + # Strategy 4: Use the entire solution_str as fallback + answer_text = solution_str.strip() + + # Clean up answer text + answer_text = answer_text.strip() + + # If answer is still empty after all strategies, mark as format error + if not answer_text: + is_format_error = True + answer_text = solution_str.strip() # Use full text as last resort + + # 4. Evaluate correctness using LLM judge + question_text = extra_info.get("question", "") if extra_info else "" + + if not client or not model_name: + logger.warning("Reward function client not initialized or model name not found.") + return 0.0 + + system_prompt = ( + "You are an expert evaluator. Your task is to determine if a model's answer is semantically equivalent to a " + "provided standard answer, given a specific question.\n" + "Your evaluation must be strict. The model's answer is only correct if it fully matches the meaning of the " + "standard answer.\n" + 'You must provide your final judgement as a single word: either "CORRECT" or "INCORRECT". Do not provide ' + "any explanation or other text." + ) + + user_prompt = ( + f"I will provide a question, a standard answer, and a model's answer. You must evaluate if the model's " + f"answer is correct.\n\n" + f"---\n" + f"**Example 1:**\n" + f"[Question]: Is the countertop tan or blue?\n" + f"[Standard Answer]: The countertop is tan.\n" + f"[Model's Answer]: tan\n" + f"[Your Judgement]: CORRECT\n" + f"---\n" + f"**Example 2:**\n" + f"[Question]: Is the man phone both blue and closed?\n" + f"[Standard Answer]: Yes, the man phone is both blue and closed.\n" + f"[Model's Answer]: No.\n" + f"[Your Judgement]: INCORRECT\n" + f"---\n" + f"**Task:**\n" + f"[Question]: {question_text}\n" + f"[Standard Answer]: {ground_truth}\n" + f"[Model's Answer]: {answer_text}\n" + f"[Your Judgement]:" + ) + + try: + chat_response = client.chat.completions.create( + model=model_name, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + seed=random.randint(0, 1000000), + temperature=0.1, # Lower temperature for more deterministic judgement + extra_body={ + "chat_template_kwargs": {"enable_thinking": False}, + }, + ) + response = chat_response.choices[0].message.content.strip() + except Exception as e: + logger.warning(f" [WARNING] Chat completion request failed: {e}") + return 0.0 + + # Parse LLM judge response + if re.search(r"\bCORRECT\b", response, re.IGNORECASE): + acc_reward = 1.0 + elif re.search(r"\bINCORRECT\b", response, re.IGNORECASE): + acc_reward = 0.0 + else: + logger.warning( + f" [WARNING] Judgement format error. Expected 'CORRECT' or 'INCORRECT'.\n" + f"Response: '{response}'\n" + f"Model Answer: '{answer_text}'\n" + f"Ground Truth: '{ground_truth}'" + ) + acc_reward = 0.0 + + # Penalize excessively long answers (potential judge hacking) + if len(answer_text) >= 1000: + acc_reward = 0.0 + is_format_error = True + + # 5. Check tool usage - look for tool_call/tool_response patterns instead of vision tokens + has_tool_usage = bool( + re.search(r".*?", solution_str, re.DOTALL) + or re.search(r".*?", solution_str, re.DOTALL) + ) + + # Tool reward: only give if tools were used AND answer is correct + tool_reward = 1.0 if has_tool_usage and acc_reward > 0.5 else 0.0 + + # Format reward: penalty for format errors + format_reward = -1.0 if is_format_error else 0.0 + + # Log debug information for problematic cases + if is_format_error or not answer_text: + logger.debug( + f"Format issue detected:\n" + f"Solution: {solution_str[:200]}...\n" + f"Extracted answer: '{answer_text}'\n" + f"Format error: {is_format_error}\n" + f"Tool usage: {has_tool_usage}" + ) + + # Final weighted score + final_score = 0.8 * acc_reward + 0.2 * format_reward + 1.2 * tool_reward + + return final_score + + +if __name__ == "__main__": + # Test case 1: Original test case + predict_str = "The answer is 2 + 2 = 4 right left " + ground_truth = "left" + extra_info = { + "answer": "The woman is to the left of the man who is holding the camera.", + "id": 0, + "image": "/cpfs/user/honglingyi/DATA/LLM/Vstar/gqa/images/713270.jpg", + "pred_ans": "The woman is to the right of the man who is holding the camera.", + "question": "Is the woman to the left or to the right of the man who is holding the camera?", + } + print("=== Test Case 1: Original test ===") + import time + + time_start = time.time() + score = compute_score("common_reasoning", predict_str, ground_truth, extra_info) + print(f"Score: {score}") + time_end = time.time() + print(f"Time: {time_end - time_start}") + + # Test case 2: Problematic case mentioned by user + problematic_solution = """ +{"name": "image_zoom_in_tool", "arguments": {"bbox_2d": [226, 399, 265, 464], "label": "white van"}} +user + +Zoomed in on the image to the region [226, 399, 265, 464] with label white van. + +assistant +The white van is visible in the lower section of the image, near the diagonal road.""" + + problematic_ground_truth = "Yes, the white van is indeed situated in the bottom part of the picture." + problematic_extra_info = { + "question": "Is the white van in the bottom part of the picture?", + } + + print("\n=== Test Case 2: Problematic case (no answer tags) ===") + print(f"Solution: {problematic_solution}") + print(f"Ground truth: {problematic_ground_truth}") + + time_start = time.time() + score2 = compute_score("common_reasoning", problematic_solution, problematic_ground_truth, problematic_extra_info) + print(f"Score: {score2}") + time_end = time.time() + print(f"Time: {time_end - time_start}") + + # Test case 3: Well-formatted case with tools + well_formatted_solution = """ +I need to use the image zoom tool to get a better look at the specific area. + + +{"name": "image_zoom_in_tool", "arguments": {"bbox_2d": [226, 399, 265, 464], "label": "white van"}} + + +Zoomed in on the image to the region [226, 399, 265, 464] with label white van. + +Yes, the white van is indeed situated in the bottom part of the picture.""" + + print("\n=== Test Case 3: Well-formatted case ===") + time_start = time.time() + score3 = compute_score( + "common_reasoning", well_formatted_solution, problematic_ground_truth, problematic_extra_info + ) + print(f"Score: {score3}") + time_end = time.time() + print(f"Time: {time_end - time_start}") diff --git a/e2c/verl/recipe/deepeyes/run_deepeyes_grpo.sh b/e2c/verl/recipe/deepeyes/run_deepeyes_grpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..3a25332817a7513536e73a68b2afd79a67599a76 --- /dev/null +++ b/e2c/verl/recipe/deepeyes/run_deepeyes_grpo.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -x + +export LLM_AS_A_JUDGE_BASE="your llm-as-a-judge server/v1" +export WANDB_API_KEY="your wandb key" + +PROJECT_NAME="your_project_name" +EXPERIMENT_NAME="your_experiment_name" + +BASEDIR=base_dir +SAVE_CHECKPOINT_DIR=${BASEDIR}/verl_checkpoints +DATASET_TRAIN=${BASEDIR}/dataset/train.parquet +DATASET_VAL=${BASEDIR}/dataset/val.parquet + +REF_MODEL_PATH=ref_model_path + +PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \ + --config-path=${BASEDIR}/recipe/deepeyes/configs \ + --config-name='deepeyes_multiturn_grpo' \ + data.train_files=${DATASET_TRAIN} \ + data.val_files=[${DATASET_VAL}] \ + data.train_batch_size=128 \ + data.max_prompt_length=8192 \ + data.max_response_length=16384 \ + data.return_raw_chat=True \ + data.filter_overlong_prompts=True \ + algorithm.adv_estimator=grpo \ + algorithm.kl_ctrl.kl_coef=0.0 \ + actor_rollout_ref.model.path=${REF_MODEL_PATH} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0.0 \ + actor_rollout_ref.actor.checkpoint.save_contents=['model','hf_model','optimizer','extra'] \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.multi_turn.enable=True \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=5 \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=5 \ + actor_rollout_ref.rollout.multi_turn.max_parallel_calls=1 \ + actor_rollout_ref.rollout.multi_turn.tool_config_path=recipe/deepeyes/configs/image_zoom_in_tool_config.yaml \ + trainer.critic_warmup=0 \ + trainer.logger=['console','wandb','tensorboard'] \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=8 \ + trainer.test_freq=80 \ + trainer.project_name=${PROJECT_NAME} \ + trainer.experiment_name=${EXPERIMENT_NAME} \ + trainer.default_local_dir=${SAVE_CHECKPOINT_DIR}/${PROJECT_NAME}/${EXPERIMENT_NAME} \ + +trainer.tensorboard_dir=${SAVE_CHECKPOINT_DIR}/logs/tensorboard \ + +trainer.rl_logging_board_dir=${SAVE_CHECKPOINT_DIR}/logs/rl_logging_board \ + trainer.total_epochs=1 2>&1 | tee ./logs/${EXPERIMENT_NAME}.log diff --git a/e2c/verl/recipe/entropy/32b_clip_cov.sh b/e2c/verl/recipe/entropy/32b_clip_cov.sh new file mode 100644 index 0000000000000000000000000000000000000000..db7ec5b289bf7eddd4c5cee722c62d24ec462754 --- /dev/null +++ b/e2c/verl/recipe/entropy/32b_clip_cov.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export WANDB_API_KEY=YOUR_WANDB_API_KEY +# export VLLM_USE_V1=1 + +project_name='Qwen2.5-32B' +exp_name='clipcov' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=1 +clip_ratio_high=1 +clip_cov_ratio=0.0002 +clip_cov_lb=1.0 +clip_cov_ub=5.0 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=False +overlong_buffer_len=$((1024 * 2)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" +loss_mode="clip_cov" +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=256 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +train_prompt_mini_bsz=32 +n_resp_per_prompt=8 +max_token=20480 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-4} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"/YOUR_MODELPATH"} +CKPTS_DIR=${CKPTS_DIR:-"/YOUR_CKPTS_PATH"} +TRAIN_FILE=${TRAIN_FILE:-"/YOUR_TRAIN_FILE_PATH"} +TEST_FILE=${TEST_FILE:-["/YOUR_TRAIN_FILE_PATH"]} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +ppo_kl_coef=1 +kl_cov_ratio=0.02 + +# Mathematically equivalent +use_dynamic_bsz=True +infer_micro_batch_size=null +train_micro_batch_size=null +offload=False + +HYDRA_FULL_ERROR=1 python -m recipe.entropy.main_entropy \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.filter_overlong_prompts=False \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + data.return_raw_chat=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.policy_loss.clip_cov_ratio=${clip_cov_ratio} \ + actor_rollout_ref.actor.policy_loss.clip_cov_lb=${clip_cov_lb} \ + actor_rollout_ref.actor.policy_loss.clip_cov_ub=${clip_cov_ub} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.mode=sync \ + actor_rollout_ref.rollout.name=vllm \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.weight_decay=0 \ + actor_rollout_ref.actor.optim.warmup_style=constant \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${train_micro_batch_size} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.clip_cov_ratio=${clip_cov_ratio} \ + actor_rollout_ref.actor.clip_cov_lb=${clip_cov_lb} \ + actor_rollout_ref.actor.clip_cov_ub=${clip_cov_ub} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=${max_token} \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=4 \ + trainer.save_freq=32 \ + trainer.total_epochs=1000 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=disable diff --git a/e2c/verl/recipe/entropy/32b_kl_cov.sh b/e2c/verl/recipe/entropy/32b_kl_cov.sh new file mode 100644 index 0000000000000000000000000000000000000000..e6f4455e8c78118ede3463b5dd2be6b8820947ea --- /dev/null +++ b/e2c/verl/recipe/entropy/32b_kl_cov.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export WANDB_API_KEY=YOUR_WANDB_API_KEY +# export VLLM_USE_V1=1 + +project_name='Qwen2.5-32B' +exp_name='klcov' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.2 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=False +overlong_buffer_len=$((1024 * 2)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" +loss_mode="kl_cov" +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=256 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +train_prompt_mini_bsz=32 +n_resp_per_prompt=8 +max_token=20480 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-4} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"/YOUR_MODELPATH"} +CKPTS_DIR=${CKPTS_DIR:-"/YOUR_CKPTS_PATH"} +TRAIN_FILE=${TRAIN_FILE:-"/YOUR_TRAIN_FILE_PATH"} +TEST_FILE=${TEST_FILE:-["/YOUR_TRAIN_FILE_PATH"]} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +ppo_kl_coef=1 +kl_cov_ratio=0.0002 + +# Mathematically equivalent +use_dynamic_bsz=True +infer_micro_batch_size=null +train_micro_batch_size=null +offload=False + +HYDRA_FULL_ERROR=1 python -m recipe.entropy.main_entropy \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.filter_overlong_prompts=False \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + data.return_raw_chat=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.policy_loss.kl_cov_ratio=${kl_cov_ratio} \ + actor_rollout_ref.actor.policy_loss.ppo_kl_coef=${ppo_kl_coef} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.mode=sync \ + actor_rollout_ref.rollout.name=vllm \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.weight_decay=0 \ + actor_rollout_ref.actor.optim.warmup_style=constant \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${train_micro_batch_size} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=${max_token} \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=4 \ + trainer.save_freq=32 \ + trainer.total_epochs=1000 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=disable diff --git a/e2c/verl/recipe/entropy/32b_kl_cov_mininbsz.sh b/e2c/verl/recipe/entropy/32b_kl_cov_mininbsz.sh new file mode 100644 index 0000000000000000000000000000000000000000..95cc683fcb9c9530441c7b69fc1047f200bcaf4f --- /dev/null +++ b/e2c/verl/recipe/entropy/32b_kl_cov_mininbsz.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export WANDB_API_KEY=YOUR_WANDB_API_KEY +# export VLLM_USE_V1=1 + +project_name='Qwen2.5-32B' +exp_name='klcov' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.2 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=False +overlong_buffer_len=$((1024 * 2)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" +loss_mode="kl_cov" +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=256 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +train_prompt_mini_bsz=16 +n_resp_per_prompt=8 +max_token=20480 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-4} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"/YOUR_MODELPATH"} +CKPTS_DIR=${CKPTS_DIR:-"/YOUR_CKPTS_PATH"} +TRAIN_FILE=${TRAIN_FILE:-"/YOUR_TRAIN_FILE_PATH"} +TEST_FILE=${TEST_FILE:-["/YOUR_TRAIN_FILE_PATH"]} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +ppo_kl_coef=1 +kl_cov_ratio=0.0002 + +# Mathematically equivalent +use_dynamic_bsz=True +infer_micro_batch_size=null +train_micro_batch_size=null +offload=False + +HYDRA_FULL_ERROR=1 python -m recipe.entropy.main_entropy \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.filter_overlong_prompts=False \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + data.return_raw_chat=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.policy_loss.kl_cov_ratio=${kl_cov_ratio} \ + actor_rollout_ref.actor.policy_loss.ppo_kl_coef=${ppo_kl_coef} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.mode=sync \ + actor_rollout_ref.rollout.name=vllm \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.weight_decay=0 \ + actor_rollout_ref.actor.optim.warmup_style=constant \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${train_micro_batch_size} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=${max_token} \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=4 \ + trainer.save_freq=32 \ + trainer.total_epochs=1000 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=disable diff --git a/e2c/verl/recipe/entropy/7b_clip_cov.sh b/e2c/verl/recipe/entropy/7b_clip_cov.sh new file mode 100644 index 0000000000000000000000000000000000000000..affdf61729431a1fcea651eac6de292dc2ea0eb1 --- /dev/null +++ b/e2c/verl/recipe/entropy/7b_clip_cov.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export WANDB_API_KEY=YOUR_WANDB_API_KEY +# export VLLM_USE_V1=1 + +project_name='Qwen2.5-7B' +exp_name='clipcov' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=1 +clip_ratio_high=1 +clip_cov_ratio=0.0002 +clip_cov_lb=1.0 +clip_cov_ub=5.0 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=False +overlong_buffer_len=$((1024 * 2)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" +loss_mode="clip_cov" +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=256 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +train_prompt_mini_bsz=32 +n_resp_per_prompt=8 +max_token=30720 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-4} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"/YOUR_MODELPATH"} +CKPTS_DIR=${CKPTS_DIR:-"/YOUR_CKPTS_PATH"} +TRAIN_FILE=${TRAIN_FILE:-"/YOUR_TRAIN_FILE_PATH"} +TEST_FILE=${TEST_FILE:-["/YOUR_TRAIN_FILE_PATH"]} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +ppo_kl_coef=1 +kl_cov_ratio=0.2 + +# Mathematically equivalent +use_dynamic_bsz=True +infer_micro_batch_size=null +train_micro_batch_size=null +offload=False + +HYDRA_FULL_ERROR=1 python -m recipe.entropy.main_entropy \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.filter_overlong_prompts=False \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + data.return_raw_chat=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.policy_loss.clip_cov_ratio=${clip_cov_ratio} \ + actor_rollout_ref.actor.policy_loss.clip_cov_lb=${clip_cov_lb} \ + actor_rollout_ref.actor.policy_loss.clip_cov_ub=${clip_cov_ub} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.mode=sync \ + actor_rollout_ref.rollout.name=vllm \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.weight_decay=0 \ + actor_rollout_ref.actor.optim.warmup_style=constant \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${train_micro_batch_size} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=${max_token} \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=4 \ + trainer.save_freq=32 \ + trainer.total_epochs=1000 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=disable diff --git a/e2c/verl/recipe/entropy/7b_kl_cov.sh b/e2c/verl/recipe/entropy/7b_kl_cov.sh new file mode 100644 index 0000000000000000000000000000000000000000..a7938fdd25fbbd82f84e6cd9182c61105f71d5f6 --- /dev/null +++ b/e2c/verl/recipe/entropy/7b_kl_cov.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export WANDB_API_KEY=YOUR_WANDB_API_KEY +# export VLLM_USE_V1=1 + +project_name='Qwen2.5-7B' +exp_name='klcov' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.2 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=False +overlong_buffer_len=$((1024 * 2)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" +loss_mode="kl_cov" +enable_filter_groups=True +filter_groups_metric=acc +max_num_gen_batches=10 +train_prompt_bsz=256 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +train_prompt_mini_bsz=32 +n_resp_per_prompt=8 +max_token=30720 + +# Ray +RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +WORKING_DIR=${WORKING_DIR:-"${PWD}"} +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-4} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"/YOUR_MODELPATH"} +CKPTS_DIR=${CKPTS_DIR:-"/YOUR_CKPTS_PATH"} +TRAIN_FILE=${TRAIN_FILE:-"/YOUR_TRAIN_FILE_PATH"} +TEST_FILE=${TEST_FILE:-["/YOUR_TRAIN_FILE_PATH"]} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +ppo_kl_coef=1 +kl_cov_ratio=0.002 + +# Mathematically equivalent +use_dynamic_bsz=True +infer_micro_batch_size=null +train_micro_batch_size=null +offload=False + +HYDRA_FULL_ERROR=1 python -m recipe.entropy.main_entropy \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.filter_overlong_prompts=False \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + data.return_raw_chat=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + actor_rollout_ref.actor.policy_loss.kl_cov_ratio=${kl_cov_ratio} \ + actor_rollout_ref.actor.policy_loss.ppo_kl_coef=${ppo_kl_coef} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.mode=sync \ + actor_rollout_ref.rollout.name=vllm \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${max_token} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.weight_decay=0 \ + actor_rollout_ref.actor.optim.warmup_style=constant \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${train_micro_batch_size} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=${max_token} \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=dapo \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=4 \ + trainer.save_freq=32 \ + trainer.total_epochs=1000 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=disable diff --git a/e2c/verl/recipe/entropy/README.md b/e2c/verl/recipe/entropy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5238cec84bbe9b28fac75e97a100341bfa2e1267 --- /dev/null +++ b/e2c/verl/recipe/entropy/README.md @@ -0,0 +1,110 @@ +
+ +# The Entropy Mechanism of Reinforcement Learning for Large Language Model Reasoning. + +[![Paper](https://img.shields.io/badge/paper-A42C25?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/pdf/2505.22617) [![Github](https://img.shields.io/badge/PRIME-000000?style=for-the-badge&logo=github&logoColor=000&logoColor=white)](https://github.com/PRIME-RL/Entropy-Mechanism-of-RL) [![alphaXiv](https://img.shields.io/badge/discussion-A42C25?style=for-the-badge&logo=arxiv&logoColor=white&color=blue +)](https://www.alphaxiv.org/abs/2505.22617) [![Twitter](https://img.shields.io/badge/Twitter-%23000000.svg?style=for-the-badge&logo=twitter&logoColor=white)](https://x.com/stingning/status/1928088554166505667) [![Twitter](https://img.shields.io/badge/Twitter-%23000000.svg?style=for-the-badge&logo=twitter&logoColor=white)](https://x.com/charlesfornlp/status/1928089451080585283) [![Twitter-ak](https://img.shields.io/badge/Twitter-%23000000.svg?style=for-the-badge&logo=twitter&logoColor=white)](https://x.com/_akhaliq/status/1928077929105268861) + + + + +
+ + +# 🎉News + +- **[2025/05/29]** 🎉 Ranked **#1** of the day on [Huggingface Daily Papers](https://huggingface.co/papers?date=2025-05-29). +- **[2025/05/29]** Released our Paper on arXiv. See [here](https://arxiv.org/pdf/2505.22617). We provide insights into the entropy mechanism of RL for LLMs and propose two simple yet effective strategies to alleviate the entropy collapse. + + + +# ✨Getting started + +After preparing the training data, for training Qwen2.5-7B on a single node, taking the KL-Cov approach as an example, you can simply run: + +``` +cd verl +conda activate your_env +bash recipe/dapo/7b_kl_cov.sh +``` + +While for training Qwen2.5-32B on multi nodes, you can run the following commands: + +``` +cd verl +conda activate your_env +bash recipe/dapo/32b_kl_cov.sh +``` + +# 📖Introduction + +
+ issue +
+ +This paper addresses the entropy collapse issue in scaling reinforcement learning (RL) for large language models (LLMs), where policy entropy drops sharply during training, leading to overconfidence and performance saturation. We empirically establish a relationship between entropy ($H$) and performance ($R$): $R=−aexp(H)+b$, showing performance is bottlenecked by entropy exhaustion. + +
+ issue +
+ +Theoretically, we find entropy changes are driven by the covariance between action probability and logit updates, which correlates with advantage in Policy Gradient methods. High-probability, high-advantage actions reduce entropy, while rare, high-advantage actions increase it. Empirically, the covariance term remains positive, explaining entropy’s monotonic decline. To mitigate this, we propose ​​Clip-Cov​​ and ​​KL-Cov​​, which restrict updates for high-covariance tokens. These methods effectively prevent entropy collapse, and improve performance. + +# 📃Evaluation + +
+ issue +
+ + +Our method is able to maintain a considerably higher level of entropy throughout training. For example, when the baseline's entropy reaches a plateau and can no longer be consumed, the KL-Cov method still sustains an entropy level over 10 times higher. Meanwhile, the response length of the policy model steadily increases, and its performance on the test set consistently surpasses that of the baseline. This indicates that our model is able to explore more freely during training, learning better policy through RL. +| **Method** | **AIME24** | **AIME25** | **AMC** | **MATH-500** | **OMNI-MATH** | **OlympiadBench** | **Minerva** | **Avg.** | +| ----------------- | ---------: | ---------: | -------: | -----------: | ------------: | ----------------: | ----------: | -------: | +| *Qwen2.5-7B* | | | | | | | | | +| GRPO | 21.2 | 9.6 | 58.7 | 78.8 | 27.9 | 40.7 | 36.7 | 38.6 | +| w. Clip-higher | 18.1 | 11.5 | 56.6 | 79.2 | 29.8 | 43.3 | 40.4 | 38.8 | +| w. **`CLIP-Cov`** | 22.1 | **15.8** | 58.2 | 80.4 | **30.5** | **44.1** | **41.1** | 40.4 | +| w. **`KL-Cov`** | **22.6** | 12.9 | **61.4** | **80.8** | 29.1 | 42.6 | 38.2 | **40.6** | +| *Qwen2.5-32B* | | | | | | | | | +| GRPO | 21.8 | 16.2 | 69.7 | 84.2 | 35.2 | 43.6 | 45.5 | 45.8 | +| w. Clip-higher | 35.6 | 22.3 | 69.5 | 77.2 | 35.1 | 42.5 | 43.0 | 47.2 | +| w. **`CLIP-Cov`** | 32.3 | 22.7 | 67.2 | **87.0** | **42.0** | **57.2** | 46.0 | 50.3 | +| w. **`KL-Cov`** | **36.8** | **30.8** | **74.5** | 84.6 | 39.1 | 49.0 | **46.3** | **52.2** | + +Our two approaches both achieve non-trivial improvements across all benchmarks. Compared to GRPO, our method outperforms it by 2.0% on average for the 7B model and by 6.4% for the 32B model. Moreover, we observe that our method yields more substantial gains on the larger Qwen2.5-32B. Specifically, our method achieves improvements of 15.0% and 14.6% compared to GRPO on the most challenging benchmarks, AIME24 and AIME25, respectively. + + +# 🎈Citation +If you find this paper or repo helpful, please cite us. + +```bibtex +@article{cui2025entropy, + title={The Entropy Mechanism of Reinforcement Learning for Reasoning Language Models}, + author={Cui, Ganqu and Zhang, Yuchen and Chen, Jiacheng and Yuan, Lifan and Wang, Zhi and Zuo, Yuxin and Li, Haozhan and Fan, Yuchen and Chen, Huayu and Chen, Weize and others}, + journal={arXiv preprint arXiv:2505.22617}, + year={2025} +} +``` +# 🌻Acknowledgement +We implement our reinforcement learning algorithm extending from [verl](https://github.com/volcengine/verl). We utilize [vLLM](https://github.com/vllm-project/vllm) for inference. Our models are trained primarily on [Qwen2.5 family](https://github.com/QwenLM/Qwen2.5). Our training data is built from [DAPO-MATH](https://huggingface.co/datasets/BytedTsinghua-SIA/DAPO-Math-17k). Thanks for their great contributions! + +# 📬 Contact + +For questions, discussion, or collaboration opportunities, feel free to contact: +- Ganqu Cui: cuiganqu@pjlab.org.cn +- Yuchen Zhang: yuchen.zhang2003@gmail.com +- Jiacheng Chen: jackchan9345@gmail.com +- Ning Ding: ningding.cs@gmail.com + diff --git a/e2c/verl/recipe/entropy/config/entropy_trainer.yaml b/e2c/verl/recipe/entropy/config/entropy_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..969c72946af0989aa592e10e3dbfc1d63bdd084e --- /dev/null +++ b/e2c/verl/recipe/entropy/config/entropy_trainer.yaml @@ -0,0 +1,39 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + gen_batch_size: ${data.train_batch_size} + +reward_model: + reward_kwargs: + overlong_buffer_cfg: ${reward_model.overlong_buffer} + reward_manager: dapo + overlong_buffer: + enable: False + len: 0 + penalty_factor: 0.0 + log: False + +algorithm: + filter_groups: + enable: False # We try to avoid forgetting to set enable + metric: null # acc / score / seq_reward / seq_final_reward / ... + max_num_gen_batches: 0 # Non-positive values mean no upper limit + +trainer: + project_name: verl-entropy + +actor_rollout_ref: + actor: + policy_loss: + loss_mode: "vanilla" # /clip-cov / kl-cov from https://arxiv.org/abs/2505. + clip_cov_ratio: 0.0002 # for clip-cov loss + clip_cov_lb: 1.0 # for clip-cov loss + clip_cov_ub: 5.0 # for clip-cov loss + kl_cov_ratio: 0.0002 # for kl-cov loss + ppo_kl_coef: 0.1 # for kl-cov loss \ No newline at end of file diff --git a/e2c/verl/recipe/entropy/entropy_ray_trainer.py b/e2c/verl/recipe/entropy/entropy_ray_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..0b0b04318c98df4151082cb6576a5c152940f697 --- /dev/null +++ b/e2c/verl/recipe/entropy/entropy_ray_trainer.py @@ -0,0 +1,347 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +FSDP PPO Trainer with Ray-based single controller. +This trainer supports model-agonistic model initialization with huggingface +""" + +import uuid +from collections import defaultdict +from copy import deepcopy +from pprint import pprint + +import numpy as np +import torch +from tqdm import tqdm + +from verl import DataProto +from verl.trainer.ppo.metric_utils import ( + compute_data_metrics, + compute_throughout_metrics, + compute_timing_metrics, + reduce_metrics, +) +from verl.trainer.ppo.ray_trainer import ( + AdvantageEstimator, + RayPPOTrainer, + apply_kl_penalty, + compute_advantage, + compute_response_mask, +) +from verl.utils.profiler import simple_timer + + +class RayEntropyTrainer(RayPPOTrainer): + """ + Note that this trainer runs on the driver process on a single CPU/GPU node. + """ + + def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the worker group through RPC + to construct the PPO dataflow. + The light-weight advantage computation is done on the driver process. + """ + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + self.global_steps = 0 + + # load checkpoint before doing anything + self._load_checkpoint() + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + val_metrics = self._validate() + assert val_metrics, f"{val_metrics=}" + pprint(f"Initial validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if self.config.trainer.get("val_only", False): + return + + # add tqdm + progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress") + + # we start from step 1 + self.global_steps += 1 + last_val_metrics = None + + timing_raw = defaultdict(float) + batch = None + num_prompt_in_batch = 0 + num_gen_batches = 0 + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + metrics = {} + + new_batch: DataProto = DataProto.from_single_dict(batch_dict) + num_gen_batches += 1 + # pop those keys for generation + if "multi_modal_inputs" in new_batch.non_tensor_batch.keys(): + gen_batch = new_batch.pop( + batch_keys=["input_ids", "attention_mask", "position_ids"], + non_tensor_batch_keys=["raw_prompt_ids", "multi_modal_data", "multi_modal_inputs"], + ) + else: + gen_batch = new_batch.pop( + batch_keys=["input_ids", "attention_mask", "position_ids"], + non_tensor_batch_keys=["raw_prompt_ids"], + ) + gen_batch = gen_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + + is_last_step = self.global_steps >= self.total_training_steps + + with simple_timer("step", timing_raw): + # generate a batch + # with simple_timer("gen", timing_raw): + # gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + with simple_timer("gen", timing_raw): + if not self.async_rollout_mode: + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + else: + gen_batch_output = self.async_rollout_manager.generate_sequences(gen_batch) + + if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: + with simple_timer("gen_max", timing_raw): + gen_baseline_batch = deepcopy(gen_batch) + gen_baseline_batch.meta_info["do_sample"] = False + gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) + + new_batch = new_batch.union(gen_baseline_output) + reward_baseline_tensor = self.reward_fn(new_batch) + reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) + + new_batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) + + new_batch.batch["reward_baselines"] = reward_baseline_tensor + + del gen_baseline_batch, gen_baseline_output + + new_batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(new_batch.batch))], dtype=object + ) + # repeat to align with repeated responses in rollout + new_batch = new_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + new_batch = new_batch.union(gen_batch_output) + + with simple_timer("reward", timing_raw): + # compute scores. Support both model and function-based. + # We first compute the scores using reward model. Then, we call reward_fn to combine + # the results from reward model and rule-based results. + if self.use_rm: + # we first compute reward model score + reward_tensor = self.rm_wg.compute_rm_score(new_batch) + new_batch = new_batch.union(reward_tensor) + + # we combine with rule-based rm + reward_extra_infos_dict: dict[str, list] + try: + reward_result = self.reward_fn(new_batch, return_dict=True) + reward_tensor = reward_result["reward_tensor"] + reward_extra_infos_dict = reward_result["reward_extra_info"] + except Exception as e: + print(f"Error in reward_fn: {e}") + reward_tensor = self.reward_fn(new_batch) + reward_extra_infos_dict = {} + + new_batch.batch["token_level_scores"] = reward_tensor + + print(f"{list(reward_extra_infos_dict.keys())=}") + if reward_extra_infos_dict: + new_batch.non_tensor_batch.update( + {k: np.array(v) for k, v in reward_extra_infos_dict.items()} + ) + + # compute rewards. apply_kl_penalty if available + if self.config.algorithm.use_kl_in_reward: + new_batch, kl_metrics = apply_kl_penalty( + new_batch, kl_ctrl=self.kl_ctrl_in_reward, kl_penalty=self.config.algorithm.kl_penalty + ) + metrics.update( + kl_metrics + ) # TODO: This will be cleared if we use multiple genenration batches + else: + new_batch.batch["token_level_rewards"] = new_batch.batch["token_level_scores"] + + if not self.config.algorithm.filter_groups.enable: + batch = new_batch + else: # NOTE: When prompts after filtering is less than train batch size, + # we skip to the next generation batch + metric_name = self.config.algorithm.filter_groups.metric + if metric_name == "seq_final_reward": + # Turn to numpy for easier filtering + new_batch.non_tensor_batch["seq_final_reward"] = ( + new_batch.batch["token_level_rewards"].sum(dim=-1).numpy() + ) + elif metric_name == "seq_reward": + new_batch.non_tensor_batch["seq_reward"] = ( + new_batch.batch["token_level_scores"].sum(dim=-1).numpy() + ) + + # Collect the sequence reward for each trajectory + prompt_uid2metric_vals = defaultdict(list) + for uid, metric_val in zip( + new_batch.non_tensor_batch["uid"], new_batch.non_tensor_batch[metric_name], strict=True + ): + prompt_uid2metric_vals[uid].append(metric_val) + + prompt_uid2metric_std = {} + for prompt_uid, metric_vals in prompt_uid2metric_vals.items(): + prompt_uid2metric_std[prompt_uid] = np.std(metric_vals) + + kept_prompt_uids = [ + uid + for uid, std in prompt_uid2metric_std.items() + if std > 0 or len(prompt_uid2metric_vals[uid]) == 1 + ] + num_prompt_in_batch += len(kept_prompt_uids) + + kept_traj_idxs = [] + for idx, traj_from_prompt_uid in enumerate(new_batch.non_tensor_batch["uid"]): + if traj_from_prompt_uid in kept_prompt_uids: + kept_traj_idxs.append(idx) + + new_batch = new_batch[kept_traj_idxs] + batch = new_batch if batch is None else DataProto.concat([batch, new_batch]) + + prompt_bsz = self.config.data.train_batch_size + if num_prompt_in_batch < prompt_bsz: + print(f"{num_prompt_in_batch=} < {prompt_bsz=}") + max_num_gen_batches = self.config.algorithm.filter_groups.max_num_gen_batches + if max_num_gen_batches <= 0 or num_gen_batches < max_num_gen_batches: + print(f"{num_gen_batches=}. Keep generating...") + continue + else: + raise ValueError( + f"{num_gen_batches=} >= {max_num_gen_batches=}." + + " Generated too many. Please check if your data are too difficult." + + " You could also try set max_num_gen_batches=0 to enable endless trials." + ) + else: + # Align the batch + traj_bsz = self.config.data.train_batch_size * self.config.actor_rollout_ref.rollout.n + print( + f"Collected {num_prompt_in_batch} / {self.config.data.train_batch_size} prompt. " + f"Collecting finished." + ) + batch = batch[:traj_bsz] + + # === Updating === + + batch.batch["response_mask"] = compute_response_mask(batch) + + # balance the number of valid tokens on each dp rank. + # Note that this breaks the order of data inside the batch. + # Please take care when you implement group based adv computation such as GRPO and rloo + if self.config.trainer.balance_batch: + self._balance_batch(batch, metrics=metrics) + + # compute global_valid tokens + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + # recompute old_log_probs + with simple_timer("old_log_prob", timing_raw): + old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) + batch = batch.union(old_log_prob) + + if self.use_reference_policy: + # compute reference log_prob + with simple_timer("ref", timing_raw): + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + + # compute values + if self.use_critic: + with simple_timer("values", timing_raw): + values = self.critic_wg.compute_values(batch) + batch = batch.union(values) + + with simple_timer("adv", timing_raw): + # compute advantages, executed on the driver process + norm_adv_by_std_in_grpo = self.config.algorithm.get("norm_adv_by_std_in_grpo", True) + batch = compute_advantage( + batch, + adv_estimator=self.config.algorithm.adv_estimator, + gamma=self.config.algorithm.gamma, + lam=self.config.algorithm.lam, + num_repeat=self.config.actor_rollout_ref.rollout.n, + norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo, + ) + + # update critic + if self.use_critic: + with simple_timer("update_critic", timing_raw): + critic_output = self.critic_wg.update_critic(batch) + critic_output_metrics = reduce_metrics(critic_output.meta_info["metrics"]) + metrics.update(critic_output_metrics) + + # implement critic warmup + if self.config.trainer.critic_warmup <= self.global_steps: + # update actor + with simple_timer("update_actor", timing_raw): + actor_output = self.actor_rollout_wg.update_actor(batch) + actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"]) + metrics.update(actor_output_metrics) + + # validate + if ( + self.val_reward_fn is not None + and self.config.trainer.test_freq > 0 + and (is_last_step or self.global_steps % self.config.trainer.test_freq == 0) + ): + with simple_timer("testing", timing_raw): + val_metrics: dict = self._validate() + if is_last_step: + last_val_metrics = val_metrics + metrics.update(val_metrics) + + if self.config.trainer.save_freq > 0 and ( + is_last_step or self.global_steps % self.config.trainer.save_freq == 0 + ): + with simple_timer("save_checkpoint", timing_raw): + self._save_checkpoint() + + # collect metrics + metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) + metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) + # TODO: implement actual tflpo and theoretical tflpo + n_gpus = self.resource_pool_manager.get_n_gpus() + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) + timing_raw = defaultdict(float) # clear timing + + metrics["train/num_gen_batches"] = num_gen_batches + batch = None + num_prompt_in_batch = 0 + num_gen_batches = 0 + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=self.global_steps) + + if is_last_step: + pprint(f"Final validation metrics: {last_val_metrics}") + progress_bar.close() + return + + progress_bar.update(1) + self.global_steps += 1 diff --git a/e2c/verl/recipe/entropy/main_entropy.py b/e2c/verl/recipe/entropy/main_entropy.py new file mode 100644 index 0000000000000000000000000000000000000000..f662e89f2bad722c1140f922a9dbc5844c47cf03 --- /dev/null +++ b/e2c/verl/recipe/entropy/main_entropy.py @@ -0,0 +1,249 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. +""" + +import hydra +import ray +from omegaconf import OmegaConf + +from .entropy_ray_trainer import RayEntropyTrainer +from .reward import load_reward_manager + + +@hydra.main(config_path="config", config_name="entropy_trainer", version_base=None) +def main(config): + run_ppo(config) + + +def run_ppo(config) -> None: + if not ray.is_initialized(): + # this is for local ray cluster + default_runtime_env = { + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "WARN", + "WANDB_API_KEY": "YOUR_WANDB_API_KEY", + } + } + ray_init_kwargs = config.ray_kwargs.get("ray_init", {}) + runtime_env_kwargs = ray_init_kwargs.get("runtime_env", {}) + runtime_env = OmegaConf.merge(default_runtime_env, runtime_env_kwargs) + ray_init_kwargs = OmegaConf.create({**ray_init_kwargs, "runtime_env": runtime_env}) + print(f"ray init kwargs: {ray_init_kwargs}") + ray.init(**OmegaConf.to_container(ray_init_kwargs)) + + runner = TaskRunner.remote() + ray.get(runner.run.remote(config)) + + +def merge_dict(a: dict, b: dict) -> dict: + """Return a new dict that has `a` updated with `b` (b wins on conflicts). + + Example:: + + >>> d1 = {"x": 1, "y": 2} + >>> d2 = {"y": 20, "z": 3} + >>> new_dict = merge_dict(d1, d2) + >>> print(new_dict) # {'x': 1, 'y': 20, 'z': 3} + >>> print(d1) # {"x": 1, "y": 2} (unchanged) + >>> print(d2) # {"y": 20, "z": 3} (unchanged) + """ + return a | b + + +@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head +class TaskRunner: + def run(self, config): + # print initial config + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_to_local + + pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values + OmegaConf.resolve(config) + + # download the checkpoint from hdfs + local_path = copy_to_local(config.actor_rollout_ref.model.path) + print(f"{config.actor_rollout_ref.model.path}") + # instantiate tokenizer + from verl.utils import hf_processor, hf_tokenizer + + trust_remote_code = config.data.get("trust_remote_code", False) + tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) + processor = hf_processor(local_path, use_fast=True) # used for multimodal LLM, could be none + + # define worker classes + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + assert config.critic.strategy in {"fsdp", "fsdp2"} + from verl.single_controller.ray import RayWorkerGroup + from verl.workers.fsdp_workers import ActorRolloutRefWorker, AsyncActorRolloutRefWorker, CriticWorker + + actor_rollout_cls = ( + AsyncActorRolloutRefWorker + if config.actor_rollout_ref.rollout.mode == "async" + else ActorRolloutRefWorker + ) + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.single_controller.ray import RayWorkerGroup + from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker + + actor_rollout_cls = ActorRolloutRefWorker + ray_worker_group_cls = RayWorkerGroup + + else: + raise NotImplementedError + + from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role + + role_worker_mapping = { + Role.ActorRollout: ray.remote(actor_rollout_cls), + Role.Critic: ray.remote(CriticWorker), + } + + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + Role.Critic: global_pool_id, + } + + # we should adopt a multi-source reward function here + # - for rule-based rm, we directly call a reward score + # - for model-based rm, we call a model + # - for code related prompt, we send to a sandbox if there are test cases + # - finally, we combine all the rewards together + # - The reward type depends on the tag of the data + if config.reward_model.enable: + if config.reward_model.strategy in {"fsdp", "fsdp2"}: + from verl.workers.fsdp_workers import RewardModelWorker + elif config.reward_model.strategy == "megatron": + from verl.workers.megatron_workers import RewardModelWorker + else: + raise NotImplementedError + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + mapping[Role.RewardModel] = global_pool_id + + # use reference model + if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) + mapping[Role.RefPolicy] = global_pool_id + + reward_kwargs = { + "max_resp_len": config.data.max_response_length, + "overlong_buffer_cfg": config.reward_model.overlong_buffer, + } + cfg_reward_kwargs = config.reward_model.get("reward_kwargs", {}) + reward_fn = load_reward_manager( + config, tokenizer, num_examine=0, **OmegaConf.merge(OmegaConf.create(reward_kwargs), cfg_reward_kwargs) + ) + val_reward_fn = load_reward_manager(config, tokenizer, num_examine=1, **reward_kwargs) + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + + from verl.utils.dataset.rl_dataset import collate_fn + + train_dataset = create_rl_dataset(config.data.train_files, config.data, tokenizer, processor) + val_dataset = create_rl_dataset(config.data.val_files, config.data, tokenizer, processor) + train_sampler = create_rl_sampler(config.data, train_dataset) + trainer = RayEntropyTrainer( + config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + train_dataset=train_dataset, + val_dataset=val_dataset, + collate_fn=collate_fn, + train_sampler=train_sampler, + ) + trainer.init_workers() + trainer.fit() + + +def create_rl_dataset(data_paths, data_config, tokenizer, processor): + """Create a dataset. + + Arguments: + data_config: The data config. + tokenizer (Tokenizer): The tokenizer. + processor (Processor): The processor. + + Returns: + dataset (Dataset): The dataset. + """ + from torch.utils.data import Dataset + + from verl.utils.dataset.rl_dataset import RLHFDataset + + if "custom_cls" in data_config and data_config.custom_cls.get("path", None) is not None: + from verl.utils.import_utils import load_extern_type + + dataset_cls = load_extern_type(data_config.custom_cls.path, data_config.custom_cls.name) + if not issubclass(dataset_cls, Dataset): + raise TypeError( + f"The custom dataset class '{data_config.custom_cls.name}' from '{data_config.custom_cls.path}' " + f"must inherit from torch.utils.data.Dataset" + ) + else: + dataset_cls = RLHFDataset + print(f"Using dataset class: {dataset_cls.__name__}") + + dataset = dataset_cls( + data_files=data_paths, + tokenizer=tokenizer, + processor=processor, + config=data_config, + ) + + return dataset + + +def create_rl_sampler(data_config, dataset): + """Create a sampler for the dataset. + + Arguments: + data_config: The data config. + dataset (Dataset): The dataset. + + Returns: + sampler (Sampler): The sampler. + """ + import torch + from torch.utils.data import RandomSampler, SequentialSampler + + # use sampler for better ckpt resume + if data_config.shuffle: + train_dataloader_generator = torch.Generator() + train_dataloader_generator.manual_seed(data_config.get("seed", 1)) + sampler = RandomSampler(data_source=dataset, generator=train_dataloader_generator) + else: + sampler = SequentialSampler(data_source=dataset) + + return sampler + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/recipe/entropy/reward.py b/e2c/verl/recipe/entropy/reward.py new file mode 100644 index 0000000000000000000000000000000000000000..36b8b65a4d2d7aa2e5977a1214e3ef5c4f9e4b4a --- /dev/null +++ b/e2c/verl/recipe/entropy/reward.py @@ -0,0 +1,86 @@ +# Copyright 2025 Individual Contributor: Thibaut Barroyer +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing +from functools import partial + +import ray + +from verl import DataProto +from verl.trainer.ppo.reward import compute_reward, get_custom_reward_fn + +from .reward_score import _default_compute_score + + +def load_reward_manager(config, tokenizer, num_examine, **reward_kwargs): + """ + Load and initialize a reward manager based on the configuration. + + Args: + config: PPO trainer configuration object containing reward_model fields. + tokenizer: Tokenizer object used for processing text. + num_examine: Number of samples to examine. + **reward_kwargs: Additional keyword arguments for the reward manager. + + Returns: + An instance of the specified reward manager class. + """ + from verl.workers.reward_manager import get_reward_manager_cls + + # The list of pre-defined reward managers are defined in `verl/workers/reward_manager/`: + # naive: NaiveRewardManager + # prime: PrimeRewardManager + # batch: BatchRewardManager + # dapo: DAPORewardManager + # Note(haibin.lin): For custom reward managers, please make sure they are imported and + # registered via `verl.workers.reward_manager.register` + # By default reward_manager is set to naive (NaiveRewardManager) + reward_manager_name = config.reward_model.get("reward_manager", "naive") + reward_manager_cls = get_reward_manager_cls(reward_manager_name) + + # Try to get a custom reward function based on the configuration + compute_score = get_custom_reward_fn(config) + final_compute_score = compute_score + + if compute_score is None: + sandbox_config = config.reward_model.get("sandbox_fusion") + sandbox_url = sandbox_config.get("url") if sandbox_config else None + if sandbox_url: + sandbox_manager = multiprocessing.Manager() + # Create a semaphore to control concurrent access to the sandbox + _concurrent_semaphore = sandbox_manager.Semaphore(sandbox_config.get("max_concurrent", 64)) + final_compute_score = partial( + _default_compute_score, sandbox_fusion_url=sandbox_url, concurrent_semaphore=_concurrent_semaphore + ) + else: + final_compute_score = _default_compute_score + + # Instantiate and return the reward manager with the specified parameters + return reward_manager_cls( + tokenizer=tokenizer, + num_examine=num_examine, + compute_score=final_compute_score, + reward_fn_key=config.data.reward_fn_key, + **reward_kwargs, + ) + + +@ray.remote(num_cpus=1) +def compute_reward_async(data: DataProto, config, tokenizer): + """ + Load the reward manager and compute the reward for a batch of data. + This is meant to be run in a separate Ray worker. + """ + reward_fn = load_reward_manager(config, tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {})) + return compute_reward(data, reward_fn) diff --git a/e2c/verl/recipe/entropy/reward_score/__init__.py b/e2c/verl/recipe/entropy/reward_score/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7224bf3c37113dea3ea9d75b20567078ab0b3501 --- /dev/null +++ b/e2c/verl/recipe/entropy/reward_score/__init__.py @@ -0,0 +1,38 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from . import gsm8k, math, prime_math, prime_code + +import traceback + +from . import entropy_math + + +def _default_compute_score( + data_source, solution_str, ground_truth, extra_info=None, sandbox_fusion_url=None, concurrent_semaphore=None +): + try: + res = entropy_math.compute_score(solution_str, str(ground_truth)) + # print(f"data_source: {data_source}") + # raise NotImplementedError(f"Reward function is not implemented for {data_source=}") + + if isinstance(res, dict): + return res + elif isinstance(res, int | float | bool): + return float(res) + else: + return float(res[0]) + except Exception as e: + print(f"[ERROR] Error in process_completion for task : {str(e)}") + traceback.print_exc() # 打印完整堆栈 + raise # 重新抛出异常以便上层捕获 diff --git a/e2c/verl/recipe/entropy/reward_score/entropy_math/__init__.py b/e2c/verl/recipe/entropy/reward_score/entropy_math/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1b2ba647d4056e1c203a2abf4f7dd17db0713911 --- /dev/null +++ b/e2c/verl/recipe/entropy/reward_score/entropy_math/__init__.py @@ -0,0 +1,1062 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except Exception in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides a math answer grading function with high recall. +Based on HF math_verify, verl, open reasoner zero, etc. +""" + +import os +import re +import signal +from itertools import islice, zip_longest +from math import isclose +from typing import Optional + +import sympy +from latex2sympy2_extended import latex2sympy +from math_verify import ExprExtractionConfig, LatexExtractionConfig, parse, verify +from pylatexenc import latex2text +from sympy import N, simplify +from sympy.parsing import sympy_parser +from sympy.parsing.latex import parse_latex +from sympy.parsing.sympy_parser import parse_expr + +""" +This code is adapted from: Dr. GRPO (https://github.com/sail-sg/understand-r1-zero/blob/main/understand_r1_zero/math_grader.py). +""" + + +def timeout_ours(timeout_seconds: int = 8): + if os.name == "posix": + import signal + + def decorator(func): + def handler(signum, frame): + raise TimeoutError("Operation timed out!") + + def wrapper(*args, **kwargs): + old_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, handler) + signal.alarm(timeout_seconds) + + try: + return func(*args, **kwargs) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + return wrapper + + return decorator + else: + raise NotImplementedError(f"Unsupported OS: {os.name}") + + +# Dan Hendrycks' code +def mathd_normalize_answer(answer: Optional[str]) -> Optional[str]: + if answer is None: + return None + answer = answer.strip() + try: + # Remove enclosing `\text{}`. + m = re.search("^\\\\text\{(?P.+?)\}$", answer) + if m is not None: + answer = m.group("text").strip() + return _strip_string(answer) + except Exception: + return answer + + +# units mainly from MathQA +unit_texts = [ + "east", + "degree", + "mph", + "kmph", + "ft", + "m sqaure", + " m east", + "sq m", + "deg", + "mile", + "q .", + "monkey", + "prime", + "ratio", + "profit of rs", + "rd", + "o", + "gm", + "p . m", + "lb", + "tile", + "per", + "dm", + "lt", + "gain", + "ab", + "way", + "west", + "a .", + "b .", + "c .", + "d .", + "e .", + "f .", + "g .", + "h .", + "t", + "a", + "h", + "no change", + "men", + "soldier", + "pie", + "bc", + "excess", + "st", + "inches", + "noon", + "percent", + "by", + "gal", + "kmh", + "c", + "acre", + "rise", + "a . m", + "th", + "π r 2", + "sq", + "mark", + "l", + "toy", + "coin", + "sq . m", + "gallon", + "° f", + "profit", + "minw", + "yr", + "women", + "feet", + "am", + "pm", + "hr", + "cu cm", + "square", + "v â € ™", + "are", + "rupee", + "rounds", + "cubic", + "cc", + "mtr", + "s", + "ohm", + "number", + "kmph", + "day", + "hour", + "minute", + "min", + "second", + "man", + "woman", + "sec", + "cube", + "mt", + "sq inch", + "mp", + "∏ cm ³", + "hectare", + "more", + "sec", + "unit", + "cu . m", + "cm 2", + "rs .", + "rs", + "kg", + "g", + "month", + "km", + "m", + "cm", + "mm", + "apple", + "liter", + "loss", + "yard", + "pure", + "year", + "increase", + "decrease", + "d", + "less", + "Surface", + "litre", + "pi sq m", + "s .", + "metre", + "meter", + "inch", +] + +unit_texts.extend([t + "s" for t in unit_texts]) + + +def _strip_string(string): + def _fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except Exception: + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + + def _fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + a = int(a) + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except Exception: + return string + + def _remove_right_units(string): + # "\\text{ " only ever occurs (at least in the val set) when describing units + if "\\text{ " in string: + splits = string.split("\\text{ ") + assert len(splits) == 2 + return splits[0] + else: + return string + + def _fix_sqrt(string): + if "\\sqrt" not in string: + return string + splits = string.split("\\sqrt") + new_string = splits[0] + for split in splits[1:]: + if split[0] != "{": + a = split[0] + new_substr = "\\sqrt{" + a + "}" + split[1:] + else: + new_substr = "\\sqrt" + split + new_string += new_substr + return new_string + + # linebreaks + string = string.replace("\n", "") + # print(string) + + # remove inverse spaces + string = string.replace("\\!", "") + # print(string) + + # replace \\ with \ + string = string.replace("\\\\", "\\") + # print(string) + + # matrix + string = re.sub(r"\\begin\{array\}\{.*?\}", r"\\begin{pmatrix}", string) + string = re.sub(r"\\end\{array\}", r"\\end{pmatrix}", string) + string = string.replace("bmatrix", "pmatrix") + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + string = string.replace("\\neq", "\\ne").replace("\\leq", "\\le").replace("\\geq", "\\ge") + # print(string) + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + # print(string) + + # Remove unit: miles, dollars if after is not none + _string = re.sub(r"\\text{.*?}$", "", string).strip() + if _string != "" and _string != string: + # print("Warning: unit not removed: '{}' -> '{}'".format(string, _string)) + string = _string + + # Remove unit: texts + for _ in range(2): + for unit_text in unit_texts: + # use regex, the prefix should be either the start of the string or a non-alphanumeric character + # the suffix should be either the end of the string or a non-alphanumeric character + _string = re.sub(r"(^|\W)" + unit_text + r"($|\W)", r"\1\2", string) + if _string != "": + string = _string + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "") + string = string.replace("^\\circ", "") + + # remove dollar signs + string = string.replace("\\$", "") + + # remove units (on the right) + string = _remove_right_units(string) + + # remove percentage + string = string.replace("\\%", "") + string = string.replace("\%", "") + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + if len(string.split("=")) == 2: + if len(string.split("=")[0]) <= 2: + string = string.split("=")[1] + + # fix sqrt3 --> sqrt{3} + string = _fix_sqrt(string) + + # remove spaces + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). + # Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # manually change 0.5 --> \frac{1}{2} + if string == "0.5": + string = "\\frac{1}{2}" + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + return string + + +SUBSTITUTIONS = [ + ("an ", ""), + ("a ", ""), + (".$", "$"), + ("\\$", ""), + (r"\ ", ""), + (" ", ""), + ("mbox", "text"), + (",\\text{and}", ","), + ("\\text{and}", ","), + ("\\text{m}", "\\text{}"), +] + + +REMOVED_EXPRESSIONS = [ + "square", + "ways", + "integers", + "dollars", + "mph", + "inches", + "ft", + "hours", + "km", + "units", + "\\ldots", + "sue", + "points", + "feet", + "minutes", + "digits", + "cents", + "degrees", + "cm", + "gm", + "pounds", + "meters", + "meals", + "edges", + "students", + "childrentickets", + "multiples", + "\\text{s}", + "\\text{.}", + "\\text{\ns}", + "\\text{}^2", + "\\text{}^3", + "\\text{\n}", + "\\text{}", + r"\mathrm{th}", + r"^\circ", + r"^{\circ}", + r"\;", + r",\!", + "{,}", + '"', + "\\dots", +] + + +def normalize_final_answer(final_answer: str) -> str: + """ + Normalize a final answer to a quantitative reasoning question. + This code comes from https://arxiv.org/pdf/2206.14858.pdf, page18. + """ + # final_answer = final_answer.split("=")[-1] + + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, "") + + # Extract answer that is in LaTeX math, is bold, + # is surrounded by a box, etc. + final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) + final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) + + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) + final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) + final_answer = final_answer.replace("$", "") + + # Normalize 100,000 -> 100000 + if final_answer.replace(",", "").isdigit(): + final_answer = final_answer.replace(",", "") + + return final_answer + + +def repeatness(s: str): + def ranks(seq): + index = {v: i for i, v in enumerate(sorted(set(seq)))} + return [index[v] for v in seq] + + def suffixArray(s): + line = ranks(s) + n, k, ans, sa = len(s), 1, line, [0] * len(s) + while k < n - 1: + line = ranks(list(zip_longest(line, islice(line, k, None), fillvalue=-1))) + ans, k = line, k << 1 + for i, k in enumerate(ans): + sa[k] = i + return ans, sa + + def lcp(arr, suffixArr, inv_suff): + n, ans, k = len(arr), [0] * len(arr), 0 + + for i in range(n): + if inv_suff[i] == n - 1: + k = 0 + continue + + j = suffixArr[inv_suff[i] + 1] + while i + k < n and j + k < n and arr[i + k] == arr[j + k]: + k += 1 + + ans[inv_suff[i]] = k + if k > 0: + k -= 1 + + return ans + + arr = [ord(i) for i in s] + n = len(arr) + if n <= 1: + return 0 + c, sa = suffixArray(arr) + cnt = sum(lcp(arr, sa, c)) + + return (cnt * 2 / (n * (n + 1))) > 0.2 + + +class timeout: + def __init__(self, seconds=1, error_message="Timeout"): + self.seconds = seconds + self.error_message = error_message + + def handle_timeout(self, signum, frame): + raise TimeoutError(self.error_message) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + signal.alarm(0) + + +def latex_eval(latex): + sym = parse_latex(latex) + val = sym.evalf() + return sym, val + + +def numeric_equal(prediction: float, reference: float): + # Note that relative tolerance has significant impact + # on the result of the synthesized GSM-Hard dataset + # if reference.is_integer(): + # return isclose(reference, round(prediction), abs_tol=1e-4) + # else: + # prediction = round(prediction, len(str(reference).split(".")[-1])) + return isclose(reference, prediction, rel_tol=1e-4) + + +@timeout_ours(timeout_seconds=5) +def symbolic_equal(a, b): + def _parse(s): + for f in [parse_latex, parse_expr, latex2sympy]: + try: + return f(s.replace("\\\\", "\\")) + except Exception: + try: + return f(s) + except Exception: + pass + return s + + a = _parse(a) + b = _parse(b) + + # direct equal + try: + if str(a) == str(b) or a == b: + return True + except Exception: + pass + + # simplify equal + try: + if a.equals(b) or simplify(a - b) == 0: + return True + except Exception: + pass + + # equation equal + try: + if (abs(a.lhs - a.rhs)).equals(abs(b.lhs - b.rhs)): + return True + except Exception: + pass + + try: + if numeric_equal(float(N(a)), float(N(b))): + return True + except Exception: + pass + + # matrix + try: + # if a and b are matrix + if a.shape == b.shape: + _a = a.applyfunc(lambda x: round(x, 3)) + _b = b.applyfunc(lambda x: round(x, 3)) + if _a.equals(_b): + return True + except Exception: + pass + + return False + + +def _is_latex_equal(str1, str2): + try: + sym1, val1 = latex_eval(str1) + sym2, val2 = latex_eval(str2) + if sym1 == sym2 or val1 == val2: + return True + else: + raise ValueError + except Exception: # noqa + try: + norm1, norm2 = normalize_final_answer(str1), normalize_final_answer(str2) + sym1, val1 = latex_eval(norm1) + sym2, val2 = latex_eval(norm2) + if sym1 == sym2 or val1 == val2: + return True + except Exception: # noqa + return norm1 == norm2 + return False + + +def is_latex_equal(given_answer: str, ground_truth: str) -> bool: + try: + with timeout(1): + try: + if (len(given_answer) > 128 and repeatness(given_answer)) or ( + len(ground_truth) > 128 and repeatness(ground_truth) + ): + return False + # First conduct normalized string matching. + ground_truth_normalized = _normalize(ground_truth) + given_normalized = _normalize(given_answer) + if ground_truth_normalized is None: + return False + if ground_truth_normalized == given_normalized: + return True + + # Next call math verify. + given_answer.replace("\n", "") + ground_truth.replace("\n", "") + if "$" not in given_answer: + given_answer = f"${given_answer}$" + if "$" not in ground_truth: + ground_truth = f"${ground_truth}$" + return verify( + parse( + ground_truth, + extraction_config=( + LatexExtractionConfig(boxed_match_priority=0), + ExprExtractionConfig(), + ), + fallback_mode="no_fallback", + extraction_mode=["first_match"], + parsing_timeout=1, + ), + parse( + given_answer, + extraction_config=( + LatexExtractionConfig(boxed_match_priority=0), + ExprExtractionConfig(), + ), + fallback_mode="no_fallback", + extraction_mode=["first_match"], + parsing_timeout=1, + ), + timeout_seconds=1, + ) + # or symbolic_equal(ground_truth, given_answer) + except Exception: + return False + except TimeoutError: + return False + + +def is_value_equal(given_answer: str, ground_truth: str) -> bool: + assert ground_truth is not None + ground_truth_normalized_mathd = mathd_normalize_answer(ground_truth) + given_answer_normalized_mathd = mathd_normalize_answer(given_answer) + + str_equal = ground_truth_normalized_mathd == given_answer_normalized_mathd + try: + number_equal = float(ground_truth_normalized_mathd) == float(given_answer_normalized_mathd) + return str_equal or number_equal + except Exception: + return str_equal + + +# sympy might hang -- we don't care about trying to be lenient in these cases +BAD_SUBSTRINGS = ["^{", "^("] +BAD_REGEXES = ["\^[0-9]+\^", "\^[0-9][0-9]+"] +TUPLE_CHARS = "()[]" + + +def _sympy_parse(expr: str): + """Parses an expression with sympy.""" + py_expr = expr.replace("^", "**") + return sympy_parser.parse_expr( + py_expr, + transformations=(sympy_parser.standard_transformations + (sympy_parser.implicit_multiplication_application,)), + ) + + +def _parse_latex(expr: str) -> str: + """Attempts to parse latex to an expression sympy can read.""" + expr = expr.replace("\\tfrac", "\\frac") + expr = expr.replace("\\dfrac", "\\frac") + expr = expr.replace("\\frac", " \\frac") # Play nice with mixed numbers. + expr = latex2text.LatexNodes2Text().latex_to_text(expr) + + # Replace the specific characters that this parser uses. + expr = expr.replace("√", "sqrt") + expr = expr.replace("π", "pi") + expr = expr.replace("∞", "inf") + expr = expr.replace("∪", "U") + expr = expr.replace("·", "*") + expr = expr.replace("×", "*") + + return expr.strip() + + +def _is_float(num: str) -> bool: + try: + float(num) + return True + except ValueError: + return False + + +def _is_int(x: float) -> bool: + try: + return abs(x - int(round(x))) <= 1e-7 + except Exception: + return False + + +def _is_frac(expr: str) -> bool: + return bool(re.search(r"^-?[0-9]+.?/0*[1-9][0-9]*.?$", expr)) + + +def _str_is_int(x: str) -> bool: + try: + x = _strip_properly_formatted_commas(x) + x = float(x) + return abs(x - int(round(x))) <= 1e-7 + except Exception: + return False + + +def _str_to_int(x: str) -> bool: + x = x.replace(",", "") + x = float(x) + return int(x) + + +def _inject_implicit_mixed_number(step: str): + """ + Automatically make a mixed number evalable + e.g. 7 3/4 => 7+3/4 + """ + p1 = re.compile("([0-9]) +([0-9])") + step = p1.sub("\\1+\\2", step) ## implicit mults + return step + + +def _strip_properly_formatted_commas(expr: str): + # We want to be careful because we don't want to strip tuple commas + p1 = re.compile("(\d)(,)(\d\d\d)($|\D)") + while True: + next_expr = p1.sub("\\1\\3\\4", expr) + if next_expr == expr: + break + expr = next_expr + return next_expr + + +def _normalize(expr: str) -> str: + """Normalize answer expressions.""" + if expr is None: + return None + + # Remove enclosing `\text{}`. + m = re.search("^\\\\text\{(?P.+?)\}$", expr) + if m is not None: + expr = m.group("text") + + expr = expr.replace("\\%", "%") + expr = expr.replace("\\$", "$") + expr = expr.replace("$", "") + expr = expr.replace("%", "") + expr = expr.replace(" or ", " , ") + expr = expr.replace(" and ", " , ") + + expr = expr.replace("million", "*10^6") + expr = expr.replace("billion", "*10^9") + expr = expr.replace("trillion", "*10^12") + + for unit in [ + "degree", + "cm", + "centimeter", + "meter", + "mile", + "second", + "minute", + "hour", + "day", + "week", + "month", + "year", + "foot", + "feet", + "inch", + "yard", + ]: + expr = re.sub(f"{unit}(es)?(s)? *(\^[0-9]+)?", "", expr) + expr = re.sub("\^ *\\\\circ", "", expr) + + if len(expr) > 0 and expr[0] == "{" and expr[-1] == "}": + expr = expr[1:-1] + + expr = re.sub(",\\\\! *", "", expr) + if _is_float(expr) and _is_int(float(expr)): + expr = str(int(round(float(expr)))) + if "\\" in expr: + try: + expr = _parse_latex(expr) + except Exception: + pass + + # edge case with mixed numbers and negative signs + expr = re.sub("- *", "-", expr) + + expr = _inject_implicit_mixed_number(expr) + expr = expr.replace(" ", "") + + # if we somehow still have latex braces here, just drop them + expr = expr.replace("{", "") + expr = expr.replace("}", "") + + # don't be case sensitive for text answers + expr = expr.lower() + + if _str_is_int(expr): + expr = str(_str_to_int(expr)) + + return expr + + +def count_unknown_letters_in_expr(expr: str): + expr = expr.replace("sqrt", "") + expr = expr.replace("frac", "") + letters_in_expr = set([x for x in expr if x.isalpha()]) + return len(letters_in_expr) + + +def should_allow_eval(expr: str): + # we don't want to try parsing unknown text or functions of more than two variables + if count_unknown_letters_in_expr(expr) > 2: + return False + + for bad_string in BAD_SUBSTRINGS: + if bad_string in expr: + return False + + for bad_regex in BAD_REGEXES: + if re.search(bad_regex, expr) is not None: + return False + + return True + + +@timeout_ours(timeout_seconds=5) +def are_equal_under_sympy(ground_truth_normalized: str, given_normalized: str): + are_equal = False + try: + expr = f"({ground_truth_normalized})-({given_normalized})" + if should_allow_eval(expr): + sympy_diff = _sympy_parse(expr) + simplified = sympy.simplify(sympy_diff) + if simplified == 0: + are_equal = True + except Exception: + pass + return are_equal + + +def split_tuple(expr: str): + """ + Split the elements in a tuple/interval, while handling well-formatted commas in large numbers + """ + expr = _strip_properly_formatted_commas(expr) + if len(expr) == 0: + return [] + if ( + len(expr) > 2 + and expr[0] in TUPLE_CHARS + and expr[-1] in TUPLE_CHARS + and all([ch not in expr[1:-1] for ch in TUPLE_CHARS]) + ): + elems = [elem.strip() for elem in expr[1:-1].split(",")] + else: + elems = [expr] + return elems + + +def last_boxed_only_string(string): + idx = string.rfind("\\boxed") + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + if right_brace_idx is None: + retval = None + else: + retval = string[idx : right_brace_idx + 1] + + return retval + + +def remove_boxed(s): + left = "\\boxed{" + try: + assert s[: len(left)] == left + assert s[-1] == "}" + return s[len(left) : -1] + except Exception: + return None + + +def extract_boxed_answer(solution: str) -> str: + """Extract the answer from inside a LaTeX \\boxed{} command""" + solution = last_boxed_only_string(solution) + solution = remove_boxed(solution) + return solution + + +def grade_answer_sympy(given_answer: str, ground_truth: str) -> bool: + ground_truth_normalized = _normalize(ground_truth) + given_normalized = _normalize(given_answer) + + if ground_truth_normalized is None: + return False + + if ground_truth_normalized == given_normalized: + return True + + if len(given_normalized) == 0: + return False + + ground_truth_elems = split_tuple(ground_truth_normalized) + given_elems = split_tuple(given_normalized) + + if len(ground_truth_elems) > 1 and ( + ground_truth_normalized[0] != given_normalized[0] or ground_truth_normalized[-1] != given_normalized[-1] + ): + is_correct = False + elif len(ground_truth_elems) != len(given_elems): + is_correct = False + else: + for ground_truth_elem, given_elem in zip(ground_truth_elems, given_elems, strict=True): + if _is_frac(ground_truth_elem) and _is_frac(given_elem): + # if fractions aren't reduced, then shouldn't be marked as correct + # so, we don't want to allow sympy.simplify in this case + is_correct = ground_truth_elem == given_elem + elif _str_is_int(ground_truth_elem) != _str_is_int(given_elem): + # if the ground truth answer is an integer, we require the given answer to be a strict match + # (no sympy.simplify) + is_correct = False + else: + is_correct = are_equal_under_sympy(ground_truth_elem, given_elem) + if not is_correct: + break + + return is_correct + + +def grade_answer_mathd(given_answer: str, ground_truth: str) -> bool: + ground_truth_normalized_mathd = mathd_normalize_answer(ground_truth) + given_answer_normalized_mathd = mathd_normalize_answer(given_answer) + + # be at least as lenient as mathd + if ground_truth_normalized_mathd == given_answer_normalized_mathd: + return True + return False + + +def extract_answer(passage: str) -> str: + if "\\boxed" in passage: + return extract_boxed_answer(passage) + return None + + +def grade(model_answer: str, gt_answer: str, fast: bool = True): + if "\\boxed" in gt_answer: + gt_answer = extract_answer(gt_answer) + correct = grade_answer_mathd(model_answer, gt_answer) or grade_answer_sympy(model_answer, gt_answer) + if not fast: + # This mode further uses math_verify to recall originally false positives. + # Will be a bit slower, and sensitive to bad inputs. + correct = correct or is_latex_equal( + model_answer, + gt_answer, + ) + return correct + + +def compute_score(model_response, gt_answer, fast=False): + model_answer = extract_answer(model_response) + if model_answer is None: + return { + "score": 0.0, + "format_score": 0.0, + "acc": False, + "extracted_gt": gt_answer, + # "extracted_pred": None, + } + # return 0.0, 0.0 # Cannot even parse anything. + is_correct = False + if isinstance(gt_answer, float) or isinstance(gt_answer, int): + gt_answer = str(gt_answer) + if isinstance(gt_answer, str): + is_correct = grade(model_answer, gt_answer, fast) + elif isinstance(gt_answer, list): + is_correct = False + for gt in gt_answer: + is_correct |= grade(model_answer, gt, fast) + if is_correct: + return { + "score": 1.0, + "format_score": 1.0, + "acc": True, + "extracted_gt": gt_answer, + # "extracted_pred": None, + } + else: + return { + "score": 0.0, + "format_score": 1.0, + "acc": False, + "extracted_gt": gt_answer, + # "extracted_pred": None, + } diff --git a/e2c/verl/recipe/entropy/reward_score/entropy_math/grader.py b/e2c/verl/recipe/entropy/reward_score/entropy_math/grader.py new file mode 100644 index 0000000000000000000000000000000000000000..02507e359646662ea001df73986fa0f3f38328ce --- /dev/null +++ b/e2c/verl/recipe/entropy/reward_score/entropy_math/grader.py @@ -0,0 +1,384 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) Microsoft Corporation. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE + +# Copyright (c) 2023 OpenAI +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# Copyright (c) 2021 Dan Hendrycks +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This logic is largely copied from the Hendrycks' MATH release (math_equivalence), and borrowed from: +- https://github.com/microsoft/ToRA/blob/main/src/eval/grader.py +- https://github.com/microsoft/ProphetNet/tree/master/CRITIC +- https://github.com/openai/prm800k +""" + +import contextlib +import math +import re +from math import isclose + +# sympy related +from sympy import N, simplify +from sympy.parsing.latex import parse_latex +from sympy.parsing.sympy_parser import parse_expr + +# verl related +from verl.utils.py_functional import timeout_limit + + +def is_digit(s): + try: + if "{,}" in str(s): + num = float(str(s).replace("{,}", "")) + return True, num + + num = float(str(s).replace(",", "")) + return True, num + except ValueError: + return False, None + + +def normalize(answer, pi) -> str: + # checking if answer is $ and removing $ in that case to compare + if isinstance(answer, str) and bool(re.match(r"\$\d+(\.\d+)?", answer)): + return answer[1:] + + # checking if answer is % or \\% and removing % + if isinstance(answer, str) and ( + bool(re.match(r"^\d+(\.\d+)?%$", answer)) or bool(re.match(r"^\d+(\.\d+)?\\%$", answer)) + ): + return answer.replace("\\%", "").replace("%", "") + + # handle base + answer = handle_base(answer) + + # handle pi + answer = handle_pi(answer, pi) + + return answer + + +def handle_base(x) -> str: + if isinstance(x, str) and "_" in x: + # Due to base + x = x.split("_")[0] + x = float(x) + return int(x) + return x + + +def handle_pi(string, pi): + if isinstance(string, str) and "\pi" in string: + # Find the first occurrence of "\pi" + idx = string.find("\pi") + + # Iterate over the string and find all occurrences of "\pi" with a valid previous character + while idx != -1: + if idx > 0 and string[idx - 1].isdigit(): + # Replace "\pi" with "*math.pi" if the previous character is a digit + string = string[:idx] + f"*{pi}" + string[idx + 3 :] + else: + # Replace "\pi" with "1*math.pi" if the previous character is not a digit + string = string[:idx] + f"1*{pi}" + string[idx + 3 :] + + # Find the next occurrence of "\pi" + idx = string.find("\pi", idx + 1) + + # Evaluate the expression using eval() function + with contextlib.suppress(Exception): + string = eval(string) + + return string + + +def math_equal( + prediction: bool | float | str, + reference: float | str, + include_percentage: bool = True, + tolerance: float = 1e-4, + timeout: float = 10.0, + pi: float = math.pi, +) -> bool: + """ + Exact match of math if and only if: + 1. numerical equal: both can convert to float and are equal + 2. symbolic equal: both can convert to sympy expression and are equal + """ + + prediction = normalize(prediction, pi) + reference = normalize(reference, pi) + + if isinstance(prediction, str) and len(prediction) > 1000: # handling weird corner-cases + prediction = prediction[:1000] + + # 0. string comparison + if isinstance(prediction, str) and isinstance(reference, str): + if prediction.strip().lower() == reference.strip().lower(): + return True + if prediction.replace(" ", "") == reference.replace(" ", ""): + return True + + try: # 1. numerical equal + if is_digit(prediction)[0] and is_digit(reference)[0]: + prediction = is_digit(prediction)[1] + reference = is_digit(reference)[1] + # number questions + gt_result = [reference / 100, reference, reference * 100] if include_percentage else [reference] + for item in gt_result: + try: + if isclose(item, prediction, rel_tol=tolerance): + return True + except Exception: + continue + return False + except Exception: + pass + + if not prediction and prediction not in [0, False]: + return False + + # 2. symbolic equal + reference = str(reference).strip() + prediction = str(prediction).strip() + + ## deal with [], (), {} + prediction = format_intervals(prediction) + + pred_str, ref_str = prediction, reference + if (prediction.startswith("[") and prediction.endswith("]") and not reference.startswith("(")) or ( + prediction.startswith("(") and prediction.endswith(")") and not reference.startswith("[") + ): + pred_str = pred_str.strip("[]()") + ref_str = ref_str.strip("[]()") + for s in ["{", "}", "(", ")"]: + ref_str = ref_str.replace(s, "") + pred_str = pred_str.replace(s, "") + if pred_str == ref_str: + return True + + ## [a, b] vs. [c, d], return a==c and b==d + if ( + prediction + and reference + and prediction[0] in "([" + and prediction[-1] in ")]" + and prediction[0] == reference[0] + and prediction[-1] == reference[-1] + ): + pred_parts = prediction[1:-1].split(",") + ref_parts = reference[1:-1].split(",") + if len(pred_parts) == len(ref_parts) and all( + [ + math_equal(pred_pt, ref_pt, include_percentage, tolerance) + for pred_pt, ref_pt in zip(pred_parts, ref_parts, strict=True) + ] + ): + return True + + if "," in prediction and "," in reference: + pred_parts = [item.strip() for item in prediction.split(",")] + ref_parts = [item.strip() for item in reference.split(",")] + + if len(pred_parts) == len(ref_parts): + return bool( + all( + [ + math_equal(pred_parts[i], ref_parts[i], include_percentage, tolerance) + for i in range(len(pred_parts)) + ] + ) + ) + + # if we have point == tuple of values + if prediction.startswith("Point") and reference[0] == "(" and reference[-1] == ")": + pred_parts = prediction[prediction.find("(") + 1 : -1].split(",") + ref_parts = reference[1:-1].split(",") + if len(pred_parts) == len(ref_parts) and all( + [ + math_equal(pred_pt, ref_pt, include_percentage, tolerance) + for pred_pt, ref_pt in zip(pred_parts, ref_parts, strict=True) + ] + ): + return True + + # if reference is a matrix + if "\begin{pmatrix}" in reference and prediction.startswith("Matrix"): + try: + pred_matrix = parse_expr(prediction) + ref_matrix_items = reference.split()[1:-1:2] + if len(pred_matrix) == len(ref_matrix_items) and all( + [ + math_equal(pred, ref, include_percentage, tolerance) + for ref, pred in zip(ref_matrix_items, pred_matrix, strict=True) + ] + ): + return True + except Exception: + pass + elif "\begin{pmatrix}" in reference and prediction.startswith("[") and prediction.endswith("]"): + if isinstance(eval(prediction), list): + try: + pred_matrix = eval(prediction) + # ref_matrix_items = reference.split()[1:-1:2] + ref_matrix_items = ( + reference.lstrip("\\begin{pmatrix}") # noqa: B005 + .lstrip("\begin{pmatrix}") + .rstrip("\\end{pmatrix}") + .rstrip("\end{pmatrix}") + ) # noqa: B005 + ref_matrix_items = ref_matrix_items.split("\\") + ref_matrix_items = [row.split("&") if "&" in row else row for row in ref_matrix_items] + if len(pred_matrix) == len(ref_matrix_items) and all( + [ + math_equal(pred, ref, include_percentage, tolerance) + for ref, pred in zip(ref_matrix_items, pred_matrix, strict=True) + ] + ): + return True + except Exception: + pass + + return symbolic_equal(prediction, reference, tolerance, timeout) + + +def symbolic_equal(a, b, tolerance, timeout=10.0): + def _parse(s): + for f in [parse_expr, parse_latex]: + try: + with timeout_limit(seconds=timeout): + return f(s) + except TimeoutError: + print(f"Parsing timed out for {s}") + continue + except Exception: + continue + return s + + a = _parse(a) + b = _parse(b) + + try: + with timeout_limit(seconds=timeout): + if simplify(a - b) == 0: + return True + except TimeoutError: + print(f"Simplification timed out for {a} - {b}") + pass + except Exception: + pass + + try: + with timeout_limit(seconds=timeout): + if isclose(N(a), N(b), rel_tol=tolerance): + return True + except TimeoutError: + print(f"Numerical evaluation timed out for {a}, {b}") + pass + except Exception: + pass + return False + + +def format_intervals(prediction): + patterns = { + "Interval(": r"^Interval\((.*)\)$", + "Interval.Ropen(": r"^Interval\.Ropen\((.*)\)$", + "Interval.Lopen(": r"^Interval\.Lopen\((.*)\)$", + "Interval.open(": r"^Interval\.open\((.*)\)$", + } + + for key, pattern in patterns.items(): + match = re.match(pattern, prediction) + if match: + inner_content = match.group(1) + + if key == "Interval(": # Intarval(a, b) == [a, b] + return f"[{inner_content}]" + elif key == "Interval.Ropen(": # Intarval.Ropen(a, b) == [a, b) + return f"[{inner_content})" + elif key == "Interval.Lopen(": # Intarval.Lopen(a, b) == (a, b] + return f"({inner_content}]" + elif key == "Interval.open(": # Intarval.open(a, b) == (a, b) + return f"({inner_content})" + + return prediction diff --git a/e2c/verl/recipe/entropy/reward_score/entropy_math/math_normalize.py b/e2c/verl/recipe/entropy/reward_score/entropy_math/math_normalize.py new file mode 100644 index 0000000000000000000000000000000000000000..74d94cc41cd7cca3c3e3051751c56f9140a775fa --- /dev/null +++ b/e2c/verl/recipe/entropy/reward_score/entropy_math/math_normalize.py @@ -0,0 +1,192 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) 2021 Dan Hendrycks +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +""" +This logic is largely copied from the Hendrycks' MATH release (math_equivalence). + +From: https://github.com/openai/prm800k/blob/main/prm800k/grading/math_normalize.py +""" + +import re +from typing import Optional + + +def normalize_answer(answer: Optional[str]) -> Optional[str]: + if answer is None: + return None + answer = answer.strip() + try: + # Remove enclosing `\text{}`. + m = re.search("^\\\\text\{(?P.+?)\}$", answer) + if m is not None: + answer = m.group("text").strip() + return _strip_string(answer) + except: # noqa: E722 + return answer + + +def _fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except: # noqa: E722 + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + + +def _fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + a = int(a) + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except: # noqa: E722 + return string + + +def _remove_right_units(string): + # "\\text{ " only ever occurs (at least in the val set) when describing units + if "\\text{ " in string: + splits = string.split("\\text{ ") + assert len(splits) == 2 + return splits[0] + else: + return string + + +def _fix_sqrt(string): + if "\\sqrt" not in string: + return string + splits = string.split("\\sqrt") + new_string = splits[0] + for split in splits[1:]: + if split[0] != "{": + a = split[0] + new_substr = "\\sqrt{" + a + "}" + split[1:] + else: + new_substr = "\\sqrt" + split + new_string += new_substr + return new_string + + +def _strip_string(string): + # linebreaks + string = string.replace("\n", "") + + # remove inverse spaces + string = string.replace("\\!", "") + + # replace \\ with \ + string = string.replace("\\\\", "\\") + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "") + string = string.replace("^\\circ", "") + + # remove dollar signs + string = string.replace("\\$", "") + + # remove units (on the right) + string = _remove_right_units(string) + + # remove percentage + string = string.replace("\\%", "") + string = string.replace("\%", "") + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + if len(string.split("=")) == 2 and len(string.split("=")[0]) <= 2: + string = string.split("=")[1] + + # fix sqrt3 --> sqrt{3} + string = _fix_sqrt(string) + + # remove spaces + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). + # Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # manually change 0.5 --> \frac{1}{2} + if string == "0.5": + string = "\\frac{1}{2}" + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + return string diff --git a/e2c/verl/recipe/genrm_remote/README.md b/e2c/verl/recipe/genrm_remote/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1a800fd882c60d20d1211828362d9f2acccec579 --- /dev/null +++ b/e2c/verl/recipe/genrm_remote/README.md @@ -0,0 +1,39 @@ +# Generative Reward Model + +## Scripts + +### Step 1: Launch a vLLM Server (Optional) + +Deploy the pretrained GenRM model using vLLM. Skip this step if you want to use an external api service. + +```bash +vllm serve verl-team/GenRM-CI-Test-1.5B --served-model-name genrm-demo +``` + +### Step 2: Perform RL using GenRM + +```bash +bash recipe/api-genrm/run_genrm_remote.sh +``` + +The implementation works by passing a customized reward function (see `reward_function.py`) + +For convenience, we run both the RL training and server on the same machine. To use an external server, configure the `BASE_URL` and `API_KEY` in `reward_function.py` first. + +## Advanced: Customizing Your GenRM + +You can use sglang server with data parallel for faster inference: + +```bash +CUDA_VISIBLE_DEVICES=0,1,2,3 python -m sglang_router.launch_server --model-path verl-team/GenRM-CI-Test-1.5B --dp-size 4 +``` + +Note that you should modify the `BASE_URL` in `reward_function.py` to match your SGLang Server address. + +You can also create your own customized GenRM by implementing a custom reward function. Here are some tips for customizing your own GenRM based on `reward_function.py`: + +- Design appropriate prompts for your GenRM +- Convert GenRM responses into RL rewards +- ... + +Since these aspects are highly flexible, we only provide a demo implementation. The actual design and implementation of GenRM is left to the user's discretion. diff --git a/e2c/verl/recipe/genrm_remote/reward_function.py b/e2c/verl/recipe/genrm_remote/reward_function.py new file mode 100644 index 0000000000000000000000000000000000000000..35b3af3993379737ce4cd2cf04235e3e426e806a --- /dev/null +++ b/e2c/verl/recipe/genrm_remote/reward_function.py @@ -0,0 +1,110 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from concurrent.futures import ThreadPoolExecutor +from time import sleep + +import requests + +from verl.utils.reward_score.math_reward import last_boxed_only_string, remove_boxed + +BASE_URL = "http://localhost:30000" +API_KEY = "EMPTY" +MAX_RETRIES = 3 +BASE_DELAY = 2 +MAX_WORKERS = 32 +MODEL_NAME = "genrm-demo" +GENRM_PROMPT_TEMPLATE = """ +The following is a math problem and an AI solution: + +[Math Problem] + +{problem} + +[AI Solution] + +{solution} + +Your task is to review and critique the solution step by step, and output whether the AI solution is correct. + +Please put your final answer (i.e., 'True' or 'False') in \\boxed{{}}. +""".strip() + + +def get_response(problem, solution_str, ground_truth): + prompt = GENRM_PROMPT_TEMPLATE.format(problem=problem, solution=solution_str) + messages = [{"role": "user", "content": prompt}] + for attempt in range(MAX_RETRIES): + try: + headers = {"Content-Type": "application/json"} + chat_url = f"{BASE_URL}/v1/chat/completions" + data = {"model": MODEL_NAME, "messages": messages} + output = requests.post(chat_url, headers=headers, json=data, timeout=30) + response = output.json()["choices"][0]["message"]["content"] + return response + except Exception as e: + if attempt < MAX_RETRIES - 1: + print("Exception: ", repr(e)) + delay = BASE_DELAY * (2**attempt) + print(f"Retrying in {delay} seconds...") + sleep(delay) + else: + print(f"Failed after {MAX_RETRIES} attempts. Error: {e}") + + raise ConnectionRefusedError(f"Failed to run the model for {prompt}!") + + +def compute_reward(response): + reward_score = 0.0 + try: + boxed_result = last_boxed_only_string(response) + if boxed_result is not None: + result = remove_boxed(boxed_result) + reward_score = float(result == "True") + except Exception as e: + print(e) + return reward_score + + +def compute_score(data_source, solution_str, ground_truth, extra_info): + split = extra_info["split"] + from verl.utils.reward_score import default_compute_score + + func_rm_score = default_compute_score(data_source, solution_str, ground_truth, extra_info) + + if split == "test": + return func_rm_score + else: + problem = extra_info["question"] + response = get_response(problem, solution_str, ground_truth) + if response is not None: + reward_score = compute_reward(response) + else: + reward_score = 0.0 + + return reward_score + + +def compute_score_batch(data_sources, solution_strs, ground_truths, extra_infos): + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + futures = [] + for data_source, solution_str, ground_truth, extra_info in zip( + data_sources, solution_strs, ground_truths, extra_infos, strict=True + ): + future = executor.submit(compute_score, data_source, solution_str, ground_truth, extra_info) + futures.append(future) + + results = [future.result() for future in futures] + + return results diff --git a/e2c/verl/recipe/genrm_remote/run_genrm_remote.sh b/e2c/verl/recipe/genrm_remote/run_genrm_remote.sh new file mode 100644 index 0000000000000000000000000000000000000000..6656dc8a7e340dae924715a03adf49171a4f3582 --- /dev/null +++ b/e2c/verl/recipe/genrm_remote/run_genrm_remote.sh @@ -0,0 +1,45 @@ +# vllm server +# CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve verl-team/GenRM-CI-Test-1.5B --served_model_name genrm-demo + +# sglang server +# CUDA_VISIBLE_DEVICES=0,1,2,3 python -m sglang_router.launch_server --model-path verl-team/GenRM-CI-Test-1.5B --dp-size 4 + +set -x + +CUDA_VISIBLE_DEVICES=4,5,6,7 python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=${HOME}/data/gsm8k/train.parquet \ + data.val_files=${HOME}/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=8 \ + algorithm.use_kl_in_reward=False \ + reward_model.reward_manager=batch \ + custom_reward_function.path=recipe/genrm_remote/reward_function.py \ + custom_reward_function.name=compute_score_batch \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_func_rm_example_gsm8k' \ + trainer.experiment_name='qwen2_5_3b_gen_rm' \ + trainer.n_gpus_per_node=4 \ + trainer.val_before_train=True \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=5 \ + trainer.total_epochs=10 \ + trainer.resume_mode='disable' diff --git a/e2c/verl/recipe/gspo/test_gspo_3b_math.sh b/e2c/verl/recipe/gspo/test_gspo_3b_math.sh new file mode 100644 index 0000000000000000000000000000000000000000..40cb9fab332c7a78d384784eb3ac5ecf957ec9e9 --- /dev/null +++ b/e2c/verl/recipe/gspo/test_gspo_3b_math.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +#SBATCH --job-name=rl-gspo-3B +#SBATCH --partition=main +#SBATCH --nodes=1 # Number of nodes +#SBATCH --ntasks-per-node=1 # One task per node +#SBATCH --cpus-per-task=128 # cpu-cores per task +#SBATCH --gres=gpu:8 +#SBATCH --mem=0 +#SBATCH --exclusive +#SBATCH --time=500:00:00 +#SBATCH --output=/rl/logs/Qwen2.5-3B/gspo/math/vllm_%x_%j.out +#SBATCH --error=/rl/logs/Qwen2.5-3B/gspo/math/vllm_%x_%j.err + +set -xeuo pipefail + +# activate the venv +echo "Activating verl environment..." +eval "$(conda shell.bash hook)" +conda deactivate +conda activate verl + +# can make training faster, depends on your infrastructure +export NCCL_IBEXT_DISABLE=1 +export NCCL_NVLS_ENABLE=1 +export NCCL_IB_HCA=mlx5 +export UCX_NET_DEVICES=mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1 + +# Set how many GPUs we actually have on this node. +export GPUS_PER_NODE=8 + +NNODES=${SLURM_JOB_NUM_NODES} +export NNODES + +export VLLM_ATTENTION_BACKEND=FLASH_ATTN +export RAY_LOGGING_LEVEL=DEBUG +export HYDRA_FULL_ERROR=1 +export WANDB_API_KEY=... # your wandb API key + +echo "Using $NNODES nodes for training..." + +# ------------------------------------- Setup xp params --------------------------------------- +project_name='RL-GSPO' + +adv_estimator=grpo +loss_mode=gspo +loss_agg_mode="seq-mean-token-mean" +MODEL_PATH=Qwen/Qwen2.5-3B-Instruct +offload=false # it's a small model, offloading will just slow-down training +rollout_engine=vllm +rollout_mode=sync # can be async to speedup large scale xps +gpu_memory_utilization=0.8 +reward_manager=dapo +adv_estimator=grpo +shuffle_dataset=true +first_time_dataset_prep=true # prepare dataset + +test_freq=10 +save_freq=10 +total_epochs=10 +total_training_steps=500 +val_before_train=false + +use_kl_in_reward=false +kl_coef=0.0 +use_kl_loss=false +kl_loss_coef=0.0 + +clip_ratio_low=0.0003 # as recommended by the paper, see Sec. 5.1 +clip_ratio_high=0.0004 # as recommended by the paper, see Sec. 5.1 +train_batch_size=512 +ppo_mini_batch_size=128 # maintain 4 mini-batches as recommended by the paper, see Sec. 5.1 +ppo_micro_batch_size_per_gpu=8 # setup depending on your GPU memory +n_resp_per_prompt=16 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +# dapo reward manager params +enable_overlong_buffer=false # true +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +# Paths and namings +SFT_MODEL=$(basename $MODEL_PATH) +exp_name="${loss_mode}-epslow-${clip_ratio_low}-epshigh-${clip_ratio_high}-${SFT_MODEL}-RL" +CKPTS_DIR=/rl/checkpoints/experimental/4b/${loss_mode}/${exp_name} + +# Sampling params at rollouts +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=1 +use_dynamic_bsz=true +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=true +gen_tp=1 +entropy_checkpointing=true # This enables entropy recomputation specifically for the entropy calculation, lowering memory usage during training. + +# ------------------------------------- train/val data preparation --------------------------------------- +if [ "$first_time_dataset_prep" = true ]; then + echo "Preprocessing GSM8K dataset..." + python examples/data_preprocess/gsm8k.py --local_save_dir /data/gsm8k/ +fi + +gsm8k_train_path=/data/gsm8k/train.parquet +gsm8k_test_path=/data/gsm8k/test.parquet + +# set the paths +train_files="['$gsm8k_train_path']" +test_files="['$gsm8k_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=${adv_estimator} \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + data.train_files="${train_files}" \ + data.val_files="${test_files}" \ + data.shuffle=$shuffle_dataset \ + data.prompt_key=prompt \ + data.truncation='error' \ + data.filter_overlong_prompts=true \ + data.train_batch_size=${train_batch_size} \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.model.use_remove_padding=true \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.name=${rollout_engine} \ + actor_rollout_ref.rollout.mode=${rollout_mode} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=true \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${ppo_mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${ppo_micro_batch_size_per_gpu} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=${gpu_memory_utilization} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=true \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=true \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.entropy_checkpointing=${entropy_checkpointing} \ + reward_model.reward_manager=${reward_manager} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=false \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${GPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=${val_before_train} \ + trainer.test_freq=${test_freq} \ + trainer.save_freq=${save_freq} \ + trainer.total_epochs=${total_epochs} \ + trainer.total_training_steps=${total_training_steps} \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=2 \ + $@ diff --git a/e2c/verl/recipe/gspo/test_gspo_3b_math_slurm.sh b/e2c/verl/recipe/gspo/test_gspo_3b_math_slurm.sh new file mode 100644 index 0000000000000000000000000000000000000000..d657b66109c6b5f56f403c50a5e4a4bfeeed37d7 --- /dev/null +++ b/e2c/verl/recipe/gspo/test_gspo_3b_math_slurm.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +#SBATCH --job-name=rl-gspo-3B +#SBATCH --partition=main +#SBATCH --nodes=1 # Number of nodes +#SBATCH --ntasks-per-node=1 # One task per node +#SBATCH --cpus-per-task=128 # cpu-cores per task +#SBATCH --gres=gpu:8 +#SBATCH --mem=0 +#SBATCH --exclusive +#SBATCH --time=500:00:00 +#SBATCH --output=/rl/logs/Qwen2.5-3B/gspo/math/vllm_%x_%j.out +#SBATCH --error=/rl/logs/Qwen2.5-3B/gspo/math/vllm_%x_%j.err + +set -xeuo pipefail + +# activate the venv +echo "Activating verl environment..." +eval "$(conda shell.bash hook)" +conda deactivate +conda activate verl + +# can make training faster, depends on your infrastructure +export NCCL_IBEXT_DISABLE=1 +export NCCL_NVLS_ENABLE=1 +export NCCL_IB_HCA=mlx5 +export UCX_NET_DEVICES=mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1 + +# Set how many GPUs we actually have on this node. +export GPUS_PER_NODE=8 + +NNODES=${SLURM_JOB_NUM_NODES} +export NNODES + +export VLLM_ATTENTION_BACKEND=FLASH_ATTN +export RAY_memory_monitor_refresh_ms=0 +export RAY_LOGGING_LEVEL=DEBUG +export HYDRA_FULL_ERROR=1 +export WANDB_API_KEY=... # your wandb API key + +# Let Ray know how many nodes to expect +export RAY_NUM_NODES=$NNODES + +echo "Using $NNODES nodes for training..." + +# ------------------------------------- Setup xp params --------------------------------------- +project_name='RL-GSPO' + +adv_estimator=grpo +loss_mode=gspo +loss_agg_mode="seq-mean-token-mean" +MODEL_PATH=Qwen/Qwen2.5-3B-Instruct +offload=false # it's a small model, offloading will just slow-down training +rollout_engine=vllm +rollout_mode=sync # can be async to speedup large scale xps +gpu_memory_utilization=0.8 +reward_manager=dapo +adv_estimator=grpo +shuffle_dataset=true +first_time_dataset_prep=true # prepare dataset + +test_freq=10 +save_freq=10 +total_epochs=10 +total_training_steps=500 +val_before_train=false + +use_kl_in_reward=false +kl_coef=0.0 +use_kl_loss=false +kl_loss_coef=0.0 + +clip_ratio_low=0.0003 # as recommended by the paper, see Sec. 5.1 +clip_ratio_high=0.0004 # as recommended by the paper, see Sec. 5.1 +train_batch_size=512 +ppo_mini_batch_size=128 # maintain 4 mini-batches as recommended by the paper, see Sec. 5.1 +ppo_micro_batch_size_per_gpu=8 # setup depending on your GPU memory +n_resp_per_prompt=16 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +# dapo reward manager params +enable_overlong_buffer=false # true +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +# Paths and namings +SFT_MODEL=$(basename $MODEL_PATH) +exp_name="${loss_mode}-epslow-${clip_ratio_low}-epshigh-${clip_ratio_high}-${SFT_MODEL}-RL" +CKPTS_DIR=/rl/checkpoints/experimental/4b/${loss_mode}/${exp_name} + +# Sampling params at rollouts +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +sp_size=1 +use_dynamic_bsz=true +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=true +gen_tp=1 +entropy_checkpointing=true # This enables entropy recomputation specifically for the entropy calculation, lowering memory usage during training. + +# ------------------------------------- train/val data preparation --------------------------------------- +if [ "$first_time_dataset_prep" = true ]; then + echo "Preprocessing GSM8K dataset..." + python examples/data_preprocess/gsm8k.py --local_save_dir /data/gsm8k/ +fi + +gsm8k_train_path=/data/gsm8k/train.parquet +gsm8k_test_path=/data/gsm8k/test.parquet + +# set the paths +train_files="['$gsm8k_train_path']" +test_files="['$gsm8k_test_path']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=${adv_estimator} \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + data.train_files="${train_files}" \ + data.val_files="${test_files}" \ + data.shuffle=$shuffle_dataset \ + data.prompt_key=prompt \ + data.truncation='error' \ + data.filter_overlong_prompts=true \ + data.train_batch_size=${train_batch_size} \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.model.use_remove_padding=true \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.name=${rollout_engine} \ + actor_rollout_ref.rollout.mode=${rollout_mode} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=true \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${ppo_mini_batch_size} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${ppo_micro_batch_size_per_gpu} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=${gpu_memory_utilization} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=true \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=true \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.entropy_checkpointing=${entropy_checkpointing} \ + reward_model.reward_manager=${reward_manager} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=false \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${GPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=${val_before_train} \ + trainer.test_freq=${test_freq} \ + trainer.save_freq=${save_freq} \ + trainer.total_epochs=${total_epochs} \ + trainer.total_training_steps=${total_training_steps} \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=2 \ + $@ diff --git a/e2c/verl/recipe/gspo/test_gspo_qwen30b_a3b_ep.sh b/e2c/verl/recipe/gspo/test_gspo_qwen30b_a3b_ep.sh new file mode 100644 index 0000000000000000000000000000000000000000..94809c6f08ba8ecc78d64e343c6e86a5ead338d2 --- /dev/null +++ b/e2c/verl/recipe/gspo/test_gspo_qwen30b_a3b_ep.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export NCCL_DEBUG=WARN +# export VERL_LOGGING_LEVEL=DEBUG + +project_name='DAPO' +exp_name='GSPO-Qwen3-30B-A3B-Base-MATH' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=3e-4 +clip_ratio_high=4e-4 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" +loss_mode=gspo + +train_prompt_bsz=256 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-2} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +# RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +# MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B-Base"} +# CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +# TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +# TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +MODEL_PATH=$HDFS_ROOT/model/Qwen3-30B-A3B-Base +CKPTS_DIR=$DATA_ROOT/checkpoint/${project_name}/${exp_name} +TRAIN_FILE=$DATA_ROOT/dataset/BytedTsinghua-SIA/DAPO-Math-17k/data/dapo-math-17k.parquet +aime24_test_path=$DATA_ROOT/dataset/aime-2024.parquet + +TEST_FILE="['$aime24_test_path']" + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 1)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True + +# gen +rollout_name=vllm # vllm or sglang +gen_tp=1 +gen_dp=4 +gen_ep=4 + +# train +train_tp=4 +train_pp=1 +EP=4 +ETP=1 + +python3 -m verl.trainer.main_ppo \ + --config-path=config \ + --config-name='ppo_megatron_trainer.yaml' \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.return_raw_chat=True \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.clip_grad=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=${rollout_name} \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.calculate_log_probs=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.data_parallel_size=${gen_dp} \ + actor_rollout_ref.rollout.expert_parallel_size=${gen_ep} \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=$EP \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=$ETP \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.use_mbridge=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}-tp${gen_tp}-ep${gen_ep}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=False \ + trainer.test_freq=10 \ + trainer.save_freq=30 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=300 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/infigui-g1/README.md b/e2c/verl/recipe/infigui-g1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..01ec072aa8cc001b26b4479b80baf98be67fe141 --- /dev/null +++ b/e2c/verl/recipe/infigui-g1/README.md @@ -0,0 +1,56 @@ +# Recipe for InfiGUI-G1 + +This directory contains the official implementation for the paper [InfiGUI-G1: Advancing GUI Grounding with Adaptive Exploration Policy Optimization](https://arxiv.org/abs/2508.05731). + +This work introduces Adaptive Exploration Policy Optimization (AEPO), a policy optimization framework designed to enhance GUI grounding in Multimodal Large Language Models (MLLMs). AEPO improves exploration efficiency by employing a multi-answer generation strategy and a theoretically grounded Adaptive Exploration Reward (AER) function. This approach effectively addresses the challenge of semantic alignment in complex GUI grounding tasks. + +We provide training scripts for both 3B and 7B models, configured for a single machine with 8 GPUs by default. + +## Environment Setup + +Please follow the main environment setup guide for `verl`. + +The provided scripts use the following Docker image: `verlai/verl:app-verl0.5-transformers4.55.4-sglang0.4.10.post2-mcore0.13.0-te2.2` + +## Data Preparation + +Before starting the training, you need to download the example dataset. This dataset is a filtered version of [omniact](https://huggingface.co/datasets/Writer/omniact), containing only grounding tasks and excluding easy samples. + +The data is hosted on the Hugging Face. You can download it using the `huggingface-cli`: + +```bash +huggingface-cli download --repo-type dataset --resume-download InfiX-ai/omniact_grounding_filtered --local-dir data/omniact_grounding_filtered +``` + +This command will download the training and validation parquet files into the `data/omniact_grounding_filtered` directory, which is the default path used by the scripts. + +## Training + +We provide scripts to train the 3B and 7B models. Please run them from the root directory of `verl`. + +- **Train the 3B model:** + + ```bash + bash recipe/infigui-g1/run_3b.sh + ``` + +- **Train the 7B model:** + + ```bash + bash recipe/infigui-g1/run_7b.sh + ``` + +## Using Custom Data + +If you wish to train on your own dataset, please format your data to match the structure of the example files located in `data/omniact_grounding_filtered`. + +Once your data is ready, you need to update the data path arguments in the training script. + +In `run_3b.sh` or `run_7b.sh`, modify the following lines: + +```bash + data.train_files=./path/to/your/train_data.parquet \ + data.val_files=./path/to/your/val_data.parquet \ +``` + +Replace the paths with the location of your custom data files. diff --git a/e2c/verl/recipe/infigui-g1/reward_fn.py b/e2c/verl/recipe/infigui-g1/reward_fn.py new file mode 100644 index 0000000000000000000000000000000000000000..9d5db0eef48b780f6982202394ee2c4af3e9838d --- /dev/null +++ b/e2c/verl/recipe/infigui-g1/reward_fn.py @@ -0,0 +1,388 @@ +# Copyright 2025 Individual Contributor: InfiX.ai +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import math +import re +from itertools import combinations + +FMT_RATIO = 1.0 +ACC_RATIO = 1.0 + + +# ============================================================================ +# Utility Functions +# ============================================================================ + + +def extract_think_format(predict_str: str) -> None | dict[str, str]: + """ + Check if the predicted string meets format requirements and extract thinking and answer parts. + + Args: + predict_str: The predicted string + + Returns: + If format requirements are met, returns a dictionary containing thinking and answer parts; + otherwise returns None + """ + if not predict_str or not isinstance(predict_str, str): + return None + + # Check if is at the beginning + if not predict_str.startswith(""): + return None + + # Check if there is ... format + pattern = r"(.*?)" + think_match = re.search(pattern, predict_str, re.DOTALL) + if not think_match: + return None + + if predict_str.count("") != 1 or predict_str.count("") != 1: + return None + + # Extract thinking content + think_content = think_match.group(1).strip() + if not think_content: + return None + + # Get content after + think_end_pos = predict_str.find("") + len("") + post_think_content = predict_str[think_end_pos:].strip() + + # Check if there is non-empty content after + if not post_think_content: + return None + + return {"think": think_content, "answer": post_think_content} + + +def extract_and_parse_json(input_string, wrapper): + """ + Try to extract and parse JSON from a string. + + Args: + input_string: The input string + wrapper: JSON wrapper symbols, can be '{}' or '[]' + + Returns: + Parsed JSON object, returns None if parsing fails + """ + if len(wrapper) != 2: + raise ValueError("Wrapper must be exactly two characters long") + + start_char, end_char = wrapper + start_index = input_string.find(start_char) + + if start_index == -1: + return None + + # Find the matching end character by balancing brackets/braces + balance = 1 + end_index = -1 + for i in range(start_index + 1, len(input_string)): + if input_string[i] == start_char: + balance += 1 + elif input_string[i] == end_char: + balance -= 1 + + if balance == 0: + end_index = i + break + + if end_index == -1: + return None + + json_string = input_string[start_index : end_index + 1] + + try: + return json.loads(json_string) + except json.JSONDecodeError: + return None + + +# ============================================================================ +# AER Reward Functions +# ============================================================================ + + +def _extract_verifiable_answer(answer): + """ + Extract and verify the format of the point list from the answer string. + + A valid format is a JSON list of dictionaries, where each dictionary + has a "point_2d" key with a list of two numbers as the value. + + Args: + answer: The answer string to extract points from + + Returns: + List of valid points or None if format is invalid + """ + points = extract_and_parse_json(answer, "[]") + if points is None or not isinstance(points, list): + return None + + # Verify each point in the list + for point in points: + if isinstance(point, dict) and "point_2d" in point: + point_2d = point["point_2d"] + if isinstance(point_2d, list) and len(point_2d) == 2: + continue + + # If any point is malformed, the whole answer is invalid + return None + + return points + + +def _format_reward(answer): + """ + Calculate the format reward for 'point' type data. + + This function is now primarily used as a check to see if the format is valid. + + Args: + answer: The answer string to validate + + Returns: + Tuple of (reward, is_collinear) where reward is 1.0 for valid format, 0.0 otherwise + """ + points = _extract_verifiable_answer(answer) + if points is None: + return 0.0, 0 + + points_2d = [item["point_2d"] for item in points] + if _check_collinear(points_2d): + return 0.0, 1 + + return 1.0, 0 + + +def _check_collinear(points_2d): + """ + Check if 3 or more points in the list are collinear on any straight line. + + This uses the cross-product method to avoid division and handle all line types. + + Args: + points_2d: A list of [x, y] coordinates + + Returns: + True if 3 or more points are collinear, False otherwise + """ + if len(points_2d) < 3: + return False + + # Iterate through all unique combinations of 3 points + for p1, p2, p3 in combinations(points_2d, 3): + x1, y1 = p1 + x2, y2 = p2 + x3, y3 = p3 + + # Check for collinearity using the cross-product method. + # If (y2 - y1) * (x3 - x1) == (y3 - y1) * (x2 - x1), the points are collinear. + # This is equivalent to checking if the area of the triangle formed by the points is 0. + if math.isclose((y2 - y1) * (x3 - x1), (y3 - y1) * (x2 - x1)): + return True + + return False + + +def _accuracy_reward(answer, ground_truth): + """ + Calculate the accuracy reward based on the symmetric zero-centered formula. + + The reward is in the range [-1, 1]. + + Args: + answer: The answer string containing predicted points + ground_truth: Ground truth bounding box dictionary + + Returns: + Tuple containing: + - accuracy (float): The calculated reward + - extracted_answer (str): The JSON string of the predicted points + - num_pred (int): The number of predicted points + - first_correct (int): 1 if the first predicted point is correct, 0 otherwise + """ + pred_points = _extract_verifiable_answer(answer) + + # If no valid points are extracted, this is considered a format error, return -1 reward + if pred_points is None: + return -1.0, "", 0, 0 + + num_pred = len(pred_points) + extracted_answer = json.dumps(pred_points) + + if num_pred == 0: + return -1.0, extracted_answer, 0, 0 + + # Find the rank 'k' of the first correct point + first_correct_rank = -1 + for i, item in enumerate(pred_points): + point_2d = item["point_2d"] + if ( + ground_truth["x1"] <= point_2d[0] <= ground_truth["x2"] + and ground_truth["y1"] <= point_2d[1] <= ground_truth["y2"] + ): + first_correct_rank = i + 1 # 1-based index + break + + # Calculate reward based on the zero-centered symmetric formula + accuracy = 0.0 + if first_correct_rank != -1: + # Case a: Correct point found (Positive reward space) + k = first_correct_rank + accuracy = 1.0 / math.sqrt(num_pred * k) + else: + # Case b: No correct point found (Negative reward space) + accuracy = -1.0 / num_pred + + first_correct = 1 if first_correct_rank == 1 else 0 + + return accuracy, extracted_answer, num_pred, first_correct + + +def calculate_point_reward(solution_str, ground_truth, extra_info=None, fmt_ratio=1.0, acc_ratio=1.0, **kwargs): + """ + Calculate the final reward for 'point' type data. + + Implements the full logic including format checks, collinearity checks, + and the zero-centered symmetric reward calculation. + + Args: + solution_str: The solution string from the model + ground_truth: Ground truth data + extra_info: Extra information dictionary + fmt_ratio: Format reward ratio + acc_ratio: Accuracy reward ratio + **kwargs: Additional keyword arguments + + Returns: + Dictionary containing detailed reward information + """ + if extra_info.get("no_think", False): + answer = solution_str + else: + solution_dict = extract_think_format(solution_str) + # If the overall 'think'/'answer' format is wrong, return score of -1 + if solution_dict is None: + return { + "score": -1.0, + "format": 0.0, + "accuracy": -1.0, + "pred": "", + "num_pred": 0, + "has_correct": 0, + "first_correct": 0, + "only_correct": 0, + "is_collinear": 0, + } + + answer = solution_dict["answer"] + + # Reuse _format_reward to check the format of the 'answer' part + # If it's invalid, return score of -1 + format_reward, is_collinear = _format_reward(answer) + if format_reward == 0.0: + return { + "score": -1.0, + "format": 0.0, + "accuracy": -1.0, + "pred": "", + "num_pred": 0, + "has_correct": 0, + "first_correct": 0, + "only_correct": 0, + "is_collinear": is_collinear, + } + + # If format is OK, calculate the accuracy reward + accuracy_reward, extracted_answer, num_pred, first_correct = _accuracy_reward(answer, ground_truth) + + return { + "score": fmt_ratio * format_reward + acc_ratio * accuracy_reward, + "format": format_reward, + "accuracy": accuracy_reward, + "pred": extracted_answer, + "num_pred": num_pred, + "has_correct": 1 if accuracy_reward > 0 else 0, + "first_correct": first_correct, + "only_correct": 1 if num_pred == 1 and accuracy_reward > 0 else 0, + "is_collinear": 0, + } + + +# ============================================================================ +# AER Reward Handler Registry +# ============================================================================ + +# Dictionary to map data_source to the respective reward calculation function +AER_REWARD_HANDLERS = { + "point": calculate_point_reward, +} + + +def aer_gui_reward_function(data_source, solution_str, ground_truth, extra_info=None, **kwargs): + """ + Main reward function dispatcher for the Adaptive Exploration Reward (AER) system. + + Delegates reward calculation to specific functions based on the data_source using a dictionary lookup. + + Args: + data_source: The source or type of the data (e.g., "point", "bbox") + solution_str: The solution string generated by the model + ground_truth: The ground truth data + extra_info: Any extra information passed along (optional) + **kwargs: Additional keyword arguments that might be passed from the PPO trainer config + + Returns: + Dictionary containing detailed reward information with keys: + - score: The final calculated reward score + - format: Format validation score + - accuracy: Accuracy score + - pred: Extracted prediction string + - num_pred: Number of predictions + - has_correct: Whether any correct prediction exists + - first_correct: Whether first prediction is correct + - only_correct: Whether only one correct prediction exists + - is_collinear: Whether points are collinear (for point type) + """ + handler = AER_REWARD_HANDLERS.get(data_source, None) + + if handler: + try: + return handler( + solution_str, ground_truth, extra_info=extra_info, fmt_ratio=FMT_RATIO, acc_ratio=ACC_RATIO, **kwargs + ) + except Exception as e: + logging.exception( + f"Error executing reward handler for data_source '{data_source}': {e}", + ) + return { + "score": -1.0, + "format": 0.0, + "accuracy": -1.0, + "pred": "", + "num_pred": 0, + "has_correct": 0, + "first_correct": 0, + "only_correct": 0, + "is_collinear": 0, + } # Return a default penalty score on error + else: + raise ValueError(f"Unknown data_source: '{data_source}'. No specific reward handler defined.") diff --git a/e2c/verl/recipe/infigui-g1/run_3b.sh b/e2c/verl/recipe/infigui-g1/run_3b.sh new file mode 100644 index 0000000000000000000000000000000000000000..811af25c7232aa89c2162e90aa1bfa2c94cad293 --- /dev/null +++ b/e2c/verl/recipe/infigui-g1/run_3b.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -x +ulimit -n 65535 + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=rloo \ + data.train_files=./data/omniact_grounding_filtered/omniact_filtered_train.parquet \ + data.val_files=./data/omniact_grounding_filtered/omniact_filtered_val.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=7168 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=False \ + data.truncation='error' \ + data.image_key=images \ + custom_reward_function.path=./recipe/infigui-g1/reward_fn.py \ + custom_reward_function.name=aer_gui_reward_function \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \ + actor_rollout_ref.model.enable_activation_offload=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=False \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=0 \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.clip_ratio_high=0.4 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.max_num_batched_tokens=8192 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=False \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.logger=['console','wandb'] \ + trainer.project_name='infigui-g1' \ + trainer.experiment_name='3b' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=16 \ + trainer.test_freq=16 \ + trainer.total_epochs=6 diff --git a/e2c/verl/recipe/infigui-g1/run_7b.sh b/e2c/verl/recipe/infigui-g1/run_7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..480d7bb90db67f1cb6ac7e2e49559ecf1992c58e --- /dev/null +++ b/e2c/verl/recipe/infigui-g1/run_7b.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -x +ulimit -n 65535 + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=rloo \ + data.train_files=./data/omniact_grounding_filtered/omniact_filtered_train.parquet \ + data.val_files=./data/omniact_grounding_filtered/omniact_filtered_val.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=7168 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=False \ + data.truncation='error' \ + data.image_key=images \ + custom_reward_function.path=./recipe/infigui-g1/reward_fn.py \ + custom_reward_function.name=aer_gui_reward_function \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \ + actor_rollout_ref.model.enable_activation_offload=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=False \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=0 \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.clip_ratio_high=0.4 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.max_num_batched_tokens=8192 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=False \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.logger=['console','wandb'] \ + trainer.project_name='infigui-g1' \ + trainer.experiment_name='7b' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=16 \ + trainer.test_freq=16 \ + trainer.total_epochs=6 diff --git a/e2c/verl/recipe/langgraph_agent/__init__.py b/e2c/verl/recipe/langgraph_agent/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/recipe/langgraph_agent/chat_model.py b/e2c/verl/recipe/langgraph_agent/chat_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5a3b98f74317c487b3ca1f7f80659aa7b76ef465 --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/chat_model.py @@ -0,0 +1,366 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Ref: https://python.langchain.com/docs/how_to/custom_chat_model/ +""" + +import asyncio +import json +import logging +import os +import uuid +from typing import Any, Optional + +from langchain_core.language_models import BaseChatModel +from langchain_core.language_models.base import LanguageModelInput +from langchain_core.messages import ( + AIMessage, + BaseMessage, + convert_to_openai_messages, +) +from langchain_core.messages.tool import InvalidToolCall, ToolCall +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.runnables import Runnable, RunnableConfig +from langchain_core.tools import StructuredTool +from langchain_core.utils.function_calling import convert_to_openai_tool +from pydantic import Field + +from verl.experimental.agent_loop.agent_loop import AgentLoopOutput, AsyncLLMServerManager +from verl.experimental.agent_loop.tool_parser import ToolParser + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class MaxTokenExceededError(Exception): + """Indicate that history chat messages + tool message exceeds LLM max_tokens.""" + + pass + + +class ChatModel(BaseChatModel): + model_name: str = Field(alias="model") + """The name of the model""" + + client: AsyncLLMServerManager + """AsyncLLM server manager""" + + tokenizer: Any + """Tokenizer for the model""" + + max_tokens: int + """Max tokens to generate""" + + tool_parser: str = "hermes" + """Tool parser for the model""" + + max_parallel_calls: int = 1 + """Max parallel tool calls""" + + temperature: float = 1.0 + """Temperature for sampling""" + + top_p: float = 1.0 + """Top p for sampling""" + + repetition_penalty: float = 1.0 + """Repetition penalty for sampling""" + + def bind_tools(self, tools, **kwargs) -> Runnable[LanguageModelInput, BaseMessage]: + """Bind tools to the model. + + Args: + tools: Sequence of tools to bind to the model. + + Returns: + A Runnable that returns a message. + """ + formatted_tools: list = [convert_to_openai_tool(tool) for tool in tools] + + # used to remove system prompt prefix when encoding tool response + system_prompt = self.tokenizer.apply_chat_template([{}], add_generation_prompt=False, tokenize=True) + kwargs["system_prompt"] = system_prompt + + return self.bind(tools=formatted_tools, **kwargs) + + def with_structured_output( + self, + schema: dict | type, + *, + include_raw: bool = False, + **kwargs: Any, + ) -> Runnable[LanguageModelInput, dict | BaseChatModel]: + """Ref: https://langchain-ai.github.io/langgraph/how-tos/react-agent-structured-output/""" + raise NotImplementedError + + def _generate( + self, + messages: list[BaseMessage], + stop: Optional[list[str]] = None, + **kwargs: Any, + ) -> ChatResult: + raise NotImplementedError + + async def _agenerate( + self, + messages: list[BaseMessage], + stop: Optional[list[str]] = None, + **kwargs: Any, + ) -> ChatResult: + """Asynchronously generate chat completion message. + + Args: + messages (list[BaseMessage]): List of list of messages. + stop (Optional[list[str]], optional): Stop words to use when generating. Model output is cut off at the + first occurrence of any of these substrings. Defaults to None. + + Returns: + ChatResult: Chat result. + """ + request_id, prompt_ids, response_mask = await self._preprocess(messages, **kwargs) + + sampling_params = { + "temperature": self.temperature, + "top_p": self.top_p, + "repetition_penalty": self.repetition_penalty, + } + if "sampling_params" in kwargs: + sampling_params.update(kwargs["sampling_params"]) + + output = await self.client.generate( + request_id=request_id, prompt_ids=prompt_ids, sampling_params=sampling_params + ) + + message = await self._postprocess(request_id, prompt_ids, response_mask, output.token_ids, **kwargs) + generation = ChatGeneration(message=message) + return ChatResult(generations=[generation]) + + @property + def _llm_type(self) -> str: + """Get the type of language model used by this chat model.""" + return self.model_name + + async def _preprocess(self, messages: list[BaseMessage], **kwargs: Any) -> tuple[str, list[int], list[int]]: + """Preprocess messages for chat completion. + + To ensure strong consistency with policy model, AsyncLLM server generate response with token in token out + instead of messages list. + + But all agent frameworks use messages list to represent chat history. To mitigate the gap, we store trajectory + (prompt_ids, response_mask) in lastest AIMessage.response_metadata. + + 1. Encode ToolMessage to token ids. + 2. Retrieve trajectory (prompt_ids, response_mask) from lastest AIMessage.response_metadata. + 3. Append ToolMessage token ids to prompt_ids, and append 0 to response_mask. + + Ref: https://python.langchain.com/docs/concepts/chat_history/ + + Args: + messages (list[BaseMessage]): List of messages. + + Returns: + tuple[str, list[int], list[int]]: Request id, prompt ids, response mask. + """ + # messages: [system], human, ai, human|tool, ai, human|tool, ... + assert messages[-1].type in ["human", "tool"], ( + f"Last message must be human or tool, but got {messages[-1].type}" + ) + loop = asyncio.get_running_loop() + + # Case 1: initial chat completion: [system], human + if messages[-1].type == "human" and (len(messages) == 1 or messages[-2].type != "ai"): + prompt_ids = await loop.run_in_executor( + None, + lambda: self.tokenizer.apply_chat_template( + convert_to_openai_messages(messages), + tools=kwargs.get("tools"), + add_generation_prompt=True, + tokenize=True, + ), + ) + return str(uuid.uuid4()), prompt_ids, [] + + # Case 2: follow up chat completion with tool/human response: [system], human, ai, human|tool, ... + for i in range(len(messages) - 1, -1, -1): + if messages[i].type == "ai": + break + assert "prompt_ids" in messages[i].response_metadata, "Last message must have prompt_ids in response_metadata" + assert "response_mask" in messages[i].response_metadata, ( + "Last message must have response_mask in response_metadata" + ) + + # encode tool response + tool_responses = convert_to_openai_messages(messages[i + 1 :]) + tool_response_ids = await loop.run_in_executor( + None, + lambda messages=tool_responses: self.tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=True + ), + ) + tool_response_ids = tool_response_ids[len(kwargs["system_prompt"]) :] + + # stop generation if response length exceeds max response length + if len(messages[i].response_metadata["response_mask"]) + len(tool_response_ids) >= self.max_tokens: + raise MaxTokenExceededError(f"Max response length {self.max_tokens} exceeded") + + # append tool response to prompt + request_id = messages[i].response_metadata.pop("request_id") + prompt_ids = messages[i].response_metadata.pop("prompt_ids") + response_mask = messages[i].response_metadata.pop("response_mask") + prompt_ids += tool_response_ids + response_mask += [0] * len(tool_response_ids) + + return request_id, prompt_ids, response_mask + + async def _postprocess( + self, request_id: str, prompt_ids: list[int], response_mask: list[int], response_ids: list[int], **kwargs: Any + ) -> AIMessage: + """Postprocess response_ids when chat completion is done. + + 1. Decode response_ids, parse tool calls to AIMessage. + 2. Append response_ids to prompt_ids, and append 1 to response_mask. + 3. Store trajectory (prompt_ids, response_mask) in AIMessage.response_metadata. + + Args: + request_id (str): Unique request id. + prompt_ids (list[int]): Input prompt token ids in this chat completion. + response_mask (list[int]): Response mask before this chat completion. + response_ids (list[int]): LLM generated token ids in this chat completion. + + Returns: + AIMessage: Postprocessed message. + """ + prompt_ids += response_ids + response_mask += [1] * len(response_ids) + + tool_parser = ToolParser.get_tool_parser(self.tool_parser, self.tokenizer) + content, function_calls = await tool_parser.extract_tool_calls(response_ids) + + tool_calls, invalid_tool_calls = [], [] + + for function_call in function_calls: + error = None + try: + args = json.loads(function_call.arguments) + if not isinstance(args, dict): + error = f"Tool arguments must be a JSON object, got {type(args).__name__}" + except json.JSONDecodeError as e: + error = f"Invalid JSON tool arguments: {e}" + + if error: + logger.warning(error) + invalid_tool_calls.append( + InvalidToolCall( + name=function_call.name, + args=function_call.arguments, + id=str(uuid.uuid4()), + error=error, + ) + ) + else: + tool_calls.append( + ToolCall( + name=function_call.name, + args=args, + id=str(uuid.uuid4()), + ) + ) + + message = AIMessage( + content=content, + tool_calls=tool_calls[: self.max_parallel_calls], + invalid_tool_calls=invalid_tool_calls[: self.max_parallel_calls], + response_metadata={ + "request_id": request_id, + "prompt_ids": prompt_ids, + "response_mask": response_mask, + }, + ) + return message + + +class TruncateStructuredTool(StructuredTool): + """Structured tool with response truncation.""" + + tool_response_truncate_side: str + """truncate side of tool response: left, middle, right""" + + max_tool_response_length: int + """max length of tool response""" + + async def _arun( + self, + *args: Any, + config: RunnableConfig, + **kwargs: Any, + ) -> Any: + tool_response = await super()._arun(*args, config=config, **kwargs) + tool_response = str(tool_response) + + if len(tool_response) > self.max_tool_response_length: + if self.tool_response_truncate_side == "left": + tool_response = tool_response[: self.max_tool_response_length] + "...(truncated)" + elif self.tool_response_truncate_side == "right": + tool_response = "(truncated)..." + tool_response[-self.max_tool_response_length :] + else: + length = self.max_tool_response_length // 2 + tool_response = tool_response[:length] + "...(truncated)..." + tool_response[-length:] + + return tool_response + + +def convert_to_agent_output(messages: list[BaseMessage], response_length: int) -> AgentLoopOutput: + """Convert messages to AgentLoopOutput. + + Args: + messages (List[BaseMessage]): List of messages, last message must be assistant + with response_metadata containing `prompt_ids` and `response_mask`. + response_length (int): Max length of response. + + Returns: + AgentLoopOutput: agent loop output trajectory used for training. + """ + # skip last tool calls + for i in range(len(messages) - 1, -1, -1): + if messages[i].type != "tool": + break + last_message = messages[i] + assert last_message.type == "ai", f"Last message must be assistant, but got {last_message.type}" + assert "prompt_ids" in last_message.response_metadata, "Last message must have prompt_ids in response_metadata" + assert "response_mask" in last_message.response_metadata, ( + "Last message must have response_mask in response_metadata" + ) + + num_turns = 0 + for i in range(len(messages)): + if messages[i].type == "system": + continue + # parallel tool calls are in single turn + if i == 0 or messages[i].type != messages[i - 1].type: + num_turns += 1 + + prompt_ids = last_message.response_metadata["prompt_ids"] + response_mask = last_message.response_metadata["response_mask"] + + response_ids = prompt_ids[-len(response_mask) :] + prompt_ids = prompt_ids[: len(prompt_ids) - len(response_mask)] + + output = AgentLoopOutput( + prompt_ids=prompt_ids, + response_ids=response_ids[:response_length], + response_mask=response_mask[:response_length], + num_turns=num_turns, + metrics={}, + ) + return output diff --git a/e2c/verl/recipe/langgraph_agent/example/README.md b/e2c/verl/recipe/langgraph_agent/example/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4540c51b4e9382afbefe9651f6754a6037f292ee --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/example/README.md @@ -0,0 +1,138 @@ +# MathExpression: LangGraph Agent Example + +MathExpression is a tiny example to demonstrate multi-turn rollout with [LangGraph ReactAgent](https://langchain-ai.github.io/langgraph/agents/overview/). + +### Define react agent with tool +Firstly, to force ReactAgent to evaluate math expression by tool, we define a special operand `@`: +```python +@tool(parse_docstring=True) +def calculate(a: int, b: int, operand: str) -> int: + """ + Compute the results using operand with two integers + + Args: + a: the first operand + b: the second operand + operand: '+' or '-' or '*' or '@' + """ + assert operand in ["+", "-", "*", "@"], f"unknown operand {operand}" + if operand == "@": + return 3 * a - 2 * b + return eval(f"{a} {operand} {b}") +``` + +Without calling `calculate`, ReactAgent is impossible to evaluate math expression correctly. + +Then, we can equip ReactAgent with `calculate` tool: +```python +class MathExpressionReactAgentLoop(ReactAgentLoop): + @classmethod + def init_class(cls, config, tokenizer): + cls.tools = [calculate] + super().init_class(config, tokenizer) +``` + +We can define agent loop config in yaml file, which will be used by AgentLoopWorker to dynamic load custom AgentLoop class. +```yaml +- name: math_expression + _target_: recipe.langgraph_agent.example.math_expression.MathExpressionReactAgentLoop +``` + +### Prepare dataset +Now, let's prepare two small datasets for training and evaluation: +```bash +python recipe/langgraph_agent/example/create_dataset.py +``` + +- Parameters: `--train_size` (default: 5000), `--test_size` (default: 500), `--output_dir` (default: `data/math_expression_tool`). +- Example with custom sizes/output: +```bash +python recipe/langgraph_agent/example/create_dataset.py \ + --train_size 10000 \ + --test_size 1000 \ + --output_dir data/math_expression_tool +``` + +Note that dataset should contain a column `agent_name` with `math_expression`, which is used by `AgentLoopWorker` to select the +agent loop class. +| prompt | reward_model | agent_name | +|--------------------------------------|------------------------------|-----------------| +| [{'role': 'user', 'content': '...'}] | {'ground_truth': '-10', ...} | math_expression | +| [{'role': 'user', 'content': '...'}] | {'ground_truth': '-10', ...} | math_expression | + +Generated math expressions are like below, requiring model to call `calculate` multiple times to solve sub expressions. +``` +(2 @ (8 @ 8 @ 5 @ 5 @ 3) @ 6 @ (1 @ 4 @ 4 @ 4) @ 2) @ 6 +(4.6 @ (9.05 @ 4.0) @ 8.3 @ 1.21) @ 8.6 +9 @ 4 +((2 @ 2) @ (3 @ 3)) @ 4 +``` + +### Training +Hook all these up and start training: +```bash +bash recipe/langgraph_agent/example/run_qwen2.5_3b.sh 2>&1 | tee train.log +``` + +To submit on a SLURM cluster (the script contains SBATCH headers): +```bash +sbatch recipe/langgraph_agent/example/run_qwen2.5_3b.sh +``` + +**Note on `GPUS_PER_NODE` and `NNODES`:** + +- `GPUS_PER_NODE`: GPUs per node. + Detection order: `SLURM_GPUS_ON_NODE` (if set) → `GPUS_PER_NODE` → `2`. +- `NNODES`: number of nodes. + Detection order: `SLURM_JOB_NUM_NODES` (if set) → `NNODES` → `1`. +- Total GPUs = `GPUS_PER_NODE × NNODES` (must be ≥ 2). + +Local override (no `SLURM_*` set): +```bash +GPUS_PER_NODE=4 NNODES=2 bash recipe/langgraph_agent/example/run_qwen2.5_3b.sh +``` + +After total 39 steps, model should achieve 100% accuray on test dataset: +- val-aux/lighteval/MATH/reward: 1.0 +- val-aux/num_turns/mean: 9.0, average number of messages include assistant and tool turns. + +We can see model can solve math expression by calling `calculate` tool: +``` +system +You are Qwen, created by Alibaba Cloud. You are a helpful assistant. +user +We define a new math operator @, where you can only call an external tool to compute. Please put your final answer inside \boxed{} only in the last turn. Now answer the following questions: +Compute 5 @ (3 @ 6 @ 8) +assistant +Sure, let's break this down step by step. We need to compute the expression \(5 @ (3 @ 6 @ 8)\). First, we need to compute the inner expression \(3 @ 6 @ 8\). + +Let's compute \(3 @ 6\) first using the external tool. + +{"name": "calculate", "arguments": {"a": 3, "b": 6, "operand": "@"}} +user + +-3 + +assistant +We have computed that \(3 @ 6 = -3\). Now we need to compute \(-3 @ 8\). + +Let's use the external tool to compute this. + +{"name": "calculate", "arguments": {"a": -3, "b": 8, "operand": "@"}} +user + +-25 + +assistant +We have computed that \(-3 @ 8 = -25\). Now we can put this result into the original expression to get the final result of \(5 @ (3 @ 6 @ 8)\), which is \(5 @ (-25)\). + +Let's compute \(5 @ (-25)\) using the external tool. + +{"name": "calculate", "arguments": {"a": 5, "b": -25, "operand": "@"}} +user + +65 + +assistant +The final result of \(5 @ (3 @ 6 @ 8)\) is \(\boxed{65}\). +``` diff --git a/e2c/verl/recipe/langgraph_agent/example/agent.yaml b/e2c/verl/recipe/langgraph_agent/example/agent.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbd8fb9ebb9023c2368cd5bce94b3a589262cbe7 --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/example/agent.yaml @@ -0,0 +1,2 @@ +- name: math_expression + _target_: recipe.langgraph_agent.example.math_expression.MathExpressionReactAgentLoop diff --git a/e2c/verl/recipe/langgraph_agent/example/create_dataset.py b/e2c/verl/recipe/langgraph_agent/example/create_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..07cf19e8b8718683c9049edd1b6f1f21d3fed76b --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/example/create_dataset.py @@ -0,0 +1,289 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Create dataset for calculator +""" + +import argparse +import os +import random + +import pandas as pd + + +def generate_math_expression(min_terms=2, max_terms=5, min_number=1, max_number=10, allow_decimals=False, max_depth=2): + """ + Generate a random mathematical expression with operators +, -, *, /, and parentheses. + + Args: + min_terms (int): Minimum number of terms in the expression. + max_terms (int): Maximum number of terms in the expression. + max_number (int): Maximum value for numbers in the expression. + allow_decimals (bool): Whether to allow decimal numbers. + max_depth (int): Maximum nesting depth for parentheses. + + Returns: + str: A valid mathematical expression as a string. + """ + + def generate_number(): + """Generate a random number (integer or float).""" + assert min_number < max_number + num = random.uniform(min_number, max_number) + if not allow_decimals: + num = int(num) + else: + num = round(num, random.randint(0, 2)) # Round to 0-2 decimal places + return str(num) + + def generate_term(depth=0): + """Generate a term (number or parenthesized expression).""" + if depth < max_depth and random.random() < 0.5: # 50% chance to add parentheses + expr = generate_expression(depth + 1) + return f"({expr})" + else: + return generate_number() + + def generate_expression(depth=0): + """Generate a full expression with multiple terms and operators.""" + num_terms = random.randint(min_terms, max_terms) + terms = [generate_term(depth) for _ in range(num_terms)] + + # Randomly select operators + operators = ["+", "-", "*", "/", "@"] + expr = terms[0] + + for i in range(1, num_terms): + # Bias towards + and - for readability + op = random.choices( + operators, + weights=[0, 0, 0, 0, 1], # + and - are 1.5x more likely than * and / + )[0] + expr += f" {op} " + terms[i] + + return expr + + return generate_expression() + + +def test(): + # Example 1: Basic integer expression + print(generate_math_expression()) + # Output: (3 + 7) * 2 - 5 + + # Example 2: Expression with decimals + print(generate_math_expression(allow_decimals=True)) + # Output: 4.5 / (2.1 + 3.7) - 1.2 + + # Example 3: More complex expression with higher depth + print(generate_math_expression(max_terms=6, max_depth=3)) + # Output: ((5 * 2) - (3 + 1)) / (7 - 2) + 4 + + # Example 4: Simplified expression + print(generate_math_expression(min_terms=2, max_terms=3, max_number=5)) + # Output: 4 - 2 * 3 + + +def calculate(expression: str) -> float: + """ + Evaluate a mathematical expression with +, -, *, /, @, and parentheses. + The @ operator is defined as: a @ b = 3a - 2b. + + Args: + expression (str): Input mathematical expression (e.g., "3@2+4"). + + Returns: + float: Result of the evaluated expression. + + Raises: + ValueError: For invalid expressions (e.g., mismatched parentheses, division by zero). + """ + + def tokenize(s: str) -> list: + """Convert the input string into tokens (numbers, operators, parentheses).""" + tokens = [] + i = 0 + while i < len(s): + if s[i].isdigit() or s[i] == ".": + # Parse number (integer or float) + j = i + while j < len(s) and (s[j].isdigit() or s[j] == "."): + j += 1 + tokens.append(s[i:j]) + i = j + elif s[i] in "+-*/@()": + # Operator or parenthesis + tokens.append(s[i]) + i += 1 + elif s[i].isspace(): + # Skip whitespace + i += 1 + else: + raise ValueError(f"Invalid character: {s[i]}") + return tokens + + def infix_to_postfix(tokens: list) -> list: + """Convert infix notation to postfix notation (Reverse Polish Notation).""" + output = [] + stack = [] + # Higher precedence for @ (between * and +) + precedence = {"@": 3, "*": 2, "/": 2, "+": 1, "-": 1} + + for token in tokens: + if token.isdigit() or "." in token: + output.append(token) + elif token == "(": + stack.append(token) + elif token == ")": + while stack and stack[-1] != "(": + output.append(stack.pop()) + if not stack or stack[-1] != "(": + raise ValueError("Mismatched parentheses") + stack.pop() # Discard '(' + else: # Operator + while stack and stack[-1] != "(" and precedence.get(stack[-1], 0) >= precedence.get(token, 0): + output.append(stack.pop()) + stack.append(token) + + # Pop remaining operators + while stack: + if stack[-1] in "()": + raise ValueError("Mismatched parentheses") + output.append(stack.pop()) + + return output + + def evaluate_postfix(postfix: list) -> float: + """Evaluate postfix expression using a stack.""" + stack = [] + for token in postfix: + if token.isdigit() or "." in token: + stack.append(float(token)) + else: + if len(stack) < 2: + raise ValueError("Invalid expression") + b = stack.pop() + a = stack.pop() + if token == "+": + res = a + b + elif token == "-": + res = a - b + elif token == "*": + res = a * b + elif token == "/": + if b == 0: + raise ValueError("Division by zero") + res = a / b + elif token == "@": + res = 3 * a - 2 * b # Custom @ operator implementation + else: + raise ValueError(f"Invalid operator: {token}") + stack.append(res) + + if len(stack) != 1: + raise ValueError("Invalid expression") + return stack[0] + + # Remove spaces and validate parentheses + expression = expression.replace(" ", "") + if expression.count("(") != expression.count(")"): + raise ValueError("Mismatched parentheses") + + tokens = tokenize(expression) + postfix = infix_to_postfix(tokens) + result = evaluate_postfix(postfix) + + # Convert integers to integer representation + if result.is_integer(): + return int(result) + return result + + +def generate_data(total_num_dataset, split): + rl_dataset = { + "prompt": [], + "data_source": [], + "ability": [], + "reward_model": [], + "extra_info": [], + "agent_name": [], + } + + for idx in range(total_num_dataset): + while True: + try: + expression: str = generate_math_expression( + min_terms=2, max_terms=3, min_number=1, max_number=10, allow_decimals=False, max_depth=1 + ) + + num_plus = expression.count("+") + num_minus = expression.count("-") + num_mul = expression.count("*") + num_star = expression.count("@") + + answer = str(calculate(expression)) + # answer = str(eval(expression)) + break + except Exception as e: + print(e) + continue + + num_tool_calls = num_plus + num_minus + num_mul + num_star + + prompt = ( + f"We define a new math operator @, where you can only call an external tool to compute. " + f"Please put your final answer inside \\boxed{{}} only in the last turn. Now answer the " + f"following questions:\nCompute {expression}" + ) + prompt_with_template = [ + { + "role": "user", + "content": prompt, + } + ] + + rl_dataset["prompt"].append(prompt_with_template) + rl_dataset["data_source"].append("lighteval/MATH") + rl_dataset["ability"].append("math") + rl_dataset["reward_model"].append({"style": "lighteval/MATH", "ground_truth": answer}) + rl_dataset["extra_info"].append( + {"index": idx, "expression": expression, "split": split, "expected_tool_calls": num_tool_calls} + ) + rl_dataset["agent_name"].append("math_expression") + + rl_dataset = pd.DataFrame(data=rl_dataset) + return rl_dataset + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Math Expression Dataset Generator") + parser.add_argument("--train_size", type=int, default=5000, help="Number of training samples") + parser.add_argument("--test_size", type=int, default=500, help="Number of testing samples") + parser.add_argument("--output_dir", default="data/math_expression_tool", help="Directory to save the dataset") + args = parser.parse_args() + + # print(calculate("3@2")) # Output: 5 (3*3 - 2*2) + # print(calculate("3@2+4")) # Output: 9 (5 + 4) + # print(calculate("3*(4@2)")) # Output: 24 (3 * 8) + # print(calculate("(5@3)*2")) # Output: 18 (9 * 2) + + train_dataset = generate_data(total_num_dataset=args.train_size, split="train") + test_dataset = generate_data(total_num_dataset=args.test_size, split="test") + + # Make sure the dataset directory exists + os.makedirs(args.output_dir, exist_ok=True) + + # Save the datasets to parquet files + train_dataset.to_parquet(os.path.join(args.output_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(args.output_dir, "test.parquet")) diff --git a/e2c/verl/recipe/langgraph_agent/example/math_expression.py b/e2c/verl/recipe/langgraph_agent/example/math_expression.py new file mode 100644 index 0000000000000000000000000000000000000000..4532c8af3c42087c98d5ae3ee0a63690cd715691 --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/example/math_expression.py @@ -0,0 +1,39 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from langchain_core.tools import tool + +from recipe.langgraph_agent.react_agent_loop import ReactAgentLoop + + +@tool(parse_docstring=True) +def calculate(a: int, b: int, operand: str) -> int: + """ + Compute the results using operand with two integers + + Args: + a: the first operand + b: the second operand + operand: '+' or '-' or '*' or '@' + """ + assert operand in ["+", "-", "*", "@"], f"unknown operand {operand}" + if operand == "@": + return 3 * a - 2 * b + return eval(f"{a} {operand} {b}") + + +class MathExpressionReactAgentLoop(ReactAgentLoop): + @classmethod + def init_class(cls, config, tokenizer, **kwargs): + cls.tools = [calculate] + super().init_class(config, tokenizer) diff --git a/e2c/verl/recipe/langgraph_agent/example/run_qwen2.5_3b.sh b/e2c/verl/recipe/langgraph_agent/example/run_qwen2.5_3b.sh new file mode 100644 index 0000000000000000000000000000000000000000..4e4cc020ae05db344ea995a4f8310068b84a8670 --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/example/run_qwen2.5_3b.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +#SBATCH --job-name=rl-langgraph-3B +#SBATCH --partition=main +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=64 +#SBATCH --gres=gpu:4 +#SBATCH --mem=0 +#SBATCH --time=10:00:00 +#SBATCH --output=%x_%j.out +#SBATCH --error=%x_%j.err + +set -xeuo pipefail + +# ================= cluster topology ================= +export GPUS_PER_NODE=${SLURM_GPUS_ON_NODE:-${GPUS_PER_NODE:-2}} # GPUs on this node +NNODES=${SLURM_JOB_NUM_NODES:-${NNODES:-1}} +export NNODES +export RAY_NUM_NODES=$NNODES + +# Require at least 2 GPUs +TOTAL_GPUS=$((GPUS_PER_NODE * NNODES)) +if [ "$TOTAL_GPUS" -lt 2 ]; then + echo "Error: at least 2 GPUs are required, detected $TOTAL_GPUS." >&2 + exit 1 +fi + +echo "Using $NNODES nodes and $GPUS_PER_NODE GPUs per node..." + +# ================= data/model/tool ================= +HDFS_ROOT=${HDFS_ROOT:-$PWD} +DATA_ROOT=${DATA_ROOT:-$PWD} + +# Prefer local model if present, otherwise fall back to HF hub path +model_path=${model_path:-$DATA_ROOT/model/Qwen2.5-3B-Instruct} +if [ ! -d "$model_path" ]; then + model_path=Qwen/Qwen2.5-3B-Instruct +fi + +# Use the default output directory produced by create_dataset.py +train_files=$DATA_ROOT/data/math_expression_tool/train.parquet +test_files=$DATA_ROOT/data/math_expression_tool/test.parquet + +# Agent config +agent_loop_config_path=recipe/langgraph_agent/example/agent.yaml + +# =================== wandb =================== +project_name=math_expression_tool +experiment_name=qwen2.5-3b +default_local_dir=$DATA_ROOT/checkpoint/$experiment_name + +# ================= algorithm ================= +adv_estimator=grpo + +use_kl_in_reward=false +kl_coef=0.0 +use_kl_loss=false +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_turns=8 +max_prompt_length=1024 +max_response_length=2048 +actor_lr=1e-6 + +train_batch_size=128 +ppo_mini_batch_size=16 +n_resp_per_prompt=8 +n_resp_per_prompt_val=1 + +# =================== logging =================== +export RAY_LOGGING_LEVEL=DEBUG +export HYDRA_FULL_ERROR=1 + +# ================= performance ================= +export NCCL_IBEXT_DISABLE=1 +export NCCL_NVLS_ENABLE=1 +export NCCL_IB_HCA=mlx5 +export UCX_NET_DEVICES=mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1 +export VLLM_USE_V1=1 +export VLLM_ATTENTION_BACKEND=FLASH_ATTN + +infer_tp=2 # vLLM tensor parallel size +train_sp=4 # Ulysses sequence parallel size for actor +offload=true + +actor_max_token_len_per_gpu=$(( (max_prompt_length + max_response_length) * 4 )) +log_prob_max_token_len_per_gpu=$(( actor_max_token_len_per_gpu * 2 )) + +train_files="['$train_files']" +test_files="['$test_files']" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=$adv_estimator \ + algorithm.use_kl_in_reward=$use_kl_in_reward \ + algorithm.kl_ctrl.kl_coef=$kl_coef \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.return_raw_chat=true \ + data.train_batch_size=$train_batch_size \ + data.max_prompt_length=$max_prompt_length \ + data.max_response_length=$max_response_length \ + data.filter_overlong_prompts=true \ + data.truncation='error' \ + actor_rollout_ref.model.path="$model_path" \ + actor_rollout_ref.model.use_remove_padding=true \ + actor_rollout_ref.model.enable_gradient_checkpointing=true \ + actor_rollout_ref.actor.use_kl_loss=$use_kl_loss \ + actor_rollout_ref.actor.kl_loss_coef=$kl_loss_coef \ + actor_rollout_ref.actor.clip_ratio_low=$clip_ratio_low \ + actor_rollout_ref.actor.clip_ratio_high=$clip_ratio_high \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.optim.lr=$actor_lr \ + actor_rollout_ref.actor.use_dynamic_bsz=true \ + actor_rollout_ref.actor.ppo_mini_batch_size=$ppo_mini_batch_size \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=$actor_max_token_len_per_gpu \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=$train_sp \ + actor_rollout_ref.actor.fsdp_config.param_offload=$offload \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=$offload \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=$log_prob_max_token_len_per_gpu \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$infer_tp \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=$max_turns \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=$max_turns \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.agent.agent_loop_config_path=$agent_loop_config_path \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.rollout.n=$n_resp_per_prompt \ + actor_rollout_ref.rollout.val_kwargs.top_p=0.6 \ + actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \ + actor_rollout_ref.rollout.val_kwargs.n=$n_resp_per_prompt_val \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=$project_name \ + trainer.experiment_name=$experiment_name \ + trainer.n_gpus_per_node="$GPUS_PER_NODE" \ + trainer.val_before_train=true \ + trainer.log_val_generations=50 \ + trainer.nnodes="$NNODES" \ + trainer.save_freq=-1 \ + trainer.default_local_dir="$default_local_dir" \ + trainer.test_freq=5 \ + trainer.total_epochs=1 "$@" \ No newline at end of file diff --git a/e2c/verl/recipe/langgraph_agent/react_agent_loop.py b/e2c/verl/recipe/langgraph_agent/react_agent_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..fcb6aa00a8c1f70e0014697b44f37060cd69aa83 --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/react_agent_loop.py @@ -0,0 +1,135 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +LangGraph React Agent Loop. + +This implementation is exact same as `ToolAgentLoop`. + +Ref: https://langchain-ai.github.io/langgraph/tutorials/workflows/ +""" + +from typing import Any, Literal + +from langchain_core.runnables import RunnableConfig +from langgraph.graph import END, MessagesState, StateGraph +from langgraph.prebuilt import ToolNode + +from recipe.langgraph_agent.chat_model import ( + ChatModel, + MaxTokenExceededError, + convert_to_agent_output, +) +from verl.experimental.agent_loop.agent_loop import AgentLoopBase, AgentLoopOutput + + +async def call_model(state: MessagesState, config: RunnableConfig): + model = config["configurable"]["model"] + sampling_params = config["configurable"]["sampling_params"] + try: + message = await model.ainvoke(state["messages"], sampling_params=sampling_params) + return {"messages": [message]} + except MaxTokenExceededError: + # last message is ToolMessage + return {"messages": []} + + +def should_continue(state: MessagesState, config: RunnableConfig) -> Literal["tools", END]: + max_assistant_turns = config["configurable"]["max_assistant_turns"] + num_assistant_turns = 0 + for message in state["messages"]: + if message.type == "ai": + num_assistant_turns += 1 + + last_message = state["messages"][-1] + + # LLM call failed, e.g: max response length exceeded + if last_message.type == "tool": + return END + + # max assistant turns exceeded + if max_assistant_turns and num_assistant_turns >= max_assistant_turns: + return END + + # no tool calls + if not last_message.tool_calls: + return END + + return "tools" + + +class ReactAgentLoop(AgentLoopBase): + @classmethod + def init_class(cls, config, tokenizer, **kwargs): + if cls._class_initialized: + return + cls._class_initialized = True + print("Performing class-level ReactAgentLoop initialization") + + # build graph + cls.graph = cls.build_graph() + + @classmethod + def build_graph(cls) -> StateGraph: + workflow = StateGraph(MessagesState) + + workflow.add_node("agent", call_model) + workflow.add_node("tools", ToolNode(cls.tools)) + workflow.set_entry_point("agent") + workflow.add_conditional_edges( + "agent", + should_continue, + { + "tools": "tools", + END: END, + }, + ) + + workflow.add_edge("tools", "agent") + graph = workflow.compile() + return graph + + async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput: + messages = list(kwargs["raw_prompt"]) + + model_path = self.config.actor_rollout_ref.model.path + model_name = "/".join(model_path.split("/")[-2:]) + + rollout = self.config.actor_rollout_ref.rollout + model = ChatModel( + model=model_name, + client=self.server_manager, + tokenizer=self.tokenizer, + max_tokens=rollout.response_length, + max_parallel_calls=rollout.multi_turn.max_parallel_calls, + tool_parser=rollout.multi_turn.format, + ) + + model = model.bind_tools(self.tools, tool_choice="any") + + config = { + "configurable": { + "model": model, + "sampling_params": sampling_params, + "max_user_turns": rollout.multi_turn.max_user_turns, + "max_assistant_turns": rollout.multi_turn.max_assistant_turns, + } + } + + # TODO: how to handle multiple trajectories in an graph invocation? + # Each graph node may has its own LLM calls and state, e.g: + # https://github.com/google-gemini/gemini-fullstack-langgraph-quickstart + state = await self.graph.ainvoke(input={"messages": messages}, config=config) + + output = convert_to_agent_output(state["messages"], rollout.response_length) + return output diff --git a/e2c/verl/recipe/langgraph_agent/test_react_agent_loop.py b/e2c/verl/recipe/langgraph_agent/test_react_agent_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..88509cc4819706a116e38e340939307bc94a3c62 --- /dev/null +++ b/e2c/verl/recipe/langgraph_agent/test_react_agent_loop.py @@ -0,0 +1,202 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os + +import numpy as np +import pytest +import ray +from langchain_core.tools import tool +from omegaconf import DictConfig + +from recipe.langgraph_agent.react_agent_loop import ReactAgentLoop +from tests.experimental.agent_loop.agent_utils import init_agent_loop_manager +from verl.protocol import DataProto +from verl.utils import hf_tokenizer + + +@pytest.fixture +def init_config() -> DictConfig: + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + config = compose(config_name="ppo_trainer") + model_path = "Qwen/Qwen2.5-1.5B-Instruct" + config.actor_rollout_ref.model.path = model_path + config.actor_rollout_ref.rollout.name = os.getenv("ROLLOUT_NAME", "vllm") + config.actor_rollout_ref.rollout.mode = "async" + config.actor_rollout_ref.rollout.prompt_length = 4096 + config.actor_rollout_ref.rollout.response_length = 4096 + config.actor_rollout_ref.rollout.n = 4 + config.actor_rollout_ref.rollout.agent.num_workers = 2 + + config.actor_rollout_ref.actor.use_dynamic_bsz = True + # test sleep/wake_up with fsdp offload + config.actor_rollout_ref.actor.fsdp_config.param_offload = True + config.actor_rollout_ref.actor.fsdp_config.optimizer_offload = True + + return config + + +@tool(parse_docstring=True) +def get_current_temperature(location: str, unit: str = "celsius"): + """Get current temperature at a location. + + Args: + location: The location to get the temperature for, in the format "City, State, Country". + unit: The unit to return the temperature in. Defaults to "celsius". (choices: ["celsius", "fahrenheit"]) + + Returns: + the temperature, the location, and the unit in a dict + """ + print(f"[DEBUG] get_current_temperature: {location}, {unit}") + return { + "temperature": 26.1, + "location": location, + "unit": unit, + } + + +@tool(parse_docstring=True) +def get_temperature_date(location: str, date: str, unit: str = "celsius"): + """Get temperature at a location and date. + + Args: + location: The location to get the temperature for, in the format "City, State, Country". + date: The date to get the temperature for, in the format "Year-Month-Day". + unit: The unit to return the temperature in. Defaults to "celsius". (choices: ["celsius", "fahrenheit"]) + + Returns: + the temperature, the location, the date and the unit in a dict + """ + print(f"[DEBUG] get_temperature_date: {location}, {date}, {unit}") + return { + "temperature": 25.9, + "location": location, + "date": date, + "unit": unit, + } + + +class TestReactAgentLoop(ReactAgentLoop): + @classmethod + def init_class(cls, config, tokenizer, **kwargs): + # TODO: find better way to configure tools + cls.tools = [get_current_temperature, get_temperature_date] + super().init_class(config, tokenizer, **kwargs) + + +def test_react_agent(init_config): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + } + ) + + # =========================== 1. Init rollout manager =========================== + agent_loop_config = [ + { + "_target_": "recipe.langgraph_agent.test_react_agent_loop.TestReactAgentLoop", + "name": "react_agent", + }, + ] + agent_loop_config_path = "/tmp/agent_loop_config.json" + with open(agent_loop_config_path, "w") as f: + json.dump(agent_loop_config, f) + + n = 2 + init_config.actor_rollout_ref.rollout.n = n + # init_config.actor_rollout_ref.rollout.multi_turn.tool_config_path = tool_config_path + init_config.actor_rollout_ref.rollout.multi_turn.max_parallel_calls = 2 + init_config.actor_rollout_ref.rollout.agent.agent_loop_config_path = agent_loop_config_path + agent_loop_manager = init_agent_loop_manager(init_config) + + # =========================== 2. Generate sequences =========================== + raw_prompts = [ + [ + {"role": "user", "content": "How are you?"}, + ], + [ + {"role": "user", "content": "What's the temperature in Los Angeles now?"}, + ], + [ + {"role": "user", "content": "What's the temperature in New York now?"}, + ], + [ + { + "role": "system", + "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.\n\n" + "Current Date: 2024-09-30", + }, + {"role": "user", "content": "What's the temperature in San Francisco now? How about tomorrow?"}, + ], + ] + batch = DataProto( + non_tensor_batch={ + "raw_prompt": np.array([np.array(prompt) for prompt in raw_prompts], dtype=object), + "agent_name": np.array(["react_agent"] * len(raw_prompts)), + "data_source": np.array(["openai/gsm8k"] * len(raw_prompts)), + "reward_model": np.array([{"style": "rule", "ground_truth": "1.0"}] * len(raw_prompts)), + }, + ) + batch = batch.repeat(n) + result = agent_loop_manager.generate_sequences(prompts=batch) + assert len(result) == len(raw_prompts) * n + + # Check turns + num_turns = result.non_tensor_batch["__num_turns__"] + print(f"num_turns: {num_turns}") + for i in range(len(num_turns)): + if i // n == 0: + # [user, assistant] + assert num_turns[i] == 2 + else: + # [user, assistant, tool, assistant] + assert num_turns[i] == 4 + + # Check response_mask + tokenizer = hf_tokenizer(init_config.actor_rollout_ref.model.path) + responses = result.batch["responses"] + response_mask = result.batch["response_mask"] + attention_mask = result.batch["attention_mask"] + assert responses.size() == response_mask.size(), f"{responses.size()} != {response_mask.size()}" + response_length = response_mask.size(1) + + for i in range(len(responses)): + # response with tool response + valid_tokens = responses[i][attention_mask[i][-response_length:].bool()] + response_with_obs = tokenizer.decode(valid_tokens) + + # response without tool response + valid_tokens = responses[i][response_mask[i].bool()] + response_without_obs = tokenizer.decode(valid_tokens) + + assert "" not in response_without_obs, ( + f"found in response: {response_without_obs}" + ) + assert "" not in response_without_obs, ( + f"found in response: {response_without_obs}" + ) + print("=========================") + print(response_with_obs) + print("---") + print(response_without_obs) + + print("Test passed!") + ray.shutdown() diff --git a/e2c/verl/recipe/minicpmo/rl_dataset.py b/e2c/verl/recipe/minicpmo/rl_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..5f6eb1b3f0cb893ca173902f76fb8aa51c1b7714 --- /dev/null +++ b/e2c/verl/recipe/minicpmo/rl_dataset.py @@ -0,0 +1,571 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import logging +import math +import os +import re +from typing import Optional + +import datasets +import torch +from omegaconf import DictConfig, ListConfig +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms +from transformers import PreTrainedTokenizer, ProcessorMixin + +import verl.utils.torch_functional as verl_F +from verl.utils.dataset.vision_utils import process_image +from verl.utils.model import compute_position_id_with_mask + +logger = logging.getLogger(__name__) + + +def build_transform(): + IMAGENET_INCEPTION_MEAN = (0.5, 0.5, 0.5) # timm.data.IMAGENET_INCEPTION_MEAN + IMAGENET_INCEPTION_STD = (0.5, 0.5, 0.5) # timm.data.IMAGENET_INCEPTION_STD + return transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize(mean=IMAGENET_INCEPTION_MEAN, std=IMAGENET_INCEPTION_STD), + ] + ) + + +def build_image_bound(input_ids, tokenizer, new_schema=True, logger=None): + if new_schema: + start_cond = (input_ids == tokenizer.im_start_id) | (input_ids == tokenizer.slice_start_id) + end_cond = (input_ids == tokenizer.im_end_id) | (input_ids == tokenizer.slice_end_id) + else: + start_cond = input_ids == tokenizer.im_start_id + end_cond = input_ids == tokenizer.im_end_id + image_start_tokens = torch.where(start_cond)[0] + image_start_tokens += 1 + image_end_tokens = torch.where(end_cond)[0] + if len(image_start_tokens) != len(image_end_tokens): + logger.error("image start token != image end tokens") + raise Exception("image start token != image end tokens") + if len(image_start_tokens) > 0: + image_bound = torch.hstack([image_start_tokens.unsqueeze(-1), image_end_tokens.unsqueeze(-1)]) + else: + image_bound = [] + return image_bound + + +def preprocess( + images_dict, + conversations, + tokenizer, + transform, + query_nums=64, + slice_config=None, + llm_type=None, + patch_size=14, + batch_vision=False, + max_length=2048, + truncation="error", + apply_chat_template_kwargs=None, + logger=None, +): + """ + single(multi) image(s) preprocess, the image(s) will be placed at the top of the conversation + """ + conversations = copy.deepcopy(conversations) + assert conversations[0]["role"] == "user", "the first role must be user" + + if slice_config is not None: + assert isinstance(slice_config, dict) + assert "patch_size" in slice_config + assert "max_slice_nums" in slice_config + assert "scale_resolution" in slice_config + default_image_placeholder = tokenizer.im_start + tokenizer.unk_token * query_nums + tokenizer.im_end + new_schema = False + use_image_id = False + if llm_type == "qwen": + new_schema = True + use_image_id = True + image_placeholder_dict = {} + images = [] + image_id_cnt = 0 + for img_name, image in images_dict.items(): + if slice_config: + source_image, patches, best_grid = slice_image( + image, + slice_config["max_slice_nums"], + slice_config["scale_resolution"], + slice_config["patch_size"], + ) + images.append(source_image) + image_placeholder = default_image_placeholder + if len(patches) > 0: + for i in range(len(patches)): + for j in range(len(patches[0])): + images.append(patches[i][j]) + if use_image_id: + image_placeholder = ( + f"{tokenizer.im_id_start}{image_id_cnt}{tokenizer.im_id_end}" + image_placeholder + ) + image_id_cnt += 1 + image_placeholder += get_grid_placeholder(tokenizer, best_grid, query_nums, new_schema=new_schema) + image_placeholder_dict[img_name] = image_placeholder + else: + images.append(image) + if use_image_id: + image_placeholder = f"{tokenizer.im_id_start}{image_id_cnt}{tokenizer.im_id_end}" + image_placeholder + image_id_cnt += 1 + else: + image_placeholder = default_image_placeholder + image_placeholder_dict[img_name] = image_placeholder + + images = [transform(i) for i in images] + + if len(images_dict) == 1 and "" in images_dict: + if "" in conversations[0]["content"]: + conversations[0]["content"] = conversations[0]["content"].replace("", image_placeholder) + else: + conversations[0]["content"] = image_placeholder + "\n" + conversations[0]["content"] + else: + pattern = r"" + new_conversations = [] + for conversation in conversations: + content = conversation["content"] + parts = re.split(f"({pattern})", content) + for i, part in enumerate(parts): + if not part.strip(): + continue + if re.match(pattern, part): + if part in image_placeholder_dict: + parts[i] = image_placeholder_dict[part] + else: + raise Exception(f"not found {part} in image dict") + conversation["content"] = "\n".join(parts) + new_conversations.append(conversation) + conversations = new_conversations + + # TODO change role in conversation for different llm + prompt_with_chat_template = tokenizer.apply_chat_template( + conversations, add_generation_prompt=True, tokenize=False, **(apply_chat_template_kwargs or {}) + ) + + input_ids, attention_mask = verl_F.tokenize_and_postprocess_data( + prompt=prompt_with_chat_template, + tokenizer=tokenizer, + max_length=max_length, + pad_token_id=tokenizer.pad_token_id, + left_pad=True, + truncation=truncation, + ) + position_ids = compute_position_id_with_mask(attention_mask) + image_bound = build_image_bound(input_ids[0], tokenizer, new_schema, logger) + + input_dict = { + "input_ids": input_ids[0], + "attention_mask": attention_mask[0], + "position_ids": position_ids[0], + "image_bound": image_bound, + } + + if batch_vision: + tgt_sizes = [] + reshape_images = [] + for image in images: + H, W = image.shape[1:] + reshape_image = reshape_by_patch(image, patch_size) + reshape_images.append(reshape_image) + tgt_sizes.append([H // patch_size, W // patch_size]) + if tgt_sizes: + tgt_sizes = torch.Tensor(tgt_sizes).type(torch.int32) + + input_dict["pixel_values"] = reshape_images + input_dict["tgt_sizes"] = tgt_sizes + + else: + input_dict["pixel_values"] = images + input_dict["tgt_sizes"] = [] + + return input_dict + + +def slice_image(image, max_slice_nums=9, scale_resolution=448, patch_size=14, never_split=False): + original_size = image.size + original_width, original_height = original_size + log_ratio = math.log(original_width / original_height) + ratio = original_width * original_height / (scale_resolution * scale_resolution) + multiple = min(math.ceil(ratio), max_slice_nums) + + source_image = None + best_grid = None + patches = [] + + if multiple <= 1 or never_split: + # dont need to slice, upsample + best_size = find_best_resize(original_size, scale_resolution, patch_size, allow_upscale=True) + source_image = image.resize(best_size, Image.Resampling.BICUBIC) + else: + candidate_split_grids_nums = [] + for i in [multiple - 1, multiple, multiple + 1]: + if i == 1 or i > max_slice_nums: + continue + candidate_split_grids_nums.append(i) + + # source image, down-sampling and ensure divided by patch_size + best_resize = find_best_resize(original_size, scale_resolution, patch_size) + source_image = image.copy().resize(best_resize, Image.Resampling.BICUBIC) + candidate_grids = [] + + # find best grid + for split_grids_nums in candidate_split_grids_nums: + m = 1 + while m <= split_grids_nums: + if split_grids_nums % m == 0: + candidate_grids.append([m, split_grids_nums // m]) + m += 1 + + best_grid = [1, 1] + min_error = float("inf") + for grid in candidate_grids: + error = abs(log_ratio - math.log(grid[0] / grid[1])) + if error < min_error: + best_grid = grid + min_error = error + + refine_size = get_refine_size(original_size, best_grid, scale_resolution, patch_size, allow_upscale=True) + + refine_image = image.resize(refine_size, Image.Resampling.BICUBIC) + patches = split_to_patches(refine_image, best_grid) + + return source_image, patches, best_grid + + +def ensure_divide(length, patch_size): + return max(round(length / patch_size) * patch_size, patch_size) + + +def find_best_resize(original_size, scale_resolution, patch_size, allow_upscale=False): + width, height = original_size + if (width * height > scale_resolution * scale_resolution) or allow_upscale: + r = width / height + height = int(scale_resolution / math.sqrt(r)) + width = int(height * r) + best_width = ensure_divide(width, patch_size) + best_height = ensure_divide(height, patch_size) + return (best_width, best_height) + + +def get_refine_size(original_size, grid, scale_resolution, patch_size, allow_upscale=False): + width, height = original_size + grid_x, grid_y = grid + + refine_width = ensure_divide(width, grid_x) + refine_height = ensure_divide(height, grid_y) + + grid_width = refine_width / grid_x + grid_height = refine_height / grid_y + + best_grid_size = find_best_resize( + (grid_width, grid_height), + scale_resolution, + patch_size, + allow_upscale=allow_upscale, + ) + + refine_size = (best_grid_size[0] * grid_x, best_grid_size[1] * grid_y) + + return refine_size + + +def split_to_patches(image, grid): + patches = [] + width, height = image.size + grid_x = int(width / grid[0]) + grid_y = int(height / grid[1]) + + for i in range(0, height, grid_y): + images = [] + for j in range(0, width, grid_x): + box = (j, i, j + grid_x, i + grid_y) + patch = image.crop(box) + images.append(patch) + patches.append(images) + + return patches + + +def get_grid_placeholder(tokenizer, grid, query_num, new_schema=False): + if new_schema: + image_placeholder = tokenizer.slice_start + tokenizer.unk_token * query_num + tokenizer.slice_end + else: + image_placeholder = tokenizer.im_start + tokenizer.unk_token * query_num + tokenizer.im_end + + cols = grid[0] + rows = grid[1] + slices = [] + for i in range(rows): + lines = [] + for j in range(cols): + lines.append(image_placeholder) + slices.append("".join(lines)) + if new_schema: + slice_placeholder = "\n".join(slices) + else: + slice_placeholder = tokenizer.slice_start + "\n".join(slices) + tokenizer.slice_end + return slice_placeholder + + +def reshape_by_patch(image_tensor, patch_size): + """ + :param image_tensor: shape [3, H, W] + :param patch_size: + :return: [3, patch_size, HW/patch_size] + """ + patches = torch.nn.functional.unfold(image_tensor, (patch_size, patch_size), stride=(patch_size, patch_size)) + + patches = patches.reshape(image_tensor.size(0), patch_size, patch_size, -1) + patches = patches.permute(0, 1, 3, 2).reshape(image_tensor.size(0), patch_size, -1) + return patches + + +def init_minicpmo_config(processor, config): + """Initialize MiniCPM-o specific configuration""" + minicpmo_config = { + "transform": build_transform(), + "patch_size": config.get("patch_size", 14), + "query_nums": config.get("query_nums", 64), + "slice_config": config.get( + "slice_config", {"max_slice_nums": 9, "patch_size": config.get("patch_size", 14), "scale_resolution": 448} + ), + "llm_type": config.get("llm_type", "qwen"), + "batch_vision": config.get("batch_vision", True), + } + return minicpmo_config + + +def process_minicpmo_data( + row_dict, + messages, + tokenizer, + minicpmo_config, + image_key, + max_prompt_length, + truncation, + apply_chat_template_kwargs, + logger, +): + """Process data for MiniCPM-o model""" + if len(row_dict[image_key]) == 1: + multi_modal_data = {} + image = process_image(row_dict.pop(image_key)[0]) + multi_modal_data["image"] = [image] + images_dict = {"": image} + else: + raise NotImplementedError + + model_inputs = preprocess( + images_dict, + messages, + tokenizer, + minicpmo_config["transform"], + query_nums=minicpmo_config["query_nums"], + slice_config=minicpmo_config["slice_config"], + llm_type=minicpmo_config["llm_type"], + patch_size=minicpmo_config["patch_size"], + batch_vision=minicpmo_config["batch_vision"], + max_length=max_prompt_length, + truncation=truncation, + apply_chat_template_kwargs=apply_chat_template_kwargs, + logger=logger, + ) + + raw_prompt = tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=False, **(apply_chat_template_kwargs or {}) + ) + raw_prompt = raw_prompt.replace("", "(./)") + + return model_inputs, multi_modal_data, raw_prompt + + +class RLHFDataset(Dataset): + """ + Load and preprocess RLHF data from Parquet files. + + - Caches files locally. + - Reads into a HuggingFace Dataset and tokenizes prompts. + - Optionally handles images/videos via a ProcessorMixin. + - Filters prompts over a max length. + - Supports resuming from checkpoints. + + Args: + data_files (str or list): Path(s) to Parquet file(s). + tokenizer (PreTrainedTokenizer): For the tokenization of text to token IDs. + config (DictConfig): Options like cache_dir, prompt_key, max_prompt_length, truncation, etc. + processor (ProcessorMixin, optional): Multimodal preprocessor for images/videos. + """ + + def __init__( + self, + data_files: str | list[str], + tokenizer: PreTrainedTokenizer, + config: DictConfig, + processor: Optional[ProcessorMixin] = None, + ): + if not isinstance(data_files, list | ListConfig): + data_files = [data_files] + + self.data_files = copy.deepcopy(data_files) + self.original_data_files = copy.deepcopy(data_files) # use for resume + self.tokenizer = tokenizer + self.processor = processor + self.config = config + + self.cache_dir = os.path.expanduser(config.get("cache_dir", "~/.cache/verl/rlhf")) + self.prompt_key = config.get("prompt_key", "prompt") + self.image_key = config.get("image_key", "images") + self.video_key = config.get("video_key", "videos") + self.max_prompt_length = config.get("max_prompt_length", 1024) + self.return_raw_chat = config.get("return_raw_chat", False) + self.return_full_prompt = config.get("return_full_prompt", False) + self.truncation = config.get("truncation", "error") + self.filter_overlong_prompts = config.get("filter_overlong_prompts", True) + self.apply_chat_template_kwargs = config.get("apply_chat_template_kwargs", {}) + + self.num_workers = config.get("filter_overlong_prompts_workers", max(1, os.cpu_count() // 4)) + self.num_workers = min(self.num_workers, os.cpu_count()) + self.use_shm = config.get("use_shm", False) + self.chat_template_func = config.get("chat_template_func", None) + self.need_tools_kwargs = config.get("need_tools_kwargs", False) + self.filter_prompts = config.get("filter_prompts", True) + self.serialize_dataset = False + self.minicpmo_config = init_minicpmo_config(self.processor, config) + self._download() + self._read_files_and_tokenize() + + def _download(self, use_origin_parquet=False): + from verl.utils.fs import copy_to_local + + data_files = self.data_files if not use_origin_parquet else self.original_data_files + for i, parquet_file in enumerate(data_files): + self.data_files[i] = copy_to_local(src=parquet_file, cache_dir=self.cache_dir, use_shm=self.use_shm) + + def _read_files_and_tokenize(self): + dataframes = [] + for parquet_file in self.data_files: + # read parquet files and cache + dataframe = datasets.load_dataset("parquet", data_files=parquet_file)["train"] + dataframes.append(dataframe) + self.dataframe: datasets.Dataset = datasets.concatenate_datasets(dataframes) + + print(f"dataset len: {len(self.dataframe)}") + + def resume_dataset_state(self): + self.serialize_dataset = not hasattr(self, "original_data_files") + # resume dataframe if not it's serialized in data.pt + if not self.serialize_dataset: + self._download(use_origin_parquet=True) # download and resume from original parquet files + self._read_files_and_tokenize() + else: + print(r"old dataloader ckpt file is used, please train from scratch for better ckpt performance") + + def __len__(self): + return len(self.dataframe) + + def _build_messages(self, example: dict): + return example.pop(self.prompt_key) + + def __getitem__(self, item): + """ + Note that we also return the raw_input_ids so that it can be combined with other chat template + """ + row_dict: dict = self.dataframe[item] + messages = self._build_messages(row_dict) + model_inputs = {} + + if self.processor is not None: + model_inputs, multi_modal_data, raw_prompt = process_minicpmo_data( + row_dict, + messages, + self.tokenizer, + self.minicpmo_config, + self.image_key, + self.max_prompt_length, + self.truncation, + self.apply_chat_template_kwargs, + logger, + ) + input_ids = model_inputs.pop("input_ids") + attention_mask = model_inputs.pop("attention_mask") + position_ids = model_inputs.pop("position_ids") + + # There's a trap here, multi_modal_inputs has to be a dict, not BatchFeature + row_dict["multi_modal_data"] = multi_modal_data + row_dict["multi_modal_inputs"] = dict(model_inputs) + else: + raw_prompt = self.tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=False, **self.apply_chat_template_kwargs + ) + model_inputs = self.tokenizer(raw_prompt, return_tensors="pt", add_special_tokens=False) + input_ids = model_inputs.pop("input_ids") + attention_mask = model_inputs.pop("attention_mask") + position_ids = compute_position_id_with_mask(attention_mask) + + row_dict["input_ids"] = input_ids + row_dict["attention_mask"] = attention_mask + row_dict["position_ids"] = position_ids + + raw_prompt_ids = self.tokenizer.encode(raw_prompt, add_special_tokens=False) + if len(raw_prompt_ids) > self.max_prompt_length: + if self.truncation == "left": + raw_prompt_ids = raw_prompt_ids[-self.max_prompt_length :] + elif self.truncation == "right": + raw_prompt_ids = raw_prompt_ids[: self.max_prompt_length] + elif self.truncation == "middle": + left_half = self.max_prompt_length // 2 + right_half = self.max_prompt_length - left_half + raw_prompt_ids = raw_prompt_ids[:left_half] + raw_prompt_ids[-right_half:] + elif self.truncation == "error": + raise RuntimeError(f"Prompt length {len(raw_prompt_ids)} is longer than {self.max_prompt_length}.") + + row_dict["raw_prompt_ids"] = raw_prompt_ids + # encode prompts without chat template + if self.return_raw_chat: + row_dict["raw_prompt"] = messages + + # get prompts with chat template + if self.return_full_prompt: + row_dict["full_prompts"] = raw_prompt # array of strings + + # add index for each prompt + index = row_dict.get("extra_info", {}).get("index", 0) + tools_kwargs = row_dict.get("extra_info", {}).get("tools_kwargs", {}) + interaction_kwargs = row_dict.get("extra_info", {}).get("interaction_kwargs", {}) + need_tools_kwargs = row_dict.get("extra_info", {}).get("need_tools_kwargs", self.need_tools_kwargs) + if need_tools_kwargs and not tools_kwargs: + logger.warning("tools_kwargs is empty for index {}, data source: {}", index, row_dict["data_source"]) + row_dict["index"] = index + row_dict["tools_kwargs"] = tools_kwargs + row_dict["interaction_kwargs"] = interaction_kwargs + return row_dict + + def __getstate__(self): + if not self.serialize_dataset: + state = self.__dict__.copy() + + if "dataframe" in state: + del state["dataframe"] + return state + + return self.__dict__.copy() diff --git a/e2c/verl/recipe/one_step_off_policy/README.md b/e2c/verl/recipe/one_step_off_policy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e9ac98deaeb63abc3b7601102246b3dae2c991d7 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/README.md @@ -0,0 +1,298 @@ +# Recipe: One Step Off Policy Async Trainer + +**Author:** `https://github.com/meituan-search` + +Last updated: 07/17/2025. + +## Introduction + +### Background + +The current reinforcement learning training process implemented by verl is synchronous, adhering to the algorithmic +workflows of established methods like PPO, GRPO, and DAPO. In each step, training samples are generated by the latest +model, and the model is updated after training completes. While this approach aligns with off-policy reinforcement +learning and stabilizes RL training, but it suffers from severe efficiency issues. +Model updates must wait for the longest output in the generation phase to complete. +During the generation of long-tail samples, GPUs remain idle, resulting in significant underutilization. +The more severe the long-tail problem in sample generation, the lower the overall training efficiency. +For example, in DAPO 32B training, the Rollout phase accounts for approximately 70% of the total time, +and increasing resources does not reduce the Rollout duration. + +![DAPO 32B Math Performance]( +https://raw.githubusercontent.com/eric-haibin-lin/verl-community/refs/heads/main/docs/dapo_32b_math.png) +> source data: https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/workspace?nw=nwusertongyuxuan361 + +### Solution + +We have implemented the **One Step Off Async Trainer** to help alleviate this issue. This approach parallelizes the +generation and training processes, utilizing samples generated in the previous step for current training. +It also involves appropriately partitioning resources, allocating dedicated resources for generation while automatically +assigning the remainder to training. By reducing resources allocated to the generation phase, we mitigate GPU idle time +during long-tail sample generation. Throughout this process, generation and training parameters maintain a one-step off +policy. + +![One Step Off Policy Diagram]( +https://raw.githubusercontent.com/eric-haibin-lin/verl-community/refs/heads/main/docs/one_step_off_policy.png) +> reference: [AReaL: A Large-Scale Asynchronous Reinforcement Learning System for Language Reasoning]( +> https://arxiv.org/abs/2505.24298) +> original work: [Asynchronous RLHF: Faster and More Efficient Off-Policy RL for Language Models](https://arxiv.org/abs/2410.18252) + +Our core contributions include: + +1. **Parallel Generation and Training**: + Samples for the next batch are asynchronously generated while the current batch is being trained. + +2. **Resource Isolation**: + Unlike `hybrid_engine`, this method requires explicit resource allocation for rollout, with remaining resources + automatically assigned to training. + +3. **NCCL Parameter Synchronization**: + Employs NCCL communication primitives for seamless parameter transfer between generation and training modules. + +### Experimental Results + +- **Machine Configuration**: 2 nodes with 16 H20 GPUs each + - Generation: 4 GPUs + - Training: 12 GPUs +- **Model**: Qwen2.5-Math-7B +- **Rollout Configuration**: +- **Max Response Length**: FSDP2: 20,480 tokens; Megatron: 8,192 tokens +- **Algorithm**: DAPO +- **Rollout Engine**: vLLM + +| training mode | engine | step | gen | wait_prev_gen | generate_sequences | old_log_prob | update_actor | total time | acc/best@32/mean | acc/maj@32/mean | +|------------------------|---------------|------|-----|---------------|--------------------|--------------|--------------|---------------|------------------|-----------------| +| colocate sync | VLLM+FSDP2 | 749 | 321 | - | 247 | 88 | 286 | 19h18m | 0.5948 | 0.417 | +| one-step-overlap async | VLLM+FSDP2 | 520 | - | 45 | 458 | 108 | 337 | 15h34m(+23%) | 0.6165 | 0.494 | +| colocate sync | VLLM+Megatron | 699 | 207 | - | 162 | 119 | 344 | 18h21m | 0.605 | 0.4217 | +| one-step-overlap async | VLLM+Megatron | 566 | - | 59 | 501 | 120 | 347 | 13h06m (+40%) | 0.6569 | 0.4038 | + +* colocate sync: step ≈ gen + old_log_prob + update_actor +* one-step-overlap async: step ≈ wait_prev_gen + old_log_prob + update_actor + +![One Step Off Megatron Performance]( +https://raw.githubusercontent.com/eric-haibin-lin/verl-community/refs/heads/main/docs/one_step_off_megatron.png) + +> source data: https://wandb.ai/hou-zg-meituan/one-step-off-policy?nw=nwuserhouzg + +## Implementation + +### One Step Off Policy Async Pipline + +Our implemented **One Step Off Policy Async Pipeline** integrates seamlessly into existing training logic at minimal +cost, +eliminating the need for additional sample storage management. The core mechanism uses `async_gen_next_batch` +for asynchronous rollout generation while maintaining continuous operation during epoch transitions +via `create_continuous_iterator`. + +```python +# iterator generator, simplify one-step integration of the training process +def _create_continuous_iterator(self): + for epoch in range(self.config.trainer.total_epochs): + iterator = iter(self.train_dataloader) + for batch_dict in iterator: + yield epoch, batch_dict + + +# read next batch samples, parameters sync and launch asyn gen_seq +def _async_gen_next_batch(self, continuous_iterator): + # read train_data + try: + epoch, batch_dict = next(continuous_iterator) + except StopIteration: + return None + batch = DataProto.from_single_dict(batch_dict) + gen_batch = batch_pocess(batch) + # sync weights from actor to rollout + self.sync_rollout_weights() + # async generation + gen_batch_output = self.rollout_wg.async_generate_sequences(gen_batch) + # future encapsulated + return GenerationBatchFuture(epoch, batch, gen_batch_output) + + +continuous_iterator = self._create_continuous_iterator() +# run rollout first to achieve one-step-off +batch_data_future = self._async_gen_next_batch(continuous_iterator) + +while batch_data_future is not None: + # wait for the gen_seq result from the previous step + batch = batch_data_future.get() + # launch the next async call to generate sequences + batch_data_future = self._async_gen_next_batch(continuous_iterator) + + # compute advantages + batch = critic.compute_values(batch) + batch = reference.compute_log_prob(batch) + batch = reward.compute_reward(batch) + batch = compute_advantages(batch) + + # model update + critic_metrics = critic.update_critic(batch) + actor_metrics = actor.update_actor(batch) +``` + +### Parameter Synchronization + +The exciting point is that our nccl based weights updating for rollout model has great performance. +At most of time, the latency is under 300ms, which is negligible for RLHF. + +> **sync_rollout_weights**:The time for synchronizing parameters from actor to rollout is extremely fast and can almost +> be ignored because it is implemented with nccl. + +```python +class ActorRolloutRefWorker: + # actor acquires the meta-info of model parameters for parameter sync + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def get_actor_weights_info(self): + params = self._get_actor_params() + ret = [] + for key, tensor in params.items(): + ret.append((key, tensor.size(), tensor.dtype)) + self._weights_info = ret + return ret + + # rollout sets the meta-info of model parameters for parameter sync + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_actor_weights_info(self, weights_info): + self._weights_info = weights_info + + +class AsyncRayPPOTrainer(RayPPOTrainer): + def init_workers(self): + + +... +# rollout obtains the meta-info of model parameters from the actor for parameter sync +weights_info = self.actor_wg.get_actor_weights_info()[0] +self.rollout_wg.set_actor_weights_info(weights_info) + +# Create an actor-rollout communication group for parameter sync +actor_rollout_workers = self.actor_wg.workers + self.rollout_wg.workers +collective.create_collective_group( + actor_rollout_workers, + len(actor_rollout_workers), + list(range(0, len(actor_rollout_workers))), + backend="nccl", + group_name="actor_rollout" +) +``` + +```python +# drive process call the actor and rollout respectively to sync parameters by nccl +def sync_rollout_weights(self): + self.actor_wg.sync_rollout_weights() + ray.get(self.rollout_wg.sync_rollout_weights()) + + +# fsdp model parameter sync +@register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) +def sync_rollout_weights(self): + params = self._get_actor_params() if self._is_actor else None + if self._is_rollout: + inference_model = ( + self.rollout.inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner.model + ) + from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader + patch_vllm_moe_model_weight_loader(inference_model) + # Model parameters are broadcast tensor-by-tensor from actor to rollout + for key, shape, dtype in self._weights_info: + tensor = torch.empty(shape, dtype=dtype, device=get_torch_device().current_device()) + if self._is_actor: + assert key in params + origin_data = params[key] + if hasattr(origin_data, "full_tensor"): + origin_data = origin_data.full_tensor() + if torch.distributed.get_rank() == 0: + tensor.copy_(origin_data) + from ray.util.collective import collective + + collective.broadcast(tensor, src_rank=0, group_name="actor_rollout") + if self._is_rollout: + inference_model.load_weights([(key, tensor)]) +``` + +## Usage + +### FSDP2 Configuration Example + +```shell +python3 -m recipe.one_step_off_policy.async_main_ppo \ + --config-path=config \ + --config-name='one_step_off_ppo_trainer.yaml' \ + actor_rollout_ref.actor.strategy=fsdp2 \ + # actor and rollout are placed separately + actor_rollout_ref.hybrid_engine=False \ + # actor and rollout resource + trainer.nnodes=1 \ + trainer.n_gpus_per_node=6 \ + rollout.nnodes=1 \ + rollout.n_gpus_per_node=2 +``` + +### Megatron Configuration Example + +```shell +python3 -m recipe.one_step_off_policy.async_main_ppo \ + --config-path=config \ + --config-name='one_step_off_ppo_megatron_trainer.yaml' \ + actor_rollout_ref.actor.strategy=megatron \ + # actor and rollout are placed separately + actor_rollout_ref.hybrid_engine=False \ + # actor and rollout resource + trainer.nnodes=1 \ + trainer.n_gpus_per_node=6 \ + rollout.nnodes=1 \ + rollout.n_gpus_per_node=2 +``` + +### Configuration Guidelines + +1. **Card Number Relationships** + Maintain either of these relationships for optimal batch distribution: + - `actor_rollout_ref.rollout.n` should be an integer divisor of: + `trainer.n_gpus_per_node * trainer.nnodes` + - `actor_rollout_ref.rollout.n * data.train_batch_size` should be evenly divisible by: + `trainer.n_gpus_per_node * trainer.nnodes` + + > Rationale: Ensures training samples can be evenly distributed across training GPUs when using partial resources for + generation. + +2. **Dynamic Resource Tuning** + Adjust `trainer.nnodes` `trainer.n_gpus_per_node` `rollout.nnodes` `rollout.n_gpus_per_node` based on phase + durations: + - **Ideal state**: Rollout and training phases have comparable durations + - **Diagnostic metrics**: + - Monitor `wait_prev_gen` duration + - Analyze `sequence_length` distribution + - **Adjustment strategy**: + - High `wait_prev_gen` + uniform sequence lengths → Increase rollout resources + - High `wait_prev_gen` + long-tail sequences → Optimize stopping criteria (resource increase won't help) + > **wait_prev_gen**:The time consumed waiting for the previous rollout to end (the part that is not fully + overlapped). + **Resource Configuration Strategies:** + - **Resource-constrained scenario**: Optimize resource utilization by adjusting GPU allocation ratios, + keeping the number of nodes equal to allow training and rollout to share nodes; + - Configure `trainer.nnodes = rollout.nnodes` with + `trainer.n_gpus_per_node + rollout.n_gpus_per_node = physical_gpus_per_node`. Control rollout resource + allocation by adjusting `n_gpus_per_node`. + - **Resource-abundant scenario**: Optimize performance by adjusting the number of nodes, + keeping the number of GPUs per node equal to enable independent scaling of training and rollout + parallelism. + - Configure `trainer.n_gpus_per_node = rollout.n_gpus_per_node` and control rollout resource allocation by + adjusting `trainer.nnodes` and `rollout.nnodes`to achieve optimal performance. + > **Note**: The total number of nodes required by the system is not simply `trainer.nnodes + rollout.nnodes`. The + > actual calculation depends on GPU capacity: + > - When `trainer.n_gpus_per_node + rollout.n_gpus_per_node <= physical_gpus_per_node`, + > the required node count is `max(trainer.nnodes, rollout.nnodes)` + > - When `trainer.n_gpus_per_node + rollout.n_gpus_per_node > physical_gpus_per_node`, + > the required node count is `trainer.nnodes + rollout.nnodes` + +## Functional Support + +| Category | Support Situation | +|--------------------|-----------------------------------------------------------------------------------------------------------------| +| train engine | FSDP2
Megatron | +| rollout engine | vLLM | +| AdvantageEstimator | GRPO
GRPO_PASSK
REINFORCE_PLUS_PLUS
RLOO
OPO
REINFORCE_PLUS_PLUS_BASELINE
GPG | +| Reward | all | diff --git a/e2c/verl/recipe/one_step_off_policy/config/one_step_off_ppo_megatron_trainer.yaml b/e2c/verl/recipe/one_step_off_policy/config/one_step_off_ppo_megatron_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6ea846e200351ab20483e817fec96debccf3228 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/config/one_step_off_ppo_megatron_trainer.yaml @@ -0,0 +1,14 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_megatron_trainer + - _self_ + +# config for the rollout (only for resource isolation) +rollout: + # Number of nodes used in the rollout + nnodes: 1 + # Number of GPUs per node + n_gpus_per_node: 8 diff --git a/e2c/verl/recipe/one_step_off_policy/config/one_step_off_ppo_trainer.yaml b/e2c/verl/recipe/one_step_off_policy/config/one_step_off_ppo_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..366d7b0fd72628f73ff5f901fe0b06fbff97f654 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/config/one_step_off_ppo_trainer.yaml @@ -0,0 +1,14 @@ +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +# config for the rollout (only for resource isolation) +rollout: + # Number of nodes used in the rollout + nnodes: 1 + # Number of GPUs per node + n_gpus_per_node: 8 diff --git a/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_fsdp2_4_12.sh b/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_fsdp2_4_12.sh new file mode 100644 index 0000000000000000000000000000000000000000..aaa4b537a3691a5290fd9dbd48357dfce2344aae --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_fsdp2_4_12.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7b-MATH-0527a1-fsdp2-one-step-off-4-12' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=12 +train_prompt_mini_bsz=32 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-2} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} + +n_gpus_rollout=2 +n_gpus_training=$((NGPUS_PER_NODE - n_gpus_rollout)) + +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +ref_offload=True +actor_offload=False +gen_tp=2 +sp_size=4 +fsdp_size=2 + +python3 -m recipe.one_step_off_policy.main_ppo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.strategy=fsdp2 \ + critic.strategy=fsdp2 \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.hybrid_engine=False \ + +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','tensorboard'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=-1 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=100 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 \ + trainer.nnodes="${NNODES}" \ + trainer.n_gpus_per_node="${n_gpus_training}" \ + rollout.nnodes="${NNODES}" \ + rollout.n_gpus_per_node="${n_gpus_rollout}" diff --git a/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_fsdp2_colocate.sh b/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_fsdp2_colocate.sh new file mode 100644 index 0000000000000000000000000000000000000000..617d7a7c8479e68d0804841f4d49c81c96b91d88 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_fsdp2_colocate.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7b-MATH-0527a1-fsdp2-colocate' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=12 +train_prompt_mini_bsz=32 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-2} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=2 +sp_size=4 +fsdp_size=2 + +# reference run wandb: https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl/runs/ow47vvon?nw=nwusertongyuxuan361 + +python3 -m verl.trainer.main_ppo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.strategy=fsdp2 \ + critic.strategy=fsdp2 \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.model.use_remove_padding=True \ + +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','tensorboard'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=-1 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=100 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_megatron_4_12.sh b/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_megatron_4_12.sh new file mode 100644 index 0000000000000000000000000000000000000000..b438b686822b166ef4fc40c9c8d4cf5421e6c47c --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_megatron_4_12.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7b-MATH-0527a1-megatron-one-step-off-4-12' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=12 +train_prompt_mini_bsz=32 + + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-2} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} + +n_gpus_rollout=2 +n_gpus_training=$((NGPUS_PER_NODE - n_gpus_rollout)) + +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +ref_offload=True +actor_offload=False +gen_tp=2 +train_tp=2 +train_pp=2 + +# TODO: support dynamic_bsz for megatron +# actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ +# actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ +# actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + +python3 -m recipe.one_step_off_policy.main_ppo \ + --config-path=config \ + --config-name='one_step_off_ppo_megatron_trainer.yaml' \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.strategy=megatron \ + critic.strategy=megatron \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.hybrid_engine=False \ + +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.megatron.param_offload=${actor_offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${actor_offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${actor_offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.clip_grad=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.param_offload=${ref_offload} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','tensorboard'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=-1 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=100 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 \ + trainer.nnodes="${NNODES}" \ + trainer.n_gpus_per_node="${n_gpus_training}" \ + rollout.nnodes="${NNODES}" \ + rollout.n_gpus_per_node="${n_gpus_rollout}" diff --git a/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_megatron_colocate.sh b/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_megatron_colocate.sh new file mode 100644 index 0000000000000000000000000000000000000000..df0c451e845b9d003305731df28059145a7dca87 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/dapo_7b_math_megatron_colocate.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name='DAPO' +exp_name='DAPO-Qwen2.5-7b-MATH-0519a1-megatron-colocate' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 8)) +enable_overlong_buffer=True +overlong_buffer_len=$((1024 * 4)) +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +train_prompt_bsz=512 +n_resp_per_prompt=16 +train_prompt_mini_bsz=32 + +# Ray +# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} +# WORKING_DIR=${WORKING_DIR:-"${PWD}"} +# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} +NNODES=${NNODES:-2} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout +val_top_p=0.7 + +# Performance Related Parameter +use_dynamic_bsz=True +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3)) +offload=True +gen_tp=2 +train_tp=2 +train_pp=2 + +# TODO: support dynamic_bsz for megatron +# actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ +# actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ +# actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ +# actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + +python3 -m verl.trainer.main_ppo \ + --config-path=config \ + --config-name='ppo_megatron_trainer.yaml' \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.strategy=megatron \ + critic.strategy=megatron \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.clip_grad=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + reward_model.reward_manager=dapo \ + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward_model.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','tensorboard'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=10 \ + trainer.save_freq=-1 \ + trainer.total_epochs=10 \ + trainer.total_training_steps=100 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 diff --git a/e2c/verl/recipe/one_step_off_policy/distributed_util.py b/e2c/verl/recipe/one_step_off_policy/distributed_util.py new file mode 100644 index 0000000000000000000000000000000000000000..75403ae1423bf42ce10d47daf39edb1928a2b455 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/distributed_util.py @@ -0,0 +1,40 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from verl.utils.device import is_npu_available + + +def stateless_init_process_group(master_address, master_port, rank, world_size, device): + """ + vLLM provides `StatelessProcessGroup` to create a process group + without considering the global process group in torch.distributed. + It is recommended to create `StatelessProcessGroup`, and then initialize + the data-plane communication (NCCL) between external (train processes) + and vLLM workers. + """ + # NOTE: If it is necessary to support weight synchronization with the sglang backend in the future, + # the following can be used: + # from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator + # from sglang.srt.distributed.utils import statelessprocessgroup + if is_npu_available: + from vllm_ascend.distributed.device_communicators.pyhccl import ( + PyHcclCommunicator as PyNcclCommunicator, + ) + else: + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + from vllm.distributed.utils import StatelessProcessGroup + + pg = StatelessProcessGroup.create(host=master_address, port=master_port, rank=rank, world_size=world_size) + pynccl = PyNcclCommunicator(pg, device=device) + return pynccl diff --git a/e2c/verl/recipe/one_step_off_policy/fsdp_workers.py b/e2c/verl/recipe/one_step_off_policy/fsdp_workers.py new file mode 100644 index 0000000000000000000000000000000000000000..2c805a6aeb73c1f2be69bd78f9bcf5a5959f6c08 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/fsdp_workers.py @@ -0,0 +1,295 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright 2025 Meituan Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import torch +import torch.distributed +from omegaconf import DictConfig, OmegaConf +from torch.distributed.device_mesh import init_device_mesh +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from transformers import AutoConfig + +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.utils import hf_processor, hf_tokenizer, omega_conf_to_dataclass +from verl.utils.device import ( + get_device_id, + get_device_name, + get_nccl_backend, + get_torch_device, +) +from verl.utils.fs import copy_to_local +from verl.utils.fsdp_utils import ( + fsdp_version, +) +from verl.utils.import_utils import import_external_libs +from verl.utils.model import get_generation_config, update_model_config +from verl.utils.profiler import DistProfiler, DistProfilerExtension, ProfilerConfig, log_gpu_memory_usage, simple_timer +from verl.utils.profiler.performance import reduce_timing, topk_reduce_ratio_min_max +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.fsdp_workers import ActorRolloutRefWorker as ARRWorker +from verl.workers.fsdp_workers import CriticWorker +from verl.workers.rollout import get_rollout_class + +from .distributed_util import stateless_init_process_group + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +device_name = get_device_name() + +__all__ = ["ActorRolloutRefWorker", "AsyncActorRolloutRefWorker", "CriticWorker", "RolloutWorker"] + + +class ActorRolloutRefWorker(ARRWorker): + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + def create_weight_sync_group(self, master_address, master_port, rank_offset, world_size): + rank = torch.distributed.get_rank() + rank_offset + self._weight_sync_group = stateless_init_process_group( + master_address, + master_port, + rank, + world_size, + get_torch_device().current_device(), + ) + + def _get_actor_params(self): + assert self._is_actor + params = self.actor_module_fsdp.state_dict() + from verl.utils.model import convert_weight_keys + + params = convert_weight_keys( + params, getattr(self.actor_module_fsdp, "_fsdp_wrapped_module", self.actor_module_fsdp) + ) + return params + + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + def sync_rollout_weights(self): + assert (self._is_actor or self._is_rollout) and not self.config.hybrid_engine + assert hasattr(self, "_weights_info") and self._weights_info is not None + + params = self._get_actor_params() if self._is_actor else None + if self._is_rollout: + inference_model = ( + self.rollout.inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner.model + ) + from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader + + patch_vllm_moe_model_weight_loader(inference_model) + for key, shape, dtype in self._weights_info: + tensor = torch.empty(shape, dtype=dtype, device=get_torch_device().current_device()) + if self._is_actor: + assert key in params + origin_data = params[key] + if hasattr(origin_data, "full_tensor"): + origin_data = origin_data.full_tensor() + if torch.distributed.get_rank() == 0: + tensor.copy_(origin_data) + + self._weight_sync_group.broadcast(tensor, src=0, stream=get_torch_device().current_stream()) + if self._is_rollout: + inference_model.load_weights([(key, tensor)]) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def get_actor_weights_info(self): + assert self._is_actor + if hasattr(self, "_weights_info"): + return self._weights_info + if fsdp_version(self.actor_module_fsdp) == 1: + from torch.distributed.fsdp.api import ShardedStateDictConfig, StateDictType + + FSDP.set_state_dict_type( + self.actor_module_fsdp, + state_dict_type=StateDictType.SHARDED_STATE_DICT, + state_dict_config=ShardedStateDictConfig(), + ) + params = self._get_actor_params() + ret = [] + for key, tensor in params.items(): + ret.append((key, tensor.size(), tensor.dtype)) + self._weights_info = ret + return ret + + +class RolloutWorker(ActorRolloutRefWorker): + def __init__(self, config: DictConfig, role: str): + Worker.__init__(self) + assert role == "rollout" + self.config = config + import torch.distributed + + if not torch.distributed.is_initialized(): + rank = int(os.environ.get("RANK", 0)) + world_size = int(os.environ.get("WORLD_SIZE", 1)) + torch.distributed.init_process_group( + backend=f"cpu:gloo,{get_device_name()}:{get_nccl_backend()}", + rank=rank, + world_size=world_size, + init_method=os.environ.get("DIST_INIT_METHOD", None), + ) + # TODO(haibin.lin): + # As of now the type of config is DictConfig, if we assign config.profiler with ProfilerConfig, + # it will actually convert the ProfilerConfig dataclass back to a DictConfig. + # We can still use ProfilerConfig for testing purpose (tests/utils/test_nvtx_profile.py) + # as they provides DictConfig-like interface + # The benefit of creating the dataclass config is to perform validation during __post_init__ + omega_profiler_config = config.get("profiler", {}) + profiler_config = omega_conf_to_dataclass(omega_profiler_config, dataclass_type=ProfilerConfig) + if omega_profiler_config.get("tool", None) in ["npu", "nsys", "torch", "torch_memory"]: + tool_config = omega_conf_to_dataclass( + omega_profiler_config.get("tool_config", {}).get(omega_profiler_config.get("tool")) + ) + else: + tool_config = None + DistProfilerExtension.__init__( + self, DistProfiler(rank=self.rank, config=profiler_config, tool_config=tool_config) + ) + self._is_rollout = True + self._is_actor = False + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + # This is used to import external_lib into the huggingface systems + import_external_libs(self.config.model.get("external_lib", None)) + override_model_config = OmegaConf.to_container(OmegaConf.create(self.config.model.get("override_config", {}))) + + use_shm = self.config.model.get("use_shm", False) + local_path = copy_to_local(self.config.model.path, use_shm=use_shm) + trust_remote_code = self.config.model.get("trust_remote_code", False) + + self.tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) + self.processor = hf_processor(local_path, trust_remote_code=trust_remote_code) + + if self.config.model.get("custom_chat_template", None) is not None: + if self.processor is not None: + self.processor.chat_template = self.config.model.custom_chat_template + else: + self.tokenizer.chat_template = self.config.model.custom_chat_template + + # override model kwargs + actor_model_config = AutoConfig.from_pretrained( + local_path, trust_remote_code=trust_remote_code, attn_implementation="flash_attention_2" + ) + + # patch for kimi-vl + if getattr(actor_model_config, "model_type", None) == "kimi_vl": + actor_model_config.text_config.topk_method = "greedy" + + self.generation_config = get_generation_config(local_path, trust_remote_code=trust_remote_code) + + override_config_kwargs = { + "bos_token_id": self.tokenizer.bos_token_id, + "eos_token_id": self.tokenizer.eos_token_id, + "pad_token_id": self.tokenizer.pad_token_id, + } + override_config_kwargs.update(override_model_config) + update_model_config(actor_model_config, override_config_kwargs=override_config_kwargs) + if self.rank == 0: + print(f"Model config after override: {actor_model_config}") + + infer_tp = self.config.rollout.tensor_model_parallel_size + dp = self.world_size // infer_tp + assert self.world_size % infer_tp == 0, ( + f"rollout world_size: {self.world_size} is not divisible by infer_tp: {infer_tp}" + ) + rollout_device_mesh = init_device_mesh( + device_name, mesh_shape=(dp, infer_tp), mesh_dim_names=["dp", "infer_tp"] + ) + + is_collect = rollout_device_mesh["infer_tp"].get_local_rank() == 0 + self._register_dispatch_collect_info( + "rollout", dp_rank=rollout_device_mesh["dp"].get_local_rank(), is_collect=is_collect + ) + + rollout_name = self.config.rollout.name + assert rollout_name == "vllm" + + rollout_config: RolloutConfig = omega_conf_to_dataclass(self.config.rollout) + model_config: HFModelConfig = omega_conf_to_dataclass(self.config.model, dataclass_type=HFModelConfig) + self.model_config = model_config + + log_gpu_memory_usage(f"Before building {rollout_name} rollout", logger=logger) + rollout = get_rollout_class(rollout_config.name, rollout_config.mode)( + config=rollout_config, model_config=model_config, device_mesh=rollout_device_mesh + ) + log_gpu_memory_usage(f"After building {rollout_name} rollout", logger=logger) + from .vllm_sharding_manager import VLLMShardingManager + + rollout_sharding_manager = VLLMShardingManager( + inference_engine=rollout.inference_engine, device_mesh=rollout_device_mesh + ) + + log_gpu_memory_usage("After building sharding manager", logger=logger) + + self.rollout = rollout + self.rollout_sharding_manager = rollout_sharding_manager + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="rollout"), blocking=False) + def async_generate_sequences(self, prompts): + # Support all hardwares + prompts = prompts.to(get_device_id()) + + assert self._is_rollout + + meta_info = { + "eos_token_id": self.generation_config.eos_token_id + if self.generation_config is not None + else self.tokenizer.eos_token_id, + "pad_token_id": self.generation_config.pad_token_id + if self.generation_config is not None + else self.tokenizer.pad_token_id, + } + prompts.meta_info.update(meta_info) + timing_generate = {} + with self.rollout_sharding_manager: + log_gpu_memory_usage("After entering rollout sharding manager", logger=logger) + + with simple_timer("generate_sequences", timing_generate): + output = self.rollout.generate_sequences(prompts=prompts) + + log_gpu_memory_usage("After rollout generation", logger=logger) + + timing_generate.update(self.rollout_sharding_manager.timing) + # We calculate the average timing across all ranks + # to make sure meta_info["timing"] is the same + timing_generate_topk_ratio, timing_generate_min, timing_generate_max = topk_reduce_ratio_min_max( + timing_generate["generate_sequences"] + ) + timing_generate = reduce_timing(timing_generate) + timing_generate.update( + { + "generation_timing/max": timing_generate_max, + "generation_timing/min": timing_generate_min, + "generation_timing/topk_ratio": timing_generate_topk_ratio, + } + ) + output.meta_info["timing"] = timing_generate + output = output.to("cpu") + + # clear kv cache + get_torch_device().empty_cache() + return output + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_actor_weights_info(self, weights_info): + assert self._is_rollout + self._weights_info = weights_info + + +class AsyncActorRolloutRefWorker(ActorRolloutRefWorker): + def __init__(self, *args, **kwargs): + raise NotImplementedError diff --git a/e2c/verl/recipe/one_step_off_policy/grpo_0.6b_gsm8k_fsdp2_2_6.sh b/e2c/verl/recipe/one_step_off_policy/grpo_0.6b_gsm8k_fsdp2_2_6.sh new file mode 100644 index 0000000000000000000000000000000000000000..09048fd0340d987a58958f7e35828cce711574a0 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/grpo_0.6b_gsm8k_fsdp2_2_6.sh @@ -0,0 +1,65 @@ +set -x + +project_name='GRPO' +exp_name='GRPO-Qwen3-0.6b-gsm8k-fsdp2-one-step-off-2-6' + +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-0.6B"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/gsm8k/train.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/gsm8k/test.parquet"} + +NNODES=${NNODES:-1} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} + +n_gpus_rollout=2 +n_gpus_training=$((NGPUS_PER_NODE - n_gpus_rollout)) + + +python3 -m recipe.one_step_off_policy.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.train_batch_size=1152 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.actor.strategy=fsdp2 \ + critic.strategy=fsdp2 \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.hybrid_engine=False \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=192 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.val_before_train=True \ + trainer.logger=['console','tensorboard'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=2 \ + trainer.nnodes="${NNODES}" \ + trainer.n_gpus_per_node="${n_gpus_training}" \ + rollout.nnodes="${NNODES}" \ + rollout.n_gpus_per_node="${n_gpus_rollout}" $@ \ No newline at end of file diff --git a/e2c/verl/recipe/one_step_off_policy/grpo_3b_gsm8k_fsdp2_2_6.sh b/e2c/verl/recipe/one_step_off_policy/grpo_3b_gsm8k_fsdp2_2_6.sh new file mode 100644 index 0000000000000000000000000000000000000000..a0d3bdb8ce8f0b116d3939372a7d88ee5ab27898 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/grpo_3b_gsm8k_fsdp2_2_6.sh @@ -0,0 +1,64 @@ +set -x + +project_name='GRPO' +exp_name='GRPO-Qwen3-0.6b-gsm8k-fsdp2-one-step-off-2-6' + +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} +MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen/Qwen2.5-3B-Instruct"} +CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} +TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/gsm8k/train.parquet"} +TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/gsm8k/test.parquet"} + +NNODES=${NNODES:-1} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} + +n_gpus_rollout=2 +n_gpus_training=$((NGPUS_PER_NODE - n_gpus_rollout)) + +python3 -m recipe.one_step_off_policy.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.train_batch_size=1152 \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.actor.strategy=fsdp2 \ + critic.strategy=fsdp2 \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.hybrid_engine=False \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=192 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.val_before_train=True \ + trainer.logger=['console','tensorboard'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=2 \ + trainer.nnodes="${NNODES}" \ + trainer.n_gpus_per_node="${n_gpus_training}" \ + rollout.nnodes="${NNODES}" \ + rollout.n_gpus_per_node="${n_gpus_rollout}" $@ \ No newline at end of file diff --git a/e2c/verl/recipe/one_step_off_policy/main_ppo.py b/e2c/verl/recipe/one_step_off_policy/main_ppo.py new file mode 100644 index 0000000000000000000000000000000000000000..344fe4b9f0c758e60812dff786eaaa24786b450b --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/main_ppo.py @@ -0,0 +1,242 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2025 Meituan Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. +""" + +import os +import socket + +import hydra +import ray +from omegaconf import OmegaConf + +from recipe.one_step_off_policy.utils import need_critic +from verl.trainer.constants_ppo import get_ppo_ray_runtime_env +from verl.trainer.main_ppo import create_rl_dataset, create_rl_sampler +from verl.trainer.ppo.reward import load_reward_manager +from verl.trainer.ppo.utils import need_reference_policy +from verl.utils.config import validate_config + +from .ray_trainer import OneStepOffRayTrainer + + +@hydra.main(config_path="config", config_name="one_step_off_ppo_trainer", version_base=None) +def main(config): + run_ppo(config) + + +# Define a function to run the PPO-like training process +def run_ppo(config) -> None: + # Check if Ray is not initialized + if not ray.is_initialized(): + # Initialize Ray with a local cluster configuration + # Set environment variables in the runtime environment to control tokenizer parallelism, + # NCCL debug level, VLLM logging level, and allow runtime LoRA updating + # `num_cpus` specifies the number of CPU cores Ray can use, obtained from the configuration + default_runtime_env = get_ppo_ray_runtime_env() + ray_init_kwargs = config.ray_kwargs.get("ray_init", {}) + runtime_env_kwargs = ray_init_kwargs.get("runtime_env", {}) + runtime_env = OmegaConf.merge(default_runtime_env, runtime_env_kwargs) + ray_init_kwargs = OmegaConf.create({**ray_init_kwargs, "runtime_env": runtime_env}) + print(f"ray init kwargs: {ray_init_kwargs}") + ray.init(**OmegaConf.to_container(ray_init_kwargs)) + + # Create a remote instance of the TaskRunner class, and + # Execute the `run` method of the TaskRunner instance remotely and wait for it to complete + if ( + config.global_profiler.tool == "nsys" + and OmegaConf.select(config.global_profiler, "steps") is not None + and len(OmegaConf.select(config.global_profiler, "steps")) > 0 + ): + nsight_options = OmegaConf.to_container(config.global_profiler.tool_config.nsys.controller_nsight_options) + runner = TaskRunner.options(runtime_env={"nsight": nsight_options}).remote() + else: + runner = TaskRunner.remote() + ray.get(runner.run.remote(config)) + + # [Optional] get the path of the timeline trace file from the configuration, default to None + # This file is used for performance analysis + timeline_json_file = config.ray_kwargs.get("timeline_json_file", None) + if timeline_json_file: + ray.timeline(filename=timeline_json_file) + + +@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head +class TaskRunner: + def run(self, config): + # Print the initial configuration. `resolve=True` will evaluate symbolic values. + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_to_local + + print(f"TaskRunner hostname: {socket.gethostname()}, PID: {os.getpid()}") + + pprint(OmegaConf.to_container(config, resolve=True)) + + OmegaConf.resolve(config) + + # Define worker classes based on the actor strategy. + if config.actor_rollout_ref.actor.strategy == "fsdp2": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.single_controller.ray import RayWorkerGroup + + from .fsdp_workers import ( + ActorRolloutRefWorker, + AsyncActorRolloutRefWorker, + CriticWorker, + RolloutWorker, + ) + + actor_rollout_cls = ( + AsyncActorRolloutRefWorker + if config.actor_rollout_ref.rollout.mode == "async" + else ActorRolloutRefWorker + ) + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.single_controller.ray import RayWorkerGroup + + from .megatron_workers import ( + ActorRolloutRefWorker, + AsyncActorRolloutRefWorker, + CriticWorker, + RolloutWorker, + ) + + actor_rollout_cls = ( + AsyncActorRolloutRefWorker + if config.actor_rollout_ref.rollout.mode == "async" + else ActorRolloutRefWorker + ) + ray_worker_group_cls = RayWorkerGroup + + else: + raise NotImplementedError + + from .ray_trainer import ResourcePoolManager, Role + + role_worker_mapping = { + Role.Actor: ray.remote(actor_rollout_cls), + Role.Rollout: ray.remote(RolloutWorker), + Role.Critic: ray.remote(CriticWorker), + } + + global_pool_id = "actor_pool" + + assert config.trainer.n_gpus_per_node > 0, "config.trainer.n_gpus_per_node must be greater than 0" + assert config.trainer.nnodes > 0, "config.trainer.nnodes must be greater than 0" + assert config.rollout.n_gpus_per_node > 0, "config.rollout.n_gpus_per_node must be greater than 0" + assert config.rollout.nnodes > 0, "config.rollout.nnodes must be greater than 0" + + actor_pool = [config.trainer.n_gpus_per_node] * config.trainer.nnodes + rollout_pool = [config.rollout.n_gpus_per_node] * config.rollout.nnodes + + resource_pool_spec = { + "actor_pool": actor_pool, + "rollout_pool": rollout_pool, + } + mapping = { + Role.Actor: "actor_pool", + Role.Rollout: "rollout_pool", + Role.Critic: "actor_pool", + } + print(f"resource_pool_spec: {resource_pool_spec}") + # We should adopt a multi-source reward function here: + # - for rule-based rm, we directly call a reward score + # - for model-based rm, we call a model + # - for code related prompt, we send to a sandbox if there are test cases + # finally, we combine all the rewards together + # The reward type depends on the tag of the data + if config.reward_model.enable: + if config.reward_model.strategy in ["fsdp2"]: + from verl.workers.fsdp_workers import RewardModelWorker + elif config.reward_model.strategy == "megatron": + from verl.workers.megatron_workers import RewardModelWorker + else: + raise NotImplementedError + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + mapping[Role.RewardModel] = global_pool_id + + # Add a reference policy worker if KL loss or KL reward is used. + if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) + mapping[Role.RefPolicy] = global_pool_id + + # validate config + validate_config( + config=config, + use_reference_policy=need_reference_policy(role_worker_mapping), + use_critic=need_critic(config), + ) + + # Download the checkpoint from HDFS to the local machine. + # `use_shm` determines whether to use shared memory, which could lead to faster model loading if turned on + local_path = copy_to_local( + config.actor_rollout_ref.model.path, use_shm=config.actor_rollout_ref.model.get("use_shm", False) + ) + + # Instantiate the tokenizer and processor. + from verl.utils import hf_processor, hf_tokenizer + + trust_remote_code = config.data.get("trust_remote_code", False) + tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) + # Used for multimodal LLM, could be None + processor = hf_processor(local_path, trust_remote_code=trust_remote_code, use_fast=True) + + # Load the reward manager for training and validation. + reward_fn = load_reward_manager( + config, tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {}) + ) + val_reward_fn = load_reward_manager( + config, tokenizer, num_examine=1, **config.reward_model.get("reward_kwargs", {}) + ) + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + + from verl.utils.dataset.rl_dataset import collate_fn + + # Create training and validation datasets. + train_dataset = create_rl_dataset(config.data.train_files, config.data, tokenizer, processor) + val_dataset = create_rl_dataset(config.data.val_files, config.data, tokenizer, processor) + train_sampler = create_rl_sampler(config.data, train_dataset) + + # Initialize the PPO trainer. + trainer = OneStepOffRayTrainer( + config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + train_dataset=train_dataset, + val_dataset=val_dataset, + collate_fn=collate_fn, + train_sampler=train_sampler, + device_name=config.trainer.device, + ) + # Initialize the workers of the trainer. + trainer.init_workers() + # Start the training process. + trainer.fit() + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/recipe/one_step_off_policy/megatron_workers.py b/e2c/verl/recipe/one_step_off_policy/megatron_workers.py new file mode 100644 index 0000000000000000000000000000000000000000..8d72edf2138596f252d9fb8370c7d8bad7782ef4 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/megatron_workers.py @@ -0,0 +1,209 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright 2025 Meituan Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import torch +import torch.distributed +from omegaconf import DictConfig, OmegaConf + +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.utils.config import omega_conf_to_dataclass +from verl.utils.debug import ( + log_gpu_memory_usage, +) +from verl.utils.device import get_device_name, get_torch_device +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.megatron_workers import ActorRolloutRefWorker as ARRWorker +from verl.workers.megatron_workers import CriticWorker, RewardModelWorker +from verl.workers.rollout import get_rollout_class + +from .distributed_util import stateless_init_process_group + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +__all__ = ["ActorRolloutRefWorker", "AsyncActorRolloutRefWorker", "CriticWorker", "RewardModelWorker", "RolloutWorker"] + + +class ActorRolloutRefWorker(ARRWorker): + def __init__(self, config: DictConfig, role: str): + assert role in ["actor", "ref"] + tmp_role = "ref" if role == "ref" else "actor_rollout" + super().__init__(config, tmp_role) + if role == "actor": + self._is_rollout = False + self.role = role + + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + def create_weight_sync_group(self, master_address, master_port, rank_offset, world_size): + rank = torch.distributed.get_rank() + rank_offset + self._weight_sync_group = stateless_init_process_group( + master_address, + master_port, + rank, + world_size, + get_torch_device().current_device(), + ) + + def _get_actor_params_generator(self): + assert self._is_actor + from verl.models.mcore import get_mcore_weight_converter + from verl.utils.megatron_utils import per_tensor_generator + + layer_name_mapping = { + "qkv_layer_name": "self_attention.linear_qkv.", + "gate_proj_layer_name": "linear_fc1.", + } + weight_converter = get_mcore_weight_converter(self.actor_model_config, self.dtype) + generator = per_tensor_generator( + self.actor.actor_module, + self.actor_model_config, + weight_converter, + self.tf_config, + layer_name_mapping, + ) + return generator + + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + def sync_rollout_weights(self): + assert (self._is_actor or self._is_rollout) and not self.config.hybrid_engine + assert hasattr(self, "_weights_info") and self._weights_info is not None + + params_generator = self._get_actor_params_generator() if self._is_actor else None + if self._is_rollout: + inference_model = ( + self.rollout.inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner.model + ) + from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader + + patch_vllm_moe_model_weight_loader(inference_model) + for key, shape, dtype in self._weights_info: + if self._is_actor: + weight_key, weight = next(params_generator) + assert key == weight_key + assert shape == weight.size() + assert dtype == weight.dtype + + tensor = torch.empty(shape, dtype=dtype, device=get_torch_device().current_device()) + if self._is_actor and torch.distributed.get_rank() == 0: + tensor.copy_(weight) + + self._weight_sync_group.broadcast(tensor, src=0, stream=get_torch_device().current_stream()) + if self._is_rollout: + inference_model.load_weights([(key, tensor)]) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def get_actor_weights_info(self): + assert self._is_actor + if hasattr(self, "_weights_info"): + return self._weights_info + + params_generator = self._get_actor_params_generator() + ret = [] + for key, tensor in params_generator: + ret.append((key, tensor.size(), tensor.dtype)) + + self._weights_info = ret + return ret + + +class RolloutWorker(ActorRolloutRefWorker): + def __init__(self, config: DictConfig, role: str): + assert role == "rollout" + ARRWorker.__init__(self, config, role) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + if self.config.model.get("external_lib", None) is not None: + # This is used to import external_lib into the huggingface systems + import importlib + + importlib.import_module(self.config.model.external_lib) + + from verl.utils.torch_dtypes import PrecisionType + + override_model_config = OmegaConf.to_container(OmegaConf.create(self.config.model.get("override_config", {}))) + override_transformer_config = {} + self.param_dtype = torch.bfloat16 + self.dtype = PrecisionType.to_dtype(self.param_dtype) + trust_remote_code = self.config.model.get("trust_remote_code", False) + + from verl.utils.model import get_generation_config + + self._init_hf_config_and_tf_config( + self.config.model.path, + self.config.model.path, + self.dtype, + override_model_config, + override_transformer_config, + trust_remote_code, + ) + self.generation_config = get_generation_config(self.local_path) + + from torch.distributed.device_mesh import init_device_mesh + + assert self.config.rollout.name == "vllm" + assert self.config.rollout.mode == "sync" + + from .vllm_sharding_manager import VLLMShardingManager + + # NOTE(sgm): If the QKV and gate_up projection layer are concate together in actor, + # we will reorganize their weight format when resharding from actor to rollout. + + infer_tp = self.config.rollout.tensor_model_parallel_size + dp = self.world_size // infer_tp + assert self.world_size % infer_tp == 0, ( + f"rollout world_size: {self.world_size} is not divisible by infer_tp: {infer_tp}" + ) + rollout_device_mesh = init_device_mesh( + get_device_name(), mesh_shape=(dp, infer_tp), mesh_dim_names=["dp", "infer_tp"] + ) + is_collect = rollout_device_mesh["infer_tp"].get_local_rank() == 0 + self._register_dispatch_collect_info( + "rollout", dp_rank=rollout_device_mesh["dp"].get_local_rank(), is_collect=is_collect + ) + log_gpu_memory_usage("Before building vllm rollout", logger=None) + + rollout_config: RolloutConfig = omega_conf_to_dataclass(self.config.rollout) + model_config: HFModelConfig = omega_conf_to_dataclass(self.config.model, dataclass_type=HFModelConfig) + rollout = get_rollout_class(rollout_config.name, rollout_config.mode)( + config=rollout_config, model_config=model_config, device_mesh=rollout_device_mesh + ) + log_gpu_memory_usage("After building vllm rollout", logger=logger) + + sharding_manager = VLLMShardingManager( + inference_engine=rollout.inference_engine, + device_mesh=rollout_device_mesh, + ) + log_gpu_memory_usage("After building sharding manager", logger=logger) + + self.rollout, self.sharding_manager = rollout, sharding_manager + self.rollout.sharding_manager = sharding_manager + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="rollout"), blocking=False) + def async_generate_sequences(self, *args, **kwargs): + return super().generate_sequences(*args, **kwargs) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_actor_weights_info(self, weights_info): + assert self._is_rollout + self._weights_info = weights_info + + +class AsyncActorRolloutRefWorker(ActorRolloutRefWorker): + def __init__(self, *args, **kwargs): + raise NotImplementedError diff --git a/e2c/verl/recipe/one_step_off_policy/ray_trainer.py b/e2c/verl/recipe/one_step_off_policy/ray_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..4781c8b155ae82aaa906c82d61168c785e7a6ed0 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/ray_trainer.py @@ -0,0 +1,679 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# Copyright 2025 Meituan Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This trainer supports model-agonistic model initialization with huggingface +""" + +import uuid +from pprint import pprint + +import numpy as np +import ray +import torch +from omegaconf import OmegaConf +from torch.utils.data import Dataset, Sampler +from tqdm import tqdm + +from recipe.one_step_off_policy.utils import need_critic +from verl import DataProto +from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup +from verl.single_controller.ray.base import create_colocated_worker_cls +from verl.trainer.ppo import core_algos +from verl.trainer.ppo.core_algos import agg_loss +from verl.trainer.ppo.metric_utils import ( + compute_data_metrics, + compute_throughout_metrics, + compute_timing_metrics, +) +from verl.trainer.ppo.ray_trainer import ( + RayPPOTrainer, + ResourcePoolManager, + apply_kl_penalty, + compute_advantage, + compute_response_mask, +) +from verl.trainer.ppo.reward import compute_reward, compute_reward_async +from verl.trainer.ppo.utils import Role, WorkerType, need_reference_policy, need_reward_model +from verl.utils.debug import marked_timer +from verl.utils.metric import ( + reduce_metrics, +) +from verl.utils.tracking import ValidationGenerationsLogger + + +class GenerationBatchFuture: + """ + Wrapper class for encapsulating batch generation results + """ + + def __init__(self, epoch, batch, gen_batch_output, future_reward=None): + """ + :param epoch: current epoch + :param batch: Input batch data + :param gen_batch_output: Generated sequences from the main model (DataProtoFuture) + :param future_reward: Future for reward computation (optional) + """ + self.epoch = epoch + self.batch = batch + self.gen_batch_output = gen_batch_output + self.future_reward = future_reward + + def get(self): + """ + Get the actual results by calling get() method on gen_batch_output + + Returns: + tuple: (epoch, batch, gen_batch_result, future_reward) + - epoch: Current epoch + - batch: Original input batch data + - gen_batch_result: Result from gen_batch_output.get() or gen_batch_output itself + - future_reward: Future for reward computation if available, else None + """ + # Call get() method on gen_batch_output if available + if hasattr(self.gen_batch_output, "get"): + gen_batch_result = self.gen_batch_output.get() + else: + gen_batch_result = self.gen_batch_output + + return self.epoch, self.batch, gen_batch_result, self.future_reward + + +class OneStepOffRayTrainer(RayPPOTrainer): + # TODO: support each role have individual ray_worker_group_cls, + # i.e., support different backend of different role + def __init__( + self, + config, + tokenizer, + role_worker_mapping: dict[Role, WorkerType], + resource_pool_manager: ResourcePoolManager, + ray_worker_group_cls: RayWorkerGroup = RayWorkerGroup, + processor=None, + reward_fn=None, + val_reward_fn=None, + train_dataset: Dataset | None = None, + val_dataset: Dataset | None = None, + collate_fn=None, + train_sampler: Sampler | None = None, + device_name="cuda", + ): + """ + Initialize distributed PPO trainer with Ray backend. + Note that this trainer runs on the driver process on a single CPU/GPU node. + + Args: + config: Configuration object containing training parameters. + tokenizer: Tokenizer used for encoding and decoding text. + role_worker_mapping (dict[Role, WorkerType]): Mapping from roles to worker classes. + resource_pool_manager (ResourcePoolManager): Manager for Ray resource pools. + ray_worker_group_cls (RayWorkerGroup, optional): Class for Ray worker groups. Defaults to RayWorkerGroup. + processor: Optional data processor, used for multimodal data + reward_fn: Function for computing rewards during training. + val_reward_fn: Function for computing rewards during validation. + train_dataset (Optional[Dataset], optional): Training dataset. Defaults to None. + val_dataset (Optional[Dataset], optional): Validation dataset. Defaults to None. + collate_fn: Function to collate data samples into batches. + train_sampler (Optional[Sampler], optional): Sampler for the training dataset. Defaults to None. + device_name (str, optional): Device name for training (e.g., "cuda", "cpu"). Defaults to "cuda". + """ + + # Store the tokenizer for text processing + self.tokenizer = tokenizer + self.processor = processor + self.config = config + self.reward_fn = reward_fn + self.val_reward_fn = val_reward_fn + + self.hybrid_engine = config.actor_rollout_ref.hybrid_engine + + assert not self.hybrid_engine + + self.role_worker_mapping = role_worker_mapping + self.resource_pool_manager = resource_pool_manager + self.use_reference_policy = need_reference_policy(self.role_worker_mapping) + self.use_rm = need_reward_model(self.role_worker_mapping) + self.use_critic = need_critic(config) + self.ray_worker_group_cls = ray_worker_group_cls + self.device_name = device_name + self.validation_generations_logger = ValidationGenerationsLogger() + + # if ref_in_actor is True, the reference policy will be actor without lora applied + self.ref_in_actor = config.actor_rollout_ref.model.get("lora_rank", 0) > 0 + + # define in-reward KL control + # kl loss control currently not suppoorted + if config.algorithm.use_kl_in_reward: + self.kl_ctrl_in_reward = core_algos.get_kl_controller(config.algorithm.kl_ctrl) + + self._create_dataloader(train_dataset, val_dataset, collate_fn, train_sampler) + + def _validate(self): + self.actor_rollout_wg = self.rollout_wg + ret = super()._validate() + self.actor_rollout_wg = self.actor_wg + return ret + + def init_workers(self): + """Initialize distributed training workers using Ray backend. + + Creates: + 1. Ray resource pools from configuration + 2. Worker groups for each role (actor, critic, etc.) + """ + self.resource_pool_manager.create_resource_pool() + + self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} + + # create actor and rollout + for role, role_name in [(Role.Actor, "actor"), (Role.Rollout, "rollout")]: + resource_pool = self.resource_pool_manager.get_resource_pool(role) + role_cls = RayClassWithInitArgs( + cls=self.role_worker_mapping[role], + config=self.config.actor_rollout_ref, + role=role_name, + ) + self.resource_pool_to_cls[resource_pool][role_name] = role_cls + + # create critic + if self.use_critic: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.Critic) + critic_cls = RayClassWithInitArgs(cls=self.role_worker_mapping[Role.Critic], config=self.config.critic) + self.resource_pool_to_cls[resource_pool]["critic"] = critic_cls + + # create reference policy if needed + if self.use_reference_policy: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) + ref_policy_cls = RayClassWithInitArgs( + self.role_worker_mapping[Role.RefPolicy], + config=self.config.actor_rollout_ref, + role="ref", + ) + self.resource_pool_to_cls[resource_pool]["ref"] = ref_policy_cls + + # create a reward model if reward_fn is None + if self.use_rm: + # we create a RM here + resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) + rm_cls = RayClassWithInitArgs(self.role_worker_mapping[Role.RewardModel], config=self.config.reward_model) + self.resource_pool_to_cls[resource_pool]["rm"] = rm_cls + + # initialize WorkerGroup + # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, + # you should not use `create_colocated_worker_cls`. + # Instead, directly pass different resource pool to different worker groups. + # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. + all_wg = {} + wg_kwargs = {} # Setting up kwargs for RayWorkerGroup + if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None: + wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout + if OmegaConf.select(self.config.global_profiler, "steps") is not None: + wg_kwargs["profile_steps"] = OmegaConf.select(self.config.trainer, "steps") + assert ( + OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") + is not None + ), "worker_nsight_options must be set when profile_steps is set" + wg_kwargs["worker_nsight_options"] = OmegaConf.to_container( + OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") + ) + + for resource_pool, class_dict in self.resource_pool_to_cls.items(): + worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + wg_dict = self.ray_worker_group_cls( + resource_pool=resource_pool, + ray_cls_with_init=worker_dict_cls, + device_name=self.device_name, + **wg_kwargs, + ) + spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + all_wg.update(spawn_wg) + + if self.use_critic: + self.critic_wg = all_wg["critic"] + self.critic_wg.init_model() + + if self.use_reference_policy and not self.ref_in_actor: + self.ref_policy_wg = all_wg["ref"] + self.ref_policy_wg.init_model() + + if self.use_rm: + self.rm_wg = all_wg["rm"] + self.rm_wg.init_model() + + self.actor_wg = all_wg["actor"] + self.rollout_wg = all_wg["rollout"] + self.actor_wg.init_model() + self.rollout_wg.init_model() + self.actor_rollout_wg = self.actor_wg # to be compatible with the functions that not be modified + weights_info = self.actor_wg.get_actor_weights_info()[0] + self.rollout_wg.set_actor_weights_info(weights_info) + + self.create_weight_sync_group() + self.sync_rollout_weights() + + # create async rollout manager and request scheduler + self.async_rollout_mode = False + if self.config.actor_rollout_ref.rollout.mode == "async" and self._is_rollout: + from verl.workers.rollout.async_server import AsyncLLMServerManager + + self.async_rollout_mode = True + self.async_rollout_manager = AsyncLLMServerManager( + config=self.config, + worker_group=self.rollout_wg, + ) + + def create_weight_sync_group(self): + master_address = ray.get(self.actor_wg.workers[0]._get_node_ip.remote()) + master_port = ray.get(self.actor_wg.workers[0]._get_free_port.remote()) + world_size = len(self.actor_wg.workers + self.rollout_wg.workers) + self.actor_wg.create_weight_sync_group( + master_address, + master_port, + 0, + world_size, + ) + ray.get( + self.rollout_wg.create_weight_sync_group( + master_address, + master_port, + len(self.actor_wg.workers), + world_size, + ) + ) + + def sync_rollout_weights(self): + if not self.hybrid_engine: + self.actor_wg.sync_rollout_weights() + ray.get(self.rollout_wg.sync_rollout_weights()) + + def _create_continuous_iterator(self): + """ + Create a continuous data iterator across epoch + """ + for epoch in range(self.config.trainer.total_epochs): + iterator = iter(self.train_dataloader) + for batch_dict in iterator: + yield epoch, batch_dict + + def _async_gen_next_batch(self, continuous_iterator): + """ + Call parameter synchronization and asynchronous sequence generation. + """ + try: + epoch, batch_dict = next(continuous_iterator) + except StopIteration: + return None + except Exception as e: + print(f"Error in async_gen_next_batch: {e}") + return None + + # Create the initial batch from the data loader + batch = DataProto.from_single_dict(batch_dict) + + # pop those keys for generation + batch_keys_to_pop = ["input_ids", "attention_mask", "position_ids"] + non_tensor_batch_keys_to_pop = ["raw_prompt_ids"] + if "multi_modal_data" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("multi_modal_data") + if "raw_prompt" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("raw_prompt") + if "tools_kwargs" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("tools_kwargs") + if "interaction_kwargs" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("interaction_kwargs") + + gen_batch = batch.pop( + batch_keys=batch_keys_to_pop, + non_tensor_batch_keys=non_tensor_batch_keys_to_pop, + ) + gen_batch = gen_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + + # sync weights from actor to rollout + self.sync_rollout_weights() + + # async generation + gen_batch_output = self.rollout_wg.async_generate_sequences(gen_batch) + + # Launch individual reward computations as each generation completes + future_reward = None + if self.config.reward_model.launch_reward_fn_async: + # Store the object reference and set up callback + future_reward = self._launch_individual_rewards.remote( + gen_batch_output, self.config, self.tokenizer, batch.non_tensor_batch + ) + + # Return the original, now-modified `batch` and the `future_reward` + return GenerationBatchFuture(epoch, batch, gen_batch_output, future_reward) + + @staticmethod + @ray.remote + def _launch_individual_rewards(gen_batch_output, config, tokenizer, original_non_tensor_batch): + # Get generation results + gen_batch_result = gen_batch_output.get() + + # Repeat non_tensor_batch to match the number of responses + n = config.actor_rollout_ref.rollout.n + repeated_non_tensor_batch = {} + for key, value in original_non_tensor_batch.items(): + repeated_non_tensor_batch[key] = np.repeat(value, n, axis=0) + + # Split into individual responses with preserved non_tensor_batch + responses_split = [] + for i in range(len(gen_batch_result)): + response_data = gen_batch_result[i : i + 1] # Get single response + # Add repeated non_tensor_batch values + for key in repeated_non_tensor_batch: + response_data.non_tensor_batch[key] = repeated_non_tensor_batch[key][i : i + 1] + responses_split.append(response_data) + + # Launch async reward computation + reward_futures = [ + compute_reward_async.remote(response_data, config, tokenizer) for response_data in responses_split + ] + + # Wait for results and combine + results = ray.get(reward_futures) + rewards_list = [r[0] for r in results] + extras_list = [r[1] for r in results] + + combined_reward_tensor = torch.cat(rewards_list, dim=0) + combined_extras_dict = {} + if extras_list and extras_list[0]: + for key in extras_list[0].keys(): + combined_extras_dict[key] = [d[key] for d in extras_list if key in d] + + return combined_reward_tensor, combined_extras_dict + + def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the worker group through RPC + to construct the PPO dataflow. + The light-weight advantage computation is done on the driver process. + """ + + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + self.global_steps = 0 + + # load checkpoint before doing anything + self._load_checkpoint() + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + val_metrics = self._validate() + assert val_metrics, f"{val_metrics=}" + pprint(f"Initial validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if self.config.trainer.get("val_only", False): + return + + # add tqdm + progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress") + + # we start from step 1 + self.global_steps += 1 + last_val_metrics = None + + # across epoch iterator + continuous_iterator = self._create_continuous_iterator() + + # Start the first asynchronous generation task. + batch_data_future = self._async_gen_next_batch(continuous_iterator) + + while batch_data_future is not None: + do_profile = ( + self.global_steps in self.config.global_profiler.steps + if self.config.global_profiler.steps is not None + else False + ) + if do_profile: + self.actor_wg.start_profile() + if not self.hybrid_engine: + self.rollout_wg.start_profile() + if self.use_reference_policy: + self.ref_policy_wg.start_profile() + if self.use_critic: + self.critic_wg.start_profile() + if self.use_rm: + self.rm_wg.start_profile() + + metrics = {} + timing_raw = {} + is_last_step = self.global_steps >= self.total_training_steps + + with marked_timer("step", timing_raw): + # wait for the previous batch + with marked_timer("wait_prev_gen", timing_raw, color="red"): + epoch, batch, gen_batch_output, future_reward = batch_data_future.get() + timing_raw.update(gen_batch_output.meta_info["timing"]) + gen_batch_output.meta_info.pop("timing", None) + + # asys next generation (with syns weights from actor to rollout) + with marked_timer("sync_rollout_weights", timing_raw, color="purple"): + if not is_last_step: + batch_data_future = self._async_gen_next_batch(continuous_iterator) + + batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(batch.batch))], dtype=object + ) + # repeat to align with repeated responses in rollout + batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + batch = batch.union(gen_batch_output) + + batch.batch["response_mask"] = compute_response_mask(batch) + # Balance the number of valid tokens across DP ranks. + # NOTE: This usually changes the order of data in the `batch`, + # which won't affect the advantage calculation (since it's based on uid), + # but might affect the loss calculation (due to the change of mini-batching). + # TODO: Decouple the DP balancing and mini-batching. + if self.config.trainer.balance_batch: + self._balance_batch(batch, metrics=metrics) + + # compute global_valid tokens + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + with marked_timer("reward", timing_raw, color="yellow"): + # compute reward model score + if self.use_rm: + reward_tensor = self.rm_wg.compute_rm_score(batch) + batch = batch.union(reward_tensor) + + # Use the pre-launched future reward if available + if self.config.reward_model.launch_reward_fn_async: + # future_reward was already started in _async_gen_next_batch + reward_tensor, reward_extra_infos_dict = ray.get(future_reward) + else: + reward_tensor, reward_extra_infos_dict = compute_reward(batch, self.reward_fn) + + # recompute old_log_probs + with marked_timer("old_log_prob", timing_raw, color="blue"): + old_log_prob = self.actor_wg.compute_log_prob(batch) + entropys = old_log_prob.batch["entropys"] + response_masks = batch.batch["response_mask"] + loss_agg_mode = self.config.actor_rollout_ref.actor.loss_agg_mode + entropy_agg = agg_loss(loss_mat=entropys, loss_mask=response_masks, loss_agg_mode=loss_agg_mode) + old_log_prob_metrics = {"actor/entropy": entropy_agg.detach().item()} + metrics.update(old_log_prob_metrics) + old_log_prob.batch.pop("entropys") + batch = batch.union(old_log_prob) + + if "rollout_log_probs" in batch.batch.keys(): + # TODO: we may want to add diff of probs too. + rollout_old_log_probs = batch.batch["rollout_log_probs"] + actor_old_log_probs = batch.batch["old_log_probs"] + attention_mask = batch.batch["attention_mask"] + responses = batch.batch["responses"] + response_length = responses.size(1) + response_mask = attention_mask[:, -response_length:] + + rollout_probs = torch.exp(rollout_old_log_probs) + actor_probs = torch.exp(actor_old_log_probs) + rollout_probs_diff = torch.abs(rollout_probs - actor_probs) + rollout_probs_diff = torch.masked_select(rollout_probs_diff, response_mask.bool()) + rollout_probs_diff_max = torch.max(rollout_probs_diff) + rollout_probs_diff_mean = torch.mean(rollout_probs_diff) + rollout_probs_diff_std = torch.std(rollout_probs_diff) + metrics.update( + { + "training/rollout_probs_diff_max": rollout_probs_diff_max.detach().item(), + "training/rollout_probs_diff_mean": rollout_probs_diff_mean.detach().item(), + "training/rollout_probs_diff_std": rollout_probs_diff_std.detach().item(), + } + ) + + if self.use_reference_policy: + # compute reference log_prob + with marked_timer("ref", timing_raw, color="olive"): + if not self.ref_in_actor: + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + else: + ref_log_prob = self.actor_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + + # compute values + if self.use_critic: + with marked_timer("values", timing_raw, color="cyan"): + values = self.critic_wg.compute_values(batch) + batch = batch.union(values) + + with marked_timer("adv", timing_raw, color="brown"): + # we combine with rule-based rm + reward_extra_infos_dict: dict[str, list] + batch.batch["token_level_scores"] = reward_tensor + + if reward_extra_infos_dict: + batch.non_tensor_batch.update({k: np.array(v) for k, v in reward_extra_infos_dict.items()}) + + # compute rewards. apply_kl_penalty if available + if self.config.algorithm.use_kl_in_reward: + batch, kl_metrics = apply_kl_penalty( + batch, kl_ctrl=self.kl_ctrl_in_reward, kl_penalty=self.config.algorithm.kl_penalty + ) + metrics.update(kl_metrics) + else: + batch.batch["token_level_rewards"] = batch.batch["token_level_scores"] + + # compute advantages, executed on the driver process + + norm_adv_by_std_in_grpo = self.config.algorithm.get( + "norm_adv_by_std_in_grpo", True + ) # GRPO adv normalization factor + + batch = compute_advantage( + batch, + adv_estimator=self.config.algorithm.adv_estimator, + gamma=self.config.algorithm.gamma, + lam=self.config.algorithm.lam, + num_repeat=self.config.actor_rollout_ref.rollout.n, + norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo, + config=self.config.algorithm, + ) + + # update critic + if self.use_critic: + with marked_timer("update_critic", timing_raw, color="pink"): + critic_output = self.critic_wg.update_critic(batch) + critic_output_metrics = reduce_metrics(critic_output.meta_info["metrics"]) + metrics.update(critic_output_metrics) + + # implement critic warmup + if self.config.trainer.critic_warmup <= self.global_steps: + # update actor + with marked_timer("update_actor", timing_raw, color="red"): + batch.meta_info["multi_turn"] = self.config.actor_rollout_ref.rollout.multi_turn.enable + actor_output = self.actor_wg.update_actor(batch) + actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"]) + metrics.update(actor_output_metrics) + + # Log rollout generations if enabled + rollout_data_dir = self.config.trainer.get("rollout_data_dir", None) + if rollout_data_dir: + with marked_timer("dump_rollout_generations", timing_raw, color="green"): + inputs = self.tokenizer.batch_decode(batch.batch["prompts"], skip_special_tokens=True) + outputs = self.tokenizer.batch_decode(batch.batch["responses"], skip_special_tokens=True) + scores = batch.batch["token_level_scores"].sum(-1).cpu().tolist() + self._dump_generations( + inputs=inputs, + outputs=outputs, + scores=scores, + reward_extra_infos_dict=reward_extra_infos_dict, + dump_path=rollout_data_dir, + ) + + # validate + if ( + self.val_reward_fn is not None + and self.config.trainer.test_freq > 0 + and (is_last_step or self.global_steps % self.config.trainer.test_freq == 0) + ): + with marked_timer("testing", timing_raw, color="green"): + val_metrics: dict = self._validate() + if is_last_step: + last_val_metrics = val_metrics + metrics.update(val_metrics) + + if self.config.trainer.save_freq > 0 and ( + is_last_step or self.global_steps % self.config.trainer.save_freq == 0 + ): + with marked_timer("save_checkpoint", timing_raw, color="green"): + self._save_checkpoint() + + # training metrics + metrics.update( + { + "training/global_step": self.global_steps, + "training/epoch": epoch, + } + ) + # collect metrics + metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) + metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) + # TODO: implement actual tflpo and theoretical tflpo + n_gpus = self.resource_pool_manager.get_n_gpus() + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=self.global_steps) + + progress_bar.update(1) + self.global_steps += 1 + + if do_profile: + self.actor_wg.stop_profile() + if not self.hybrid_engine: + self.rollout_wg.stop_profile() + if self.use_reference_policy: + self.ref_policy_wg.stop_profile() + if self.use_critic: + self.critic_wg.stop_profile() + if self.use_rm: + self.rm_wg.stop_profile() + + if is_last_step: + pprint(f"Final validation metrics: {last_val_metrics}") + progress_bar.close() + return diff --git a/e2c/verl/recipe/one_step_off_policy/utils.py b/e2c/verl/recipe/one_step_off_policy/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1879b0672fa68eda19a1b8e6553f4354b17816fe --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/utils.py @@ -0,0 +1,38 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2025 Meituan Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from omegaconf import DictConfig + +from verl.trainer.ppo.core_algos import AdvantageEstimator + + +def need_critic(config: DictConfig) -> bool: + """Given a config, do we need critic""" + if config.algorithm.adv_estimator == AdvantageEstimator.GAE: + return True + elif config.algorithm.adv_estimator in [ + AdvantageEstimator.GRPO, + AdvantageEstimator.GRPO_PASSK, + AdvantageEstimator.REINFORCE_PLUS_PLUS, + # AdvantageEstimator.REMAX, # TODO:REMAX advantage estimator is not yet supported in one_step_off_policy + AdvantageEstimator.RLOO, + AdvantageEstimator.OPO, + AdvantageEstimator.REINFORCE_PLUS_PLUS_BASELINE, + AdvantageEstimator.GPG, + ]: + return False + else: + raise NotImplementedError diff --git a/e2c/verl/recipe/one_step_off_policy/vllm_sharding_manager.py b/e2c/verl/recipe/one_step_off_policy/vllm_sharding_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..c33ba58547092f6e4725f542294f1d32a5eb6e79 --- /dev/null +++ b/e2c/verl/recipe/one_step_off_policy/vllm_sharding_manager.py @@ -0,0 +1,74 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright 2025 Meituan Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from torch.distributed.device_mesh import DeviceMesh + +from verl import DataProto +from verl.protocol import all_gather_data_proto +from verl.third_party.vllm import parallel_state as vllm_ps +from verl.utils.debug import GPUMemoryLogger +from verl.utils.device import get_torch_device +from verl.utils.torch_functional import check_device_is_available +from verl.workers.sharding_manager.base import BaseShardingManager + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class VLLMShardingManager(BaseShardingManager): + @check_device_is_available() + def __init__(self, inference_engine, device_mesh: DeviceMesh): + self.device_mesh = device_mesh + self.inference_engine = inference_engine + inference_engine.wake_up() + assert device_mesh is not None + assert inference_engine is not None + self.tp_size = self.device_mesh["infer_tp"].size() + self.tp_rank = self.device_mesh["infer_tp"].get_local_rank() + self.timing = {} + gen_dp_rank = self.device_mesh["dp"].get_local_rank() + get_torch_device().manual_seed(gen_dp_rank + 1000) + self.gen_random_states = get_torch_device().get_rng_state() + + @GPUMemoryLogger(role="vllm sharding_manager", logger=logger) + def __enter__(self): + get_torch_device().set_rng_state(self.gen_random_states) + + @GPUMemoryLogger(role="vllm sharding_manager", logger=logger) + def __exit__(self, exc_type, exc_value, traceback): + self.gen_random_states = get_torch_device().get_rng_state() + self.inference_engine.reset_prefix_cache() + + @GPUMemoryLogger(role="vllm sharding_manager", logger=logger) + def preprocess_data(self, data: DataProto) -> DataProto: + """All gather across tp group to make each rank has identical input.""" + if self.tp_size == 1: + return data + + group = vllm_ps.get_tensor_model_parallel_group().device_group + + all_gather_data_proto(data=data, process_group=group) + return data + + @GPUMemoryLogger(role="vllm sharding_manager", logger=logger) + def postprocess_data(self, data: DataProto) -> DataProto: + """Get chunk data of this tp rank since we do all gather in preprocess.""" + if self.tp_size == 1: + return data + + return data.chunk(chunks=self.tp_size)[self.tp_rank] diff --git a/e2c/verl/recipe/prime/__init__.py b/e2c/verl/recipe/prime/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6b76ea65c919de7f0b6544338c10026251d17100 --- /dev/null +++ b/e2c/verl/recipe/prime/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/recipe/prime/config/prime_trainer.yaml b/e2c/verl/recipe/prime/config/prime_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40aed453200964756e6ec611d4a4f0e15342bcd6 --- /dev/null +++ b/e2c/verl/recipe/prime/config/prime_trainer.yaml @@ -0,0 +1,75 @@ +# the prime config will override default ppo_trainer.yaml + +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +data: + filter_accuracy: True + accuracy_lower_bound: 0.2 + accuracy_upper_bound: 0.8 + oversample_factor: 4.0 # Sample more responses than the batch size. prompts satisfying the filter will be prioritized. + filter_truncate: True + truncation: right + +actor_rollout_ref: + hybrid_engine: True + model: + use_remove_padding: True + rollout: + # number of responses (i.e. num sample times) + n: 4 + actor: + entropy_coeff: 0.001 + +reward_model: + enable: True + strategy: fsdp + model: + ref_path: ${reward_model.model.path} + use_remove_padding: True + use_fused_kernels: ${actor_rollout_ref.model.use_fused_kernels} + fused_kernel_options: + impl_backend: torch # triton, torch + tokenizer_path: ${actor_rollout_ref.model.path} + enable_gradient_checkpointing: ${actor_rollout_ref.model.enable_gradient_checkpointing} + ref_type: freeze + fsdp_config: + min_num_params: 0 + param_offload: ${actor_rollout_ref.actor.fsdp_config.param_offload} + optimizer_offload: ${actor_rollout_ref.actor.fsdp_config.optimizer_offload} + update: before # ``before`` for double-forward, ``after`` for single-forward + optim: + lr: 1e-6 + lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime + min_lr_ratio: null + warmup_style: constant + total_training_steps: -1 # must be overridden by program + weight_decay: 0. + grad_clip: 10.0 + beta_train: 0.05 + loss_type: ce # currently only supports ce loss + prime_granularity: token + prime_norm: batch_norm # batch_norm or none. if set to none, the normalizer is beta_train + mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} + reward_manager: prime + +algorithm: + adv_estimator: rloo + # now supports rloo. it treats different source of reward separately. + kl_ctrl: + type: fixed + kl_coef: 0.000 + reward_gt_coef: 5 + reward_dpo_coef: 5 + +trainer: + project_name: prime + experiment_name: examples + val_before_train: False + balance_batch: False diff --git a/e2c/verl/recipe/prime/main_prime.py b/e2c/verl/recipe/prime/main_prime.py new file mode 100644 index 0000000000000000000000000000000000000000..39d20de4326bbea9957326405903abd3c5c02ecb --- /dev/null +++ b/e2c/verl/recipe/prime/main_prime.py @@ -0,0 +1,163 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. +""" + +import hydra +import ray +from omegaconf import OmegaConf + +from verl.trainer.ppo.utils import need_reference_policy +from verl.utils.config import validate_config + +from .prime_ray_trainer import RayPRIMETrainer + + +@hydra.main(config_path="config", config_name="prime_trainer", version_base=None) +def main(config): + run_prime(config) + + +def run_prime(config, compute_score=None): + if not ray.is_initialized(): + default_runtime_env = {"env_vars": {"TOKENIZERS_PARALLELISM": "true", "NCCL_DEBUG": "WARN"}} + ray_init_kwargs = config.ray_kwargs.get("ray_init", {}) + runtime_env_kwargs = ray_init_kwargs.get("runtime_env", {}) + runtime_env = OmegaConf.merge(default_runtime_env, runtime_env_kwargs) + ray_init_kwargs = OmegaConf.create({**ray_init_kwargs, "runtime_env": runtime_env}) + print(f"ray init kwargs: {ray_init_kwargs}") + # this is for local ray cluster + ray.init(**OmegaConf.to_container(ray_init_kwargs)) + + ray.get(main_task.remote(config, compute_score)) + + +@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head +def main_task(config, compute_score=None): + # print initial config + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_local_path_from_hdfs + + pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values + OmegaConf.resolve(config) + + # define worker classes + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + assert config.critic.strategy in {"fsdp", "fsdp2"} + from verl.single_controller.ray import RayWorkerGroup + from verl.workers.fsdp_workers import ActorRolloutRefWorker + + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.single_controller.ray import RayWorkerGroup + from verl.workers.megatron_workers import ActorRolloutRefWorker + + ray_worker_group_cls = RayWorkerGroup + + else: + raise NotImplementedError + + from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role + + role_worker_mapping = { + Role.ActorRollout: ray.remote(ActorRolloutRefWorker), + } + + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + } + + # use reference model + if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) + mapping[Role.RefPolicy] = global_pool_id + + if config.reward_model.enable: + from .prime_fsdp_workers import PRIMERewardModelWorker + + role_worker_mapping[Role.RewardModel] = ray.remote(PRIMERewardModelWorker) + mapping[Role.RewardModel] = global_pool_id + + # validate config + # TODO: Additional config checks can be added with proper function under prime recipe + validate_config( + config=config, + use_reference_policy=need_reference_policy(role_worker_mapping), + use_critic=False, + ) + + # download the checkpoint from hdfs + local_path = copy_local_path_from_hdfs(config.actor_rollout_ref.model.path) + + # instantiate tokenizer + from verl.utils import hf_tokenizer + + tokenizer = hf_tokenizer(local_path) + reward_manager_name = config.reward_model.get("reward_manager", "naive") + if reward_manager_name == "naive": + from verl.workers.reward_manager import NaiveRewardManager + + reward_manager_cls = NaiveRewardManager + elif reward_manager_name == "prime": + from verl.workers.reward_manager import PrimeRewardManager + + reward_manager_cls = PrimeRewardManager + else: + raise NotImplementedError + reward_fn = reward_manager_cls(tokenizer=tokenizer, num_examine=0, compute_score=compute_score) + + # Note that we always use function-based RM for validation + val_reward_fn = reward_manager_cls(tokenizer=tokenizer, num_examine=1, compute_score=compute_score) + + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + + trainer = RayPRIMETrainer( + config=config, + tokenizer=tokenizer, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + ) + trainer.init_workers() + trainer.fit() + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/recipe/prime/prime_core_algos.py b/e2c/verl/recipe/prime/prime_core_algos.py new file mode 100644 index 0000000000000000000000000000000000000000..825671216ee12874d5eedf5900ae90de3298d968 --- /dev/null +++ b/e2c/verl/recipe/prime/prime_core_algos.py @@ -0,0 +1,147 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +import verl +import verl.utils.torch_functional as verl_F + + +def compute_rloo_advantage_return(data: verl.DataProto, response_mask: torch.Tensor, n_samples, config): + # calculate rloo reward on different reward sources, and sum again + def masked_rloo(reward_tensor_original, mask_tensor): + reward_tensor = reward_tensor_original.clone() + reward_tensor[~mask_tensor] = 0 + for start_pos in range(0, reward_tensor.shape[0], n_samples): + cur_rewards_mean = torch.cat( + [ + reward_tensor[pos : pos + 1][mask_tensor[pos : pos + 1]].mean(dim=0, keepdim=True) + for pos in range(start_pos, start_pos + n_samples) + ], + dim=0, + ) + cur_rewards_sum = cur_rewards_mean.sum() + cur_reward_baseline = cur_rewards_sum / (n_samples - 1) + reward_tensor[start_pos : start_pos + n_samples][mask_tensor[start_pos : start_pos + n_samples]] = ( + reward_tensor[start_pos : start_pos + n_samples][mask_tensor[start_pos : start_pos + n_samples]] + * (n_samples / (n_samples - 1)) + - cur_reward_baseline + ) + + return reward_tensor + + reward_tensors = [] + + with torch.no_grad(): + if "rm_scores" in data.batch.keys() and config.algorithm.reward_dpo_coef != 0.0: + reward_tensor = data.batch["rm_scores"] + reward_mask = response_mask.bool() + + reward_tensors.append(masked_rloo(reward_tensor, reward_mask) * config.algorithm.reward_dpo_coef) + + if "acc" in data.batch.keys() and config.algorithm.reward_gt_coef != 0.0: + reward_tensor = torch.zeros_like(response_mask, dtype=torch.float32) + reward_mask = torch.zeros_like(response_mask, dtype=torch.bool) + + prompt_ids = data.batch["prompts"] + prompt_length = prompt_ids.shape[-1] + valid_response_length = data.batch["attention_mask"][:, prompt_length:].sum(-1) + + reward_mask[ + torch.arange(0, valid_response_length.shape[0], dtype=torch.long, device=valid_response_length.device), + valid_response_length - 1, + ] = True + reward_tensor[ + torch.arange(0, valid_response_length.shape[0], dtype=torch.long, device=valid_response_length.device), + valid_response_length - 1, + ] = data.batch["acc"] + + reward_tensors.append(masked_rloo(reward_tensor, reward_mask) * config.algorithm.reward_gt_coef) + + final_reward_tensor = sum(reward_tensors) + + returns = (final_reward_tensor * response_mask).flip(dims=[-1]).cumsum(dim=-1).flip(dims=[-1]) + + advantages = returns.clone() + advantages = verl_F.masked_whiten(advantages, response_mask) + + return advantages, returns + + +def compute_ce_dpo_loss_rm(token_level_scores, acc, response_mask, beta): + cur_scores = ((token_level_scores * response_mask).sum(dim=1) * beta).sigmoid() + cur_dpo_loss = torch.nn.functional.binary_cross_entropy(cur_scores, acc) + return cur_dpo_loss + + +def compute_detach_dpo_loss_rm(token_level_scores, acc, Q_bc, acc_bc, response_mask, beta, bon_mode="none"): + # we always assume that the BoN size equals n_samples + # mode1: use acc as rm + # mode2: use Q as rm + cur_Q = (token_level_scores * response_mask).sum(dim=1) * beta + other_Q = torch.zeros_like(cur_Q) + for i in range(token_level_scores.shape[0]): + Q_chosen = Q_bc[i][acc_bc[i] < acc[i]] if acc[i] > 0 else Q_bc[i][acc_bc[i] > acc[i]] + if len(Q_chosen) > 0: + other_Q[i] = Q_chosen.mean() * beta + else: + other_Q[i] = 0 + dpo_loss = -torch.log(torch.sigmoid((cur_Q - other_Q) * ((acc > 0).float() * 2 - 1))) + if bon_mode == "none": + dpo_loss = dpo_loss.mean() + else: + weight = torch.zeros_like(dpo_loss) + n_samples = acc_bc.shape[1] + if bon_mode == "bon_rm": + for i in range(token_level_scores.shape[0]): + weight[i] = n_samples * torch.pow((Q_bc[i] * beta <= cur_Q[i]).float().mean(), n_samples - 1) + elif bon_mode == "bon_acc": + for i in range(token_level_scores.shape[0]): + weight[i] = n_samples * torch.pow((acc_bc[i] <= acc[i]).float().mean(), n_samples - 1) + else: + raise NotImplementedError + dpo_loss = (dpo_loss * weight).sum() + + return dpo_loss + + +def compute_dpo_accuracy(token_level_scores, acc, response_mask, n_samples): + dpo_acc = [] + for start_id in range(0, token_level_scores.shape[0], n_samples): + cur_scores = ( + token_level_scores[start_id : start_id + n_samples] * response_mask[start_id : start_id + n_samples] + ).sum(dim=1) + + def get_upper_triangle(tensor_x): + diff_matrix = tensor_x.unsqueeze(1) - tensor_x.unsqueeze(0) + upper_tri_indices = torch.triu(torch.ones_like(diff_matrix).bool(), diagonal=1) + return diff_matrix[upper_tri_indices] + + cur_acc_diff = get_upper_triangle(acc[start_id : start_id + n_samples]) # in range [-1,1] + cur_score_diff = get_upper_triangle(cur_scores) # in R + cur_score_prediction = (cur_score_diff > 0).float() # in [0,1] + if cur_acc_diff.abs().sum() == 0: + cur_acc = torch.zeros_like(cur_score_prediction[0]) + 0.5 + else: + cur_acc = ( + ((cur_score_diff > 0) == (cur_acc_diff > 0)).float() * cur_acc_diff.abs() + ).sum() / cur_acc_diff.abs().sum() + + dpo_acc.append(cur_acc.unsqueeze(0)) + + return torch.cat(dpo_acc, dim=0).mean() + + +def compute_dpo_abs_accuracy(token_level_scores, acc, response_mask, n_samples): + return (torch.sign((token_level_scores * response_mask).sum(dim=-1)) == torch.sign(acc * 2 - 1)).float().mean() diff --git a/e2c/verl/recipe/prime/prime_dp_rm.py b/e2c/verl/recipe/prime/prime_dp_rm.py new file mode 100644 index 0000000000000000000000000000000000000000..d15d772f07f8ba7ae3b352aa5e2555b73743ddce --- /dev/null +++ b/e2c/verl/recipe/prime/prime_dp_rm.py @@ -0,0 +1,400 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Implement a multiprocess PPOCritic +""" + +import itertools + +import torch +import torch.distributed +from flash_attn.bert_padding import index_first_axis, pad_input, rearrange, unpad_input +from torch import nn, optim +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + +import verl.utils.torch_functional as verl_F +from verl import DataProto +from verl.utils.device import get_device_name +from verl.utils.py_functional import append_to_dict +from verl.utils.seqlen_balancing import get_reverse_idx, rearrange_micro_batches +from verl.utils.ulysses import gather_outputs_and_unpad, ulysses_pad_and_slice_inputs + +from .prime_core_algos import compute_ce_dpo_loss_rm, compute_detach_dpo_loss_rm + +__all__ = ["DataParallelPRIMERewardModel"] + + +class DataParallelPRIMERewardModel: + def __init__(self, config, reward_module: nn.Module, ref_module: nn.Module, reward_optimizer: optim.Optimizer): + self.config = config + self.reward_module = reward_module + self.ref_module = ref_module + self.reward_optimizer = reward_optimizer + self.use_remove_padding = self.config.model.get("use_remove_padding", False) + print(f"Reward model use_remove_padding={self.use_remove_padding}") + self.use_fused_kernels = self.config.model.get("use_fused_kernels", False) + print(f"Reward model use_fused_kernels={self.use_fused_kernels}") + + self.ulysses_sequence_parallel_size = self.config.get("ulysses_sequence_parallel_size", 1) + + def _forward_micro_batch(self, micro_batch, prompt_length): + input_ids = micro_batch["input_ids"] + batch_size, seqlen = input_ids.shape + attention_mask = micro_batch["attention_mask"] + position_ids = micro_batch["position_ids"] + + num_actions = micro_batch["input_ids"].shape[-1] - prompt_length + max_positions = micro_batch["attention_mask"][:, prompt_length:].sum(-1) + + if self.use_remove_padding: + input_ids_rmpad, indices, *_ = unpad_input( + input_ids.unsqueeze(-1), attention_mask + ) # input_ids_rmpad (total_nnz, ...) + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) + + # unpad the position_ids to align the rotary + position_ids_rmpad = index_first_axis( + rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices + ).transpose(0, 1) + + # for compute the log_prob + input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=1) # (1, total_nnz) + + # pad and slice the inputs if sp > 1 + if self.ulysses_sequence_parallel_size > 1: + input_ids_rmpad, position_ids_rmpad, pad_size = ulysses_pad_and_slice_inputs( + input_ids_rmpad, position_ids_rmpad, sp_size=self.ulysses_sequence_parallel_size + ) + input_ids_rmpad_rolled, _, _ = ulysses_pad_and_slice_inputs( + input_ids_rmpad_rolled, None, self.ulysses_sequence_parallel_size + ) + + input_ids_rmpad_rolled = input_ids_rmpad_rolled.squeeze(0) + output = self.reward_module( + input_ids=input_ids_rmpad, + attention_mask=None, + position_ids=position_ids_rmpad, + use_cache=False, + return_dict=self.use_fused_kernels, + ) + + if self.use_fused_kernels: + rm_log_labels = output.log_probs.squeeze(0) # (total_nnz,) + rm_log_labels = rm_log_labels.to(torch.float32) + + else: + rm_output_logits = output.logits.squeeze(0) + rm_log_labels = verl_F.logprobs_from_logits( + logits=rm_output_logits, + labels=input_ids_rmpad_rolled, + ) + + if self.ulysses_sequence_parallel_size > 1: + rm_log_labels = gather_outputs_and_unpad( + rm_log_labels, gather_dim=0, unpad_dim=0, padding_size=pad_size + ) + rm_log_labels = pad_input( + hidden_states=rm_log_labels.unsqueeze(-1), indices=indices, batch=batch_size, seqlen=seqlen + ).squeeze(-1)[:, -num_actions - 1 : -1] + + else: + output = self.reward_module( + input_ids=micro_batch["input_ids"], + attention_mask=micro_batch["attention_mask"], + position_ids=micro_batch["position_ids"], + use_cache=False, + return_dict=self.use_fused_kernels, + ) + + if self.use_fused_kernels: + rm_log_labels = output.log_probs[:, :-1] # (bsz, seq_length) + rm_log_labels = rm_log_labels.to(torch.float32) + + else: + rm_output_logits = output.logits + rm_log_prob = torch.nn.functional.log_softmax( + rm_output_logits[:, :-1, :], dim=-1 + ) # (batch_size, seq_length, vocab_size) + rm_log_labels = rm_log_prob.gather(dim=-1, index=micro_batch["input_ids"][:, 1:].unsqueeze(-1)).squeeze( + -1 + ) # (batch, seq_length) + + if self.ref_module is not None: + # do not have to pad again + with torch.no_grad(), torch.autocast(device_type=get_device_name(), dtype=torch.bfloat16): + if self.ulysses_sequence_parallel_size > 1 and self.use_remove_padding: + ref_output = self.ref_module( + input_ids=input_ids_rmpad, + attention_mask=None, + position_ids=position_ids_rmpad, + use_cache=False, + ) + + if self.use_fused_kernels: + ref_log_labels = ref_output.log_probs.squeeze(0) # (total_nnz,) + ref_log_labels = ref_log_labels.to(torch.float32) + + else: + ref_output_logits = ref_output.logits.squeeze(0) + ref_log_labels = verl_F.logprobs_from_logits( + logits=ref_output_logits, labels=input_ids_rmpad_rolled + ) + + ref_log_labels = gather_outputs_and_unpad( + ref_log_labels, gather_dim=0, unpad_dim=0, padding_size=pad_size + ) + ref_log_labels = pad_input( + hidden_states=ref_log_labels.unsqueeze(-1), indices=indices, batch=batch_size, seqlen=seqlen + ).squeeze(-1)[:, -num_actions - 1 : -1] + else: + ref_output = self.ref_module( + input_ids=micro_batch["input_ids"], + attention_mask=micro_batch["attention_mask"], + position_ids=micro_batch["position_ids"], + use_cache=False, + ) + + if self.use_fused_kernels: + ref_log_labels = ref_output.log_probs[:, :-1] # (batch_size, seq_length) + ref_log_labels = ref_log_labels.to(torch.float32) + + else: + ref_output_logits = ref_output.logits + ref_log_prob = torch.nn.functional.log_softmax( + ref_output_logits[:, :-1, :], dim=-1 + ) # (batch_size, seq_length, vocab_size) + ref_log_labels = ref_log_prob.gather( + dim=-1, index=micro_batch["input_ids"][:, 1:].unsqueeze(-1) + ).squeeze(-1) # (batch, seq_length) + + else: + ref_log_labels = micro_batch["old_log_probs"] + + ref_log_labels.to(rm_log_labels.dtype) + q = rm_log_labels[:, -num_actions:] - ref_log_labels[:, -num_actions:] # this is actually diff of q + + # trim unnecessary logprobs here + for i in range(micro_batch["input_ids"].shape[0]): + q[i, max_positions[i] :] = 0 + + # reward computation does not need gradient. only q needs + with torch.no_grad(): + # generalized estimation of r should go before the reward filling. r means process reward for policy + # model, or the advantage of reward model. + lam = self.config.get("lambda", 0.0) + beta = self.config.model.get("beta_train", 0.05) + if lam == 0.0: + r = q * beta + else: + # reward coefficient takes no effect here + acc = micro_batch["acc"] + q_ = q * beta + r = torch.zeros_like(q) + lastgaelam = 0 + # change the last token and mask out all paddings to make this process easier if we rely on + # outcome reward to calculate V + for i in range(q.shape[0]): + if self.config.prime_use_gt: + q_[i, max_positions[i] - 1] = acc[i] - q_[i, : max_positions[i] - 1].sum() + q_[i, max_positions[i] :] = 0 + + for t in reversed(range(num_actions)): + delta = q_[:, t] + lastgaelam = delta + lam * lastgaelam + r[:, t] = lastgaelam + + token_level_score = torch.zeros_like(q) + + if self.config.prime_granularity == "token": + for i in range(micro_batch["input_ids"].shape[0]): + token_level_score[i, : max_positions[i] - 1] = r[i, : max_positions[i] - 1] + elif self.config.prime_granularity == "whole": + for i in range(micro_batch["input_ids"].shape[0]): + token_level_score[i, max_positions[i] - 1] = r[i, : max_positions[i]] + else: + raise NotImplementedError + + return token_level_score, q + + def _optimizer_step(self): + assert self.config.model.optim.grad_clip is not None + + if isinstance(self.reward_module, FSDP): + grad_norm = self.reward_module.clip_grad_norm_(self.config.model.optim.grad_clip) + else: + grad_norm = torch.nn.utils.clip_grad_norm_( + self.reward_module.parameters(), max_norm=self.config.model.optim.grad_clip + ) + self.reward_optimizer.step() + return grad_norm + + def prime_norm(self, token_level_scores): + if self.config.prime_norm == "batch_norm": + reverse_cumsum = torch.cumsum(token_level_scores.flip(dims=[1]), dim=-1).flip(dims=[1]) + token_level_scores = token_level_scores / (reverse_cumsum.abs().max() + 1e-6) + return token_level_scores + + def compute_rm_score(self, data: DataProto): + self.reward_module.eval() + self.ref_module.eval() + micro_batch_size = data.meta_info["micro_batch_size"] + select_keys = ["responses", "input_ids", "attention_mask", "position_ids", "acc"] + batch = data.select(batch_keys=select_keys).batch + use_dynamic_bsz = data.meta_info["use_dynamic_bsz"] + prompt_length = data.batch["input_ids"].shape[-1] - data.batch["responses"].shape[-1] + + if use_dynamic_bsz: + # split using dynamic bsz + max_token_len = data.meta_info["max_token_len"] * self.ulysses_sequence_parallel_size + micro_batches, indices = rearrange_micro_batches(batch=batch, max_token_len=max_token_len) + else: + micro_batches = batch.split(micro_batch_size) + + rm_scores_lst = [] + q_lst = [] + for micro_batch in micro_batches: + with torch.no_grad(): + rm_score, q = self._forward_micro_batch(micro_batch, prompt_length) + rm_scores_lst.append(rm_score) + q_lst.append(q) + rm_scores = torch.concat(rm_scores_lst, dim=0) + q = torch.concat(q_lst, dim=0) + + rm_scores = self.prime_norm(rm_scores) + + if use_dynamic_bsz: + indices = list(itertools.chain.from_iterable(indices)) + assert len(indices) == rm_scores.size(0), f"{len(indices)} vs. {rm_scores.size()}" + revert_indices = torch.tensor(get_reverse_idx(indices), dtype=torch.long) + rm_scores = rm_scores[revert_indices] + + return ( + rm_scores, + q.detach(), + { + "reward_model/reward": rm_scores.sum(dim=-1).mean().item(), + "reward_model/raw_reward": q.sum(dim=-1).mean().item(), + }, + ) + + def update_rm(self, data: DataProto): + # make sure we are in training mode + self.reward_module.train() + metrics = {} + + beta = self.config.model.get("beta_train", 0.05) + + select_keys = ["input_ids", "responses", "attention_mask", "position_ids", "acc", "prompts"] + + for key in ["Q_bc", "acc_bc"]: + if key in data.batch.keys(): + select_keys.append(key) + + batch = data.select(batch_keys=select_keys).batch + # Split to make minibatch iterator for updating the actor + # See PPO paper for details. https://arxiv.org/abs/1707.06347 + dataloader = batch.split(self.config.mini_batch_size) + + rm_scores_lst = [] + q_lst = [] + + for batch_idx, data in enumerate(dataloader): + # split batch into micro_batches + mini_batch = data + if self.config.use_dynamic_bsz: + max_token_len = self.config.ppo_max_token_len_per_gpu * self.ulysses_sequence_parallel_size + micro_batches, _ = rearrange_micro_batches(batch=mini_batch, max_token_len=max_token_len) + else: + micro_batches = mini_batch.split(self.config.micro_batch_size_per_gpu) + self.gradient_accumulation = self.config.mini_batch_size // self.config.micro_batch_size_per_gpu + + self.reward_optimizer.zero_grad() + + for data in micro_batches: + data = data.to(get_device_name()) + attention_mask = data["attention_mask"] + acc = data["acc"] + + prompt_ids = data["prompts"] + prompt_length = prompt_ids.shape[-1] + + response_mask = attention_mask[:, prompt_length:] + + rm_score, q = self._forward_micro_batch(data, prompt_length) + + rm_scores_lst.append(rm_score) + q_lst.append(q.detach()) + + if self.config.model.loss_type == "ce": + dpo_loss = compute_ce_dpo_loss_rm(q, acc, response_mask=response_mask, beta=beta) + elif self.config.model.loss_type == "dpo": + # the implementation of dpo is actually detached, which means we have to know the average + # value of w/l reward before the update. + dpo_loss = compute_detach_dpo_loss_rm( + q, acc, Q_bc=data["Q_bc"], acc_bc=data["acc_bc"], response_mask=response_mask, beta=beta + ) + elif self.config.model.loss_type == "bon_acc": + # change the original distribution of each sample to BoN distribution, then update reward model + dpo_loss = compute_detach_dpo_loss_rm( + q, + acc, + Q_bc=data["Q_bc"], + acc_bc=data["acc_bc"], + response_mask=response_mask, + beta=beta, + bon_mode="bon_acc", + ) + elif self.config.model.loss_type == "bon_rm": + dpo_loss = compute_detach_dpo_loss_rm( + q, + acc, + Q_bc=data["Q_bc"], + acc_bc=data["acc_bc"], + response_mask=response_mask, + beta=beta, + bon_mode="bon_rm", + ) + else: + raise NotImplementedError + + data = {"reward_model/dpo_loss": dpo_loss.detach().item()} + + if self.config.use_dynamic_bsz: + # relative to the dynamic bsz + loss = dpo_loss * (len(data) / self.config.ppo_mini_batch_size) + else: + loss = dpo_loss / self.gradient_accumulation + + loss.backward() + + append_to_dict(metrics, data) + + grad_norm = self._optimizer_step() + data = {"reward_model/grad_norm": grad_norm.detach().item()} + append_to_dict(metrics, data) + self.reward_optimizer.zero_grad() + + rm_scores = torch.cat(rm_scores_lst, dim=0) + q = torch.concat(q_lst, dim=0) + + rm_scores = self.prime_norm(rm_scores) + + metrics.update( + { + "reward_model/reward": rm_scores.sum(dim=-1).mean().item(), + "reward_model/raw_reward": q.sum(dim=-1).mean().item(), + } + ) + + return rm_scores, metrics diff --git a/e2c/verl/recipe/prime/prime_fsdp_workers.py b/e2c/verl/recipe/prime/prime_fsdp_workers.py new file mode 100644 index 0000000000000000000000000000000000000000..d41487034f751e6cd430959446e710d033e4f37d --- /dev/null +++ b/e2c/verl/recipe/prime/prime_fsdp_workers.py @@ -0,0 +1,380 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +import warnings + +import torch +import torch.distributed +from omegaconf import OmegaConf +from torch.distributed.device_mesh import init_device_mesh + +from verl import DataProto +from verl.models.transformers.monkey_patch import apply_monkey_patch +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, register +from verl.utils import hf_tokenizer +from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager +from verl.utils.device import get_device_id, get_device_name, get_nccl_backend +from verl.utils.flops_counter import FlopsCounter +from verl.utils.fs import copy_local_path_from_hdfs +from verl.utils.fsdp_utils import ( + get_fsdp_wrap_policy, + get_init_weight_context_manager, + init_fn, + load_fsdp_model_to_gpu, + load_fsdp_optimizer, + offload_fsdp_model_to_cpu, + offload_fsdp_optimizer, +) +from verl.utils.import_utils import import_external_libs +from verl.utils.profiler import log_gpu_memory_usage +from verl.workers.fsdp_workers import create_device_mesh, get_sharding_strategy +from verl.workers.sharding_manager.fsdp_ulysses import FSDPUlyssesShardingManager + +from .prime_core_algos import compute_dpo_abs_accuracy, compute_dpo_accuracy + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class PRIMERewardModelWorker(Worker): + def __init__(self, config): + super().__init__() + import torch.distributed + + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group(backend=get_nccl_backend()) + self.config = config + + # build device mesh for Ulysses Sequence Parallel + world_size = torch.distributed.get_world_size() + + fsdp_size = self.config.model.fsdp_config.fsdp_size + self.device_mesh = create_device_mesh(world_size=world_size, fsdp_size=fsdp_size) + + self.ulysses_device_mesh = None + self.ulysses_sequence_parallel_size = self.config.get("ulysses_sequence_parallel_size", 1) + dp = world_size // self.ulysses_sequence_parallel_size + if self.ulysses_sequence_parallel_size > 1: + self.ulysses_device_mesh = init_device_mesh( + get_device_name(), mesh_shape=(dp, self.ulysses_sequence_parallel_size), mesh_dim_names=["dp", "sp"] + ) + + self.ulysses_sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) + + # set FSDP offload params + self._is_offload_param = self.config.model.fsdp_config.param_offload + self._is_offload_optimizer = self.config.model.fsdp_config.optimizer_offload + + # normalize config + self.config.mini_batch_size //= torch.distributed.get_world_size() // self.ulysses_sequence_parallel_size + if self.config.micro_batch_size is not None: + self.config.micro_batch_size //= torch.distributed.get_world_size() // self.ulysses_sequence_parallel_size + self.config.micro_batch_size_per_gpu = self.config.micro_batch_size + assert self.config.mini_batch_size % self.config.micro_batch_size_per_gpu == 0 + + def _build_reward_ref_model_optimizer(self, config): + # the following line is necessary + from torch import optim + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + from torch.distributed.fsdp import MixedPrecision + + from verl.utils.model import print_model_size + from verl.utils.torch_dtypes import PrecisionType + + local_path = copy_local_path_from_hdfs(config.model.path) + + tokenizer_path = copy_local_path_from_hdfs(config.model.tokenizer_path) + self.tokenizer = hf_tokenizer(tokenizer_path, trust_remote_code=config.model.get("trust_remote_code", False)) + + override_config = OmegaConf.to_container(OmegaConf.create(self.config.model.get("override_config", {}))) + override_config_kwargs = { + "bos_token_id": self.tokenizer.bos_token_id, + "eos_token_id": self.tokenizer.eos_token_id, + "pad_token_id": self.tokenizer.pad_token_id, + } + override_config_kwargs.update(override_config) + if self.rank == 0: + print(f"Reward model overriding config {override_config_kwargs}") + + torch_dtype = self.config.model.fsdp_config.get("model_dtype", "fp32") + torch_dtype = PrecisionType.to_dtype(torch_dtype) + + from transformers import AutoConfig, AutoModelForCausalLM + + trust_remote_code = False + reward_model_config = AutoConfig.from_pretrained(local_path, trust_remote_code=trust_remote_code) + reward_model_config.num_labels = 1 + + init_context = get_init_weight_context_manager(use_meta_tensor=not reward_model_config.tie_word_embeddings) + with init_context(), warnings.catch_warnings(): + warnings.simplefilter("ignore") + reward_model_config.classifier_dropout = 0.0 + reward_model_config.hidden_dropout = "0" + reward_module = AutoModelForCausalLM.from_pretrained( + pretrained_model_name_or_path=local_path, + torch_dtype=torch_dtype, + config=reward_model_config, + attn_implementation="flash_attention_2", + trust_remote_code=trust_remote_code, + ) + + fused_kernel_options = config.model.get("fused_kernel_options", None) + fused_kernels_backend = ( + fused_kernel_options.get("impl_backend", None) if fused_kernel_options is not None else None + ) + + apply_monkey_patch( + model=reward_module, + ulysses_sp_size=self.ulysses_sequence_parallel_size, + use_remove_padding=config.model.get("use_remove_padding", False), + use_fused_kernels=config.model.get("use_fused_kernels", False), + fused_kernels_backend=fused_kernels_backend, + ) + + # some parameters may not in torch_dtype + reward_module.to(torch_dtype) + + if config.model.get("enable_gradient_checkpointing", False): + reward_module.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + if self.rank == 0: + print_model_size(reward_module) + + self.reward_model_config = reward_model_config + + fsdp_config = self.config.model.fsdp_config + mixed_precision_config = fsdp_config.get("mixed_precision", None) + if mixed_precision_config is not None: + param_dtype = PrecisionType.to_dtype(mixed_precision_config.get("param_dtype", "bf16")) + reduce_dtype = PrecisionType.to_dtype(mixed_precision_config.get("reduce_dtype", "fp32")) + buffer_dtype = PrecisionType.to_dtype(mixed_precision_config.get("buffer_dtype", "fp32")) + else: + param_dtype = torch.bfloat16 + reduce_dtype = torch.float32 + buffer_dtype = torch.float32 + + mixed_precision = MixedPrecision(param_dtype=param_dtype, reduce_dtype=reduce_dtype, buffer_dtype=buffer_dtype) + + auto_wrap_policy = get_fsdp_wrap_policy(module=reward_module, config=self.config.model.fsdp_config.wrap_policy) + + log_gpu_memory_usage("Before reward model FSDP", logger=None) + + fsdp_mesh = self.device_mesh + sharding_strategy = get_sharding_strategy(fsdp_mesh) + + with init_context(), warnings.catch_warnings(): + warnings.simplefilter("ignore") + reward_model_config.classifier_dropout = 0.0 + reward_model_config.hidden_dropout = "0" + ref_module = AutoModelForCausalLM.from_pretrained( + pretrained_model_name_or_path=copy_local_path_from_hdfs(config.model.ref_path), + torch_dtype=torch_dtype, + config=reward_model_config, + attn_implementation="flash_attention_2", + trust_remote_code=trust_remote_code, + ) + + # some parameters may not in torch_dtype + ref_module.to(torch_dtype) + + reward_module = FSDP( + reward_module, + param_init_fn=init_fn, + use_orig_params=False, + auto_wrap_policy=auto_wrap_policy, + device_id=get_device_id(), + sharding_strategy=sharding_strategy, + mixed_precision=mixed_precision, + sync_module_states=True, + forward_prefetch=False, + device_mesh=self.device_mesh, + cpu_offload=None, + ) + + log_gpu_memory_usage("After reward FSDP", logger=None) + + ref_module = FSDP( + ref_module, + param_init_fn=init_fn, + use_orig_params=False, + auto_wrap_policy=auto_wrap_policy, + device_id=get_device_id(), + sharding_strategy=sharding_strategy, + mixed_precision=mixed_precision, + sync_module_states=True, + forward_prefetch=False, + device_mesh=self.device_mesh, + cpu_offload=None, + ) + + reward_optimizer = optim.AdamW( + reward_module.parameters(), + lr=config.model.optim.lr, + betas=config.model.optim.get("betas", (0.9, 0.999)), + weight_decay=config.model.optim.get("weight_decay", 1e-2), + ) + + total_steps = config.model.optim.get("total_training_steps", 0) + num_warmup_steps = int(config.model.optim.get("lr_warmup_steps", -1)) + if num_warmup_steps < 0: + num_warmup_steps_ratio = config.model.optim.get("lr_warmup_steps_ratio", 0.0) + num_warmup_steps = int(num_warmup_steps_ratio * total_steps) + + print(f"Total steps: {total_steps}, num_warmup_steps: {num_warmup_steps}") + + from verl.utils.torch_functional import get_constant_schedule_with_warmup + + reward_lr_scheduler = get_constant_schedule_with_warmup( + optimizer=reward_optimizer, num_warmup_steps=num_warmup_steps + ) + + return reward_module, ref_module, reward_optimizer, reward_lr_scheduler + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + # This is used to import external_lib into the huggingface systems + import_external_libs(self.config.model.get("external_lib", None)) + + from .prime_dp_rm import DataParallelPRIMERewardModel + + self.reward_module, self.ref_module, self.reward_optimizer, self.reward_lr_scheduler = ( + self._build_reward_ref_model_optimizer(config=self.config) + ) + + if self._is_offload_param: + offload_fsdp_model_to_cpu(self.reward_module) + offload_fsdp_model_to_cpu(self.ref_module) + if self._is_offload_optimizer: + offload_fsdp_optimizer(optimizer=self.reward_optimizer) + + self.rm = DataParallelPRIMERewardModel( + config=self.config, + reward_module=self.reward_module, + ref_module=self.ref_module, + reward_optimizer=self.reward_optimizer, + ) + + self.flops_counter = FlopsCounter(self.reward_model_config) + self.checkpoint_manager = FSDPCheckpointManager( + model=self.reward_module, + optimizer=self.reward_optimizer, + lr_scheduler=self.reward_lr_scheduler, + tokenizer=self.tokenizer, + ) + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def compute_rm_score(self, data: DataProto): + data = data.to(get_device_name()) + + if self._is_offload_param: + load_fsdp_model_to_gpu(self.reward_module) + load_fsdp_model_to_gpu(self.ref_module) + micro_batch_size = self.config.micro_batch_size_per_gpu + data.meta_info["micro_batch_size"] = micro_batch_size + data.meta_info["max_token_len"] = self.config.forward_max_token_len_per_gpu + data.meta_info["use_dynamic_bsz"] = self.config.use_dynamic_bsz + # perform forward computation + with self.ulysses_sharding_manager: + data = self.ulysses_sharding_manager.preprocess_data(data=data) + rm_scores, q, metrics = self.rm.compute_rm_score(data=data) + + prompt_length = data.batch["prompts"].shape[-1] + response_mask = data.batch["attention_mask"][:, prompt_length:] + acc = data.batch["acc"] + + dpo_acc = compute_dpo_accuracy(rm_scores, acc, response_mask=response_mask, n_samples=data.meta_info["n"]) + dpo_acc_abs = compute_dpo_abs_accuracy(rm_scores, acc, response_mask, n_samples=data.meta_info["n"]) + + metrics["reward_model/dpo_acc"] = dpo_acc.detach().item() + metrics["reward_model/dpo_acc_abs"] = dpo_acc_abs.detach().item() + + output = DataProto.from_dict(tensors={"rm_scores": rm_scores, "q": q}, meta_info={"metrics": metrics}) + output = self.ulysses_sharding_manager.postprocess_data(data=output) + + output = output.to("cpu") + if self._is_offload_param: + offload_fsdp_model_to_cpu(self.reward_module) + offload_fsdp_model_to_cpu(self.ref_module) + return output + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def update_rm(self, data: DataProto): + data = data.to(get_device_name()) + if self._is_offload_param: + load_fsdp_model_to_gpu(self.ref_module) + load_fsdp_model_to_gpu(self.reward_module) + if self._is_offload_optimizer: + load_fsdp_optimizer(optimizer=self.reward_optimizer, device_id=get_device_id()) + + # perform forward computation + with self.ulysses_sharding_manager: + data = self.ulysses_sharding_manager.preprocess_data(data=data) + + rm_scores, metrics = self.rm.update_rm(data=data) + + self.reward_lr_scheduler.step() + lr = self.reward_lr_scheduler.get_last_lr()[0] + metrics["rm/lr"] = lr + + prompt_length = data.batch["prompts"].shape[-1] + response_mask = data.batch["attention_mask"][:, prompt_length:] + acc = data.batch["acc"] + + dpo_acc_before = compute_dpo_accuracy( + rm_scores, acc, response_mask=response_mask, n_samples=data.meta_info["n"] + ) + dpo_acc_abs = compute_dpo_abs_accuracy(rm_scores, acc, response_mask, n_samples=data.meta_info["n"]) + + metrics["reward_model/dpo_acc_before"] = dpo_acc_before.detach().item() + metrics["reward_model/dpo_acc_abs_before"] = dpo_acc_abs.detach().item() + + output = DataProto.from_dict(tensors={"rm_scores": rm_scores}, meta_info={"metrics": metrics}) + output = self.ulysses_sharding_manager.postprocess_data(data=output) + + if self._is_offload_param: + offload_fsdp_model_to_cpu(self.reward_module) + offload_fsdp_model_to_cpu(self.ref_module) + if self._is_offload_optimizer: + offload_fsdp_optimizer(optimizer=self.reward_optimizer) + output = output.to("cpu") + return output + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): + import torch + + if self._is_offload_param: + load_fsdp_model_to_gpu(self.reward_module) + + self.checkpoint_manager.save_checkpoint( + local_path=local_path, hdfs_path=hdfs_path, global_step=global_step, max_ckpt_to_keep=max_ckpt_to_keep + ) + + torch.distributed.barrier() + if self._is_offload_param: + offload_fsdp_model_to_cpu(self.reward_module) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def load_checkpoint(self, local_path, del_local_after_load=True): + import torch + + if self._is_offload_param: + load_fsdp_model_to_gpu(self.reward_module) + + self.checkpoint_manager.load_checkpoint(local_path=local_path, del_local_after_load=del_local_after_load) + + torch.distributed.barrier() + if self._is_offload_param: + offload_fsdp_model_to_cpu(self.reward_module) diff --git a/e2c/verl/recipe/prime/prime_ray_trainer.py b/e2c/verl/recipe/prime/prime_ray_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..6782b32256a2e37c8a115047bdb7c69c09d4f5ee --- /dev/null +++ b/e2c/verl/recipe/prime/prime_ray_trainer.py @@ -0,0 +1,572 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +FSDP PPO Trainer with Ray-based single controller. +This trainer supports model-agonistic model initialization with huggingface +""" + +import os +import statistics +import uuid +from copy import deepcopy +from pprint import pprint + +import numpy as np +import torch +from omegaconf import OmegaConf, open_dict + +from verl import DataProto +from verl.single_controller.ray import RayWorkerGroup +from verl.trainer.ppo.core_algos import agg_loss +from verl.trainer.ppo.metric_utils import _compute_response_info +from verl.trainer.ppo.ray_trainer import RayPPOTrainer, ResourcePoolManager +from verl.trainer.ppo.utils import Role, WorkerType +from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path +from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn +from verl.utils.metric import reduce_metrics +from verl.utils.profiler.performance import simple_timer + +from . import prime_core_algos + + +def compute_advantage(data: DataProto, adv_estimator, config): + if adv_estimator == "rloo": + responses = data.batch["responses"] + response_length = responses.size(-1) + attention_mask = data.batch["attention_mask"] + response_mask = attention_mask[:, -response_length:] + advantages, returns = prime_core_algos.compute_rloo_advantage_return( + data, response_mask, config.actor_rollout_ref.rollout.n, config + ) + data.batch["advantages"] = advantages + data.batch["returns"] = returns + else: + raise NotImplementedError + return data + + +def compute_data_metrics(batch, use_critic=True): + advantages = batch.batch["advantages"] + returns = batch.batch["returns"] + + max_response_length = batch.batch["responses"].shape[-1] + + prompt_mask = batch.batch["attention_mask"][:, :-max_response_length].bool() + response_mask = batch.batch["attention_mask"][:, -max_response_length:].bool() + + max_prompt_length = prompt_mask.size(-1) + + response_info = _compute_response_info(batch) + prompt_length = response_info["prompt_length"] + response_length = response_info["response_length"] + + valid_adv = torch.masked_select(advantages, response_mask) + valid_returns = torch.masked_select(returns, response_mask) + + if use_critic: + values = batch.batch["values"] + valid_values = torch.masked_select(values, response_mask) + return_diff_var = torch.var(valid_returns - valid_values) + return_var = torch.var(valid_returns) + + metrics = { + # adv + "critic/advantages/mean": torch.mean(valid_adv).detach().item(), + "critic/advantages/max": torch.max(valid_adv).detach().item(), + "critic/advantages/min": torch.min(valid_adv).detach().item(), + # returns + "critic/returns/mean": torch.mean(valid_returns).detach().item(), + "critic/returns/max": torch.max(valid_returns).detach().item(), + "critic/returns/min": torch.min(valid_returns).detach().item(), + **( + { + # values + "critic/values/mean": torch.mean(valid_values).detach().item(), + "critic/values/max": torch.max(valid_values).detach().item(), + "critic/values/min": torch.min(valid_values).detach().item(), + # vf explained var + "critic/vf_explained_var": (1.0 - return_diff_var / (return_var + 1e-5)).detach().item(), + } + if use_critic + else {} + ), + # response length + "response_length/mean": torch.mean(response_length).detach().item(), + "response_length/max": torch.max(response_length).detach().item(), + "response_length/min": torch.min(response_length).detach().item(), + "response_length/clip_ratio": torch.mean(torch.eq(response_length, max_response_length).float()) + .detach() + .item(), + # prompt length + "prompt_length/mean": torch.mean(prompt_length).detach().item(), + "prompt_length/max": torch.max(prompt_length).detach().item(), + "prompt_length/min": torch.min(prompt_length).detach().item(), + "prompt_length/clip_ratio": torch.mean(torch.eq(prompt_length, max_prompt_length).float()).detach().item(), + } + return metrics + + +def compute_response_mask(data: DataProto): + responses = data.batch["responses"] + response_length = responses.size(1) + attention_mask = data.batch["attention_mask"] + return attention_mask[:, -response_length:] + + +def compute_timing_metrics(batch, timing_raw): + response_info = _compute_response_info(batch) + num_prompt_tokens = torch.sum(response_info["prompt_length"]).item() + num_response_tokens = torch.sum(response_info["response_length"]).item() + num_overall_tokens = num_prompt_tokens + num_response_tokens + + num_tokens_of_section = { + "gen": num_response_tokens, + **{name: num_overall_tokens for name in ["ref", "values", "adv", "update_critic", "update_actor"]}, + } + + return { + **{f"timing_s/{name}": value for name, value in timing_raw.items()}, + **{ + f"timing_per_token_ms/{name}": timing_raw[name] * 1000 / num_tokens_of_section[name] + for name in set(num_tokens_of_section.keys()) & set(timing_raw.keys()) + }, + } + + +class RayPRIMETrainer(RayPPOTrainer): + """ + Note that this trainer runs on the driver process on a single CPU/GPU node. + """ + + # TODO: support each role have individual ray_worker_group_cls, + # i.e., support different backend of different role + def __init__( + self, + config, + tokenizer, + role_worker_mapping: dict[Role, WorkerType], + resource_pool_manager: ResourcePoolManager, + ray_worker_group_cls: RayWorkerGroup = RayWorkerGroup, + reward_fn=None, + val_reward_fn=None, + device_name="cuda", + ): + # assert get_torch_device().is_available(), 'cuda must be available on driver' + + super().__init__( + config, + tokenizer, + role_worker_mapping, + resource_pool_manager, + ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + device_name=device_name, + ) + + self.use_critic = False + + def _create_dataloader(self, *args, **kwargs): + from torch.utils.data import DataLoader, RandomSampler, SequentialSampler + + # TODO: we have to make sure the batch size is divisible by the dp size + self.train_dataset = RLHFDataset( + data_files=self.config.data.train_files, tokenizer=self.tokenizer, config=self.config.data + ) + # use sampler for better ckpt resume + if self.config.data.shuffle: + train_dataloader_generator = torch.Generator() + train_dataloader_generator.manual_seed(self.config.data.get("seed", 1)) + sampler = RandomSampler(data_source=self.train_dataset, generator=train_dataloader_generator) + else: + sampler = SequentialSampler(data_source=self.train_dataset) + + self.train_dataloader = DataLoader( + dataset=self.train_dataset, + batch_size=int(self.config.data.train_batch_size * self.config.data.oversample_factor), + drop_last=True, + collate_fn=collate_fn, + sampler=sampler, + ) + + self.val_dataset = RLHFDataset( + data_files=self.config.data.val_files, tokenizer=self.tokenizer, config=self.config.data + ) + self.val_dataloader = DataLoader( + dataset=self.val_dataset, + batch_size=len(self.val_dataset), + shuffle=True, + drop_last=True, + collate_fn=collate_fn, + ) + + assert len(self.train_dataloader) >= 1 + assert len(self.val_dataloader) >= 1 + + print(f"Size of train dataloader: {len(self.train_dataloader)}") + print(f"Size of val dataloader: {len(self.val_dataloader)}") + + # inject total_training_steps to actor/critic optim_config. This is hacky. + total_training_steps = len(self.train_dataloader) * self.config.trainer.total_epochs + + if self.config.trainer.total_training_steps is not None: + total_training_steps = self.config.trainer.total_training_steps + + self.total_training_steps = total_training_steps + print(f"Total training steps: {self.total_training_steps}") + + OmegaConf.set_struct(self.config, True) + with open_dict(self.config): + self.config.actor_rollout_ref.actor.optim.total_training_steps = total_training_steps + self.config.critic.optim.total_training_steps = total_training_steps + + def _save_checkpoint(self): + # path: given_path + `/global_step_{global_steps}` + `/actor` + local_global_step_folder = os.path.join( + self.config.trainer.default_local_dir, f"global_step_{self.global_steps}" + ) + print(f"local_global_step_folder: {local_global_step_folder}") + actor_local_path = os.path.join(local_global_step_folder, "actor") + + actor_remote_path = ( + None + if self.config.trainer.default_hdfs_dir is None + else os.path.join(self.config.trainer.default_hdfs_dir, f"global_step_{self.global_steps}", "actor") + ) + self.actor_rollout_wg.save_checkpoint( + actor_local_path, + actor_remote_path, + self.global_steps, + ) + + if self.use_rm: + reward_local_path = os.path.join(local_global_step_folder, "reward") + reward_remote_path = ( + None + if self.config.trainer.default_hdfs_dir is None + else os.path.join(self.config.trainer.default_hdfs_dir, f"global_step_{self.global_steps}", "reward") + ) + self.rm_wg.save_checkpoint( + reward_local_path, + reward_remote_path, + self.global_steps, + ) + + # save dataloader + dataloader_local_path = os.path.join(local_global_step_folder, "data.pt") + import dill + + torch.save(self.train_dataloader, dataloader_local_path, pickle_module=dill) + + # latest checkpointed iteration tracker (for atomic usage) + local_latest_checkpointed_iteration = os.path.join( + self.config.trainer.default_local_dir, "latest_checkpointed_iteration.txt" + ) + with open(local_latest_checkpointed_iteration, "w") as f: + f.write(str(self.global_steps)) + + def _load_checkpoint(self): + if self.config.trainer.resume_mode == "disable": + return 0 + + # load from hdfs + if self.config.trainer.default_hdfs_dir is not None: + NotImplementedError("load from hdfs is not implemented yet") + else: + checkpoint_folder = self.config.trainer.default_local_dir # TODO: check path + if not os.path.isabs(checkpoint_folder): + working_dir = os.getcwd() + checkpoint_folder = os.path.join(working_dir, checkpoint_folder) + global_step_folder = find_latest_ckpt_path(checkpoint_folder) # None if no latest + + # find global_step_folder + if self.config.trainer.resume_mode == "auto": + if global_step_folder is None: + print("Training from scratch") + return 0 + else: + if self.config.trainer.resume_mode == "resume_path": + assert isinstance(self.config.trainer.resume_from_path, str), "resume ckpt must be str type" + assert "global_step_" in self.config.trainer.resume_from_path, ( + "resume ckpt must specify the global_steps" + ) + global_step_folder = self.config.trainer.resume_from_path + if not os.path.isabs(global_step_folder): + working_dir = os.getcwd() + global_step_folder = os.path.join(working_dir, global_step_folder) + print(f"Load from checkpoint folder: {global_step_folder}") + # set global step + self.global_steps = int(global_step_folder.split("global_step_")[-1]) + + print(f"Setting global step to {self.global_steps}") + print(f"Resuming from {global_step_folder}") + + actor_path = os.path.join(global_step_folder, "actor") + reward_path = os.path.join(global_step_folder, "reward") + # load actor + self.actor_rollout_wg.load_checkpoint( + actor_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load + ) + # load rm + if self.use_rm: + self.rm_wg.load_checkpoint(reward_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load) + + # load dataloader, + # TODO: from remote not implemented yet + dataloader_local_path = os.path.join(global_step_folder, "data.pt") + self.train_dataloader = torch.load(dataloader_local_path) + if isinstance(self.train_dataloader.dataset, RLHFDataset): + self.train_dataloader.dataset.resume_dataset_state() + + def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the worker group through RPC to + construct the PPO dataflow. The light-weight advantage computation is done on the driver process. + """ + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + self.global_steps = 0 + + # load checkpoint before doing anything + self._load_checkpoint() + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + val_metrics = self._validate() + assert val_metrics, f"{val_metrics=}" + pprint(f"Initial validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if self.config.trainer.get("val_only", False): + return + + # we start from step 1 + self.global_steps += 1 + + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + metrics = {} + timing_raw = {} + + batch: DataProto = DataProto.from_single_dict(batch_dict) + + # pop those keys for generation + gen_batch = batch.pop(batch_keys=["input_ids", "attention_mask", "position_ids"]) + gen_batch = gen_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + + with simple_timer("step", timing_raw): + # generate a batch + with simple_timer("gen", timing_raw): + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + timing_raw.update(gen_batch_output.meta_info["timing"]) + gen_batch_output.meta_info.pop("timing", None) + + if self.config.algorithm.adv_estimator == "remax": + with simple_timer("gen_max", timing_raw): + gen_baseline_batch = deepcopy(gen_batch) + gen_baseline_batch.meta_info["do_sample"] = False + gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) + + batch = batch.union(gen_baseline_output) + reward_baseline_tensor = self.reward_fn(batch) + reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) + + batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) + + batch.batch["reward_baselines"] = reward_baseline_tensor + + del gen_baseline_batch, gen_baseline_output + + batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(batch.batch))], dtype=object + ) + # repeat to align with repeated responses in rollout + batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + batch = batch.union(gen_batch_output) + + # Balance the number of valid tokens across DP ranks. + # NOTE: This usually changes the order of data in the `batch`, + # which won't affect the advantage calculation (since it's based on uid), + # but might affect the loss calculation (due to the change of mini-batching). + # TODO: Decouple the DP balancing and mini-batching. + if self.config.trainer.balance_batch: + self._balance_batch(batch, metrics=metrics) + + # compute global_valid tokens + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + # verify + with simple_timer("verify", timing_raw): + scores = self.reward_fn.verify(batch) + metrics["acc"] = statistics.mean(scores) + + # filter the batch. 1/oversample_factor samples will be kept. + # If there is a filter, prompts passing it will be prioritized. + + batch = self.filter_and_downsample(scores, batch) + batch.meta_info["n"] = self.config.actor_rollout_ref.rollout.n + n_samples = self.config.actor_rollout_ref.rollout.n + + # recompute old_log_probs + with simple_timer("old_log_prob", timing_raw): + old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) + entropys = old_log_prob.batch["entropys"] + response_masks = compute_response_mask(batch) + loss_agg_mode = self.config.actor_rollout_ref.actor.loss_agg_mode + entropy_agg = agg_loss(loss_mat=entropys, loss_mask=response_masks, loss_agg_mode=loss_agg_mode) + old_log_prob_metrics = {"actor/entropy": entropy_agg.detach().item()} + metrics.update(old_log_prob_metrics) + old_log_prob.batch.pop("entropys") + batch = batch.union(old_log_prob) + + if self.use_reference_policy: + # compute reference log_prob + with simple_timer("ref", timing_raw): + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + + with simple_timer("adv", timing_raw): + if self.use_rm: + update_style = self.config.reward_model.model.get("update", "none") + if update_style == "none": # only run forward + reward_output = self.rm_wg.compute_rm_score(batch) + elif update_style == "after": # update and directly return the reward + reward_output = self.rm_wg.update_rm(batch) + elif update_style == "before": # update reward model, and then run forward + reward_output = self.rm_wg.update_rm(batch) + if "metrics" in reward_output.meta_info.keys(): + reward_output_metrics = reduce_metrics(reward_output.meta_info["metrics"]) + metrics.update(reward_output_metrics) + + reward_output = self.rm_wg.compute_rm_score(batch) + elif ( + update_style == "reverse" + ): # run forward to calculate statistics, then update reward model + reward_output = self.rm_wg.compute_rm_score(batch) + # broadcast q and acc tensor to each result + bc_td = DataProto.from_dict( + tensors={ + "Q_bc": reward_output.batch["q"] + .sum(dim=-1) + .view(-1, n_samples) + .unsqueeze(1) + .expand(-1, n_samples, -1) + .reshape(-1, n_samples), + "acc_bc": batch.batch["acc"] + .view(-1, n_samples) + .unsqueeze(1) + .expand(-1, n_samples, -1) + .reshape(-1, n_samples), + } + ) + batch = batch.union(bc_td) + reward_output = self.rm_wg.update_rm(batch) + else: + raise NotImplementedError + batch = batch.union(reward_output) + if "metrics" in reward_output.meta_info.keys(): + reward_output_metrics = reduce_metrics(reward_output.meta_info["metrics"]) + metrics.update(reward_output_metrics) + + # compute advantages, executed on the driver process + batch = compute_advantage( + batch, adv_estimator=self.config.algorithm.adv_estimator, config=self.config + ) + + # update actor + with simple_timer("update_actor", timing_raw): + actor_output = self.actor_rollout_wg.update_actor(batch) + actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"]) + metrics.update(actor_output_metrics) + + # validate + if ( + self.val_reward_fn is not None + and self.config.trainer.test_freq > 0 + and self.global_steps % self.config.trainer.test_freq == 0 + ): + with simple_timer("testing", timing_raw): + val_metrics: dict = self._validate() + metrics.update(val_metrics) + + if self.config.trainer.save_freq > 0 and self.global_steps % self.config.trainer.save_freq == 0: + with simple_timer("save_checkpoint", timing_raw): + self._save_checkpoint() + + # collect metrics + metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) + metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=self.global_steps) + + self.global_steps += 1 + + if self.global_steps >= self.total_training_steps: + # perform validation after training + if self.val_reward_fn is not None: + val_metrics = self._validate() + pprint(f"Final validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if ( + self.config.trainer.save_freq > 0 + and (self.global_steps - 1) % self.config.trainer.save_freq != 0 + ): + with simple_timer("save_checkpoint", timing_raw): + self._save_checkpoint() + return + + def filter_and_downsample(self, scores, batch: DataProto): + """ + downsample the batch according to oversample_factor + samples passing the filters will be prioritized + """ + n_samples = int(self.config.actor_rollout_ref.rollout.n) + reward_matrix = torch.tensor(scores).reshape(-1, n_samples) + + filter_mask = torch.ones((reward_matrix.shape[0]), dtype=torch.bool) + + if self.config.data.filter_accuracy: + acc_tensor = torch.mean(reward_matrix, dim=-1) + filter_mask[ + (acc_tensor > self.config.data.accuracy_upper_bound) + | (acc_tensor < self.config.data.accuracy_lower_bound) + ] = False + + if self.config.data.filter_truncate: + length_matrix = ( + batch.batch["attention_mask"][:, -batch.batch["responses"].shape[-1] :] + .sum(dim=-1) + .reshape(-1, n_samples) + ) + length_tensor = torch.max(length_matrix, dim=-1)[0] + filter_mask[length_tensor >= self.config.data.max_response_length - 1] = False + + reorder_index = torch.argsort(filter_mask, descending=True) + reorder_index = (reorder_index.unsqueeze(-1) * n_samples + torch.arange(0, n_samples).unsqueeze(0)).view(-1) + batch.reorder( + reorder_index[: int(len(batch) // self.config.data.oversample_factor)] + ) # this operation is inplace + + return batch diff --git a/e2c/verl/recipe/prime/run_prime_qwen.sh b/e2c/verl/recipe/prime/run_prime_qwen.sh new file mode 100644 index 0000000000000000000000000000000000000000..145f31b7bada41456f2b5b069016a51eeab82602 --- /dev/null +++ b/e2c/verl/recipe/prime/run_prime_qwen.sh @@ -0,0 +1,64 @@ +set -x + + +gsm8k_train_path=$HOME/data/gsm8k/train.parquet +gsm8k_test_path=$HOME/data/gsm8k/test.parquet + +# download from https://huggingface.co/datasets/PRIME-RL/Eurus-2-RL-Data +math_train_path=$HOME/data/math/train.parquet +math_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path', '$math_train_path']" +test_files="['$gsm8k_test_path', '$math_test_path']" + +model_path=PRIME-RL/Eurus-2-7B-SFT +# model_path=Qwen/Qwen2.5-0.5B-Instruct + +python3 -m recipe.prime.main_prime \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=64 \ + data.val_batch_size=6312 \ + data.max_prompt_length=1024 \ + data.max_response_length=3072 \ + data.filter_overlong_prompts=True \ + data.filter_accuracy=True \ + data.accuracy_lower_bound=0.2 \ + data.accuracy_upper_bound=0.8 \ + data.oversample_factor=4 \ + actor_rollout_ref.model.path=$model_path \ + actor_rollout_ref.actor.optim.lr=5e-7 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + algorithm.adv_estimator=rloo \ + algorithm.use_kl_in_reward=True \ + algorithm.kl_penalty=kl \ + algorithm.kl_ctrl.kl_coef=0.001 \ + reward_model.model.path=$model_path \ + reward_model.micro_batch_size_per_gpu=1 \ + reward_model.model.update=before \ + reward_model.model.beta_train=0.05 \ + reward_model.model.optim.lr=1e-6 \ + reward_model.model.optim.grad_clip=10.0 \ + reward_model.model.input_tokenizer=null \ + reward_model.mini_batch_size=64 \ + trainer.val_before_train=False \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='prime_example' \ + trainer.experiment_name='Eurus-2-7B-SFT-gsm8k' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=64 \ + trainer.test_freq=64 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/recipe/prime/run_prime_qwen_code.sh b/e2c/verl/recipe/prime/run_prime_qwen_code.sh new file mode 100644 index 0000000000000000000000000000000000000000..e179c0858ab0f4819a4f9cd7ebf58cf5b7acd194 --- /dev/null +++ b/e2c/verl/recipe/prime/run_prime_qwen_code.sh @@ -0,0 +1,61 @@ +set -x + + +# download from https://huggingface.co/datasets/PRIME-RL/Eurus-2-RL-Data +code_train_path=$HOME/data/code/train.parquet +code_test_path=$HOME/data/code/test.parquet + +train_files="['$code_train_path']" +test_files="['$code_test_path']" + +model_path=PRIME-RL/Eurus-2-7B-SFT +# model_path=Qwen/Qwen2.5-0.5B-Instruct + +python3 -m recipe.prime.main_prime \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=64 \ + data.val_batch_size=6312 \ + data.max_prompt_length=1024 \ + data.max_response_length=3072 \ + data.filter_overlong_prompts=True \ + data.filter_accuracy=True \ + data.accuracy_lower_bound=0.2 \ + data.accuracy_upper_bound=0.8 \ + data.oversample_factor=4 \ + actor_rollout_ref.model.path=$model_path \ + actor_rollout_ref.actor.optim.lr=5e-7 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + algorithm.adv_estimator=rloo \ + algorithm.use_kl_in_reward=True \ + algorithm.kl_penalty=kl \ + algorithm.kl_ctrl.kl_coef=0.001 \ + reward_model.model.path=$model_path \ + reward_model.micro_batch_size_per_gpu=1 \ + reward_model.model.update=before \ + reward_model.model.beta_train=0.05 \ + reward_model.model.optim.lr=1e-6 \ + reward_model.model.optim.grad_clip=10.0 \ + reward_model.model.input_tokenizer=null \ + reward_model.mini_batch_size=64 \ + trainer.val_before_train=False \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='prime_example' \ + trainer.experiment_name='Eurus-2-7B-SFT-code' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=64 \ + trainer.test_freq=64 \ + trainer.total_epochs=15 $@ diff --git a/e2c/verl/recipe/r1/README.md b/e2c/verl/recipe/r1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ddd23bcc3abe7560c50af6082a2a2bdb6601fe39 --- /dev/null +++ b/e2c/verl/recipe/r1/README.md @@ -0,0 +1,26 @@ +# DeepSeek R1 Reproduction + +This recipe is under development, if you are interested, checkout the TODO list and join this project! https://github.com/volcengine/verl/issues/708 + +## Reproducing Evaluation + +Eval Results of DS-R1-Distill-Qwen2.5-1.5B (k=8) + +Dataset | Test Results | Reported +-- | -- | -- +GPQA Diamond | 35.3 | 33.8 +LiveCodeBench | 16.9 | 16.9 +AIME 2024 | 30.4 | 28.9 +CNMO 2024 (en) | 45.1 | - +CNMO 2024 (zh) | 41.0 | - + +--- + +Eval Results (DS-R1) + +Dataset | Test Results (k=1) | Test Results (k=4) | Reported +-- | -- | -- | -- +GPQA Diamond | 67.7 | 69.6 | 71.5 +LiveCodeBench | 64.7 | 63.1 | 65.9 +AIME 2024 | 86.7 | 79.2 | 79.8 +CNMO 2024 | 75.0 | 78.5 | 78.8 diff --git a/e2c/verl/recipe/r1/__init__.py b/e2c/verl/recipe/r1/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/recipe/r1/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/recipe/r1/config/evaluation.yaml b/e2c/verl/recipe/r1/config/evaluation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fe664ae43aa28584f8d946e11b06d346e9cab86 --- /dev/null +++ b/e2c/verl/recipe/r1/config/evaluation.yaml @@ -0,0 +1,14 @@ +data: + path: /tmp/math_Qwen2-7B-Instruct.parquet + prompt_key: prompt + response_key: responses + data_source_key: data_source + reward_model_key: reward_model + +custom_reward_function: + path: null + name: compute_score + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. \ No newline at end of file diff --git a/e2c/verl/recipe/r1/data_process.py b/e2c/verl/recipe/r1/data_process.py new file mode 100644 index 0000000000000000000000000000000000000000..fb41c814371aa21e4f08af449b43c0a4e5753634 --- /dev/null +++ b/e2c/verl/recipe/r1/data_process.py @@ -0,0 +1,203 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the dataset to parquet format +""" + +import argparse +import os +from functools import partial + +from datasets import concatenate_datasets, load_dataset + +from verl.utils.hdfs_io import copy, makedirs + + +def example_map_fn(example, idx, process_fn, data_source, ability, split): + question, solution = process_fn(example) + data = { + "data_source": data_source, + "prompt": [{"role": "user", "content": question}], + "ability": ability, + "reward_model": {"style": "rule", "ground_truth": solution}, + "extra_info": {"split": split, "index": idx}, + } + return data + + +def build_aime2024_dataset(): + def process_aime2024(example): + return example["Problem"], str(example["Answer"]) + + data_source = "Maxwell-Jia/AIME_2024" + print(f"Loading the {data_source} dataset from huggingface...", flush=True) + dataset = load_dataset(data_source, split="train") + map_fn = partial( + example_map_fn, process_fn=process_aime2024, data_source=data_source, ability="English", split="test" + ) + dataset = dataset.map(map_fn, with_indices=True, remove_columns=dataset.column_names) + return dataset + + +def build_gpqa_dimond_dataset(): + import random + + GPQA_QUERY_TEMPLATE = ( + "Answer the following multiple choice question. The last line of your response should be of the following " + "format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before " + "answering.\n\n{Question}\n\nA) {A}\nB) {B}\nC) {C}\nD) {D}" + ) + + def process_gpqa_diamond(example): + choices = [example["Incorrect Answer 1"], example["Incorrect Answer 2"], example["Incorrect Answer 3"]] + random.shuffle(choices) + gold_index = random.randint(0, 3) + choices.insert(gold_index, example["Correct Answer"]) + query_prompt = GPQA_QUERY_TEMPLATE.format( + A=choices[0], B=choices[1], C=choices[2], D=choices[3], Question=example["Question"] + ) + gold_choice = "ABCD"[gold_index] + return query_prompt, gold_choice + + data_source = "Idavidrein/gpqa" + print(f"Loading the {data_source} dataset from huggingface...", flush=True) + + dataset = load_dataset(data_source, "gpqa_diamond", split="train") + map_fn = partial( + example_map_fn, process_fn=process_gpqa_diamond, data_source=data_source, ability="Math", split="test" + ) + dataset = dataset.map(map_fn, with_indices=True, remove_columns=dataset.column_names) + return dataset + + +def build_cnmo2024_dataset(): + def process_cnmo2024(example): + return example["question"], example["answer"] + + data_source = "opencompass/LiveMathBench" + print(f"Loading the {data_source} dataset from huggingface...", flush=True) + + dataset_en = load_dataset(data_source, "v202412_CNMO_en", split="test") + map_fn_en = partial( + example_map_fn, process_fn=process_cnmo2024, data_source="opencompass/cnmo2024_en", ability="Math", split="test" + ) + dataset_en = dataset_en.map(map_fn_en, with_indices=True, remove_columns=dataset_en.column_names) + + dataset_zh = load_dataset(data_source, "v202412_CNMO_cn", split="test") + map_fn_zh = partial( + example_map_fn, process_fn=process_cnmo2024, data_source="opencompass/cnmo2024_zh", ability="Math", split="test" + ) + dataset_zh = dataset_zh.map(map_fn_zh, with_indices=True, remove_columns=dataset_zh.column_names) + + dataset = concatenate_datasets([dataset_en, dataset_zh]) + return dataset + + +def build_livecodebench_dataset(): + import base64 + import json + import pickle + import zlib + + def process_livecodebench(example): + # Construct Query Prompt + # From https://github.com/LiveCodeBench/LiveCodeBench/blob/998c52d394b836f15fff3b9a29866191108ff81b/lcb_runner/prompts/code_generation.py#L140 + query_prompt = ( + f"You will be given a question (problem specification) and will generate a correct Python program " + f"that matches the specification and passes all tests.\n\nQuestion: {example['question_content']}\n\n" + ) + if example["starter_code"]: + query_prompt += ( + f"You will use the following starter code to write the solution to the problem and enclose your " + f"code within delimiters.\n```python\n{example['starter_code']}\n```" + ) + else: + query_prompt += ( + "Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test " + "on the sample inputs). Enclose your code within delimiters as follows. Ensure that when the python " + "program runs, it reads the inputs, runs the algorithm and writes output to STDOUT." + "```python\n# YOUR CODE HERE\n```" + ) + + # Construct test cases + public_test_cases = json.loads(example["public_test_cases"]) + try: + private_test_cases = json.loads(example["private_test_cases"]) + except Exception as e: + print(f"Error loading private test cases: {e}") + private_test_cases = json.loads( + pickle.loads(zlib.decompress(base64.b64decode(example["private_test_cases"].encode("utf-8")))) + ) + full_test_cases = public_test_cases + private_test_cases + + metadata = json.loads(example["metadata"]) + test_cases = { + "inputs": [t["input"] for t in full_test_cases], + "outputs": [t["output"] for t in full_test_cases], + "fn_name": metadata.get("func_name", None), + } + text_cases_compressed = base64.b64encode(zlib.compress(pickle.dumps(json.dumps(test_cases)))).decode("utf-8") + return query_prompt, text_cases_compressed + + data_source = "livecodebench/code_generation_lite" + print(f"Loading the {data_source} dataset from huggingface...", flush=True) + dataset = load_dataset(data_source, split="test") + # R1 Evaluation use LiveCodeBench 24.08-25.01 + dataset = dataset.filter(lambda line: "2024-08-00T00:00:00" <= line["contest_date"] < "2025-01-00T00:00:00") + map_fn = partial( + example_map_fn, process_fn=process_livecodebench, data_source=data_source, ability="Code", split="test" + ) + + dataset = dataset.map(map_fn, with_indices=True, remove_columns=dataset.column_names, num_proc=8) + return dataset + + +TASK2DATA = { + "aime2024": build_aime2024_dataset, + "gpqa_diamond": build_gpqa_dimond_dataset, + "cnmo2024": build_cnmo2024_dataset, + "livecodebench": build_livecodebench_dataset, +} +SUPPORTED_TASKS = TASK2DATA.keys() + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default="~/data/r1") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--tasks", default="all") + + args = parser.parse_args() + + if args.tasks.lower() == "all": + args.tasks = SUPPORTED_TASKS + else: + args.tasks = [task.strip() for task in args.tasks.split(",") if task.strip()] + for task in args.tasks: + if task not in SUPPORTED_TASKS: + raise NotImplementedError(f"{task} has not been supported.") + + datasets = [] + for task in args.tasks: + datasets.append(TASK2DATA[task]()) + test_dataset = concatenate_datasets(datasets) + + local_dir = args.local_dir + hdfs_dir = args.hdfs_dir + + test_dataset.to_parquet(os.path.join(local_dir, "test.parquet")) + + if hdfs_dir is not None: + makedirs(hdfs_dir) + + copy(src=local_dir, dst=hdfs_dir) diff --git a/e2c/verl/recipe/r1/main_eval.py b/e2c/verl/recipe/r1/main_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..5c0e735a1a582071bde0a9eaf6681085c8b4272c --- /dev/null +++ b/e2c/verl/recipe/r1/main_eval.py @@ -0,0 +1,81 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Offline evaluate the performance of a generated file using reward model and ground truth verifier. +The input is a parquet file that contains N generated sequences and (optional) the ground truth. + +""" + +from collections import defaultdict + +import hydra +import numpy as np +import pandas as pd +import ray +from omegaconf import OmegaConf +from tqdm import tqdm + +from verl.trainer.ppo.reward import get_custom_reward_fn +from verl.utils.fs import copy_to_local + + +@ray.remote +def process_item(config, data_source, response_lst, reward_data): + reward_fn = get_custom_reward_fn(config) + ground_truth = reward_data["ground_truth"] + score_lst = [reward_fn(data_source, r, ground_truth) for r in response_lst] + return data_source, np.mean(score_lst) + + +@hydra.main(config_path="config", config_name="evaluation", version_base=None) +def main(config): + local_path = copy_to_local(config.data.path) + dataset = pd.read_parquet(local_path) + responses = dataset[config.data.response_key] + data_sources = dataset[config.data.data_source_key] + reward_model_data = dataset[config.data.reward_model_key] + + total = len(dataset) + + # Initialize Ray + if not ray.is_initialized(): + ray.init(**OmegaConf.to_container(config.ray_kwargs.get("ray_init", {}))) + + # evaluate test_score based on data source + data_source_reward = defaultdict(list) + + # Create remote tasks + remote_tasks = [ + process_item.remote(config, data_sources[i], responses[i], reward_model_data[i]) for i in range(total) + ] + + # Process results as they come in + with tqdm(total=total) as pbar: + while len(remote_tasks) > 0: + # Use ray.wait to get completed tasks + done_ids, remote_tasks = ray.wait(remote_tasks) + for result_id in done_ids: + data_source, score = ray.get(result_id) + data_source_reward[data_source].append(score) + pbar.update(1) + + metric_dict = {} + for data_source, rewards in data_source_reward.items(): + metric_dict[f"test_score/{data_source}"] = np.mean(rewards) + + print(metric_dict) + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/recipe/r1/reward_score.py b/e2c/verl/recipe/r1/reward_score.py new file mode 100644 index 0000000000000000000000000000000000000000..9aeced911412327bb36dc65b159e0db5222a59b2 --- /dev/null +++ b/e2c/verl/recipe/r1/reward_score.py @@ -0,0 +1,30 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def reward_func(data_source, solution_str, ground_truth, extra_info=None): + if data_source in ["Maxwell-Jia/AIME_2024", "opencompass/cnmo2024_en", "opencompass/cnmo2024_zh"]: + from recipe.r1.tasks import math_reward + + return math_reward.compute_score(solution_str, ground_truth) + elif data_source == "Idavidrein/gpqa": + from recipe.r1.tasks import gpqa + + return gpqa.compute_score(solution_str, ground_truth) + elif data_source in ["livecodebench/code_generation_lite", "livecodebench/code_generation"]: + from recipe.r1.tasks import livecodebench + + return livecodebench.compute_score(solution_str, ground_truth) + else: + raise NotImplementedError diff --git a/e2c/verl/recipe/r1/run_r1_distill_qwen.sh b/e2c/verl/recipe/r1/run_r1_distill_qwen.sh new file mode 100644 index 0000000000000000000000000000000000000000..a1aa9edccc43ddd0b63d382cda59c92f143cd8f7 --- /dev/null +++ b/e2c/verl/recipe/r1/run_r1_distill_qwen.sh @@ -0,0 +1,33 @@ +MODEL_PATH=Qwen/DeepSeek-R1-Distill-Qwen-1.5B +DATA_PATH=/workspace/datasets/r1_bench + +# Eval Data Process +python3 -m recipe.r1.data_process \ + --local_dir $DATA_PATH \ + --tasks all + +# Generation +python3 -m verl.trainer.main_generation \ + trainer.nnodes=1 \ + trainer.n_gpus_per_node=8 \ + data.path=$DATA_PATH/test.parquet \ + data.prompt_key=prompt \ + data.batch_size=1024 \ + data.n_samples=8 \ + data.output_path=$DATA_PATH/test-output-8.parquet \ + model.path=$MODEL_PATH \ + rollout.temperature=0.6 \ + rollout.top_p=0.95 \ + rollout.prompt_length=1024 \ + rollout.response_length=32768 \ + rollout.tensor_model_parallel_size=1 \ + rollout.gpu_memory_utilization=0.9 \ + rollout.max_num_batched_tokens=65536 + +# Evaluation +python3 -m recipe.r1.main_eval \ + data.path=$DATA_PATH/test-output-8.parquet \ + data.prompt_key=prompt \ + data.response_key=responses \ + custom_reward_function.path=recipe/r1/reward_score.py \ + custom_reward_function.name=reward_func diff --git a/e2c/verl/recipe/r1/tasks/__init__.py b/e2c/verl/recipe/r1/tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/recipe/r1/tasks/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/recipe/r1/tasks/gpqa.py b/e2c/verl/recipe/r1/tasks/gpqa.py new file mode 100644 index 0000000000000000000000000000000000000000..65b37e91662923f2e1acef29297213addfcc50f3 --- /dev/null +++ b/e2c/verl/recipe/r1/tasks/gpqa.py @@ -0,0 +1,25 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +# Extraction Template from https://github.com/openai/simple-evals/blob/90e3e821cabba2aeb6be651dcb662b253df04225/common.py#L25 +ANSWER_PATTERN_MULTICHOICE = r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?" + + +def compute_score(solution_str, ground_truth) -> float: + match = re.search(ANSWER_PATTERN_MULTICHOICE, solution_str) + extracted_answer = match.group(1) if match else None + score = 1.0 if extracted_answer == ground_truth else 0.0 + return score diff --git a/e2c/verl/recipe/r1/tasks/livecodebench.py b/e2c/verl/recipe/r1/tasks/livecodebench.py new file mode 100644 index 0000000000000000000000000000000000000000..f0cbab681d7ee1b2a830b879b528a4170dc1faf7 --- /dev/null +++ b/e2c/verl/recipe/r1/tasks/livecodebench.py @@ -0,0 +1,72 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import multiprocessing +import pickle +import zlib + +# Reuse `run_test` for convenience +from verl.utils.reward_score.prime_code.testing_util import run_test + + +def _temp_run(in_outs, generation, debug, result, metadata_list, timeout): + res, metadata = run_test(in_outs, test=generation, debug=debug, timeout=timeout) + result.append(res) + metadata_list.append(metadata) + + +def check_correctness(in_outs, generation, timeout, debug=True): + """Check correctness of code generation with a global timeout. + The global timeout is to catch some extreme/rare cases not handled by the timeouts + inside `run_test`""" + + manager = multiprocessing.Manager() + result = manager.list() + metadata_list = manager.list() + p = multiprocessing.Process( + target=_temp_run, + args=(in_outs, generation, debug, result, metadata_list, timeout), + ) + p.start() + p.join(timeout=(timeout + 1) * len(in_outs["inputs"]) + 5) + if p.is_alive(): + p.kill() + if not result: + # consider that all tests failed + result = [[-1 for i in range(len(in_outs["inputs"]))]] + if debug: + print("global timeout") + return result[0], metadata_list[0] + + +def compute_score(completion, test_cases): + solution = completion.split("```python")[-1].split("```")[0] + + # extract test cases + try: + in_outs = json.loads(test_cases) + except Exception as e: + print(f"Error loading test cases: {e}") + in_outs = json.loads(pickle.loads(zlib.decompress(base64.b64decode(test_cases.encode("utf-8"))))) + + success = False + try: + res, metadata = check_correctness(in_outs=in_outs, generation=solution, timeout=6, debug=False) + success = all(map(lambda x: x is True, res)) + except Exception: + pass + + return success diff --git a/e2c/verl/recipe/r1/tasks/math_reward.py b/e2c/verl/recipe/r1/tasks/math_reward.py new file mode 100644 index 0000000000000000000000000000000000000000..5ecde5494ef8f7a21400cc2861abcdf4e3a48aa6 --- /dev/null +++ b/e2c/verl/recipe/r1/tasks/math_reward.py @@ -0,0 +1,35 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import contextlib + +try: + from math_verify.metric import math_metric + from math_verify.parser import ExprExtractionConfig, LatexExtractionConfig +except ImportError: + print("To use Math-Verify, please install it first by running `pip install math-verify`.") + + +def compute_score(model_output: str, ground_truth: str) -> bool: + verify_func = math_metric( + gold_extraction_target=(LatexExtractionConfig(),), + pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig()), + ) + ret_score = 0.0 + + # Wrap the ground truth in \boxed{} format for verification + ground_truth_boxed = "\\boxed{" + ground_truth + "}" + with contextlib.suppress(Exception): + ret_score, _ = verify_func([ground_truth_boxed], [model_output]) + + return ret_score diff --git a/e2c/verl/recipe/retool/README.md b/e2c/verl/recipe/retool/README.md new file mode 100644 index 0000000000000000000000000000000000000000..85791e1591efe2f61ce77cd2ac45f2426cfc70c2 --- /dev/null +++ b/e2c/verl/recipe/retool/README.md @@ -0,0 +1,44 @@ +# Retool +[ReTool: Reinforcement Learning for Strategic Tool Use in LLMs](https://arxiv.org/abs/2504.11536) + +## Overview +- Base model: [Qwen/Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct) +- SFT dataset: [JoeYing/ReTool-SFT](https://huggingface.co/datasets/JoeYing/ReTool-SFT) +- RL dataset: [BytedTsinghua-SIA/DAPO-Math-17k](https://huggingface.co/datasets/BytedTsinghua-SIA/DAPO-Math-17k) +- Val dataset: [yentinglin/aime_2025](https://huggingface.co/datasets/yentinglin/aime_2025) + +## SFT +1. Data preparation +```bash +python3 recipe/retool/retool_sft_preprocess.py +``` + +2. Training +```bash +bash recipe/retool/run_qwen2-32b_sft.sh +``` + +After 6 epoches, validation metrics: +- val-core/aime_2025/acc/mean@30: 0.24 +- val-aux/num_turns/mean: 7.2 + +## RL + +### GRPO +```bash +bash recipe/retool/run_qwen2-32b_dapo.sh +``` + +After 150 steps, validation metrics: +- val-core/aime_2025/acc/mean@30: 0.6 +- val-aux/num_turns/mean: 10 + +### PPO + +```bash +bash recipe/retool/run_qwen2-32b_ppo.sh +``` + +After 250 steps, validation metrics: +- val-core/aime_2025/acc/mean@30: 0.55 +- val-aux/num_turns/mean: 8.3 diff --git a/e2c/verl/recipe/retool/retool.py b/e2c/verl/recipe/retool/retool.py new file mode 100644 index 0000000000000000000000000000000000000000..7bcc70453eec95318378bad03c1dc0d7a0d574cc --- /dev/null +++ b/e2c/verl/recipe/retool/retool.py @@ -0,0 +1,120 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import re +from typing import Any + +import datasets + +from verl.tools.base_tool import OpenAIFunctionToolSchema +from verl.tools.sandbox_fusion_tools import SandboxFusionTool +from verl.utils.dataset import RLHFDataset +from verl.utils.reward_score import math_dapo +from verl.utils.rollout_trace import rollout_trace_op + +logger = logging.getLogger(__name__) + + +class CustomSandboxFusionTool(SandboxFusionTool): + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + super().__init__(config, tool_schema) + self.code_pattern = re.compile(r"```python(.*?)```", re.DOTALL) + + @rollout_trace_op + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[str, float, dict]: + code = parameters["code"] + matches = self.code_pattern.findall(code) + if matches: + code = matches[0].strip() + + # NOTE: some script may not explicitly print result, we need to add a print statement to the end of the script + lines = code.split("\n") + for i, line in reversed(list(enumerate(lines))): + if line == "": + continue + if not lines[i].startswith("print"): + lines[i] = f"print({line})" + break + code = "\n".join(lines) + + timeout = parameters.get("timeout", self.default_timeout) + language = parameters.get("language", self.default_language) + if not isinstance(code, str): + code = str(code) + + result = await self.execution_pool.execute.remote(self.execute_code, instance_id, code, timeout, language) + # sandbox has no score or metrics, use Nones + return result, None, None + + +answer_format = """\nThe answer format must be: \\boxed{'The final answer goes here.'}""" + + +class CustomRLHFDataset(RLHFDataset): + """Custom dataset class to process Maxwell-Jia/AIME_2024, yentinglin/aime_2025 datasets.""" + + def _read_files_and_tokenize(self): + dataframes = [] + for parquet_file in self.data_files: + # read parquet files and cache + dataframe = datasets.load_dataset(parquet_file)["train"] + data_source = "/".join(parquet_file.split("/")[-2:]) + if data_source in ["Maxwell-Jia/AIME_2024", "yentinglin/aime_2025"]: + dataframe = dataframe.map( + self.map_fn, fn_kwargs={"data_source": data_source}, remove_columns=dataframe.column_names + ) + else: + dataframe = dataframe.map(self.map_fn2, num_proc=16) + dataframes.append(dataframe) + self.dataframe: datasets.Dataset = datasets.concatenate_datasets(dataframes) + + print(f"dataset len: {len(self.dataframe)}") + + def map_fn(self, row: dict, *, data_source: str = None): + if data_source == "Maxwell-Jia/AIME_2024": + problem, answer = row["Problem"], row["Answer"] + elif data_source == "yentinglin/aime_2025": + problem, answer = row["problem"], row["answer"] + + prompt = problem + answer_format + data = { + "data_source": data_source.split("/")[1].lower(), # aime_2024, aime_2025 + "prompt": [{"role": "user", "content": prompt}], + "ability": "MATH", + "reward_model": {"ground_truth": str(answer)}, + "agent_name": "tool_agent", + } + return data + + def map_fn2(self, row: dict): + content = row["prompt"][0]["content"] + row["prompt"][0]["content"] = content + answer_format + row["agent_name"] = "tool_agent" + return row + + +def compute_score(data_source, solution_str, ground_truth, extra_info): + # use \\boxed{...} answer + result = math_dapo.compute_score(solution_str, ground_truth, strict_box_verify=True) + + # encourage model to call tools + num_turns = extra_info["num_turns"] + if result["score"] < 0: + tool_call_reward = (num_turns - 2) / 2 * 0.1 + result["score"] = min(-0.6, result["score"] + tool_call_reward) + + if result["pred"] is None: + result["pred"] = "" + + return result diff --git a/e2c/verl/recipe/retool/retool_sft_preprocess.py b/e2c/verl/recipe/retool/retool_sft_preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..57d53c91c6290675771ccd4932ffab142fb4335f --- /dev/null +++ b/e2c/verl/recipe/retool/retool_sft_preprocess.py @@ -0,0 +1,136 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Convert JoeYing/ReTool-SFT to standard multi-turn tool calling messages. +""" + +import json +import os +import re +from typing import Any + +import datasets +from omegaconf import OmegaConf + +code_pattern = re.compile(r"```python(.*?)```", re.DOTALL) + + +def extract_code_message(content: str) -> tuple[dict[str, Any], str]: + start, stop = "", "" + i = content.find(start) + if i == -1: + return None, content + j = content.find(stop) + assert j > i + + code = content[i + len(start) : j] + matches = code_pattern.findall(code) + if matches: + code = matches[0].strip() + + message = { + "role": "assistant", + "content": content[:i].strip(), + "tool_calls": [ + { + "type": "function", + "function": { + "name": "code_interpreter", + "arguments": {"code": code}, + }, + }, + ], + } + return message, content[j + len(stop) :] + + +def extract_answer_message(content: str) -> tuple[dict[str, Any], str]: + start, stop = "", "" + i = content.find(start) + if i == -1: + return None, content + j = content.find(stop) + assert j > i + + answer = content[:i] + content[i + len(start) : j] + message = { + "role": "assistant", + "content": answer.strip(), + } + return message, content[j + len(stop) :] + + +def extract_interpreter_message(content: str) -> tuple[dict[str, Any], str]: + start, stop = "", "" + i = content.find(start) + if i == -1: + return None, content + j = content.find(stop) + assert j > i + + interpreter = content[i + len(start) : j] + message = { + "role": "tool", + "content": interpreter.strip(), + } + return message, content[j + len(stop) :] + + +def process(row: dict, *, tools: str): + messages = [] + + # extract problem + content = row["messages"][0]["content"] + start = "*user question:*" + i = content.find(start) + assert i != -1 + prompt = content[i + len(start) :].replace("", "").replace("", "").strip() + messages.append( + { + "role": "user", + "content": prompt, + } + ) + + # extract multi turns + content = row["messages"][1]["content"] + role = "assistant" + while len(content) > 0: + if role == "assistant": + message, content = extract_code_message(content) + if message is None: + message, content = extract_answer_message(content) + assert message is not None + messages.append(message) + role = "tool" + else: + message, content = extract_interpreter_message(content) + assert message is not None + messages.append(message) + role = "assistant" + + tools = json.loads(tools) + return {"messages": messages, "tools": tools} + + +if __name__ == "__main__": + tools_config_file = "recipe/retool/sandbox_fusion_tool_config.yaml" + tools_config = OmegaConf.load(tools_config_file) + tool_schema = OmegaConf.to_container(tools_config["tools"][0]["tool_schema"]) + tools = json.dumps([tool_schema]) + + data = datasets.load_dataset("JoeYing/ReTool-SFT")["train"] + data = data.map(process, fn_kwargs={"tools": tools}) + save_path = os.path.expanduser("~/ReTool-SFT/data/train-00000-of-00001.parquet") + data.to_parquet(save_path) diff --git a/e2c/verl/recipe/retool/run_qwen2-32b_dapo.sh b/e2c/verl/recipe/retool/run_qwen2-32b_dapo.sh new file mode 100644 index 0000000000000000000000000000000000000000..2df380da24cfe3872665e407489d4450aeff20c9 --- /dev/null +++ b/e2c/verl/recipe/retool/run_qwen2-32b_dapo.sh @@ -0,0 +1,107 @@ +set -x + +# ================= data/model/tool ================= +HDFS_ROOT=${HDFS_ROOT:-$PWD} +DATA_ROOT=${DATA_ROOT:-$PWD} + +dapo_math_17k=$DATA_ROOT/dataset/BytedTsinghua-SIA/DAPO-Math-17k +aime_2024=$DATA_ROOT/dataset/Maxwell-Jia/AIME_2024 +aime_2025=$DATA_ROOT/dataset/yentinglin/aime_2025 +model_path=$HDFS_ROOT/checkpoint/multiturn-sft-qwen-2.5-32b-instruct/global_step_372 + +train_files="['$dapo_math_17k']" +test_files="['$aime_2025']" + +# tool +tool_config_path=recipe/retool/sandbox_fusion_tool_config.yaml + +# wandb +project_name=wuxibin_retool +experiment_name=qwen2.5-32b_dapo +default_local_dir=$DATA_ROOT/checkpoint/$experiment_name + +# ================= algorithm ================= +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_turns=8 +max_prompt_length=2048 +max_response_length=16384 +actor_lr=1e-6 + +train_batch_size=512 +ppo_mini_batch_size=64 +n_resp_per_prompt=16 +n_resp_per_prompt_val=30 + +# ================= perfomance ================= +infer_tp=4 # vllm +train_sp=8 # train +offload=True + +actor_max_token_len_per_gpu=$(( (max_prompt_length + max_response_length) * 1 )) +log_prob_max_token_len_per_gpu=$(( actor_max_token_len_per_gpu * 4 )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=$adv_estimator \ + algorithm.use_kl_in_reward=$use_kl_in_reward \ + algorithm.kl_ctrl.kl_coef=$kl_coef \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.return_raw_chat=True \ + data.train_batch_size=$train_batch_size \ + data.max_prompt_length=$max_prompt_length \ + data.max_response_length=$max_response_length \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.custom_cls.path=recipe/retool/retool.py \ + data.custom_cls.name=CustomRLHFDataset \ + custom_reward_function.path=recipe/retool/retool.py \ + custom_reward_function.name=compute_score \ + actor_rollout_ref.model.path=$model_path \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.use_kl_loss=$use_kl_loss \ + actor_rollout_ref.actor.kl_loss_coef=$kl_loss_coef \ + actor_rollout_ref.actor.clip_ratio_low=$clip_ratio_low \ + actor_rollout_ref.actor.clip_ratio_high=$clip_ratio_high \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.optim.lr=$actor_lr \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=$ppo_mini_batch_size \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=$actor_max_token_len_per_gpu \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=$train_sp \ + actor_rollout_ref.actor.fsdp_config.param_offload=$offload \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=$offload \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=$log_prob_max_token_len_per_gpu \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$infer_tp \ + actor_rollout_ref.rollout.multi_turn.enable=True \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=$max_turns \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=$max_turns \ + actor_rollout_ref.rollout.multi_turn.tool_config_path=$tool_config_path \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.rollout.n=$n_resp_per_prompt \ + actor_rollout_ref.rollout.val_kwargs.top_p=0.6 \ + actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \ + actor_rollout_ref.rollout.val_kwargs.n=$n_resp_per_prompt_val \ + trainer.logger=['console','wandb'] \ + trainer.project_name=$project_name \ + trainer.experiment_name=$experiment_name \ + trainer.n_gpus_per_node=8 \ + trainer.val_before_train=True \ + trainer.log_val_generations=100 \ + trainer.nnodes=2 \ + trainer.save_freq=30 \ + trainer.default_local_dir=$default_local_dir \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ diff --git a/e2c/verl/recipe/retool/run_qwen2-32b_ppo.sh b/e2c/verl/recipe/retool/run_qwen2-32b_ppo.sh new file mode 100644 index 0000000000000000000000000000000000000000..1e3ef2cd7fbb0bafe12bcdb95ce9696060cc8e2b --- /dev/null +++ b/e2c/verl/recipe/retool/run_qwen2-32b_ppo.sh @@ -0,0 +1,123 @@ +set -x + +# ================= data/model/tool ================= +HDFS_ROOT=${HDFS_ROOT:-$PWD} +DATA_ROOT=${DATA_ROOT:-$PWD} + +dapo_math_17k=$DATA_ROOT/dataset/BytedTsinghua-SIA/DAPO-Math-17k +aime_2024=$DATA_ROOT/dataset/Maxwell-Jia/AIME_2024 +aime_2025=$DATA_ROOT/dataset/yentinglin/aime_2025 +actor_model_path=$HDFS_ROOT/checkpoint/multiturn-sft-qwen-2.5-32b-instruct/global_step_372 +critic_model_path=$actor_model_path + +train_files="['$dapo_math_17k']" +test_files="['$aime_2025']" + +# tool +tool_config_path=recipe/retool/sandbox_fusion_tool_config.yaml + +# wandb +project_name=wuxibin_retool +experiment_name=qwen2.5-32b_ppo +default_local_dir=$DATA_ROOT/checkpoint/$experiment_name + +# ================= algorithm ================= +adv_estimator=gae + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_turns=8 +max_prompt_length=2048 +max_response_length=16384 +actor_lr=1e-6 +critic_lr=2e-6 +gae_gamma=1.0 +gae_lam=1.0 + +critic_warmup=20 + +train_batch_size=1024 +ppo_mini_batch_size=256 +n_resp_per_prompt_val=30 + +# ================= perfomance ================= +infer_tp=4 # vllm +train_sp=4 # train + +offload=True + +actor_max_token_len_per_gpu=$(( (max_prompt_length + max_response_length) * 2 )) +critic_max_token_len_per_gpu=$(( (max_prompt_length + max_response_length) * 4 )) + + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=$adv_estimator \ + algorithm.use_kl_in_reward=$use_kl_in_reward \ + algorithm.kl_ctrl.kl_coef=$kl_coef \ + algorithm.gamma=$gae_gamma \ + algorithm.lam=$gae_lam \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.return_raw_chat=True \ + data.train_batch_size=$train_batch_size \ + data.max_prompt_length=$max_prompt_length \ + data.max_response_length=$max_response_length \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.custom_cls.path=recipe/retool/retool.py \ + data.custom_cls.name=CustomRLHFDataset \ + custom_reward_function.path=recipe/retool/retool.py \ + custom_reward_function.name=compute_score \ + actor_rollout_ref.model.path=$actor_model_path \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.use_kl_loss=$use_kl_loss \ + actor_rollout_ref.actor.kl_loss_coef=$kl_loss_coef \ + actor_rollout_ref.actor.clip_ratio_low=$clip_ratio_low \ + actor_rollout_ref.actor.clip_ratio_high=$clip_ratio_high \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.optim.lr=$actor_lr \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=$ppo_mini_batch_size \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=$actor_max_token_len_per_gpu \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=$train_sp \ + actor_rollout_ref.actor.fsdp_config.param_offload=$offload \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=$offload \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$infer_tp \ + actor_rollout_ref.rollout.multi_turn.enable=True \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=$max_turns \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=$max_turns \ + actor_rollout_ref.rollout.multi_turn.tool_config_path=$tool_config_path \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.rollout.val_kwargs.top_p=0.6 \ + actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \ + actor_rollout_ref.rollout.val_kwargs.n=$n_resp_per_prompt_val \ + critic.optim.lr=$critic_lr \ + critic.model.use_remove_padding=True \ + critic.model.path=$critic_model_path \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_max_token_len_per_gpu=$critic_max_token_len_per_gpu \ + critic.ulysses_sequence_parallel_size=$train_sp \ + critic.model.fsdp_config.param_offload=$offload \ + critic.model.fsdp_config.optimizer_offload=$offload \ + trainer.critic_warmup=$critic_warmup \ + trainer.logger=['console','wandb'] \ + trainer.project_name=$project_name \ + trainer.experiment_name=$experiment_name \ + trainer.n_gpus_per_node=8 \ + trainer.val_before_train=True \ + trainer.log_val_generations=100 \ + trainer.nnodes=2 \ + trainer.save_freq=30 \ + trainer.default_local_dir=$default_local_dir \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ diff --git a/e2c/verl/recipe/retool/run_qwen2-32b_sft.sh b/e2c/verl/recipe/retool/run_qwen2-32b_sft.sh new file mode 100644 index 0000000000000000000000000000000000000000..d218b0e7eb87bdd5236fbb5eed61e8f2ccd306f8 --- /dev/null +++ b/e2c/verl/recipe/retool/run_qwen2-32b_sft.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -x + +nnodes=2 +nproc_per_node=8 +master_addr= +master_port= + +experiment_name=multiturn-sft-qwen-2.5-32b-instruct +HDFS_ROOT=${HDFS_ROOT:-$PWD} +DATA_ROOT=${DATA_ROOT:-$PWD} + +TRAIN_DATA=$DATA_ROOT/dataset/wuxibin/ReTool-SFT/data/train-00000-of-00001.parquet +EVAL_DATA=$DATA_ROOT/dataset/wuxibin/ReTool-SFT/data/train-00000-of-00001.parquet +MODEL_PATH=$HDFS_ROOT/model/Qwen2.5-32B-Instruct +SAVE_PATH=$DATA_ROOT/checkpoint/$experiment_name + +torchrun --nnodes=$nnodes \ + --nproc_per_node=$nproc_per_node \ + --master-addr=$master_addr \ + --master-port=$master_port \ + --node-rank=$node_rank \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$TRAIN_DATA \ + data.val_files=$EVAL_DATA \ + data.max_length=16384 \ + data.train_batch_size=32 \ + data.multiturn.enable=true \ + data.multiturn.messages_key=messages \ + data.multiturn.tools_key=tools \ + data.micro_batch_size_per_gpu=4 \ + model.partial_pretrain=$MODEL_PATH \ + model.strategy=fsdp \ + trainer.default_local_dir=$SAVE_PATH \ + trainer.project_name=wuxibin-multiturn-sft \ + trainer.experiment_name=$experiment_name \ + trainer.logger='["console","wandb"]' \ + trainer.total_epochs=6 \ + ulysses_sequence_parallel_size=4 \ + use_remove_padding=true \ No newline at end of file diff --git a/e2c/verl/recipe/retool/run_qwen2_7b_dapo.sh b/e2c/verl/recipe/retool/run_qwen2_7b_dapo.sh new file mode 100644 index 0000000000000000000000000000000000000000..f1187a3d12d3b308e41c27e801319dc839691f8c --- /dev/null +++ b/e2c/verl/recipe/retool/run_qwen2_7b_dapo.sh @@ -0,0 +1,109 @@ +set -x + +export VLLM_USE_V1=1 + +# ================= data/model/tool ================= +HDFS_ROOT=${HDFS_ROOT:-$PWD} +DATA_ROOT=${DATA_ROOT:-$PWD} + +dapo_math_17k=$DATA_ROOT/dataset/BytedTsinghua-SIA/DAPO-Math-17k +aime_2024=$DATA_ROOT/dataset/Maxwell-Jia/AIME_2024 +aime_2025=$DATA_ROOT/dataset/yentinglin/aime_2025 +model_path=$HDFS_ROOT/checkpoint/multiturn-sft-qwen-2.5-7b-instruct/global_step_372 + +train_files="['$dapo_math_17k']" +test_files="['$aime_2025', '$aime_2024']" + +# tool +tool_config_path=recipe/retool/sandbox_fusion_tool_config.yaml + +# wandb +project_name=retool +experiment_name=qwen2.5-7b_dapo +default_local_dir=$DATA_ROOT/checkpoint/$experiment_name + +# ================= algorithm ================= +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_turns=16 +max_prompt_length=2048 +max_response_length=16384 +actor_lr=1e-6 + +train_batch_size=64 +ppo_mini_batch_size=16 +n_resp_per_prompt=16 +n_resp_per_prompt_val=30 + +# ================= perfomance ================= +infer_tp=4 # vllm +train_sp=4 # train +offload=True + +actor_max_token_len_per_gpu=$(( (max_prompt_length + max_response_length) * 1 )) +log_prob_max_token_len_per_gpu=$(( actor_max_token_len_per_gpu * 4 )) + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=$adv_estimator \ + algorithm.use_kl_in_reward=$use_kl_in_reward \ + algorithm.kl_ctrl.kl_coef=$kl_coef \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.return_raw_chat=True \ + data.train_batch_size=$train_batch_size \ + data.max_prompt_length=$max_prompt_length \ + data.max_response_length=$max_response_length \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.custom_cls.path=recipe/retool/retool.py \ + data.custom_cls.name=CustomRLHFDataset \ + custom_reward_function.path=recipe/retool/retool.py \ + custom_reward_function.name=compute_score \ + actor_rollout_ref.model.path=$model_path \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.use_kl_loss=$use_kl_loss \ + actor_rollout_ref.actor.kl_loss_coef=$kl_loss_coef \ + actor_rollout_ref.actor.clip_ratio_low=$clip_ratio_low \ + actor_rollout_ref.actor.clip_ratio_high=$clip_ratio_high \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.optim.lr=$actor_lr \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=$ppo_mini_batch_size \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=$actor_max_token_len_per_gpu \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=$train_sp \ + actor_rollout_ref.actor.fsdp_config.param_offload=$offload \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=$offload \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=$log_prob_max_token_len_per_gpu \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$infer_tp \ + actor_rollout_ref.rollout.multi_turn.enable=True \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=$max_turns \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=$max_turns \ + actor_rollout_ref.rollout.multi_turn.tool_config_path=$tool_config_path \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.rollout.n=$n_resp_per_prompt \ + actor_rollout_ref.rollout.val_kwargs.top_p=0.6 \ + actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \ + actor_rollout_ref.rollout.val_kwargs.n=$n_resp_per_prompt_val \ + trainer.logger=['console','wandb'] \ + trainer.project_name=$project_name \ + trainer.experiment_name=$experiment_name \ + trainer.n_gpus_per_node=8 \ + trainer.val_before_train=True \ + trainer.log_val_generations=20 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.default_local_dir=$default_local_dir \ + trainer.test_freq=10 \ + trainer.total_epochs=1 $@ diff --git a/e2c/verl/recipe/retool/run_qwen2_7b_sft.sh b/e2c/verl/recipe/retool/run_qwen2_7b_sft.sh new file mode 100644 index 0000000000000000000000000000000000000000..e4369e167e3bb62ffe11d1658d155a76c58af510 --- /dev/null +++ b/e2c/verl/recipe/retool/run_qwen2_7b_sft.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -x + +nnodes=1 +nproc_per_node=8 +master_addr= +master_port= +node_rank=${ARNOLD_ID:-0} + +project_name=retool +experiment_name=multiturn-sft-qwen-2.5-7b-instruct + +HDFS_ROOT=${HDFS_ROOT:-$PWD} +DATA_ROOT=${DATA_ROOT:-$PWD} + +TRAIN_DATA=$DATA_ROOT/dataset/wuxibin/ReTool-SFT/data/train-00000-of-00001.parquet +EVAL_DATA=$DATA_ROOT/dataset/wuxibin/ReTool-SFT/data/train-00000-of-00001.parquet +MODEL_PATH=$HDFS_ROOT/model/Qwen2.5-7B-Instruct +SAVE_PATH=$DATA_ROOT/checkpoint/$experiment_name + +torchrun --nnodes=$nnodes \ + --nproc_per_node=$nproc_per_node \ + --master-addr=$master_addr \ + --master-port=$master_port \ + --node-rank=$node_rank \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$TRAIN_DATA \ + data.val_files=$EVAL_DATA \ + data.max_length=16384 \ + data.train_batch_size=32 \ + data.multiturn.enable=true \ + data.multiturn.messages_key=messages \ + data.multiturn.tools_key=tools \ + data.micro_batch_size_per_gpu=4 \ + model.partial_pretrain=$MODEL_PATH \ + model.strategy=fsdp \ + trainer.default_local_dir=$SAVE_PATH \ + trainer.project_name=wuxibin-multiturn-sft \ + trainer.experiment_name=$experiment_name \ + trainer.logger='["console","wandb"]' \ + trainer.total_epochs=6 \ + trainer.save_freq=62 \ + ulysses_sequence_parallel_size=4 \ + use_remove_padding=true \ No newline at end of file diff --git a/e2c/verl/recipe/retool/run_qwen2_7b_sft_npu.sh b/e2c/verl/recipe/retool/run_qwen2_7b_sft_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..ba203e6bee001198f1bd2b20b415397980162aaf --- /dev/null +++ b/e2c/verl/recipe/retool/run_qwen2_7b_sft_npu.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -x + +nnodes=1 +nproc_per_node=8 + +project_name=retool_sft +experiment_name=multiturn-sft-qwen-2.5-7b-instruct + +TRAIN_DATA=PATH/TO/ReTool-SFT/data/train-00000-of-00001.parquet +EVAL_DATA=PATH/TO/ReTool-SFT/data/train-00000-of-00001.parquet +MODEL_PATH=PATH/TO/Qwen2.5-7B-Instruct +SAVE_PATH=PATH/TO/checkpoint/$experiment_name + +torchrun --nnodes=$nnodes \ + --nproc_per_node=$nproc_per_node \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$TRAIN_DATA \ + data.val_files=$EVAL_DATA \ + data.max_length=16384 \ + data.train_batch_size=64 \ + data.multiturn.enable=true \ + data.multiturn.messages_key=messages \ + data.multiturn.tools_key=tools \ + data.micro_batch_size_per_gpu=8 \ + model.partial_pretrain=$MODEL_PATH \ + model.strategy=fsdp \ + trainer.default_local_dir=$SAVE_PATH \ + trainer.project_name=$project_name \ + trainer.experiment_name=$experiment_name \ + trainer.logger='["console"]' \ + trainer.total_epochs=6 \ + trainer.save_freq=10 \ + trainer.device=npu \ + ulysses_sequence_parallel_size=4 \ + use_remove_padding=true \ No newline at end of file diff --git a/e2c/verl/recipe/retool/sandbox_fusion_tool_config.yaml b/e2c/verl/recipe/retool/sandbox_fusion_tool_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71b10e50ec95bbb42cca13ffd218a44b8d759ef0 --- /dev/null +++ b/e2c/verl/recipe/retool/sandbox_fusion_tool_config.yaml @@ -0,0 +1,24 @@ +tools: + - class_name: "recipe.retool.retool.CustomSandboxFusionTool" + config: + sandbox_fusion_url: "http://localhost:8080/run_code" + num_workers: 128 + enable_global_rate_limit: true + rate_limit: 128 + default_timeout: 30 + default_language: "python" + memory_limit_mb: 1024 + type: native + + tool_schema: + type: "function" + function: + name: "code_interpreter" + description: "A tool for executing code." + parameters: + type: "object" + properties: + code: + type: "string" + description: "The code to execute." + required: ["code"] diff --git a/e2c/verl/recipe/spin/README.md b/e2c/verl/recipe/spin/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a80a5c8a76cd53e296f0c33c1a72310d7368825b --- /dev/null +++ b/e2c/verl/recipe/spin/README.md @@ -0,0 +1,179 @@ +# SPIN: Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models + +This repository hosts a `verl` recipe inspired by the paper **"Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models"** (SPIN). SPIN is a language model finetuning algorithm that enables iterative self-improvement through a self-play mechanism inspired by game theory. + +**Core Idea:** Models learn by playing against themselves, reducing reliance on external preference datasets or stronger teacher models: + +1. **Synthetic Data Generation:** The current model generates responses, creating its own training data from previous iterations. +2. **Two-Player Game Setup:** A game involving two players acted by a single LLM. +3. **Iterative Training:** The model progressively improves by refining its policy, with each iteration's model becoming the opponent for the next iteration. + +Paper Authors: [Zixiang Chen](https://github.com/uclaml/SPIN)\*, [Yihe Deng](https://github.com/uclaml/SPIN)\*, [Huizhuo Yuan](https://scholar.google.com/citations?user=8foZzX4AAAAJ)\*, [Kaixuan Ji](https://scholar.google.com/citations?user=FOoKDukAAAAJ), [Quanquan Gu](https://web.cs.ucla.edu/~qgu/) + +[[Webpage](https://uclaml.github.io/SPIN/)] [[Huggingface](https://huggingface.co/papers/2401.01335)] [[Paper](https://arxiv.org/abs/2401.01335)] [[Original Implementation](https://github.com/uclaml/SPIN)] + +verl Implementation Authors: [Chendong Wang](https://cdwang96.github.io/), [Chenyang Zhao](https://github.com/zhaochenyang20) + +--- + +## Key Function (compute_online_dpo_loss) and Related works +SPIN (Chen et al., 2024) proposes an iterative self-play mechanism to fine-tune language models. In each iteration, SPIN's training objective, when using a logistic loss function, is equivalent to Direct Preference Optimization (DPO) loss (Rafailov et al., 2023). + +This `verl` recipe realizes SPIN's core concept by using DPO loss iteratively (Xu et al., 2023; Xiong et al., 2023; Snorkel AI, 2024). This means that in each iteration, we fine-tune the LLM using DPO loss for preference optimization. Notably, Xu et al. (2023) explored iterative preference optimization with pairwise cringe loss, while Xiong et al. (2023) discussed how to bridge theory and practice for RLHF under KL constraints using iterative training. The concept of iterative preference learning was also explored in online DPO (Guo et al., 2024), which focuses on direct alignment from online AI feedback. In online DPO, preference data is dynamically updated during training, allowing the model to learn from its own generated data. + +Specifically, we developed the **`compute_online_dpo_loss`** function and built this SPIN recipe on top of it. By incorporating online preference generation, this approach enables continuously refining language models without relying on fixed external preference datasets. + +**Reference Papers:** +* [Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models](https://arxiv.org/abs/2401.01335) (Chen et al., 2024) +* [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://arxiv.org/abs/2305.18290) (Rafailov et al., 2023) +* [Somethings are more cringe than others: Preference optimization with the pairwise cringe loss](https://arxiv.org/abs/2312.16682) (Xu et al., 2023) +* [Iterative preference learning from human feedback: Bridging theory and practice for rlhf under kl-constraint](https://arxiv.org/abs/2312.11456) (Xiong et al., 2023) +* [Snorkel-Mistral-PairRM-DPO](https://huggingface.co/snorkelai/Snorkel-Mistral-PairRM-DPO) (Snorkel AI, 2024) +* [Direct language model alignment from online ai feedback](https://arxiv.org/abs/2402.04792) (Guo et al., 2024) + + +## Our Online DPO Implementation + +Our `compute_online_dpo_loss` function adapts `verl`'s existing PPO infrastructure (based on `verl` v0.3.0.post1) for this iterative online DPO. Key aspects of our implementation include: + +* **No Critic:** Unlike PPO, we omit the value function critic. +* **Dynamic Reference Model:** An explicit reference policy (`ref_policy_wg`) is used for DPO loss. This reference model's weights can be periodically updated from the actor (`ref_update_freq`), providing a dynamic baseline. +* **Online Preference Generation:** The `compute_onlineDPO_pref` function (in `core_algos.py`) dynamically creates chosen/rejected pairs based on a reward source (e.g., rule-based ranking for math problems). +* **DPO Loss Integration:** We replace PPO's policy loss with our `compute_online_dpo_loss` (in `core_algos.py`) within the actor update (`dp_actor.py`), directly optimizing the policy using the generated preferences. +* **Iterative Training Orchestration:** The `SpinTrainer` (in `spin_trainer.py`) manages the entire self-play loop: generation, preference labeling, optional reference model updates, and policy updates, enabling continuous self-improvement aligned with SPIN's principles. + +--- +## Algorithm + +This recipe implements an Online algorithm adapted to the `verl` Reinforcement Learning framework, which provides an alternative to PPO for fine-tuning language models. + +**Online Loop:** Instead of maximizing a scalar reward signal in PPO, this approach directly optimizes the policy model to align with preference data generated *online* during training: + +1. **Generation:** The current model generates multiple responses for each prompt in a batch. +2. **Preference Labeling:** A function evaluates these generated responses to determine which one is preferred (chosen) and which is dispreferred (rejected). This can be done using a reward function or implicit ranking based on specific rules. (In this recipe, we use rule-based ranking on the math problem). +3. **Update:** This preference tuple (`prompt`, `chosen_response`, `rejected_response`) is used to update the actor model using `compute_online_dpo_loss`, comparing against a reference model. + +**Connection with SPIN:** +Instead of only using a fixed target data distribution, the online generation loop in step 2 will dynamically change the target data distribution by using a certain Preference Labeling method (rule-based ranking on the math problem by selecting the better one in this recipe). This explores the direction mentioned in SPIN's paper Section 7 about "dynamically changing target data distribution" to potentially elevate LLM performance beyond the fixed human-annotated data ceiling. + +--- + +## Reproduce the Experiment (Example Setup) + +The following steps outline how to set up the environment and run the SPIN recipe, based on the provided test log using GSM8K and Qwen2.5-3B-Instruct. + +1. **Setup Environment (Example using Docker):** + ```bash + # Start a container with GPU access and shared memory + docker run -it --name spin_test --gpus all \ + --shm-size=32g \ + --ipc=host \ + -v /path/to/host/.cache:/root/.cache \ + -e HF_TOKEN= \ + lmsysorg/sglang:latest \ + /bin/bash + + # Inside the container or on your host machine: + # Ensure /tmp is writable + mkdir -p /tmp + chmod 1777 /tmp + + # Install Python 3.10 (if not present) and venv + sudo apt update + sudo apt install -y python3.10 python3.10-venv tmux + python3 -m ensurepip --upgrade + + # Create and activate a virtual environment + python3 -m venv ~/.python/spin_env + source ~/.python/spin_env/bin/activate + + # Install uv (fast package installer) + python3 -m pip install uv + ``` + +2. **Install verl and Dependencies:** + ```bash + # Clone the verl repository and checkout the spin branch + cd ~ + git clone git@github.com:volcengine/verl.git && cd verl + + # Install flash-attn (handle potential build issues) + python3 -m uv pip install wheel packaging + python3 -m uv pip install flash-attn --no-build-isolation --no-deps + + # Install verl with sglang extras + python3 -m uv pip install -e ".[sglang]" + ``` + *Note: If `flash-attn` installation fails, try the manual steps again or consult its documentation.* + +3. **Login & Download Data/Model:** + ```bash + # Login to Weights & Biases (optional, for logging) + export WANDB_API_KEY= + # wandb login + + # Download the GSM8K dataset + python3 examples/data_preprocess/gsm8k.py --local_save_dir ~/data/gsm8k # Adjusted path + + # Download the base model (Example: Qwen2.5-3B-Instruct) + huggingface-cli download Qwen/Qwen2.5-3B-Instruct --local-dir $HOME/models/Qwen2.5-3B-Instruct + ``` + +4. **Configure:** + * Modify the configuration file (e.g., `config/spin_trainer.yaml` or the one specified in the run script) with correct paths to your downloaded model, data, desired hyperparameters (`dpo_beta`, learning rate, etc.), and distributed training settings (nodes, GPUs per node). + * Pay attention to `actor_rollout_ref.model_path`, `data` paths, `reward_model` config (if using one), and `trainer.ref_update_freq`. + +5. **Run Training:** + ```bash + # Set CUDA visible devices (adjust based on your hardware and config) + export CUDA_VISIBLE_DEVICES=0,1,2,3 + + # Launch the training script (e.g., test.sh or a custom script) + # Ensure test.sh points to the correct config and main script + bash recipe/spin/run_spin.sh + ``` + +--- + +## Configuration + +* The primary configuration is typically managed through a YAML file specified in the launch script (e.g., `config/spin_trainer.yaml`). +* Key configuration sections: + * `data`: Paths to training/validation prompt files, batch sizes, sequence lengths. + * `actor_rollout_ref`: Paths to the base model (used for actor and initial reference), FSDP settings, optimization parameters (learning rate, scheduler). + * `reward_model`: Configuration for the reward model used for online preference labeling (path, batch size, etc.). Can be omitted if using a simpler reward function. + * `algorithm`: DPO-specific hyperparameters like `dpo_beta`, `dpo_loss_type`. + * `trainer`: Distributed training settings (nodes, GPUs per node), logging (WandB), checkpointing frequency, and `ref_update_freq` (set > 0 to enable periodic reference model updates from the actor). + +--- + +## Key Files + +* `main_spin.py`: Main entry point using Hydra to load the config and launch the `SpinTrainer`. +* `spin_trainer.py`: Defines the `SpinTrainer` class, orchestrating the Online DPO training loop. +* `fsdp_workers.py`: Implements Ray workers (Actor, Reference) potentially using FSDP. +* `dp_actor.py`: Contains the actor class, including the DPO policy update logic. +* `core_algos.py`: Includes helper functions for `compute_online_dpo_loss` and `compute_onlineDPO_pref`. +* `config/spin_trainer.yaml` (or similar): Main Hydra configuration file for the recipe. +* `run_spin.sh` (or similar): Example bash script for launching a training run. +* `README.md`: This file. + +--- + +## Acknowledgement + +We sincerely thank the contribution and guidance from the `verl` community and advisors, including (adapted from SPPO): + +* [Zixiang Chen](https://sites.google.com/view/zxchen) +* [Yuhao Yang](https://github.com/yhyang201) +* [Yifan Zhang](https://github.com/yifanzhang-pro) +* [Yongan Xiang](https://github.com/BearBiscuit05) +* [Junrong Lin](https://github.com/ocss884) +* [Yuxuan Tong](https://github.com/tongyx361) +* [Guangming Shen](https://github.com/PeterSH6) +* [Biao He](https://www.linkedin.com/in/biao-he/) +* [Qingquan Song](https://qingquansong.github.io/) +* [Chenyang Zhao](https://zhaochenyang20.github.io/Chayenne/) +* [Quanquan Gu](https://web.cs.ucla.edu/~qgu/) + +--- diff --git a/e2c/verl/recipe/spin/config/spin_trainer.yaml b/e2c/verl/recipe/spin/config/spin_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee105c4213efa9e67a7c59e3548fa0c3998423a1 --- /dev/null +++ b/e2c/verl/recipe/spin/config/spin_trainer.yaml @@ -0,0 +1,28 @@ +# the sppo config will override default ppo_trainer.yaml + +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +actor_rollout_ref: + actor: + dpo_beta: 0.1 + optim: + lr_warmup_steps: 15 + rollout: + name: sglang + tensor_model_parallel_size: 2 + gpu_memory_utilization: 0.5 + val_kwargs: + n: 2 # 2 will trigger validation, 1 will bypass + +algorithm: + adv_estimator: null + +trainer: + log_val_generations: 0 + ref_update_freq: 1 \ No newline at end of file diff --git a/e2c/verl/recipe/spin/core_algos.py b/e2c/verl/recipe/spin/core_algos.py new file mode 100644 index 0000000000000000000000000000000000000000..c48027e54106ab496c09ddb80107fb7df210f2b6 --- /dev/null +++ b/e2c/verl/recipe/spin/core_algos.py @@ -0,0 +1,206 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import numpy as np +import torch + + +class AdaptiveKLController: + """ + Adaptive KL controller described in the paper: + https://arxiv.org/pdf/1909.08593.pdf + """ + + def __init__(self, init_kl_coef, target_kl, horizon): + self.value = init_kl_coef + self.target = target_kl + self.horizon = horizon + + def update(self, current_kl, n_steps): + target = self.target + proportional_error = np.clip(current_kl / target - 1, -0.2, 0.2) + mult = 1 + proportional_error * n_steps / self.horizon + self.value *= mult + + +class FixedKLController: + """Fixed KL controller.""" + + def __init__(self, kl_coef): + self.value = kl_coef + + def update(self, current_kl, n_steps): + pass + + +def get_kl_controller(kl_ctrl): + if kl_ctrl.type == "fixed": + return FixedKLController(kl_coef=kl_ctrl.kl_coef) + elif kl_ctrl.type == "adaptive": + assert kl_ctrl.horizon > 0, f"horizon must be larger than 0. Got {kl_ctrl.horizon}" + return AdaptiveKLController(init_kl_coef=kl_ctrl.kl_coef, target_kl=kl_ctrl.target_kl, horizon=kl_ctrl.horizon) + else: + raise NotImplementedError + + +def compute_onlinedpo_pref( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, +) -> torch.Tensor: + """ + Computes preferences between pairs of sequences based on summed rewards + and returns a mask aligned with the interleaved batch. + + Assumes inputs are interleaved: [Resp1_Prompt0, Resp2_Prompt0, Resp1_Prompt1, Resp2_Prompt1, ...] + + Args: + token_level_rewards: Tensor of shape [batch_size * 2, seq_len] + response_mask: Tensor of shape [batch_size * 2, seq_len] + + Returns: + torch.Tensor: A boolean mask of shape [batch_size * 2], where True indicates + the corresponding entry is the chosen response for its pair. + Example: [True, False, False, True, ...] means for prompt 0, + response 1 was chosen; for prompt 1, response 2 was chosen. + """ + # print(f"---- [DEBUG] Inside compute_onlinedpo_pref ----") + if token_level_rewards.shape[0] % 2 != 0 or response_mask.shape[0] % 2 != 0: + raise ValueError( + f"Input tensor batch dimension must be even for pair comparison, got shapes: " + f"{token_level_rewards.shape}, {response_mask.shape}" + ) + if token_level_rewards.shape != response_mask.shape: + raise ValueError(f"Shape mismatch between rewards {token_level_rewards.shape} and mask {response_mask.shape}") + + # 1. Calculate Sequence Scores + scores = (token_level_rewards * response_mask).sum(dim=-1) + # print(f" Calculated sequence scores shape: {scores.shape}") # [batch_size * 2] + + # 2. Reshape scores to group pairs: [batch_size, 2] + try: + score_pairs = scores.view(-1, 2) + except RuntimeError as e: + print(f"ERROR reshaping scores (shape {scores.shape}) into pairs: {e}") + raise e + print(f" Reshaped score pairs shape: {score_pairs.shape}") # [batch_size, 2] + + # 3. Compare scores to find which index (0 or 1) is the winner within each pair + # winner_indices[i] = 0 if score_pairs[i, 0] >= score_pairs[i, 1] else 1 + winner_indices = torch.argmax(score_pairs, dim=1) # 0 if first is max, 1 if second is max + # Handle ties explicitly if argmax behavior isn't guaranteed (usually picks first max) + # Alternatively: winner_mask_original = score_pairs[:, 0] >= score_pairs[:, 1] + # print(f" Winner indices shape: {winner_indices.shape}") # [batch_size] + # print(f" Number where Response 2 (index 1) is preferred: {winner_indices.sum().item()}") # Counts number of 1s + + # 4. Create the final [batch_size * 2] mask + num_pairs = score_pairs.shape[0] + full_batch_size = num_pairs * 2 + # Create indices for the full batch [0, 1, 2, 3, ..., N*2-1] + # full_indices = torch.arange(full_batch_size, device=scores.device) + # Create indices corresponding to the winner within each pair's original index + # E.g., if winner_indices is [0, 1, 0], pair_indices is [0, 1, 2] + # winner_global_indices = (pair_indices * 2) + winner_indices -> [ (0*2)+0, (1*2)+1, (2*2)+0 ] -> [0, 3, 4] + pair_indices = torch.arange(num_pairs, device=scores.device) + winner_global_indices = (pair_indices * 2) + winner_indices + + # Create boolean mask - True at the winner's position + output_preference_mask = torch.zeros(full_batch_size, dtype=torch.bool, device=scores.device) + output_preference_mask[winner_global_indices] = True + + # print(f" Output preference mask shape: {output_preference_mask.shape}") # Should be [batch_size * 2] + # print(f" Output mask True count (Chosen): {output_preference_mask.sum().item()}") # Should be batch_size + # print(f" Output mask False count (Rejected): {(~output_preference_mask).sum().item()}") # Should be batch_size + # print(f"---- [DEBUG] Exiting compute_onlinedpo_pref ----") + + return output_preference_mask + + +def compute_online_dpo_loss( + policy_chosen_logps: torch.Tensor, + policy_rejected_logps: torch.Tensor, + reference_chosen_logps: torch.Tensor, + reference_rejected_logps: torch.Tensor, + beta: float, + label_smoothing: float = 0.0, + loss_type: str = "sigmoid", + reference_free: bool = False, +) -> torch.Tensor: + import torch.nn.functional as F + + pi_logratios = policy_chosen_logps - policy_rejected_logps + ref_logratios = reference_chosen_logps - reference_rejected_logps + + if reference_free: + ref_logratios = torch.zeros_like(pi_logratios) + + logits = pi_logratios - ref_logratios + + if loss_type == "sigmoid": + losses = -F.logsigmoid(beta * logits) * (1 - label_smoothing) - F.logsigmoid(-beta * logits) * label_smoothing + elif loss_type == "ipo": + losses = (logits - 1 / (2 * beta)) ** 2 + else: + raise ValueError(f"Unsupported loss_type: {loss_type}. Choose 'sigmoid', 'ipo', or 'hinge'.") + + return losses.mean() + + +def get_batch_logps( + logits: torch.FloatTensor, labels: torch.LongTensor, average_log_prob: bool = False +) -> torch.FloatTensor: + """ + Compute the log probabilities of the given labels under the given logits. + + Args: + logits: Logits of the model (e.g., huggingface CausalLMOutputs `logits`). + Shape: (batch_size, sequence_length, vocab_size) + labels: Labels for computing the sequence log probabilities. Shape: (batch_size, sequence_length) + average_log_prob: If True, return the average log probability per sequence. Otherwise, return the sum. + + Returns: + A tensor of shape (batch_size,) containing the average/sum log probabilities of the given sequences. + """ + if logits.shape[:-1] != labels.shape: + raise ValueError("Logits and labels must have the same shape[:-1]") + + # Ensure labels are contiguous and on the same device as logits + labels = labels.contiguous().to(logits.device) + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + + # Calculate per token log probability + loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100, reduction="none") + per_token_logps = -loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + per_token_logps = per_token_logps.view( + shift_logits.size(0), shift_logits.size(1) + ) # Reshape back to (batch_size, seq_len-1) + + # Create a mask for the labels that are not -100 + loss_mask = shift_labels != -100 + + # Apply the mask to the per token log probabilities + masked_logps = per_token_logps * loss_mask + + # Calculate the sum or average log probability per sequence + sequence_logps = masked_logps.sum(dim=-1) + + if average_log_prob: + # Avoid division by zero for sequences with no valid tokens + num_valid_tokens = loss_mask.sum(dim=-1) + return sequence_logps / torch.clamp(num_valid_tokens, min=1) + else: + return sequence_logps diff --git a/e2c/verl/recipe/spin/dp_actor.py b/e2c/verl/recipe/spin/dp_actor.py new file mode 100644 index 0000000000000000000000000000000000000000..35caa29c7004756dc286e200b31018d8ba0fc8c7 --- /dev/null +++ b/e2c/verl/recipe/spin/dp_actor.py @@ -0,0 +1,288 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import itertools +import math +from collections import defaultdict + +import numpy as np +import torch + +from recipe.spin.core_algos import compute_online_dpo_loss, get_batch_logps +from verl import DataProto +from verl.utils.device import get_device_name +from verl.utils.seqlen_balancing import get_reverse_idx, rearrange_micro_batches +from verl.workers.actor import DataParallelPPOActor + +__all__ = ["DataParallelPPOActor"] + + +class SPINDataParallelPPOActor(DataParallelPPOActor): + def compute_log_prob(self, data: DataProto) -> torch.Tensor: + """Compute the log probability of the responses given input_ids, attention_mask and position_ids + + Args: + data (DataProto): a DataProto containing keys + + ``input_ids``: tensor of shape [batch_size, sequence_length]. torch.int64. Note that input_ids is the + concatenation of prompt and response. Note that ``sequence_length = prompt_length + response_length``. + + ``attention_mask``: tensor of shape [batch_size, sequence_length]. torch.int64. + + ``position_ids``: tensor of shape [batch_size, sequence_length]. torch.int64. + + ``responses``: tensor of shape [batch_size, response_length]. torch.int64. + + Returns: + torch.Tensor: the log_prob tensor + """ + # set to eval + self.actor_module.eval() + + micro_batch_size = data.meta_info["micro_batch_size"] + temperature = data.meta_info["temperature"] # temperature must be in the data.meta_info to avoid silent error + use_dynamic_bsz = data.meta_info["use_dynamic_bsz"] + + select_keys = ["responses", "input_ids", "attention_mask", "position_ids"] + batch = data.select(batch_keys=select_keys).batch + has_multi_modal_inputs = "multi_modal_inputs" in data.non_tensor_batch.keys() + + if has_multi_modal_inputs: + num_micro_batches = data.batch.batch_size[0] // micro_batch_size + non_tensor_select_keys = ["multi_modal_inputs"] + micro_batches = data.select(select_keys, non_tensor_select_keys).chunk(num_micro_batches) + elif use_dynamic_bsz: + # split using dynamic bsz + max_token_len = data.meta_info["max_token_len"] * self.ulysses_sequence_parallel_size + micro_batches, indices = rearrange_micro_batches(batch=batch, max_token_len=max_token_len) + else: + micro_batches = batch.split(micro_batch_size) + + log_probs_lst = [] + for micro_batch in micro_batches: + if isinstance(micro_batch, DataProto): + micro_batch = {**micro_batch.batch, **micro_batch.non_tensor_batch} + + with torch.no_grad(): + _, log_probs = self._forward_micro_batch(micro_batch, temperature=temperature) + log_probs_lst.append(log_probs) + log_probs = torch.concat(log_probs_lst, dim=0) + + if use_dynamic_bsz: + indices = list(itertools.chain.from_iterable(indices)) + assert len(indices) == log_probs.size(0), f"{len(indices)} vs. {log_probs.size()}" + revert_indices = torch.tensor(get_reverse_idx(indices), dtype=torch.long) + log_probs = log_probs[revert_indices] + + return log_probs + + def update_policy_dpo_with_ref(self, data: DataProto): + """ + Performs the DPO update step using pre-calculated reference log probs + from an external, periodically updated reference model. + """ + self.actor_module.train() # Ensure training mode + + # --- Retrieve necessary data --- + try: + # Expects batch prepared by fit_dpo loop, including reference log probs + batch_td = data.batch + chosen_labels = batch_td["chosen_labels"] + rejected_labels = batch_td["rejected_labels"] + # ... other needed tensors like chosen/rejected input_ids, attention_mask, position_ids ... + + # === Get PRE-CALCULATED reference log probs from input data === + reference_chosen_logps = batch_td["reference_chosen_logps"] # Should be sequence-level logps + reference_rejected_logps = batch_td["reference_rejected_logps"] # Should be sequence-level logps + # ============================================================ + + # Get DPO params from meta_info + # beta = data.meta_info.get('dpo_beta', 0.1) # Default beta + beta = self.config.get("dpo_beta", 0.1) # Default beta + loss_type = data.meta_info.get("dpo_loss_type", "sigmoid") + label_smoothing = data.meta_info.get("dpo_label_smoothing", 0.0) + # reference_free should now be False as we provide ref logps + reference_free = data.meta_info.get("reference_free", False) # Default False + + except KeyError as e: + print(f"ERROR: Missing required key for DPO update (in update_policy_dpo): {e}") + print(f"Available keys in data.batch: {list(batch_td.keys())}") # Debug print + return {} # Return empty metrics on error + except Exception as e_data: + print(f"ERROR accessing data for DPO update (in update_policy_dpo): {e_data}") + return {} + + # --- Micro-batching Setup --- + micro_batch_size = self.config.get("ppo_micro_batch_size_per_gpu") + if micro_batch_size is None: + # Fallback or default if not set, or raise error + micro_batch_size = 1 # Example fallback, adjust as needed + print(f"Warning: 'ppo_micro_batch_size_per_gpu' not set, defaulting to {micro_batch_size}") + # raise ValueError("Config 'ppo_micro_batch_size_per_gpu' must be set.") + + # Ensure chosen_input_ids exists before getting shape + if "chosen_input_ids" not in batch_td: + print("ERROR: 'chosen_input_ids' not found in batch_td for DPO update.") + return {} + bsz = batch_td["chosen_input_ids"].shape[0] + + if bsz == 0: + print("Warning: DPO batch size is 0 in update_policy_dpo. Skipping update.") + return {"actor/dpo_loss": 0.0, "actor/grad_norm": 0.0} # Return zero metrics if batch is empty + + num_micro_batches = math.ceil(bsz / micro_batch_size) + gradient_accumulation_steps = num_micro_batches + + # --- Metrics Accumulation --- + total_loss = 0.0 + accumulated_metrics = defaultdict(list) + metrics = {} # Final metrics dict + + # --- Zero Gradients --- + self.actor_optimizer.zero_grad(set_to_none=True) + + # --- Micro-batch Loop --- + for i in range(num_micro_batches): + start_idx = i * micro_batch_size + end_idx = min(start_idx + micro_batch_size, bsz) + if start_idx >= end_idx: + continue + + # Slice the full DPO batch into micro-batches + # Important: Slice ALL required tensors, including labels and inputs + micro_batch_chosen_labels = chosen_labels[start_idx:end_idx] + micro_batch_rejected_labels = rejected_labels[start_idx:end_idx] + micro_batch_chosen_inputs = { + "input_ids": batch_td["chosen_input_ids"][start_idx:end_idx], + "attention_mask": batch_td["chosen_attention_mask"][start_idx:end_idx], + } + if "chosen_position_ids" in batch_td: + micro_batch_chosen_inputs["position_ids"] = batch_td["chosen_position_ids"][start_idx:end_idx] + + micro_batch_rejected_inputs = { + "input_ids": batch_td["rejected_input_ids"][start_idx:end_idx], + "attention_mask": batch_td["rejected_attention_mask"][start_idx:end_idx], + } + if "rejected_position_ids" in batch_td: + micro_batch_rejected_inputs["position_ids"] = batch_td["rejected_position_ids"][start_idx:end_idx] + + # Determine autocast dtype + autocast_dtype = torch.bfloat16 # Or get dynamically from config/FSDP settings + # --- Autocast Forward Pass --- + with torch.autocast(device_type=get_device_name(), dtype=autocast_dtype): + # --- Step 1: Forward pass for CURRENT policy log probs (with grad) --- + policy_chosen_outputs = self.actor_module(**micro_batch_chosen_inputs, use_cache=False) + policy_rejected_outputs = self.actor_module(**micro_batch_rejected_inputs, use_cache=False) + + # --- Step 2: Calculate CURRENT policy log probs using get_batch_logps --- + policy_chosen_logps = get_batch_logps( + policy_chosen_outputs.logits, micro_batch_chosen_labels, average_log_prob=False + ) + policy_rejected_logps = get_batch_logps( + policy_rejected_outputs.logits, micro_batch_rejected_labels, average_log_prob=False + ) + + # --- Step 3: Retrieve PRE-CALCULATED reference log probs (NO grad needed) --- + # Slice the full batch reference logps for the current micro-batch + micro_ref_chosen_logps = reference_chosen_logps[start_idx:end_idx] + micro_ref_rejected_logps = reference_rejected_logps[start_idx:end_idx] + # --- The ActorAsRef calculation block is REMOVED --- + + # --- Step 4: Calculate DPO Logits and Loss --- + pi_logratios = policy_chosen_logps - policy_rejected_logps + ref_logratios = micro_ref_chosen_logps - micro_ref_rejected_logps # Uses pre-calculated values + logits = pi_logratios - ref_logratios # DPO logits + + loss = compute_online_dpo_loss( + policy_chosen_logps=policy_chosen_logps, # Has grad + policy_rejected_logps=policy_rejected_logps, # Has grad + reference_chosen_logps=micro_ref_chosen_logps, # No grad (from input) + reference_rejected_logps=micro_ref_rejected_logps, # No grad (from input) + beta=beta, + label_smoothing=label_smoothing, + loss_type=loss_type, + reference_free=reference_free, # Should be False now + ) + + # --- Scale loss for gradient accumulation --- + scaled_loss = loss / gradient_accumulation_steps + + # --- Accumulate Metrics --- + total_loss += loss.item() # Unscaled loss + accumulated_metrics["actor/dpo_loss_batch"].append(loss.item()) + accumulated_metrics["actor/dpo_logits_batch"].append(logits.mean().item()) + # Accumulate policy and reference log probs/ratios if needed for debugging + accumulated_metrics["actor/policy_chosen_logps_batch"].append(policy_chosen_logps.mean().item()) + accumulated_metrics["actor/policy_rejected_logps_batch"].append(policy_rejected_logps.mean().item()) + accumulated_metrics["actor/reference_chosen_logps_batch"].append(micro_ref_chosen_logps.mean().item()) + accumulated_metrics["actor/reference_rejected_logps_batch"].append( + micro_ref_rejected_logps.mean().item() + ) + + # --- Backward Pass (outside autocast) --- + # Check if loss requires grad before backward + if scaled_loss.requires_grad: + scaled_loss.backward() + else: + print(f"Warning: Scaled loss at micro-batch {i} does not require grad. Skipping backward.") + + # --- End Micro-batch Loop --- + + # --- Optimizer Step (after accumulating gradients for all micro-batches) --- + grad_norm = self._optimizer_step() + + # --- Populate Final Metrics --- + if num_micro_batches > 0 and bsz > 0: # Check if any processing happened + metrics["actor/dpo_loss"] = total_loss / num_micro_batches + metrics["actor/grad_norm"] = ( + grad_norm.item() if torch.is_tensor(grad_norm) and torch.isfinite(grad_norm) else float("inf") + ) + # Average other accumulated metrics + for key, val_list in accumulated_metrics.items(): + if val_list: + metrics[key.replace("_batch", "")] = np.mean(val_list) + + # Calculate accuracy / rewards / margins based on averaged logprobs if desired + if ( + "actor/policy_chosen_logps" in metrics + and "actor/policy_rejected_logps" in metrics + and "actor/reference_chosen_logps" in metrics + and "actor/reference_rejected_logps" in metrics + ): + policy_ratio_mean = metrics["actor/policy_chosen_logps"] - metrics["actor/policy_rejected_logps"] + ref_ratio_mean = metrics["actor/reference_chosen_logps"] - metrics["actor/reference_rejected_logps"] + logits_mean = policy_ratio_mean - ref_ratio_mean + metrics["actor/rewards_chosen"] = beta * ( + metrics["actor/policy_chosen_logps"] - metrics["actor/reference_chosen_logps"] + ) + metrics["actor/rewards_rejected"] = beta * ( + metrics["actor/policy_rejected_logps"] - metrics["actor/reference_rejected_logps"] + ) + metrics["actor/rewards_accuracies"] = float(logits_mean > 0) # Mean accuracy proxy + metrics["actor/rewards_margins"] = metrics["actor/rewards_chosen"] - metrics["actor/rewards_rejected"] + + else: # Handle case where no micro-batches were run (e.g., bsz=0) + metrics["actor/dpo_loss"] = 0.0 + metrics["actor/grad_norm"] = 0.0 + # Initialize other metrics to 0 or NaN as appropriate + for key in accumulated_metrics.keys(): + metrics[key.replace("_batch", "")] = 0.0 + metrics["actor/rewards_chosen"] = 0.0 + metrics["actor/rewards_rejected"] = 0.0 + metrics["actor/rewards_accuracies"] = 0.0 + metrics["actor/rewards_margins"] = 0.0 + + return metrics # Return aggregated metrics diff --git a/e2c/verl/recipe/spin/fsdp_workers.py b/e2c/verl/recipe/spin/fsdp_workers.py new file mode 100644 index 0000000000000000000000000000000000000000..ccee0cb76f7c1022d6f695131778a30e3d33880e --- /dev/null +++ b/e2c/verl/recipe/spin/fsdp_workers.py @@ -0,0 +1,598 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import warnings + +import numpy as np +import psutil +import torch +import torch.distributed +from codetiming import Timer +from omegaconf import OmegaConf, open_dict +from torch.distributed.device_mesh import init_device_mesh + +import verl.utils.torch_functional as verl_F +from verl import DataProto +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.utils import hf_tokenizer +from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager +from verl.utils.device import get_device_id, get_device_name, get_nccl_backend, get_torch_device +from verl.utils.flops_counter import FlopsCounter +from verl.utils.fs import copy_to_local +from verl.utils.fsdp_utils import ( + get_fsdp_wrap_policy, + get_init_weight_context_manager, + init_fn, + load_fsdp_model_to_gpu, + load_fsdp_optimizer, + offload_fsdp_model_to_cpu, + offload_fsdp_optimizer, +) +from verl.utils.import_utils import import_external_libs +from verl.utils.model import compute_position_id_with_mask +from verl.utils.profiler import log_gpu_memory_usage +from verl.workers.fsdp_workers import ActorRolloutRefWorker +from verl.workers.sharding_manager.fsdp_ulysses import FSDPUlyssesShardingManager + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_PPO_LOGGING_LEVEL", "WARN")) + + +def create_device_mesh(world_size, fsdp_size): + if fsdp_size < 0 or fsdp_size >= world_size: + device_mesh = init_device_mesh(get_device_name(), mesh_shape=(world_size,), mesh_dim_names=["fsdp"]) + else: + device_mesh = init_device_mesh( + get_device_name(), mesh_shape=(world_size // fsdp_size, fsdp_size), mesh_dim_names=["ddp", "fsdp"] + ) + return device_mesh + + +def get_sharding_strategy(device_mesh): + from torch.distributed.fsdp import ShardingStrategy + + if device_mesh.ndim == 1: + sharding_strategy = ShardingStrategy.FULL_SHARD + elif device_mesh.ndim == 2: + sharding_strategy = ShardingStrategy.HYBRID_SHARD + else: + raise NotImplementedError(f"Get device mesh ndim={device_mesh.ndim}, but only support 1 or 2") + return sharding_strategy + + +class SPINRolloutRefWorker(ActorRolloutRefWorker): + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + from recipe.spin.dp_actor import SPINDataParallelPPOActor as DataParallelPPOActor + + # This is used to import external_lib into the huggingface systems + import_external_libs(self.config.model.get("external_lib", None)) + + override_model_config = OmegaConf.to_container(OmegaConf.create(self.config.model.get("override_config", {}))) + use_remove_padding = self.config.model.get("use_remove_padding", False) + use_fused_kernels = self.config.model.get("use_fused_kernels", False) + + if self._is_actor or self._is_rollout or self._is_ref: + # we need the model for actor and rollout + if self._is_actor or self._is_ref: + optim_config = self.config.actor.optim + fsdp_config = self.config.actor.fsdp_config + else: + optim_config = None + fsdp_config = OmegaConf.create() + self.actor_module_fsdp, self.actor_optimizer, self.actor_lr_scheduler, self.actor_model_config = ( + self._build_model_optimizer( + model_path=self.config.model.path, + fsdp_config=fsdp_config, + optim_config=optim_config, + override_model_config=override_model_config, + use_remove_padding=use_remove_padding, + use_fused_kernels=use_fused_kernels, + enable_gradient_checkpointing=self.config.model.get("enable_gradient_checkpointing", False), + trust_remote_code=self.config.model.get("trust_remote_code", False), + use_liger=self.config.model.get("use_liger", False), + role="actor", + ) + ) + + # get the original unwrapped module + self.actor_module = self.actor_module_fsdp._fsdp_wrapped_module + + if self._is_offload_optimizer: + offload_fsdp_optimizer(optimizer=self.actor_optimizer) + log_gpu_memory_usage("After offload actor optimizer during init", logger=logger) + # load from checkpoint + if self._is_actor or self._is_ref: + OmegaConf.set_struct(self.config.actor, True) + with open_dict(self.config.actor): + self.config.actor.use_remove_padding = use_remove_padding + self.config.actor.use_fused_kernels = use_fused_kernels + self.actor = DataParallelPPOActor( + config=self.config.actor, actor_module=self.actor_module_fsdp, actor_optimizer=self.actor_optimizer + ) + + if self._is_rollout: + self._build_rollout(trust_remote_code=self.config.model.get("trust_remote_code", False)) + + if self._is_ref: + self.ref_module_fsdp = self._build_model_optimizer( + model_path=self.config.model.path, + fsdp_config=self.config.ref.fsdp_config, + optim_config=None, + override_model_config=override_model_config, + use_remove_padding=use_remove_padding, + use_fused_kernels=use_fused_kernels, + trust_remote_code=self.config.model.get("trust_remote_code", False), + use_liger=self.config.model.get("use_liger", False), + role="ref", + )[0] + OmegaConf.set_struct(self.config.ref, True) + with open_dict(self.config.ref): + self.config.ref.use_remove_padding = use_remove_padding + self.config.ref.use_fused_kernels = use_fused_kernels + self.ref_policy = DataParallelPPOActor(config=self.config.ref, actor_module=self.ref_module_fsdp) + self.checkpoint_manager = FSDPCheckpointManager( + model=self.actor_module_fsdp, + optimizer=self.actor.actor_optimizer, + lr_scheduler=self.actor_lr_scheduler, + processing_class=self.processor if self.processor is not None else self.tokenizer, + checkpoint_config=self.config.actor.checkpoint, + ) + + if self._is_actor: + self.flops_counter = FlopsCounter(self.actor_model_config) + self.checkpoint_manager = FSDPCheckpointManager( + model=self.actor_module_fsdp, + optimizer=self.actor.actor_optimizer, + lr_scheduler=self.actor_lr_scheduler, + processing_class=self.processor if self.processor is not None else self.tokenizer, + checkpoint_config=self.config.actor.checkpoint, + ) + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + def compute_ref_log_prob(self, data: DataProto): + assert self._is_ref + + # Support all hardwares + data = data.to(get_device_id()) + + micro_batch_size = self.config.ref.log_prob_micro_batch_size_per_gpu + data.meta_info["micro_batch_size"] = micro_batch_size + data.meta_info["temperature"] = self.config.rollout.temperature + data.meta_info["max_token_len"] = self.config.ref.log_prob_max_token_len_per_gpu + data.meta_info["use_dynamic_bsz"] = self.config.ref.log_prob_use_dynamic_bsz + with self.ulysses_sharding_manager: + output = self.ref_policy.compute_log_prob(data=data) + output = DataProto.from_dict(tensors={"ref_log_prob": output}) + + output = output.to("cpu") + + # https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes + # unshard the root FSDP module + if self.world_size > 1: + self.ref_policy.actor_module._handle.reshard(True) + + return output + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + def compute_log_prob(self, data: DataProto): + assert self._is_actor + if self._is_offload_param: + load_fsdp_model_to_gpu(self.actor_module_fsdp) + + # Support all hardwares + data = data.to(get_device_id()) + # we should always recompute old_log_probs when it is HybridEngine + data.meta_info["micro_batch_size"] = self.config.rollout.log_prob_micro_batch_size_per_gpu + data.meta_info["max_token_len"] = self.config.rollout.log_prob_max_token_len_per_gpu + data.meta_info["use_dynamic_bsz"] = self.config.rollout.log_prob_use_dynamic_bsz + data.meta_info["temperature"] = self.config.rollout.temperature + # perform recompute log_prob + with self.ulysses_sharding_manager: + output = self.actor.compute_log_prob(data=data) + output = DataProto.from_dict( + tensors={"old_log_probs": output}, meta_info={"temperature": self.config.rollout.temperature} + ) + + output = output.to("cpu") + + # https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes + # unshard the root FSDP module + if self.world_size > 1: + self.actor.actor_module._handle.reshard(True) + + if self._is_offload_param: + offload_fsdp_model_to_cpu(self.actor_module_fsdp) + + log_gpu_memory_usage("After compute_log_prob", logger=logger) + return output + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + def update_actor_dpo(self, data: DataProto): + """ + Wrapper for actor update step. Handles FSDP state management. + Calls self.actor.update_policy which now contains DPO logic based + on pre-calculated log probabilities. + """ + # Support all hardwares + data = data.to(get_device_id()) + + assert self._is_actor # Make sure this worker has the actor role + if self.actor is None: + raise RuntimeError("Actor instance (self.actor) not initialized in worker.") + + # --- FSDP State Management --- + if self._is_offload_param: + load_fsdp_model_to_gpu(self.actor_module_fsdp) + if self._is_offload_optimizer: + load_fsdp_optimizer(optimizer=self.actor_optimizer, device_id=get_device_id()) + + log_gpu_memory_usage("Before update policy (DPO via PPO path)", logger=logger) + + # --- Ulysses Sharding (if used) --- + with self.ulysses_sharding_manager: + # --- Call the core update method (now containing DPO logic) --- + with Timer(name="update_policy_dpo_via_ppo", logger=None) as timer: # Use a distinct timer name + # Calls the modified update_policy method + metrics = self.actor.update_policy_dpo_with_ref(data=data) # <-- THIS CALLS THE MODIFIED FUNCTION + delta_time = timer.last + + # --- Add Performance Metrics --- + # MFU calculation might be less accurate/meaningful here for DPO + metrics["perf/approx_tokens_processed"] = torch.sum( + data.batch.get("attention_mask", torch.tensor(0)) + ).item() # Approx tokens + metrics["perf/max_memory_allocated_gb"] = get_torch_device().max_memory_allocated() / (1024**3) + metrics["perf/max_memory_reserved_gb"] = get_torch_device().max_memory_reserved() / (1024**3) + metrics["perf/cpu_memory_used_gb"] = psutil.virtual_memory().used / (1024**3) + global_num_tokens = data.meta_info["global_token_num"] + estimated_flops, promised_flops = self.flops_counter.estimate_flops(global_num_tokens, delta_time) + metrics["perf/mfu/actor"] = estimated_flops * self.config.ppo_epochs / promised_flops / self.world_size + + # --- LR Scheduler Step --- + lr = self.actor_lr_scheduler.get_last_lr()[0] + metrics["actor/lr"] = lr + self.actor_lr_scheduler.step() + + log_gpu_memory_usage("After update policy (DPO via PPO path)", logger=logger) + + # --- Prepare Output --- + output = DataProto(meta_info={"metrics": metrics}) + output = output.to("cpu") + + # --- FSDP State Management (Offload) --- + if self._is_offload_param: + offload_fsdp_model_to_cpu(self.actor_module_fsdp) + if self._is_offload_optimizer: + offload_fsdp_optimizer(optimizer=self.actor_optimizer) + + return output + + +# TODO(sgm): we may need to extract it to dp_reward_model.py +class RewardModelWorker(Worker): + """ + Note that we only implement the reward model that is subclass of AutoModelForTokenClassification. + """ + + def __init__(self, config): + super().__init__() + import torch.distributed + + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group(backend=get_nccl_backend()) + self.config = config + + # build device mesh for Ulysses Sequence Parallel + world_size = torch.distributed.get_world_size() + from torch.distributed.device_mesh import init_device_mesh + + fsdp_size = self.config.model.fsdp_config.fsdp_size + self.device_mesh = create_device_mesh(world_size=world_size, fsdp_size=fsdp_size) + + self.ulysses_device_mesh = None + self.ulysses_sequence_parallel_size = self.config.get("ulysses_sequence_parallel_size", 1) + dp = world_size // self.ulysses_sequence_parallel_size + if self.ulysses_sequence_parallel_size > 1: + self.ulysses_device_mesh = init_device_mesh( + get_device_name(), mesh_shape=(dp, self.ulysses_sequence_parallel_size), mesh_dim_names=["dp", "sp"] + ) + + if self.ulysses_device_mesh is not None: + is_collect = self.ulysses_device_mesh["sp"].get_local_rank() == 0 + self._register_dispatch_collect_info( + "reward", dp_rank=self.ulysses_device_mesh["dp"].get_local_rank(), is_collect=is_collect + ) + else: + self._register_dispatch_collect_info("reward", dp_rank=self.rank, is_collect=True) + + self.ulysses_sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) + + self.use_remove_padding = self.config.model.get("use_remove_padding", False) + + # normalize config + if self.config.micro_batch_size is not None: + self.config.micro_batch_size //= torch.distributed.get_world_size() + self.config.micro_batch_size_per_gpu = self.config.micro_batch_size + + def _build_model(self, config): + # the following line is necessary + from torch.distributed.fsdp import CPUOffload + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + from transformers import AutoConfig, AutoModelForTokenClassification + + # download the checkpoint from hdfs + local_path = copy_to_local(config.model.path) + + if self.config.model.input_tokenizer is None: + self._do_switch_chat_template = False + else: + self._do_switch_chat_template = True + input_tokenizer_local_path = copy_to_local(config.model.input_tokenizer) + self.input_tokenizer = hf_tokenizer( + input_tokenizer_local_path, trust_remote_code=config.model.get("trust_remote_code", False) + ) + self.tokenizer = hf_tokenizer(local_path, trust_remote_code=config.model.get("trust_remote_code", False)) + + trust_remote_code = config.model.get("trust_remote_code", False) + model_config = AutoConfig.from_pretrained(local_path, trust_remote_code=trust_remote_code) + model_config.num_labels = 1 + + # note that we have to create model in fp32. Otherwise, the optimizer is in bf16, which is incorrect + init_context = get_init_weight_context_manager( + use_meta_tensor=not model_config.tie_word_embeddings, mesh=self.device_mesh + ) + + with init_context(), warnings.catch_warnings(): + warnings.simplefilter("ignore") + model_config.classifier_dropout = 0.0 + reward_module = AutoModelForTokenClassification.from_pretrained( + pretrained_model_name_or_path=local_path, + config=model_config, + torch_dtype=torch.bfloat16, + attn_implementation="flash_attention_2", + trust_remote_code=trust_remote_code, + ) + + if config.model.get("use_remove_padding", False) or self.ulysses_sequence_parallel_size > 1: + from verl.models.transformers.monkey_patch import apply_monkey_patch + + apply_monkey_patch(model=reward_module, ulysses_sp_size=self.ulysses_sequence_parallel_size) + + reward_module.to(torch.bfloat16) + + auto_wrap_policy = get_fsdp_wrap_policy(module=reward_module, config=self.config.model.fsdp_config) + + fsdp_mesh = self.device_mesh + sharding_strategy = get_sharding_strategy(fsdp_mesh) + + reward_module = FSDP( + reward_module, + param_init_fn=init_fn, + use_orig_params=False, + auto_wrap_policy=auto_wrap_policy, + device_id=get_device_id(), + sharding_strategy=sharding_strategy, # zero3 + sync_module_states=True, + cpu_offload=CPUOffload(offload_params=True), + forward_prefetch=False, + device_mesh=self.device_mesh, + ) + + return reward_module + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + # This is used to import external_lib into the huggingface systems + import_external_libs(self.config.model.get("external_lib", None)) + self.reward_module = self._build_model(config=self.config) + + def _forward_micro_batch(self, micro_batch): + from flash_attn.bert_padding import index_first_axis, pad_input, rearrange, unpad_input + + from verl.utils.ulysses import gather_outputs_and_unpad, ulysses_pad_and_slice_inputs + + with torch.no_grad(), torch.autocast(device_type=get_device_name(), dtype=torch.bfloat16): + input_ids = micro_batch["input_ids"] + batch_size, seqlen = input_ids.shape + attention_mask = micro_batch["attention_mask"] + position_ids = micro_batch["position_ids"] + + if self.use_remove_padding: + input_ids_rmpad, indices, *_ = unpad_input( + input_ids.unsqueeze(-1), attention_mask + ) # input_ids_rmpad (total_nnz, ...) + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) + + # unpad the position_ids to align the rotary + position_ids_rmpad = index_first_axis( + rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices + ).transpose(0, 1) + + # pad and slice the inputs if sp > 1 + if self.ulysses_sequence_parallel_size > 1: + input_ids_rmpad, position_ids_rmpad, pad_size = ulysses_pad_and_slice_inputs( + input_ids_rmpad, position_ids_rmpad, sp_size=self.ulysses_sequence_parallel_size + ) + + # only pass input_ids and position_ids to enable flash_attn_varlen + output = self.reward_module( + input_ids=input_ids_rmpad, attention_mask=None, position_ids=position_ids_rmpad, use_cache=False + ) # prevent model thinks we are generating + reward_rmpad = output.logits + reward_rmpad = reward_rmpad.squeeze(0) # (total_nnz) + + # gather output if sp > 1 + if self.ulysses_sequence_parallel_size > 1: + reward_rmpad = gather_outputs_and_unpad( + reward_rmpad, gather_dim=0, unpad_dim=0, padding_size=pad_size + ) + + # pad it back + rm_score = pad_input(reward_rmpad, indices=indices, batch=batch_size, seqlen=seqlen).squeeze(-1) + else: + output = self.reward_module( + input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, use_cache=False + ) + rm_score = output.logits # (batch_size, seq_len, 1) + rm_score = rm_score.squeeze(-1) + + # extract the result of the last valid token + eos_mask_idx = torch.argmax(position_ids * attention_mask, dim=-1) # (bsz,) + rm_score = rm_score[torch.arange(batch_size), eos_mask_idx] + return rm_score + + def _expand_to_token_level(self, data: DataProto, scores: torch.Tensor): + batch_size = data.batch.batch_size[0] + # expand as token_level_reward + attention_mask = data.batch["attention_mask"] + position_ids = data.batch["position_ids"] + response_length = data.batch["responses"].shape[-1] + eos_mask_idx = torch.argmax(position_ids * attention_mask, dim=-1) # (bsz,) + token_level_scores = torch.zeros_like(attention_mask, dtype=scores.dtype) # (bsz, seqlen) + token_level_scores[torch.arange(batch_size), eos_mask_idx] = scores + + # select the response part + token_level_scores = token_level_scores[:, -response_length:] + + return token_level_scores + + def _switch_chat_template(self, data: DataProto): + src_max_length = data.batch["attention_mask"].shape[-1] + + src_tokenizer = self.input_tokenizer + target_tokenizer = self.tokenizer + + rm_input_ids = [] + rm_attention_mask = [] + + for i in range(data.batch.batch_size[0]): + if not isinstance(data.non_tensor_batch["raw_prompt"][i], list | np.ndarray): + raise TypeError( + f"raw_prompt must be a list or numpy array, got {type(data.non_tensor_batch['raw_prompt'][i])}" + ) + + # extract raw prompt + chat: list = list(data.non_tensor_batch["raw_prompt"][i]) + + # extract response + response_ids = data.batch["responses"][i] + response_length = response_ids.shape[-1] + valid_response_length = data.batch["attention_mask"][i][-response_length:].sum() + valid_response_ids = response_ids[:valid_response_length] + + # decode + response = src_tokenizer.decode(valid_response_ids) + # remove bos and eos + response = response.replace(src_tokenizer.eos_token, "") + + chat.append({"role": "assistant", "content": response}) + + prompt_with_chat_template = target_tokenizer.apply_chat_template( + chat, add_generation_prompt=False, tokenize=False + ) + if self.rank == 0 and i == 0: + # for debugging purpose + print(f"Switch template. chat: {prompt_with_chat_template}") + + # the maximum length is actually determined by the reward model itself + max_length = self.config.get("max_length", src_max_length) + if max_length is None: + max_length = src_max_length + + model_inputs = target_tokenizer(prompt_with_chat_template, return_tensors="pt", add_special_tokens=False) + input_ids, attention_mask = verl_F.postprocess_data( + input_ids=model_inputs["input_ids"], + attention_mask=model_inputs["attention_mask"], + max_length=max_length, + pad_token_id=target_tokenizer.pad_token_id, + left_pad=False, # right padding + truncation=self.config.get("truncation", "right"), + ) # truncate from the right + + rm_input_ids.append(input_ids) + rm_attention_mask.append(attention_mask) + + rm_input_ids = torch.cat(rm_input_ids, dim=0) + rm_attention_mask = torch.cat(rm_attention_mask, dim=0) + + rm_position_ids = compute_position_id_with_mask(rm_attention_mask) + + rm_inputs = {"input_ids": rm_input_ids, "attention_mask": rm_attention_mask, "position_ids": rm_position_ids} + + return DataProto.from_dict(rm_inputs) + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="reward")) + def compute_rm_score(self, data: DataProto): + import itertools + + from verl.utils.seqlen_balancing import get_reverse_idx, rearrange_micro_batches + + # Support all hardwares + data = data.to(get_device_id()) + if self._do_switch_chat_template: + rm_data = self._switch_chat_template(data) + else: + rm_input_ids = data.batch["input_ids"] + rm_attention_mask = data.batch["attention_mask"] + rm_position_ids = data.batch["position_ids"] + rm_inputs = { + "input_ids": rm_input_ids, + "attention_mask": rm_attention_mask, + "position_ids": rm_position_ids, + } + rm_data = DataProto.from_dict(rm_inputs) + + # Support all hardwares + rm_data.batch = rm_data.batch.to(get_device_id()) + + # perform forward computation + with self.ulysses_sharding_manager: + rm_data = self.ulysses_sharding_manager.preprocess_data(data=rm_data) + data = self.ulysses_sharding_manager.preprocess_data(data=data) + + use_dynamic_bsz = self.config.use_dynamic_bsz + if use_dynamic_bsz: + max_token_len = self.config.forward_max_token_len_per_gpu * self.ulysses_sequence_parallel_size + micro_batches, indices = rearrange_micro_batches(batch=rm_data.batch, max_token_len=max_token_len) + else: + micro_batches = rm_data.batch.split(self.config.micro_batch_size_per_gpu) + output = [] + for micro_batch in micro_batches: + rm_score = self._forward_micro_batch(micro_batch) + output.append(rm_score) + scores = torch.cat(output, dim=0) # (batch_size) + + if use_dynamic_bsz: + indices = list(itertools.chain.from_iterable(indices)) + assert len(indices) == scores.size(0), f"{len(indices)} vs. {scores.size()}" + revert_indices = torch.tensor(get_reverse_idx(indices), dtype=torch.long) + scores = scores[revert_indices] + + token_level_scores = self._expand_to_token_level(data, scores) + # Note that this is only the scores, may not be the final rewards used to train RL + output = DataProto.from_dict(tensors={"rm_scores": token_level_scores}) + output = self.ulysses_sharding_manager.postprocess_data(data=output) + + # https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes + # unshard the root FSDP module + self.reward_module._handle.reshard(True) + + output = output.to("cpu") + return output diff --git a/e2c/verl/recipe/spin/main_spin.py b/e2c/verl/recipe/spin/main_spin.py new file mode 100644 index 0000000000000000000000000000000000000000..0001a3841cd93b416a730184222fb56e36cc099e --- /dev/null +++ b/e2c/verl/recipe/spin/main_spin.py @@ -0,0 +1,167 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import hydra +import ray + +from recipe.spin.spin_trainer import RaySPINTrainer +from recipe.spin.utils import validate_config +from verl.trainer.ppo.reward import get_custom_reward_fn +from verl.trainer.ppo.utils import need_reference_policy + + +@hydra.main(config_path="config", config_name="spin_trainer", version_base=None) +def main(config): + run_ppo(config) + + +def run_ppo(config) -> None: + # TODO(linjunrong.ocss884): this ENV is left for resolving SGLang conflict with ray devices + # isolation, will solve in the future + os.environ["ENSURE_CUDA_VISIBLE_DEVICES"] = os.environ.get("CUDA_VISIBLE_DEVICES", "") + if not ray.is_initialized(): + # this is for local ray cluster + ray.init( + runtime_env={ + "env_vars": {"TOKENIZERS_PARALLELISM": "true", "NCCL_DEBUG": "WARN", "VLLM_LOGGING_LEVEL": "WARN"} + } + ) + + runner = TaskRunner.remote() + ray.get(runner.run.remote(config)) + + +@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head +class TaskRunner: + def run(self, config): + # print initial config + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_to_local + + pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values + OmegaConf.resolve(config) + + # define worker classes + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + assert config.critic.strategy in {"fsdp", "fsdp2"} + # from recipe.spin.fsdp_workers import ActorRolloutRefWorker + from recipe.spin.fsdp_workers import SPINRolloutRefWorker + from verl.single_controller.ray import RayWorkerGroup + + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.single_controller.ray import RayWorkerGroup + + ray_worker_group_cls = RayWorkerGroup + + else: + raise NotImplementedError + + from recipe.spin.spin_trainer import ResourcePoolManager, Role + + role_worker_mapping = { + # Role.ActorRollout: ray.remote(ActorRolloutRefWorker), + Role.ActorRollout: ray.remote(SPINRolloutRefWorker), + # Role.Critic: ray.remote(CriticWorker), + } + + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + # Role.Critic: global_pool_id, + } + + if config.reward_model.enable: + if config.reward_model.strategy in {"fsdp", "fsdp2"}: + from recipe.spin.fsdp_workers import RewardModelWorker + elif config.reward_model.strategy == "megatron": + from verl.workers.megatron_workers import RewardModelWorker + else: + raise NotImplementedError + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + mapping[Role.RewardModel] = global_pool_id + + # use reference model + # if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + # role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) + role_worker_mapping[Role.RefPolicy] = ray.remote(SPINRolloutRefWorker) + mapping[Role.RefPolicy] = global_pool_id + + # validate config + validate_config( + config=config, + use_reference_policy=need_reference_policy(role_worker_mapping), + use_critic=False, + ) + + # download the checkpoint from hdfs + local_path = copy_to_local(config.actor_rollout_ref.model.path) + + # instantiate tokenizer + from verl.utils import hf_processor, hf_tokenizer + + trust_remote_code = config.data.get("trust_remote_code", False) + tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) + processor = hf_processor(local_path, use_fast=True) # used for multimodal LLM, could be none + + from verl.workers.reward_manager import get_reward_manager_cls + + # Note(haibin.lin): please make sure custom reward managers are imported and + # registered via `verl.workers.reward_manager.register` + reward_manager_name = config.reward_model.get("reward_manager", "naive") + reward_manager_cls = get_reward_manager_cls(reward_manager_name) + + compute_score = get_custom_reward_fn(config) + reward_kwargs = dict(config.reward_model.get("reward_kwargs", {})) + reward_fn = reward_manager_cls( + tokenizer=tokenizer, + num_examine=0, + compute_score=compute_score, + reward_fn_key=config.data.reward_fn_key, + **reward_kwargs, + ) + + # Note that we always use function-based RM for validation + val_reward_fn = reward_manager_cls( + tokenizer=tokenizer, num_examine=1, compute_score=compute_score, reward_fn_key=config.data.reward_fn_key + ) + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + + trainer = RaySPINTrainer( + config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + ) + trainer.init_workers() + trainer.fit_dpo() + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/recipe/spin/run_spin.sh b/e2c/verl/recipe/spin/run_spin.sh new file mode 100644 index 0000000000000000000000000000000000000000..798dedabed0fae0c601899d83bd38f5adde909ea --- /dev/null +++ b/e2c/verl/recipe/spin/run_spin.sh @@ -0,0 +1,29 @@ +set -e +set -x +VISIBLE_DEVICES="4,5,6,7" +export HYDRA_FULL_ERROR=1 + +CUDA_VISIBLE_DEVICES=${VISIBLE_DEVICES} python3 -m recipe.spin.main_spin \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size=8 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=64 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=64 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=console \ + trainer.val_before_train=True \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=1 \ + +trainer.log_freq=1 \ + trainer.ref_update_freq=1 \ + trainer.total_epochs=1000 2>&1 | tee verl_demo.log \ No newline at end of file diff --git a/e2c/verl/recipe/spin/spin_trainer.py b/e2c/verl/recipe/spin/spin_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..d312e7e4841ff4280da5ca5448ac76dff8e9c7c8 --- /dev/null +++ b/e2c/verl/recipe/spin/spin_trainer.py @@ -0,0 +1,1308 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import traceback +import uuid +from collections import defaultdict +from contextlib import contextmanager +from dataclasses import dataclass, field +from pprint import pprint +from typing import Any, Optional + +import numpy as np +import ray +import torch +from codetiming import Timer +from omegaconf import OmegaConf, open_dict +from torch.utils.data import Dataset, Sampler +from torchdata.stateful_dataloader import StatefulDataLoader +from tqdm import tqdm + +from recipe.spin import core_algos +from verl import DataProto +from verl.protocol import pad_dataproto_to_divisor, unpad_dataproto +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup +from verl.single_controller.ray.base import create_colocated_worker_cls +from verl.trainer.ppo.metric_utils import ( + compute_throughout_metrics, + compute_timing_metrics, + process_validation_metrics, + reduce_metrics, +) +from verl.trainer.ppo.utils import Role, WorkerType, need_reference_policy, need_reward_model +from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path +from verl.utils.seqlen_balancing import get_seqlen_balanced_partitions, log_seqlen_unbalance +from verl.utils.torch_functional import masked_mean +from verl.utils.tracking import ValidationGenerationsLogger + + +@dataclass +class ResourcePoolManager: + """ + Define a resource pool specification. Resource pool will be initialized first. + Mapping + """ + + resource_pool_spec: dict[str, list[int]] + mapping: dict[Role, str] + resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) + + def create_resource_pool(self): + for resource_pool_name, process_on_nodes in self.resource_pool_spec.items(): + # max_colocate_count means the number of WorkerGroups (i.e. processes) in each RayResourcePool + # For FSDP backend, we recommend using max_colocate_count=1 that merge all WorkerGroups into one. + # For Megatron backend, we recommend using max_colocate_count>1 that can utilize different + # WorkerGroup for different models + resource_pool = RayResourcePool( + process_on_nodes=process_on_nodes, use_gpu=True, max_colocate_count=1, name_prefix=resource_pool_name + ) + self.resource_pool_dict[resource_pool_name] = resource_pool + + self._check_resource_available() + + def get_resource_pool(self, role: Role) -> RayResourcePool: + """Get the resource pool of the worker_cls""" + return self.resource_pool_dict[self.mapping[role]] + + def get_n_gpus(self) -> int: + """Get the number of gpus in this cluster.""" + return sum([n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes]) + + def _check_resource_available(self): + """Check if the resource pool can be satisfied in this ray cluster.""" + node_available_resources = ray._private.state.available_resources_per_node() + node_available_gpus = {node: node_info.get("GPU", 0) for node, node_info in node_available_resources.items()} + + # check total required gpus can be satisfied + total_available_gpus = sum(node_available_gpus.values()) + total_required_gpus = sum( + [n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes] + ) + if total_available_gpus < total_required_gpus: + raise ValueError( + f"Total available GPUs {total_available_gpus} is less than total desired GPUs {total_required_gpus}" + ) + + # check each resource pool can be satisfied, O(#resource_pools * #nodes) + for resource_pool_name, process_on_nodes in self.resource_pool_spec.items(): + num_gpus, num_nodes = process_on_nodes[0], len(process_on_nodes) + for node, available_gpus in node_available_gpus.items(): + if available_gpus >= num_gpus: + node_available_gpus[node] -= num_gpus + num_nodes -= 1 + if num_nodes == 0: + break + if num_nodes > 0: + raise ValueError( + f"Resource pool {resource_pool_name}: {num_gpus}*{num_nodes} cannot be satisfied in this " + f"ray cluster" + ) + + +def _compute_response_info(batch: DataProto) -> dict[str, Any]: + """Placeholder: Computes prompt and response lengths.""" + try: + # Assuming 'prompts' and 'responses' keys exist after generation/union + prompt_len = batch.batch["prompts"].shape[1] + resp_len = batch.batch["responses"].shape[1] + # This is simplified - real implementation might use attention masks + # to get actual lengths per sample. + batch_size = batch.batch.batch_size[0] + prompt_lengths_tensor = torch.full((batch_size,), prompt_len, dtype=torch.float32, device=batch.batch.device) + response_lengths_tensor = torch.full((batch_size,), resp_len, dtype=torch.float32, device=batch.batch.device) + + # Try getting actual lengths from attention mask if possible (more accurate) + if "response_mask" in batch.batch: + response_lengths_tensor = batch.batch["response_mask"].sum(dim=1).float() + # if "attention_mask" in batch.batch and "response_mask" in batch.batch: + # full_mask = batch.batch["attention_mask"] + # resp_mask = batch.batch["response_mask"] + # Infer prompt mask length based on where response mask starts or total length + # This logic depends heavily on how your masks are constructed. + # Example: prompt_lengths_tensor = full_mask.sum(dim=1).float() - response_lengths_tensor + # Fallback to using prompt shape if mask logic is complex: + prompt_lengths_tensor = torch.tensor( + [batch.batch["prompts"].shape[1]] * batch_size, dtype=torch.float32, device=batch.batch.device + ) + + return { + "prompt_length": prompt_lengths_tensor, + "response_length": response_lengths_tensor, + "max_response_length": resp_len, + "max_prompt_length": prompt_len, # Or from config if fixed padding + } + except KeyError as e: + print(f"Warning: Missing key in _compute_response_info: {e}. Returning defaults.") + # Return default/dummy values if keys are missing + b_size = batch.batch.batch_size[0] if batch.batch.batch_size else 1 + max_resp = batch.batch.get("responses").shape[1] if batch.batch.get("responses") is not None else 0 + max_prompt = batch.batch.get("prompts").shape[1] if batch.batch.get("prompts") is not None else 0 + return { + "prompt_length": torch.zeros(b_size), + "response_length": torch.zeros(b_size), + "max_response_length": max_resp, + "max_prompt_length": max_prompt, + } + + +# --- Modified Metric Function --- +def compute_dpo_data_metrics(batch: DataProto) -> dict[str, Any]: + """ + Computes and returns metrics relevant for the DPO-like process. + Assumes 'batch' contains results after generation and preference marking, + potentially including 'dpo_logits', 'preferences', 'chosen_logps', etc. + Removes PPO-specific advantage/return/critic metrics. + """ + print("---- [DEBUG] Computing DPO Data Metrics ----") + metrics = {} + try: + # --- Scores and Rewards (from reward_fn) --- + if "token_level_scores" in batch.batch and batch.batch["token_level_scores"] is not None: + sequence_score = batch.batch["token_level_scores"].sum(-1) + metrics.update( + { + "reward/score/mean": torch.mean(sequence_score).item(), + "reward/score/max": torch.max(sequence_score).item(), + "reward/score/min": torch.min(sequence_score).item(), + } + ) + else: + print("DEBUG compute_dpo_data_metrics: 'token_level_scores' not found.") + + if "token_level_rewards" in batch.batch and batch.batch["token_level_rewards"] is not None: + sequence_reward = batch.batch["token_level_rewards"].sum(-1) + metrics.update( + { + "reward/rewards/mean": torch.mean(sequence_reward).item(), + "reward/rewards/max": torch.max(sequence_reward).item(), + "reward/rewards/min": torch.min(sequence_reward).item(), + } + ) + else: + print("DEBUG compute_dpo_data_metrics: 'token_level_rewards' not found.") + + # --- DPO Specific Metrics (if stored previously) --- + if "dpo_logits" in batch.batch and batch.batch["dpo_logits"] is not None: + metrics["actor/dpo_logits"] = batch.batch["dpo_logits"].mean().item() + else: + print("DEBUG compute_dpo_data_metrics: 'dpo_logits' not found.") + + if "chosen_logps" in batch.batch and batch.batch["chosen_logps"] is not None: + metrics["actor/chosen_logps"] = batch.batch["chosen_logps"].mean().item() + else: + print("DEBUG compute_dpo_data_metrics: 'chosen_logps' not found.") + + if "rejected_logps" in batch.batch and batch.batch["rejected_logps"] is not None: + metrics["actor/rejected_logps"] = batch.batch["rejected_logps"].mean().item() + else: + print("DEBUG compute_dpo_data_metrics: 'rejected_logps' not found.") + + # Add metrics based on the 'preferences' mask if available + # if "preferences" in batch.batch and batch.batch["preferences"] is not None: + # prefs_mask = batch.batch["preferences"] # Shape [batch_size * n] + # Calculate accuracy based on RM scores (assuming higher score -> True in mask) + # Requires chosen/rejected scores to be available or recalculated + # This is complex here, better calculated in the main loop or update function + + # --- Length Metrics --- + response_info = _compute_response_info(batch) + prompt_length = response_info["prompt_length"] + response_length = response_info["response_length"] + max_response_length = response_info["max_response_length"] + max_prompt_length = response_info["max_prompt_length"] # Use calculated or from config + + metrics.update( + { + "response_length/mean": torch.mean(response_length).item(), + "response_length/max": torch.max(response_length).item(), + "response_length/min": torch.min(response_length).item(), + "response_length/clip_ratio": torch.mean(torch.eq(response_length, max_response_length).float()).item(), + "prompt_length/mean": torch.mean(prompt_length).item(), + "prompt_length/max": torch.max(prompt_length).item(), + "prompt_length/min": torch.min(prompt_length).item(), + # Prompt clip ratio might need adjustment based on how max_prompt_length is defined + "prompt_length/clip_ratio": torch.mean(torch.eq(prompt_length, max_prompt_length).float()).item(), + } + ) + + except KeyError as e: + print(f"ERROR in compute_dpo_data_metrics: Missing key {e}") + except Exception as e: + print(f"ERROR in compute_dpo_data_metrics: {e}") + traceback.print_exc() + + print(f"---- [DEBUG] Calculated DPO Data Metrics: {list(metrics.keys())} ----") + return metrics + + +def apply_kl_penalty(data: DataProto, kl_ctrl: core_algos.AdaptiveKLController, kl_penalty="kl"): + responses = data.batch["responses"] + response_length = responses.size(1) + token_level_scores = data.batch["token_level_scores"] + batch_size = data.batch.batch_size[0] + attention_mask = data.batch["attention_mask"] + response_mask = attention_mask[:, -response_length:] + + # compute kl between ref_policy and current policy + # When apply_kl_penalty, algorithm.use_kl_in_reward=True, so the reference model has been enabled. + kld = core_algos.kl_penalty( + data.batch["old_log_probs"], data.batch["ref_log_prob"], kl_penalty=kl_penalty + ) # (batch_size, response_length) + kld = kld * response_mask + beta = kl_ctrl.value + + token_level_rewards = token_level_scores - beta * kld + + current_kl = masked_mean(kld, mask=response_mask, axis=-1) # average over sequence + current_kl = torch.mean(current_kl, dim=0).item() + + # according to https://github.com/huggingface/trl/blob/951ca1841f29114b969b57b26c7d3e80a39f75a0/trl/trainer/ppo_trainer.py#L837 + kl_ctrl.update(current_kl=current_kl, n_steps=batch_size) + data.batch["token_level_rewards"] = token_level_rewards + + metrics = {"actor/reward_kl_penalty": current_kl, "actor/reward_kl_penalty_coeff": beta} + + return data, metrics + + +def compute_response_mask(data: DataProto): + responses = data.batch["responses"] + response_length = responses.size(1) + attention_mask = data.batch["attention_mask"] + return attention_mask[:, -response_length:] + + +def compute_onlineDPO_pref(data: DataProto): + """ + Wrapper to compute DPO preference and add it to the DataProto batch. + Includes debugging prints. + """ + # print(f"\n---- [DEBUG] Entering compute_onlineDPO_pref ----") + # print(f" Input batch keys: {list(data.batch.keys())}") + + # Check inputs + rewards_tensor = data.batch.get("token_level_rewards") + mask_tensor = data.batch.get("response_mask") + + if rewards_tensor is None or mask_tensor is None: + print(" ERROR: Missing 'token_level_rewards' or 'response_mask' in input data!") + # Handle error case - maybe return original data or raise? + # Returning original data for now to potentially allow skipping + return data + + try: + preferences = core_algos.compute_onlinedpo_pref(token_level_rewards=rewards_tensor, response_mask=mask_tensor) + # Store the result + data.batch["preferences"] = preferences + + except AttributeError: + print("ERROR: Function 'compute_online_dpo_preference' not found in core_algos.py!") + # Assign dummy value or raise error + data.batch["preferences"] = None # Indicate failure + except Exception as e_pref: + print(f"ERROR during core_algos.compute_online_dpo_preference: {e_pref}") + import traceback + + traceback.print_exc() + data.batch["preferences"] = None # Indicate failure + + # print(f"---- [DEBUG] Exiting compute_onlineDPO_pref ----") + return data + + +@contextmanager +def _timer(name: str, timing_raw: dict[str, float]): + with Timer(name=name, logger=None) as timer: + yield + timing_raw[name] = timer.last + + +class RaySPINTrainer: + """ + Note that this trainer runs on the driver process on a single CPU/GPU node. + """ + + # TODO: support each role have individual ray_worker_group_cls, + # i.e., support different backend of different role + def __init__( + self, + config, + tokenizer, + role_worker_mapping: dict[Role, WorkerType], + resource_pool_manager: ResourcePoolManager, + ray_worker_group_cls: RayWorkerGroup = RayWorkerGroup, + processor=None, + reward_fn=None, + val_reward_fn=None, + train_dataset: Optional[Dataset] = None, + val_dataset: Optional[Dataset] = None, + collate_fn=None, + train_sampler: Optional[Sampler] = None, + device_name=None, + ): + # assert get_torch_device().is_available(), 'cuda must be available on driver' + + self.tokenizer = tokenizer + self.processor = processor + self.config = config + self.reward_fn = reward_fn + self.val_reward_fn = val_reward_fn + + self.hybrid_engine = config.actor_rollout_ref.hybrid_engine + assert self.hybrid_engine, "Currently, only support hybrid engine" + + if self.hybrid_engine: + assert Role.ActorRollout in role_worker_mapping, f"{role_worker_mapping.keys()=}" + + self.role_worker_mapping = role_worker_mapping + self.resource_pool_manager = resource_pool_manager + self.use_reference_policy = need_reference_policy(role_worker_mapping) + self.use_rm = need_reward_model(role_worker_mapping) + self.use_critic = False + self.ray_worker_group_cls = ray_worker_group_cls + self.validation_generations_logger = ValidationGenerationsLogger() + self.async_rollout_mode = False + self.device_name = device_name if device_name else self.config.trainer.device + + # define in-reward KL control + # kl loss control currently not suppoorted + if config.algorithm.use_kl_in_reward: + self.kl_ctrl_in_reward = core_algos.get_kl_controller(config.algorithm.kl_ctrl) + + self._create_dataloader(train_dataset, val_dataset, collate_fn, train_sampler) + + def _create_dataloader(self, train_dataset, val_dataset, collate_fn, train_sampler): + """ + Creates the train and validation dataloaders. + """ + # TODO: we have to make sure the batch size is divisible by the dp size + from verl.trainer.main_ppo import create_rl_dataset, create_rl_sampler + + if train_dataset is None: + train_dataset = create_rl_dataset( + self.config.data.train_files, self.config.data, self.tokenizer, self.processor + ) + if val_dataset is None: + val_dataset = create_rl_dataset( + self.config.data.val_files, self.config.data, self.tokenizer, self.processor + ) + self.train_dataset, self.val_dataset = train_dataset, val_dataset + + if train_sampler is None: + train_sampler = create_rl_sampler(self.config.data, self.train_dataset) + if collate_fn is None: + from verl.utils.dataset.rl_dataset import collate_fn as default_collate_fn + + collate_fn = default_collate_fn + + self.train_dataloader = StatefulDataLoader( + dataset=self.train_dataset, + batch_size=self.config.data.get("gen_batch_size", self.config.data.train_batch_size), + num_workers=self.config.data.get("dataloader_num_workers", 8), + drop_last=True, + collate_fn=collate_fn, + sampler=train_sampler, + ) + + val_batch_size = self.config.data.val_batch_size # Prefer config value if set + if val_batch_size is None: + val_batch_size = len(self.val_dataset) + + self.val_dataloader = StatefulDataLoader( + dataset=self.val_dataset, + batch_size=val_batch_size, + num_workers=self.config.data.get("dataloader_num_workers", 8), + shuffle=False, + drop_last=False, + collate_fn=collate_fn, + ) + + assert len(self.train_dataloader) >= 1, "Train dataloader is empty!" + assert len(self.val_dataloader) >= 1, "Validation dataloader is empty!" + + print( + f"Size of train dataloader: {len(self.train_dataloader)}, " + f"Size of val dataloader: {len(self.val_dataloader)}" + ) + + total_training_steps = len(self.train_dataloader) * self.config.trainer.total_epochs + + if self.config.trainer.total_training_steps is not None: + total_training_steps = self.config.trainer.total_training_steps + + self.total_training_steps = total_training_steps + print(f"Total training steps: {self.total_training_steps}") + + try: + OmegaConf.set_struct(self.config, True) + with open_dict(self.config): + if OmegaConf.select(self.config, "actor_rollout_ref.actor.optim"): + self.config.actor_rollout_ref.actor.optim.total_training_steps = total_training_steps + if OmegaConf.select(self.config, "critic.optim"): + self.config.critic.optim.total_training_steps = total_training_steps + except Exception as e: + print(f"Warning: Could not set total_training_steps in config. Structure missing? Error: {e}") + + def _maybe_log_val_generations(self, inputs, outputs, scores): + """Log a table of validation samples to the configured logger (wandb or swanlab)""" + + generations_to_log = self.config.trainer.log_val_generations + + if generations_to_log == 0: + return + + import numpy as np + + # Create tuples of (input, output, score) and sort by input text + samples = list(zip(inputs, outputs, scores, strict=True)) + samples.sort(key=lambda x: x[0]) # Sort by input text + + # Use fixed random seed for deterministic shuffling + rng = np.random.RandomState(42) + rng.shuffle(samples) + + # Take first N samples after shuffling + samples = samples[:generations_to_log] + + # Log to each configured logger + self.validation_generations_logger.log(self.config.trainer.logger, samples, self.global_steps) + + def _validate(self): + data_source_lst = [] + reward_extra_infos_dict: dict[str, list] = defaultdict(list) + + # Lists to collect samples for the table + sample_inputs = [] + sample_outputs = [] + sample_scores = [] + + for test_data in self.val_dataloader: + test_batch = DataProto.from_single_dict(test_data) + + # repeat test batch + test_batch = test_batch.repeat( + repeat_times=self.config.actor_rollout_ref.rollout.val_kwargs.n, interleave=True + ) + + # we only do validation on rule-based rm + if self.config.reward_model.enable and test_batch[0].non_tensor_batch["reward_model"]["style"] == "model": + return {} + + # Store original inputs + input_ids = test_batch.batch["input_ids"] + # TODO: Can we keep special tokens except for padding tokens? + input_texts = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in input_ids] + sample_inputs.extend(input_texts) + + batch_keys_to_pop = ["input_ids", "attention_mask", "position_ids"] + non_tensor_batch_keys_to_pop = ["raw_prompt_ids"] + if "multi_modal_inputs" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.extend(["multi_modal_data", "multi_modal_inputs"]) + if "raw_prompt" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("raw_prompt") + if "tools_kwargs" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("tools_kwargs") + test_gen_batch = test_batch.pop( + batch_keys=batch_keys_to_pop, + non_tensor_batch_keys=non_tensor_batch_keys_to_pop, + ) + + test_gen_batch.meta_info = { + "eos_token_id": self.tokenizer.eos_token_id, + "pad_token_id": self.tokenizer.pad_token_id, + "recompute_log_prob": False, + "do_sample": self.config.actor_rollout_ref.rollout.val_kwargs.do_sample, + "validate": True, + } + print(f"test_gen_batch meta info: {test_gen_batch.meta_info}") + + # pad to be divisible by dp_size + test_gen_batch_padded, pad_size = pad_dataproto_to_divisor(test_gen_batch, self.actor_rollout_wg.world_size) + if not self.async_rollout_mode: + test_output_gen_batch_padded = self.actor_rollout_wg.generate_sequences(test_gen_batch_padded) + else: + test_output_gen_batch_padded = self.async_rollout_manager.generate_sequences(test_gen_batch_padded) + + # unpad + test_output_gen_batch = unpad_dataproto(test_output_gen_batch_padded, pad_size=pad_size) + print("validation generation end") + + # Store generated outputs + output_ids = test_output_gen_batch.batch["responses"] + output_texts = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in output_ids] + sample_outputs.extend(output_texts) + + test_batch = test_batch.union(test_output_gen_batch) + + # evaluate using reward_function + result = self.val_reward_fn(test_batch, return_dict=True) + reward_tensor = result["reward_tensor"] + scores = reward_tensor.sum(-1).cpu().tolist() + sample_scores.extend(scores) + + reward_extra_infos_dict["reward"].extend(scores) + if "reward_extra_info" in result: + for key, lst in result["reward_extra_info"].items(): + reward_extra_infos_dict[key].extend(lst) + + data_source_lst.append(test_batch.non_tensor_batch.get("data_source", ["unknown"] * reward_tensor.shape[0])) + + self._maybe_log_val_generations(inputs=sample_inputs, outputs=sample_outputs, scores=sample_scores) + + # dump generations + val_data_dir = self.config.trainer.get("validation_data_dir", None) + if val_data_dir: + sample_gts = [ + item.non_tensor_batch.get("reward_model", {}).get("ground_truth", None) for item in test_batch + ] + self._dump_generations( + inputs=sample_inputs, + outputs=sample_outputs, + gts=sample_gts, + scores=sample_scores, + reward_extra_infos_dict=reward_extra_infos_dict, + dump_path=val_data_dir, + ) + + for key_info, lst in reward_extra_infos_dict.items(): + assert len(lst) == 0 or len(lst) == len(sample_scores), f"{key_info}: {len(lst)=}, {len(sample_scores)=}" + + data_sources = np.concatenate(data_source_lst, axis=0) + print(f"DEBUG: Data sources shape: {data_sources.shape}") # Added Print + print(f"DEBUG: reward_extra_infos_dict keys before processing: {reward_extra_infos_dict.keys()}") # Added Print + + data_src2var2metric2val = process_validation_metrics(data_sources, sample_inputs, reward_extra_infos_dict) + print( + f"DEBUG: Output of process_validation_metrics (data_src2var2metric2val): {data_src2var2metric2val}" + ) # Added Print + metric_dict = {} + for data_source, var2metric2val in data_src2var2metric2val.items(): + core_var = "acc" if "acc" in var2metric2val else "reward" + for var_name, metric2val in var2metric2val.items(): + n_max = max([int(name.split("@")[-1].split("/")[0]) for name in metric2val.keys()]) + for metric_name, metric_val in metric2val.items(): + if ( + (var_name == core_var) + and any(metric_name.startswith(pfx) for pfx in ["mean", "maj", "best"]) + and (f"@{n_max}" in metric_name) + ): + metric_sec = "val-core" + else: + metric_sec = "val-aux" + pfx = f"{metric_sec}/{data_source}/{var_name}/{metric_name}" + metric_dict[pfx] = metric_val + + return metric_dict + + def init_workers(self): + """Init resource pool and worker group""" + self.resource_pool_manager.create_resource_pool() + + self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} + + # create actor and rollout + if self.hybrid_engine: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.ActorRollout) + actor_rollout_cls = RayClassWithInitArgs( + cls=self.role_worker_mapping[Role.ActorRollout], + config=self.config.actor_rollout_ref, + role="actor_rollout", + ) + self.resource_pool_to_cls[resource_pool]["actor_rollout"] = actor_rollout_cls + else: + raise NotImplementedError + + # create critic + if self.use_critic: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.Critic) + critic_cls = RayClassWithInitArgs(cls=self.role_worker_mapping[Role.Critic], config=self.config.critic) + self.resource_pool_to_cls[resource_pool]["critic"] = critic_cls + + # create reference policy if needed + if self.use_reference_policy: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) + ref_policy_cls = RayClassWithInitArgs( + self.role_worker_mapping[Role.RefPolicy], config=self.config.actor_rollout_ref, role="ref" + ) + self.resource_pool_to_cls[resource_pool]["ref"] = ref_policy_cls + + # create a reward model if reward_fn is None + if self.use_rm: + # we create a RM here + resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) + rm_cls = RayClassWithInitArgs(self.role_worker_mapping[Role.RewardModel], config=self.config.reward_model) + self.resource_pool_to_cls[resource_pool]["rm"] = rm_cls + + # initialize WorkerGroup + # NOTE: if you want to use a different resource pool for each role, which can support different + # parallel size, + # you should not use `create_colocated_worker_cls`. Instead, directly pass different resource pool to + # different worker groups. + # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. + all_wg = {} + self.wg_dicts = [] + wg_kwargs = {} # Setting up kwargs for RayWorkerGroup + if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None: + wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout + wg_kwargs["device_name"] = self.device_name + + for resource_pool, class_dict in self.resource_pool_to_cls.items(): + worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + wg_dict = self.ray_worker_group_cls( + resource_pool=resource_pool, + ray_cls_with_init=worker_dict_cls, + **wg_kwargs, + ) + spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + all_wg.update(spawn_wg) + # keep the referece of WorkerDict to support ray >= 2.31. Ref: https://github.com/ray-project/ray/pull/45699 + self.wg_dicts.append(wg_dict) + + if self.use_critic: + self.critic_wg = all_wg["critic"] + self.critic_wg.init_model() + + if self.use_reference_policy: + self.ref_policy_wg = all_wg["ref"] + self.ref_policy_wg.init_model() + + if self.use_rm: + self.rm_wg = all_wg["rm"] + self.rm_wg.init_model() + + # we should create rollout at the end so that vllm can have a better estimation of kv cache memory + self.actor_rollout_wg = all_wg["actor_rollout"] + self.actor_rollout_wg.init_model() + + def _save_checkpoint(self): + # path: given_path + `/global_step_{global_steps}` + `/actor` + local_global_step_folder = os.path.join( + self.config.trainer.default_local_dir, f"global_step_{self.global_steps}" + ) + + print(f"local_global_step_folder: {local_global_step_folder}") + actor_local_path = os.path.join(local_global_step_folder, "actor") + + actor_remote_path = ( + None + if self.config.trainer.default_hdfs_dir is None + else os.path.join(self.config.trainer.default_hdfs_dir, f"global_step_{self.global_steps}", "actor") + ) + + remove_previous_ckpt_in_save = self.config.trainer.get("remove_previous_ckpt_in_save", False) + if remove_previous_ckpt_in_save: + print( + "Warning: remove_previous_ckpt_in_save is deprecated, set max_actor_ckpt_to_keep=1 and " + "max_critic_ckpt_to_keep=1 instead" + ) + max_actor_ckpt_to_keep = ( + self.config.trainer.get("max_actor_ckpt_to_keep", None) if not remove_previous_ckpt_in_save else 1 + ) + max_critic_ckpt_to_keep = ( + self.config.trainer.get("max_critic_ckpt_to_keep", None) if not remove_previous_ckpt_in_save else 1 + ) + + self.actor_rollout_wg.save_checkpoint( + actor_local_path, actor_remote_path, self.global_steps, max_ckpt_to_keep=max_actor_ckpt_to_keep + ) + + if self.use_critic: + critic_local_path = os.path.join(local_global_step_folder, "critic") + critic_remote_path = ( + None + if self.config.trainer.default_hdfs_dir is None + else os.path.join(self.config.trainer.default_hdfs_dir, f"global_step_{self.global_steps}", "critic") + ) + self.critic_wg.save_checkpoint( + critic_local_path, critic_remote_path, self.global_steps, max_ckpt_to_keep=max_critic_ckpt_to_keep + ) + + # save dataloader + dataloader_local_path = os.path.join(local_global_step_folder, "data.pt") + dataloader_state_dict = self.train_dataloader.state_dict() + torch.save(dataloader_state_dict, dataloader_local_path) + + # latest checkpointed iteration tracker (for atomic usage) + local_latest_checkpointed_iteration = os.path.join( + self.config.trainer.default_local_dir, "latest_checkpointed_iteration.txt" + ) + with open(local_latest_checkpointed_iteration, "w") as f: + f.write(str(self.global_steps)) + + def _load_checkpoint(self): + if self.config.trainer.resume_mode == "disable": + return 0 + + # load from hdfs + if self.config.trainer.default_hdfs_dir is not None: + raise NotImplementedError("load from hdfs is not implemented yet") + else: + checkpoint_folder = self.config.trainer.default_local_dir # TODO: check path + if not os.path.isabs(checkpoint_folder): + working_dir = os.getcwd() + checkpoint_folder = os.path.join(working_dir, checkpoint_folder) + global_step_folder = find_latest_ckpt_path(checkpoint_folder) # None if no latest + + # find global_step_folder + if self.config.trainer.resume_mode == "auto": + if global_step_folder is None: + print("Training from scratch") + return 0 + else: + if self.config.trainer.resume_mode == "resume_path": + assert isinstance(self.config.trainer.resume_from_path, str), "resume ckpt must be str type" + assert "global_step_" in self.config.trainer.resume_from_path, ( + "resume ckpt must specify the global_steps" + ) + global_step_folder = self.config.trainer.resume_from_path + if not os.path.isabs(global_step_folder): + working_dir = os.getcwd() + global_step_folder = os.path.join(working_dir, global_step_folder) + print(f"Load from checkpoint folder: {global_step_folder}") + # set global step + self.global_steps = int(global_step_folder.split("global_step_")[-1]) + + print(f"Setting global step to {self.global_steps}") + print(f"Resuming from {global_step_folder}") + + actor_path = os.path.join(global_step_folder, "actor") + critic_path = os.path.join(global_step_folder, "critic") + # load actor + self.actor_rollout_wg.load_checkpoint( + actor_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load + ) + # load critic + if self.use_critic: + self.critic_wg.load_checkpoint( + critic_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load + ) + + # load dataloader, + # TODO: from remote not implemented yet + dataloader_local_path = os.path.join(global_step_folder, "data.pt") + if os.path.exists(dataloader_local_path): + dataloader_state_dict = torch.load(dataloader_local_path, weights_only=False) + self.train_dataloader.load_state_dict(dataloader_state_dict) + else: + print(f"Warning: No dataloader state found at {dataloader_local_path}, will start from scratch") + + def _balance_batch(self, batch: DataProto, metrics, logging_prefix="global_seqlen"): + """Reorder the data on single controller such that each dp rank gets similar total tokens""" + attention_mask = batch.batch["attention_mask"] + batch_size = attention_mask.shape[0] + global_seqlen_lst = batch.batch["attention_mask"].view(batch_size, -1).sum(-1).tolist() # (train_batch_size,) + world_size = self.actor_rollout_wg.world_size + global_partition_lst = get_seqlen_balanced_partitions( + global_seqlen_lst, k_partitions=world_size, equal_size=True + ) + # reorder based on index. The data will be automatically equally partitioned by dispatch function + global_idx = torch.tensor([j for partition in global_partition_lst for j in partition]) + batch.reorder(global_idx) + global_balance_stats = log_seqlen_unbalance( + seqlen_list=global_seqlen_lst, partitions=global_partition_lst, prefix=logging_prefix + ) + metrics.update(global_balance_stats) + + def fit_dpo(self): # Renamed for clarity as standard PPO loop + """ + The training loop of Online DPO using a periodically updated reference model. + The driver process calls worker groups for computation. + Advantage computation is replaced by DPO logic. + """ + import traceback # Ensure traceback is imported + + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + # Initialize logger + logger = None + try: + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True, throw_on_missing=False), + ) + except Exception as e: + print(f"Warning: Failed to initialize logger: {e}") + + self.global_steps = 0 + # Load checkpoint before doing anything + loaded_step = self._load_checkpoint() + self.global_steps = loaded_step + 1 if loaded_step is not None and loaded_step > 0 else 1 + print( + f"Starting Online DPO training from global step {self.global_steps}. " + f"Total steps: {self.total_training_steps}" + ) + print(f"Reference model update frequency: {self.config.trainer.get('ref_update_freq', 'Not Set')}") + + # Check if reference policy is configured correctly for this mode + if not self.use_reference_policy: + print( + "WARNING: 'use_reference_policy' is False. Periodic reference model update requires a " + "reference policy worker. DPO updates might fail or use incorrect logic." + ) + # Consider raising an error if strict adherence is required: + # raise ValueError("Periodic reference model update requires 'use_reference_policy' to be True " + # "and a configured reference worker.") + + # Perform validation before training + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + print("Running validation before Online DPO training...") + val_metrics = self._validate() + pprint(f"Initial validation metrics: {val_metrics}") + if logger and val_metrics: + logger.log(data=val_metrics, step=max(0, self.global_steps - 1)) + if self.config.trainer.get("val_only", False): + print("Validation only mode enabled. Exiting training.") + if logger and hasattr(logger, "finish"): + logger.finish() + return + + # Add tqdm progress bar + progress_bar = tqdm( + total=self.total_training_steps, + initial=self.global_steps, + desc="Online DPO Training Progress", + position=0, + leave=True, + ) + + last_val_metrics = None + should_stop = False + + for epoch in range(self.config.trainer.total_epochs): + if should_stop: + break + print(f"--- Starting Online DPO Epoch {epoch} ---") + try: + train_iterator = iter(self.train_dataloader) + except TypeError: + print("Warning: Dataloader is not iterable.") + train_iterator = self.train_dataloader # Fallback attempt + + for batch_idx, batch_dict in enumerate(train_iterator): + if self.global_steps > self.total_training_steps: + should_stop = True + break + + metrics = {} + timing_raw = {} + step_timer = Timer(logger=None) + ref_log_prob_computed = False # Flag to track if ref log probs were computed + + try: # Outer try-except for the whole step + step_timer.start() + with _timer("step", timing_raw): + batch: DataProto = DataProto.from_single_dict(batch_dict) + current_batch_size = batch.batch.batch_size[0] + print( + f"\n[Step {self.global_steps}, Batch {batch_idx}] Processing batch size: " + f"{current_batch_size}" + ) + + # --- Reference Model Update --- + ref_update_freq = self.config.trainer.get("ref_update_freq", -1) + if ( + self.use_reference_policy + and ref_update_freq > 0 + and self.global_steps % ref_update_freq == 0 + ): + print(f"\n[Step {self.global_steps}] Updating Reference Model Weights from Actor...") + try: + # --- This requires careful implementation with FSDP --- + # 1. Save actor state dict (potentially to CPU memory or disk) + # This needs to be done collectively across actor worker ranks. + # The checkpoint_manager might be adaptable, or use FSDP APIs directly. + # Example placeholder using a conceptual save/load mechanism: + actor_state_path = "/tmp/actor_state_mid" # Temporary path + self.actor_rollout_wg.save_checkpoint(actor_state_path) # Adapt save logic + + # 2. Load the state dict onto the reference model worker group + # This also needs collective loading on the ref worker ranks. + self.ref_policy_wg.load_checkpoint(actor_state_path, None, True) # Adapt load logic + + print(f"[Step {self.global_steps}] Reference Model Weights Updated.") + # Optionally remove the temporary state file + # os.remove(actor_state_path) # Needs rank-aware removal or shared storage + + except Exception as sync_e: + print(f"ERROR during reference model sync at step {self.global_steps}: {sync_e}") + traceback.print_exc() + + # Pop keys for generation + pop_batch_keys = ["input_ids", "attention_mask"] + if "position_ids" in batch.batch: + pop_batch_keys.append("position_ids") + pop_non_tensor_keys = ["raw_prompt_ids"] if "raw_prompt_ids" in batch.non_tensor_batch else [] + if "multi_modal_inputs" in batch.non_tensor_batch.keys(): + pop_non_tensor_keys.extend(["multi_modal_data", "multi_modal_inputs"]) + original_non_tensor_data = batch.non_tensor_batch + gen_batch = batch.pop( + batch_keys=pop_batch_keys, + non_tensor_batch_keys=pop_non_tensor_keys, + ) + gen_batch = gen_batch.repeat( + repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True + ) + # (Add Debug prints for gen_batch if needed) + + # Generate sequences (chosen/rejected pairs) + with _timer("gen", timing_raw): + try: + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + # (Add Debug prints for gen_batch_output if needed) + except Exception as gen_e: + print(f"\n!!!!!!!! ERROR DURING GENERATION (Step {self.global_steps}) !!!!!!!!") + print(gen_e) + traceback.print_exc() + print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + step_timer.stop() + continue + + # Combine original prompts with generated sequences + batch.non_tensor_batch = original_non_tensor_data # Restore non-tensor data + batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(current_batch_size)], dtype=object + ) + batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + batch = batch.union(gen_batch_output) + # (Add Debug prints after union if needed) + + # Compute response mask (needed for ref logprob calc and DPO prep) + batch.batch["response_mask"] = compute_response_mask(batch) + + if self.config.trainer.balance_batch: + self._balance_batch(batch, metrics=metrics) + + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + # --- Compute Log Probs for the CURRENT policy (used for KL if enabled, or ActorAsRef + # fallback) --- + # Note: For pure DPO with external ref, this 'old_log_probs' might not be strictly needed + # unless used for other metrics or a fallback. Keep it for now. + with _timer("policy_log_prob", timing_raw): + policy_log_prob_output = self.actor_rollout_wg.compute_log_prob(batch) + batch = batch.union(policy_log_prob_output) # Adds 'old_log_probs' + # (Debug prints for old_log_probs) + + # --- Compute Log Probs using the EXTERNAL Reference Model --- + if self.use_reference_policy: + with _timer("ref_log_prob_dpo", timing_raw): + # print(f"---- [Step {self.global_steps}] DEBUG DPO: Calling compute_ref_log_prob ----") + try: + # 'batch' contains interleaved chosen/rejected sequences + ref_log_prob_output = self.ref_policy_wg.compute_ref_log_prob( + batch + ) # Returns DataProto with 'ref_log_prob' + batch = batch.union( + ref_log_prob_output + ) # Adds 'ref_log_prob' key [batch_size * n, seq_len] + ref_log_prob_computed = True # Mark success + # print(f"---- [Step {self.global_steps}] DEBUG DPO: ref_log_prob tensor shape: " + # f"{batch.batch['ref_log_prob'].shape} ----") + except Exception as ref_e: + print(f"ERROR computing reference log probs at step {self.global_steps}: {ref_e}") + traceback.print_exc() + batch.batch["ref_log_prob"] = None # Mark as failed + ref_log_prob_computed = False + else: + print( + "Warning: Skipping external reference log prob calculation as use_reference_policy " + "is False." + ) + # DPO update will likely fail unless ActorAsRef logic is re-enabled in dp_actor + + # --- Compute Rewards/Scores (used to determine preference) --- + with _timer("reward_calc", timing_raw): + # (Reward calculation logic using RM or reward_fn as before) + # ... Ensure this calculates 'token_level_rewards' or similar ... + if self.use_rm: + reward_tensor_rm = self.rm_wg.compute_rm_score(batch) + batch = batch.union(reward_tensor_rm) # Adds 'rm_scores' + + reward_extra_infos_dict = {} + try: + if self.reward_fn is None: + # print(f"---- [DEBUG Step {self.global_steps}] ERROR: self.reward_fn is None! " + # f"Using dummy rewards. ----") + # Use rm_scores if available, otherwise zeros + reward_tensor = batch.batch.get( + "rm_scores", torch.zeros_like(batch.batch["response_mask"], dtype=torch.float32) + ) + else: + reward_result = self.reward_fn(batch, return_dict=True) + reward_tensor = reward_result["reward_tensor"] # Final combined reward + reward_extra_infos_dict = reward_result.get("reward_extra_info", {}) + + except Exception: + # print(f'---- [DEBUG Step {self.global_steps}] Error in reward_fn call: {e}. ' + # f'Using dummy rewards. ----') + traceback.print_exc() + reward_tensor = torch.zeros_like(batch.batch["response_mask"], dtype=torch.float32) + reward_extra_infos_dict = {} + + # Use 'token_level_rewards' as the key for preference calculation + batch.batch["token_level_rewards"] = reward_tensor + if reward_extra_infos_dict: + batch.non_tensor_batch.update( + {k: np.array(v) for k, v in reward_extra_infos_dict.items()} + ) + + # --- Determine Preferences --- + # Uses 'token_level_rewards' to determine chosen/rejected based on score + batch = compute_onlineDPO_pref(batch) # Adds 'preferences' key + + # --- Prepare DPO Batch --- + dpo_update_batch_proto = None # Initialize + with _timer("prepare_dpo_batch", timing_raw): + try: + if "preferences" not in batch.batch or batch.batch["preferences"] is None: + raise ValueError("'preferences' key missing or None after compute_onlineDPO_pref.") + + # Check if reference log probs were computed successfully (if needed) + if self.use_reference_policy and not ref_log_prob_computed: + raise ValueError("Reference log probs required but failed to compute.") + + # Check required base keys + required_keys = ["input_ids", "attention_mask", "response_mask"] + for rk in required_keys: + if rk not in batch.batch or batch.batch[rk] is None: + raise KeyError(f"Required key '{rk}' missing from batch for DPO prep.") + + preferences_mask = batch.batch["preferences"] # Shape [batch_size * n] + not_preferences_mask = ~preferences_mask + + # Gather Chosen/Rejected Base Tensors + chosen_input_ids = batch.batch["input_ids"][preferences_mask] + chosen_attention_mask = batch.batch["attention_mask"][preferences_mask] + rejected_input_ids = batch.batch["input_ids"][not_preferences_mask] + rejected_attention_mask = batch.batch["attention_mask"][not_preferences_mask] + chosen_position_ids = ( + batch.batch.get("position_ids")[preferences_mask] + if "position_ids" in batch.batch + else None + ) + rejected_position_ids = ( + batch.batch.get("position_ids")[not_preferences_mask] + if "position_ids" in batch.batch + else None + ) + + # Create Labels + print("WARNING: Creating DPO labels using configured max_prompt_length...") + prompt_len = self.config.data.max_prompt_length + chosen_labels = chosen_input_ids.clone() + chosen_labels[:, :prompt_len] = -100 + rejected_labels = rejected_input_ids.clone() + rejected_labels[:, :prompt_len] = -100 + + # Calculate and Gather Reference Log Probs (Sequence Level) + if self.use_reference_policy: + ref_log_prob_tensor = batch.batch["ref_log_prob"] # Token level [bsz * n, seq_len] + response_mask_full = batch.batch[ + "response_mask" + ] # Response mask [bsz * n, seq_len] + ref_sequence_logps = (ref_log_prob_tensor * response_mask_full).sum( + dim=-1 + ) # Sequence level [bsz * n] + reference_chosen_logps = ref_sequence_logps[preferences_mask] + reference_rejected_logps = ref_sequence_logps[not_preferences_mask] + else: + # If not using external ref, DPO needs ActorAsRef logic in dp_actor + # We won't add the keys here, dp_actor will handle it (or fail if not modified) + print( + "Info: Not adding explicit reference logps to DPO batch " + "(use_reference_policy=False)." + ) + reference_chosen_logps = None # Explicitly None + reference_rejected_logps = None + + # Package Tensors + dpo_tensors = { + "chosen_input_ids": chosen_input_ids, + "chosen_attention_mask": chosen_attention_mask, + "chosen_labels": chosen_labels, + "rejected_input_ids": rejected_input_ids, + "rejected_attention_mask": rejected_attention_mask, + "rejected_labels": rejected_labels, + } + # Conditionally add reference logps if computed + if reference_chosen_logps is not None: + dpo_tensors["reference_chosen_logps"] = reference_chosen_logps + if reference_rejected_logps is not None: + dpo_tensors["reference_rejected_logps"] = reference_rejected_logps + # Add position ids if they exist + if chosen_position_ids is not None: + dpo_tensors["chosen_position_ids"] = chosen_position_ids + if rejected_position_ids is not None: + dpo_tensors["rejected_position_ids"] = rejected_position_ids + + # Prepare Meta Info + dpo_meta = { + "dpo_beta": OmegaConf.select(self.config.algorithm, "dpo_beta", default=0.1), + "dpo_loss_type": OmegaConf.select( + self.config.algorithm, "dpo_loss_type", default="sigmoid" + ), + "dpo_label_smoothing": OmegaConf.select( + self.config.algorithm, "dpo_label_smoothing", default=0.0 + ), + "use_reference_policy": self.use_reference_policy, + "reference_free": not self.use_reference_policy, # False if using external ref + "global_step": self.global_steps, + } + + dpo_update_batch_proto = DataProto.from_dict(tensors=dpo_tensors, meta_info=dpo_meta) + # print(f"---- [Step {self.global_steps}] DEBUG DPO: Prepared DPO Update Batch ----") + # print(f" Keys: {list(dpo_update_batch_proto.batch.keys())}") + # print(f" Meta Info: {dpo_meta}") + + except Exception as e_prep: + print(f"ERROR preparing DPO batch at step {self.global_steps}: {e_prep}") + traceback.print_exc() + dpo_update_batch_proto = None # Skip update on error + + # --- Actor Update Step --- + actor_output = None + if self.config.trainer.critic_warmup <= self.global_steps and dpo_update_batch_proto: + with _timer("update_actor", timing_raw): + # Pass the batch containing reference log probs (if computed) + # The modified update_actor_dpo expects them if reference_free=False + actor_output = self.actor_rollout_wg.update_actor_dpo(dpo_update_batch_proto) + if actor_output and "metrics" in actor_output.meta_info: + metrics.update(reduce_metrics(actor_output.meta_info["metrics"])) + elif dpo_update_batch_proto is None: + print( + f"Skipping actor update at step {self.global_steps} due to DPO batch preparation error." + ) + + # --- Validation and Saving --- + test_freq = OmegaConf.select(self.config.trainer, "test_freq", default=-1) + is_last_step = self.global_steps >= self.total_training_steps + if ( + self.val_reward_fn is not None + and test_freq > 0 + and (is_last_step or self.global_steps % test_freq == 0) + ): + print(f"\nRunning DPO validation at step {self.global_steps}...") + val_timing_raw = {} + with _timer("testing", val_timing_raw): + val_metrics: dict = self._validate() + if is_last_step: + last_val_metrics = val_metrics + if val_metrics: + metrics["time/validation_run"] = val_timing_raw.get("testing", 0) + metrics.update(val_metrics) + else: + print("Validation skipped or returned no metrics.") + + save_freq = OmegaConf.select(self.config.trainer, "save_freq", default=-1) + if save_freq > 0 and (is_last_step or self.global_steps % save_freq == 0): + print(f"\nSaving DPO checkpoint at step {self.global_steps}...") + with _timer("save_checkpoint", timing_raw): + self._save_checkpoint() # Saves actor (and potentially critic if used elsewhere) + metrics["time/save_checkpoint"] = timing_raw.get("save_checkpoint", 0) + + # --- End main step timer context --- + + # --- Metrics calculation AFTER the 'step' timer block --- + metrics.update(compute_dpo_data_metrics(batch=batch)) # Use DPO-specific metrics + metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) + n_gpus = self.resource_pool_manager.get_n_gpus() + if "step" in timing_raw: + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) + else: + print( + f"Warning: 'step' key missing from timing_raw at step {self.global_steps}. " + f"Skipping throughput." + ) + + step_timer.stop() + metrics["time/step"] = step_timer.last + + # Log metrics + log_freq = OmegaConf.select(self.config.trainer, "log_freq", default=1) + if logger and self.global_steps % log_freq == 0: + log_payload = metrics.copy() + # Add learning rate to log payload + if actor_output and "actor/lr" in metrics: + log_payload["actor/lr"] = metrics["actor/lr"] + + print(f"[Step {self.global_steps} DPO] Logging Step Payload Keys: {list(log_payload.keys())}") + try: + logger.log(data=log_payload, step=self.global_steps) + except Exception as e: + print(f"Logging failed at step {self.global_steps}: {e}") + + # Update progress bar + postfix_metrics = { + k: f"{v:.3f}" if isinstance(v, float) else v + for k, v in metrics.items() + if isinstance(v, int | float) + } + progress_bar.set_postfix(postfix_metrics) + + except Exception as step_e: + print(f"\n!!!!!!!! ERROR DURING DPO Step {self.global_steps} !!!!!!!!") + print(f"Caught Exception: {step_e}") + traceback.print_exc() + print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + step_timer.stop() + should_stop = True + break + + if is_last_step or should_stop: + print(f"Stopping DPO training at step {self.global_steps}.") + break + + self.global_steps += 1 + progress_bar.update(1) + + # End of epoch handling + if hasattr(self.train_dataloader, "reset"): + try: + self.train_dataloader.reset() + except Exception as e: + print(f"Warning: Failed to reset train dataloader state: {e}") + if should_stop: + break + + # --- Final cleanup and logging --- + progress_bar.close() + final_step = max(0, self.global_steps - 1) + print(f"Online DPO Training finished at step {final_step}.") + # Save final checkpoint + save_freq = OmegaConf.select(self.config.trainer, "save_freq", default=-1) + if not self.config.trainer.get("val_only", False) and (save_freq <= 0 or final_step % save_freq != 0): + print(f"Saving final DPO checkpoint at step {final_step}...") + self._save_checkpoint() + + # Final validation run + if self.val_reward_fn and last_val_metrics is None and not self.config.trainer.get("val_only", False): + print("Running final validation...") + last_val_metrics = self._validate() + if last_val_metrics and logger: + last_val_metrics["final_validation"] = True + try: + logger.log(data=last_val_metrics, step=final_step) + except Exception as e: + print(f"[Final Val Metrics Log Error]: {e}") + + pprint(f"Final validation metrics: {last_val_metrics}") + if logger and hasattr(logger, "finish"): + logger.finish() + print("Online DPO Training Run Complete.") diff --git a/e2c/verl/recipe/spin/utils.py b/e2c/verl/recipe/spin/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..571ad1e91546c99e5341e2d027c6f8d0ad03c0f9 --- /dev/null +++ b/e2c/verl/recipe/spin/utils.py @@ -0,0 +1,160 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from omegaconf import DictConfig + + +def validate_config( + config: DictConfig, + use_reference_policy: bool, + use_critic: bool, +) -> None: + """ + Validate an OmegaConf DictConfig + + Args: + config: The OmegaConf DictConfig to validate. + use_reference_policy (bool): is ref policy needed + use_critic (bool): is critic needed + """ + # number of GPUs total + n_gpus = config.trainer.n_gpus_per_node * config.trainer.nnodes + + # 1. Check total batch size for data correctness + real_train_batch_size = config.data.train_batch_size * config.actor_rollout_ref.rollout.n + assert real_train_batch_size % n_gpus == 0, ( + f"real_train_batch_size ({real_train_batch_size}) must be divisible by total n_gpus ({n_gpus})." + ) + + # A helper function to check "micro_batch_size" vs "micro_batch_size_per_gpu" + # We throw an error if the user sets both. The new convention is "..._micro_batch_size_per_gpu". + def check_mutually_exclusive(mbs, mbs_per_gpu, name: str): + settings = { + "actor_rollout_ref.actor": "micro_batch_size", + "critic": "micro_batch_size", + "reward_model": "micro_batch_size", + "actor_rollout_ref.ref": "log_prob_micro_batch_size", + "actor_rollout_ref.rollout": "log_prob_micro_batch_size", + } + + if name in settings: + param = settings[name] + param_per_gpu = f"{param}_per_gpu" + + if mbs is None and mbs_per_gpu is None: + raise ValueError(f"[{name}] Please set at least one of '{name}.{param}' or '{name}.{param_per_gpu}'.") + + if mbs is not None and mbs_per_gpu is not None: + raise ValueError( + f"[{name}] You have set both '{name}.{param}' AND '{name}.{param_per_gpu}'. " + f"Please remove '{name}.{param}' because only '*_{param_per_gpu}' is supported " + f"(the former is deprecated)." + ) + + if not config.actor_rollout_ref.actor.use_dynamic_bsz: + # actor: ppo_micro_batch_size vs. ppo_micro_batch_size_per_gpu + check_mutually_exclusive( + config.actor_rollout_ref.actor.ppo_micro_batch_size, + config.actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu, + "actor_rollout_ref.actor", + ) + + if use_reference_policy: + # reference: log_prob_micro_batch_size vs. log_prob_micro_batch_size_per_gpu + check_mutually_exclusive( + config.actor_rollout_ref.ref.log_prob_micro_batch_size, + config.actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu, + "actor_rollout_ref.ref", + ) + + # The rollout section also has log_prob_micro_batch_size vs. log_prob_micro_batch_size_per_gpu + check_mutually_exclusive( + config.actor_rollout_ref.rollout.log_prob_micro_batch_size, + config.actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu, + "actor_rollout_ref.rollout", + ) + + if use_critic and not config.critic.use_dynamic_bsz: + # Check for critic micro-batch size conflicts + check_mutually_exclusive( + config.critic.ppo_micro_batch_size, config.critic.ppo_micro_batch_size_per_gpu, "critic" + ) + + # Check for reward model micro-batch size conflicts + if config.reward_model.enable and not config.reward_model.use_dynamic_bsz: + check_mutually_exclusive( + config.reward_model.micro_batch_size, config.reward_model.micro_batch_size_per_gpu, "reward_model" + ) + + # Actor + # check if train_batch_size is larger than ppo_mini_batch_size + # if NOT dynamic_bsz, we must ensure: + # ppo_mini_batch_size is divisible by ppo_micro_batch_size + # ppo_micro_batch_size * sequence_parallel_size >= n_gpus + if not config.actor_rollout_ref.actor.use_dynamic_bsz: + assert config.data.train_batch_size >= config.actor_rollout_ref.actor.ppo_mini_batch_size + sp_size = config.actor_rollout_ref.actor.get("ulysses_sequence_parallel_size", 1) + if config.actor_rollout_ref.actor.ppo_micro_batch_size is not None: + assert ( + config.actor_rollout_ref.actor.ppo_mini_batch_size % config.actor_rollout_ref.actor.ppo_micro_batch_size + == 0 + ) + assert config.actor_rollout_ref.actor.ppo_micro_batch_size * sp_size >= n_gpus + + assert config.actor_rollout_ref.actor.loss_agg_mode in [ + "token-mean", + "seq-mean-token-sum", + "seq-mean-token-mean", + ], f"Invalid loss_agg_mode: {config.actor_rollout_ref.actor.loss_agg_mode}" + + if config.algorithm.use_kl_in_reward and config.actor_rollout_ref.actor.use_kl_loss: + print("NOTICE: You have both enabled in-reward kl and kl loss.") + + # critic + if use_critic and not config.critic.use_dynamic_bsz: + assert config.data.train_batch_size >= config.critic.ppo_mini_batch_size + sp_size = config.critic.get("ulysses_sequence_parallel_size", 1) + if config.critic.ppo_micro_batch_size is not None: + assert config.critic.ppo_mini_batch_size % config.critic.ppo_micro_batch_size == 0 + assert config.critic.ppo_micro_batch_size * sp_size >= n_gpus + + # Check if use_remove_padding is enabled when using sequence parallelism for fsdp + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + if ( + config.actor_rollout_ref.actor.get("ulysses_sequence_parallel_size", 1) > 1 + or config.actor_rollout_ref.ref.get("ulysses_sequence_parallel_size", 1) > 1 + ): + assert config.actor_rollout_ref.model.use_remove_padding, ( + "When using sequence parallelism for actor/ref policy, you must enable `use_remove_padding`." + ) + + if use_critic and config.critic.strategy in {"fsdp", "fsdp2"}: + if config.critic.get("ulysses_sequence_parallel_size", 1) > 1: + assert config.critic.model.use_remove_padding, ( + "When using sequence parallelism for critic, you must enable `use_remove_padding`." + ) + + if config.data.get("val_batch_size", None) is not None: + print( + "WARNING: val_batch_size is deprecated. Validation datasets are sent to inference engines " + "as a whole batch, which will schedule the memory themselves." + ) + + # check eval config + if config.actor_rollout_ref.rollout.val_kwargs.do_sample: + assert config.actor_rollout_ref.rollout.temperature > 0, ( + "validation gen temperature should be greater than 0 when enabling do_sample" + ) + + print("[validate_config] All configuration checks passed successfully!") diff --git a/e2c/verl/recipe/sppo/README.md b/e2c/verl/recipe/sppo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f87efa853b87857d7fd19de4e4159275619edec3 --- /dev/null +++ b/e2c/verl/recipe/sppo/README.md @@ -0,0 +1,50 @@ +# SPPO: Self-Play Preference Optimization for Language Model Alignment + +This repository hosts the community implementation for the paper [Self-Play Preference Optimization for Language Model Alignment](https://arxiv.org/abs/2405.00675). SPPO can significantly enhance the performance of an LLM without strong external signals such as responses or preferences from GPT-4. It can outperform the model trained with iterative direct preference optimization (DPO), among other methods. SPPO is theoretically grounded, ensuring that the LLM can converge to the von Neumann winner (i.e., Nash equilibrium) under general, potentially intransitive preference, and empirically validated through extensive evaluations on multiple datasets. + +Paper Authors: [Yue Wu](https://yuewu.us/)\*, [Zhiqing Sun](https://www.cs.cmu.edu/~zhiqings/)\*, [Huizhuo Yuan](https://scholar.google.com/citations?user=8foZzX4AAAAJ)\*, [Kaixuan Ji](https://scholar.google.com/citations?user=FOoKDukAAAAJ), [Yiming Yang](https://www.cs.cmu.edu/~yiming/), [Quanquan Gu](https://web.cs.ucla.edu/~qgu/) + +verl Implementation Authors: [Yuhao Yang](https://github.com/yhyang201), [Chenyang Zhao](https://github.com/zhaochenyang20) + +[[Webpage](https://uclaml.github.io/SPPO/)] [[Huggingface](https://huggingface.co/papers/2405.00675)] [[Paper](https://arxiv.org/abs/2405.00675)][[Original Implementation](https://github.com/uclaml/SPPO)] + +## Reproduce the Experiment + +We evaluate the performance of SPPO on the MATH dataset. Starting from an initial score of 46.6 with Qwen2.5-7B-Instruct, we achieve a score of 65.6 after 20 epochs of training, placing our model approximately in the top 20 on the [MATH leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math). It's important to note that verl's internal evaluation metrics may not perfectly align with the official evaluation methodology for Qwen2.5-7B-Instruct. Therefore, for consistency and fair comparison, we report only the results based on verl's evaluation framework. + +``` +git clone git@github.com:volcengine/verl.git +cd verl +python3 -m uv pip install -e ".[sglang]" + +export WANDB_API_KEY= + +python3 examples/data_preprocess/math_dataset.py --local_dir ~/data/math +huggingface-cli download Qwen/Qwen2.5-7B-Instruct --local-dir $HOME/models/Qwen2.5-7B-Instruct + +export CUDA_VISIBLE_DEVICES=0,1,2,3 +bash recipe/sppo/run_qwen2.5-7b_rm.sh +``` + +Note that the installation would occasionally fail to install flash-attn. If this happens, you can install it manually by running: + +```bash +python3 -m uv pip install wheel +python3 -m uv pip install packaging +python3 -m uv pip install flash-attn --no-build-isolation --no-deps +``` + +## Acknowledgement + +We sincerely thank the contribution and guidance from: + +- [Yue Wu](https://yuewu.us/) +- [Chendong Wang](https://cdwang96.github.io/) +- [Yifan Zhang](https://github.com/yifanzhang-pro) +- [Yongan Xiang](https://github.com/BearBiscuit05) +- [Junrong Lin](https://github.com/ocss884) +- [Yuxuan Tong](https://github.com/tongyx361) +- [Guangming Shen](https://github.com/PeterSH6) +- [Biao He](https://www.linkedin.com/in/biao-he/) +- [Qingquan Song](https://qingquansong.github.io/) +- [Quanquan Gu](https://web.cs.ucla.edu/~qgu/) diff --git a/e2c/verl/recipe/sppo/__init__.py b/e2c/verl/recipe/sppo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc88468e3aa17ae3dd07e0492b253c60c0d71d03 --- /dev/null +++ b/e2c/verl/recipe/sppo/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/recipe/sppo/config.py b/e2c/verl/recipe/sppo/config.py new file mode 100644 index 0000000000000000000000000000000000000000..6894e1d7cf234db441a500013e85d5aeb6c3cb6b --- /dev/null +++ b/e2c/verl/recipe/sppo/config.py @@ -0,0 +1,22 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + +from verl.workers.config import FSDPActorConfig + + +@dataclass +class SPPOActorConfig(FSDPActorConfig): + sppo_eta: float = 1.0 diff --git a/e2c/verl/recipe/sppo/config/sppo_trainer.yaml b/e2c/verl/recipe/sppo/config/sppo_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf750eea06cf4358575e69871e2fa22ba3fd8735 --- /dev/null +++ b/e2c/verl/recipe/sppo/config/sppo_trainer.yaml @@ -0,0 +1,38 @@ +# the sppo config will override default ppo_trainer.yaml + +hydra: + searchpath: + - file://verl/trainer/config + +defaults: + - ppo_trainer + - _self_ + +actor_rollout_ref: + actor: + _target_: recipe.sppo.config.SPPOActorConfig + + # sppo_eta is an additional hyperparameter for SPPO, not available in + # verl core. specifying _target_ with SPPOActorConfig is needed to + # extend verl ActorConfig with custom fields. + # additional, it is also possible to use the `extra` field natively supported + # by all verl core dataclasses, without having to define SPPOActorConfig + # extra: + # sppo_eta: 1.0 + sppo_eta: 1.0 + + optim: + lr_warmup_steps: 15 + rollout: + name: sglang + tensor_model_parallel_size: 2 + gpu_memory_utilization: 0.5 + val_kwargs: + n: 2 # 2 will trigger validation, 1 will bypass + +algorithm: + adv_estimator: null + sppo_eta: 1.0 + +trainer: + log_val_generations: 0 \ No newline at end of file diff --git a/e2c/verl/recipe/sppo/dp_actor.py b/e2c/verl/recipe/sppo/dp_actor.py new file mode 100644 index 0000000000000000000000000000000000000000..df14c0b4ed60be3ebc4f0b67aecf242199544b5d --- /dev/null +++ b/e2c/verl/recipe/sppo/dp_actor.py @@ -0,0 +1,187 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import torch + +import verl.utils.torch_functional as verl_F +from verl import DataProto +from verl.trainer.ppo.core_algos import agg_loss, kl_penalty +from verl.utils.device import get_device_id +from verl.utils.profiler import GPUMemoryLogger +from verl.utils.py_functional import append_to_dict +from verl.utils.seqlen_balancing import rearrange_micro_batches +from verl.workers.actor.dp_actor import DataParallelPPOActor + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +def compute_sppo_loss( + old_log_prob: torch.Tensor, # (bs, seq_len) + log_prob: torch.Tensor, # (bs, seq_len) + rewards: torch.Tensor, # (bs,) + response_mask: torch.Tensor, # (bs, seq_len) + eta: float = 1.0, + loss_agg_mode: str = "token-mean", +): + """ + SPPO Loss computation. + """ + # Compute log-ratios over masked tokens + log_prob_sum = (log_prob * response_mask).sum(dim=1) # (bs,) + old_log_prob_sum = (old_log_prob * response_mask).sum(dim=1) # (bs,) + log_ratios = log_prob_sum - old_log_prob_sum # (bs,) + + scaled_rewards = eta * (rewards) + loss_vec = (log_ratios - scaled_rewards) ** 2 # (bs,) + + if loss_agg_mode == "token-mean": + sample_mask = response_mask.any(dim=1).float() # (bs,) + loss = verl_F.masked_mean(loss_vec, sample_mask) + + return loss, log_ratios, scaled_rewards + + +class DataParallelSPPOActor(DataParallelPPOActor): + @GPUMemoryLogger(role="dp actor", logger=logger) + def update_policy(self, data: DataProto): + # make sure we are in training mode + self.actor_module.train() + + temperature = data.meta_info["temperature"] # temperature must be in the data.meta_info to avoid slient error + multi_turn = data.meta_info.get("multi_turn", False) + + select_keys = ["responses", "input_ids", "attention_mask", "position_ids", "old_log_probs", "seq_level_rewards"] + if multi_turn: + select_keys.append("loss_mask") + if self.config.use_kl_loss: + select_keys.append("ref_log_prob") + batch = data.select(batch_keys=select_keys).batch + has_multi_modal_inputs = "multi_modal_inputs" in data.non_tensor_batch.keys() + + # Split to make minibatch iterator for updating the actor + # See PPO paper for details. https://arxiv.org/abs/1707.06347 + if has_multi_modal_inputs: + num_mini_batches = data.batch.batch_size[0] // self.config.ppo_mini_batch_size + non_tensor_select_keys = ["multi_modal_inputs"] + dataloader = data.select(select_keys, non_tensor_select_keys).chunk(num_mini_batches) + else: + dataloader = batch.split(self.config.ppo_mini_batch_size) + + metrics = {} + for epoch in range(self.config.ppo_epochs): + for batch_idx, data in enumerate(dataloader): + # split batch into micro_batches + mini_batch = data + if has_multi_modal_inputs: + self.gradient_accumulation = ( + self.config.ppo_mini_batch_size // self.config.ppo_micro_batch_size_per_gpu + ) + num_micro_batches = mini_batch.batch.batch_size[0] // self.config.ppo_micro_batch_size_per_gpu + micro_batches = data.select(select_keys, non_tensor_select_keys).chunk(num_micro_batches) + elif self.config.use_dynamic_bsz: + max_token_len = self.config.ppo_max_token_len_per_gpu * self.ulysses_sequence_parallel_size + micro_batches, _ = rearrange_micro_batches(batch=mini_batch, max_token_len=max_token_len) + else: + self.gradient_accumulation = ( + self.config.ppo_mini_batch_size // self.config.ppo_micro_batch_size_per_gpu + ) + # split batch into micro_batches + micro_batches = mini_batch.split(self.config.ppo_micro_batch_size_per_gpu) + + self.actor_optimizer.zero_grad() + + for data in micro_batches: + # Support all hardwares + if isinstance(data, DataProto): + data = {**data.batch.to(get_device_id()), **data.non_tensor_batch} + else: + data = data.to(get_device_id()) # actor device is cpu when using offload + responses = data["responses"] + response_length = responses.size(1) + attention_mask = data["attention_mask"] + if multi_turn: + response_mask = data["loss_mask"][:, -response_length:] + else: + response_mask = attention_mask[:, -response_length:] + + old_log_prob = data["old_log_probs"] + rewards = data["seq_level_rewards"] + + entropy_coeff = self.config.entropy_coeff + loss_agg_mode = self.config.loss_agg_mode + eta = self.config.get("sppo_eta", 1.0) + + # all return: (bsz, response_length) + calculate_entropy = False + if entropy_coeff != 0: + calculate_entropy = True + entropy, log_prob = self._forward_micro_batch( + micro_batch=data, temperature=temperature, calculate_entropy=calculate_entropy + ) + + pg_loss, log_ratios, preference = compute_sppo_loss( + old_log_prob=old_log_prob, + log_prob=log_prob, + rewards=rewards, + response_mask=response_mask, + eta=eta, + loss_agg_mode=loss_agg_mode, + ) + + if entropy_coeff != 0: + entropy_loss = agg_loss(loss_mat=entropy, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + + # compute policy loss + policy_loss = pg_loss - entropy_loss * entropy_coeff + else: + policy_loss = pg_loss + + if self.config.use_kl_loss: + ref_log_prob = data["ref_log_prob"] + # compute kl loss + kld = kl_penalty( + logprob=log_prob, ref_logprob=ref_log_prob, kl_penalty=self.config.kl_loss_type + ) + kl_loss = agg_loss( + loss_mat=kld, loss_mask=response_mask, loss_agg_mode=self.config.loss_agg_mode + ) + + policy_loss = policy_loss + kl_loss * self.config.kl_loss_coef + metrics["actor/kl_loss"] = kl_loss.detach().item() + metrics["actor/kl_coef"] = self.config.kl_loss_coef + + if self.config.use_dynamic_bsz: + # relative to the dynamic bsz + loss = policy_loss * (len(data) / self.config.ppo_mini_batch_size) + else: + loss = policy_loss / self.gradient_accumulation + loss.backward() + + data = { + "actor/loss": loss.detach().item(), + "actor/log_ratio_mean": log_ratios.mean().detach().item(), + "actor/preference_mean": preference.mean().detach().item(), + } + append_to_dict(metrics, data) + + grad_norm = self._optimizer_step() + data = {"actor/grad_norm": grad_norm.detach().item()} + append_to_dict(metrics, data) + self.actor_optimizer.zero_grad() + return metrics diff --git a/e2c/verl/recipe/sppo/main_sppo.py b/e2c/verl/recipe/sppo/main_sppo.py new file mode 100644 index 0000000000000000000000000000000000000000..7f5a9e2c9ad63316364eef146299e2ed1c12d419 --- /dev/null +++ b/e2c/verl/recipe/sppo/main_sppo.py @@ -0,0 +1,166 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. +""" + +import os + +import hydra +import ray +from omegaconf import OmegaConf + +from verl.trainer.ppo.reward import load_reward_manager +from verl.trainer.ppo.utils import need_reference_policy +from verl.utils.config import validate_config + +from .sppo_ray_trainer import RaySPPOTrainer + + +@hydra.main(config_path="config", config_name="sppo_trainer", version_base=None) +def main(config): + run_ppo(config) + + +def run_ppo(config) -> None: + # TODO(linjunrong.ocss884): this ENV is left for resolving SGLang conflict with ray devices + # isolation, will solve in the future + os.environ["ENSURE_CUDA_VISIBLE_DEVICES"] = os.environ.get("CUDA_VISIBLE_DEVICES", "") + if not ray.is_initialized(): + # this is for local ray cluster + default_runtime_env = { + "env_vars": {"TOKENIZERS_PARALLELISM": "true", "NCCL_DEBUG": "WARN", "VLLM_LOGGING_LEVEL": "WARN"} + } + ray_init_kwargs = config.ray_kwargs.get("ray_init", {}) + runtime_env_kwargs = ray_init_kwargs.get("runtime_env", {}) + runtime_env = OmegaConf.merge(default_runtime_env, runtime_env_kwargs) + ray_init_kwargs = OmegaConf.create({**ray_init_kwargs, "runtime_env": runtime_env}) + print(f"ray init kwargs: {ray_init_kwargs}") + ray.init(**OmegaConf.to_container(ray_init_kwargs)) + + runner = TaskRunner.remote() + ray.get(runner.run.remote(config)) + + +@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head +class TaskRunner: + def run(self, config): + # print initial config + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_to_local + + pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values + OmegaConf.resolve(config) + + # define worker classes + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + assert config.critic.strategy in {"fsdp", "fsdp2"} + from verl.single_controller.ray import RayWorkerGroup + + from .sppo_worker import SPPOActorRolloutRefWorker # , CriticWorker + + actor_rollout_cls = SPPOActorRolloutRefWorker + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.single_controller.ray import RayWorkerGroup + from verl.workers.megatron_workers import ActorRolloutRefWorker + + actor_rollout_cls = ActorRolloutRefWorker + ray_worker_group_cls = RayWorkerGroup + + else: + raise NotImplementedError + + from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role + + # sppo does not use critic + role_worker_mapping = { + Role.ActorRollout: ray.remote(actor_rollout_cls), + } + + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + } + + # we should adopt a multi-source reward function here + # - for rule-based rm, we directly call a reward score + # - for model-based rm, we call a model + # - for code related prompt, we send to a sandbox if there are test cases + # - finally, we combine all the rewards together + # - The reward type depends on the tag of the data + if config.reward_model.enable: + if config.reward_model.strategy in {"fsdp", "fsdp2"}: + from verl.workers.fsdp_workers import RewardModelWorker + elif config.reward_model.strategy == "megatron": + from verl.workers.megatron_workers import RewardModelWorker + else: + raise NotImplementedError + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + mapping[Role.RewardModel] = global_pool_id + + # use reference model + if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + role_worker_mapping[Role.RefPolicy] = ray.remote(SPPOActorRolloutRefWorker) + mapping[Role.RefPolicy] = global_pool_id + + # validate config + validate_config( + config=config, + use_reference_policy=need_reference_policy(role_worker_mapping), + use_critic=False, + ) + + # download the checkpoint from hdfs + local_path = copy_to_local(config.actor_rollout_ref.model.path) + + # instantiate tokenizer + from verl.utils import hf_processor, hf_tokenizer + + trust_remote_code = config.data.get("trust_remote_code", False) + tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) + processor = hf_processor(local_path, use_fast=True) # used for multimodal LLM, could be none + + reward_fn = load_reward_manager( + config, tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {}) + ) + val_reward_fn = load_reward_manager(config, tokenizer, num_examine=1) + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + + trainer = RaySPPOTrainer( + config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + ) + trainer.init_workers() + trainer.fit() + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/recipe/sppo/run_qwen2.5-7b_rm.sh b/e2c/verl/recipe/sppo/run_qwen2.5-7b_rm.sh new file mode 100644 index 0000000000000000000000000000000000000000..cc614d02511f5d3c97a90d97f7dc5d8420ff1bdc --- /dev/null +++ b/e2c/verl/recipe/sppo/run_qwen2.5-7b_rm.sh @@ -0,0 +1,56 @@ +# Discliamer: the model used in the script is only for academic purpose. +set -x + +# Data preparation scripts are available in ``examples/data_preprocess``. +# Example usage: +# +# python3 examples/data_preprocess/math_dataset.py --local_dir ~/data/math +# python3 examples/data_preprocess/gsm8k.py --local_save_dir ~/data/gsm8k + +gsm8k_train_path=$HOME/data/math/train.parquet +gsm8k_test_path=$HOME/data/math/test.parquet + +train_files="['$gsm8k_train_path']" +test_files="['$gsm8k_test_path']" + +# prepare model ckpt +huggingface-cli download Qwen/Qwen2.5-7B-Instruct --local-dir $HOME/models/Qwen2.5-7B-Instruct & +# huggingface-cli download sfairXC/FsfairX-LLaMA3-RM-v0.1 --local-dir $HOME/models/FsfairX-LLaMA3-RM-v0.1 & +wait + +python3 -m recipe.sppo.main_sppo \ + data.train_files="$train_files" \ + data.val_files="$test_files" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path="$HOME/models/Qwen2.5-7B-Instruct" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='sppo-sglang' \ + trainer.val_before_train=True \ + trainer.experiment_name='Qwen2-7B-Instruct_hybrid_rm' \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=1 \ + trainer.total_epochs=1000 $@ + # Note that we set lr_warmup_steps = 15 in config/sppo_trainer.yaml + # The experiment will converge to 0.656 on MATH dataset after 20 epochs \ No newline at end of file diff --git a/e2c/verl/recipe/sppo/sppo_ray_trainer.py b/e2c/verl/recipe/sppo/sppo_ray_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..e075beec8c3e5d1a3d34e9aabc2993797778bf26 --- /dev/null +++ b/e2c/verl/recipe/sppo/sppo_ray_trainer.py @@ -0,0 +1,348 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +FSDP PPO Trainer with Ray-based single controller. +This trainer supports model-agonistic model initialization with huggingface +""" + +import uuid +from copy import deepcopy +from pprint import pprint +from typing import Optional + +import numpy as np +import ray +import torch +from torch.utils.data import Dataset, Sampler +from tqdm import tqdm + +from verl import DataProto +from verl.single_controller.ray import RayWorkerGroup +from verl.trainer.ppo import core_algos +from verl.trainer.ppo.core_algos import agg_loss +from verl.trainer.ppo.metric_utils import reduce_metrics +from verl.trainer.ppo.ray_trainer import ( + AdvantageEstimator, + RayPPOTrainer, + ResourcePoolManager, + apply_kl_penalty, + compute_response_mask, +) +from verl.trainer.ppo.reward import compute_reward, compute_reward_async +from verl.trainer.ppo.utils import Role, WorkerType, need_reference_policy, need_reward_model +from verl.utils.profiler.performance import simple_timer +from verl.utils.tracking import ValidationGenerationsLogger + + +def softmean(x: torch.Tensor, beta: float, dim: int = -1, keepdim: bool = False) -> torch.Tensor: + """ + Compute SoftMean_β(x) = (1/β) * log( (1/n) * Σ exp(β * x_i) ) + Falls back to arithmetic mean when β=0. + """ + if beta == 0.0: + return x.mean(dim=dim, keepdim=keepdim) + + # cast beta to tensor on same device/dtype + beta_t = x.new_tensor(beta) + # numerically-stable logsumexp(β x) + lse = torch.logsumexp(x * beta_t, dim=dim, keepdim=keepdim) + n = x.size(dim) + log_n = x.new_tensor(n).log() + + return (lse - log_n) / beta_t + + +def compute_advantage(data: DataProto, beta=1.0): + rewards = data.batch["token_level_rewards"].sum(axis=-1) # (bs, ) + s_mean = softmean(rewards, beta, keepdim=True) # (bs, ) + rewards = rewards - s_mean # (bs, ) + data.batch["seq_level_rewards"] = rewards # (bs, ) + return data + + +class RaySPPOTrainer(RayPPOTrainer): + """ + Note that this trainer runs on the driver process on a single CPU/GPU node. + """ + + # TODO: support each role have individual ray_worker_group_cls, + # i.e., support different backend of different role + def __init__( + self, + config, + tokenizer, + role_worker_mapping: dict[Role, WorkerType], + resource_pool_manager: ResourcePoolManager, + ray_worker_group_cls: RayWorkerGroup = RayWorkerGroup, + processor=None, + reward_fn=None, + val_reward_fn=None, + train_dataset: Optional[Dataset] = None, + val_dataset: Optional[Dataset] = None, + collate_fn=None, + train_sampler: Optional[Sampler] = None, + device_name=None, + ): + self.tokenizer = tokenizer + self.processor = processor + self.config = config + self.reward_fn = reward_fn + self.val_reward_fn = val_reward_fn + + self.hybrid_engine = config.actor_rollout_ref.hybrid_engine + assert self.hybrid_engine, "Currently, only support hybrid engine" + + if self.hybrid_engine: + assert Role.ActorRollout in role_worker_mapping, f"{role_worker_mapping.keys()=}" + + self.role_worker_mapping = role_worker_mapping + self.resource_pool_manager = resource_pool_manager + self.use_reference_policy = need_reference_policy(role_worker_mapping) + self.use_rm = need_reward_model(role_worker_mapping) + self.use_critic = False + self.ray_worker_group_cls = ray_worker_group_cls + self.validation_generations_logger = ValidationGenerationsLogger() + self.device_name = device_name if device_name else self.config.trainer.device + + # define in-reward KL control + # kl loss control currently not supported + if config.algorithm.use_kl_in_reward: + self.kl_ctrl_in_reward = core_algos.get_kl_controller(config.algorithm.kl_ctrl) + + self._create_dataloader(train_dataset, val_dataset, collate_fn, train_sampler) + + def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the + worker group through RPC to construct the PPO dataflow. + The light-weight advantage computation is done on the driver process. + """ + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + self.global_steps = 0 + + # load checkpoint before doing anything + self._load_checkpoint() + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + val_metrics = self._validate() + pprint(f"Initial validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if self.config.trainer.get("val_only", False): + return + + # add tqdm + progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress") + + # we start from step 1 + self.global_steps += 1 + last_val_metrics = None + + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + metrics = {} + timing_raw = {} + batch: DataProto = DataProto.from_single_dict(batch_dict) + + # pop those keys for generation + batch_keys_to_pop = ["input_ids", "attention_mask", "position_ids"] + non_tensor_batch_keys_to_pop = ["raw_prompt_ids"] + if "multi_modal_data" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("multi_modal_data") + if "raw_prompt" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("raw_prompt") + if "tools_kwargs" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("tools_kwargs") + gen_batch = batch.pop( + batch_keys=batch_keys_to_pop, + non_tensor_batch_keys=non_tensor_batch_keys_to_pop, + ) + gen_batch = gen_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + + is_last_step = self.global_steps >= self.total_training_steps + + with simple_timer("step", timing_raw): + # generate a batch + with simple_timer("gen", timing_raw): + if not self.async_rollout_mode: + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + else: + gen_batch_output = self.async_rollout_manager.generate_sequences(gen_batch) + timing_raw.update(gen_batch_output.meta_info["timing"]) + gen_batch_output.meta_info.pop("timing", None) + + if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: + with simple_timer("gen_max", timing_raw): + gen_baseline_batch = deepcopy(gen_batch) + gen_baseline_batch.meta_info["do_sample"] = False + gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) + + batch = batch.union(gen_baseline_output) + reward_baseline_tensor = self.reward_fn(batch) + reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) + + batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) + + batch.batch["reward_baselines"] = reward_baseline_tensor + + del gen_baseline_batch, gen_baseline_output + + batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(batch.batch))], dtype=object + ) + # repeat to align with repeated responses in rollout + batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + batch = batch.union(gen_batch_output) + + batch.batch["response_mask"] = compute_response_mask(batch) + # Balance the number of valid tokens across DP ranks. + # NOTE: This usually changes the order of data in the `batch`, + # which won't affect the advantage calculation (since it's based on uid), + # but might affect the loss calculation (due to the change of mini-batching). + # TODO: Decouple the DP balancing and mini-batching. + if self.config.trainer.balance_batch: + self._balance_batch(batch, metrics=metrics) + + # compute global_valid tokens + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + with simple_timer("reward", timing_raw): + # compute reward model score + if self.use_rm: + reward_tensor = self.rm_wg.compute_rm_score(batch) + batch = batch.union(reward_tensor) + + if self.config.reward_model.launch_reward_fn_async: + future_reward = compute_reward_async.remote(batch, self.config, self.tokenizer) + else: + reward_tensor, reward_extra_infos_dict = compute_reward(batch, self.reward_fn) + + # recompute old_log_probs + with simple_timer("old_log_prob", timing_raw): + old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) + entropys = old_log_prob.batch["entropys"] + response_masks = batch.batch["response_mask"] + loss_agg_mode = self.config.actor_rollout_ref.actor.loss_agg_mode + entropy_agg = agg_loss(loss_mat=entropys, loss_mask=response_masks, loss_agg_mode=loss_agg_mode) + old_log_prob_metrics = {"actor/entropy": entropy_agg.detach().item()} + metrics.update(old_log_prob_metrics) + old_log_prob.batch.pop("entropys") + batch = batch.union(old_log_prob) + + if self.use_reference_policy: + # compute reference log_prob + with simple_timer("ref", timing_raw): + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + + # compute values + if self.use_critic: + with simple_timer("values", timing_raw): + values = self.critic_wg.compute_values(batch) + batch = batch.union(values) + + with simple_timer("adv", timing_raw): + # we combine with rule-based rm + reward_extra_infos_dict: dict[str, list] + if self.config.reward_model.launch_reward_fn_async: + reward_tensor, reward_extra_infos_dict = ray.get(future_reward) + batch.batch["token_level_scores"] = reward_tensor + + if reward_extra_infos_dict: + batch.non_tensor_batch.update({k: np.array(v) for k, v in reward_extra_infos_dict.items()}) + + # compute rewards. apply_kl_penalty if available + if self.config.algorithm.use_kl_in_reward: + batch, kl_metrics = apply_kl_penalty( + batch, kl_ctrl=self.kl_ctrl_in_reward, kl_penalty=self.config.algorithm.kl_penalty + ) + metrics.update(kl_metrics) + else: + batch.batch["token_level_rewards"] = batch.batch["token_level_scores"] + batch.batch["seq_level_rewards"] = batch.batch["token_level_scores"] + + beta = self.config.algorithm.sppo_eta + batch = compute_advantage(batch, beta=beta) + + # update critic + if self.use_critic: + with simple_timer("update_critic", timing_raw): + critic_output = self.critic_wg.update_critic(batch) + critic_output_metrics = reduce_metrics(critic_output.meta_info["metrics"]) + metrics.update(critic_output_metrics) + + # implement critic warmup + if self.config.trainer.critic_warmup <= self.global_steps: + # update actor + with simple_timer("update_actor", timing_raw): + batch.meta_info["multi_turn"] = self.config.actor_rollout_ref.rollout.multi_turn.enable + actor_output = self.actor_rollout_wg.update_actor(batch) + actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"]) + metrics.update(actor_output_metrics) + + # Log rollout generations if enabled + rollout_data_dir = self.config.trainer.get("rollout_data_dir", None) + if rollout_data_dir: + self._log_rollout_data(batch, reward_extra_infos_dict, timing_raw, rollout_data_dir) + + # validate + if ( + self.val_reward_fn is not None + and self.config.trainer.test_freq > 0 + and (is_last_step or self.global_steps % self.config.trainer.test_freq == 0) + ): + with simple_timer("testing", timing_raw): + val_metrics: dict = self._validate() + if is_last_step: + last_val_metrics = val_metrics + metrics.update(val_metrics) + + if self.config.trainer.save_freq > 0 and ( + is_last_step or self.global_steps % self.config.trainer.save_freq == 0 + ): + with simple_timer("save_checkpoint", timing_raw): + self._save_checkpoint() + + # training metrics + metrics.update( + { + "training/global_step": self.global_steps, + "training/epoch": epoch, + } + ) + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=self.global_steps) + + if is_last_step: + pprint(f"Final validation metrics: {last_val_metrics}") + progress_bar.close() + return + + progress_bar.update(1) + self.global_steps += 1 diff --git a/e2c/verl/recipe/sppo/sppo_worker.py b/e2c/verl/recipe/sppo/sppo_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..3353159b1820055e17b3605820fea6006054fc96 --- /dev/null +++ b/e2c/verl/recipe/sppo/sppo_worker.py @@ -0,0 +1,122 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from omegaconf import OmegaConf, open_dict + +from verl.single_controller.base.decorator import Dispatch, register +from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager +from verl.utils.flops_counter import FlopsCounter +from verl.utils.fsdp_utils import offload_fsdp_model_to_cpu, offload_fsdp_optimizer +from verl.utils.import_utils import import_external_libs +from verl.utils.profiler import log_gpu_memory_usage +from verl.workers.fsdp_workers import ActorRolloutRefWorker + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_PPO_LOGGING_LEVEL", "WARN")) + + +class SPPOActorRolloutRefWorker(ActorRolloutRefWorker): + """ + This worker can be instantiated as a standalone actor or a standalone rollout or a standalone reference policy + or a hybrid engine based on the config.rollout + """ + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + from .dp_actor import DataParallelSPPOActor + + # This is used to import external_lib into the huggingface systems + import_external_libs(self.config.model.get("external_lib", None)) + + override_model_config = OmegaConf.to_container(OmegaConf.create(self.config.model.get("override_config", {}))) + use_remove_padding = self.config.model.get("use_remove_padding", False) + use_fused_kernels = self.config.model.get("use_fused_kernels", False) + + if self._is_actor or self._is_rollout: + # we need the model for actor and rollout + if self._is_actor: + optim_config = self.config.actor.optim + fsdp_config = self.config.actor.fsdp_config + else: + optim_config = None + fsdp_config = OmegaConf.create() + self.actor_module_fsdp, self.actor_optimizer, self.actor_lr_scheduler, self.actor_model_config = ( + self._build_model_optimizer( + model_path=self.config.model.path, + fsdp_config=fsdp_config, + optim_config=optim_config, + override_model_config=override_model_config, + use_remove_padding=use_remove_padding, + use_fused_kernels=use_fused_kernels, + enable_gradient_checkpointing=self.config.model.get("enable_gradient_checkpointing", False), + trust_remote_code=self.config.model.get("trust_remote_code", False), + use_liger=self.config.model.get("use_liger", False), + role="actor", + ) + ) + + # get the original unwrapped module + self.actor_module = self.actor_module_fsdp._fsdp_wrapped_module + + if self._is_offload_param: + offload_fsdp_model_to_cpu(self.actor_module_fsdp) + log_gpu_memory_usage("After offload actor model during init", logger=logger) + + if self._is_offload_optimizer: + offload_fsdp_optimizer(optimizer=self.actor_optimizer) + log_gpu_memory_usage("After offload actor optimizer during init", logger=logger) + # load from checkpoint + if self._is_actor: + OmegaConf.set_struct(self.config.actor, True) + with open_dict(self.config.actor): + self.config.actor.use_remove_padding = use_remove_padding + self.config.actor.use_fused_kernels = use_fused_kernels + self.actor = DataParallelSPPOActor( + config=self.config.actor, actor_module=self.actor_module_fsdp, actor_optimizer=self.actor_optimizer + ) + + if self._is_rollout: + self._build_rollout(trust_remote_code=self.config.model.get("trust_remote_code", False)) + + if self._is_ref: + self.ref_module_fsdp = self._build_model_optimizer( + model_path=self.config.model.path, + fsdp_config=self.config.ref.fsdp_config, + optim_config=None, + override_model_config=override_model_config, + use_remove_padding=use_remove_padding, + use_fused_kernels=use_fused_kernels, + trust_remote_code=self.config.model.get("trust_remote_code", False), + use_liger=self.config.model.get("use_liger", False), + role="ref", + )[0] + OmegaConf.set_struct(self.config.ref, True) + with open_dict(self.config.ref): + self.config.ref.use_remove_padding = use_remove_padding + self.config.ref.use_fused_kernels = use_fused_kernels + self.ref_policy = DataParallelSPPOActor(config=self.config.ref, actor_module=self.ref_module_fsdp) + + if self._is_actor: + self.flops_counter = FlopsCounter(self.actor_model_config) + self.checkpoint_manager = FSDPCheckpointManager( + model=self.actor_module_fsdp, + optimizer=self.actor.actor_optimizer, + lr_scheduler=self.actor_lr_scheduler, + processing_class=self.processor if self.processor is not None else self.tokenizer, + checkpoint_config=self.config.actor.checkpoint, + ) diff --git a/e2c/verl/requirements-cuda.txt b/e2c/verl/requirements-cuda.txt new file mode 100644 index 0000000000000000000000000000000000000000..7bfe8efeb555b9f509c7584045461076993424ce --- /dev/null +++ b/e2c/verl/requirements-cuda.txt @@ -0,0 +1 @@ +flash-attn \ No newline at end of file diff --git a/e2c/verl/requirements-npu.txt b/e2c/verl/requirements-npu.txt new file mode 100644 index 0000000000000000000000000000000000000000..ae9ed11615696ffe3302a0b958ccab40d02e1de2 --- /dev/null +++ b/e2c/verl/requirements-npu.txt @@ -0,0 +1,21 @@ +# requirements.txt records the full set of dependencies for development +accelerate +codetiming +datasets +dill +hydra-core +numpy<2.0.0 +pandas +peft>=0.15.2 +pyarrow>=15.0.0 +pybind11 +pylatexenc +tensordict>=0.8.0,<=0.10.0,!=0.9.0 +transformers==4.52.4 +ray==2.46.0 +wandb +mathruler +torchdata +einops +qwen_vl_utils +torchvision==0.20.1 diff --git a/e2c/verl/requirements.txt b/e2c/verl/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d4f236ea5d654408a5ab94c25e9b9a6c7621d3f --- /dev/null +++ b/e2c/verl/requirements.txt @@ -0,0 +1,26 @@ +# requirements.txt records the full set of dependencies for development +accelerate +codetiming +datasets +dill +hydra-core +liger-kernel +numpy<2.0.0 +pandas +peft +pyarrow>=19.0.0 +pybind11 +pylatexenc +pre-commit +ray[default] +tensordict>=0.8.0,<=0.10.0,!=0.9.0 +torchdata +transformers +# vllm==0.8.4 +wandb +packaging>=20.0 +uvicorn +fastapi +latex2sympy2_extended +math_verify +tensorboard \ No newline at end of file diff --git a/e2c/verl/requirements_sglang.txt b/e2c/verl/requirements_sglang.txt new file mode 100644 index 0000000000000000000000000000000000000000..113bca0d3e7ee9b976a854b48d271b3d27a88e81 --- /dev/null +++ b/e2c/verl/requirements_sglang.txt @@ -0,0 +1,21 @@ +# requirements.txt records the full set of dependencies for development +accelerate +codetiming +datasets +dill +flash-attn +hydra-core +numpy<2.0.0 +pandas +peft +pyarrow>=19.0.0 +pybind11 +pylatexenc +ray[default]>=2.10 +tensordict>=0.8.0,<=0.10.0,!=0.9.0 +torchdata +torchvision +transformers +wandb +sglang[all]==0.5.2 +huggingface_hub diff --git a/e2c/verl/scripts/__init__.py b/e2c/verl/scripts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/scripts/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/scripts/converter_hf_to_mcore.py b/e2c/verl/scripts/converter_hf_to_mcore.py new file mode 100644 index 0000000000000000000000000000000000000000..b14eb932584b7dd175771b3458d3e74144046836 --- /dev/null +++ b/e2c/verl/scripts/converter_hf_to_mcore.py @@ -0,0 +1,562 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import warnings +from contextlib import contextmanager +from importlib.metadata import version +from typing import Any, Callable, ContextManager, Optional + +import numpy as np +import torch +import torch.distributed as dist + +try: + # NPU patch + import mindspeed.megatron_adaptor # noqa: F401 +except ImportError: + pass + +from accelerate import init_empty_weights +from megatron.core import dist_checkpointing +from megatron.core import parallel_state as mpu +from megatron.core.dist_checkpointing.mapping import ShardedTensor +from megatron.core.dist_checkpointing.serialization import StrictHandling +from megatron.core.models.gpt.gpt_model import ModelType +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from packaging.version import Version +from transformers import AutoConfig + +from verl.model_merger.megatron_model_merger import get_dynamic_pipeline_shards +from verl.models.mcore import hf_to_mcore_config +from verl.utils.device import get_device_name, get_torch_device +from verl.utils.megatron_utils import get_model + + +def _init_args(): + """ + Examples: + + 1. single rank conversion for any model: + > python converter_hf_to_mcore.py --hf_model_path %{hf_model} --output_path ${output_path} + 2. distributed conversion for DeepseekV3 671B: + > torchrun --nproc_per_node 1 --nnodes 4 --node_rank ${RANK} converter_hf_to_mcore.py \ + --hf_model_path %{hf_model} --output_path ${output_path} + """ + parser = argparse.ArgumentParser() + parser.add_argument("--hf_model_path", type=str, required=True, help="The path for the huggingface model") + parser.add_argument("--output_path", type=str, required=True, help="The path for the output mcore model") + parser.add_argument("--use_cpu_initialization", action="store_true", help="Whether to use cpu initialization") + parser.add_argument("--test", action="store_true", help="Whether to test the conversion") + parser.add_argument("--trust_remote_code", action="store_true", help="Whether to trust remote code") + args = parser.parse_args() + return args + + +def test_conversion(megatron_model_provider, tfconfig, output_path, model): + ########### test ########### + # load model + model_test = get_model( + model_provider_func=megatron_model_provider, + model_type=ModelType.encoder_or_decoder, + wrap_with_ddp=True, + transformer_config=tfconfig, + ) + ref_state_dict = model_test[0].module.sharded_state_dict() + dist_checkpointing.load(ref_state_dict, output_path, strict=StrictHandling.ASSUME_OK_UNEXPECTED) + + dut_state_dict = model[0].module.state_dict() + for name in dut_state_dict.keys(): + if dut_state_dict[name] is None: + print(f"[Warning] {name} is none in dut_state_dict") + continue + dut_data = dut_state_dict[name].data + if name in ref_state_dict: + ref_data = ref_state_dict[name] + if isinstance(ref_data, ShardedTensor): + ref_data = ref_data.data.view(ref_data.local_shape) + else: + ref_data = ref_data.data + assert dut_data.shape == ref_data.shape, f"{name=} {dut_data.shape=} {ref_data.shape=}" + assert (dut_data == ref_data).all(), f"{name} is not equal" + print(f"{name} is equal") + else: + print(f"[Warning] {name} is not in ref_state_dict") + for name in ref_state_dict.keys(): + if ref_state_dict[name] is None: + print(f"[Warning] {name} is none in ref_state_dict") + continue + ref_data = ref_state_dict[name] + if isinstance(ref_data, ShardedTensor): + ref_data = ref_data.data.view(ref_data.local_shape) + else: + ref_data = ref_data.data + if name in dut_state_dict: + dut_data = dut_state_dict[name].data + assert dut_data.shape == ref_data.shape, f"{name=} {dut_data.shape=} {ref_data.shape=}" + assert (dut_data == ref_data).all(), f"{name} is not equal" + print(f"{name} is equal") + else: + print(f"[Warning] {name} is not in dut_state_dict") + print("Conversion test passed!") + + +@torch.inference_mode() +def convert_checkpoint_from_transformers_to_megatron( + hf_model, model, hf_config, layer_start_end: Optional[tuple[int, int]] = None +): + if layer_start_end is None: + layer_start_end = (0, len(model.decoder.layers)) + layer_start, layer_end = layer_start_end + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + numel = 0 + + num_attention_heads = hf_config.num_attention_heads + num_key_value_heads = hf_config.num_key_value_heads + hidden_dim = hf_config.hidden_size + head_dim = getattr(hf_config, "head_dim", hidden_dim // num_attention_heads) + if num_attention_heads != num_key_value_heads: + print("[WARNING] Converting GQA model") + has_qkv_bias = getattr(hf_config, "qkv_bias", False) or getattr(hf_config, "attention_bias", False) + has_share_expert = getattr(hf_config, "shared_expert_intermediate_size", None) + if pp_rank == 0: + numel += safe_copy(hf_model.model.embed_tokens.weight, model.embedding.word_embeddings.weight) + + assert len(model.decoder.layers) == (layer_end - layer_start), ( + f"Expected {len(model.decoder.layers)} layers, but got {layer_end - layer_start}" + ) + for layer_idx, (layer, hf_layer) in enumerate( + zip(model.decoder.layers, hf_model.model.layers[layer_start:layer_end], strict=True) + ): + global_layer_idx = layer_idx + layer_start + numel_cur = numel + numel += safe_copy(hf_layer.input_layernorm.weight, layer.self_attention.linear_qkv.layer_norm_weight) + + q = hf_layer.self_attn.q_proj.weight.view( + [num_key_value_heads, head_dim * num_attention_heads // num_key_value_heads, -1] + ) + k = hf_layer.self_attn.k_proj.weight.view([num_key_value_heads, head_dim, -1]) + v = hf_layer.self_attn.v_proj.weight.view([num_key_value_heads, head_dim, -1]) + qkv = torch.cat([q, k, v], dim=1).view(-1, hidden_dim).contiguous() + numel += safe_copy(qkv, layer.self_attention.linear_qkv.weight) + + if has_qkv_bias: + q_bias = hf_layer.self_attn.q_proj.bias.view([num_key_value_heads, -1]) + k_bias = hf_layer.self_attn.k_proj.bias.view([num_key_value_heads, -1]) + v_bias = hf_layer.self_attn.v_proj.bias.view([num_key_value_heads, -1]) + qkv_bias = torch.cat([q_bias, k_bias, v_bias], dim=1).view(-1).contiguous() + numel += safe_copy(qkv_bias, layer.self_attention.linear_qkv.bias) + + if hasattr(hf_layer.self_attn, "q_norm"): + numel += safe_copy(hf_layer.self_attn.q_norm.weight.data, layer.self_attention.q_layernorm.weight) + numel += safe_copy(hf_layer.self_attn.k_norm.weight.data, layer.self_attention.k_layernorm.weight) + + numel += safe_copy(hf_layer.self_attn.o_proj.weight, layer.self_attention.linear_proj.weight) + numel += safe_copy(hf_layer.post_attention_layernorm.weight, layer.pre_mlp_layernorm.weight) + + numel += safe_copy(hf_layer.mlp.gate.weight, layer.mlp.router.weight) + + for idx, hf_expert in enumerate(hf_layer.mlp.experts): + fc1_weight = torch.cat([hf_expert.gate_proj.weight, hf_expert.up_proj.weight]) + numel += safe_copy(fc1_weight, layer.mlp.experts.linear_fc1._parameters[f"weight{idx}"]) + numel += safe_copy(hf_expert.down_proj.weight, layer.mlp.experts.linear_fc2._parameters[f"weight{idx}"]) + + if has_share_expert: + numel += safe_copy(hf_layer.mlp.shared_expert_gate.weight, layer.mlp.shared_experts.gate_weight) + shared_fc1_weight = torch.cat( + [hf_layer.mlp.shared_expert.gate_proj.weight, hf_layer.mlp.shared_expert.up_proj.weight] + ) + numel += safe_copy(shared_fc1_weight, layer.mlp.shared_experts.linear_fc1.weight) + numel += safe_copy(hf_layer.mlp.shared_expert.down_proj.weight, layer.mlp.shared_experts.linear_fc2.weight) + print(f"{pp_rank=} {global_layer_idx=} {layer_idx=} {numel=} numel this layer={numel - numel_cur}") + + if pp_rank == pp_size - 1: + numel += safe_copy(hf_model.model.norm.weight, model.decoder.final_layernorm.weight) + numel += safe_copy(hf_model.lm_head.weight, model.output_layer.weight) + return numel + + +def safe_copy( + src_tensor: torch.Tensor, + dst_tensor: torch.Tensor, + skip_dtype_assert: bool = False, +): + if not skip_dtype_assert: + if src_tensor.dtype != dst_tensor.dtype: + raise ValueError(f"Get source dtype {src_tensor.dtype}, but target dtype {dst_tensor.dtype}") + assert src_tensor.shape == dst_tensor.shape + dst_tensor.data.copy_(src_tensor.data) + return src_tensor.numel() + + +@torch.inference_mode() +def convert_checkpoint_from_transformers_to_megatron_qwen2_5_vl(hfmodel, mgmodel, hf_config): + mgmodel = mgmodel.bfloat16() + hfmodel = hfmodel.bfloat16() + num_attention_heads = hf_config.num_attention_heads + num_query_groups = hf_config.num_key_value_heads + hidden_size = hf_config.hidden_size + head_dim = hidden_size // num_attention_heads + + # 1. vision model + if Version(version("transformers")) < Version("4.52.0"): + print("Using transformers < 4.52 API to load vision model") + hfvision = hfmodel.visual + else: + hfvision = hfmodel.model.visual + mgvision = mgmodel.vision_model + vision_hidden_size = mgvision.config.hidden_size + vision_num_query_groups = mgvision.config.num_query_groups + vision_head_dim = vision_hidden_size // mgvision.config.num_attention_heads + copied_numel = 0 + safe_copy(hfvision.rotary_pos_emb.inv_freq, mgvision.rotary_pos_emb.inv_freq) + copied_numel += safe_copy(hfvision.patch_embed.proj.weight, mgvision.patch_embed.proj.weight) + for hfblock, mgblock in zip(hfvision.blocks, mgvision.decoder.layers, strict=True): + # norm1 --> linear_qkv.norm + copied_numel += safe_copy(hfblock.norm1.weight, mgblock.self_attention.linear_qkv.layer_norm_weight) + # norm2 --> mlp.linear_fc1.norm + copied_numel += safe_copy(hfblock.norm2.weight, mgblock.mlp.linear_fc1.layer_norm_weight) + # qkv --> self_attention.linear_qkv + converted_weight = ( + hfblock.attn.qkv.weight.view(3, vision_num_query_groups, -1, vision_head_dim, vision_hidden_size) + .transpose(0, 1) + .flatten(1, 2) + .reshape(-1, vision_hidden_size) + .contiguous() + ) + copied_numel += safe_copy(converted_weight, mgblock.self_attention.linear_qkv.weight) + converted_bias = ( + hfblock.attn.qkv.bias.view(3, vision_num_query_groups, -1) + .transpose(0, 1) + .flatten(1, 2) + .view(-1) + .contiguous() + ) + copied_numel += safe_copy(converted_bias, mgblock.self_attention.linear_qkv.bias) + # proj --> self_attention.linear_proj + copied_numel += safe_copy(hfblock.attn.proj.weight, mgblock.self_attention.linear_proj.weight) + copied_numel += safe_copy(hfblock.attn.proj.bias, mgblock.self_attention.linear_proj.bias) + # mlp --> mlp: gate + fc1_weight = torch.cat([hfblock.mlp.gate_proj.weight, hfblock.mlp.up_proj.weight]) + fc1_bias = torch.cat([hfblock.mlp.gate_proj.bias, hfblock.mlp.up_proj.bias]) + copied_numel += safe_copy(fc1_weight, mgblock.mlp.linear_fc1.weight) + copied_numel += safe_copy(fc1_bias, mgblock.mlp.linear_fc1.bias) + copied_numel += safe_copy(hfblock.mlp.down_proj.weight, mgblock.mlp.linear_fc2.weight) + copied_numel += safe_copy(hfblock.mlp.down_proj.bias, mgblock.mlp.linear_fc2.bias) + + # 2. vision projector + hfprojector = hfvision.merger + mgprojector = mgvision.projection + copied_numel += safe_copy(hfprojector.ln_q.weight, mgvision.decoder.final_layernorm.weight) + + copied_numel += safe_copy(hfprojector.mlp[0].weight, mgprojector.encoder.linear_fc1.weight) + copied_numel += safe_copy(hfprojector.mlp[0].bias, mgprojector.encoder.linear_fc1.bias) + copied_numel += safe_copy(hfprojector.mlp[2].weight, mgprojector.encoder.linear_fc2.weight) + copied_numel += safe_copy(hfprojector.mlp[2].bias, mgprojector.encoder.linear_fc2.bias) + n_params = sum([t.numel() for t in hfvision.state_dict().values()]) + assert n_params == copied_numel, f"n_params={n_params} != copied_numel={copied_numel}" + # 3. llm [just Qwen2] + if Version(version("transformers")) < Version("4.52.0"): + print("Using transformers < 4.52 API to load llm") + hfllm = hfmodel.model + else: + hfllm = hfmodel.model.language_model + mgllm = mgmodel.language_model + copied_numel = 0 + copied_numel += safe_copy(hfllm.embed_tokens.weight, mgllm.embedding.word_embeddings.weight) + layermaps = zip(mgllm.decoder.layers, hfllm.layers, strict=True) + for mglayer, hflayer in layermaps: + copied_numel += safe_copy(hflayer.input_layernorm.weight, mglayer.self_attention.linear_qkv.layer_norm_weight) + + q_proj_weight = hflayer.self_attn.q_proj.weight.view(num_query_groups, -1, head_dim, hidden_size) + k_proj_weight = hflayer.self_attn.k_proj.weight.view(num_query_groups, -1, head_dim, hidden_size) + v_proj_weight = hflayer.self_attn.v_proj.weight.view(num_query_groups, -1, head_dim, hidden_size) + qkv_proj = torch.cat([q_proj_weight, k_proj_weight, v_proj_weight], dim=1).view(-1, hidden_size).contiguous() + copied_numel += safe_copy(qkv_proj, mglayer.self_attention.linear_qkv.weight) + + q_proj_bias = hflayer.self_attn.q_proj.bias.view(num_query_groups, -1) + k_proj_bias = hflayer.self_attn.k_proj.bias.view(num_query_groups, -1) + v_proj_bias = hflayer.self_attn.v_proj.bias.view(num_query_groups, -1) + qkv_bias = torch.cat([q_proj_bias, k_proj_bias, v_proj_bias], dim=1).view(-1).contiguous() + copied_numel += safe_copy(qkv_bias, mglayer.self_attention.linear_qkv.bias) + copied_numel += safe_copy(hflayer.self_attn.o_proj.weight, mglayer.self_attention.linear_proj.weight) + + fc1_weight = torch.cat([hflayer.mlp.gate_proj.weight, hflayer.mlp.up_proj.weight]) + copied_numel += safe_copy(fc1_weight, mglayer.mlp.linear_fc1.weight) + + copied_numel += safe_copy(hflayer.mlp.down_proj.weight, mglayer.mlp.linear_fc2.weight) + copied_numel += safe_copy(hflayer.post_attention_layernorm.weight, mglayer.mlp.linear_fc1.layer_norm_weight) + + copied_numel += safe_copy(hfllm.norm.weight, mgllm.decoder.final_layernorm.weight) + if not hf_config.tie_word_embeddings: + safe_copy(hfmodel.lm_head.weight, mgllm.output_layer.weight) + + n_params = sum([t.numel() for t in hfllm.state_dict().values()]) + + assert n_params == copied_numel, f"n_params={n_params} != copied_numel={copied_numel}" + + +@torch.inference_mode() +def convert_checkpoint_from_transformers_to_megatron_dpskv3( + hf_model, + model, + hf_config, + tfconfig, + layer_start_end: Optional[tuple[int, int]] = None, +): + warnings.warn("MTP model is not supported yet", stacklevel=2) + if layer_start_end is None: + layer_start_end = (0, len(model.decoder.layers)) + layer_start, layer_end = layer_start_end + numel: int = 0 + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + if pp_rank == 0: + numel += safe_copy(hf_model.model.embed_tokens.weight, model.embedding.word_embeddings.weight) + + assert len(model.decoder.layers) == (layer_end - layer_start), ( + f"Expected {len(model.decoder.layers)} layers, but got {layer_end - layer_start}" + ) + for layer_idx, (layer, hf_layer) in enumerate( + zip(model.decoder.layers, hf_model.model.layers[layer_start:layer_end], strict=True) + ): + global_layer_idx = layer_idx + layer_start + numel_cur: int = numel + numel += safe_copy(hf_layer.input_layernorm.weight, layer.input_layernorm.weight) + + if hf_config.q_lora_rank is None: + numel += safe_copy(hf_layer.self_attn.q_proj.weight, layer.self_attention.linear_q_proj.weight) + else: + numel += safe_copy(hf_layer.self_attn.q_a_proj.weight, layer.self_attention.linear_q_down_proj.weight) + numel += safe_copy(hf_layer.self_attn.q_b_proj.weight, layer.self_attention.linear_q_up_proj.weight) + numel += safe_copy( + hf_layer.self_attn.q_a_layernorm.weight, layer.self_attention.linear_q_up_proj.layer_norm_weight + ) + + numel += safe_copy( + hf_layer.self_attn.kv_a_proj_with_mqa.weight, layer.self_attention.linear_kv_down_proj.weight + ) + numel += safe_copy(hf_layer.self_attn.kv_b_proj.weight, layer.self_attention.linear_kv_up_proj.weight) + numel += safe_copy( + hf_layer.self_attn.kv_a_layernorm.weight, layer.self_attention.linear_kv_up_proj.layer_norm_weight + ) + numel += safe_copy(hf_layer.self_attn.o_proj.weight, layer.self_attention.linear_proj.weight) + + if not hasattr(layer.mlp, "router"): + numel += safe_copy(hf_layer.post_attention_layernorm.weight, layer.mlp.linear_fc1.layer_norm_weight) + numel += safe_copy( + torch.cat([hf_layer.mlp.gate_proj.weight, hf_layer.mlp.up_proj.weight]), layer.mlp.linear_fc1.weight + ) + numel += safe_copy(hf_layer.mlp.down_proj.weight, layer.mlp.linear_fc2.weight) + else: + numel += safe_copy(hf_layer.mlp.gate.weight, layer.mlp.router.weight) + # NOTE: the e_score_correction_bias in mcore model will be initialized with bfloat16 and \ + # recover to fp32 in the first forward. There is always a diff in the bias between two models (~0.3%) + numel += safe_copy( + hf_layer.mlp.gate.e_score_correction_bias, layer.mlp.router.expert_bias, skip_dtype_assert=True + ) + if tfconfig.moe_grouped_gemm: + for i, hf_expert in enumerate(hf_layer.mlp.experts): + fc1_weight = torch.cat([hf_expert.gate_proj.weight, hf_expert.up_proj.weight]) + linear_fc1_weighti = getattr(layer.mlp.experts.linear_fc1, "weight" + str(i)) + numel += safe_copy(fc1_weight, linear_fc1_weighti) + linear_fc2_weighti = getattr(layer.mlp.experts.linear_fc2, "weight" + str(i)) + numel += safe_copy(hf_expert.down_proj.weight, linear_fc2_weighti) + else: + for i, hf_expert in enumerate(hf_layer.mlp.experts): + expert = layer.mlp.experts.local_experts[i] + fc1_weight = torch.cat([hf_expert.gate_proj.weight, hf_expert.up_proj.weight]) + numel += safe_copy(fc1_weight, expert.linear_fc1.weight) + numel += safe_copy(hf_expert.down_proj.weight, expert.linear_fc2.weight) + numel += safe_copy(hf_layer.post_attention_layernorm.weight, layer.pre_mlp_layernorm.weight) + shared_fc1_weight = torch.cat( + [hf_layer.mlp.shared_experts.gate_proj.weight, hf_layer.mlp.shared_experts.up_proj.weight] + ) + numel += safe_copy(shared_fc1_weight, layer.mlp.shared_experts.linear_fc1.weight) + numel += safe_copy(hf_layer.mlp.shared_experts.down_proj.weight, layer.mlp.shared_experts.linear_fc2.weight) + print(f"{pp_rank=} {global_layer_idx=} {layer_idx=} {numel=} numel this layer={numel - numel_cur}") + assert numel - numel_cur == sum([i.numel() for i in hf_layer.state_dict().values()]), "numel mismatch" + + if pp_rank == pp_size - 1: + numel += safe_copy(hf_model.model.norm.weight, model.decoder.final_layernorm.weight) + if not hf_config.tie_word_embeddings: + numel += safe_copy(hf_model.lm_head.weight, model.output_layer.weight) + print(f"{pp_rank=} {numel=}") + return numel + + +@contextmanager +def noop_context() -> Any: + yield + + +def support_distributed_convert(hf_config: AutoConfig) -> bool: + for arch in ["DeepseekV3ForCausalLM", "Qwen3MoeForCausalLM", "Qwen2MoeForCausalLM"]: + if arch in hf_config.architectures: + return True + return False + + +def convert_hf_to_mcore(hf_model_path, output_path, use_cpu_initialization=False, test=False, trust_remote_code=False): + os.makedirs(output_path, exist_ok=True) + if len(os.listdir(output_path)) > 0 and not test: + print(f"Output path {output_path} is not empty, skipping conversion") + return + + # init torch distributed and mpu + if "WORLD_SIZE" not in os.environ: + os.environ["RANK"] = "0" + os.environ["WORLD_SIZE"] = "1" + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12355" + + torch.distributed.init_process_group("nccl") + + rank = dist.get_rank() + local_rank = os.getenv("LOCAL_RANK", 0) + world_size = dist.get_world_size() + get_torch_device().set_device(f"{get_device_name()}:{local_rank}") + + mpu.initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=world_size, + virtual_pipeline_model_parallel_size=None, + context_parallel_size=1, + expert_model_parallel_size=1, + ) + model_parallel_cuda_manual_seed(0) + + # init hf config + hf_config = AutoConfig.from_pretrained(hf_model_path) + print(hf_config, flush=True) + + if world_size > 1 and not support_distributed_convert(hf_config): + raise NotImplementedError(f"distributed conversion is not supported for {hf_config.architectures} yet.") + + pipeline_shards = get_dynamic_pipeline_shards(hf_config.num_hidden_layers, world_size) + print(f"Pipeline shards: {pipeline_shards}", flush=True) + + tfconfig = hf_to_mcore_config( + hf_config, + torch.bfloat16, + num_layers_in_first_pipeline_stage=pipeline_shards[0] if len(pipeline_shards) > 1 else None, + num_layers_in_last_pipeline_stage=pipeline_shards[-1] if len(pipeline_shards) > 2 else None, + ) + tfconfig.use_cpu_initialization = use_cpu_initialization + tie_word_embeddings = getattr(hf_config, "tie_word_embeddings", False) + + # init megatron model + def megatron_model_provider(pre_process, post_process): + from verl.models.mcore import init_mcore_model + + parallel_model = init_mcore_model( + tfconfig, + hf_config, + pre_process, + post_process, + share_embeddings_and_output_weights=tie_word_embeddings, + value=False, + ) + return parallel_model + + context: Callable[..., ContextManager] = init_empty_weights if use_cpu_initialization else noop_context + with context(): + model = get_model( + model_provider_func=megatron_model_provider, + model_type=ModelType.encoder_or_decoder, + wrap_with_ddp=False, + transformer_config=tfconfig, + ) + + if use_cpu_initialization: + # convert meta device to empty tensor so it can use `copy_` function + model[0].module = model[0].module.to_empty(device="cpu") + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + from transformers import AutoModelForCausalLM, AutoModelForImageTextToText + + # init hf model + if "Qwen2_5_VLForConditionalGeneration" in hf_config.architectures: + hf_model = AutoModelForImageTextToText.from_pretrained( + hf_model_path, torch_dtype=torch.bfloat16, trust_remote_code=trust_remote_code + ) + else: + hf_model = AutoModelForCausalLM.from_pretrained( + hf_model_path, torch_dtype=torch.bfloat16, trust_remote_code=trust_remote_code + ) + hf_state_dict = hf_model.state_dict() + + # distributed convert + if world_size > 1 and support_distributed_convert(hf_config): + pipeline_cumsum = np.cumsum(pipeline_shards) + layer_start = 0 if rank == 0 else pipeline_cumsum[rank - 1] + layer_end = pipeline_cumsum[rank] + if "DeepseekV3ForCausalLM" in hf_config.architectures: + numel_partial: int = convert_checkpoint_from_transformers_to_megatron_dpskv3( + hf_model, model[0].module, hf_config, tfconfig=tfconfig, layer_start_end=(layer_start, layer_end) + ) + elif "Qwen3MoeForCausalLM" in hf_config.architectures or "Qwen2MoeForCausalLM" in hf_config.architectures: + numel_partial: int = convert_checkpoint_from_transformers_to_megatron( + hf_model, model[0].module, hf_config, layer_start_end=(layer_start, layer_end) + ) + else: + raise NotImplementedError(f"Distributed conversion is not supported for {hf_config.architectures} yet.") + + numel_tensor = torch.tensor([numel_partial]).to(get_device_name()) + dist.all_reduce(numel_tensor, op=dist.ReduceOp.SUM) + numel = int(numel_tensor.cpu().item()) + print(f"total numel={numel} vs {hf_model.num_parameters()=}") + if numel != hf_model.num_parameters(): + warnings.warn(f"numel mismatch: {numel=} != {hf_model.num_parameters()=}", stacklevel=1) + + # load hf state dict to megatron model + elif "Qwen2MoeForCausalLM" in hf_config.architectures: + convert_checkpoint_from_transformers_to_megatron(hf_model, model[0].module, hf_config) + elif "Qwen2_5_VLForConditionalGeneration" in hf_config.architectures: + convert_checkpoint_from_transformers_to_megatron_qwen2_5_vl(hf_model, model[0].module, hf_config) + elif "DeepseekV3ForCausalLM" in hf_config.architectures: + convert_checkpoint_from_transformers_to_megatron_dpskv3(hf_model, model[0].module, hf_config, tfconfig=tfconfig) + elif "Qwen3MoeForCausalLM" in hf_config.architectures: + convert_checkpoint_from_transformers_to_megatron(hf_model, model[0].module, hf_config) + else: + assert not use_cpu_initialization, "use_cpu_initialization is only supported for MoE model" + from verl.models.mcore.loader import load_state_dict_to_megatron_gptmodel + + load_state_dict_to_megatron_gptmodel( + state_dict=hf_state_dict, + wrapped_models=model, + config=hf_config, + params_dtype=torch.bfloat16, + is_value_model=False, + ) + + megatron_state_dict = model[0].module.sharded_state_dict() + del hf_state_dict, hf_model + + # save megatron model + if len(os.listdir(output_path)) == 0: + dist_checkpointing.save(megatron_state_dict, output_path, sharded_strategy=None, async_sharded_save=False) + if test: + test_conversion(megatron_model_provider, tfconfig, output_path, model) + + +if __name__ == "__main__": + args = _init_args() + convert_hf_to_mcore( + args.hf_model_path, args.output_path, args.use_cpu_initialization, args.test, args.trust_remote_code + ) diff --git a/e2c/verl/scripts/diagnose.py b/e2c/verl/scripts/diagnose.py new file mode 100644 index 0000000000000000000000000000000000000000..cb78f9e5c6297a8ba8e84262253ff385f49e0d2a --- /dev/null +++ b/e2c/verl/scripts/diagnose.py @@ -0,0 +1,312 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Diagnose script for checking OS/hardware/python/pip/verl/network. +The output of this script can be a very good hint to issue/problem. +""" + +import os +import platform +import socket +import subprocess +import sys +import time + +import psutil + +try: + from urllib.parse import urlparse + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen + from urlparse import urlparse +import argparse +import importlib.metadata + +import torch + +URLS = { + "PYPI": "https://pypi.python.org/pypi/pip", +} + +REGIONAL_URLS = { + "cn": { + "PYPI(douban)": "https://pypi.douban.com/", + "Conda(tsinghua)": "https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/", + } +} + + +def test_connection(name, url, timeout=10): + """Simple connection test""" + urlinfo = urlparse(url) + start = time.time() + try: + socket.gethostbyname(urlinfo.netloc) + except Exception as e: + print("Error resolving DNS for {}: {}, {}".format(name, url, e)) + return + dns_elapsed = time.time() - start + start = time.time() + try: + _ = urlopen(url, timeout=timeout) + except Exception as e: + print("Error open {}: {}, {}, DNS finished in {} sec.".format(name, url, e, dns_elapsed)) + return + load_elapsed = time.time() - start + print("Timing for {}: {}, DNS: {:.4f} sec, LOAD: {:.4f} sec.".format(name, url, dns_elapsed, load_elapsed)) + + +def check_python(): + print("----------Python Info----------") + print("Version :", platform.python_version()) + print("Compiler :", platform.python_compiler()) + print("Build :", platform.python_build()) + print("Arch :", platform.architecture()) + + +def check_pip(): + print("------------Pip Info-----------") + try: + import pip + + print("Version :", pip.__version__) + print("Directory :", os.path.dirname(pip.__file__)) + except ImportError: + print("No corresponding pip install for current python.") + + +def _get_current_git_commit(): + try: + result = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + print(f"Error running git command: {e.stderr.strip()}") + return None + except FileNotFoundError: + print("Did not find command: git") + return None + + +def check_verl(): + print("----------verl Info-----------") + try: + sys.path.insert(0, os.getcwd()) + import verl + + print("Version :", verl.__version__) + verl_dir = os.path.dirname(verl.__file__) + print("Directory :", verl_dir) + try: + commit_hash = _get_current_git_commit() + print("Commit Hash :", commit_hash) + except AttributeError: + print("Commit hash not found. ") + except ImportError as e: + print(f"No verl installed: {e}") + except Exception as e: + import traceback + + if not isinstance(e, IOError): + print("An error occurred trying to import verl.") + print("This is very likely due to missing or incompatible library files.") + print(traceback.format_exc()) + + +def check_os(): + print("----------Platform Info----------") + print("Platform :", platform.platform()) + print("system :", platform.system()) + print("node :", platform.node()) + print("release :", platform.release()) + print("version :", platform.version()) + + +def check_hardware(): + print("----------Hardware Info----------") + print("machine :", platform.machine()) + print("processor :", platform.processor()) + if sys.platform.startswith("darwin"): + pipe = subprocess.Popen(("sysctl", "-a"), stdout=subprocess.PIPE) + output = pipe.communicate()[0] + for line in output.split(b"\n"): + if b"brand_string" in line or b"features" in line: + print(line.strip()) + elif sys.platform.startswith("linux"): + subprocess.call(["lscpu"]) + elif sys.platform.startswith("win32"): + subprocess.call(["wmic", "cpu", "get", "name"]) + + +def check_network(args): + print("----------Network Test----------") + if args.timeout > 0: + print("Setting timeout: {}".format(args.timeout)) + socket.setdefaulttimeout(10) + for region in args.region.strip().split(","): + r = region.strip().lower() + if not r: + continue + if r in REGIONAL_URLS: + URLS.update(REGIONAL_URLS[r]) + else: + import warnings + + warnings.warn("Region {} do not need specific test, please refer to global sites.".format(r), stacklevel=2) + for name, url in URLS.items(): + test_connection(name, url, args.timeout) + + +def check_environment(): + print("----------Environment----------") + for k, v in os.environ.items(): + if k.startswith("VERL_") or k.startswith("OMP_") or k.startswith("KMP_") or k == "CC" or k == "CXX": + print('{}="{}"'.format(k, v)) + + +def check_pip_package_versions(): + packages = ["vllm", "sglang", "ray", "torch"] + for package in packages: + try: + version = importlib.metadata.version(package) + print(f"{package}\t : {version}") + except importlib.metadata.PackageNotFoundError: + print(f"{package}\t : not found.") + + +def check_cuda_versions(): + if torch.cuda.is_available(): + try: + cuda_runtime_version = torch.version.cuda + print(f"CUDA Runtime : {cuda_runtime_version}") + import subprocess + + nvcc_output = subprocess.check_output(["nvcc", "--version"]).decode("utf-8") + cuda_compiler_version = next((line for line in nvcc_output.splitlines() if "release" in line), None) + if cuda_compiler_version: + print(f"CUDA Compiler : {cuda_compiler_version.strip()}") + else: + print("Could not determine CUDA compiler version.") + except FileNotFoundError as e: + print(f"CUDA compiler : Not found: {e}") + except Exception as e: + print(f"An error occurred while checking CUDA versions: {e}") + else: + print("CUDA is not available.") + + +def _get_cpu_memory(): + """ + Get the total CPU memory capacity in GB. + """ + memory = psutil.virtual_memory() + return memory.total / (1024**3) + + +def _get_gpu_info(): + """ + Get GPU type, GPU memory, and GPU count using nvidia-smi command. + """ + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=gpu_name,memory.total", "--format=csv,noheader,nounits"], + capture_output=True, + text=True, + check=True, + ) + gpu_lines = result.stdout.strip().split("\n") + gpu_count = len(gpu_lines) + gpu_info = [] + for line in gpu_lines: + gpu_name, gpu_memory = line.split(", ") + gpu_info.append( + { + "type": gpu_name, + "memory": float(gpu_memory) / 1024, # Convert to GB + } + ) + return gpu_count, gpu_info + except (subprocess.CalledProcessError, FileNotFoundError): + print("Failed to execute nvidia-smi command.") + return 0, [] + + +def _get_system_info(): + """ + Get CPU memory capacity, GPU type, GPU memory, and GPU count. + """ + cpu_memory = _get_cpu_memory() + gpu_count, gpu_info = _get_gpu_info() + return {"cpu_memory": cpu_memory, "gpu_count": gpu_count, "gpu_info": gpu_info} + + +def check_system_info(): + print("----------System Info----------") + system_info = _get_system_info() + print(f"CPU Memory\t: {system_info['cpu_memory']:.2f} GB") + print(f"GPU Count\t: {system_info['gpu_count']}") + for i, gpu in enumerate(system_info["gpu_info"]): + print(f"GPU {i + 1}\tType : {gpu['type']}") + print(f"GPU {i + 1}\tMemory : {gpu['memory']:.2f} GB") + + +def parse_args(): + """Parse arguments.""" + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Diagnose script for checking the current system.", + ) + choices = ["python", "pip", "verl", "system", "os", "environment"] + for choice in choices: + parser.add_argument("--" + choice, default=1, type=int, help="Diagnose {}.".format(choice)) + parser.add_argument("--network", default=0, type=int, help="Diagnose network.") + parser.add_argument("--hardware", default=0, type=int, help="Diagnose hardware.") + parser.add_argument( + "--region", + default="", + type=str, + help="Additional sites in which region(s) to test. \ + Specify 'cn' for example to test mirror sites in China.", + ) + parser.add_argument("--timeout", default=10, type=int, help="Connection test timeout threshold, 0 to disable.") + args = parser.parse_args() + return args + + +if __name__ == "__main__": + args = parse_args() + if args.python: + check_python() + + if args.pip: + check_pip() + check_pip_package_versions() + + if args.verl: + check_verl() + + if args.os: + check_os() + + if args.hardware: + check_hardware() + + if args.network: + check_network(args) + + if args.environment: + check_environment() + check_cuda_versions() + + if args.system: + check_system_info() diff --git a/e2c/verl/scripts/generate_trainer_config.sh b/e2c/verl/scripts/generate_trainer_config.sh new file mode 100644 index 0000000000000000000000000000000000000000..a40f555fd0fa40f6f3e3d4e99fa1e0db9212ce75 --- /dev/null +++ b/e2c/verl/scripts/generate_trainer_config.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euox pipefail + + +# Define config specifications: "config_name:output_file:config_arg" +CONFIG_SPECS=( + "ppo_trainer:_generated_ppo_trainer.yaml:" + "ppo_megatron_trainer:_generated_ppo_megatron_trainer.yaml:--config-name=ppo_megatron_trainer.yaml" +) + +generate_config() { + local config_name="$1" + local output_file="$2" + local config_arg="$3" + + local target_cfg="verl/trainer/config/${output_file}" + local tmp_header=$(mktemp) + local tmp_cfg=$(mktemp) + + echo "# This reference configration yaml is automatically generated via 'scripts/generate_trainer_config.sh'" > "$tmp_header" + echo "# in which it invokes 'python3 scripts/print_cfg.py --cfg job ${config_arg}' to flatten the 'verl/trainer/config/${config_name}.yaml' config fields into a single file." >> "$tmp_header" + echo "# Do not modify this file directly." >> "$tmp_header" + echo "# The file is usually only for reference and never used." >> "$tmp_header" + echo "" >> "$tmp_header" + + python3 scripts/print_cfg.py --cfg job ${config_arg} > "$tmp_cfg" + + cat "$tmp_header" > "$target_cfg" + sed -n '/^actor_rollout_ref/,$p' "$tmp_cfg" >> "$target_cfg" + + rm "$tmp_cfg" "$tmp_header" + + echo "Generated: $target_cfg" +} + +for spec in "${CONFIG_SPECS[@]}"; do + IFS=':' read -r config_name output_file config_arg <<< "$spec" + generate_config "$config_name" "$output_file" "$config_arg" +done + +for spec in "${CONFIG_SPECS[@]}"; do + IFS=':' read -r config_name output_file config_arg <<< "$spec" + target_cfg="verl/trainer/config/${output_file}" + if ! git diff --exit-code -- "$target_cfg" >/dev/null; then + echo "✖ $target_cfg is out of date. Please regenerate via 'scripts/generate_trainer_config.sh' and commit the changes." + exit 1 + fi +done + +echo "All good" +exit 0 diff --git a/e2c/verl/scripts/init_random_model.py b/e2c/verl/scripts/init_random_model.py new file mode 100644 index 0000000000000000000000000000000000000000..2804bc2a24ffd7b8386b7a515c3f5dd831c9357d --- /dev/null +++ b/e2c/verl/scripts/init_random_model.py @@ -0,0 +1,95 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script override a model with custom config and random weights, mainly for create small models for +debugging purposes. + +Usage: + python scripts/init_random_model.py \ + --hf_model_path \ + --new_config_path \ + --output_path + +""" + +import argparse +import json +import os +import warnings +from typing import Any + +from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, PretrainedConfig + + +def _init_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--hf_model_path", type=str, required=True, help="The path for the huggingface model") + parser.add_argument("--new_config_path", type=str, required=True, help="The path for the new config file") + parser.add_argument("--output_path", type=str, required=True, help="The path for the output random model") + args = parser.parse_args() + return args + + +def check_output_path(output_path: str): + if os.path.exists(output_path): + warnings.warn(f"Output path '{output_path}' already exists. Will do nothing.", stacklevel=2) + exit() + else: + os.makedirs(output_path, exist_ok=True) + print(f"Output path '{output_path}' created.") + + +def check_configs(original_config: dict[str, Any], new_config: dict[str, Any]) -> bool: + """ + Check if the original config and new config are compatible. + This is a placeholder function; actual implementation may vary based on requirements. + """ + # Example check: ensure 'model_type' is the same + if new_config.get("model_type", None) is not None and original_config.get("model_type") != new_config.get( + "model_type" + ): + raise RuntimeError("Model types do not match.") + for key in new_config: + if key not in original_config: + warnings.warn( + f"Key '{key}' in new config does not exist in original config, may not take effect.", stacklevel=2 + ) + + +def init_random_model(hf_model_path, new_config_path, output_path): + config = AutoConfig.from_pretrained(hf_model_path) + tokenizer = AutoTokenizer.from_pretrained(hf_model_path) + config_dict = PretrainedConfig.get_config_dict(hf_model_path)[0] + print(config_dict) + with open(new_config_path) as f: + new_config_dict = json.load(f) + check_configs(config_dict, new_config_dict) + config_dict.update(new_config_dict) + new_confg = config.from_dict(config_dict) + print(f"new_config: {new_confg}") + model = AutoModelForCausalLM.from_config(new_confg) + model.save_pretrained(output_path) + tokenizer.save_pretrained(output_path) + new_confg.save_pretrained(output_path) + print(f"Random model initialized and saved to {output_path}") + + +if __name__ == "__main__": + args = _init_args() + check_output_path(args.output_path) + init_random_model( + hf_model_path=args.hf_model_path, new_config_path=args.new_config_path, output_path=args.output_path + ) diff --git a/e2c/verl/scripts/install_vllm_sglang_mcore.sh b/e2c/verl/scripts/install_vllm_sglang_mcore.sh new file mode 100644 index 0000000000000000000000000000000000000000..0e305c5d80fed04af65b5222f28a2770a09f11ad --- /dev/null +++ b/e2c/verl/scripts/install_vllm_sglang_mcore.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +USE_MEGATRON=${USE_MEGATRON:-1} +USE_SGLANG=${USE_SGLANG:-1} + +export MAX_JOBS=32 + +echo "1. install inference frameworks and pytorch they need" +if [ $USE_SGLANG -eq 1 ]; then + pip install "sglang[all]==0.4.6.post1" --no-cache-dir --find-links https://flashinfer.ai/whl/cu124/torch2.6/flashinfer-python && pip install torch-memory-saver --no-cache-dir +fi +pip install --no-cache-dir "vllm==0.8.5.post1" "torch==2.6.0" "torchvision==0.21.0" "torchaudio==2.6.0" "tensordict==0.6.2" torchdata + +echo "2. install basic packages" +pip install "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=15.0.0" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler \ + pytest py-spy pyext pre-commit ruff tensorboard + +pip install "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + + +echo "3. install FlashAttention and FlashInfer" +# Install flash-attn-2.7.4.post1 (cxx11abi=False) +wget -nv https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl && \ + pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl + +# Install flashinfer-0.2.2.post1+cu124 (cxx11abi=False) +# vllm-0.8.3 does not support flashinfer>=0.2.3 +# see https://github.com/vllm-project/vllm/pull/15777 +wget -nv https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.2.post1/flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + pip install --no-cache-dir flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl + + +if [ $USE_MEGATRON -eq 1 ]; then + echo "4. install TransformerEngine and Megatron" + echo "Notice that TransformerEngine installation can take very long time, please be patient" + NVTE_FRAMEWORK=pytorch pip3 install --no-deps git+https://github.com/NVIDIA/TransformerEngine.git@v2.2.1 + pip3 install --no-deps git+https://github.com/NVIDIA/Megatron-LM.git@core_v0.12.2 +fi + + +echo "5. May need to fix opencv" +pip install opencv-python +pip install opencv-fixer && \ + python -c "from opencv_fixer import AutoFix; AutoFix()" + + +if [ $USE_MEGATRON -eq 1 ]; then + echo "6. Install cudnn python package (avoid being overridden)" + pip install nvidia-cudnn-cu12==9.8.0.87 +fi + +echo "Successfully installed all packages" diff --git a/e2c/verl/scripts/legacy_model_merger.py b/e2c/verl/scripts/legacy_model_merger.py new file mode 100644 index 0000000000000000000000000000000000000000..8a5224abf3fd67d056cce0737d396f232604335f --- /dev/null +++ b/e2c/verl/scripts/legacy_model_merger.py @@ -0,0 +1,781 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script is used to merge huggingface model and test verl checkpoints from FSDP and Megatron backends. + +To merge FSDP checkpoints: +```sh +python scripts/legacy_model_merger.py merge \ + --backend fsdp \ + --local_dir checkpoints/verl_fsdp_gsm8k_examples/qwen2_5_0b5_fsdp_saveload/global_step_1/actor \ + --target_dir /path/to/merged_hf_model +``` + +To merge Megatron checkpoints: +```sh +python scripts/legacy_model_merger.py merge \ + --backend megatron \ + --tie-word-embedding \ + --local_dir checkpoints/verl_megatron_gsm8k_examples/qwen2_5_0b5_megatron_saveload/global_step_1/actor \ + --target_dir /path/to/merged_hf_model +``` + +For more details, please refer to documentation: +https://verl.readthedocs.io/en/latest/advance/checkpoint.html#convert-fsdp-and-megatron-checkpoints-to-huggingface-format-model +""" + +import argparse +import os +import re +import warnings +from abc import ABC, abstractmethod +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional, Union + +import numpy as np +import torch +from accelerate import init_empty_weights +from safetensors.torch import load_file +from torch.distributed._tensor import Placement, Shard +from transformers import ( + AutoConfig, + AutoModelForCausalLM, + AutoModelForTokenClassification, + AutoModelForVision2Seq, + GenerationConfig, + PretrainedConfig, +) + +try: + # for torch 2.5+ + from torch.distributed.tensor import DTensor +except ImportError: + from torch.distributed._tensor import DTensor + +from tqdm import tqdm + +from verl.utils import hf_processor, hf_tokenizer + + +@dataclass +class ModelMergerConfig: + operation: str # 'merge' or 'test' + backend: str + local_dir: str + hf_model_config_path: str + target_dir: Optional[str] = "tmp" + hf_upload_path: Optional[str] = None + private: bool = False + test_hf_dir: Optional[str] = None + tie_word_embedding: bool = False + is_value_model: bool = False + hf_model_path: Optional[str] = None + hf_upload: bool = field(init=False) + + def __post_init__(self): + self.hf_upload = self.operation == "merge" and bool(self.hf_upload_path) + if self.operation == "test": + self.target_dir = None + self.hf_upload_path = None + self.private = False + + +class BaseModelMerger(ABC): + def __init__(self, config: ModelMergerConfig): + self.config = config + self.hf_model_config_path = config.hf_model_config_path + + if config.hf_model_path: + print( + "Warning: --hf_model_path is deprecated and will be removed in a future version. Currently verl will save huggingface model configuration files into checkpoint directories. Therefore, there is no need to provide --hf_model_path. " + ) + self.hf_model_config_path = config.hf_model_path + + self.model_config = AutoConfig.from_pretrained(self.hf_model_config_path) + + def get_transformers_auto_model_class(self): + if "ForTokenClassification" in self.model_config.architectures[0]: + return AutoModelForTokenClassification + elif "ForCausalLM" in self.model_config.architectures[0]: + return AutoModelForCausalLM + elif "ForConditionalGeneration" in self.model_config.architectures[0]: + return AutoModelForVision2Seq + + raise NotImplementedError(f"Unknown architecture {self.model_config.architectures}") + + def patch_model_generation_config(self, model): + """ + The generation_config created from model config may be different to the pretrained model, + this may lead to error when generating: https://github.com/volcengine/verl/issues/1246 + + This function patch the generation_config created from model config to the pretrained model. + """ + if model.can_generate(): + try: + model.generation_config = GenerationConfig.from_pretrained(self.hf_model_config_path) + except OSError: + print( + f"Warning: Generation config file not found in {self.hf_model_config_path}, using a generation config created from the model config." + ) + return model + + def save_lora_adapter(self, state_dict: dict[str, torch.Tensor]): + """ + Save lora adapter to safetensors. + + Returns: + lora_path: str, the path to the lora adapter. None if no lora adapter found. + + Note: + This function change the 'state_dict' in place. + """ + lora_params_names = [name for name in state_dict.keys() if "lora_" in name] + + if len(lora_params_names) == 0: + return None + + import json + from typing import OrderedDict + + import peft + from safetensors.torch import save_file + + lora_params = OrderedDict() + target_modules = set() + lora_key = None + + for name in lora_params_names: + lora_key = name.replace(".default.weight", ".weight") + target_modules.add(lora_key.split(".")[-3]) + lora_params[lora_key] = state_dict.pop(name) + + lora_rank = min(lora_params[lora_key].shape[0], lora_params[lora_key].shape[1]) + peft_dict = { + "r": lora_rank, + "lora_alpha": 0, # lora_alpha is not set. An error should be raised to inform the user to set it manually. + "target_modules": list(target_modules), + } + peft_config = peft.LoraConfig(**peft_dict).to_dict() + peft_config["task_type"] = peft_config["task_type"].value if peft_config["task_type"] else None + peft_config["peft_type"] = peft_config["peft_type"].value if peft_config["peft_type"] else None + peft_config["target_modules"] = list(peft_config["target_modules"]) + + lora_path = os.path.join(self.config.target_dir, "lora_adapter") + os.makedirs(lora_path, exist_ok=True) + with open(os.path.join(lora_path, "adapter_config.json"), "w", encoding="utf-8") as f: + json.dump(peft_config, f, ensure_ascii=False, indent=4) + save_file(lora_params, os.path.join(lora_path, "adapter_model.safetensors")) + + for name in list(state_dict.keys()): + key = ( + name.replace("base_model.model.", "") + .replace(".base_layer.weight", ".weight") + .replace(".base_layer.bias", ".bias") + ) + state_dict[key] = state_dict.pop(name) + + return lora_path + + def save_hf_model_and_tokenizer(self, state_dict: dict[str, torch.Tensor]): + auto_model_class = self.get_transformers_auto_model_class() + with init_empty_weights(): + model = auto_model_class.from_config(self.model_config, torch_dtype=torch.bfloat16) + model.to_empty(device="cpu") + model = self.patch_model_generation_config(model) + + lora_path = self.save_lora_adapter(state_dict) + if lora_path: + print(f"Saving lora adapter to {lora_path}") + + print(f"Saving model to {self.config.target_dir}") + model.save_pretrained(self.config.target_dir, state_dict=state_dict) + del state_dict + del model + + processor = hf_processor(self.hf_model_config_path) + tokenizer = hf_tokenizer(self.hf_model_config_path) + if processor is not None: + print(f"Saving processor to {self.config.target_dir}") + processor.save_pretrained(self.config.target_dir) + if tokenizer is not None: + print(f"Saving tokenizer to {self.config.target_dir}") + tokenizer.save_pretrained(self.config.target_dir) + + def upload_to_huggingface(self): + from huggingface_hub import HfApi + + api = HfApi() + api.create_repo(repo_id=self.config.hf_upload_path, private=self.config.private, exist_ok=True) + api.upload_folder(folder_path=self.config.target_dir, repo_id=self.config.hf_upload_path, repo_type="model") + + @abstractmethod + def merge_and_save(self): + raise NotImplementedError("Subclasses should implement this method") + + +class FSDPModelMerger(BaseModelMerger): + def _get_world_size(self) -> int: + """Extracts the FSDP world_size from checkpoint filenames (e.g., 'model_world_size_8_rank_0.pt').""" + for filename in os.listdir(self.config.local_dir): + match = re.match(r"model_world_size_(\d+)_rank_0\.pt", filename) + if match: + return int(match.group(1)) + raise FileNotFoundError( + f"Could not determine world size. No file matching 'model_world_size_(\d+)_rank_0.pt' found in {self.config.local_dir}" + ) + + def _load_rank_zero_state_dict(self, world_size: int) -> dict: + return torch.load( + Path(self.config.local_dir) / f"model_world_size_{world_size}_rank_0.pt", + map_location="cpu", + weights_only=False, + ) + + def _extract_device_mesh_info(self, state_dict: dict, world_size: int) -> tuple[np.ndarray, tuple[str, ...]]: + """ + Retrieves sharding information (device_mesh, mesh_dim_names) from a DTensor in the state_dict. + If no DTensor is found, infers a simple FSDP mesh based on world_size. + """ + pivot_key = sorted(list(state_dict.keys()))[0] + weight = state_dict[pivot_key] + + if isinstance(weight, DTensor): + # get sharding info + device_mesh = weight.device_mesh + mesh = device_mesh.mesh + mesh_dim_names = device_mesh.mesh_dim_names + else: + # for non-DTensor + mesh = np.array([world_size], dtype=np.int64) + mesh_dim_names = ("fsdp",) + + return mesh, mesh_dim_names + + def _calculate_shard_configuration( + self, mesh: np.ndarray, mesh_dim_names: tuple[str, ...] + ) -> tuple[int, tuple[int, ...]]: + """Calculates the total number of shards and the shape of the device mesh.""" + assert mesh_dim_names in (("fsdp",), ("ddp", "fsdp")), f"Unsupported mesh_dim_names {mesh_dim_names}" + + if "tp" in mesh_dim_names: + # TODO: "tp" is not supported yet due to the above assert + total_shards = mesh.shape[-1] * mesh.shape[-2] + mesh_shape = (mesh.shape[-2], mesh.shape[-1]) + else: + total_shards = mesh.shape[-1] + mesh_shape = (mesh.shape[-1],) + + return total_shards, mesh_shape + + def _merge_by_placement(self, tensors: list[torch.Tensor], placement: Placement) -> torch.Tensor: + """Merges a list of tensors based on their DTensor placement""" + if placement.is_replicate(): + return tensors[0] + elif placement.is_partial(): + raise NotImplementedError("Partial placement is not supported yet") + elif placement.is_shard(): + return torch.cat(tensors, dim=placement.dim).contiguous() + + raise NotImplementedError(f"Unsupported placement: {placement}") + + def _load_and_merge_state_dicts( + self, world_size: int, total_shards: int, mesh_shape: tuple[int, ...], mesh_dim_names: tuple[str, ...] + ) -> dict[str, torch.Tensor]: + model_state_dict_lst = [None] * total_shards + + def process_one_shard(rank: int, model_state_dict_lst: list): + model_path = Path(self.config.local_dir) / f"model_world_size_{world_size}_rank_{rank}.pt" + state_dict = torch.load(model_path, map_location="cpu", weights_only=False) + model_state_dict_lst[rank] = state_dict + return state_dict + + with ThreadPoolExecutor(max_workers=min(32, os.cpu_count())) as executor: + futures = [executor.submit(process_one_shard, rank, model_state_dict_lst) for rank in range(total_shards)] + for future in tqdm(futures, desc=f"Loading {total_shards} FSDP shards", total=total_shards): + future.result() + + # Merge state dicts from all shards + state_dict = {} + param_placements: dict[str, list] = {} + + for key in set(model_state_dict_lst[0].keys()): + state_dict[key] = [] + for model_state_shard in model_state_dict_lst: + # add tensor shard in order of rank to state_dict[key] + tensor = model_state_shard.pop(key) + if isinstance(tensor, DTensor): + state_dict[key].append(tensor._local_tensor.bfloat16()) + + placements = tuple(tensor.placements) + # replicated placement at dp dimension can be discarded + if mesh_dim_names[0] in ("dp", "ddp"): + placements = placements[1:] + + if key not in param_placements: + param_placements[key] = placements + else: + assert param_placements[key] == placements + else: + state_dict[key].append(tensor.bfloat16()) + + del model_state_dict_lst + + # Merge tensors + for key in sorted(state_dict): + if not isinstance(state_dict[key], list): + print(f"No need to merge key {key}") + continue + if key in param_placements: + # merge shards + placements: tuple[Shard] = param_placements[key] + if len(mesh_shape) == 1: + # 1-D list, FSDP without TP + assert len(placements) == 1 + shards = state_dict[key] + state_dict[key] = self._merge_by_placement(shards, placements[0]) + else: + # 2-D list, FSDP + TP + raise NotImplementedError("FSDP + TP is not supported yet") + else: + state_dict[key] = torch.cat(state_dict[key], dim=0) + + return state_dict + + def merge_and_save(self): + world_size = self._get_world_size() + rank_zero_state_dict = self._load_rank_zero_state_dict(world_size) + + mesh, mesh_dim_names = self._extract_device_mesh_info(rank_zero_state_dict, world_size) + print(f"Got device mesh {mesh}, mesh_dim_names {mesh_dim_names}") + + total_shards, mesh_shape = self._calculate_shard_configuration(mesh, mesh_dim_names) + print(f"Processing model shards with {total_shards} {mesh_shape} in total") + + merged_state_dict = self._load_and_merge_state_dicts(world_size, total_shards, mesh_shape, mesh_dim_names) + + if self.config.operation == "test": + if not self.config.test_hf_dir: + raise ValueError("test_hf_dir must be provided for test operation") + self._test_state_dict(merged_state_dict) + elif self.config.operation == "merge": + self.save_hf_model_and_tokenizer(merged_state_dict) + if self.config.hf_upload: + self.upload_to_huggingface() + else: + raise ValueError(f"Unknown operation: {self.config.operation}") + + def _test_state_dict(self, state_dict: dict[str, torch.Tensor]): + auto_model_class = self.get_transformers_auto_model_class() + + hf_model = auto_model_class.from_pretrained(self.config.test_hf_dir, torch_dtype=torch.bfloat16) + hf_state_dict = hf_model.state_dict() + del hf_model + + hf_model_keys = set(hf_state_dict.keys()) + collected_keys = set(state_dict.keys()) + + missing_keys = hf_model_keys - collected_keys + assert len(missing_keys) == 0, f"Missing keys in collected state dict: {list(sorted(missing_keys))}" + + extra_keys = collected_keys - hf_model_keys + assert len(extra_keys) == 0, f"Extra keys in collected state dict: {list(sorted(extra_keys))}" + + for key in hf_model_keys: + hf_shape = hf_state_dict[key].shape + collected_shape = state_dict[key].shape + assert hf_shape == collected_shape, ( + f"Shape mismatch for key '{key}': original {hf_shape} vs collected {collected_shape}" + ) + + hf_dtype = hf_state_dict[key].dtype + collected_dtype = state_dict[key].dtype + assert hf_dtype == collected_dtype, ( + f"Dtype mismatch for key '{key}': original {hf_dtype} vs collected {collected_dtype}" + ) + + torch.testing.assert_close(hf_state_dict[key], state_dict[key], atol=1e-6, rtol=1e-6) + + print("FSDP checks passed: The merged state_dict matches the hf model saved by FSDPCheckpointManager.") + + +class MegatronModelMerger(BaseModelMerger): + def __init__(self, config: ModelMergerConfig): + from verl.utils.megatron_utils import get_hf_config_and_tokenizer_checkpoint_path + + config.hf_model_config_path = get_hf_config_and_tokenizer_checkpoint_path(config.local_dir) + super().__init__(config) + + self.params_mapping = { + # megatron core gpt model name, huggingface model name + # NOTICE: It's a little bit tricky, when 2 keys have the same prefix, we need to make sure the longer key within the containing relationship is processed first. + "embedding.word_embeddings": "model.embed_tokens", + # attn + "self_attention.linear_qkv.layer_norm_weight": "input_layernorm.weight", + "self_attention.linear_qkv.layer_norm_bias": "input_layernorm.bias", + "self_attention.linear_qkv": "self_attn.qkv_proj", + "self_attention.q_layernorm": "self_attn.q_norm", + "self_attention.k_layernorm": "self_attn.k_norm", + "self_attention.linear_proj": "self_attn.o_proj", + # mla + "self_attention.linear_q_proj": "self_attn.q_proj", + "self_attention.linear_q_down_proj": "self_attn.q_a_proj", + "self_attention.linear_q_up_proj.layer_norm_weight": "self_attn.q_a_layernorm.weight", + "self_attention.linear_q_up_proj": "self_attn.q_b_proj", + "self_attention.linear_kv_down_proj": "self_attn.kv_a_proj_with_mqa", + "self_attention.linear_kv_up_proj.layer_norm_weight": "self_attn.kv_a_layernorm.weight", + "self_attention.linear_kv_up_proj": "self_attn.kv_b_proj", + # mlp + "pre_mlp_layernorm": "post_attention_layernorm", + "mlp.linear_fc1.layer_norm_weight": "post_attention_layernorm.weight", + "mlp.linear_fc1.layer_norm_bias": "post_attention_layernorm.bias", + "mlp.linear_fc1": "mlp.gate_up_proj", + "mlp.linear_fc2": "mlp.down_proj", + # moe + "mlp.router.expert_bias": "mlp.gate.e_score_correction_bias", + "mlp.router": "mlp.gate", + "mlp.shared_experts.linear_fc1": "mlp.shared_experts.gate_up_proj", + "mlp.shared_experts.linear_fc2": "mlp.shared_experts.down_proj", + "linear_fc1": "gate_up_proj", + "linear_fc2": "down_proj", + # output + "final_layernorm": "norm", + "output_layer": "lm_head", + } + + def _get_tp_pp_rank_from_sharded_dir(self, sharded_dir: str) -> tuple[int, int]: + tp_rank = pp_rank = None + rank_list = sharded_dir.split("_")[2:] + if re.match(r"mp_rank_(\d\d)_(\d\d\d)", sharded_dir): + tp_rank = int(rank_list[0]) + pp_rank = int(rank_list[1]) + elif re.match(r"mp_rank_(\d\d)", sharded_dir): + tp_rank = int(rank_list[0]) + pp_rank = 0 + + assert tp_rank is not None and pp_rank is not None, f"Invalid sharded dir {sharded_dir}" + + return tp_rank, pp_rank + + def _check_megatron_checkpoint_path(self, model_path: str) -> tuple[list[str], int, int]: + """ + Validates the Megatron checkpoint structure (presence of 'model.pt' in sharded directories). + Determines TP and PP sizes from directory names. + """ + tp_size = 0 + pp_size = 0 + sharded_dirs = sorted(os.listdir(model_path)) + for sharded_dir in sharded_dirs: + assert "model.pt" in os.listdir(Path(model_path) / sharded_dir), f"model.pt not found in {sharded_dir}" + tp_rank, pp_rank = self._get_tp_pp_rank_from_sharded_dir(sharded_dir) + tp_size = max(tp_size, tp_rank + 1) + pp_size = max(pp_size, pp_rank + 1) + return sharded_dirs, tp_size, pp_size + + def _merge_across_tp( + self, + key: str, + tp_data: list[torch.Tensor], + config: PretrainedConfig, + tp_size: int, + is_value_model: bool = False, + ) -> Union[torch.Tensor, list[torch.Tensor]]: + if "linear_fc1.weight" in key: + # if the tensor is gate and proj + gate_lst = [] + up_lst = [] + for infer_param in tp_data: + gate, up = infer_param.chunk(2) + gate_lst.append(gate) + up_lst.append(up) + gate = torch.cat(gate_lst, dim=0) + up = torch.cat(up_lst, dim=0) + return [gate, up] + elif "self_attention.linear_qkv." in key and "layer_norm" not in key: + # if the tensor is qkv, for each param on tp, split into q, k, v + # concat q, k, v separately. + q_lst = [] + k_lst = [] + v_lst = [] + assert config.num_attention_heads % config.num_key_value_heads == 0 + num_q_per_kv = config.num_attention_heads // config.num_key_value_heads + assert tp_data[0].shape[0] % (num_q_per_kv + 2) == 0 + kv_size_per_tp = tp_data[0].shape[0] // (num_q_per_kv + 2) + split_size = [kv_size_per_tp * num_q_per_kv, kv_size_per_tp, kv_size_per_tp] + + for infer_param in tp_data: + num_query_groups_per_partition = config.num_key_value_heads // tp_size + for chunk in infer_param.chunk(num_query_groups_per_partition): + split_size = [ + kv_size_per_tp * num_q_per_kv // num_query_groups_per_partition, + kv_size_per_tp // num_query_groups_per_partition, + kv_size_per_tp // num_query_groups_per_partition, + ] + q, k, v = chunk.split(split_size) + q_lst.append(q) + k_lst.append(k) + v_lst.append(v) + + q = torch.cat(q_lst, dim=0) + k = torch.cat(k_lst, dim=0) + v = torch.cat(v_lst, dim=0) + return [q, k, v] + elif "layer_norm" in key or "layernorm" in key or "router" in key or ("output_layer" in key and is_value_model): + return tp_data[0] + else: + dim = 0 + if "linear_fc2.weight" in key or "self_attention.linear_proj" in key: + dim = 1 + return torch.cat(tp_data, dim=dim) + + def _load_state_dicts( + self, model_ckpt_path: str, sharded_dirs: list[str], tp_size: int, pp_size: int + ) -> list[list[dict]]: + model_state_dict_lst = [[None for _ in range(tp_size)] for _ in range(pp_size)] + + def _process_one_megatron_shard(sharded_dir: str): + model_file_path = Path(model_ckpt_path) / sharded_dir / "model.pt" + state_dict = torch.load(model_file_path, map_location="cpu", weights_only=False) + tp_rank, pp_rank = self._get_tp_pp_rank_from_sharded_dir(sharded_dir) + model_state_dict_lst[pp_rank][tp_rank] = state_dict + + with ThreadPoolExecutor(max_workers=min(32, os.cpu_count())) as executor: + futures = [executor.submit(_process_one_megatron_shard, sharded_dir) for sharded_dir in sharded_dirs] + for future in tqdm(futures, desc=f"Loading {len(sharded_dirs)} Megatron shards", total=len(sharded_dirs)): + future.result() + + return model_state_dict_lst + + def _check_megatron_state_key(self, key: str) -> bool: + """ + Checks if the key is a valid Megatron state key. + + Now the model merger only supports keys that start with "decoder/embedding/output_layer" in TransformerLayer. + Shall not use key starts with "model." + """ + if key.startswith("model."): + raise ValueError( + f"Invalid key {key} in Megatron state_dict. Expected keys to start with 'decoder/embedding/output_layer' in TransformerLayer." + ) + + skip_checking_keys = ["embedding.word_embeddings", "output_layer"] + for skip_key in skip_checking_keys: + if skip_key in key: + print(f"skip checking key {key}") + return + + # Exclude extra state keys + if not key.startswith("decoder"): + raise ValueError( + f"Invalid key {key} in Megatron state_dict. Expected keys to start with 'decoder' in TransformerLayer." + ) + + def _merge_state_dicts( + self, model_state_dict_lst: list[list[dict]], tp_size: int, pp_size: int + ) -> dict[str, torch.Tensor]: + state_dict = {} + vpp_size = len(model_state_dict_lst[0][0]) + layers_cum = 0 + + for vpp_rank in range(vpp_size): + for pp_rank in range(pp_size): + layers_handled = 0 + keys = model_state_dict_lst[pp_rank][0][vpp_rank].keys() + for key in keys: + if "extra_state" in key: + continue + if self.config.tie_word_embedding and ("output_layer" in key): + print("skip lm_head and reward_head loading because of tie_word_embeddings") + continue + + self._check_megatron_state_key(key) + hf_name = self._replace_name(key, self.params_mapping) + assert hf_name is not None, f"Failed to convert layer name [{key}] from megatron to huggingface." + if "model.layers." in hf_name: + local_layer_no = int(hf_name.split(".")[2]) + layers_handled = max(local_layer_no, layers_handled) + global_layer_no = local_layer_no + layers_cum + new_key_list = hf_name.split(".") + new_key_list[2] = str(global_layer_no) + hf_name = ".".join(new_key_list) + else: + warnings.warn(f"hf_name {hf_name} will not be fixed with layer number", stacklevel=2) + + tp_data = [model_state_dict_lst[pp_rank][tp_rank][vpp_rank][key] for tp_rank in range(tp_size)] + merged = self._merge_across_tp(key, tp_data, self.model_config, tp_size, self.config.is_value_model) + + if not isinstance(merged, list): + state_dict[hf_name] = merged + elif len(merged) == 3: + # split qkv + for n, d in zip(["q", "k", "v"], merged): + state_dict[hf_name.replace("qkv", n)] = d + elif len(merged) == 2: + # split gate up + state_dict[hf_name.replace("gate_up", "gate")] = merged[0] + state_dict[hf_name.replace("gate_up", "up")] = merged[1] + print( + f"converted {key} to {hf_name} with shape {merged.shape if isinstance(merged, torch.Tensor) else [t.shape for t in merged]}" + ) + + layers_cum += layers_handled + 1 # zero based + + return state_dict + + def merge_and_save(self): + from verl.utils.megatron_utils import get_model_checkpoint_path + + model_ckpt_path = get_model_checkpoint_path(self.config.local_dir) + sharded_dirs, tp_size, pp_size = self._check_megatron_checkpoint_path(model_ckpt_path) + print(f"sharded_dirs: {sharded_dirs}, tp_size: {tp_size}, pp_size: {pp_size}, mp_size: {len(sharded_dirs)}") + + model_state_dict_lst = self._load_state_dicts(model_ckpt_path, sharded_dirs, tp_size, pp_size) + merged_state_dict = self._merge_state_dicts(model_state_dict_lst, tp_size, pp_size) + del model_state_dict_lst + + if self.config.operation == "test": + if not self.config.test_hf_dir: + raise ValueError("test_hf_dir must be provided for test operation") + self._test_state_dict(merged_state_dict) + elif self.config.operation == "merge": + self.save_hf_model_and_tokenizer(merged_state_dict) + if self.config.hf_upload: + self.upload_to_huggingface() + else: + raise ValueError(f"Unknown operation: {self.config.operation}") + + def _test_state_dict(self, state_dict: dict[str, torch.Tensor]): + """ + Compares the merged Megatron state_dict against a reference safetensors model. + Applies necessary name mappings from Megatron to Hugging Face conventions using _replace_name. + """ + ref_state_dict = load_file(Path(self.config.test_hf_dir) / "model.safetensors") + + for name, loaded_weight in state_dict.items(): + # name = self._replace_name(original_name, self.params_mapping) + if not name or name.endswith(".bias") and name not in ref_state_dict: + continue + if "rotary_emb.inv_freq" in name: + continue + if self.config.tie_word_embedding and "lm_head.weight" in name: + continue + if name not in ref_state_dict: + raise RuntimeError(f"key: {name} not exist in state_dict") + param = ref_state_dict[name] + assert loaded_weight.dtype == param.dtype + torch.testing.assert_close(loaded_weight, param, atol=1e-2, rtol=5e-2) + + def _replace_name(self, megatron_name: str, name_mapping: dict[str, str]) -> str: + for m_name, v_name in name_mapping.items(): + if m_name not in megatron_name: + continue + + megatron_name = megatron_name.replace("decoder", "model") + param_name = megatron_name.replace(m_name, v_name) + return param_name + + return None # Return None if no mapping found + + +def main(): + parser = argparse.ArgumentParser(description="verl model merger") + subparsers = parser.add_subparsers(dest="operation", required=True, help="Specify 'merge' or 'test' operation.") + + base_op_parser = argparse.ArgumentParser(add_help=False) + base_op_parser.add_argument( + "--backend", type=str, required=True, choices=["fsdp", "megatron"], help="The backend of the model" + ) + base_op_parser.add_argument("--local_dir", type=str, required=True, help="Path to the saved model checkpoints") + base_op_parser.add_argument( + "--hf_model_path", + type=str, + default=None, + help="(Deprecated) Path to the original Hugging Face model for config.", + ) + base_op_parser.add_argument( + "--tie-word-embedding", + action="store_true", + help="Whether to tie word embedding weights (currently only Megatron supported)", + ) + base_op_parser.add_argument( + "--is-value-model", + action="store_true", + help="Whether the model is a value model (currently only Megatron supported)", + ) + + merge_parser = subparsers.add_parser("merge", parents=[base_op_parser], help="Merge model checkpoints and save.") + merge_parser.add_argument( + "--target_dir", default="tmp", type=str, help="Directory to save the merged huggingface model" + ) + merge_parser.add_argument( + "--hf_upload_path", default=None, type=str, help="Hugging Face repository ID to upload the model" + ) + merge_parser.add_argument( + "--private", action="store_true", help="Whether to upload the model to a private Hugging Face repository" + ) + + test_parser = subparsers.add_parser( + "test", parents=[base_op_parser], help="Test merged model against a reference Hugging Face model" + ) + test_parser.add_argument( + "--test_hf_dir", type=str, required=True, help="Path to the reference Hugging Face model directory for testing" + ) + + args = parser.parse_args() + + common_config_args = { + "operation": args.operation, + "backend": args.backend, + "tie_word_embedding": args.tie_word_embedding, + "is_value_model": args.is_value_model, + "local_dir": args.local_dir, + "hf_model_path": args.hf_model_path, + "hf_model_config_path": args.local_dir, + } + + if args.operation == "merge": + config = ModelMergerConfig( + **common_config_args, + target_dir=args.target_dir, + hf_upload_path=args.hf_upload_path, + private=args.private, + test_hf_dir=None, + ) + os.makedirs(config.target_dir, exist_ok=True) + elif args.operation == "test": + config = ModelMergerConfig( + **common_config_args, + test_hf_dir=args.test_hf_dir, + # the following args are not used by test operation + target_dir=None, + hf_upload_path=None, + private=False, + ) + else: + raise NotImplementedError(f"Unknown operation: {args.operation}") + + if config.backend == "fsdp": + merger = FSDPModelMerger(config) + elif config.backend == "megatron": + merger = MegatronModelMerger(config) + else: + raise NotImplementedError(f"Unknown backend: {config.backend}") + + merger.merge_and_save() + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/scripts/print_cfg.py b/e2c/verl/scripts/print_cfg.py new file mode 100644 index 0000000000000000000000000000000000000000..287756fb1b7dbaac84b5f7ec572ba7a172e347b3 --- /dev/null +++ b/e2c/verl/scripts/print_cfg.py @@ -0,0 +1,35 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +try: + import hydra +except ImportError as e: + raise ImportError("Please install hydra-core via 'pip install hydra-core' and retry.") from e + + +@hydra.main(config_path="../verl/trainer/config", config_name="ppo_trainer", version_base=None) +def main(config): + """Main entry point for PPO training with Hydra configuration management. + + Args: + config_dict: Hydra configuration dictionary containing training parameters. + """ + print(config) + from verl.utils.config import omega_conf_to_dataclass + + profiler_config = omega_conf_to_dataclass(config.critic.profiler) + print(profiler_config) + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/scripts/rollout_viewer.py b/e2c/verl/scripts/rollout_viewer.py new file mode 100644 index 0000000000000000000000000000000000000000..1a1d1a010156ed1b498a633672283c2d3a5732f0 --- /dev/null +++ b/e2c/verl/scripts/rollout_viewer.py @@ -0,0 +1,565 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import re +import traceback +from pathlib import Path +from typing import Annotated, Optional + +import aiofiles + +try: + import ujson as json +except ImportError: + import json +import typer +from rich.highlighter import ReprHighlighter +from rich.markdown import Markdown +from rich.table import Table +from rich.text import Text +from textual import on +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical, VerticalScroll +from textual.widgets import Input, ProgressBar, Select, SelectionList, Static + +INDEX_KEY = "__IDX" +FILE_SUFFIX = ".jsonl" + + +def check_textual_version(): + # check if textual version is equal to 0.52.1 + import textual + from packaging.version import Version + + if Version(textual.__version__) != Version("0.52.1"): + raise ImportError(f"Textual version {textual.__version__} is not supported, please pip install textual==0.52.1") + + +check_textual_version() + + +async def load_path(p: Path, data: dict, mask_strs: str, idx: int, pbar): + samples = [] + async with aiofiles.open(p, encoding="utf-8") as f: + async for line in f: + d = json.loads(line) + for k in d: + if isinstance(d[k], str): + if mask_strs: + d[k] = re.sub(rf"{mask_strs}", "*", d[k]) + else: + d[k] = json.dumps(d[k], ensure_ascii=False, indent=4) + + d[INDEX_KEY] = len(samples) + samples.append(d) + data[idx] = {"samples": samples} + + print(f"path {p} loaded") + pbar.advance(1) + + +async def load_dir(path: Path, data: dict[int, dict], pbar, mask_strs: str = ""): + paths = list(path.glob(f"*{FILE_SUFFIX}")) + paths = sorted(paths, key=lambda x: int(x.stem)) + + tasks = [load_path(p, data, mask_strs, i, pbar) for i, p in enumerate(paths)] + + await asyncio.gather(*tasks) + + +class Highlighter(ReprHighlighter): + highlights = ReprHighlighter.highlights + [ + r"(?P[][\<\>{}()\|()【】\[\]=`])", + r"\<\|(?P[\w\W]*?)\|\>", + ] + + +def center_word_with_equals_exactly(word: str, total_length: int, char: str = "=") -> str: + if len(word) > total_length: + return word + + padding = total_length - len(word) + left_pad = (padding) // 2 + right_pad = (padding + 1) // 2 + return char * left_pad + " " + word + " " + char * right_pad + + +def highlight_keyword(content: str, keyword: Optional[str]): + if not keyword: + return Text(content) + text = Text() + parts = content.split(keyword) + for i, part in enumerate(parts): + text.append(part, style=None) + if i < len(parts) - 1: + # text.append(keyword, style=Style(color="#d154d1", bgcolor="yellow", bold=True)) + text.append(keyword, style="on #8f51b5") + return text + + +help_doc = """ +⌨️ keybinds: + +- `f/esc`: find/cancel +- `tab/←/→`: change focus +- `j/k`: page down/up +- `g/G`: scroll home/end +- `n/N`: next sample/step +- `p/P`: previous sample/step +- `s`: switch display mode + - plain text + - rich table + +""" + + +class JsonLineViewer(App): + BINDINGS = [ + ("left", "focus_previous", "Focus Previous"), + ("right", "focus_next", "Focus Next"), + ("s", "swith_render", "switch render"), + # control + ("n", "next_sample", "Next Sample"), + ("N", "next_step", "Next Step"), + ("p", "previous_sample", "Previous Sample"), + ("P", "previous_step", "Previous Step"), + # search + ("f", "toggle_search", "find"), + ("enter", "next_search", "find next"), + ("escape", "cancel_search", "cancel find"), + # scroll + ("j", "page_down", "page down"), + ("k", "page_up", "page up"), + ("g", "page_home", "page home"), + ("G", "page_end", "page end"), + ] + + CSS = """ + + Select:focus > SelectCurrent { + border: tall #8f51b5; + } + Select.-expanded > SelectCurrent { + border: tall #8f51b5; + } + #select-container { + width: 15%; + height: 100%; + align: center top; + } + #search-container { + height: 10%; + align: center top; + } + #search-box { + width: 50%; + } + #reqid-box { + width: 50%; + } + """ + + def __init__(self, step_num: int, data: dict[int, dict], pbar): + super().__init__() + self.step_num = step_num + + self.data = data + self.render_table = False + self.selected_step_index = 0 + self.selected_sample_index = 0 + self.pbar = pbar + + self.matches = [] + self.current_match_index = 0 + + self.highlighter = Highlighter() + + first_samples = data[list(data.keys())[0]]["samples"] + # Prepare the initial field filter list (all keys from the first sample) + self.filter_fields = [(f, f, True) for f in first_samples[0].keys()] + + # Internal set used for fast membership checks when we add new fields on the fly. + # We keep it here so that when new columns appear in later steps (e.g. `request_id`), + # they can be added to the UI automatically without restarting the viewer. + self._field_set: set[str] = set(first_samples[0].keys()) + self.sample_num = len(first_samples) + + def compose(self) -> ComposeResult: + with Horizontal(id="search-container"): + yield Input(placeholder="find something...", id="search-box") + yield Input(placeholder="request id...", id="reqid-box") + with Vertical(id="search-container2"): + yield self.pbar + yield Static("", id="search-status") + + with Horizontal(): + with Vertical(id="select-container"): + yield Static("\n") + yield Static( + renderable=Markdown( + help_doc, + ), + markup=False, + ) + yield Static("\n") + yield Select( + id="step-select", + value=0, + prompt="select step", + options=[("step: 1", 0)], + allow_blank=False, + ) + yield Select( + id="sample-select", + value=0, + prompt="select sample", + options=[("sample: 1", 0)], + allow_blank=False, + ) + yield Select( + id="sample-sort", + value=0, + prompt="排序", + options=[ + ("sort", 0), + ("score asc", 1), + ("score desc", 2), + ], + allow_blank=False, + ) + + yield SelectionList[int](("Select ALL", 1, True), id="fields-select-all") + with VerticalScroll(id="scroll-view2"): + yield SelectionList[str](*self.filter_fields, id="fields-select") + with VerticalScroll(id="scroll-view"): + yield Static(id="content", markup=False) + + async def on_mount(self) -> None: + self.step_select = self.query_one("#step-select", Select) + self.sample_select = self.query_one("#sample-select", Select) + self.sample_sort = self.query_one("#sample-sort", Select) + self.content_display = self.query_one("#content", Static) + self.search_box = self.query_one("#search-box", Input) + self.reqid_box = self.query_one("#reqid-box", Input) + self.scroll_view = self.query_one("#scroll-view", VerticalScroll) + self.search_status = self.query_one("#search-status", Static) + self.fields_select = self.query_one("#fields-select", SelectionList) + self.fields_select.border_title = "field filter" + + if self.data: + self.step_select.set_options([(f"step: {i + 1}", i) for i in range(self.step_num)]) + self.sample_select.set_options([(f"sample: {i + 1}", i) for i in range(self.sample_num)]) + self.step_select.focus() + await self.update_content() + + def update_result_options(self, offset: int = 0, sort_desc: Optional[bool] = None): + options = [] + if isinstance(self.selected_step_index, int) and self.selected_step_index < len(self.data): + if self.sample_num is None or sort_desc is not None: + samples = self.data[self.selected_step_index].get("samples", []) + if not samples: + self.selected_sample_index = offset + return + if sort_desc is not None: + samples = sorted( + samples, + key=lambda x: x.get("score", x.get("score_1", 0)), + reverse=sort_desc, + ) + + options = [(f"sample: {r[INDEX_KEY] + 1}", r[INDEX_KEY]) for r in samples] + self.sample_select.set_options(options) + self.sample_num = len(samples) + + if sort_desc is not None and options: + self.selected_sample_index = options[0][1] + else: + self.selected_sample_index = offset + + async def update_content(self, search_keyword: Optional[str] = None): + content = "" + try: + samples = self.data[self.selected_step_index].get("samples", []) + content_dict_full = samples[self.selected_sample_index] + + # Dynamically track any NEW keys that appear and add them to the field filter. + self._update_fields_select(content_dict_full.keys()) + + # Apply field selection filter (only show selected fields) + content_dict = {k: v for k, v in content_dict_full.items() if k in self.fields_select.selected} + if self.render_table: + content = Table("key", "value", show_lines=True) + for k in content_dict: + v = content_dict[k] + v = f"{v}" + content.add_row( + k, + self.highlighter(highlight_keyword(v, search_keyword)), + ) + else: + text = Text() + for k in content_dict: + v = content_dict[k] + s = center_word_with_equals_exactly(k, 64) + f"\n{v}\n" + text.append(highlight_keyword(s, search_keyword)) + content = self.highlighter(text) + except KeyError: + content = f"Loading data asynchronously, progress: {len(self.data)}/{self.step_num} step" + + except Exception: + content = self.highlighter(traceback.format_exc()) + + self.content_display.update(content) + + # --------------------------------------------------------------------- + # Request-ID jump logic + # --------------------------------------------------------------------- + + @on(Input.Submitted, "#reqid-box") + async def on_reqid_submitted(self, event: Input.Submitted) -> None: + """Jump to the sample that has a matching `request_id`.""" + + req_id_raw = event.value.strip() + # Remove hyphens so search is tolerant to different id formats + req_id = req_id_raw.replace("-", "") + if not req_id: + return + + found = False + for step_idx, step_data in self.data.items(): + for sample in step_data.get("samples", []): + sample_id = str(sample.get("request_id", "")) + if sample_id.replace("-", "") == req_id: + # Update selected indices + self.selected_step_index = step_idx + self.step_select.value = step_idx + + # Ensure sample list is updated and select sample + self.update_result_options(offset=sample[INDEX_KEY]) + self.selected_sample_index = sample[INDEX_KEY] + self.sample_select.value = sample[INDEX_KEY] + + await self._clear_search() + await self.update_content() + + found = True + break + if found: + break + + if not found: + self.search_status.update(Text(f"request_id '{req_id_raw}' not found", style="bold red")) + else: + # Keep the typed id in the input box so users see what was searched. + pass + + # --------------------------------------------------------------------- + # Helper: add new fields to SelectionList on-the-fly + # --------------------------------------------------------------------- + + def _update_fields_select(self, keys): + """Add any unseen *keys* to the field-selection widget so they can be toggled. + + The viewer is often launched with only the first step loaded. Later steps may + introduce new columns (e.g. `request_id`). This helper ensures those fields + become visible without requiring a restart. + """ + # Ensure we have the widget (only after on_mount) + if not hasattr(self, "fields_select"): + return + + for k in keys: + if k not in self._field_set: + self._field_set.add(k) + try: + # By default, new fields are selected so they appear immediately. + self.fields_select.add_option(k, k, selected=True) + except Exception: + # Fallback for older textual versions where signature is different. + self.fields_select.add_option((k, k, True)) + + @on(Select.Changed, "#step-select") + async def step_changed(self, event): + self.selected_step_index = event.value + self.update_result_options() + await self.update_content() + + @on(Select.Changed, "#sample-select") + async def sample_changed(self, event): + self.selected_sample_index = event.value + await self._clear_search() + await self.update_content() + + @on(Select.Changed, "#sample-sort") + async def sort_changed(self, event): + v = event.value + self.update_result_options(sort_desc=None if v == 0 else False if v == 1 else True) + await self.update_content() + + @on(SelectionList.SelectedChanged, "#fields-select") + async def fields_changed(self, event): + await self.update_content() + + @on(SelectionList.SelectedChanged, "#fields-select-all") + async def fields_all_changed(self, event): + s = self.query_one("#fields-select-all", SelectionList) + if s.selected: + self.fields_select.select_all() + else: + self.fields_select.deselect_all() + + def action_focus_previous(self): + self.screen.focus_previous() + + def action_focus_next(self): + self.screen.focus_next() + + async def action_next_step(self) -> None: + self.selected_step_index += 1 + if self.selected_step_index >= self.step_num: + self.selected_step_index = 0 + self.step_select.value = self.selected_step_index + self.update_result_options() + await self.update_content() + + async def action_next_sample(self) -> None: + self.selected_sample_index += 1 + if not self.sample_num or self.selected_sample_index >= self.sample_num: + self.selected_sample_index = 0 + self.sample_select.value = self.selected_sample_index + await self._clear_search() + await self.update_content() + + async def action_previous_step(self) -> None: + self.selected_step_index -= 1 + if self.selected_step_index < 0: + self.selected_step_index = self.step_num - 1 + self.step_select.value = self.selected_step_index + self.update_result_options() + await self.update_content() + + async def action_previous_sample(self) -> None: + self.selected_sample_index -= 1 + if self.selected_sample_index < 0: + self.selected_sample_index = self.sample_num - 1 + self.sample_select.value = self.selected_sample_index + await self._clear_search() + await self.update_content() + + async def action_swith_render(self): + self.render_table = not self.render_table + await self.update_content() + + def action_toggle_search(self) -> None: + self.search_box.focus() + + async def action_cancel_search(self) -> None: + self.search_box.value = "" + await self._clear_search() + await self.update_content() + + async def _clear_search(self): + self.matches = [] + self.search_status.update("") + self.current_match_index = 0 + + @on(Input.Submitted, "#search-box") + async def on_search_submitted(self, event: Input.Submitted) -> None: + self.matches = [] + self.current_match_index = 0 + if event.value: + await self.update_content(event.value) + renderable = self.content_display.render() + if isinstance(renderable, Table): + return + + assert isinstance(renderable, Text) + console = self.content_display._console + lines = renderable.wrap(console, self.scroll_view.container_size.width) + line_idx_recorded = set() + for line_idx, line in enumerate(lines): + if line_idx in line_idx_recorded: + continue + if event.value in line: + self.matches.append( + { + "line": line_idx, + "word": event.value, + } + ) + line_idx_recorded.add(line_idx) + self.scroll_view.focus() + await self.action_next_search() + + async def action_next_search(self) -> None: + if not self.matches or self.current_match_index >= len(self.matches): + return + + target_line = self.matches[self.current_match_index]["line"] + self.scroll_view.scroll_to(x=0, y=target_line * 1, animate=False) + self.current_match_index = (self.current_match_index + 1) % len(self.matches) + self.search_status.update( + Text( + f"Find :{self.current_match_index + 1}/{len(self.matches)}", + style="bold on #8f51b5", + ) + ) + + def action_page_up(self): + self.scroll_view.scroll_page_up(animate=False) + + def action_page_down(self): + self.scroll_view.scroll_page_down(animate=False) + + def action_page_home(self): + self.scroll_view.scroll_home(animate=False) + + def action_page_end(self): + self.scroll_view.scroll_end(animate=False) + + +async def _run(path: Path, mask_str: str): + assert path.exists(), f"{path} not exist" + + paths = list(path.glob(f"*{FILE_SUFFIX}")) + paths = sorted(paths, key=lambda x: int(x.stem)) + + if not paths: + raise ValueError(f"no available reward dump files under f{path}") + + print(f"get jsonl file nums: {len(paths)}") + + pbar = ProgressBar(total=len(paths), name="data load progress") + data = {} + await load_path(paths[0], data, mask_str, 0, pbar) + app = JsonLineViewer(step_num=len(paths), data=data, pbar=pbar) + await asyncio.gather(load_dir(path, data, pbar, mask_str), app.run_async()) + + +app = typer.Typer() + + +@app.command(help="launch TUI APP") +def run( + rollout_data_dir: Path, + mask_str: Annotated[str, typer.Option(help="string that will be masked to *")] = "<\|image_pad\|>|<\|imgpad\|>", +): + loop = asyncio.get_event_loop() + loop.run_until_complete(_run(rollout_data_dir, mask_str)) + + +if __name__ == "__main__": + app() diff --git a/e2c/verl/setup.py b/e2c/verl/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..4a86f035d094c0612e534575441f531497ccbad7 --- /dev/null +++ b/e2c/verl/setup.py @@ -0,0 +1,96 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# setup.py is the fallback installation script when pyproject.toml does not work +import os +from pathlib import Path + +from setuptools import find_packages, setup + +version_folder = os.path.dirname(os.path.join(os.path.abspath(__file__))) + +with open(os.path.join(version_folder, "verl/version/version")) as f: + __version__ = f.read().strip() + +install_requires = [ + "accelerate", + "codetiming", + "datasets", + "dill", + "hydra-core", + "numpy<2.0.0", + "pandas", + "peft", + "pyarrow>=19.0.0", + "pybind11", + "pylatexenc", + "ray[default]>=2.41.0", + "torchdata", + "tensordict>=0.8.0,<=0.10.0,!=0.9.0", + "transformers", + "wandb", + "packaging>=20.0", + "tensorboard", +] + +TEST_REQUIRES = ["pytest", "pre-commit", "py-spy", "pytest-asyncio"] +PRIME_REQUIRES = ["pyext"] +GEO_REQUIRES = ["mathruler", "torchvision", "qwen_vl_utils"] +GPU_REQUIRES = ["liger-kernel", "flash-attn"] +MATH_REQUIRES = ["math-verify"] # Add math-verify as an optional dependency +VLLM_REQUIRES = ["tensordict>=0.8.0,<=0.10.0,!=0.9.0", "vllm>=0.7.3,<=0.9.1"] +SGLANG_REQUIRES = [ + "tensordict>=0.8.0,<=0.10.0,!=0.9.0", + "sglang[srt,openai]==0.5.2", + "torch==2.8.0", +] +TRL_REQUIRES = ["trl<=0.9.6"] +MCORE_REQUIRES = ["mbridge"] + +extras_require = { + "test": TEST_REQUIRES, + "prime": PRIME_REQUIRES, + "geo": GEO_REQUIRES, + "gpu": GPU_REQUIRES, + "math": MATH_REQUIRES, + "vllm": VLLM_REQUIRES, + "sglang": SGLANG_REQUIRES, + "trl": TRL_REQUIRES, + "mcore": MCORE_REQUIRES, +} + + +this_directory = Path(__file__).parent +long_description = (this_directory / "README.md").read_text() + +setup( + name="verl", + version=__version__, + package_dir={"": "."}, + packages=find_packages(where="."), + url="https://github.com/volcengine/verl", + license="Apache 2.0", + author="Bytedance - Seed - MLSys", + author_email="zhangchi.usc1992@bytedance.com, gmsheng@connect.hku.hk", + description="verl: Volcano Engine Reinforcement Learning for LLM", + install_requires=install_requires, + extras_require=extras_require, + package_data={ + "": ["version/*"], + "verl": ["trainer/config/*.yaml"], + }, + include_package_data=True, + long_description=long_description, + long_description_content_type="text/markdown", +) diff --git a/e2c/verl/tests/README.md b/e2c/verl/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..479f06933e4e536ee159b738794daa05364119bb --- /dev/null +++ b/e2c/verl/tests/README.md @@ -0,0 +1,30 @@ +# Tests layout + +Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +- `tests/trainer` for testing functionality related to `verl/trainer` +- `tests/models` for testing functionality related to `verl/models` +- ... + +There are a few folders with `special_` prefix, created for special purposes: +- `special_distributed`: unit tests that must run with multiple GPUs +- `special_e2e`: end-to-end tests with training/generation scripts +- `special_npu`: tests for NPUs +- `special_sanity`: a suite of quick sanity tests +- `special_standalone`: a set of test that are designed to run in dedicated environments + +Accelerators for tests +- By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +- For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# Workflow layout + +All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +3. End-to-end tests: `e2e_*.yml` +4. Unit tests + - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` + - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. + - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when + - new workflow yaml is added to `.github/workflows` + - new tests are added to workflow mentioned in 2. \ No newline at end of file diff --git a/e2c/verl/tests/__init__.py b/e2c/verl/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/tests/experimental/agent_loop/agent_utils.py b/e2c/verl/tests/experimental/agent_loop/agent_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fa4504c6af0cf68f3b1090ae113e350838fab1c9 --- /dev/null +++ b/e2c/verl/tests/experimental/agent_loop/agent_utils.py @@ -0,0 +1,93 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ray +from omegaconf import DictConfig + +from verl.experimental.agent_loop import AgentLoopManager +from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup +from verl.single_controller.ray.base import create_colocated_worker_cls +from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role +from verl.workers.fsdp_workers import ActorRolloutRefWorker, AsyncActorRolloutRefWorker, RewardModelWorker + + +def init_agent_loop_manager(config: DictConfig) -> AgentLoopManager | RayWorkerGroup: + # =========================== 1. Create hybrid ActorRollout workers =========================== + actor_rollout_cls = ( + AsyncActorRolloutRefWorker if config.actor_rollout_ref.rollout.mode == "async" else ActorRolloutRefWorker + ) + role_worker_mapping = { + Role.ActorRollout: ray.remote(actor_rollout_cls), + } + if config.reward_model.enable: + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + } + if config.reward_model.enable_resource_pool: + mapping[Role.RewardModel] = "reward_pool" + if config.reward_model.n_gpus_per_node <= 0: + raise ValueError("config.reward_model.n_gpus_per_node must be greater than 0") + if config.reward_model.nnodes <= 0: + raise ValueError("config.reward_model.nnodes must be greater than 0") + + reward_pool = [config.reward_model.n_gpus_per_node] * config.reward_model.nnodes + resource_pool_spec["reward_pool"] = reward_pool + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + resource_pool_manager.create_resource_pool() + resource_pool_to_cls = {pool: {} for pool in resource_pool_manager.resource_pool_dict.values()} + + # create actor and rollout + resource_pool = resource_pool_manager.get_resource_pool(Role.ActorRollout) + actor_rollout_cls = RayClassWithInitArgs( + cls=role_worker_mapping[Role.ActorRollout], config=config.actor_rollout_ref, role="actor_rollout" + ) + resource_pool_to_cls[resource_pool]["actor_rollout"] = actor_rollout_cls + + if config.reward_model.enable: + # we create a RM here + resource_pool = resource_pool_manager.get_resource_pool(Role.RewardModel) + rm_cls = RayClassWithInitArgs(role_worker_mapping[Role.RewardModel], config=config.reward_model) + resource_pool_to_cls[resource_pool]["rm"] = rm_cls + + all_wg = {} + for resource_pool, class_dict in resource_pool_to_cls.items(): + worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + wg_dict = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=worker_dict_cls) + spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + all_wg.update(spawn_wg) + actor_rollout_wg = all_wg["actor_rollout"] + actor_rollout_wg.init_model() + + if config.actor_rollout_ref.rollout.mode == "sync": + return actor_rollout_wg + + if config.reward_model.enable_resource_pool and config.reward_model.enable: + rm_wg = all_wg["rm"] + rm_wg.init_model() + else: + rm_wg = None + # =========================== 2. Create AgentLoopManager =========================== + agent_loop_manager = AgentLoopManager( + config=config, + worker_group=actor_rollout_wg, + rm_wg=rm_wg, + ) + + return agent_loop_manager diff --git a/e2c/verl/tests/experimental/agent_loop/qwen_vl_tool_chat_template.jinja2 b/e2c/verl/tests/experimental/agent_loop/qwen_vl_tool_chat_template.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..9fea57ff86b54917ff806a28b3617bb79517c494 --- /dev/null +++ b/e2c/verl/tests/experimental/agent_loop/qwen_vl_tool_chat_template.jinja2 @@ -0,0 +1,150 @@ +{% set image_count = namespace(value=0) %} +{% set video_count = namespace(value=0) %} +{%- if tools %} +{{- '<|im_start|>system\n' }} +{%- if messages[0]['role'] == 'system' %} +{%- if messages[0]['content'] is string %} +{{- messages[0]['content'] }} +{%- else %} +{{- messages[0]['content'][0]['text'] }} +{%- endif %} +{%- else %} +{{- 'You are a helpful assistant.' }} +{%- endif %} +{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} +{%- for tool in tools %} +{{- "\n" }} +{{- tool | tojson }} +{%- endfor %} +{{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{% for message in messages %} +{% if message['role'] != 'system' or loop.first == false %} +{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} +<|im_start|>{{ message['role'] }} +{% if message['content'] is string %} +{{ message['content'] }}<|im_end|> +{% else %} +{% for content in message['content'] %} +{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %} +{% set image_count.value = image_count.value + 1 %} +{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|> +{% elif content['type'] == 'video' or 'video' in content %} +{% set video_count.value = video_count.value + 1 %} +{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|> +{% elif 'text' in content %} +{{ content['text'] }} +{% endif %} +{% endfor %}<|im_end|> +{% endif %} +{%- elif message.role == "assistant" %} +{{- '<|im_start|>' + message.role }} +{%- if message.content %} +{{- '\n' + message.content }} +{%- endif %} +{%- for tool_call in message.tool_calls %} +{%- if tool_call.function is defined %} +{%- set tool_call = tool_call.function %} +{%- endif %} +{{- '\n\n{"name": "' }} +{{- tool_call.name }} +{{- '", "arguments": ' }} +{{- tool_call.arguments | tojson }} +{{- '}\n' }} +{%- endfor %} +{{- '<|im_end|>\n' }} +{%- elif message.role == "tool" %} +{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} +{{- '<|im_start|>user' }} +{%- endif %} +{{- '\n\n' }} +{% if message['content'] is string %} +{{ message.content }} +{% else %} +{% for content in message['content'] %} +{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %} +{% set image_count.value = image_count.value + 1 %} +{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|> +{% elif content['type'] == 'video' or 'video' in content %} +{% set video_count.value = video_count.value + 1 %} +{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|> +{% elif content['type'] == 'text' or 'text' in content %} +{{ content['text'] }} +{% endif %} +{% endfor %} +{% endif %} +{{- '\n' }} +{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} +{{- '<|im_end|>\n' }} +{%- endif %} +{%- endif %} +{% endif %} +{% endfor %} +{%- else %} +{% for message in messages %} +{% if loop.first and message['role'] != 'system' %} +<|im_start|>system +You are a helpful assistant.<|im_end|> +{% endif %} +{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} +<|im_start|>{{ message['role'] }} +{% if message['content'] is string %} +{{ message['content'] }}<|im_end|> +{% else %} +{% for content in message['content'] %} +{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %} +{% set image_count.value = image_count.value + 1 %} +{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|> +{% elif content['type'] == 'video' or 'video' in content %} +{% set video_count.value = video_count.value + 1 %} +{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|> +{% elif 'text' in content %} +{{ content['text'] }} +{% endif %} +{% endfor %}<|im_end|> +{% endif %} +{%- elif message.role == "assistant" %} +{{- '<|im_start|>' + message.role }} +{%- if message.content %} +{{- '\n' + message.content }} +{%- endif %} +{%- for tool_call in message.tool_calls %} +{%- if tool_call.function is defined %} +{%- set tool_call = tool_call.function %} +{%- endif %} +{{- '\n\n{"name": "' }} +{{- tool_call.name }} +{{- '", "arguments": ' }} +{{- tool_call.arguments | tojson }} +{{- '}\n' }} +{%- endfor %} +{{- '<|im_end|>\n' }} +{%- elif message.role == "tool" %} +{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} +{{- '<|im_start|>user' }} +{%- endif %} +{{- '\n\n' }} +{% if message['content'] is string %} +{{ message.content }} +{% else %} +{% for content in message['content'] %} +{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %} +{% set image_count.value = image_count.value + 1 %} +{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|> +{% elif content['type'] == 'video' or 'video' in content %} +{% set video_count.value = video_count.value + 1 %} +{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|> +{% elif content['type'] == 'text' or 'text' in content %} +{{ content['text'] }} +{% endif %} +{% endfor %} +{% endif %} +{{- '\n' }} +{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} +{{- '<|im_end|>\n' }} +{%- endif %} +{%- endif %} +{% endfor %} +{%- endif %} +{% if add_generation_prompt %} +<|im_start|>assistant +{% endif %} \ No newline at end of file diff --git a/e2c/verl/tests/experimental/agent_loop/test_agent_loop_reward.py b/e2c/verl/tests/experimental/agent_loop/test_agent_loop_reward.py new file mode 100644 index 0000000000000000000000000000000000000000..2716d0439fe24079f5de0a573b0493dd85227c96 --- /dev/null +++ b/e2c/verl/tests/experimental/agent_loop/test_agent_loop_reward.py @@ -0,0 +1,91 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import ray +from hydra import compose, initialize_config_dir +from torchdata.stateful_dataloader import StatefulDataLoader +from transformers import AutoTokenizer + +from verl.experimental.agent_loop import AgentLoopManager +from verl.protocol import DataProto +from verl.trainer.main_ppo import create_rl_sampler +from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn + + +def test_agent_loop_compute_score(): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + } + ) + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + config = compose("ppo_trainer") + + model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-1.5B-Instruct") + config.data.return_raw_chat = True + config.actor_rollout_ref.model.path = model_path + config.actor_rollout_ref.actor.use_dynamic_bsz = True + config.actor_rollout_ref.rollout.name = os.environ["ROLLOUT_NAME"] + config.actor_rollout_ref.rollout.mode = "async" + config.actor_rollout_ref.rollout.enforce_eager = True + config.actor_rollout_ref.rollout.prompt_length = 1024 + config.actor_rollout_ref.rollout.response_length = 4096 + config.actor_rollout_ref.rollout.skip_tokenizer_init = True + + # 1. init agent loop manager + agent_loop_manager = AgentLoopManager(config) + + # 2. init dataset and dataloader + local_folder = os.path.expanduser("~/data/gsm8k/") + data_files = [os.path.join(local_folder, "train.parquet")] + tokenizer = AutoTokenizer.from_pretrained(model_path) + + dataset = RLHFDataset( + data_files=data_files, + tokenizer=tokenizer, + config=config.data, + processor=None, + ) + + batch_size = 128 + sampler = create_rl_sampler(config.data, dataset) + dataloader = StatefulDataLoader( + dataset=dataset, + batch_size=batch_size, + num_workers=config.data.dataloader_num_workers, + drop_last=True, + collate_fn=collate_fn, + sampler=sampler, + ) + + # 3. generate_sequences with agent loop + batch_dict = next(iter(dataloader)) + batch = DataProto.from_single_dict(batch_dict) + gen_batch = agent_loop_manager.generate_sequences(prompts=batch) + + rm_scores = gen_batch.batch["rm_scores"] + sample_scores = rm_scores.sum(dim=1) + assert sample_scores.min() == 0.0, f"min score: {sample_scores.min()}" + assert sample_scores.max() == 1.0, f"max score: {sample_scores.max()}" + print(f"gsm8k acc: {sample_scores.mean()}") + + print("Test passed!") + ray.shutdown() diff --git a/e2c/verl/tests/experimental/agent_loop/test_agent_loop_reward_model.py b/e2c/verl/tests/experimental/agent_loop/test_agent_loop_reward_model.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c169e20aab9b9489528e27bb65c18a2c0c8d54 --- /dev/null +++ b/e2c/verl/tests/experimental/agent_loop/test_agent_loop_reward_model.py @@ -0,0 +1,100 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pytest +import ray +from hydra import compose, initialize_config_dir +from torchdata.stateful_dataloader import StatefulDataLoader +from transformers import AutoTokenizer + +from tests.experimental.agent_loop.agent_utils import AgentLoopManager +from verl.protocol import DataProto +from verl.trainer.main_ppo import create_rl_sampler +from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn + + +@pytest.mark.skip(reason="reward model is depreated and replaced by GRM") +def test_agent_loop_compute_score_with_model(): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + } + ) + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + config = compose("ppo_trainer") + + model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-1.5B-Instruct") + config.data.return_raw_chat = True + config.actor_rollout_ref.model.path = model_path + config.actor_rollout_ref.actor.use_dynamic_bsz = True + config.actor_rollout_ref.rollout.name = os.environ["ROLLOUT_NAME"] + config.actor_rollout_ref.rollout.mode = "async" + config.actor_rollout_ref.rollout.enforce_eager = True + config.actor_rollout_ref.rollout.prompt_length = 1024 + config.actor_rollout_ref.rollout.response_length = 4096 + config.actor_rollout_ref.rollout.skip_tokenizer_init = True + config.reward_model.enable = True + config.reward_model.model.path = model_path + config.reward_model.use_dynamic_bsz = True + config.reward_model.forward_max_token_len_per_gpu = 6000 + config.reward_model.micro_batch_size_per_gpu = 40 + config.reward_model.enable_resource_pool = True + config.reward_model.n_gpus_per_node = 1 + config.reward_model.nnodes = 1 + config.reward_model.model.trust_remote_code = True + config.reward_model.model.input_tokenizer = None + config.trainer.n_gpus_per_node = 4 + config.trainer.nnodes = 1 + # 1. init agent loop manager + agent_loop_manager = AgentLoopManager(config) + + # 2. init dataset and dataloader + local_folder = os.path.expanduser("~/data/gsm8k/") + data_files = [os.path.join(local_folder, "train.parquet")] + tokenizer = AutoTokenizer.from_pretrained(model_path) + + dataset = RLHFDataset( + data_files=data_files, + tokenizer=tokenizer, + config=config.data, + processor=None, + ) + + batch_size = 128 + sampler = create_rl_sampler(config.data, dataset) + dataloader = StatefulDataLoader( + dataset=dataset, + batch_size=batch_size, + num_workers=config.data.dataloader_num_workers, + drop_last=True, + collate_fn=collate_fn, + sampler=sampler, + ) + + # 3. generate_sequences with agent loop + batch_dict = next(iter(dataloader)) + batch = DataProto.from_single_dict(batch_dict) + gen_batch = agent_loop_manager.generate_sequences(prompts=batch) + + rm_scores = gen_batch.batch["rm_scores"] + sample_scores = rm_scores.sum(dim=1) + print(sample_scores) + ray.shutdown() diff --git a/e2c/verl/tests/experimental/agent_loop/test_basic_agent_loop.py b/e2c/verl/tests/experimental/agent_loop/test_basic_agent_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..6746db10137216c006ee25c94e48a2fe91b643f0 --- /dev/null +++ b/e2c/verl/tests/experimental/agent_loop/test_basic_agent_loop.py @@ -0,0 +1,446 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +from typing import Any + +import numpy as np +import pytest +import ray +from omegaconf import DictConfig +from transformers.utils import get_json_schema + +from tests.experimental.agent_loop.agent_utils import init_agent_loop_manager +from verl.experimental.agent_loop import AgentLoopManager +from verl.experimental.agent_loop.agent_loop import get_trajectory_info +from verl.protocol import DataProto +from verl.tools.base_tool import BaseTool, OpenAIFunctionToolSchema +from verl.tools.schemas import ToolResponse +from verl.trainer.ppo.reward import compute_reward, load_reward_manager +from verl.utils import hf_tokenizer + + +@pytest.fixture +def init_config() -> DictConfig: + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + config = compose( + config_name="ppo_trainer", + overrides=[ + "actor_rollout_ref.actor.use_dynamic_bsz=true", + # test sleep/wake_up with fsdp offload + "actor_rollout_ref.actor.fsdp_config.param_offload=True", + "actor_rollout_ref.actor.fsdp_config.optimizer_offload=True", + "reward_model.reward_manager=dapo", + "+reward_model.reward_kwargs.overlong_buffer_cfg.enable=False", + "+reward_model.reward_kwargs.overlong_buffer_cfg.len=3072", + "+reward_model.reward_kwargs.max_resp_len=4096", + ], + ) + + model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-1.5B-Instruct") + config.actor_rollout_ref.model.path = model_path + config.actor_rollout_ref.rollout.name = os.environ["ROLLOUT_NAME"] + config.actor_rollout_ref.rollout.mode = "async" + config.actor_rollout_ref.rollout.enforce_eager = True + config.actor_rollout_ref.rollout.prompt_length = 4096 + config.actor_rollout_ref.rollout.response_length = 4096 + config.actor_rollout_ref.rollout.n = 4 + config.actor_rollout_ref.rollout.agent.num_workers = 2 + config.actor_rollout_ref.rollout.skip_tokenizer_init = True + + return config + + +def test_single_turn(init_config): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + } + ) + + agent_loop_manager = AgentLoopManager(init_config) + tokenizer = hf_tokenizer(init_config.actor_rollout_ref.model.path) + reward_fn = load_reward_manager( + init_config, tokenizer, num_examine=0, **init_config.reward_model.get("reward_kwargs", {}) + ) + + raw_prompts = [ + [ + { + "role": "user", + "content": "Let's play a role playing game. Your name is Alice, your favorite color is blue.", + } + ], + [{"role": "user", "content": "Let's play a role playing game. Your name is Bob, your favorite color is red."}], + ] + batch = DataProto( + non_tensor_batch={ + "raw_prompt": np.array(raw_prompts), + "agent_name": np.array(["single_turn_agent"] * len(raw_prompts)), + "data_source": np.array(["openai/gsm8k"] * len(raw_prompts)), + "reward_model": np.array([{"style": "rule", "ground_truth": "1.0"}] * len(raw_prompts)), + }, + ) + n = init_config.actor_rollout_ref.rollout.n + batch = batch.repeat(n) + result = agent_loop_manager.generate_sequences(prompts=batch) + assert len(result) == len(raw_prompts) * n + + # check result + seq_len = result.batch["prompts"].size(1) + result.batch["responses"].size(1) + assert result.batch["input_ids"].size(1) == seq_len + assert result.batch["attention_mask"].size(1) == seq_len + assert result.batch["position_ids"].size(1) == seq_len + + if init_config.actor_rollout_ref.rollout.calculate_log_probs: + assert result.batch["rollout_log_probs"].size(1) == result.batch["responses"].size(1) + + # check compute score + assert result.batch["rm_scores"].shape == result.batch["responses"].shape + reward_tensor, reward_extra_info = compute_reward(result, reward_fn) + assert reward_tensor.shape == result.batch["responses"].shape + assert "acc" in reward_extra_info, f"reward_extra_info {reward_extra_info} should contain 'acc'" + assert reward_extra_info["acc"].shape == (len(result),), f"invalid acc: {reward_extra_info['acc']}" + + # check turns + num_turns = result.non_tensor_batch["__num_turns__"] + assert np.all(num_turns == 2) + + print("Test passed!") + ray.shutdown() + + +class WeatherTool(BaseTool): + def get_current_temperature(self, location: str, unit: str = "celsius"): + """Get current temperature at a location. + + Args: + location: The location to get the temperature for, in the format "City, State, Country". + unit: The unit to return the temperature in. Defaults to "celsius". (choices: ["celsius", "fahrenheit"]) + + Returns: + the temperature, the location, and the unit in a dict + """ + print(f"[DEBUG] get_current_temperature: {location}, {unit}") + return { + "temperature": 26.1, + "location": location, + "unit": unit, + } + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + schema = get_json_schema(self.get_current_temperature) + return OpenAIFunctionToolSchema(**schema) + + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + try: + result = self.get_current_temperature(**parameters) + return ToolResponse(text=json.dumps(result)), 0, {} + except Exception as e: + return ToolResponse(text=str(e)), 0, {} + + +class WeatherToolWithData(BaseTool): + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + schema = get_json_schema(self.get_temperature_date) + return OpenAIFunctionToolSchema(**schema) + + def get_temperature_date(self, location: str, date: str, unit: str = "celsius"): + """Get temperature at a location and date. + + Args: + location: The location to get the temperature for, in the format "City, State, Country". + date: The date to get the temperature for, in the format "Year-Month-Day". + unit: The unit to return the temperature in. Defaults to "celsius". (choices: ["celsius", "fahrenheit"]) + + Returns: + the temperature, the location, the date and the unit in a dict + """ + print(f"[DEBUG] get_temperature_date: {location}, {date}, {unit}") + return { + "temperature": 25.9, + "location": location, + "date": date, + "unit": unit, + } + + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + try: + result = self.get_temperature_date(**parameters) + return ToolResponse(text=json.dumps(result)), 0, {} + except Exception as e: + return ToolResponse(text=str(e)), 0, {} + + +def test_tool_agent(init_config): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + }, + ignore_reinit_error=True, + ) + + # =========================== 1. Init rollout manager =========================== + tool_config = { + "tools": [ + { + "class_name": "tests.experimental.agent_loop.test_basic_agent_loop.WeatherTool", + "config": {"type": "native"}, + }, + { + "class_name": "tests.experimental.agent_loop.test_basic_agent_loop.WeatherToolWithData", + "config": {"type": "native"}, + }, + ] + } + tool_config_path = "/tmp/tool_config.json" + with open(tool_config_path, "w") as f: + json.dump(tool_config, f) + + n = 2 + init_config.actor_rollout_ref.rollout.n = n + init_config.actor_rollout_ref.rollout.multi_turn.tool_config_path = tool_config_path + init_config.actor_rollout_ref.rollout.multi_turn.max_parallel_calls = 2 + init_config.actor_rollout_ref.rollout.calculate_log_probs = True + agent_loop_manager = AgentLoopManager(init_config) + + # =========================== 2. Generate sequences =========================== + raw_prompts = [ + [ + {"role": "user", "content": "How are you?"}, + ], + [ + {"role": "user", "content": "What's the temperature in Los Angeles now?"}, + ], + [ + {"role": "user", "content": "What's the temperature in New York now?"}, + ], + [ + { + "role": "system", + "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.\n\n" + "Current Date: 2024-09-30", + }, + {"role": "user", "content": "What's the temperature in San Francisco now? How about tomorrow?"}, + ], + ] + batch = DataProto( + non_tensor_batch={ + "raw_prompt": np.array([np.array(prompt) for prompt in raw_prompts], dtype=object), + "agent_name": np.array(["tool_agent"] * len(raw_prompts)), + "data_source": np.array(["openai/gsm8k"] * len(raw_prompts)), + "reward_model": np.array([{"style": "rule", "ground_truth": "1.0"}] * len(raw_prompts)), + }, + ) + batch = batch.repeat(n) + result = agent_loop_manager.generate_sequences(prompts=batch) + assert len(result) == len(raw_prompts) * n + + # Check turns + num_turns = result.non_tensor_batch["__num_turns__"] + print(f"num_turns: {num_turns}") + for i in range(len(num_turns)): + if i // n == 0: + # [user, assistant] + assert num_turns[i] == 2 + else: + # [user, assistant, tool, assistant] + assert num_turns[i] == 4 + + # Check response_mask + tokenizer = hf_tokenizer(init_config.actor_rollout_ref.model.path) + responses = result.batch["responses"] + response_mask = result.batch["response_mask"] + attention_mask = result.batch["attention_mask"] + assert result.batch["rm_scores"].size(1) == responses.size(1) + assert responses.size() == response_mask.size(), f"{responses.size()} != {response_mask.size()}" + assert result.batch["rollout_log_probs"].size(1) == result.batch["responses"].size(1) + + response_length = response_mask.size(1) + for i in range(len(responses)): + # response with tool response + valid_tokens = responses[i][attention_mask[i][-response_length:].bool()] + response_with_obs = tokenizer.decode(valid_tokens) + + # response without tool response + valid_tokens = responses[i][response_mask[i].bool()] + response_without_obs = tokenizer.decode(valid_tokens) + + assert "" not in response_without_obs, ( + f"found in response: {response_without_obs}" + ) + assert "" not in response_without_obs, ( + f"found in response: {response_without_obs}" + ) + print("=========================") + print(response_with_obs) + print("---") + print(response_without_obs) + + print("Test passed!") + ray.shutdown() + + +def test_tool_agent_with_interaction(init_config): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + } + ) + + # =========================== 1. Init rollout manager =========================== + tool_config = { + "tools": [ + { + "class_name": "tests.experimental.agent_loop.test_basic_agent_loop.WeatherTool", + "config": {"type": "native"}, + }, + { + "class_name": "tests.experimental.agent_loop.test_basic_agent_loop.WeatherToolWithData", + "config": {"type": "native"}, + }, + ] + } + tool_config_path = "/tmp/tool_config.json" + with open(tool_config_path, "w") as f: + json.dump(tool_config, f) + + interaction_config = { + "interaction": [ + {"name": "weather", "class_name": "verl.interactions.weather_interaction.WeatherInteraction", "config": {}} + ] + } + interaction_config_path = "/tmp/interaction_config.json" + with open(interaction_config_path, "w") as f: + json.dump(interaction_config, f) + + n = 2 + init_config.actor_rollout_ref.rollout.n = n + init_config.actor_rollout_ref.rollout.multi_turn.tool_config_path = tool_config_path + init_config.actor_rollout_ref.rollout.multi_turn.interaction_config_path = interaction_config_path + init_config.actor_rollout_ref.rollout.multi_turn.max_parallel_calls = 2 + agent_loop_manager = init_agent_loop_manager(init_config) + + # =========================== 2. Generate sequences =========================== + raw_prompts = [ + [ + {"role": "user", "content": "How are you?"}, + ], + [ + {"role": "user", "content": "What's the temperature in Los Angeles now?"}, + ], + [ + {"role": "user", "content": "What's the temperature in New York now?"}, + ], + [ + { + "role": "system", + "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.\n\n" + "Current Date: 2024-09-30", + }, + {"role": "user", "content": "What's the temperature in San Francisco now? How about tomorrow?"}, + ], + ] + batch = DataProto( + non_tensor_batch={ + "raw_prompt": np.array([np.array(prompt) for prompt in raw_prompts], dtype=object), + "agent_name": np.array(["tool_agent"] * len(raw_prompts)), + "data_source": np.array(["openai/gsm8k"] * len(raw_prompts)), + "reward_model": np.array([{"style": "rule", "ground_truth": "1.0"}] * len(raw_prompts)), + "extra_info": np.array( + [ + {"interaction_kwargs": {"name": "weather"}}, + {"interaction_kwargs": {"name": "weather"}}, + {"interaction_kwargs": {"name": "weather"}}, + {"interaction_kwargs": {"name": "weather"}}, + ] + ), + }, + ) + batch = batch.repeat(n) + result = agent_loop_manager.generate_sequences(prompts=batch) + assert len(result) == len(raw_prompts) * n + + # Check turns + num_turns = result.non_tensor_batch["__num_turns__"] + print(f"num_turns: {num_turns}") + for i in range(len(num_turns)): + if i // n == 0: + # [user, assistant, user] + assert num_turns[i] == 3 + else: + # [user, assistant, tool, assistant, user] + assert num_turns[i] == 5 + + # Check response_mask + tokenizer = hf_tokenizer(init_config.actor_rollout_ref.model.path) + responses = result.batch["responses"] + response_mask = result.batch["response_mask"] + attention_mask = result.batch["attention_mask"] + assert responses.size() == response_mask.size(), f"{responses.size()} != {response_mask.size()}" + response_length = response_mask.size(1) + + for i in range(len(responses)): + # response with tool response + valid_tokens = responses[i][attention_mask[i][-response_length:].bool()] + response_with_obs = tokenizer.decode(valid_tokens) + + # response without tool response + valid_tokens = responses[i][response_mask[i].bool()] + response_without_obs = tokenizer.decode(valid_tokens) + + assert "\udb82\udc89" not in response_without_obs, f"found \udb82\udc89 in response: {response_without_obs}" + assert "\udb82\udc8a" not in response_without_obs, f"found \udb82\udc8a in response: {response_without_obs}" + print("=========================") + print(response_with_obs) + print("---") + print(response_without_obs) + + print("Test passed!") + ray.shutdown() + + +@pytest.mark.asyncio +async def test_get_trajectory_info(): + """Tests the get_trajectory_info method.""" + # Initialize the class to set up class-level attributes + step = 10 + index = [1, 1, 3, 3] + expected_info = [ + {"step": step, "sample_index": 1, "rollout_n": 0, "validate": False}, + {"step": step, "sample_index": 1, "rollout_n": 1, "validate": False}, + {"step": step, "sample_index": 3, "rollout_n": 0, "validate": False}, + {"step": step, "sample_index": 3, "rollout_n": 1, "validate": False}, + ] + + trajectory_info = await get_trajectory_info(step, index, validate=False) + + assert trajectory_info == expected_info diff --git a/e2c/verl/tests/experimental/agent_loop/test_multi_modal.py b/e2c/verl/tests/experimental/agent_loop/test_multi_modal.py new file mode 100644 index 0000000000000000000000000000000000000000..45fc2a7f148e1a90fee091b66beec298cb969ef1 --- /dev/null +++ b/e2c/verl/tests/experimental/agent_loop/test_multi_modal.py @@ -0,0 +1,246 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +from typing import Any + +import numpy as np +import pytest +import ray +from omegaconf import DictConfig +from PIL import Image +from transformers.utils import get_json_schema + +from verl.experimental.agent_loop import AgentLoopManager +from verl.protocol import DataProto +from verl.tools.base_tool import BaseTool, OpenAIFunctionToolSchema +from verl.tools.schemas import ToolResponse +from verl.utils import hf_tokenizer + + +@pytest.fixture +def init_config() -> DictConfig: + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + config = compose( + config_name="ppo_trainer", + overrides=[ + "actor_rollout_ref.actor.use_dynamic_bsz=true", + # test sleep/wake_up with fsdp offload + "actor_rollout_ref.actor.fsdp_config.param_offload=True", + "actor_rollout_ref.actor.fsdp_config.optimizer_offload=True", + ], + ) + + model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-VL-3B-Instruct") + config.actor_rollout_ref.model.path = model_path + config.actor_rollout_ref.rollout.name = os.environ["ROLLOUT_NAME"] + config.actor_rollout_ref.rollout.mode = "async" + config.actor_rollout_ref.rollout.enforce_eager = True + config.actor_rollout_ref.rollout.prompt_length = 4096 + config.actor_rollout_ref.rollout.response_length = 4096 + config.actor_rollout_ref.rollout.n = 4 + config.actor_rollout_ref.rollout.agent.num_workers = 2 + config.actor_rollout_ref.rollout.skip_tokenizer_init = True + + return config + + +class ImageGeneratorTool(BaseTool): + def generate_image(self, description: str, size: str = "256x256"): + """Generate a simple image based on description. + + Args: + description: The description of the image to generate. + size: The size of the image. Defaults to "256x256". (choices: ["256x256", "512x512"]) + + Returns: + A generated image + """ + print(f"[DEBUG] generate_image: {description}, {size}") + # Create a simple colored image for testing + width, height = map(int, size.split("x")) + + # Create different colors based on description + if "red" in description.lower(): + color = (255, 0, 0) + elif "blue" in description.lower(): + color = (0, 0, 255) + elif "green" in description.lower(): + color = (0, 255, 0) + else: + color = (128, 128, 128) # gray + + # Create image + image = Image.new("RGB", (width, height), color) + + # Add some pattern to make it more interesting + for i in range(0, width, 50): + for j in range(0, height, 50): + # Add white squares in a grid pattern + for x in range(i, min(i + 20, width)): + for y in range(j, min(j + 20, height)): + image.putpixel((x, y), (255, 255, 255)) + + return image + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + schema = get_json_schema(self.generate_image) + return OpenAIFunctionToolSchema(**schema) + + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + try: + image = self.generate_image(**parameters) + # Return the PIL Image directly - the framework should handle the conversion + return ToolResponse(image=[image]), 0, {} + except Exception as e: + return ToolResponse(text=str(e)), 0, {} + + +def test_multimodal_tool_agent(init_config): + """Test agent loop with multimodal tool that returns images using Qwen VL model.""" + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + }, + ignore_reinit_error=True, + ) + + # Add custom chat template to enable tool calling support (same as recipe/deepeyes) + template_path = os.path.join(os.path.dirname(__file__), "qwen_vl_tool_chat_template.jinja2") + with open(template_path, encoding="utf-8") as f: + custom_chat_template = f.read() + + init_config.actor_rollout_ref.model.custom_chat_template = custom_chat_template + + # =========================== 1. Init rollout manager with image tool =========================== + tool_config = { + "tools": [ + { + "class_name": "tests.experimental.agent_loop.test_multi_modal.ImageGeneratorTool", + "config": {"type": "native"}, + }, + ] + } + tool_config_path = "/tmp/multimodal_tool_config.json" + with open(tool_config_path, "w") as f: + json.dump(tool_config, f) + + n = 2 + init_config.actor_rollout_ref.rollout.n = n + init_config.actor_rollout_ref.rollout.multi_turn.tool_config_path = tool_config_path + init_config.actor_rollout_ref.rollout.multi_turn.max_parallel_calls = 1 + init_config.actor_rollout_ref.rollout.multi_turn.max_user_turns = 1 + agent_loop_manager = AgentLoopManager(init_config) + + # =========================== 2. Generate sequences with multimodal prompts =========================== + raw_prompts = [ + [ + {"role": "user", "content": "How are you?"}, + ], + [ + {"role": "user", "content": "Please generate a red image for me."}, + ], + [ + {"role": "user", "content": "Can you create a blue picture with size 512x512?"}, + ], + [ + { + "role": "system", + "content": ( + "You are Qwen VL, created by Alibaba Cloud. You are a helpful " + "assistant that can generate and analyze images." + ), + }, + {"role": "user", "content": "Generate a green landscape image and describe what you see in it."}, + ], + ] + + batch = DataProto( + non_tensor_batch={ + "raw_prompt": np.array([np.array(prompt) for prompt in raw_prompts], dtype=object), + "agent_name": np.array(["tool_agent"] * len(raw_prompts)), + "data_source": np.array(["openai/gsm8k"] * len(raw_prompts)), + "reward_model": np.array([{"style": "rule", "ground_truth": "1.0"}] * len(raw_prompts)), + }, + ) + batch = batch.repeat(n) + result = agent_loop_manager.generate_sequences(prompts=batch) + assert len(result) == len(raw_prompts) * n + + # Check turns + num_turns = result.non_tensor_batch["__num_turns__"] + print(f"num_turns: {num_turns}") + for i in range(len(num_turns)): + if i // n == 0: + # First prompt: "How are you?" - should have 2 turns [user, assistant] + assert num_turns[i] == 2, f"Expected 2 turns but got {num_turns[i]} for sample {i}" + else: + # Tool-calling prompts should have 4 turns [user, assistant, tool, assistant] + assert num_turns[i] == 4, f"Expected 4 turns but got {num_turns[i]} for sample {i}" + + # Check that images were properly returned in the tool responses + tokenizer = hf_tokenizer(init_config.actor_rollout_ref.model.path) + responses = result.batch["responses"] + response_mask = result.batch["response_mask"] + attention_mask = result.batch["attention_mask"] + assert responses.size() == response_mask.size(), f"{responses.size()} != {response_mask.size()}" + response_length = response_mask.size(1) + + image_found_count = 0 + for i in range(len(responses)): + # response with tool response (including images) + valid_tokens = responses[i][attention_mask[i][-response_length:].bool()] + response_with_obs = tokenizer.decode(valid_tokens) + + # response without tool response + valid_tokens = responses[i][response_mask[i].bool()] + response_without_obs = tokenizer.decode(valid_tokens) + + # Check that tool responses were properly masked out from training + assert "" not in response_without_obs, ( + f"found in response: {response_without_obs}" + ) + assert "" not in response_without_obs, ( + f"found in response: {response_without_obs}" + ) + + # Check that images were included in the full response + if "" in response_with_obs or "image" in response_with_obs.lower(): + image_found_count += 1 + + print("=========================") + print("Response with tool observations:") + print(response_with_obs) + print("---") + print("Response without tool observations:") + print(response_without_obs) + + # Verify that tool-calling responses contained image-related content + print(f"Found {image_found_count} responses with image content out of {len(responses)}") + # We should have at least some image content from the tool-calling prompts + # Note: First prompt might not use tools, so we don't expect 100% image content + expected_tool_calls = sum(1 for i in range(len(num_turns)) if num_turns[i] == 4) + assert image_found_count >= 0, ( + f"No image-related content found, but expected at least some from {expected_tool_calls} tool calls" + ) + + print("Multimodal tool test passed!") + ray.shutdown() diff --git a/e2c/verl/tests/experimental/agent_loop/test_standalone_rollout.py b/e2c/verl/tests/experimental/agent_loop/test_standalone_rollout.py new file mode 100644 index 0000000000000000000000000000000000000000..a530bae8281abd2a884242fb0ad981c4838f339a --- /dev/null +++ b/e2c/verl/tests/experimental/agent_loop/test_standalone_rollout.py @@ -0,0 +1,154 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import os + +import pytest +import ray +from omegaconf import DictConfig +from openai import AsyncOpenAI, OpenAI + +from tests.experimental.agent_loop.agent_utils import init_agent_loop_manager +from verl.workers.rollout.replica import get_rollout_replica_class + + +@pytest.fixture +def init_config() -> DictConfig: + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + config = compose(config_name="ppo_trainer") + + config.trainer.n_gpus_per_node = 4 + config.trainer.nnodes = 2 + config.actor_rollout_ref.actor.use_dynamic_bsz = True + config.actor_rollout_ref.model.path = os.path.expanduser("~/models/Qwen/Qwen2.5-1.5B-Instruct") + config.actor_rollout_ref.rollout.name = os.environ["ROLLOUT_NAME"] + config.actor_rollout_ref.rollout.mode = "async" + config.actor_rollout_ref.rollout.skip_tokenizer_init = False + + return config + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tp_size", [2, 4]) +async def test_standalone_rollout(init_config, tp_size): + """Test standalone rollout single node and multi nodes.""" + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + } + ) + + init_config.actor_rollout_ref.rollout.tensor_model_parallel_size = tp_size + num_replicas = (init_config.trainer.n_gpus_per_node * init_config.trainer.nnodes) // tp_size + rollout_config = init_config.actor_rollout_ref.rollout + model_config = init_config.actor_rollout_ref.model + + # create standalone rollout server + rollout_server_class = get_rollout_replica_class(init_config.actor_rollout_ref.rollout.name) + rollout_servers = [ + rollout_server_class( + replica_rank=replica_rank, config=rollout_config, model_config=model_config, gpus_per_node=2 + ) + for replica_rank in range(num_replicas) + ] + await asyncio.gather(*[server.init_standalone() for server in rollout_servers]) + + server_handles = [server._server_handle for server in rollout_servers] + server_addresses = [server._server_address for server in rollout_servers] + assert len(server_handles) == num_replicas + assert len(server_addresses) == num_replicas + + os.environ.pop("HTTPS_PROXY", None) + os.environ.pop("HTTP_PROXY", None) + os.environ.pop("NO_PROXY", None) + + client = AsyncOpenAI( + api_key="123-abc", + base_url=f"http://{server_addresses[0]}/v1", + ) + + completion = await client.chat.completions.create( + model=init_config.actor_rollout_ref.model.path, + messages=[{"role": "user", "content": "What can you do?"}], + ) + print(completion.choices[0].message.content) + + ray.shutdown() + + +@pytest.mark.skip(reason="local test only") +def test_hybrid_rollout_with_ep(init_config): + """Test hybrid rollout with expert parallelism, DP=2, TP=4, EP=8.""" + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + } + ) + + model_path = os.path.expanduser("~/models/Qwen/Qwen3-30B-A3B-Instruct-2507") + init_config.actor_rollout_ref.model.path = model_path + + # parallelism config + init_config.actor_rollout_ref.rollout.tensor_model_parallel_size = 2 + init_config.actor_rollout_ref.rollout.data_parallel_size = 4 + init_config.actor_rollout_ref.rollout.expert_parallel_size = 8 + + # 1. init hybrid worker: FSDP+rollout + # - build FSDP model and optimizer + # - offload FSDP model and optimizer, build rollout + # - sleep rollout and load FSDP model and optimizer + agent_loop_manager = init_agent_loop_manager(init_config) + + # 2. wake up rollout + # - wake_up weights + # - load_weights from FSDP + # - wake_up kv_cache + agent_loop_manager.wake_up() + + # 3. test async openai call + server_address = agent_loop_manager.server_addresses[0] + client = OpenAI( + api_key="123-abc", + base_url=f"http://{server_address}/v1", + ) + + smapling_params = { + "temperature": 1.0, + "top_p": 1.0, + "max_tokens": 512, + } + + response = client.chat.completions.create( + model=model_path, + messages=[{"role": "user", "content": "What can you do?"}], + **smapling_params, + ) + + completion = response.choices[0].message.content + print(f"response: {completion}") + + print("Test passed!") + ray.shutdown() diff --git a/e2c/verl/tests/interactions/__init__.py b/e2c/verl/tests/interactions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b6db0fcef70b051ba5975c4a94d2b68b986e1127 --- /dev/null +++ b/e2c/verl/tests/interactions/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/tests/interactions/test_gsm8k_interaction.py b/e2c/verl/tests/interactions/test_gsm8k_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..ac40f32f42cbe52697bcddf8ebd882b620f45766 --- /dev/null +++ b/e2c/verl/tests/interactions/test_gsm8k_interaction.py @@ -0,0 +1,422 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch + +import pytest + +from verl.interactions.gsm8k_interaction import Gsm8kInteraction + + +class TestGsm8kInteraction: + """Test cases for Gsm8kInteraction class.""" + + def setup_method(self): + """Set up test environment before each test method.""" + self.config = {"name": "gsm8k"} + self.interaction = Gsm8kInteraction(self.config) + + def test_init(self): + """Test Gsm8kInteraction initialization.""" + assert self.interaction._instance_dict == {} + assert self.interaction.config == self.config + assert self.interaction.name == "gsm8k" + + @pytest.mark.asyncio + async def test_start_interaction_with_instance_id(self): + """Test start_interaction with provided instance_id.""" + instance_id = "test_instance" + ground_truth = "42" + + result_id = await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + assert result_id == instance_id + assert instance_id in self.interaction._instance_dict + assert self.interaction._instance_dict[instance_id]["response"] == "" + assert self.interaction._instance_dict[instance_id]["ground_truth"] == ground_truth + assert self.interaction._instance_dict[instance_id]["reward"] == 0.0 + + @pytest.mark.asyncio + async def test_start_interaction_without_instance_id(self): + """Test start_interaction without provided instance_id (auto-generated).""" + ground_truth = "42" + + result_id = await self.interaction.start_interaction(ground_truth=ground_truth) + + assert result_id is not None + assert len(result_id) == 36 # UUID4 length + assert result_id in self.interaction._instance_dict + assert self.interaction._instance_dict[result_id]["ground_truth"] == ground_truth + + @pytest.mark.asyncio + async def test_start_interaction_without_ground_truth(self): + """Test start_interaction without ground_truth parameter.""" + instance_id = "test_instance" + + result_id = await self.interaction.start_interaction(instance_id=instance_id) + + assert result_id == instance_id + assert self.interaction._instance_dict[instance_id]["ground_truth"] is None + + @pytest.mark.asyncio + async def test_generate_response_correct_answer_with_prefix(self): + """Test generate_response with correct answer already having #### prefix.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + messages = [{"role": "assistant", "content": "#### 42"}] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=1.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is True + assert response == "Your response is correct!" + assert reward == 1.0 + assert metadata == {} + assert self.interaction._instance_dict[instance_id]["response"] == "#### 42" + + @pytest.mark.asyncio + async def test_generate_response_correct_answer_without_prefix(self): + """Test generate_response with correct answer missing #### prefix.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + messages = [{"role": "assistant", "content": "42"}] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=1.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is True + assert response == "Your response is correct!" + assert reward == 1.0 + assert self.interaction._instance_dict[instance_id]["response"] == "#### 42" + + @pytest.mark.asyncio + async def test_generate_response_incorrect_answer(self): + """Test generate_response with incorrect answer.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + messages = [{"role": "assistant", "content": "24"}] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=0.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is False + assert response == "Your response is incorrect! You need to reflect on your answer and try again." + assert reward == 0.0 + assert self.interaction._instance_dict[instance_id]["response"] == "#### 24" + + @pytest.mark.asyncio + async def test_generate_response_multiple_messages(self): + """Test generate_response with multiple messages (should use last assistant message).""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + messages = [ + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": "### 4"}, + {"role": "user", "content": "What is 40+2?"}, + {"role": "assistant", "content": "#### 42"}, + ] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=1.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is True + assert response == "Your response is correct!" + assert self.interaction._instance_dict[instance_id]["response"] == "#### 42" + + @pytest.mark.asyncio + async def test_generate_response_no_assistant_message(self): + """Test generate_response with no assistant messages.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + messages = [{"role": "user", "content": "Hello!"}] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=0.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is False + assert self.interaction._instance_dict[instance_id]["response"] == "#### " + + @pytest.mark.asyncio + async def test_calculate_score_direct_call(self): + """Test calculate_score method directly.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + # Set a response + self.interaction._instance_dict[instance_id]["response"] = "#### 42" + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=1.0) as mock_compute: + score = await self.interaction.calculate_score(instance_id) + + assert score == 1.0 + mock_compute.assert_called_once_with("#### 42", "42", method="flexible", format_score=0.0, score=1.0) + + @pytest.mark.asyncio + async def test_calculate_score_with_kwargs(self): + """Test calculate_score method with additional kwargs.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + # Set a response + self.interaction._instance_dict[instance_id]["response"] = "#### 24" + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=0.0) as mock_compute: + score = await self.interaction.calculate_score(instance_id, extra_param="test") + + assert score == 0.0 + mock_compute.assert_called_once_with("#### 24", "42", method="flexible", format_score=0.0, score=1.0) + + @pytest.mark.asyncio + async def test_finalize_interaction(self): + """Test finalize_interaction method.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + assert instance_id in self.interaction._instance_dict + + await self.interaction.finalize_interaction(instance_id) + + assert instance_id not in self.interaction._instance_dict + + @pytest.mark.asyncio + async def test_finalize_interaction_with_kwargs(self): + """Test finalize_interaction method with additional kwargs.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + assert instance_id in self.interaction._instance_dict + + await self.interaction.finalize_interaction(instance_id, extra_param="test") + + assert instance_id not in self.interaction._instance_dict + + @pytest.mark.asyncio + async def test_finalize_nonexistent_interaction(self): + """Test finalize_interaction with non-existent instance_id.""" + instance_id = "nonexistent_instance" + + # This should raise KeyError + with pytest.raises(KeyError): + await self.interaction.finalize_interaction(instance_id) + + @pytest.mark.asyncio + async def test_full_interaction_workflow_correct(self): + """Test complete interaction workflow with correct answer.""" + ground_truth = "42" + + # Start interaction + instance_id = await self.interaction.start_interaction(ground_truth=ground_truth) + + # Generate response with correct answer + messages = [{"role": "assistant", "content": "42"}] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=1.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is True + assert reward == 1.0 + + # Finalize interaction + await self.interaction.finalize_interaction(instance_id) + assert instance_id not in self.interaction._instance_dict + + @pytest.mark.asyncio + async def test_full_interaction_workflow_incorrect(self): + """Test complete interaction workflow with incorrect answer.""" + ground_truth = "42" + + # Start interaction + instance_id = await self.interaction.start_interaction(ground_truth=ground_truth) + + # Generate response with incorrect answer + messages = [{"role": "assistant", "content": "24"}] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=0.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is False + assert reward == 0.0 + + # Continue with another attempt + messages.append({"role": "user", "content": response}) + messages.append({"role": "assistant", "content": "42"}) + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=1.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is True + assert reward == 1.0 + + # Finalize interaction + await self.interaction.finalize_interaction(instance_id) + assert instance_id not in self.interaction._instance_dict + + @pytest.mark.asyncio + async def test_multiple_concurrent_interactions(self): + """Test multiple concurrent interaction instances.""" + ground_truth_1 = "42" + ground_truth_2 = "24" + + # Start multiple interactions + instance_id_1 = await self.interaction.start_interaction(ground_truth=ground_truth_1) + instance_id_2 = await self.interaction.start_interaction(ground_truth=ground_truth_2) + + assert len(self.interaction._instance_dict) == 2 + assert instance_id_1 in self.interaction._instance_dict + assert instance_id_2 in self.interaction._instance_dict + + # Test responses for both instances + messages_1 = [{"role": "assistant", "content": "42"}] + messages_2 = [{"role": "assistant", "content": "24"}] + + with patch("verl.utils.reward_score.gsm8k.compute_score", side_effect=[1.0, 1.0]): + should_terminate_1, _, reward_1, _ = await self.interaction.generate_response(instance_id_1, messages_1) + should_terminate_2, _, reward_2, _ = await self.interaction.generate_response(instance_id_2, messages_2) + + assert should_terminate_1 is True + assert should_terminate_2 is True + assert reward_1 == 1.0 + assert reward_2 == 1.0 + + # Finalize both interactions + await self.interaction.finalize_interaction(instance_id_1) + await self.interaction.finalize_interaction(instance_id_2) + + assert len(self.interaction._instance_dict) == 0 + + @pytest.mark.asyncio + async def test_edge_case_empty_messages(self): + """Test edge case with empty messages list.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + messages = [] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=0.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is False + assert reward == 0.0 + assert self.interaction._instance_dict[instance_id]["response"] == "#### " + + @pytest.mark.asyncio + async def test_edge_case_message_without_content(self): + """Test edge case with message without content field.""" + instance_id = "test_instance" + ground_truth = "42" + + # Setup instance + await self.interaction.start_interaction(instance_id=instance_id, ground_truth=ground_truth) + + messages = [ + {"role": "assistant"} # Missing content field + ] + + with patch("verl.utils.reward_score.gsm8k.compute_score", return_value=0.0): + should_terminate, response, reward, metadata = await self.interaction.generate_response( + instance_id, messages + ) + + assert should_terminate is False + assert reward == 0.0 + assert self.interaction._instance_dict[instance_id]["response"] == "#### None" + + def test_inheritance_from_base_interaction(self): + """Test that Gsm8kInteraction properly inherits from BaseInteraction.""" + from verl.interactions.base import BaseInteraction + + assert isinstance(self.interaction, BaseInteraction) + + # Test that all required methods are implemented + assert hasattr(self.interaction, "start_interaction") + assert hasattr(self.interaction, "generate_response") + assert hasattr(self.interaction, "calculate_score") + assert hasattr(self.interaction, "finalize_interaction") + + # Test that methods are callable + assert callable(self.interaction.start_interaction) + assert callable(self.interaction.generate_response) + assert callable(self.interaction.calculate_score) + assert callable(self.interaction.finalize_interaction) + + def test_name_attribute_initialization(self): + """Test name attribute initialization with different configs.""" + # Test with explicit name in config + config_with_name = {"name": "custom_gsm8k"} + interaction_with_name = Gsm8kInteraction(config_with_name) + assert interaction_with_name.name == "custom_gsm8k" + + # Test with default name when not provided in config + config_without_name = {} + interaction_without_name = Gsm8kInteraction(config_without_name) + assert interaction_without_name.name == "interaction_agent" # Default from BaseInteraction + + # Test that name is accessible as attribute + assert hasattr(self.interaction, "name") + assert self.interaction.name == "gsm8k" diff --git a/e2c/verl/tests/interactions/test_interaction_registry.py b/e2c/verl/tests/interactions/test_interaction_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..7fe193b52eca965bb73ba3628108e7c14cce7464 --- /dev/null +++ b/e2c/verl/tests/interactions/test_interaction_registry.py @@ -0,0 +1,206 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile + +import pytest +from omegaconf import OmegaConf + +from verl.interactions.base import BaseInteraction +from verl.interactions.gsm8k_interaction import Gsm8kInteraction +from verl.interactions.utils.interaction_registry import ( + get_interaction_class, + initialize_interactions_from_config, +) + + +class TestInteractionRegistry: + def test_get_interaction_class(self): + """Test getting interaction class by name.""" + # Test getting base interaction class + base_cls = get_interaction_class("verl.interactions.base.BaseInteraction") + assert base_cls == BaseInteraction + + # Test getting gsm8k interaction class + gsm8k_cls = get_interaction_class("verl.interactions.gsm8k_interaction.Gsm8kInteraction") + assert gsm8k_cls == Gsm8kInteraction + + def test_initialize_single_interaction_from_config(self): + """Test initializing single interaction from config.""" + # Create temporary config file + config_content = { + "interaction": [ + { + "name": "test_gsm8k", + "class_name": "verl.interactions.gsm8k_interaction.Gsm8kInteraction", + "config": {}, + } + ] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(config_content, f.name) + temp_config_path = f.name + + try: + interaction_map = initialize_interactions_from_config(temp_config_path) + + # Check that interaction was created + assert len(interaction_map) == 1 + assert "test_gsm8k" in interaction_map + assert isinstance(interaction_map["test_gsm8k"], Gsm8kInteraction) + assert interaction_map["test_gsm8k"].name == "test_gsm8k" + finally: + os.unlink(temp_config_path) + + def test_initialize_multiple_interactions_from_config(self): + """Test initializing multiple interactions from config.""" + config_content = { + "interaction": [ + { + "name": "gsm8k_solver", + "class_name": "verl.interactions.gsm8k_interaction.Gsm8kInteraction", + "config": {}, + }, + { + "name": "base_agent", + "class_name": "verl.interactions.base.BaseInteraction", + "config": {"custom_param": "test_value"}, + }, + ] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(config_content, f.name) + temp_config_path = f.name + + try: + interaction_map = initialize_interactions_from_config(temp_config_path) + + # Check that both interactions were created + assert len(interaction_map) == 2 + assert "gsm8k_solver" in interaction_map + assert "base_agent" in interaction_map + + # Check types + assert isinstance(interaction_map["gsm8k_solver"], Gsm8kInteraction) + assert isinstance(interaction_map["base_agent"], BaseInteraction) + + # Check names were injected + assert interaction_map["gsm8k_solver"].name == "gsm8k_solver" + assert interaction_map["base_agent"].name == "base_agent" + + # Check custom config was passed + assert interaction_map["base_agent"].config.get("custom_param") == "test_value" + finally: + os.unlink(temp_config_path) + + def test_initialize_interaction_without_explicit_name(self): + """Test that interaction name is derived from class name when not specified.""" + config_content = { + "interaction": [{"class_name": "verl.interactions.gsm8k_interaction.Gsm8kInteraction", "config": {}}] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(config_content, f.name) + temp_config_path = f.name + + try: + interaction_map = initialize_interactions_from_config(temp_config_path) + + # Check that interaction name was derived from class name + assert len(interaction_map) == 1 + assert "gsm8k" in interaction_map # Should be "gsm8k" after removing "interaction" suffix + assert isinstance(interaction_map["gsm8k"], Gsm8kInteraction) + assert interaction_map["gsm8k"].name == "gsm8k" + finally: + os.unlink(temp_config_path) + + def test_initialize_empty_config(self): + """Test initializing from empty config.""" + config_content = {"interaction": []} + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(config_content, f.name) + temp_config_path = f.name + + try: + interaction_map = initialize_interactions_from_config(temp_config_path) + assert len(interaction_map) == 0 + finally: + os.unlink(temp_config_path) + + def test_invalid_class_name(self): + """Test handling of invalid class name.""" + config_content = { + "interaction": [{"name": "invalid", "class_name": "invalid.module.InvalidClass", "config": {}}] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(config_content, f.name) + temp_config_path = f.name + + try: + with pytest.raises(ModuleNotFoundError): + initialize_interactions_from_config(temp_config_path) + finally: + os.unlink(temp_config_path) + + def test_duplicate_interaction_names(self): + """Test handling of duplicate interaction names.""" + config_content = { + "interaction": [ + {"name": "duplicate", "class_name": "verl.interactions.base.BaseInteraction", "config": {}}, + { + "name": "duplicate", + "class_name": "verl.interactions.gsm8k_interaction.Gsm8kInteraction", + "config": {}, + }, + ] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(config_content, f.name) + temp_config_path = f.name + + try: + with pytest.raises(ValueError, match="Duplicate interaction name 'duplicate' found"): + initialize_interactions_from_config(temp_config_path) + finally: + os.unlink(temp_config_path) + + def test_auto_name_generation_edge_cases(self): + """Test automatic name generation for various class name patterns.""" + config_content = { + "interaction": [ + {"class_name": "verl.interactions.base.BaseInteraction", "config": {}}, + {"class_name": "verl.interactions.gsm8k_interaction.Gsm8kInteraction", "config": {}}, + ] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(config_content, f.name) + temp_config_path = f.name + + try: + interaction_map = initialize_interactions_from_config(temp_config_path) + + # Check that names were generated correctly + assert len(interaction_map) == 2 + assert "base" in interaction_map # BaseInteraction -> base + assert "gsm8k" in interaction_map # Gsm8kInteraction -> gsm8k + finally: + os.unlink(temp_config_path) diff --git a/e2c/verl/tests/kill_github_tests.sh b/e2c/verl/tests/kill_github_tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c76d7658d5373f4a5d73c9aa9c84a7d14b08402 --- /dev/null +++ b/e2c/verl/tests/kill_github_tests.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 YOUR_GITHUB_TOKEN" + echo "Please provide exactly one input argument for your github token." + exit 1 +fi + +# Set your GitHub repository details +OWNER="volcengine" +REPO="verl" +TOKEN=$1 + +# API URL for workflow runs +API_URL="https://api.github.com/repos/$OWNER/$REPO/actions/runs?status=queued" + +# Check required commands +command -v jq >/dev/null 2>&1 || { echo "jq is required but not installed. Aborting."; exit 1; } + +# Get queued workflow runs +response=$(curl -s -H "Authorization: token $TOKEN" -H "Accept: application/vnd.github.v3+json" "$API_URL") + +# Run this for debugging +# echo $response + +# Extract run IDs +queued_run_ids=$(echo "$response" | jq -r '.workflow_runs[] | .id') + +if [ -z "$queued_run_ids" ]; then + echo "No queued workflow runs found." + exit 0 +fi + +# Cancel each queued run +for run_id in $queued_run_ids; do + echo "Cancelling run $run_id" + cancel_url="https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/cancel" + curl -s -X POST -H "Authorization: token $TOKEN" -H "Accept: application/vnd.github.v3+json" "$cancel_url" +done + +echo "Cancelled all queued workflow runs." diff --git a/e2c/verl/tests/models/test_engine.py b/e2c/verl/tests/models/test_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..04634d4297e8d28802b24240413b4e34adfb5e8e --- /dev/null +++ b/e2c/verl/tests/models/test_engine.py @@ -0,0 +1,363 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +os.environ["NCCL_DEBUG"] = "WARN" + +from functools import partial + +import numpy as np +import pytest +import ray +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from transformers import AutoModelForCausalLM, AutoModelForTokenClassification, Qwen3Config, Qwen3MoeConfig + +from verl import DataProto +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup +from verl.trainer.config import CheckpointConfig +from verl.utils.model import compute_position_id_with_mask, create_random_mask +from verl.utils.torch_functional import logprobs_from_logits_naive +from verl.workers.config import ( + ActorConfig, + CriticConfig, + FSDPEngineConfig, + FSDPOptimizerConfig, + HFModelConfig, + McoreEngineConfig, + McoreOptimizerConfig, +) +from verl.workers.roles import ActorWorker, CriticWorker +from verl.workers.roles.utils.losses import ppo_loss, sft_loss + + +@pytest.mark.parametrize("strategy", ["megatron", "fsdp", "fsdp2"]) +def test_actor_engine(strategy): + ray.init() + + path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B-Instruct") + model_config = HFModelConfig(path=path) + + if strategy == "megatron": + engine_config = McoreEngineConfig( + forward_only=False, + use_mbridge=False, + tensor_model_parallel_size=2, + pipeline_model_parallel_size=2, + context_parallel_size=2, + ) + optimizer_config = McoreOptimizerConfig(lr_decay_steps=10) + elif strategy in ["fsdp", "fsdp2"]: + engine_config = FSDPEngineConfig( + forward_only=False, fsdp_size=4, strategy=strategy, ulysses_sequence_parallel_size=2 + ) + optimizer_config = FSDPOptimizerConfig() + else: + raise NotImplementedError(f"strategy {strategy} is not supported") + + config = ActorConfig( + model_config=model_config, + engine=engine_config, + strategy=strategy, + ppo_micro_batch_size_per_gpu=256, + ppo_mini_batch_size=4, + optim=optimizer_config, + use_dynamic_bsz=True, + rollout_n=1, + ) + ray_cls_with_init = RayClassWithInitArgs(cls=ray.remote(ActorWorker), config=config) + resource_pool = RayResourcePool(process_on_nodes=[8]) + wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) + # init model + wg.init_model() + + batch_size = 8 + seqlen = 32 + + response_length = seqlen // 2 + + torch.manual_seed(1) + np.random.seed(1) + + input_ids = torch.randint(0, model_config.hf_config.vocab_size, (batch_size, seqlen)) + attention_mask = create_random_mask( + input_ids=input_ids, max_ratio_of_valid_token=0.8, max_ratio_of_left_padding=0.2, min_ratio_of_valid_token=0.6 + ) + position_ids = compute_position_id_with_mask(attention_mask) + + global_token_num = torch.sum(attention_mask, dim=-1).tolist() + + print(input_ids.float().mean(), attention_mask.float().mean()) + + responses = input_ids[:, response_length:] + response_mask = attention_mask[:, response_length:] + + assert torch.all(response_mask[:, 0] == 1) + + data = DataProto.from_single_dict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "responses": responses, + "response_mask": response_mask, + }, + meta_info={"temperature": 1.0, "global_token_num": global_token_num}, + ) + + sft_loss_ = partial(sft_loss, config=config) + + # eval + output = wg.compute_log_prob(data) + + # load hf model and compare results with hf model + hf_model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16) + hf_output = hf_model(input_ids, attention_mask=attention_mask) + hf_logprobs = logprobs_from_logits_naive( + hf_output.logits[:, -response_length - 1 : -1, :].float(), input_ids[:, -response_length:] + ) + hf_logprobs_mean = torch.mean(hf_logprobs * response_mask) + mcore_logprobs_mean = torch.mean(output.batch["old_log_probs"] * response_mask) + + torch.testing.assert_close(hf_logprobs_mean, mcore_logprobs_mean, atol=1e-3, rtol=1e-2) + + data = data.union(output) + + wg.set_loss_fn(sft_loss_) + + # train for one step + metrics = wg.update_actor(data) + print(metrics) + + # add ppo data + data.batch["advantages"] = torch.rand_like(responses, dtype=torch.float32) + data.batch["ref_log_prob"] = torch.rand_like(responses, dtype=torch.float32) + + # set ppo loss + ppo_loss_ = partial(ppo_loss, config=config) + wg.set_loss_fn(ppo_loss_) + + # update again + ppo_metrics = wg.update_actor(data) + print(ppo_metrics) + + ray.shutdown() + + +def create_model(): + from transformers import Qwen3Config + + config = Qwen3Config(num_hidden_layers=2, num_labels=1) + model = AutoModelForTokenClassification.from_config(config) + assert model.config.num_labels == 1 + path = os.path.expanduser("~/models/test_model") + model.save_pretrained(path) + config.save_pretrained(path) + return path + + +@pytest.mark.parametrize("strategy", ["megatron", "fsdp", "fsdp2"]) +def test_critic_engine(strategy): + ray.init() + + path = create_model() + model_config = HFModelConfig(path=path, load_tokenizer=False) + + if strategy == "megatron": + engine_config = McoreEngineConfig( + forward_only=False, + use_mbridge=False, + tensor_model_parallel_size=2, + pipeline_model_parallel_size=2, + context_parallel_size=2, + ) + optimizer_config = McoreOptimizerConfig(lr_decay_steps=10) + elif strategy in ["fsdp", "fsdp2"]: + engine_config = FSDPEngineConfig( + forward_only=False, fsdp_size=4, strategy=strategy, ulysses_sequence_parallel_size=2 + ) + optimizer_config = FSDPOptimizerConfig() + else: + raise NotImplementedError(f"strategy {strategy} is not supported") + + config = CriticConfig( + model_config=model_config, + engine=engine_config, + strategy=strategy, + ppo_micro_batch_size_per_gpu=256, + ppo_mini_batch_size=4, + optim=optimizer_config, + use_dynamic_bsz=True, + rollout_n=1, + ) + ray_cls_with_init = RayClassWithInitArgs(cls=ray.remote(CriticWorker), config=config) + resource_pool = RayResourcePool(process_on_nodes=[8]) + wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) + # init model + wg.init_model() + + batch_size = 8 + seqlen = 32 + + response_length = seqlen // 2 + + torch.manual_seed(1) + np.random.seed(1) + + input_ids = torch.randint(0, model_config.hf_config.vocab_size, (batch_size, seqlen)) + attention_mask = create_random_mask( + input_ids=input_ids, max_ratio_of_valid_token=0.8, max_ratio_of_left_padding=0.2, min_ratio_of_valid_token=0.6 + ) + position_ids = compute_position_id_with_mask(attention_mask) + + global_token_num = torch.sum(attention_mask, dim=-1).tolist() + + print(input_ids.float().mean(), attention_mask.float().mean()) + + responses = input_ids[:, response_length:] + response_mask = attention_mask[:, response_length:] + + assert torch.all(response_mask[:, 0] == 1) + + data = DataProto.from_single_dict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "responses": responses, + "response_mask": response_mask, + }, + meta_info={"temperature": 1.0, "global_token_num": global_token_num}, + ) + + # eval + output = wg.compute_values(data) + + # load hf model and compare results with hf model + with torch.device("cuda"): + hf_model = AutoModelForTokenClassification.from_pretrained( + path, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + ) + hf_output = hf_model(input_ids.cuda(), attention_mask=attention_mask.cuda()) + hf_values = hf_output.logits[:, -response_length - 1 : -1, :].float().squeeze(-1).cpu() + hf_values_mean = torch.mean(hf_values * response_mask) + + engine_values = torch.mean(output.batch["values"] * response_mask) + + torch.testing.assert_close(hf_values_mean, engine_values, atol=1e-2, rtol=1e-2) + + data = data.union(output) + + # add ppo data + data.batch["values"] = torch.rand_like(responses, dtype=torch.float32) + data.batch["returns"] = torch.rand_like(responses, dtype=torch.float32) + + # update again + ppo_metrics = wg.update_critic(data) + print(ppo_metrics) + + ray.shutdown() + + +def create_actor_model(tmp_path, config): + model = AutoModelForCausalLM.from_config(config) + path = os.path.join(tmp_path, "test_model") + model.save_pretrained(path) + config.save_pretrained(path) + return path + + +def _worker(rank: int, world_size: int, rendezvous_file: str, strategy: str, model_path: str): + torch.cuda.set_device(rank) + dist.init_process_group( + backend="nccl", + init_method=f"file://{rendezvous_file}", + rank=rank, + world_size=world_size, + ) + + with torch.device("meta"): + ref_model = AutoModelForCausalLM.from_pretrained(model_path) + + from verl.workers.engine import BaseEngine, EngineRegistry + + # construct configs + model_config = HFModelConfig(path=model_path, load_tokenizer=False) + + if strategy == "megatron": + engine_config = McoreEngineConfig( + forward_only=False, + use_mbridge=True, + tensor_model_parallel_size=2, + pipeline_model_parallel_size=2, + context_parallel_size=1, + ) + optimizer_config = McoreOptimizerConfig(lr_decay_steps=10) + elif strategy in ["fsdp", "fsdp2"]: + engine_config = FSDPEngineConfig( + forward_only=False, fsdp_size=4, strategy=strategy, ulysses_sequence_parallel_size=2 + ) + optimizer_config = FSDPOptimizerConfig() + else: + raise NotImplementedError(f"strategy {strategy} is not supported") + + checkpoint_config = CheckpointConfig() + + # build model engine + engine: BaseEngine = EngineRegistry.new( + model_type="language_model", + backend=engine_config.strategy, + model_config=model_config, + engine_config=engine_config, + optimizer_config=optimizer_config, + checkpoint_config=checkpoint_config, + ) + + engine.initialize() + + # get per tensor parameter + per_tensor_params = engine.get_per_tensor_param() + + ref_state_dict = ref_model.state_dict() + + # load ground truth and compare + for key, value in per_tensor_params: + assert key in ref_state_dict, f"{key} not in ref_state_dict" + assert value.shape == ref_state_dict[key].shape, ( + f"{key} shape not equal, {value.shape} != {ref_state_dict[key].shape}" + ) + if rank == 0: + print(key, value.shape) + + dist.barrier() + dist.destroy_process_group() + + +@pytest.mark.parametrize("world_size", [8]) +@pytest.mark.parametrize("config", [Qwen3Config(num_hidden_layers=2), Qwen3MoeConfig(num_hidden_layers=2)]) +@pytest.mark.parametrize("strategy", ["megatron", "fsdp", "fsdp2"]) +def test_per_tensor_generator(world_size, tmp_path, config, strategy): + rendezvous_file = str(tmp_path / "rdzv_mask") + os.makedirs(os.path.dirname(rendezvous_file), exist_ok=True) + # create a model + model_path = create_actor_model(tmp_path, config) + # spawn workers + mp.spawn( + fn=_worker, + args=(world_size, rendezvous_file, strategy, model_path), + nprocs=world_size, + join=True, + ) diff --git a/e2c/verl/tests/models/test_transformer.py b/e2c/verl/tests/models/test_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..d78815e515155aa6ff65bc365b0c184b63da76d5 --- /dev/null +++ b/e2c/verl/tests/models/test_transformer.py @@ -0,0 +1,168 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from flash_attn.bert_padding import index_first_axis, pad_input, rearrange, unpad_input +from transformers import ( + ApertusConfig, + AutoModelForCausalLM, + AutoModelForTokenClassification, + GemmaConfig, + LlamaConfig, + MistralConfig, + Qwen2Config, +) + +from verl.utils.model import compute_position_id_with_mask, create_random_mask +from verl.utils.torch_functional import log_probs_from_logits_all_rmpad, masked_mean + +# TODO(sgm): add more models for test +# we only need one scale for each model +test_configs = [ + LlamaConfig(num_hidden_layers=1), + MistralConfig(num_hidden_layers=1), + GemmaConfig(num_hidden_layers=1), + Qwen2Config(num_hidden_layers=1), + ApertusConfig(num_hidden_layers=1), +] + + +def test_hf_casual_models(): + batch_size = 4 + seqlen = 128 + response_length = 127 + + for config in test_configs: + # config = AutoConfig.from_pretrained(test_case) + with torch.device("cuda"): + model = AutoModelForCausalLM.from_config( + config=config, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + ) + model = model.to(device="cuda") + input_ids = torch.randint(low=0, high=config.vocab_size, size=(batch_size, seqlen), device="cuda") + attention_mask = create_random_mask( + input_ids=input_ids, + max_ratio_of_left_padding=0.1, + max_ratio_of_valid_token=0.8, + min_ratio_of_valid_token=0.5, + ) + position_ids = compute_position_id_with_mask( + attention_mask + ) # TODO(sgm): we can construct the position_ids_rmpad here + + input_ids_rmpad, indices, *_ = unpad_input( + input_ids.unsqueeze(-1), attention_mask + ) # input_ids_rmpad (total_nnz, ...) + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) + + # unpad the position_ids to align the rotary + position_ids_rmpad = index_first_axis( + rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices + ).transpose(0, 1) + + # input with input_ids_rmpad and postition_ids to enable flash attention varlen + logits_rmpad = model( + input_ids_rmpad, position_ids=position_ids_rmpad, use_cache=False + ).logits # (1, total_nnz, vocab_size) + + origin_logits = model( + input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, use_cache=False + ).logits + origin_logits_rmpad, origin_logits_indices, *_ = unpad_input(origin_logits, attention_mask) + + logits_rmpad = logits_rmpad.squeeze(0) + log_probs = log_probs_from_logits_all_rmpad( + input_ids_rmpad=input_ids_rmpad, + logits_rmpad=logits_rmpad, + indices=indices, + batch_size=batch_size, + seqlen=seqlen, + response_length=response_length, + ) # (batch, seqlen) + origin_log_probs = log_probs_from_logits_all_rmpad( + input_ids_rmpad=input_ids_rmpad, + logits_rmpad=origin_logits_rmpad, + indices=origin_logits_indices, + batch_size=batch_size, + seqlen=seqlen, + response_length=response_length, + ) # (batch, seqlen) + + torch.testing.assert_close( + masked_mean(log_probs, attention_mask[:, -response_length - 1 : -1]), + masked_mean(origin_log_probs, attention_mask[:, -response_length - 1 : -1]), + atol=1e-2, + rtol=1e-5, + ) + print("Check pass") + + +def test_hf_value_models(): + batch_size = 4 + seqlen = 128 + + for config in test_configs: + # config = AutoConfig.from_pretrained(test_case) + config.num_labels = 1 + config.classifier_dropout = 0 + config.hidden_dropout = 0 + with torch.device("cuda"): + model = AutoModelForTokenClassification.from_config( + config=config, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + ) + model = model.to(device="cuda") + input_ids = torch.randint(low=0, high=config.vocab_size, size=(batch_size, seqlen), device="cuda") + attention_mask = create_random_mask( + input_ids=input_ids, + max_ratio_of_left_padding=0.1, + max_ratio_of_valid_token=0.8, + min_ratio_of_valid_token=0.5, + ) + position_ids = compute_position_id_with_mask( + attention_mask + ) # TODO(sgm): we can construct the position_ids_rmpad here + + input_ids_rmpad, indices, *_ = unpad_input( + input_ids.unsqueeze(-1), attention_mask + ) # input_ids_rmpad (total_nnz, ...) + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) + + # unpad the position_ids to align the rotary + position_ids_rmpad = index_first_axis( + rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices + ).transpose(0, 1) + + origin_logits = model( + input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, use_cache=False + ).logits + + # input with input_ids_rmpad and postition_ids to enable flash attention varlen + rmpad_logits = model( + input_ids_rmpad, position_ids=position_ids_rmpad, use_cache=False + ).logits # (1, total_nnz, 1) + rmpad_logits = rmpad_logits.squeeze(0) + pad_logits = pad_input(rmpad_logits, indices, batch_size, seqlen=seqlen) + + torch.testing.assert_close( + masked_mean(pad_logits, attention_mask[:, :, None]), + masked_mean(origin_logits, attention_mask[:, :, None]), + atol=1e-2, + rtol=1e-5, + ) + print("Value model check pass") + + +if __name__ == "__main__": + test_hf_casual_models() + test_hf_value_models() diff --git a/e2c/verl/tests/models/test_transformers_ulysses.py b/e2c/verl/tests/models/test_transformers_ulysses.py new file mode 100644 index 0000000000000000000000000000000000000000..7a7b73698ce884e33e5dd0b48b4020d60e8cbdbd --- /dev/null +++ b/e2c/verl/tests/models/test_transformers_ulysses.py @@ -0,0 +1,278 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import contextlib +import copy +from dataclasses import dataclass + +import pytest +import torch +import torch.distributed +import transformers +from flash_attn.bert_padding import index_first_axis, rearrange, unpad_input +from packaging import version +from torch.distributed import init_device_mesh +from transformers import AutoModelForCausalLM, LlamaConfig, PretrainedConfig, Qwen2Config + +from verl.models.transformers.monkey_patch import apply_monkey_patch +from verl.protocol import DataProto +from verl.utils.distributed import initialize_global_process_group +from verl.utils.model import compute_position_id_with_mask, create_random_mask +from verl.utils.ulysses import ( + gather_outputs_and_unpad, + get_ulysses_sequence_parallel_world_size, + set_ulysses_sequence_parallel_group, + ulysses_pad_and_slice_inputs, +) +from verl.workers.sharding_manager.fsdp_ulysses import FSDPUlyssesShardingManager + +# TODO(sgm): add more models for test +# we only need one scale for each model + + +@dataclass +class SequenceParallelConfig: + config: PretrainedConfig + sp_size: int + is_valid: bool + + +def test_configs(): + configs = [ + SequenceParallelConfig( + LlamaConfig(num_hidden_layers=2, num_attention_heads=32, num_key_value_heads=32), sp_size=8, is_valid=True + ), + SequenceParallelConfig( + Qwen2Config(num_hidden_layers=2, num_attention_heads=28, num_key_value_heads=4, hidden_size=3584), + sp_size=4, + is_valid=True, + ), + SequenceParallelConfig( + Qwen2Config(num_hidden_layers=2, num_attention_heads=28, num_key_value_heads=4, hidden_size=3584), + sp_size=8, + is_valid=False, + ), + SequenceParallelConfig( + Qwen2Config(num_hidden_layers=2, num_attention_heads=32, num_key_value_heads=4), sp_size=4, is_valid=True + ), + SequenceParallelConfig( + Qwen2Config(num_hidden_layers=2, num_attention_heads=32, num_key_value_heads=4), sp_size=8, is_valid=True + ), + ] + + if version.parse(transformers.__version__) >= version.parse("4.56.0"): + from transformers import ApertusConfig + + configs.append( + SequenceParallelConfig( + ApertusConfig(num_hidden_layers=2, num_attention_heads=32, num_key_value_heads=32, hidden_size=4096), + sp_size=8, + is_valid=True, + ) + ) + + return configs + + +def sync_model_parameters_global(layer): + # synchronize weights + for p in layer.parameters(): + torch.distributed.broadcast(tensor=p.data, src=0) + + +@pytest.mark.parametrize("test_config", test_configs()) +def test_hf_casual_fwd_bwd(test_config): + if not torch.distributed.is_initialized(): + initialize_global_process_group() + + context = contextlib.nullcontext() if test_config.is_valid else pytest.raises(AssertionError) + with context: + world_size = torch.distributed.get_world_size() + _hf_casual_fwd_bwd(test_config.config, test_config.sp_size, world_size // test_config.sp_size) + + # TODO: seems not work, will cause `socketStartConnect: Connect to xxx failed : Software caused connection abort` + # torch.distributed.destroy_process_group() + + +def _hf_casual_fwd(config, sp_size, dp_size): + assert torch.cuda.device_count() >= 2, "need at least 2 gpus for test" + + ulysses_device_mesh = init_device_mesh( + device_type="cuda", mesh_shape=(dp_size, sp_size), mesh_dim_names=("dp", "sp") + ) + sharding_manager = FSDPUlyssesShardingManager(ulysses_device_mesh) + + batch_size = 1 + seqlen = 128 + # response_length = 127 + + # patch before load + with torch.device("cuda"): + model = AutoModelForCausalLM.from_config( + config=config, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + ) + apply_monkey_patch(model, sp_size) + model = model.to(device="cuda") + sync_model_parameters_global(model) + + # different rank will generate different input_ids following fsdp + input_ids = torch.randint(low=0, high=config.vocab_size, size=(batch_size, seqlen), device="cuda") + attention_mask = create_random_mask( + input_ids=input_ids, max_ratio_of_left_padding=0, max_ratio_of_valid_token=0.9, min_ratio_of_valid_token=0.8 + ) + position_ids = compute_position_id_with_mask( + attention_mask + ) # TODO(sgm): we can construct the position_ids_rmpad here + + model_inputs = { + "input_ids": input_ids.cuda(), + "attention_mask": attention_mask.cuda(), + "position_ids": position_ids.int().cuda(), + } + + model_inputs = DataProto.from_dict(model_inputs) + + # 1. perform ulysses forward + with sharding_manager: + model_inputs = sharding_manager.preprocess_data(model_inputs) + input_ids = model_inputs.batch["input_ids"] + attention_mask = model_inputs.batch["attention_mask"] + position_ids = model_inputs.batch["position_ids"] + input_ids_rmpad, indices, *_ = unpad_input( + input_ids.unsqueeze(-1), attention_mask + ) # input_ids_rmpad (total_nnz, ...) + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) + # unpad the position_ids to align the rotary + position_ids_rmpad = index_first_axis( + rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices + ).transpose(0, 1) + + # slice input tensor for ulysses + # input_ids are padded and sliced + # postition_ids are only padded but not sliced + input_ids_rmpad_sliced, position_ids_rmpad_padded, pad_size = ulysses_pad_and_slice_inputs( + input_ids_rmpad, position_ids_rmpad, sp_size=get_ulysses_sequence_parallel_world_size() + ) + + # input with input_ids_rmpad and postition_ids to enable flash attention varlen + logits_split_in_seq = model( + input_ids_rmpad_sliced, position_ids=position_ids_rmpad_padded, use_cache=False + ).logits # (1, total_nnz/n, vocab_size) + + # all_gather output + logits_full = gather_outputs_and_unpad(logits_split_in_seq, gather_dim=1, unpad_dim=1, padding_size=pad_size) + + # 2. perform normal forward + set_ulysses_sequence_parallel_group(None) + logits_rmpad_local = model( + input_ids_rmpad, position_ids=position_ids_rmpad, use_cache=False + ).logits # (1, total_nnz, vocab_size) + + mean_local = logits_rmpad_local.mean() + mean_full = logits_full.mean() + torch.testing.assert_close(mean_local, mean_full, rtol=1e-2, atol=1e-5) + + +def _hf_casual_fwd_bwd(config, sp_size, dp_size): + assert torch.cuda.device_count() >= 2, "need at least 2 gpus for test" + + ulysses_device_mesh = init_device_mesh( + device_type="cuda", mesh_shape=(dp_size, sp_size), mesh_dim_names=("dp", "sp") + ) + sharding_manager = FSDPUlyssesShardingManager(ulysses_device_mesh) + + batch_size = 1 + seqlen = 128 + # response_length = 127 + + # patch before load + with torch.device("cuda"): + model = AutoModelForCausalLM.from_config( + config=config, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + ) + apply_monkey_patch(model, sp_size) + model = model.to(device="cuda") + sync_model_parameters_global(model) + + # different rank will generate different input_ids following fsdp + input_ids = torch.randint(low=0, high=config.vocab_size, size=(batch_size, seqlen), device="cuda") + attention_mask = create_random_mask( + input_ids=input_ids, max_ratio_of_left_padding=0, max_ratio_of_valid_token=0.9, min_ratio_of_valid_token=0.8 + ) + position_ids = compute_position_id_with_mask( + attention_mask + ) # TODO(sgm): we can construct the position_ids_rmpad here + + model_inputs = { + "input_ids": input_ids.cuda(), + "attention_mask": attention_mask.cuda(), + "position_ids": position_ids.int().cuda(), + } + + model_inputs = DataProto.from_dict(model_inputs) + + # 1. perform ulysses forward + with sharding_manager: + model_inputs = sharding_manager.preprocess_data(model_inputs) + input_ids = model_inputs.batch["input_ids"] + attention_mask = model_inputs.batch["attention_mask"] + position_ids = model_inputs.batch["position_ids"] + input_ids_rmpad, indices, *_ = unpad_input( + input_ids.unsqueeze(-1), attention_mask + ) # input_ids_rmpad (total_nnz, ...) + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) + # unpad the position_ids to align the rotary + position_ids_rmpad = index_first_axis( + rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices + ).transpose(0, 1) + + # slice input tensor for ulysses + # input_ids are padded and sliced + # postition_ids are only padded but not sliced + input_ids_rmpad_sliced, position_ids_rmpad_padded, pad_size = ulysses_pad_and_slice_inputs( + input_ids_rmpad, position_ids_rmpad, sp_size=get_ulysses_sequence_parallel_world_size() + ) + + # input with input_ids_rmpad and postition_ids to enable flash attention varlen + logits_split_in_seq = model( + input_ids_rmpad_sliced, position_ids=position_ids_rmpad_padded, use_cache=False + ).logits # (1, total_nnz/n, vocab_size) + + # all_gather output + logits_full = gather_outputs_and_unpad(logits_split_in_seq, gather_dim=1, unpad_dim=1, padding_size=pad_size) + + # 2. perform normal forward + set_ulysses_sequence_parallel_group(None) + input_ids_full = copy.deepcopy(input_ids_rmpad) + position_ids_full = copy.deepcopy(position_ids_rmpad) + model_no_sp = copy.deepcopy(model) + logits_rmpad_local = model_no_sp( + input_ids_full, position_ids=position_ids_full, use_cache=False + ).logits # (1, total_nnz, vocab_size) + + mean_local = logits_rmpad_local.mean() + mean_full = logits_full.mean() + + mean_full.backward() + mean_local.backward() + + # 3. check the gradients + grad = model.model.layers[0].self_attn.q_proj.weight.grad + grad_full = model_no_sp.model.layers[0].self_attn.q_proj.weight.grad + torch.testing.assert_close(mean_local, mean_full, rtol=1e-2, atol=3e-5) + # The check should be less strict because the gradient is not an averaged value. + torch.testing.assert_close(grad, grad_full, rtol=1e-2, atol=1e-3) + + +if __name__ == "__main__": + pytest.main([__file__, "-svv"]) diff --git a/e2c/verl/tests/single_controller/__init__.py b/e2c/verl/tests/single_controller/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1cd1e8433dffa0b3ba420be3e346f4f5cd062014 --- /dev/null +++ b/e2c/verl/tests/single_controller/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/tests/single_controller/base/test_decorator.py b/e2c/verl/tests/single_controller/base/test_decorator.py new file mode 100644 index 0000000000000000000000000000000000000000..5447d65ce0ecfad235d63c3c8ca02d88c4c7a9e7 --- /dev/null +++ b/e2c/verl/tests/single_controller/base/test_decorator.py @@ -0,0 +1,76 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import verl.single_controller.base.decorator as decorator_module +from verl.single_controller.base.decorator import ( + DISPATCH_MODE_FN_REGISTRY, + Dispatch, + _check_dispatch_mode, + get_predefined_dispatch_fn, + register_dispatch_mode, + update_dispatch_mode, +) + + +@pytest.fixture +def reset_dispatch_registry(): + # Store original state + original_registry = DISPATCH_MODE_FN_REGISTRY.copy() + yield + # Reset registry after test + decorator_module.DISPATCH_MODE_FN_REGISTRY.clear() + decorator_module.DISPATCH_MODE_FN_REGISTRY.update(original_registry) + + +def test_register_new_dispatch_mode(reset_dispatch_registry): + # Test registration + def dummy_dispatch(worker_group, *args, **kwargs): + return args, kwargs + + def dummy_collect(worker_group, output): + return output + + register_dispatch_mode("TEST_MODE", dummy_dispatch, dummy_collect) + + # Verify enum extension + _check_dispatch_mode(Dispatch.TEST_MODE) + + # Verify registry update + assert get_predefined_dispatch_fn(Dispatch.TEST_MODE) == { + "dispatch_fn": dummy_dispatch, + "collect_fn": dummy_collect, + } + # Clean up + Dispatch.remove("TEST_MODE") + + +def test_update_existing_dispatch_mode(reset_dispatch_registry): + # Store original implementation + original_mode = Dispatch.ONE_TO_ALL + + # New implementations + def new_dispatch(worker_group, *args, **kwargs): + return args, kwargs + + def new_collect(worker_group, output): + return output + + # Test update= + update_dispatch_mode(original_mode, new_dispatch, new_collect) + + # Verify update + assert get_predefined_dispatch_fn(original_mode)["dispatch_fn"] == new_dispatch + assert get_predefined_dispatch_fn(original_mode)["collect_fn"] == new_collect diff --git a/e2c/verl/tests/single_controller/check_worker_alive/main.py b/e2c/verl/tests/single_controller/check_worker_alive/main.py new file mode 100644 index 0000000000000000000000000000000000000000..cbdee9a8d6cf98544efc8abeb9555a66a2fd70ee --- /dev/null +++ b/e2c/verl/tests/single_controller/check_worker_alive/main.py @@ -0,0 +1,64 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import time + +import ray + +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.base.worker import Worker +from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + + +@ray.remote +class TestActor(Worker): + def __init__(self) -> None: + super().__init__() + + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + def foo(self, wait_time): + time.sleep(wait_time) + sys.exit(1) + + +if __name__ == "__main__": + wait_time = int(os.getenv("WAIT_TIME", "10")) + + ray.init() + + # test single-node-no-partition + print("test single-node-no-partition") + resource_pool = RayResourcePool([2], use_gpu=False) + class_with_args = RayClassWithInitArgs(cls=TestActor) + + print("create worker group") + wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="test") + + wg.start_worker_aliveness_check(1) + time.sleep(1) + + print(time.time(), "start foo") + + _ = wg.foo(wait_time) + print("foo started") + + print( + time.time(), + f"wait 6x wait time {wait_time * 6} to let signal returned to process but still not exceed process wait time", + ) + time.sleep(wait_time * 6) + + ray.shutdown() diff --git a/e2c/verl/tests/single_controller/detached_worker/README.md b/e2c/verl/tests/single_controller/detached_worker/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b06c4c6143e01d071458f7416033872d41d71031 --- /dev/null +++ b/e2c/verl/tests/single_controller/detached_worker/README.md @@ -0,0 +1,14 @@ +# Detached Worker +## How to run (Only on a single node) +- Start a local ray cluster: +```bash +ray start --head --port=6379 +``` +- Run the server +```bash +python3 server.py +``` +- On another terminal, Run the client +```bash +python3 client.py +``` diff --git a/e2c/verl/tests/single_controller/detached_worker/client.py b/e2c/verl/tests/single_controller/detached_worker/client.py new file mode 100644 index 0000000000000000000000000000000000000000..8c78aaf5d37f6ca5aced3ba5a42b64218cb950e1 --- /dev/null +++ b/e2c/verl/tests/single_controller/detached_worker/client.py @@ -0,0 +1,56 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +In client, we can get the server handler and send RPC request +""" + +import ray +import torch +from server import Trainer +from tensordict import TensorDict + +from verl import DataProto +from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup + + +def compute_position_id_with_mask(mask): + return torch.clip(torch.cumsum(mask, dim=-1) - 1, min=0, max=None) + + +if __name__ == "__main__": + ray.init(address="auto", namespace="verl") + # get the worker group using names + worker_names = ["trainerTrainer_0:0", "trainerTrainer_0:1"] + cls_with_init_args = RayClassWithInitArgs(cls=Trainer) + worker_group = RayWorkerGroup.from_detached(worker_names=worker_names, ray_cls_with_init=cls_with_init_args) + + batch_size = 16 + sequence_length = 1024 + + # give Trainer some data to train + input_ids = torch.randint(low=0, high=256, size=(batch_size, sequence_length), dtype=torch.int64, device="cuda") + attention_mask = torch.ones_like(input_ids) + position_ids = compute_position_id_with_mask(attention_mask) + + data = DataProto( + batch=TensorDict( + {"input_ids": input_ids, "attention_mask": attention_mask, "position_ids": position_ids}, + batch_size=batch_size, + ), + meta_info={}, + ) + + output = worker_group.train_model(data) + + print(output) diff --git a/e2c/verl/tests/single_controller/detached_worker/run.sh b/e2c/verl/tests/single_controller/detached_worker/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..a3c6387933262694bf3534066b4310fda0a9fea3 --- /dev/null +++ b/e2c/verl/tests/single_controller/detached_worker/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash +ray start --head --port=6379 +python3 server.py +python3 client.py +ray stop --force \ No newline at end of file diff --git a/e2c/verl/tests/single_controller/detached_worker/server.py b/e2c/verl/tests/single_controller/detached_worker/server.py new file mode 100644 index 0000000000000000000000000000000000000000..a25c41ed7ccd683591a89c7df7f27f5b87f55107 --- /dev/null +++ b/e2c/verl/tests/single_controller/detached_worker/server.py @@ -0,0 +1,153 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Server starts a Trainer. Client sends data to the server to train. +""" + +import os + +os.environ["MEGATRON_USE_CUDA_TIMER"] = "0" +os.environ["MEGATRON_START_PROCESS_TIMER"] = "False" +os.environ["NCCL_DEBUG"] = "WARN" + +import ray +import torch +from megatron.core import parallel_state as mpu +from megatron.core import tensor_parallel +from megatron.core.models.gpt.gpt_model import ModelType +from omegaconf import OmegaConf +from tensordict import TensorDict +from torch import nn +from transformers import LlamaConfig + +from verl import DataProto +from verl.models.llama.megatron import ParallelLlamaForCausalLMRmPadPP +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup +from verl.utils.megatron.optimizer import get_megatron_optimizer, init_megatron_optim_config +from verl.utils.megatron_utils import get_model, mcore_model_parallel_config + + +@ray.remote +class Trainer(Worker): + def __init__(self): + super().__init__() + + if not torch.distributed.is_initialized(): + rank = int(os.environ["LOCAL_RANK"]) + torch.distributed.init_process_group(backend="nccl") + torch.cuda.set_device(rank) + + mpu.initialize_model_parallel( + tensor_model_parallel_size=2, + pipeline_model_parallel_size=1, + virtual_pipeline_model_parallel_size=None, + pipeline_model_parallel_split_rank=None, + use_sharp=False, + context_parallel_size=1, + expert_model_parallel_size=1, + nccl_communicator_config_path=None, + ) + tensor_parallel.model_parallel_cuda_manual_seed(10) + + is_collect = ( + mpu.get_tensor_model_parallel_rank() == 0 + and mpu.get_pipeline_model_parallel_rank() == mpu.get_pipeline_model_parallel_world_size() - 1 + and mpu.get_context_parallel_rank() == 0 + ) + self._register_dispatch_collect_info( + mesh_name="train", dp_rank=mpu.get_data_parallel_rank(), is_collect=is_collect + ) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + actor_model_config = LlamaConfig( + vocab_size=256, + hidden_size=2048, + intermediate_size=5504, + num_hidden_layers=24, + num_attention_heads=16, + num_key_value_heads=16, + ) + + megatron_config = mcore_model_parallel_config(sequence_parallel=True, params_dtype=torch.bfloat16) + self.megatron_config = megatron_config + + def megatron_actor_model_provider(pre_process, post_process): + # vpp is not supported yet because it will hang for some reason. Need debugging + # this_megatron_config = copy.deepcopy(megatron_config) + # this_megatron_config.virtual_pipeline_model_parallel_rank = vpp_rank + parallel_model = ParallelLlamaForCausalLMRmPadPP( + config=actor_model_config, + megatron_config=megatron_config, + pre_process=pre_process, + post_process=post_process, + ) + parallel_model.cuda() + return parallel_model + + actor_module = get_model( + model_provider_func=megatron_actor_model_provider, + model_type=ModelType.encoder_or_decoder, + wrap_with_ddp=True, + ) + actor_module = nn.ModuleList(actor_module) + + optim_config = OmegaConf.create({"lr": 1e-6, "clip_grad": 1.0}) + + optim_config = init_megatron_optim_config(optim_config) + self.optimizer_config = optim_config + actor_optimizer = get_megatron_optimizer(model=actor_module, config=optim_config) + + self.model = actor_module[0] + self.optimizer = actor_optimizer + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train")) + def train_model(self, data: DataProto) -> DataProto: + input_ids = data.batch["input_ids"] + attention_mask = data.batch["attention_mask"] + position_ids = data.batch["position_ids"] + + self.optimizer.zero_grad() + self.model.zero_grad_buffer( + zero_buffer=(not self.optimizer_config.use_distributed_optimizer) + ) # use use_contiguous_buffers_in_local_ddp and no overlap_dp_param_comm + # update for 1 iteration + output = self.model(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids).logits + output.mean().backward() + + update_successful, grad_norm, num_zeros_in_grad = self.optimizer.step( + self.megatron_config, self.megatron_config.timers + ) + + return DataProto(batch=TensorDict({"loss": output.detach()}, batch_size=output.shape[0])) + + +if __name__ == "__main__": + ray.init(address="auto", namespace="verl") + + resource_pool = RayResourcePool(process_on_nodes=[2], detached=True) + cls_with_init_args = RayClassWithInitArgs(cls=Trainer) + worker_group = RayWorkerGroup( + resource_pool=resource_pool, + ray_cls_with_init=cls_with_init_args, + name_prefix="trainer", + detached=True, + ) + + worker_group.init_model() + + worker_names = worker_group.worker_names + print(worker_names) diff --git a/e2c/verl/tests/single_controller/test_auto_padding_on_cpu.py b/e2c/verl/tests/single_controller/test_auto_padding_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..b60e719c98a5ee42918b55326f2f98c443c7dd9d --- /dev/null +++ b/e2c/verl/tests/single_controller/test_auto_padding_on_cpu.py @@ -0,0 +1,152 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import ray +import torch + +from verl import DataProto +from verl.protocol import DataProtoConfig +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + +# or set env var VERL_AUTO_PADDING = "1" / "true" +DataProtoConfig.auto_padding = True + + +@ray.remote +class Actor(Worker): + def __init__(self) -> None: + super().__init__() + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def add(self, data: DataProto): + data.batch["a"] += self.rank + return data + + +def test_auto_padding(): + ray.init(num_cpus=100) + + chunk_size = 4 + actor_cls = RayClassWithInitArgs(cls=Actor) + resource_pool = RayResourcePool(process_on_nodes=[chunk_size], use_gpu=False) + actor_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=actor_cls) + + # test locally first + for test_size in range(4, 20): + local_data = DataProto.from_dict({"a": torch.zeros(test_size)}, {"na": np.zeros(test_size, dtype=object)}) + # print(f"before padding, local_data = {local_data}") + padding_size = (chunk_size - (test_size % chunk_size)) if (test_size % chunk_size > 0) else 0 + local_data.padding(padding_size) + # print(f"after padding, local_data = {local_data}") + assert len(local_data) == len(local_data) + len(local_data) % chunk_size, ( + f"expecting padded length to be {len(local_data) + len(local_data) % chunk_size}, but got {len(local_data)}" + ) + chunked = local_data.chunk(chunk_size) + assert len(chunked) == chunk_size, f"during test_size = {test_size}, expecting {chunk_size}, got {chunked}" + for dp in chunked: + assert len(dp) == test_size // chunk_size + bool(test_size % chunk_size), ( + f"test size = {test_size}, expecting dp to be length of " + f"{test_size // chunk_size + bool(test_size % chunk_size)}, but got {len(dp)}: {dp} {chunked}" + ) + + # test with RayWorkerGroup method decorated as dispatch_mode=Dispatch.DP_COMPUTE_PROTO + data = DataProto.from_dict({"a": torch.zeros(10)}, {"na": np.array([str(i) for i in range(10)], dtype=object)}) + output = actor_wg.add(data) + + print(output.batch["a"]) + assert len(output) == 10, "Failed in args split and padding." + + data = DataProto.from_dict({"a": torch.zeros(10)}, {"na": np.array([str(i) for i in range(10)], dtype=object)}) + output = actor_wg.add(data=data) + + print(output.batch["a"]) + assert len(output) == 10, "Failed in kwargs split and padding." + + data = DataProto.from_dict({"a": torch.zeros(1)}, {"na": np.array([str(i) for i in range(1)], dtype=object)}) + output = actor_wg.add(data) + + print(output.batch["a"]) + assert len(output) == 1, "Failed in args split and padding." + + data = DataProto.from_dict({"a": torch.zeros(1)}, {"na": np.array([str(i) for i in range(1)], dtype=object)}) + output = actor_wg.add(data=data) + + print(output.batch["a"]) + assert len(output) == 1, "Failed in kwargs split and padding." + + data = DataProto.from_dict({"a": torch.zeros(8)}, {"na": np.array([str(i) for i in range(8)], dtype=object)}) + output = actor_wg.add(data) + + print(output.batch["a"]) + assert len(output) == 8, "Failed in args split and padding." + + data = DataProto.from_dict({"a": torch.zeros(8)}, {"na": np.array([str(i) for i in range(8)], dtype=object)}) + output = actor_wg.add(data=data) + + print(output.batch["a"]) + assert len(output) == 8, "Failed in kwargs split and padding." + + # test data proto specific config + DataProtoConfig.auto_padding = False + + data = DataProto.from_dict( + {"a": torch.zeros(10)}, {"na": np.array([str(i) for i in range(10)], dtype=object)}, auto_padding=True + ) + output = actor_wg.add(data) + print(output.batch["a"]) + assert len(output) == 10, "Failed in args split and padding." + + data = DataProto.from_dict( + {"a": torch.zeros(10)}, {"na": np.array([str(i) for i in range(10)], dtype=object)}, auto_padding=True + ) + output = actor_wg.add(data=data) + print(output.batch["a"]) + assert len(output) == 10, "Failed in kwargs split and padding." + + data = DataProto.from_single_dict( + {"a": torch.zeros(1), "na": np.array([str(i) for i in range(1)], dtype=object)}, auto_padding=True + ) + output = actor_wg.add(data) + + print(output.batch["a"]) + assert len(output) == 1, "Failed in args split and padding." + + data = DataProto.from_single_dict( + {"a": torch.zeros(1), "na": np.array([str(i) for i in range(1)], dtype=object)}, auto_padding=True + ) + output = actor_wg.add(data=data) + + print(output.batch["a"]) + assert len(output) == 1, "Failed in kwargs split and padding." + + data = DataProto.from_single_dict({"a": torch.zeros(8), "na": np.array([str(i) for i in range(8)], dtype=object)}) + output = actor_wg.add(data) + + print(output.batch["a"]) + assert len(output) == 8, "Failed in args split and padding." + + data = DataProto.from_single_dict({"a": torch.zeros(8), "na": np.array([str(i) for i in range(8)], dtype=object)}) + output = actor_wg.add(data=data) + + print(output.batch["a"]) + assert len(output) == 8, "Failed in kwargs split and padding." + + ray.shutdown() + + +if __name__ == "__main__": + test_auto_padding() diff --git a/e2c/verl/tests/single_controller/test_colocated_workers.py b/e2c/verl/tests/single_controller/test_colocated_workers.py new file mode 100644 index 0000000000000000000000000000000000000000..cdaa74768ff5a5bb404bf086f595cbf3ab024b64 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_colocated_workers.py @@ -0,0 +1,83 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ray + +from verl import DataProto +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.ray.base import ( + RayClassWithInitArgs, + RayResourcePool, + RayWorkerGroup, + create_colocated_worker_cls, +) + + +@ray.remote +class Actor(Worker): + def __init__(self) -> None: + super().__init__() + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def add(self, data: DataProto): + data.batch["a"] += self.rank + return data + + +@ray.remote +class Critic(Worker): + def __init__(self, config) -> None: + super().__init__() + self.config = config + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + async def sub(self, data: DataProto): + data.batch["a"] -= self.config["b"] + return data + + +def test_colocated_workers(): + ray.init() + + import torch + + data = DataProto.from_dict({"a": torch.zeros(10)}) + # create separate workers on the same resource pool + actor_cls = RayClassWithInitArgs(cls=Actor) + critic_cls = RayClassWithInitArgs(cls=Critic, config={"b": 10}) + resource_pool = RayResourcePool(process_on_nodes=[2]) + + actor_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=actor_cls) + critic_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=critic_cls) + + expected_actor_output = actor_wg.add(data) + expected_critic_output = critic_wg.sub(data) + + # create colocated workers + cls_dict = {"actor": actor_cls, "critic": critic_cls} + ray_cls_with_init = create_colocated_worker_cls(cls_dict) + wg_dict = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) + spawn_wg = wg_dict.spawn(prefix_set=cls_dict.keys()) + + colocated_actor_wg = spawn_wg["actor"] + colocated_critic_wg = spawn_wg["critic"] + + actor_output = colocated_actor_wg.add(data) + critic_output = colocated_critic_wg.sub(data) + + torch.testing.assert_close(expected_actor_output.batch, actor_output.batch, atol=0, rtol=0) + torch.testing.assert_close(expected_critic_output.batch, critic_output.batch, atol=0, rtol=0) + + ray.shutdown() diff --git a/e2c/verl/tests/single_controller/test_colocated_workers_fused.py b/e2c/verl/tests/single_controller/test_colocated_workers_fused.py new file mode 100644 index 0000000000000000000000000000000000000000..93b1a728e47f6a0374c3a9ddd293d117e6d2baf6 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_colocated_workers_fused.py @@ -0,0 +1,83 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ray + +from verl import DataProto +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.ray.base import ( + RayClassWithInitArgs, + RayResourcePool, + RayWorkerGroup, + create_colocated_worker_cls_fused, +) + + +@ray.remote +class Actor(Worker): + def __init__(self) -> None: + super().__init__() + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def add(self, data: DataProto): + data.batch["a"] += self.rank + return data + + +@ray.remote +class Critic(Worker): + def __init__(self, config) -> None: + super().__init__() + self.config = config + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def sub(self, data: DataProto): + data.batch["a"] -= self.config["b"] + return data + + +def test_colocated_workers_fused(): + ray.init() + + import torch + + data = DataProto.from_dict({"a": torch.zeros(10)}) + # create separate workers on the same resource pool + actor_cls = RayClassWithInitArgs(cls=Actor) + critic_cls = RayClassWithInitArgs(cls=Critic, config={"b": 10}) + resource_pool = RayResourcePool(process_on_nodes=[2]) + + actor_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=actor_cls) + critic_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=critic_cls) + + expected_actor_output = actor_wg.add(data) + expected_critic_output = critic_wg.sub(data) + + # create colocated workers + cls_dict = {"actor": actor_cls, "critic": critic_cls} + ray_cls_with_init = create_colocated_worker_cls_fused(cls_dict) + wg_dict = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) + spawn_wg = wg_dict.spawn(prefix_set=cls_dict.keys()) + + colocated_actor_wg = spawn_wg["actor"] + colocated_critic_wg = spawn_wg["critic"] + + actor_output = colocated_actor_wg.add(data) + critic_output = colocated_critic_wg.sub(data) + + torch.testing.assert_close(expected_actor_output.batch, actor_output.batch, atol=0, rtol=0) + torch.testing.assert_close(expected_critic_output.batch, critic_output.batch, atol=0, rtol=0) + + ray.shutdown() diff --git a/e2c/verl/tests/single_controller/test_data_transfer.py b/e2c/verl/tests/single_controller/test_data_transfer.py new file mode 100644 index 0000000000000000000000000000000000000000..13777b0bd02ea9df23b6ffe77d6465bf1b7b85d8 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_data_transfer.py @@ -0,0 +1,107 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +In this test, we instantiate a data parallel worker with 8 GPUs +""" + +import ray +import tensordict +import torch +from codetiming import Timer +from torch import distributed as dist + +from verl import DataProto +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup +from verl.utils.ray_utils import parallel_put + + +@ray.remote +class DummyWorker(Worker): + def __init__(self): + super().__init__() + dist.init_process_group() + + @register(dispatch_mode=Dispatch.DP_COMPUTE, blocking=False) + def do_nothing(self, data): + for key in data.batch.keys(): + data.batch[key] += 1 + if tensordict.__version__ >= "0.5.0": + data.batch = data.batch.consolidate() + return data + + +def test_data_transfer(): + ray.init() + # construct resource pool + resource_pool = RayResourcePool([8]) + cls_with_init = RayClassWithInitArgs(cls=DummyWorker) + # construct worker group + wg = RayWorkerGroup(resource_pool, cls_with_init) + + # this is real dataset size + batch_size = 4096 + seqlen = 32768 + + data_dict = {} + + for i in range(2): + data_dict[str(i)] = torch.randint(0, 10000, (batch_size, seqlen)) + + data = DataProto.from_dict(tensors=data_dict) + + print(data) + + # we manually split data here and send to each worker + data_list = data.chunk(wg.world_size) + + for i in range(wg.world_size): + # consolidate is necessary + if tensordict.__version__ >= "0.5.0": + data_list[i].batch = data_list[i].batch.consolidate() + + with Timer(name="ray.pickle", initial_text=True): + for i in range(wg.world_size): + ray.cloudpickle.pickle.dumps(data_list[i]) + + with Timer(name="raw.pickle", initial_text=True): + import pickle + + for i in range(wg.world_size): + pickle.dumps(data_list[i]) + + # we put in advance + with Timer(name="put", initial_text=True): + # takes around 40 seconds + data_list_ref = parallel_put(data_list) + # for i in range(wg.world_size): + # data_list[i] = ray.put(data_list[i]) + + with Timer(name="launch", initial_text=True): + output_ref = wg.do_nothing(data_list_ref) + + with Timer(name="get", initial_text=True): + # takes around 40 seconds + output_lst = ray.get(output_ref) + + for input_data, output_data in zip(data_list, output_lst, strict=True): + for key in input_data.batch.keys(): + assert torch.all(torch.eq(input_data.batch[key] + 1, output_data.batch[key])), ( + input_data.batch[key], + output_data.batch[key], + key, + ) + + ray.shutdown() diff --git a/e2c/verl/tests/single_controller/test_decorator_on_cpu.py b/e2c/verl/tests/single_controller/test_decorator_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..4dfec6331f0a603a9ad52084dc93cf19acba5250 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_decorator_on_cpu.py @@ -0,0 +1,141 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import time + +import pytest +import ray +import torch +from tensordict import TensorDict + +from verl.protocol import DataProto, DataProtoFuture +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.base.worker import Worker +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + + +# Pytest fixture for Ray setup/teardown +@pytest.fixture +def ray_init_shutdown(): + ray.init(num_cpus=100) + yield + ray.shutdown() + + +# Define a simple worker for testing +@ray.remote +class DecoratorTestWorker(Worker): + def __init__(self, initial_value=0): + super().__init__() + self.value = initial_value + # Simulate some setup if needed + time.sleep(0.1) # Ensure worker init completes + + # Test method for synchronous DP compute (default behavior) + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) + def dp_compute(self, data: DataProto) -> DataProto: + time.sleep(0.1) # Simulate work + rank_value = torch.tensor(self.rank, device=data.batch["input"].device, dtype=data.batch["input"].dtype) + data.batch["output"] = data.batch["input"] + self.value + rank_value + return data + + # Test async def method with DP compute (default behavior) + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO, blocking=False) + async def async_dp_compute(self, data: DataProto) -> DataProto: + # Simulate async work + await asyncio.sleep(0.1) # Simulate async work + rank_value = torch.tensor(self.rank, device=data.batch["input"].device, dtype=data.batch["input"].dtype) + data.batch["output_async"] = data.batch["input"] * 2 + self.value + rank_value + return data + + +# Test function for synchronous DP compute +def test_decorator_dp_compute(ray_init_shutdown): + """ + Tests the default behavior of a synchronous decorated method with DP_COMPUTE_PROTO. + Verifies the result correctness. + """ + num_workers = 2 + resource_pool = RayResourcePool([num_workers], use_gpu=False, max_colocate_count=1) # Use CPU for simplicity + cls_with_args = RayClassWithInitArgs(cls=DecoratorTestWorker, initial_value=10) + worker_group = RayWorkerGroup( + resource_pool, cls_with_args, name_prefix=f"decorator_test_sync_dp_{int(time.time())}" + ) + + # Prepare input data (size 4, for 2 workers) + input_tensor = torch.arange(4, dtype=torch.float32) + data = DataProto(batch=TensorDict({"input": input_tensor}, batch_size=[4])) + + # Call the decorated method + output = worker_group.dp_compute(data) + + # Assert the result correctness + assert isinstance(output, DataProto), "Expected DataProto result" + assert "output" in output.batch.keys() + assert len(output) == len(data), "Output length should match input length" + + # Expected output calculation for DP_COMPUTE_PROTO with 2 workers + # Worker 0 gets data[0:2], Worker 1 gets data[2:4] + # Worker 0 adds initial_value(10) + rank(0) = 10 + # Worker 1 adds initial_value(10) + rank(1) = 11 + expected_output_part1 = torch.tensor([0, 1], dtype=torch.float32) + 10 + 0 + expected_output_part2 = torch.tensor([2, 3], dtype=torch.float32) + 10 + 1 + expected_output = torch.cat([expected_output_part1, expected_output_part2]) + + torch.testing.assert_close(output.batch["output"], expected_output, msg="Sync DP compute output data mismatch") + + +# Test function for async def method with DP compute +def test_decorator_async_function(ray_init_shutdown): + """ + Tests the decorator with an `async def` method using DP_COMPUTE_PROTO. + Verifies that the call returns a future and the result is correct after .get(). + """ + num_workers = 2 + resource_pool = RayResourcePool([num_workers], use_gpu=False, max_colocate_count=1) + cls_with_args = RayClassWithInitArgs(cls=DecoratorTestWorker, initial_value=5) + worker_group = RayWorkerGroup( + resource_pool, cls_with_args, name_prefix=f"decorator_test_async_dp_{int(time.time())}" + ) + + # Prepare input data (size 4, for 2 workers) + input_tensor = torch.arange(4, dtype=torch.float32) + data = DataProto(batch=TensorDict({"input": input_tensor}, batch_size=[4])) + + # Call the async decorated method - this should return a future + future_output: DataProtoFuture = worker_group.async_dp_compute(data) + + # Assert that the call returned a future + assert isinstance(future_output, DataProtoFuture), "Expected DataProtoFuture for async def call" + + # Get the result (this should block) + result_data = future_output.get() + + # Assert the result correctness + assert isinstance(result_data, DataProto) + assert "output_async" in result_data.batch.keys() + assert len(result_data) == len(data), "Output length should match input length" + + # Expected output calculation for DP_COMPUTE_PROTO with 2 workers + # Worker 0 gets data[0:2], Worker 1 gets data[2:4] + # Worker 0 calculates: input * 2 + initial_value(5) + rank(0) + # Worker 1 calculates: input * 2 + initial_value(5) + rank(1) + expected_output_part1 = (torch.tensor([0, 1], dtype=torch.float32) * 2) + 5 + 0 + expected_output_part2 = (torch.tensor([2, 3], dtype=torch.float32) * 2) + 5 + 1 + expected_output = torch.cat([expected_output_part1, expected_output_part2]) + + torch.testing.assert_close( + result_data.batch["output_async"], expected_output, msg="Async DP compute output data mismatch" + ) diff --git a/e2c/verl/tests/single_controller/test_device_mesh_register.py b/e2c/verl/tests/single_controller/test_device_mesh_register.py new file mode 100644 index 0000000000000000000000000000000000000000..2b56358d69f58644f32d88707bd10241dc5d67b3 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_device_mesh_register.py @@ -0,0 +1,100 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import ray +import torch + +from verl import DataProto +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import make_nd_compute_dataproto_dispatch_fn, register + + +@ray.remote +class TestActor(Worker): + def __init__(self): + super().__init__() + + import torch.distributed + + torch.distributed.init_process_group(backend="nccl") + self.infer_device_mesh = torch.distributed.device_mesh.init_device_mesh( + device_type="cuda", mesh_shape=[2, 4], mesh_dim_names=["dp", "tp"] + ) + self.train_device_mesh = torch.distributed.device_mesh.init_device_mesh( + device_type="cuda", mesh_shape=[2, 2, 2], mesh_dim_names=["pp", "dp", "tp"] + ) + + self._register_dispatch_collect_info( + "infer", + dp_rank=self.infer_device_mesh["dp"].get_local_rank(), + is_collect=self.infer_device_mesh["tp"].get_local_rank() == 0, + ) + self._register_dispatch_collect_info( + "train", + dp_rank=self.train_device_mesh["dp"].get_local_rank(), + is_collect=self.train_device_mesh["tp"].get_local_rank() == 0 + and self.train_device_mesh["pp"].get_local_rank() == 1, + ) + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="infer")) + def generate_data_proto(self, data: DataProto): + tp_rank = self.infer_device_mesh["tp"].get_local_rank() + dp_rank = self.infer_device_mesh["dp"].get_local_rank() + data.batch["a"] += (tp_rank + 1) * dp_rank + return data + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train")) + def train_data_proto(self, data: DataProto): + tp_rank = self.train_device_mesh["tp"].get_local_rank() + dp_rank = self.train_device_mesh["dp"].get_local_rank() + pp_rank = self.train_device_mesh["pp"].get_local_rank() + data.batch["a"] += (tp_rank + 1) * (dp_rank + 2) * (pp_rank + 3) + # tp rank 0, pp rank 1, dp rank 0, output data added: 8 + 3 = 11 + # tp rank 0, pp rank 1, dp rank 1, output data added: 12 + 4 = 16 + return data + + +def test_dist_global_info_wg(): + # create a worker group with size 8 + # register a infer dist info with tp=4, dp=2 + # register a train dist info with tp=2, dp=2, pp=2 + # test the correctness of data dispatch and computation + from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + + ray.init() + + ray_cls = RayClassWithInitArgs(TestActor) + resource_pool = RayResourcePool(process_on_nodes=[8]) + wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls) + + infer_input_data_proto = DataProto.from_single_dict(data={"a": torch.tensor([1, 2])}) + infer_output_data_proto = wg.generate_data_proto(infer_input_data_proto) + + assert wg._dispatch_info["infer"] == [0, 0, 0, 0, 1, 1, 1, 1] + + assert torch.all(torch.eq(infer_output_data_proto.batch["a"], torch.tensor([1, 3]))) + + train_input_data_proto = DataProto.from_single_dict(data={"a": torch.tensor([3, 4])}) + train_output_data_proto = wg.train_data_proto(train_input_data_proto) + + assert wg._dispatch_info["train"] == [0, 0, 1, 1, 0, 0, 1, 1] + + assert torch.all(torch.eq(train_output_data_proto.batch["a"], torch.tensor([11, 16]))) + + ray.shutdown() + + +if __name__ == "__main__": + test_dist_global_info_wg() diff --git a/e2c/verl/tests/single_controller/test_driverfunc_to_worker.py b/e2c/verl/tests/single_controller/test_driverfunc_to_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..a38d790d62516326e373e4980544a3b81146bb04 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_driverfunc_to_worker.py @@ -0,0 +1,84 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import ray +import torch +from tensordict import TensorDict + +from verl import DataProto +from verl.single_controller.base.worker import Worker +from verl.single_controller.ray import RayWorkerGroup +from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool + +os.environ["RAY_DEDUP_LOGS"] = "0" +os.environ["NCCL_DEBUG"] = "WARN" + + +@ray.remote +class ModelActor(Worker): + def __init__(self): + pass + + +class HackSelf: + def __init__(self): + pass + + +def get_aux_metrics(self, test_proto): + sequence_ids = test_proto.batch["sequence_ids"] + decode_count = [] + for i in range(sequence_ids.size(0)): + decode_count.append(len(sequence_ids[i].tolist())) + ret_proto = DataProto( + batch=TensorDict( + {"sequence_ids": sequence_ids, "decode_count": torch.tensor(decode_count)}, batch_size=sequence_ids.size(0) + ) + ) + return ret_proto + + +def test(): + # construct model + ray.init() + + # create 2 workers, each hold a GPU + resource_pool = RayResourcePool([2], use_gpu=True, name_prefix="a") + + class_with_args = RayClassWithInitArgs(cls=ModelActor) + shard_wg = RayWorkerGroup(resource_pool, class_with_args) + + test_bs = 8 + test_proto = DataProto( + TensorDict( + { + "sequence_ids": torch.ones([test_bs, 2048], dtype=torch.int64), + }, + batch_size=test_bs, + ), + meta_info={"query_length": 1536}, + ) + + # Sharding among different ranks + ret_proto1 = shard_wg.execute_with_func_generator(get_aux_metrics, test_proto) + + # compare execute on driver + hs = HackSelf() + ret_proto2 = get_aux_metrics(hs, test_proto) + + torch.testing.assert_close(ret_proto1.batch["decode_count"], ret_proto2.batch["decode_count"]) + + ray.shutdown() diff --git a/e2c/verl/tests/single_controller/test_fused_workers_on_cpu.py b/e2c/verl/tests/single_controller/test_fused_workers_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..527ddc102419bae10f01684a9b4e3e3b13530522 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_fused_workers_on_cpu.py @@ -0,0 +1,90 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ray + +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.ray.base import ( + RayClassWithInitArgs, + RayResourcePool, + RayWorkerGroup, + create_colocated_worker_raw_cls, +) + + +@ray.remote +class Actor(Worker): + def __init__(self) -> None: + super().__init__() + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def add(self, x): + x += self.rank + return x + + +@ray.remote +class Critic(Worker): + def __init__(self, val) -> None: + super().__init__() + self.val = val + + @register(dispatch_mode=Dispatch.ALL_TO_ALL) + def sub(self, x): + x -= self.val + return x + + +actor_cls = RayClassWithInitArgs(cls=Actor) +critic_cls = RayClassWithInitArgs(cls=Critic, val=10) +cls_dict = {"actor": actor_cls, "critic": critic_cls} +FusedBaseClass = create_colocated_worker_raw_cls(cls_dict) + + +@ray.remote +class HybridWorker(FusedBaseClass): + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def foo(self, x): + return self.critic.sub(self.actor.add(x)) + + +def test_fused_workers(): + ray.init(num_cpus=100) + + # create separate workers on the same resource pool + process_on_nodes = [2] + resource_pool = RayResourcePool(process_on_nodes=process_on_nodes, use_gpu=False) + + # create colocated workers + hybrid_cls_with_init = RayClassWithInitArgs(cls=HybridWorker) + hybrid_cls_with_init.fused_worker_used = True + + fused_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=hybrid_cls_with_init) + fused_wg.fuse(cls_dict.keys()) + + x = fused_wg.actor.add(0.1) + print(x) + y = fused_wg.critic.sub(x) + print(y) + z = fused_wg.foo(0.1) + print(z) + for i, j in zip(y, z, strict=True): + assert i == j + + ray.shutdown() + + +if __name__ == "__main__": + test_fused_workers() diff --git a/e2c/verl/tests/single_controller/test_high_level_scheduling_api.py b/e2c/verl/tests/single_controller/test_high_level_scheduling_api.py new file mode 100644 index 0000000000000000000000000000000000000000..52cc7c7df4d4545563e0e774de3d46590eee334d --- /dev/null +++ b/e2c/verl/tests/single_controller/test_high_level_scheduling_api.py @@ -0,0 +1,85 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import ray + +from verl.single_controller.base.worker import Worker +from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup, merge_resource_pool + + +@ray.remote +class TestActor(Worker): + # TODO: pass *args and **kwargs is bug prone and not very convincing + def __init__(self, cuda_visible_devices=None) -> None: + super().__init__(cuda_visible_devices) + + def get_node_id(self): + return ray.get_runtime_context().get_node_id() + + +def test(): + ray.init() + + # test single-node-no-partition + print("test single-node-no-partition") + resource_pool = RayResourcePool([8], use_gpu=True) + + class_with_args = RayClassWithInitArgs(cls=TestActor) + + print("create actor worker group") + actor_wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="high_level_api_actor") + print("create critic worker group") + critic_wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="hight_level_api_critic") + print("create rm worker group") + rm_wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="high_level_api_rm") + print("create ref worker group") + ref_wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="high_level_api_ref") + + assert actor_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] + assert critic_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] + assert rm_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] + assert ref_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] + + del actor_wg + del critic_wg + del rm_wg + del ref_wg + + [ray.util.remove_placement_group(pg) for pg in resource_pool.get_placement_groups()] + print("wait 5s to remove placemeng_group") + time.sleep(5) + # test single-node-multi-partition + + print("test single-node-multi-partition") + rm_resource_pool = RayResourcePool([4], use_gpu=True, name_prefix="rm") + ref_resource_pool = RayResourcePool([4], use_gpu=True, name_prefix="ref") + total_resource_pool = merge_resource_pool(rm_resource_pool, ref_resource_pool) + + assert rm_resource_pool.world_size == 4 + assert ref_resource_pool.world_size == 4 + assert total_resource_pool.world_size == 8 + + actor_wg = RayWorkerGroup(total_resource_pool, class_with_args, name_prefix="high_level_api_actor") + critic_wg = RayWorkerGroup(total_resource_pool, class_with_args, name_prefix="high_level_api_critic") + rm_wg = RayWorkerGroup(rm_resource_pool, class_with_args, name_prefix="high_level_api_rm") + ref_wg = RayWorkerGroup(ref_resource_pool, class_with_args, name_prefix="high_level_api_ref") + + assert actor_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] + assert critic_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] + assert rm_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(4)] + assert ref_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(4, 8)] + + ray.shutdown() diff --git a/e2c/verl/tests/single_controller/test_nested_worker.py b/e2c/verl/tests/single_controller/test_nested_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..e35d8f44c126668f648d58b1dc7544edffbcd49c --- /dev/null +++ b/e2c/verl/tests/single_controller/test_nested_worker.py @@ -0,0 +1,68 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import ray + +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.base.worker import Worker +from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + + +class TestActor(Worker): + # TODO: pass *args and **kwargs is bug prone and not very convincing + def __init__(self, x) -> None: + super().__init__() + self.a = x + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def get(self): + return self.a + self.rank + + +class TestHighLevelActor(Worker): + def __init__(self, x=None) -> None: + super().__init__() + self.test_actor = TestActor(x=x) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def get(self): + return self.test_actor.get() + + +def test_nested_worker(): + ray.init(num_cpus=100) + + # create 4 workers, each hold a GPU + resource_pool = RayResourcePool([4], use_gpu=True) + class_with_args = RayClassWithInitArgs(cls=ray.remote(TestActor), x=2) + + worker_group = RayWorkerGroup( + resource_pool=resource_pool, ray_cls_with_init=class_with_args, name_prefix="worker_group_basic" + ) + + output = worker_group.get() + + assert output == [2, 3, 4, 5] + + class_with_args = RayClassWithInitArgs(cls=ray.remote(TestHighLevelActor), x=2) + high_level_worker_group = RayWorkerGroup( + resource_pool=resource_pool, ray_cls_with_init=class_with_args, name_prefix="worker_group_basic_2" + ) + + output_1 = high_level_worker_group.get() + + assert output_1 == [2, 3, 4, 5] + + ray.shutdown() diff --git a/e2c/verl/tests/single_controller/test_ray_collectives.py b/e2c/verl/tests/single_controller/test_ray_collectives.py new file mode 100644 index 0000000000000000000000000000000000000000..3722a8f8029313bad6070d8d0ed2b9a29e4f3770 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_ray_collectives.py @@ -0,0 +1,113 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test for using ray collective group. +Suppose we Actor and Rollout. Actor contains 4 workers and Rollout contains 2 workers. We established a Worker to +Rollout relationship by using collective groups +Actor: rank 0, 1 - Rollout rank 0 +Rollout rank 2, 3 - Rollout rank 1 +Then, we initiate 4 p2p comms from actor to rollout +""" + +import ray +import ray.util.collective as collective +import torch + +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, register +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + + +@ray.remote +class Actor(Worker): + @register(Dispatch.ONE_TO_ALL) + def init(self): + remote_rank = self.rank // 2 + self.group_name = f"A{self.rank}_R{remote_rank}" + collective.init_collective_group(world_size=2, rank=0, backend="nccl", group_name=self.group_name) + + @register(Dispatch.ONE_TO_ALL, blocking=False) + def send_tensors(self): + tensor = torch.ones(size=(4,), dtype=torch.float32, device="cuda") * self.rank + collective.send(tensor=tensor, dst_rank=1, group_name=self.group_name) + + +@ray.remote +class Rollout(Worker): + @register(Dispatch.ONE_TO_ALL) + def init(self): + self.remote_first_rank = self.rank * 2 + self.remote_second_rank = self.remote_first_rank + 1 + self.first_group_name = f"A{self.remote_first_rank}_R{self.rank}" + self.second_group_name = f"A{self.remote_second_rank}_R{self.rank}" + + collective.init_collective_group(world_size=2, rank=1, backend="nccl", group_name=self.first_group_name) + collective.init_collective_group(world_size=2, rank=1, backend="nccl", group_name=self.second_group_name) + + @register(Dispatch.ONE_TO_ALL, blocking=False) + def receive_tensors(self): + self.tensor1 = torch.randn(size=(4,), dtype=torch.float32, device="cuda") + self.tensor2 = torch.randn(size=(4,), dtype=torch.float32, device="cuda") + + collective.recv(self.tensor1, src_rank=0, group_name=self.first_group_name) + collective.recv(self.tensor2, src_rank=0, group_name=self.second_group_name) + + @register(Dispatch.ONE_TO_ALL) + def get_tensors(self): + return {f"src_{self.remote_first_rank}": self.tensor1, f"src_{self.remote_second_rank}": self.tensor2} + + +def test_ray_collective_group(): + ray.init() + + actor_resource_pool = RayResourcePool([4]) + rollout_resource_pool = RayResourcePool([2]) + + actor_cls = RayClassWithInitArgs(cls=Actor) + rollout_cls = RayClassWithInitArgs(cls=Rollout) + + actor_wg = RayWorkerGroup( + resource_pool=actor_resource_pool, ray_cls_with_init=actor_cls, name_prefix="collective_group_actor" + ) + rollout_wg = RayWorkerGroup( + resource_pool=rollout_resource_pool, ray_cls_with_init=rollout_cls, name_prefix="collective_group_rollout" + ) + + actor_wg.init() + rollout_wg.init() + + out1 = actor_wg.send_tensors() + out2 = rollout_wg.receive_tensors() + + # block to wait + ray.get(out1) + ray.get(out2) + + output = rollout_wg.get_tensors() + + rollout_0_output = output[0] + rollout_1_output = output[1] + + output = rollout_0_output | rollout_1_output + + print(output) + + for i in range(4): + assert torch.sum(output[f"src_{i}"]).item() == 4 * i + + ray.shutdown() + + +if __name__ == "__main__": + test_ray_collective_group() diff --git a/e2c/verl/tests/single_controller/test_ray_local_envs_on_cpu.py b/e2c/verl/tests/single_controller/test_ray_local_envs_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..6c51beeaf3f8600387ce14fe63c97a5c804c4237 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_ray_local_envs_on_cpu.py @@ -0,0 +1,91 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +e2e test verl.single_controller.ray +""" + +import os + +import ray + +from verl.single_controller.base.worker import Worker +from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + + +@ray.remote +class TestActor(Worker): + def __init__(self) -> None: + super().__init__() + + def getenv(self, key): + val = os.getenv(key, f"{key} not set") + return val + + +def test_basics(): + ray.init(num_cpus=100) + + # create 4 workers, each hold a GPU + resource_pool = RayResourcePool([4], use_gpu=False) + class_with_args = RayClassWithInitArgs(cls=TestActor) + + worker_group = RayWorkerGroup( + resource_pool=resource_pool, ray_cls_with_init=class_with_args, name_prefix="worker_group_basic" + ) + + output = worker_group.execute_all_sync("getenv", key="RAY_LOCAL_WORLD_SIZE") + assert output == ["4", "4", "4", "4"] + + ray.shutdown() + + +def test_customized_worker_env(): + ray.init(num_cpus=100) + + # create 4 workers, each hold a GPU + resource_pool = RayResourcePool([4], use_gpu=False) + class_with_args = RayClassWithInitArgs(cls=TestActor) + + worker_group = RayWorkerGroup( + resource_pool=resource_pool, + ray_cls_with_init=class_with_args, + name_prefix="worker_group_customized", + worker_env={ + "test_key": "test_value", # new key will be appended + }, + ) + + output = worker_group.execute_all_sync("getenv", key="test_key") + assert output == ["test_value", "test_value", "test_value", "test_value"] + + try: + worker_group = RayWorkerGroup( + resource_pool=resource_pool, + ray_cls_with_init=class_with_args, + name_prefix="worker_group_error", + worker_env={ + "WORLD_SIZE": "100", # override system env will result in error + }, + ) + except ValueError as e: + assert "WORLD_SIZE" in str(e) + else: + raise ValueError("test failed") + + ray.shutdown() + + +if __name__ == "__main__": + test_basics() + test_customized_worker_env() diff --git a/e2c/verl/tests/single_controller/test_ray_utils_on_cpu.py b/e2c/verl/tests/single_controller/test_ray_utils_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..e36497d210f6ec5daa8b9d559987f5dcc3974af2 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_ray_utils_on_cpu.py @@ -0,0 +1,54 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import ray + +from verl.utils.ray_utils import parallel_put + + +# Initialize Ray for testing if not already done globally +@pytest.fixture() +def init_ray(): + ray.init(num_cpus=4) + yield + ray.shutdown() + + +def test_parallel_put_basic(init_ray): + data = [1, "hello", {"a": 2}, [3, 4]] + refs = parallel_put(data) + assert len(refs) == len(data) + retrieved_data = [ray.get(ref) for ref in refs] + assert retrieved_data == data + + +def test_parallel_put_empty(init_ray): + data = [] + with pytest.raises(AssertionError): + _ = parallel_put(data) + + +def test_parallel_put_workers(init_ray): + data = list(range(20)) + # Test with specific number of workers + refs = parallel_put(data, max_workers=4) + assert len(refs) == len(data) + retrieved_data = [ray.get(ref) for ref in refs] + assert retrieved_data == data + # Test with default workers (should cap) + refs_default = parallel_put(data) + assert len(refs_default) == len(data) + retrieved_data_default = [ray.get(ref) for ref in refs_default] + assert retrieved_data_default == data diff --git a/e2c/verl/tests/single_controller/test_rvdz.py b/e2c/verl/tests/single_controller/test_rvdz.py new file mode 100644 index 0000000000000000000000000000000000000000..7dea12f95cd5cb697f5fcfa20a844331bd46e8f3 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_rvdz.py @@ -0,0 +1,51 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ray + + +@ray.remote +class TestWorker: + def __init__(self, rank, world_size, group_name): + self.rank = rank + self.world_size = world_size + self.group_name = group_name + self.communicator = None + + def init(self): + from verl.utils.rendezvous.ray_backend import create_nccl_communicator_in_ray + + self.communicator = create_nccl_communicator_in_ray(self.rank, self.world_size, self.group_name) + + def test(self): + if self.communicator is None: + return None + return self.communicator.rank_id() + + +def test_rvdz(): + ray.init() + + group_name = "test_group" + world_size = 2 + + workers = [TestWorker.options(num_gpus=1).remote(rank, world_size, group_name) for rank in range(world_size)] + + ray.get([worker.init.remote() for worker in workers]) + + ranks = ray.get([worker.test.remote() for worker in workers]) + + assert ranks == [0, 1], f"expecting [0, 1], got {ranks}" + + ray.shutdown() diff --git a/e2c/verl/tests/single_controller/test_worker_group_basics.py b/e2c/verl/tests/single_controller/test_worker_group_basics.py new file mode 100644 index 0000000000000000000000000000000000000000..5c4823dfb2dc85e9464517b991a552d8a0d7c2b7 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_worker_group_basics.py @@ -0,0 +1,130 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +e2e test verl.single_controller.ray +""" + +import ray +import torch + +from verl.single_controller.base.decorator import Dispatch, Execute, collect_all_to_all, register +from verl.single_controller.base.worker import Worker +from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + + +def two_to_all_dispatch_fn(worker_group, *args, **kwargs): + """ + Assume the input is a list of 2. Duplicate the input interleaved and pass to each worker. + """ + for arg in args: + assert len(arg) == 2 + for i in range(worker_group.world_size - 2): + arg.append(arg[i % 2]) + for k, v in kwargs.items(): + assert len(v) == 2 + for i in range(worker_group.world_size - 2): + v.append(v[i % 2]) + return args, kwargs + + +@ray.remote +class TestActor(Worker): + # TODO: pass *args and **kwargs is bug prone and not very convincing + def __init__(self, x) -> None: + super().__init__() + self._x = x + + def foo(self, y): + return self._x + y + + @register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.RANK_ZERO) + def foo_rank_zero(self, x, y): + return self._x + y + x + + @register(Dispatch.ONE_TO_ALL, blocking=False) + def foo_one_to_all(self, x, y): + return self._x + y + x + + @register(Dispatch.ALL_TO_ALL, blocking=False) + def foo_all_to_all(self, x, y): + return self._x + y + x + + @register(dispatch_mode={"dispatch_fn": two_to_all_dispatch_fn, "collect_fn": collect_all_to_all}) + def foo_custom(self, x, y): + return self._x + y + x + + +@ray.remote(num_gpus=0.1) +def remote_call_wg(worker_names): + class_with_args = RayClassWithInitArgs(cls=TestActor, x=2) + worker_group = RayWorkerGroup.from_detached( + worker_names=worker_names, ray_cls_with_init=class_with_args, name_prefix=None + ) + print(worker_group.worker_names) + + output_ref = worker_group.foo_custom(x=[1, 2], y=[5, 6]) + assert output_ref == [8, 10, 8, 10] + + output_ref = worker_group.foo_rank_zero(x=1, y=2) + assert output_ref == 5 + + return worker_group.worker_names + + +def add_one(data): + data = data.to("cuda") + data += 1 + data = data.to("cpu") + return data + + +def test_basics(): + ray.init(num_cpus=100) + + # create 4 workers, each hold a GPU + resource_pool = RayResourcePool([4], use_gpu=True) + class_with_args = RayClassWithInitArgs(cls=TestActor, x=2) + + worker_group = RayWorkerGroup( + resource_pool=resource_pool, ray_cls_with_init=class_with_args, name_prefix="worker_group_basic" + ) + + print(worker_group.worker_names) + + # this will wait for all the results + output = worker_group.execute_all_sync("foo", y=3) + assert output == [5, 5, 5, 5] + + # this is a list of object reference. It won't block. + output_ref = worker_group.execute_all_async("foo", y=4) + print(output_ref) + + assert ray.get(output_ref) == [6, 6, 6, 6] + + output_ref = worker_group.foo_one_to_all(x=1, y=2) + assert ray.get(output_ref) == [5, 5, 5, 5] + + output_ref = worker_group.foo_all_to_all(x=[1, 2, 3, 4], y=[5, 6, 7, 8]) + assert ray.get(output_ref) == [8, 10, 12, 14] + + print(ray.get(remote_call_wg.remote(worker_group.worker_names))) + + output = worker_group.execute_func_rank_zero(add_one, torch.ones(2, 2)) + torch.testing.assert_close(output, torch.ones(2, 2) + 1) + + ray.shutdown() + + +if __name__ == "__main__": + test_basics() diff --git a/e2c/verl/tests/single_controller/test_worker_group_torch.py b/e2c/verl/tests/single_controller/test_worker_group_torch.py new file mode 100644 index 0000000000000000000000000000000000000000..a601c43da74099a1fa5a314245ff0b2a1bb3ef07 --- /dev/null +++ b/e2c/verl/tests/single_controller/test_worker_group_torch.py @@ -0,0 +1,111 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +os.environ["RAY_DEDUP_LOGS"] = "0" +os.environ["NCCL_DEBUG"] = "WARN" + +import ray +import torch +import torch.distributed + +from verl.single_controller.base.worker import Worker +from verl.single_controller.ray.base import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup + + +@ray.remote +class TestAllGatherActor(Worker): + def __init__(self, size) -> None: + super().__init__() + self.size = size + + def init(self): + torch.distributed.init_process_group() + self.tensor = torch.zeros(size=(self.size,), dtype=torch.int64, device="cuda") + self.tensor += self.rank + + def all_gather(self): + world_size = self._world_size + output = torch.zeros( + size=(self.tensor.shape[0] * world_size,), dtype=self.tensor.dtype, device=self.tensor.device + ) + torch.distributed.all_gather_into_tensor(output, self.tensor, async_op=False) + return output + + +@ray.remote +class TestAllGatherActorV2(Worker): + def __init__(self, size) -> None: + super().__init__() + self.size = size + + torch.distributed.init_process_group() + self.tensor = torch.zeros(size=(self.size,), dtype=torch.int64, device="cuda") + self.tensor += self.rank + + def all_gather(self): + world_size = self._world_size + output = torch.zeros( + size=(self.tensor.shape[0] * world_size,), dtype=self.tensor.dtype, device=self.tensor.device + ) + torch.distributed.all_gather_into_tensor(output, self.tensor, async_op=False) + return output + + +def test_all_gather_torch(): + """ + In this test, we instantiate 4 GPUs in a group and test the all_gather + """ + ray.init() + + # create 4 workers, each hold a GPU + resource_pool = RayResourcePool([4], use_gpu=True) + class_with_args = RayClassWithInitArgs(cls=TestAllGatherActor, size=2) + + worker_group = RayWorkerGroup(resource_pool, class_with_args, name_prefix="worker_group_torch") + + worker_group.execute_all_sync("init") + output = worker_group.execute_all_sync("all_gather") + for i in range(1, len(output)): + assert torch.all(output[i] == output[0]) + + output = output[0].cpu() + print(output) + assert torch.all(output == torch.tensor([0, 0, 1, 1, 2, 2, 3, 3], dtype=torch.int64)) + + ray.shutdown() + + +def test_all_gather_torch_v2(): + """ + In this test, we instantiate 4 GPUs in a group and test the all_gather + """ + ray.init() + + # create 4 workers, each hold a GPU + resource_pool = RayResourcePool([4], use_gpu=True) + class_with_args = RayClassWithInitArgs(cls=TestAllGatherActorV2, size=2) + + worker_group = RayWorkerGroup(resource_pool, class_with_args, name_prefix="worker_group_torch") + + output = worker_group.execute_all_sync("all_gather") + for i in range(1, len(output)): + assert torch.all(output[i] == output[0]) + + output = output[0].cpu() + print(output) + assert torch.all(output == torch.tensor([0, 0, 1, 1, 2, 2, 3, 3], dtype=torch.int64)) + + ray.shutdown() diff --git a/e2c/verl/tests/special_distributed/README.md b/e2c/verl/tests/special_distributed/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f2f865e8bf95a673a0d6f56b74c7a2c12535faf2 --- /dev/null +++ b/e2c/verl/tests/special_distributed/README.md @@ -0,0 +1 @@ +This folder is reserved for unit tests (instead of end-to-end tests) that require multiple GPUs. diff --git a/e2c/verl/tests/special_distributed/run_all.sh b/e2c/verl/tests/special_distributed/run_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..c34edf2229bcee37a2ddb6548796579ad785f914 --- /dev/null +++ b/e2c/verl/tests/special_distributed/run_all.sh @@ -0,0 +1,18 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash + +set -e -x +torchrun --nproc-per-node=4 --standalone tests/special_distributed/test_tensor_dict.py \ No newline at end of file diff --git a/e2c/verl/tests/special_distributed/test_fsdp_ckpt.py b/e2c/verl/tests/special_distributed/test_fsdp_ckpt.py new file mode 100644 index 0000000000000000000000000000000000000000..d4cd0d51ed49617764bd67ec915b9167f8e35101 --- /dev/null +++ b/e2c/verl/tests/special_distributed/test_fsdp_ckpt.py @@ -0,0 +1,162 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +import tempfile + +import torch +import torch.distributed +from torch.distributed import init_device_mesh +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import MixedPrecision, ShardingStrategy +from transformers import AutoModelForCausalLM, AutoTokenizer, Qwen2Config + +from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager +from verl.utils.distributed import initialize_global_process_group +from verl.utils.fsdp_utils import MixedPrecisionPolicy, apply_fsdp2 + + +def create_random_input_ids(batch_size, seq_len, vocab_size): + from flash_attn.bert_padding import unpad_input + + from verl.utils.model import compute_position_id_with_mask, create_random_mask + + input_ids = torch.randint(0, vocab_size, (batch_size, seq_len), device="cuda") + + attention_mask = create_random_mask( + input_ids, max_ratio_of_left_padding=0.1, min_ratio_of_valid_token=0.5, max_ratio_of_valid_token=0.7 + ) + position_ids = compute_position_id_with_mask(attention_mask) + + input_ids = unpad_input(input_ids.unsqueeze(-1), attention_mask)[0].transpose(0, 1) + position_ids = unpad_input(position_ids.unsqueeze(-1), attention_mask)[0].transpose(0, 1) + return input_ids, position_ids + + +def test_fsdp_ckpt(strategy="fsdp"): + assert torch.cuda.device_count() >= 2, "need at least 2 gpus for test" + local_rank, rank, world_size = initialize_global_process_group() + device_mesh = init_device_mesh("cuda", mesh_shape=(world_size,), mesh_dim_names=("dp",)) + + model_name = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B-Instruct") + config = Qwen2Config(num_hidden_layers=1) + + with torch.device("cuda"): + model = AutoModelForCausalLM.from_config( + config=config, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + ) + model = model.to(device="cuda") + + # Wrap model with FSDP + if strategy == "fsdp": + mixed_precision = MixedPrecision( + param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32 + ) + + model = FSDP( + model, + use_orig_params=False, + device_id=torch.cuda.current_device(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + device_mesh=device_mesh, + ) + else: + mp_policy = MixedPrecisionPolicy( + param_dtype=torch.bfloat16, reduce_dtype=torch.float32, cast_forward_inputs=True + ) + fsdp_kwargs = { + "mesh": device_mesh, + "mp_policy": mp_policy, + } + apply_fsdp2(model, fsdp_kwargs, {}) + + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4) + lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.9) + + # Create checkpoint manager + tokenizer = AutoTokenizer.from_pretrained(model_name) + checkpoint_manager = FSDPCheckpointManager( + model=model, optimizer=optimizer, lr_scheduler=lr_scheduler, tokenizer=tokenizer + ) + + # Generate sample input + batch_size = 10 + seq_len = 1024 + vocab_size = config.vocab_size + # First input for initial update + input_ids1, position_ids1 = create_random_input_ids(batch_size, seq_len, vocab_size) + + # Second input for verification + input_ids2, position_ids2 = create_random_input_ids(batch_size, seq_len, vocab_size) + + # Step 1: Initial update and save checkpoint + outputs1 = model(input_ids=input_ids1, position_ids=position_ids1) + loss1 = outputs1.logits.mean() + loss1.backward() + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Save checkpoint after first update + temp_dir = tempfile.mkdtemp() + checkpoint_path = os.path.join(temp_dir, "checkpoint") + checkpoint_manager.save_checkpoint(local_path=checkpoint_path, hdfs_path=None, global_step=0) + saved_state_dict = model.state_dict() + + # Step 2: Second update and forward pass + outputs2 = model(input_ids=input_ids2, position_ids=position_ids2) + loss2 = outputs2.logits.mean() + loss2.backward() + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Record logits after second update + with torch.no_grad(): + logits_before_load = model(input_ids=input_ids2, position_ids=position_ids2).logits + + # Step 3: Load checkpoint and repeat second update + checkpoint_manager.load_checkpoint(checkpoint_path) + loaded_state_dict = model.state_dict() + for key in loaded_state_dict: + assert key in saved_state_dict, f"Key {key} not found in saved state dict" + torch.testing.assert_close(loaded_state_dict[key], saved_state_dict[key], atol=0.0, rtol=0.0) + + # Repeat the second update with same input + outputs3 = model(input_ids=input_ids2, position_ids=position_ids2) + loss3 = outputs3.logits.mean() + loss3.backward() + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Record logits after loaded checkpoint and update + with torch.no_grad(): + logits_after_load = model(input_ids=input_ids2, position_ids=position_ids2).logits + + # Step 4: Verify outputs match + torch.testing.assert_close(logits_before_load, logits_after_load, atol=0.0, rtol=0.0) + print("Checkpoint save/load test passed!") + + # Cleanup + shutil.rmtree(temp_dir) + torch.distributed.barrier() + torch.distributed.destroy_process_group() + + +if __name__ == "__main__": + strategy = os.environ.get("STRATEGY", "fsdp") + os.environ["FLASH_ATTENTION_DETERMINISTIC"] = "1" + test_fsdp_ckpt(strategy=strategy) diff --git a/e2c/verl/tests/special_distributed/test_mcore_config_converter.py b/e2c/verl/tests/special_distributed/test_mcore_config_converter.py new file mode 100644 index 0000000000000000000000000000000000000000..18b2d3a047dd56244dfcbd0fc3284a254b1568fb --- /dev/null +++ b/e2c/verl/tests/special_distributed/test_mcore_config_converter.py @@ -0,0 +1,101 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import megatron.core.parallel_state as mpu +import torch +from megatron.core.transformer import MLATransformerConfig, TransformerConfig +from transformers import AutoConfig, PretrainedConfig + +from verl.models.mcore import hf_to_mcore_config +from verl.utils.distributed import destroy_global_process_group, initialize_global_process_group + +TEST_MODELS = [ + "Qwen/Qwen2.5-7B", # Qwen2 dense + "Qwen/Qwen3-8B", # Qwen3 dense + "deepseek-ai/deepseek-coder-1.3b-instruct", # deepseek dense + "Qwen/Qwen2-57B-A14B", # Qwen2 moe + "Qwen/Qwen3-30B-A3B", # Qwen3 moe + # "mistralai/Mixtral-8x7B-v0.1", # Mixtral # require authentication + "deepseek-ai/DeepSeek-V3-Base", # Deepseek V3 +] + + +def check_config_converter_results(tf_config: TransformerConfig | MLATransformerConfig, hf_config: PretrainedConfig): + assert tf_config.num_layers == hf_config.num_hidden_layers, ( + f"Number of layers mismatch: {tf_config.num_layers} != {hf_config.num_hidden_layers}" + ) + assert tf_config.hidden_size == hf_config.hidden_size, ( + f"Hidden size mismatch: {tf_config.hidden_size} != {hf_config.hidden_size}" + ) + assert tf_config.num_attention_heads == hf_config.num_attention_heads, ( + f"Number of attention heads mismatch: {tf_config.num_attention_heads} != {hf_config.num_attention_heads}" + ) + assert tf_config.num_query_groups == hf_config.num_key_value_heads, ( + f"Number of query groups mismatch: {tf_config.num_query_groups} != {hf_config.num_key_value_heads}" + ) + assert tf_config.ffn_hidden_size == hf_config.intermediate_size, ( + f"FFN hidden size mismatch: {tf_config.ffn_hidden_size} != {hf_config.intermediate_size}" + ) + assert tf_config.attention_dropout == hf_config.attention_dropout, ( + f"Attention dropout mismatch: {tf_config.attention_dropout} != {hf_config.attention_dropout}" + ) + assert tf_config.hidden_dropout == getattr(hf_config, "hidden_dropout", 0.0), ( + f"Hidden dropout mismatch: {tf_config.hidden_dropout} != {getattr(hf_config, 'hidden_dropout', 0.0)}" + ) + if getattr(hf_config, "head_dim", None) is not None: + assert tf_config.kv_channels == getattr(hf_config, "head_dim", None), ( + f"Head dim mismatch: {tf_config.kv_channels} != {getattr(hf_config, 'head_dim', None)}" + ) + assert tf_config.layernorm_epsilon == hf_config.rms_norm_eps, ( + f"Layernorm epsilon mismatch: {tf_config.layernorm_epsilon} != {hf_config.rms_norm_eps}" + ) + + +def modify_hf_config(name: str, hf_config: PretrainedConfig): + if name == "deepseek-ai/DeepSeek-V3-Base": + hf_config.num_nextn_predict_layers = 0 + hf_config.quantization_config = None + return hf_config + + +def test_mcore_config_converter(): + """ + Test the conversion of Hugging Face model configurations to MCore configurations. + """ + local_rank, rank, world_size = initialize_global_process_group() + mpu.initialize_model_parallel( + tensor_model_parallel_size=2, + pipeline_model_parallel_size=2, + virtual_pipeline_model_parallel_size=None, + pipeline_model_parallel_split_rank=None, + use_sharp=False, + context_parallel_size=2, + expert_model_parallel_size=1, + expert_tensor_parallel_size=None, + nccl_communicator_config_path=None, + ) + for model_name in TEST_MODELS: + print(f"testing {model_name}") + hf_config = AutoConfig.from_pretrained(os.path.expanduser(f"~/models/configs/{model_name}/config.json")) + hf_config = modify_hf_config(model_name, hf_config) + tf_config = hf_to_mcore_config(hf_config, torch.bfloat16) + check_config_converter_results(tf_config, hf_config) + + destroy_global_process_group() + + +if __name__ == "__main__": + test_mcore_config_converter() diff --git a/e2c/verl/tests/special_distributed/test_tensor_dict.py b/e2c/verl/tests/special_distributed/test_tensor_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..0a7f8039d908f33296900508b1119ee6513fac4a --- /dev/null +++ b/e2c/verl/tests/special_distributed/test_tensor_dict.py @@ -0,0 +1,121 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +os.environ["NCCL_DEBUG"] = "WARN" + +import numpy as np +import torch +import torch.distributed + +from verl.protocol import DataProto, all_gather_data_proto +from verl.utils.distributed import initialize_global_process_group + + +def test_all_gather_data_proto(): + device_mesh = torch.distributed.device_mesh.init_device_mesh("cuda", mesh_shape=[2, 2], mesh_dim_names=["dp", "tp"]) + + global_rank = torch.distributed.get_rank() + + obs = torch.tensor([[1 * global_rank, 2 * global_rank + 1], [3 * global_rank, 4 * global_rank + 1]]) + + labels = ["a", "b"] if global_rank % 2 == 0 else ["b", "a"] + labels = np.array(labels, dtype=object) + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"info": "test_info"}) + + all_gather_data_proto(data=data, process_group=device_mesh.get_group("dp")) + + if global_rank == 0: + expected_obs = torch.tensor([[0, 1], [0, 1], [2, 5], [6, 9]], device="cuda") + expected_labels = ["a", "b", "a", "b"] + elif global_rank == 1: + expected_obs = torch.tensor([[1, 3], [3, 5], [3, 7], [9, 13]], device="cuda") + expected_labels = ["b", "a", "b", "a"] + elif global_rank == 2: + expected_obs = torch.tensor([[0, 1], [0, 1], [2, 5], [6, 9]], device="cuda") + expected_labels = ["a", "b", "a", "b"] + elif global_rank == 3: + expected_obs = torch.tensor([[1, 3], [3, 5], [3, 7], [9, 13]], device="cuda") + expected_labels = ["b", "a", "b", "a"] + + torch.testing.assert_close(data.batch["obs"], expected_obs, atol=0, rtol=0) + assert (data.non_tensor_batch["labels"] == expected_labels).all() + assert data.meta_info == {"info": "test_info"} + + +def test_vocab_parallel_entropy(): + from megatron.core import parallel_state as mpu + + from verl.utils.megatron.tensor_parallel import vocab_parallel_entropy + from verl.utils.profiler import log_gpu_memory_usage + from verl.utils.torch_functional import entropy_from_logits + + mpu.initialize_model_parallel( + tensor_model_parallel_size=2, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None + ) + + batch_size = 2 + seqlen = 128 + vocab_size = 155136 + + logits = torch.randn(batch_size * seqlen, vocab_size, device="cuda", requires_grad=True) + target = torch.randint(low=0, high=vocab_size, size=(batch_size * seqlen,), device="cuda", dtype=torch.int64) + + # broadcast across tp + torch.distributed.broadcast( + logits, mpu.get_tensor_model_parallel_src_rank(), group=mpu.get_tensor_model_parallel_group() + ) + torch.distributed.broadcast( + target, mpu.get_tensor_model_parallel_src_rank(), group=mpu.get_tensor_model_parallel_group() + ) + + tp_rank = mpu.get_tensor_model_parallel_rank() + vocab_size_per_tp = vocab_size // mpu.get_tensor_model_parallel_world_size() + + # get the local logits of each tp + vocab_parallel_logits = ( + logits.clone().detach()[:, tp_rank * vocab_size_per_tp : (tp_rank + 1) * vocab_size_per_tp].requires_grad_() + ) + logits.grad = None + vocab_parallel_logits.grad = None + + log_gpu_memory_usage("begin") + output_entropy = vocab_parallel_entropy(vocab_parallel_logits) + log_gpu_memory_usage("after forward") + grad_output = torch.randn_like(output_entropy) + output_entropy.backward(grad_output) + log_gpu_memory_usage("after backward") + + target_entropy = entropy_from_logits(logits) + torch.testing.assert_close(output_entropy, target_entropy) + target_entropy.backward(grad_output) + torch.testing.assert_close( + logits.grad[:, tp_rank * vocab_size_per_tp : (tp_rank + 1) * vocab_size_per_tp], vocab_parallel_logits.grad + ) + # make sure logits is not altered + torch.testing.assert_close( + logits[:, tp_rank * vocab_size_per_tp : (tp_rank + 1) * vocab_size_per_tp], vocab_parallel_logits + ) + + if mpu.get_tensor_model_parallel_rank() == 0: + print("test_vocab_parallel_entropy passes") + + mpu.destroy_model_parallel() + + +if __name__ == "__main__": + local_rank, rank, world_size = initialize_global_process_group() + test_all_gather_data_proto() + test_vocab_parallel_entropy() diff --git a/e2c/verl/tests/special_e2e/README.md b/e2c/verl/tests/special_e2e/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3c295e844ceb11ee132564ab2949a05a2a066b3e --- /dev/null +++ b/e2c/verl/tests/special_e2e/README.md @@ -0,0 +1 @@ +This folder is reserved for end-to-end tests that typically require multiple GPUs. diff --git a/e2c/verl/tests/special_e2e/__init__.py b/e2c/verl/tests/special_e2e/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/tests/special_e2e/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/tests/special_e2e/check_custom_rwd_fn.py b/e2c/verl/tests/special_e2e/check_custom_rwd_fn.py new file mode 100644 index 0000000000000000000000000000000000000000..8d77a53729bd96b153f004eb230df85f1d32f890 --- /dev/null +++ b/e2c/verl/tests/special_e2e/check_custom_rwd_fn.py @@ -0,0 +1,33 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +def check_congratulations_in_file(output_file): + with open(output_file) as f: + output = f.read() + + success_message = "Congratulations!!! You have called my_reward_function successfully!!!" + assert success_message in output, f"Success message of my_reward_function not found in {output_file}" + print("Check passes") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_file", required=True, type=str) + + args = parser.parse_args() + + check_congratulations_in_file(args.output_file) diff --git a/e2c/verl/tests/special_e2e/check_results.py b/e2c/verl/tests/special_e2e/check_results.py new file mode 100644 index 0000000000000000000000000000000000000000..9453282fbc80c88a12429369647208347d35491b --- /dev/null +++ b/e2c/verl/tests/special_e2e/check_results.py @@ -0,0 +1,53 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import numpy as np + + +def extract_reward_from_line(line): + # TODO: this function needs error handling + try: + key_vals = line.split(" - ") + for key_val in key_vals: + key, val = key_val.split(":") + if key == "critic/rewards/mean": + reward = float(val) + return reward + return -np.inf + except Exception: + return -np.inf + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_file", required=True, type=str) + parser.add_argument("--target", type=float, default=0.2, help="target reward score") + + args = parser.parse_args() + + with open(args.output_file) as f: + output = f.read().split("\n") + + best_reward = -np.inf + for line in output: + if line.startswith("step"): + reward = extract_reward_from_line(line) + if reward > best_reward: + best_reward = reward + + print(f"Best reward is {best_reward}") + assert best_reward > args.target, f"Best reward must be greater than {args.target}. best_reward: {best_reward}" + print("Check passes") diff --git a/e2c/verl/tests/special_e2e/envs/__init__.py b/e2c/verl/tests/special_e2e/envs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..eb85e22f361e4af4635bda991ff12a1ed4911eec --- /dev/null +++ b/e2c/verl/tests/special_e2e/envs/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .digit_completion import DigitCompletion + +__all__ = ["DigitCompletion"] diff --git a/e2c/verl/tests/special_e2e/envs/digit_completion/__init__.py b/e2c/verl/tests/special_e2e/envs/digit_completion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..80893ae41d6669f4f7265ce76d7ac28579b30b6f --- /dev/null +++ b/e2c/verl/tests/special_e2e/envs/digit_completion/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from transformers import AutoTokenizer, LlamaConfig + +from .task import DigitCompletion, generate_ground_truth_response +from .tokenizer import CharTokenizer + +AutoTokenizer.register(LlamaConfig, CharTokenizer, exist_ok=True) + +__all__ = ["DigitCompletion", "generate_ground_truth_response", "CharTokenizer"] diff --git a/e2c/verl/tests/special_e2e/envs/digit_completion/task.py b/e2c/verl/tests/special_e2e/envs/digit_completion/task.py new file mode 100644 index 0000000000000000000000000000000000000000..c3643a86b867b440352ed55dc0f978135ac79bcf --- /dev/null +++ b/e2c/verl/tests/special_e2e/envs/digit_completion/task.py @@ -0,0 +1,179 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Task and environment definition for digit completion.""" + +import numpy as np + + +class DigitCompletion: + """ + The implementation of a simple digit completion task. + The prompt is a sequence of numbers with fixed difference. The task is to complete the next N numbers. + If the max number is reached, the next number should be modulo with max number. + + For example, + - prompt = [1, 2, 3] + - N = 5 + - max_number = 6 + + the response should be [4, 5, 6, 7%6, 8%6] = [4, 5, 6, 0, 1] + + Note that the tokenizer is char-level to increase the difficulty. + """ + + def __init__(self, max_number: int, max_diff: int, max_num_in_response: int, seed=0): + """ + + Args: + max_number: the maximum number allowed in the arithmetic sequence + max_diff: the maximum diff. The actual common diff will be sampled from [0, max_diff] + max_num_in_response: the maximum number in the response + """ + super().__init__() + self.max_number = max_number + self.max_diff = max_diff + self.max_num_in_response = max_num_in_response + assert self.max_num_in_response < 10 + assert self.max_number > 0 + assert self.max_diff > 0 + self.max_number_length = len(str(max_number)) + # {num1},{num2}:{max_num_in_response},{max_number} + self._prompt_length = self.max_number_length * 2 + 4 + self.max_number_length # no negative is allowed + + self.np_rng = np.random.default_rng(seed=seed) + + def __str__(self): + return ( + f"Prompt length: {self.prompt_length}. Response length: {self.response_length}, " + f"Max number: {self.max_number}. Max diff: {self.max_diff}, " + f"Max number in response: {self.max_num_in_response}" + ) + + def get_state(self): + return {"rng": self.np_rng} + + def set_state(self, state): + assert "rng" in state, "rng must be inside state" + self.np_rng = state["rng"] + + @property + def prompt_length(self): + return self._prompt_length + + @property + def response_length(self): + # number length + comma length + [EOS] + # The actual number times 1.5 to allow 'U' + return (self.max_num_in_response * self.max_number_length + (self.max_num_in_response - 1) + 1) * 2 + + def add(self, a, b): + return (a + b) % self.max_number + + def get_all_prompts(self): + all_prompts = [] + for first_num in range(self.max_number + 1): + for diff in range(0, self.max_diff + 1): + second_num = self.add(first_num, diff) + for num_to_complete in range(self.max_num_in_response + 1): + prompt = str(first_num) + "," + str(second_num) + f":{self.max_number},{num_to_complete}" + all_prompts.append(prompt) + return all_prompts + + def sample_str_prompts(self): + # step 1: sample initial numbers + first_num = self.np_rng.integers(self.max_number + 1) + diff = self.np_rng.integers(self.max_diff + 1) + second_num = self.add(first_num, diff) + num_to_complete = self.np_rng.integers(self.max_num_in_response + 1) + prompt = str(first_num) + "," + str(second_num) + f":{self.max_number},{num_to_complete}" + return prompt + + def sample_batch_str_prompts(self, batch_size): + str_prompts = [] + for _ in range(batch_size): + str_prompts.append(self.sample_str_prompts()) + return str_prompts + + +def compute_attention_mask(prompts, pad_token_id): + mask = np.ones_like(prompts) + mask[prompts == pad_token_id] = 0 + return mask + + +def compute_position_id_with_mask(mask): + return np.clip(np.cumsum(mask, axis=-1) - 1, a_min=0, a_max=None) + + +def generate_ground_truth_response(prompt: str): + """Generate ground truth response given a prompt.""" + num, info = prompt.split(":") + num1, num2 = num.split(",") + max_number, num_to_gen = info.split(",") + num1 = int(num1) + num2 = int(num2) + max_number = int(max_number) + num_to_gen = int(num_to_gen) + diff = (num2 - num1) % max_number + results = [] + last_num = num2 + for _ in range(num_to_gen): + curr = (last_num + diff) % max_number + results.append(str(curr)) + last_num = curr + response = ",".join(results) + return response + + +def compute_reward(prompt: str, response: str, sequence_reward=1.0): + """We compute dense reward here so that we can directly train RL without SFT""" + response_length = len(response) + ground_truth_response = generate_ground_truth_response(prompt) + per_token_reward = sequence_reward / (len(ground_truth_response) + 1) # including [EOS] + + # pad + reward = np.zeros(response_length, dtype=np.float32) # this assumes that each char is a token + # assign reward until mismatches + ground_truth_idx = 0 + for i in range(response_length): + if ground_truth_idx == len(ground_truth_response): + break + + ground_truth_response_token = ground_truth_response[ground_truth_idx] + response_token = response[i] + if ground_truth_response_token == response_token: + reward[i] = per_token_reward + ground_truth_idx += 1 + else: + # no matches + break + + return reward, {"ground_truth_response": ground_truth_response} + + +if __name__ == "__main__": + task = DigitCompletion(max_number=20, max_diff=3, max_num_in_response=5) + print(task.sample_str_prompts()) + + prompt = "7,8:20,0" + response = "" + print(compute_reward(prompt, response)) + + prompt = "7,8:20,0" + response = "E000" + print(compute_reward(prompt, response)) + + prompt = "9,10:20,2" + response = "11,12,13" + print(compute_reward(prompt, response)) diff --git a/e2c/verl/tests/special_e2e/envs/digit_completion/tokenizer.py b/e2c/verl/tests/special_e2e/envs/digit_completion/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..6ff471938937dc55ab528cb883e4ba2e03b35416 --- /dev/null +++ b/e2c/verl/tests/special_e2e/envs/digit_completion/tokenizer.py @@ -0,0 +1,155 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Copied from https://github.com/dariush-bahrami/character-tokenizer/blob/master/charactertokenizer/core.py + +CharacterTokenzier for Hugging Face Transformers. + +This is heavily inspired from CanineTokenizer in transformers package. +""" + +import json +import os +from pathlib import Path +from typing import Optional, Sequence + +from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer + + +class CharTokenizer(PreTrainedTokenizer): + def __init__(self, characters: Sequence[str], model_max_length: int, chat_template, **kwargs): + """Character tokenizer for Hugging Face transformers. + + Args: + characters (Sequence[str]): List of desired characters. Any character which + is not included in this list will be replaced by a special token called + [UNK] with id=6. Following are list of all of the special tokens with + their corresponding ids: + "[CLS]": 0 + "[SEP]": 1 + "[BOS]": 2 + "[MASK]": 3 + "[PAD]": 4 + "[RESERVED]": 5 + "[UNK]": 6 + an id (starting at 7) will be assigned to each character. + + model_max_length (int): Model maximum sequence length. + """ + eos_token_str = "E" + sep_token_str = "S" + pad_token_str = "P" + unk_token_str = "U" + + self.characters = characters + self.model_max_length = model_max_length + eos_token = AddedToken(eos_token_str, lstrip=False, rstrip=False) + sep_token = AddedToken(sep_token_str, lstrip=False, rstrip=False) + pad_token = AddedToken(pad_token_str, lstrip=False, rstrip=False) + unk_token = AddedToken(unk_token_str, lstrip=False, rstrip=False) + + self._vocab_str_to_int = { + sep_token_str: 0, + eos_token_str: 1, + pad_token_str: 2, + unk_token_str: 3, + **{ch: i + 4 for i, ch in enumerate(characters)}, + } + self._vocab_int_to_str = {v: k for k, v in self._vocab_str_to_int.items()} + + super().__init__( + eos_token=eos_token, + sep_token=sep_token, + pad_token=pad_token, + unk_token=unk_token, + add_prefix_space=False, + model_max_length=model_max_length, + **kwargs, + ) + + self.chat_template = chat_template + + @property + def vocab_size(self) -> int: + return len(self._vocab_str_to_int) + + def get_vocab(self): + return self._vocab_str_to_int + + def _tokenize(self, text: str) -> list[str]: + return list(text) + + def _convert_token_to_id(self, token: str) -> int: + return self._vocab_str_to_int.get(token, self._vocab_str_to_int["U"]) + + def _convert_id_to_token(self, index: int) -> str: + return self._vocab_int_to_str[index] + + def convert_tokens_to_string(self, tokens): + return "".join(tokens) + + def build_inputs_with_special_tokens( + self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None + ) -> list[int]: + sep = [self.sep_token_id] + cls = [self.cls_token_id] + result = cls + token_ids_0 + sep + if token_ids_1 is not None: + result += token_ids_1 + sep + return result + + def get_special_tokens_mask( + self, + token_ids_0: list[int], + token_ids_1: Optional[list[int]] = None, + already_has_special_tokens: bool = False, + ) -> list[int]: + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True, + ) + + result = [1] + ([0] * len(token_ids_0)) + [1] + if token_ids_1 is not None: + result += ([0] * len(token_ids_1)) + [1] + return result + + def get_config(self) -> dict: + return { + "char_ords": [ord(ch) for ch in self.characters], + "model_max_length": self.model_max_length, + "chat_template": self.chat_template, + } + + @classmethod + def from_config(cls, config: dict): + cfg = {} + cfg["characters"] = [chr(i) for i in config["char_ords"]] + cfg["model_max_length"] = config["model_max_length"] + cfg["chat_template"] = config["chat_template"] + return cls(**cfg) + + def save_pretrained(self, save_directory: str | os.PathLike, **kwargs): + cfg_file = Path(save_directory) / "tokenizer_config.json" + cfg = self.get_config() + with open(cfg_file, "w") as f: + json.dump(cfg, f, indent=4) + + @classmethod + def from_pretrained(cls, save_directory: str | os.PathLike, **kwargs): + cfg_file = Path(save_directory) / "tokenizer_config.json" + with open(cfg_file) as f: + cfg = json.load(f) + return cls.from_config(cfg) diff --git a/e2c/verl/tests/special_e2e/generation/run_gen_qwen05.sh b/e2c/verl/tests/special_e2e/generation/run_gen_qwen05.sh new file mode 100644 index 0000000000000000000000000000000000000000..61c55b157cdaa06b9fa0b977c733397f37c1ec61 --- /dev/null +++ b/e2c/verl/tests/special_e2e/generation/run_gen_qwen05.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Tested with 1 & 4 GPUs +set -xeuo pipefail + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} + +NGPUS_PER_NODE=${NGPUS_PER_NODE:-4} +OUTPUT_PATH=${OUTPUT_PATH:-$HOME/data/gen/qwen_05_gen_test.parquet} +GEN_TP=${GEN_TP:-2} # Default tensor parallel size to 2 + +python3 -m verl.trainer.main_generation \ + trainer.nnodes=1 \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + data.path="${HOME}/data/gsm8k/test.parquet" \ + data.prompt_key=prompt \ + data.n_samples=1 \ + data.output_path="${OUTPUT_PATH}" \ + model.path="${MODEL_ID}" \ + +model.trust_remote_code=True \ + rollout.temperature=1.0 \ + rollout.top_k=50 \ + rollout.top_p=0.7 \ + rollout.prompt_length=2048 \ + rollout.response_length=1024 \ + rollout.tensor_model_parallel_size="${GEN_TP}" \ + rollout.gpu_memory_utilization=0.8 diff --git a/e2c/verl/tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json b/e2c/verl/tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json new file mode 100644 index 0000000000000000000000000000000000000000..c215fa4f7ccb777035e4be513045fb6ddb204b8f --- /dev/null +++ b/e2c/verl/tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json @@ -0,0 +1,4 @@ +{ + "num_hidden_layers": 2, + "max_window_layers": 2 +} \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/ppo_trainer/run_function_reward.sh b/e2c/verl/tests/special_e2e/ppo_trainer/run_function_reward.sh new file mode 100644 index 0000000000000000000000000000000000000000..d56c150e2611593a3d99fc710796f290d3fbbcbe --- /dev/null +++ b/e2c/verl/tests/special_e2e/ppo_trainer/run_function_reward.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +NUM_GPUS=${NUM_GPUS:-8} + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +TRAIN_FILES=${TRAIN_FILES:-$HOME/data/gsm8k/train.parquet} +VAL_FILES=${VAL_FILES:-$HOME/data/gsm8k/test.parquet} +MAX_PROMPT_LEN=${MAX_PROMPT_LEN:-512} +MAX_RESPONSE_LEN=${MAX_RESPONSE_LEN:-512} + +ENGINE=${ENGINE:-vllm} +ROLLOUT_MODE=${ROLLOUT_MODE:-sync} + +RETURN_RAW_CHAT="False" +SKIP_TOKENIZER_INIT=${SKIP_TOKENIZER_INIT:-False} +if [ "$ROLLOUT_MODE" = "async" ]; then + RETURN_RAW_CHAT="True" + SKIP_TOKENIZER_INIT="True" +fi + +GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8} +ACTOR_FSDP_PARAM_OFFLOAD=${ACTOR_FSDP_PARAM_OFFLOAD:-False} +ACTOR_FSDP_OPTIMIZER_OFFLOAD=${ACTOR_FSDP_OPTIMIZER_OFFLOAD:-False} +REF_FSDP_PARAM_OFFLOAD=${REF_FSDP_PARAM_OFFLOAD:-True} +RM_PAD=${RM_PAD:-True} +FUSED_KERNELS=${FUSED_KERNELS:-False} +FUSED_KERNEL_BACKEND=${FUSED_KERNEL_BACKEND:-torch} # or 'triton' for triton backend +ADV_ESTIMATOR=${ADV_ESTIMATOR:-gae} +LOSS_MODE=${LOSS_MODE:-vanilla} +USE_KL=${USE_KL:-False} +CUSTOM_REWARD_FN=${CUSTOM_REWARD_FN:-False} +ENABLE_CHUNKED_PREFILL=${ENABLE_CHUNKED_PREFILL:-True} # For vLLM VLM placeholder issue: https://github.com/vllm-project/vllm/issues/15185 +STRATEGY=${STRATEGY:-fsdp} +# LoRA config +LORA_RANK=${LORA_RANK:-0} +LORA_ALPHA=${LORA_ALPHA:-${LORA_RANK}} +LORA_TARGET=${LORA_TARGET:-"all-linear"} +LORA_EXCLUDE=${LORA_EXCLUDE:-"DONT_EXCLUDE"} +USE_SHM=${USE_SHM:-False} +LOAD_FORMAT=${LOAD_FORMAT:-dummy} +LAYERED_SUMMON=${LAYERED_SUMMON:-False} +# Validation +VAL_BEFORE_TRAIN=${VAL_BEFORE_TRAIN:-False} +TEST_FREQ=${TEST_FREQ:--1} +# Save & Resume +RESUME_MODE=${RESUME_MODE:-disable} +SAVE_FREQ=${SAVE_FREQ:--1} +TOTAL_TRAIN_STEPS=${TOTAL_TRAIN_STEPS:-1} + +# whether to save hf_model +SAVE_HF_MODEL=${SAVE_HF_MODEL:-False} +FSDP_SIZE=${FSDP_SIZE:--1} +SP_SIZE=${SP_SIZE:-1} + +if [ "${SAVE_HF_MODEL}" = "True" ]; then + CHECKPOINT_CONTENTS="['model','hf_model','optimizer','extra']" +else + CHECKPOINT_CONTENTS="['model','optimizer','extra']" +fi + +train_traj_micro_bsz_per_gpu=2 # b +n_resp_per_prompt=4 # g + +train_traj_micro_bsz=$((train_traj_micro_bsz_per_gpu * NUM_GPUS)) # b * n +train_traj_mini_bsz=$((train_traj_micro_bsz * 2)) # 2 * b * n +train_prompt_mini_bsz=$((train_traj_mini_bsz * n_resp_per_prompt)) # 2 * b * n / g +train_prompt_bsz=$((train_prompt_mini_bsz * 2)) # 4 * b * n / g + +reward_fn_name=null +reward_fn_file_path=null +output_file="$(pwd)/output.txt" +if [ "${CUSTOM_REWARD_FN}" = "True" ]; then + reward_fn_name="my_reward_function" + reward_fn_file_path="$(pwd)/my_reward_function.py" + rm -rf "${reward_fn_file_path}" + cat < "$reward_fn_file_path" +def ${reward_fn_name}(data_source, solution_str, ground_truth, extra_info=None): + print(f"Congratulations!!! You have called ${reward_fn_name} successfully!!!") + return 0.1 +EOF + + rm -rf "${output_file}" +fi + +exp_name="${VERL_EXP_NAME:-$(basename "${MODEL_ID,,}")-function-reward-minimal}" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator="${ADV_ESTIMATOR}" \ + data.train_files="${TRAIN_FILES}" \ + data.val_files="${VAL_FILES}" \ + data.train_batch_size="${train_prompt_bsz}" \ + data.max_prompt_length="${MAX_PROMPT_LEN}" \ + data.max_response_length="${MAX_RESPONSE_LEN}" \ + data.return_raw_chat=${RETURN_RAW_CHAT} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.use_shm=${USE_SHM} \ + actor_rollout_ref.model.lora_rank=${LORA_RANK} \ + actor_rollout_ref.model.lora_alpha=${LORA_ALPHA} \ + actor_rollout_ref.model.target_modules=${LORA_TARGET} \ + actor_rollout_ref.model.exclude_modules=${LORA_EXCLUDE} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding="${RM_PAD}" \ + actor_rollout_ref.model.use_fused_kernels=${FUSED_KERNELS} \ + actor_rollout_ref.model.fused_kernel_options.impl_backend=${FUSED_KERNEL_BACKEND} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${ACTOR_FSDP_PARAM_OFFLOAD} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${ACTOR_FSDP_OPTIMIZER_OFFLOAD} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${FSDP_SIZE} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size="${SP_SIZE}" \ + actor_rollout_ref.actor.checkpoint.save_contents=${CHECKPOINT_CONTENTS} \ + actor_rollout_ref.actor.use_kl_loss="${USE_KL}" \ + actor_rollout_ref.actor.policy_loss.loss_mode="${LOSS_MODE}" \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name="${ENGINE}" \ + actor_rollout_ref.rollout.mode="${ROLLOUT_MODE}" \ + actor_rollout_ref.rollout.load_format=${LOAD_FORMAT} \ + actor_rollout_ref.rollout.layered_summon=${LAYERED_SUMMON} \ + actor_rollout_ref.rollout.skip_tokenizer_init="${SKIP_TOKENIZER_INIT}" \ + actor_rollout_ref.rollout.gpu_memory_utilization="${GPU_MEMORY_UTILIZATION}" \ + actor_rollout_ref.rollout.enable_chunked_prefill="${ENABLE_CHUNKED_PREFILL}" \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.ref.fsdp_config.param_offload="${REF_FSDP_PARAM_OFFLOAD}" \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding="${RM_PAD}" \ + critic.model.path="${MODEL_PATH}" \ + critic.model.enable_gradient_checkpointing=False \ + critic.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + custom_reward_function.path="${reward_fn_file_path}"\ + custom_reward_function.name="${reward_fn_name}"\ + algorithm.use_kl_in_reward="${USE_KL}" \ + algorithm.kl_penalty=kl \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl-test' \ + trainer.experiment_name="${exp_name}" \ + trainer.nnodes=1 \ + trainer.n_gpus_per_node="${NUM_GPUS}" \ + trainer.val_before_train="${VAL_BEFORE_TRAIN}" \ + trainer.test_freq="${TEST_FREQ}" \ + trainer.save_freq="${SAVE_FREQ}" \ + trainer.resume_mode="${RESUME_MODE}" \ + trainer.total_epochs=2 \ + trainer.device=cuda \ + trainer.total_training_steps="${TOTAL_TRAIN_STEPS}" $@ \ + | tee "${output_file}" + +if [ "${CUSTOM_REWARD_FN}" = "True" ]; then + python3 tests/special_e2e/check_custom_rwd_fn.py --output_file="${output_file}" + check_exit_code=$? + rm -rf "${reward_fn_file_path}" + rm -rf "${output_file}" + # Return the exit code of check_custom_rwd_fn.py if it fails + if [ $check_exit_code -ne 0 ]; then + exit $check_exit_code + fi +fi diff --git a/e2c/verl/tests/special_e2e/ppo_trainer/run_model_reward.sh b/e2c/verl/tests/special_e2e/ppo_trainer/run_model_reward.sh new file mode 100644 index 0000000000000000000000000000000000000000..09d6757b51138f280520a99b63d12ad0ceda5290 --- /dev/null +++ b/e2c/verl/tests/special_e2e/ppo_trainer/run_model_reward.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +NUM_GPUS=${NUM_GPUS:-8} + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +TRAIN_FILES=${TRAIN_FILES:-$HOME/data/gsm8k/train.parquet} +VAL_FILES=${VAL_FILES:-$HOME/data/gsm8k/test.parquet} + +RM_PAD=${RM_PAD:-True} +FUSED_KERNELS=${FUSED_KERNELS:-False} +FUSED_KERNEL_BACKEND=${FUSED_KERNEL_BACKEND:-torch} # or 'triton' for triton backend +SP_SIZE=${SP_SIZE:-1} +SEQ_BALANCE=${SEQ_BALANCE:-False} +LIGER=${LIGER:-False} +# Validation +VAL_BEFORE_TRAIN=${VAL_BEFORE_TRAIN:-False} +TEST_FREQ=${TEST_FREQ:--1} +# Save & Resume +RESUME_MODE=${RESUME_MODE:-disable} +SAVE_FREQ=${SAVE_FREQ:--1} +TOTAL_TRAIN_STEPS=${TOTAL_TRAIN_STEPS:-1} + +train_traj_micro_bsz_per_gpu=2 # b +n_resp_per_prompt=4 # g + +train_traj_micro_bsz=$((train_traj_micro_bsz_per_gpu * NUM_GPUS)) # b * n +train_traj_mini_bsz=$((train_traj_micro_bsz * 2)) # 2 * b * n +train_prompt_mini_bsz=$((train_traj_mini_bsz * n_resp_per_prompt)) # 2 * b * n / g +train_prompt_bsz=$((train_prompt_mini_bsz * 2)) # 4 * b * n / g + +train_max_token_num_per_gpu=32768 +infer_max_token_num_per_gpu=32768 + +exp_name="$(basename "${MODEL_ID,,}")-model-reward-minimal" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files="${TRAIN_FILES}" \ + data.val_files="${VAL_FILES}" \ + data.train_batch_size=${train_prompt_bsz} \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.use_liger="${LIGER}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding="${RM_PAD}" \ + actor_rollout_ref.model.use_fused_kernels=${FUSED_KERNELS} \ + actor_rollout_ref.model.fused_kernel_options.impl_backend=${FUSED_KERNEL_BACKEND} \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.use_dynamic_bsz="${SEQ_BALANCE}" \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${train_max_token_num_per_gpu} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size="${SP_SIZE}" \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_max_token_num_per_gpu} \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_max_token_num_per_gpu} \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + critic.optim.lr=1e-5 \ + critic.ulysses_sequence_parallel_size="${SP_SIZE}" \ + critic.model.use_remove_padding="${RM_PAD}" \ + critic.optim.lr_warmup_steps_ratio=0.05 \ + critic.model.path="${MODEL_PATH}" \ + critic.model.enable_gradient_checkpointing=False \ + critic.use_dynamic_bsz="${SEQ_BALANCE}" \ + critic.ppo_max_token_len_per_gpu=${train_max_token_num_per_gpu} \ + critic.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + critic.model.fsdp_config.param_offload=False \ + critic.model.fsdp_config.optimizer_offload=False \ + reward_model.enable=True \ + reward_model.ulysses_sequence_parallel_size="${SP_SIZE}" \ + reward_model.model.path="${MODEL_PATH}" \ + reward_model.model.use_remove_padding="${RM_PAD}" \ + reward_model.model.fsdp_config.param_offload=True \ + reward_model.use_dynamic_bsz="${SEQ_BALANCE}" \ + reward_model.forward_max_token_len_per_gpu=${infer_max_token_num_per_gpu} \ + reward_model.micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl-test' \ + trainer.experiment_name="${exp_name}" \ + trainer.nnodes=1 \ + trainer.n_gpus_per_node="${NUM_GPUS}" \ + trainer.val_before_train="${VAL_BEFORE_TRAIN}" \ + trainer.test_freq="${VAL_BEFORE_TRAIN}" \ + trainer.save_freq="${SAVE_FREQ}" \ + trainer.resume_mode="${RESUME_MODE}" \ + trainer.total_epochs=2 \ + trainer.total_training_steps="${TOTAL_TRAIN_STEPS}" $@ diff --git a/e2c/verl/tests/special_e2e/ppo_trainer/run_single_gpu.sh b/e2c/verl/tests/special_e2e/ppo_trainer/run_single_gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..7e8615a24fbaad4b01993ddaa755e2ddb79bfde1 --- /dev/null +++ b/e2c/verl/tests/special_e2e/ppo_trainer/run_single_gpu.sh @@ -0,0 +1,24 @@ +PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=512 \ + data.max_response_length=256 \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=console \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + actor_rollout_ref.rollout.name=hf \ + trainer.total_training_steps=2 \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/ppo_trainer/run_single_gpu_with_engine.sh b/e2c/verl/tests/special_e2e/ppo_trainer/run_single_gpu_with_engine.sh new file mode 100644 index 0000000000000000000000000000000000000000..9f36a9dc8605e37bf70ab3acdf22acd84cdcb0d5 --- /dev/null +++ b/e2c/verl/tests/special_e2e/ppo_trainer/run_single_gpu_with_engine.sh @@ -0,0 +1,25 @@ +PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=512 \ + data.max_response_length=256 \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + critic.optim.lr=1e-5 \ + critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ + critic.ppo_micro_batch_size_per_gpu=4 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=['console'] \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + actor_rollout_ref.rollout.name=hf \ + trainer.use_legacy_worker_impl=disable \ + trainer.total_training_steps=2 \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/run_dapo.sh b/e2c/verl/tests/special_e2e/run_dapo.sh new file mode 100644 index 0000000000000000000000000000000000000000..5e6257f5d363a288590254b5d533a89000b9b0d8 --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_dapo.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +NUM_GPUS=${NUM_GPUS:-8} + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +adv_estimator=grpo + +kl_coef=0.0 +use_kl_in_reward=False +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=1024 +max_response_length=2048 +enable_overlong_buffer=True +overlong_buffer_len=128 +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=True +filter_groups_metric=seq_reward +max_num_gen_batches=10 + +train_traj_micro_bsz_per_gpu=2 # b +n_resp_per_prompt=4 # g + +train_traj_micro_bsz=$((train_traj_micro_bsz_per_gpu * NUM_GPUS)) # b * n +train_traj_mini_bsz=$((train_traj_micro_bsz * 2)) # 2 * b * n +train_prompt_mini_bsz=$((train_traj_mini_bsz * n_resp_per_prompt)) # 2 * b * n / g +train_prompt_bsz=$((train_prompt_mini_bsz * 2)) # 4 * b * n / g + +gen_prompt_bsz=$((train_prompt_bsz * 4)) + +exp_name="$(basename "${MODEL_ID,,}")-dapo-minimal" + +python3 -m recipe.dapo.main_dapo \ + data.train_files="${HOME}/data/gsm8k/train.parquet" \ + data.val_files="${HOME}/data/gsm8k/test.parquet" \ + reward_model.reward_manager=dapo \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + data.train_batch_size=${train_prompt_bsz} \ + data.gen_batch_size=${gen_prompt_bsz} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + trainer.logger=console \ + trainer.project_name='verl-test' \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=${NUM_GPUS} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.total_epochs=2 \ + trainer.resume_mode=disable \ + trainer.val_before_train=False \ + trainer.total_training_steps=1 $@ diff --git a/e2c/verl/tests/special_e2e/run_genrm_remote.sh b/e2c/verl/tests/special_e2e/run_genrm_remote.sh new file mode 100644 index 0000000000000000000000000000000000000000..ff1c7826b736f00003e24adaa42ce7b8e15b81e4 --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_genrm_remote.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +export no_proxy="localhost,127.0.0.1" + +set -x + +# Launch a vllm server +CUDA_VISIBLE_DEVICES=0 vllm serve $HOME/models/verl-team/GenRM-CI-Test-1.5B \ + --served_model_name genrm-demo --host localhost --port 30000 > /dev/null & +SERVER_PID=$! + +# kill server when script exits +cleanup() { + echo "Cleaning up..." + kill $SERVER_PID 2>/dev/null || true + wait $SERVER_PID 2>/dev/null || true + echo "Cleanup done" +} +trap cleanup EXIT + +# wait for server to start +wait_for_server() { + local max_attempts=60 + local attempt=0 + local sleep_time=10 + + while [ $attempt -lt $max_attempts ]; do + if curl -s "http://localhost:30000/health" >/dev/null; then + echo "Server is up and running!" + return 0 + fi + echo "Waiting for server to start... (attempt $((attempt+1))/$max_attempts)" + sleep $sleep_time + ((attempt++)) + done + + echo "Error: Failed to start server after $max_attempts attempts" >&2 + return 1 +} + +if ! wait_for_server; then + exit 1 +fi + +CUDA_VISIBLE_DEVICES=4,5,6,7 python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=${HOME}/data/gsm8k/train.parquet \ + data.val_files=${HOME}/data/gsm8k/test.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=4 \ + algorithm.use_kl_in_reward=False \ + reward_model.reward_manager=batch \ + custom_reward_function.path=recipe/genrm_remote/reward_function.py \ + custom_reward_function.name=compute_score_batch \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl-test' \ + trainer.experiment_name='qwen2.5-0.5b-gen-rm' \ + trainer.n_gpus_per_node=4 \ + trainer.val_before_train=False \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.total_epochs=10 \ + trainer.resume_mode='disable' \ + trainer.total_training_steps=1 diff --git a/e2c/verl/tests/special_e2e/run_geo3k_fsdp_sgl_multiturn_w_tool.sh b/e2c/verl/tests/special_e2e/run_geo3k_fsdp_sgl_multiturn_w_tool.sh new file mode 100644 index 0000000000000000000000000000000000000000..1ef52678210c7d7d503197aace6332f93783968f --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_geo3k_fsdp_sgl_multiturn_w_tool.sh @@ -0,0 +1,58 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +#huggingface-cli download Qwen/Qwen2.5-VL-3B-Instruct --local-dir $HOME/models/Qwen/Qwen2.5-VL-3B-Instruct + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" +FSDP_STRATEGY=${FSDP_STRATEGY:-fsdp} + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='geo3k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=64 \ + data.max_prompt_length=2048 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-VL-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.strategy=$FSDP_STRATEGY \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.ref.strategy=$FSDP_STRATEGY \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='geo3k_async_rl' \ + trainer.experiment_name=qwen2.5-vl-3b_function_rm-geo3k-sgl-multi-w-tool-$FSDP_STRATEGY-rebased-0619-verify-n8 \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + data.train_files=$HOME/data/geo3k_verl_sgl_multi_turn_preprocessed/train.parquet \ + data.val_files=$HOME/data/geo3k_verl_sgl_multi_turn_preprocessed/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/geo3k_tool_config.yaml" \ + trainer.val_before_train=False \ + trainer.total_training_steps=1 $@ \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/run_grpo_lora_with_merge.sh b/e2c/verl/tests/special_e2e/run_grpo_lora_with_merge.sh new file mode 100644 index 0000000000000000000000000000000000000000..5148992594100236469061572aeff96164491aa8 --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_grpo_lora_with_merge.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# +# An e2e test script for testing the GRPO LoRA training process +# and processing the generated checkpoint using the merge_model.py script. + +set -xeuo pipefail + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +if [ ! -d "$MODEL_PATH" ]; then + echo "Downloading model to ${MODEL_PATH}..." +# huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" +else + echo "Model directory ${MODEL_PATH} already exists, skip downloading." +fi + + +BATCH_SIZE=16 +EXP_NAME="qwen2.5_0.5b_grpo_lora" +# step 1. train model with grpo-lora for 1 step +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=${BATCH_SIZE} \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + actor_rollout_ref.model.path=${MODEL_PATH} \ + actor_rollout_ref.model.use_shm=True \ + actor_rollout_ref.model.lora_rank=64 \ + actor_rollout_ref.model.lora_alpha=32 \ + actor_rollout_ref.actor.optim.lr=3e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${BATCH_SIZE} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.layered_summon=True \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name=${EXP_NAME} \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.total_training_steps=1 \ + trainer.save_freq=1 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 $@ + +# step 2. merge model +python3 -m verl.model_merger merge \ + --backend fsdp \ + --local_dir checkpoints/verl_grpo_example_gsm8k/${EXP_NAME}/global_step_1/actor/ \ + --target_dir checkpoints/verl_grpo_example_gsm8k/${EXP_NAME}/global_step_1/actor/hf + +# step 3. assert +# make sure adapter_model.safetensors exists and its size is larger than 1MB +file_path="checkpoints/verl_grpo_example_gsm8k/${EXP_NAME}/global_step_1/actor/hf/lora_adapter/adapter_model.safetensors" + +if [ ! -f "$file_path" ]; then + echo "Error: File $file_path does not exist!" + exit 1 +fi + +file_size=$(stat -c %s "$file_path") + +min_size_mb=1 +min_size=$((min_size_mb * 1024 * 1024)) # 1MB = 1048576 bytes + +if [ "$file_size" -lt "$min_size" ]; then + echo "Error: File $file_path is too small! Current size: $((file_size/1024))KB, Required: ${min_size_mb}MB" + exit 1 +fi + +echo "Check passed: File exists and size is $(($file_size/1024/1024))MB" +exit 0 diff --git a/e2c/verl/tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_sf_tool.sh b/e2c/verl/tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_sf_tool.sh new file mode 100644 index 0000000000000000000000000000000000000000..729b42554743b384e2464bda91c729be751622a1 --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_sf_tool.sh @@ -0,0 +1,62 @@ +# run on 8xH20 +# make sure your current working directory is the root of the project + +set -x + + +export PYTHONUNBUFFERED=1 +export RAY_DEDUP_LOGS=0 +export RUST_BACKTRACE=1 +export HYDRA_FULL_ERROR=1 + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_sf_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=128 \ + data.max_prompt_length=2048 \ + data.max_response_length=16384 \ + data.filter_overlong_prompts=False \ + data.truncation='error' \ + data.return_raw_chat=True \ + data.train_files=$HOME/data/retool_dapo/train.parquet \ + data.val_files=$HOME/data/retool_aime2024/train.parquet \ + actor_rollout_ref.model.path=Qwen/Qwen3-4B \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.use_liger=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.enable_activation_offloading=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/sandbox_fusion_tool_config.yaml" \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='retool_async_rl' \ + trainer.experiment_name='qwen3-4b_function_rm-retool-async-sgl-no-sft-n8-v2505271300' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=100 \ + trainer.test_freq=20 \ + trainer.total_training_steps=1000 \ + trainer.total_epochs=1 $@ \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_w_tool.sh b/e2c/verl/tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_w_tool.sh new file mode 100644 index 0000000000000000000000000000000000000000..e192f841d258723b03c9c0db05b7c769fc8e2e4e --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_gsm8k_fsdp_sgl_multiturn_w_tool.sh @@ -0,0 +1,58 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +#huggingface-cli download Qwen/Qwen2.5-3B-Instruct --local-dir $HOME/models/Qwen/Qwen2.5-3B-Instruct + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" +FSDP_STRATEGY=${FSDP_STRATEGY:-fsdp} + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.strategy=$FSDP_STRATEGY \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.strategy=$FSDP_STRATEGY \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='gsm8k_async_rl' \ + trainer.experiment_name=qwen2.5-3b_function_rm-gsm8k-sgl-multi-w-tool-$FSDP_STRATEGY-rebased-0427-verify-n16 \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + data.train_files=$HOME/data/gsm8k_verl_sgl_multi_turn_preprocessed/train.parquet \ + data.val_files=$HOME/data/gsm8k_verl_sgl_multi_turn_preprocessed/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + trainer.val_before_train=False \ + trainer.total_training_steps=1 $@ diff --git a/e2c/verl/tests/special_e2e/run_one_step_off_policy.sh b/e2c/verl/tests/special_e2e/run_one_step_off_policy.sh new file mode 100644 index 0000000000000000000000000000000000000000..3f8b908b6e178d6f12dd56706348c1b1985e160d --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_one_step_off_policy.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +# Test script for one_step_off_policy E2E regression testing +# This script runs one_step_off_policy with both FSDP2 and Megatron backends +# to ensure the asynchronous training mechanism works correctly + +NUM_GPUS=${NUM_GPUS:-8} +ACTOR_STRATEGY=${ACTOR_STRATEGY:-"fsdp2"} # fsdp2 or megatron + +# Download model if not exists +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +# Algorithm parameters +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +# Response length parameters +max_prompt_length=1024 +max_response_length=2048 +enable_overlong_buffer=True +overlong_buffer_len=128 +overlong_penalty_factor=1.0 + +# Training parameters +loss_agg_mode="token-mean" +train_prompt_bsz=8 +n_resp_per_prompt=3 +train_prompt_mini_bsz=4 + +# Temperature parameters +temperature=1.0 +top_p=1.0 +top_k=-1 +val_top_p=0.7 + +# One-step-off-policy specific parameters +# Allocate 2 GPUs for rollout, remaining for training +n_gpus_rollout=2 +n_gpus_training=$((NUM_GPUS - n_gpus_rollout)) + +exp_name="$(basename "${MODEL_ID,,}")-one-step-off-policy-${ACTOR_STRATEGY}-minimal" + +echo "Running one_step_off_policy with ${ACTOR_STRATEGY} strategy" +echo "Total GPUs: ${NUM_GPUS}, Rollout GPUs: ${n_gpus_rollout}, Training GPUs: ${n_gpus_training}" + +# Common parameters for both FSDP2 and Megatron +common_params=( + data.train_files="${HOME}/data/gsm8k/train.parquet" + data.val_files="${HOME}/data/gsm8k/test.parquet" + data.prompt_key=prompt + data.truncation='left' + data.max_prompt_length=${max_prompt_length} + data.max_response_length=${max_response_length} + data.train_batch_size=${train_prompt_bsz} + actor_rollout_ref.rollout.n=${n_resp_per_prompt} + algorithm.adv_estimator=${adv_estimator} + algorithm.use_kl_in_reward=${use_kl_in_reward} + algorithm.kl_ctrl.kl_coef=${kl_coef} + actor_rollout_ref.hybrid_engine=False \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} + actor_rollout_ref.actor.clip_ratio_c=10.0 + actor_rollout_ref.model.path="${MODEL_PATH}" + actor_rollout_ref.actor.optim.lr=1e-6 + actor_rollout_ref.actor.optim.lr_warmup_steps=-1 + actor_rollout_ref.actor.optim.weight_decay=0.1 + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} + actor_rollout_ref.actor.entropy_coeff=0 + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 + actor_rollout_ref.rollout.temperature=${temperature} + actor_rollout_ref.rollout.top_p=${top_p} + actor_rollout_ref.rollout.top_k=${top_k} + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} + actor_rollout_ref.rollout.val_kwargs.do_sample=True + actor_rollout_ref.rollout.val_kwargs.n=1 + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.name=vllm \ + reward_model.reward_manager=dapo + +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} + +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} + +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} + +reward_model.reward_kwargs.overlong_buffer_cfg.log=False + +reward_model.reward_kwargs.max_resp_len=${max_response_length} + trainer.logger=['console'] + trainer.project_name='verl-test' + trainer.experiment_name="${exp_name}" + trainer.val_before_train=False + trainer.test_freq=-1 + trainer.save_freq=-1 + trainer.total_epochs=2 + trainer.total_training_steps=2 + trainer.resume_mode=disable + trainer.nnodes=1 + trainer.n_gpus_per_node=${n_gpus_training} + rollout.nnodes=1 + rollout.n_gpus_per_node=${n_gpus_rollout} + +) + +if [ "${ACTOR_STRATEGY}" == "fsdp2" ]; then + echo "Running with FSDP2 strategy..." + # FSDP2 specific parameters + gen_tp=2 + sp_size=2 + fsdp_size=2 + ref_offload=True + actor_offload=False + + python3 -m recipe.one_step_off_policy.main_ppo \ + "${common_params[@]}" \ + actor_rollout_ref.actor.strategy=fsdp2 \ + critic.strategy=fsdp2 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} $@ + +elif [ "${ACTOR_STRATEGY}" == "megatron" ]; then + echo "Running with Megatron strategy..." + # Megatron specific parameters + gen_tp=2 + train_tp=1 + train_pp=2 + ref_offload=True + actor_offload=False + + python3 -m recipe.one_step_off_policy.main_ppo \ + --config-path=config \ + --config-name='one_step_off_ppo_megatron_trainer.yaml' \ + "${common_params[@]}" \ + actor_rollout_ref.actor.strategy=megatron \ + critic.strategy=megatron \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.megatron.param_offload=${actor_offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${actor_offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${actor_offload} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.param_offload=${ref_offload} $@ +else + echo "Error: Unknown strategy ${ACTOR_STRATEGY}. Please use 'fsdp2' or 'megatron'" + exit 1 +fi + +echo "One-step-off-policy E2E test completed successfully with ${ACTOR_STRATEGY} strategy" \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/run_ppo_trainer_megatron.sh b/e2c/verl/tests/special_e2e/run_ppo_trainer_megatron.sh new file mode 100644 index 0000000000000000000000000000000000000000..f79e5d7f2a239413c67f29aacddfe998c3bc3b37 --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_ppo_trainer_megatron.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping +export VERL_LOGGING_LEVEL=INFO +export VERL_PPO_LOGGING_LEVEL=INFO + +NUM_GPUS=${NUM_GPUS:-8} + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +USE_DUMMY_MODEL=${USE_DUMMY_MODEL:-False} +DUMMY_MODEL_PATH=${DUMMY_MODEL_PATH:-${HOME}/dummy_models/${MODEL_ID}} +if [ "$USE_DUMMY_MODEL" = "True" ]; then + if [ -z "${DUMMY_MODEL_CONFIG_PATH}" ]; then + echo "[ERROR] DUMMY_MODEL_CONFIG_PATH not set" + exit 1 + fi + + python scripts/init_random_model.py \ + --hf_model_path "${MODEL_PATH}" \ + --new_config_path "${DUMMY_MODEL_CONFIG_PATH}" \ + --output_path "${DUMMY_MODEL_PATH}" + + MODEL_PATH="${DUMMY_MODEL_PATH}" +fi + +TRAIN_FILES=${TRAIN_FILES:-${HOME}/data/gsm8k/train.parquet} +VAL_FILES=${VAL_FILES:-${HOME}/data/gsm8k/test.parquet} + +ADV_ESTIMATOR=${ADV_ESTIMATOR:-gae} +# Validation +VAL_BEFORE_TRAIN=${VAL_BEFORE_TRAIN:-False} +TEST_FREQ=${TEST_FREQ:--1} +# Save & Resume +RESUME_MODE=${RESUME_MODE:-disable} +SAVE_FREQ=${SAVE_FREQ:--1} +TOTAL_TRAIN_STEPS=${TOTAL_TRAIN_STEPS:-1} + +USE_DYNAMIC_BSZ=${USE_DYNAMIC_BSZ:-True} +ppo_max_token_len_per_gpu=${PPO_MAX_TOKEN_LEN:-2400} +forward_max_token_len_per_gpu=${FWD_MAX_TOKEN_LEN:-4800} +train_traj_micro_bsz_per_gpu=${MICRO_BSZ:-2} # b +n_resp_per_prompt=4 # g + +train_traj_micro_bsz=$((train_traj_micro_bsz_per_gpu * NUM_GPUS)) # b * n +train_traj_mini_bsz=$((train_traj_micro_bsz * 2)) # 2 * b * n +train_prompt_mini_bsz=$((train_traj_mini_bsz * n_resp_per_prompt)) # 2 * b * n / g +train_prompt_bsz=$((train_prompt_mini_bsz * 2)) # 4 * b * n / g + +MAX_PROMPT_LENGTH=${MAX_PROMPT_LENGTH:-512} +MAX_RESPONSE_LENGTH=${MAX_RESPONSE_LENGTH:-512} + +COMMON_PP=${COMMON_PP:-2} +COMMON_VPP=${COMMON_VPP:-2} +COMMON_CP=${COMMON_CP:-2} +COMMON_TP=${COMMON_TP:-2} +COMMON_EP=${COMMON_EP:-1} +COMMON_ETP=${COMMON_ETP:-1} + +TRAIN_TP=${TRAIN_TP:-$COMMON_TP} +INFER_TP=${INFER_TP:-$COMMON_TP} + +ACTOR_PP=${ACTOR_PP:-$COMMON_PP} +ACTOR_VPP=${ACTOR_VPP:-$COMMON_VPP} +ACTOR_CP=${ACTOR_CP:-$COMMON_CP} +ACTOR_TP=${ACTOR_TP:-$TRAIN_TP} +ACTOR_EP=${ACTOR_EP:-$COMMON_EP} +ACTOR_ETP=${ACTOR_ETP:-$COMMON_ETP} +ROLLOUT_TP=${ROLLOUT_TP:-$INFER_TP} +REF_PP=${REF_PP:-$COMMON_PP} +REF_VPP=${REF_VPP:-$COMMON_VPP} +REF_CP=${REF_CP:-$COMMON_CP} +REF_TP=${REF_TP:-$TRAIN_TP} +REF_EP=${REF_EP:-$COMMON_EP} +REF_ETP=${REF_ETP:-$COMMON_ETP} +CRITIC_PP=${CRITIC_PP:-$COMMON_PP} +CRITIC_VPP=${CRITIC_VPP:-$COMMON_VPP} +CRITIC_CP=${CRITIC_CP:-$COMMON_CP} +CRITIC_TP=${CRITIC_TP:-$TRAIN_TP} +CRITIC_EP=${CRITIC_EP:-$COMMON_EP} +CRITIC_ETP=${CRITIC_ETP:-$COMMON_ETP} +RM_PP=${RM_PP:-$COMMON_PP} +RM_VPP=${RM_VPP:-$COMMON_VPP} +RM_CP=${RM_CP:-$COMMON_CP} +RM_TP=${RM_TP:-$TRAIN_TP} +RM_EP=${RM_EP:-$COMMON_EP} +RM_ETP=${RM_ETP:-$COMMON_ETP} + +ALL_OFFLOAD=${ALL_OFFLOAD:-False} +COMMON_PARAM_OFFLOAD=${COMMON_PARAM_OFFLOAD:-$ALL_OFFLOAD} +COMMON_GRAD_OFFLOAD=${COMMON_GRAD_OFFLOAD:-$ALL_OFFLOAD} +COMMON_OPTIMIZER_OFFLOAD=${COMMON_OPTIMIZER_OFFLOAD:-$ALL_OFFLOAD} + +ACTOR_PARAM_OFFLOAD=${ACTOR_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +ACTOR_GRAD_OFFLOAD=${ACTOR_GRAD_OFFLOAD:-$COMMON_GRAD_OFFLOAD} +ACTOR_OPTIMIZER_OFFLOAD=${ACTOR_OPTIMIZER_OFFLOAD:-$COMMON_OPTIMIZER_OFFLOAD} +REF_PARAM_OFFLOAD=${REF_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +CRITIC_PARAM_OFFLOAD=${CRITIC_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +CRITIC_GRAD_OFFLOAD=${CRITIC_GRAD_OFFLOAD:-$COMMON_GRAD_OFFLOAD} +CRITIC_OPTIMIZER_OFFLOAD=${CRITIC_OPTIMIZER_OFFLOAD:-$COMMON_OPTIMIZER_OFFLOAD} +RM_PARAM_OFFLOAD=${RM_PARAM_OFFLOAD:-$COMMON_PARAM_OFFLOAD} +USE_MBRIDGE=${USE_MBRIDGE:-False} +USE_FUSED_KERNELS=${USE_FUSED_KERNELS:-False} + +LR_WARMUP_STEPS=${LR_WARMUP_STEPS:-null} + +CHECKPOINT_CONTENTS=['model','hf_model','optimizer','extra'] +SKIP_SAVE_HF_MODEL=${SKIP_SAVE_HF_MODEL:-0} +if [ $SKIP_SAVE_HF_MODEL -eq 1 ]; then + CHECKPOINT_CONTENTS=['model','optimizer','extra'] +fi + +USE_DIST_CKPT=${USE_DIST_CKPT:-False} +DIST_CKPT_PATH=${DIST_CKPT_PATH:-${HOME}/dist_ckpt/${MODEL_ID}} +if [ "$USE_DIST_CKPT" = "True" ]; then + if [ "$USE_DUMMY_MODEL" = "True" ]; then + DIST_CKPT_PATH=${HOME}/dist_ckpt_dummy/${MODEL_ID} + fi + python scripts/converter_hf_to_mcore.py \ + --hf_model_path "${MODEL_PATH}" \ + --output_path "${DIST_CKPT_PATH}" +fi + +ENGINE=${ENGINE:-"vllm"} + +exp_name="$(basename "${MODEL_ID,,}")-megatron-gsm8k-minimal" + +if [ "$ENGINE" = "vllm" ]; then + MODE=${MODE:-"sync"} + ROLLOUT_MODE_ARG="actor_rollout_ref.rollout.mode=${MODE}" + if [ "$MODE" = "async" ]; then + ROLLOUT_MODE_ARG="${ROLLOUT_MODE_ARG} data.return_raw_chat=True" + fi +else + ROLLOUT_MODE_ARG="" +fi + +OPTIM_MEMORY_EFFICIENT=${OPTIM_MEMORY_EFFICIENT:-False} + +PROFILE_ENABLE=${PROFILE_ENABLE:-False} +PROFILE_STEPS=${PROFILE_STEPS:-[1]} +PROFILE_RANKS_ALL=${PROFILE_RANKS_ALL:-True} +PROFILE_RANKS=${PROFILE_RANKS:-[0,1,2,3]} +DISCRETE=${DISCRETE:-True} # or True + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml'\ + algorithm.adv_estimator="${ADV_ESTIMATOR}" \ + data.train_files="${TRAIN_FILES}" \ + data.val_files="${VAL_FILES}" \ + data.train_batch_size=${train_prompt_bsz} \ + data.max_prompt_length=${MAX_PROMPT_LENGTH} \ + data.max_response_length=${MAX_RESPONSE_LENGTH} \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.use_fused_kernels=${USE_FUSED_KERNELS} \ + actor_rollout_ref.actor.optim.lr_warmup_steps=$LR_WARMUP_STEPS \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=$OPTIM_MEMORY_EFFICIENT \ + +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=$OPTIM_MEMORY_EFFICIENT \ + +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=$OPTIM_MEMORY_EFFICIENT \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.actor.use_dynamic_bsz=${USE_DYNAMIC_BSZ} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${ppo_max_token_len_per_gpu} \ + actor_rollout_ref.actor.megatron.use_mbridge=${USE_MBRIDGE} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=$ACTOR_PP \ + actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=$ACTOR_VPP \ + actor_rollout_ref.actor.megatron.context_parallel_size=$ACTOR_CP \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=$ACTOR_TP \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=$ACTOR_EP \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=$ACTOR_ETP \ + actor_rollout_ref.actor.megatron.param_offload=${ACTOR_PARAM_OFFLOAD} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${ACTOR_OPTIMIZER_OFFLOAD} \ + actor_rollout_ref.actor.megatron.grad_offload=${ACTOR_GRAD_OFFLOAD} \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ + actor_rollout_ref.actor.megatron.dist_checkpointing_path=${DIST_CKPT_PATH} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.checkpoint.save_contents=$CHECKPOINT_CONTENTS \ + actor_rollout_ref.actor.profiler.enable=$PROFILE_ENABLE \ + actor_rollout_ref.actor.profiler.ranks=$PROFILE_RANKS \ + actor_rollout_ref.actor.profiler.all_ranks=$PROFILE_RANKS_ALL \ + actor_rollout_ref.rollout.name="${ENGINE}" ${ROLLOUT_MODE_ARG}\ + actor_rollout_ref.rollout.tensor_model_parallel_size=$ROLLOUT_TP \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.rollout.update_weights_bucket_megabytes=128 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.ref.megatron.use_mbridge=${USE_MBRIDGE} \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=$REF_PP \ + actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=$REF_VPP \ + actor_rollout_ref.ref.megatron.context_parallel_size=$REF_CP \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=$REF_TP \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=$REF_EP \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=$REF_ETP \ + actor_rollout_ref.ref.megatron.param_offload=${REF_PARAM_OFFLOAD} \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ + actor_rollout_ref.ref.megatron.dist_checkpointing_path=${DIST_CKPT_PATH} \ + critic.optim.lr=2e-5 \ + critic.optim.lr_warmup_steps=$LR_WARMUP_STEPS \ + +critic.optim.override_optimizer_config.optimizer_cpu_offload=$OPTIM_MEMORY_EFFICIENT \ + +critic.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=$OPTIM_MEMORY_EFFICIENT \ + +critic.optim.override_optimizer_config.use_precision_aware_optimizer=$OPTIM_MEMORY_EFFICIENT \ + critic.model.path="${MODEL_PATH}" \ + critic.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + critic.ppo_max_token_len_per_gpu=${forward_max_token_len_per_gpu} \ + critic.megatron.use_mbridge=${USE_MBRIDGE} \ + critic.megatron.pipeline_model_parallel_size=$CRITIC_PP \ + critic.megatron.virtual_pipeline_model_parallel_size=$CRITIC_VPP \ + critic.megatron.context_parallel_size=$CRITIC_CP \ + critic.megatron.tensor_model_parallel_size=$CRITIC_TP \ + critic.megatron.expert_model_parallel_size=$CRITIC_EP \ + critic.megatron.expert_tensor_parallel_size=$CRITIC_ETP \ + critic.megatron.param_offload=${CRITIC_PARAM_OFFLOAD} \ + critic.megatron.optimizer_offload=${CRITIC_OPTIMIZER_OFFLOAD} \ + critic.megatron.grad_offload=${CRITIC_GRAD_OFFLOAD} \ + critic.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ + critic.megatron.dist_checkpointing_path=${DIST_CKPT_PATH} \ + critic.checkpoint.save_contents=$CHECKPOINT_CONTENTS \ + critic.profiler.enable=$PROFILE_ENABLE \ + critic.profiler.ranks=$PROFILE_RANKS \ + critic.profiler.all_ranks=$PROFILE_RANKS_ALL \ + reward_model.enable=True \ + reward_model.model.path="${MODEL_PATH}" \ + reward_model.micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + reward_model.megatron.use_mbridge=${USE_MBRIDGE} \ + reward_model.megatron.pipeline_model_parallel_size=$RM_PP \ + reward_model.megatron.virtual_pipeline_model_parallel_size=$RM_VPP \ + reward_model.megatron.context_parallel_size=$RM_CP \ + reward_model.megatron.tensor_model_parallel_size=$RM_TP \ + reward_model.megatron.expert_model_parallel_size=$RM_EP \ + reward_model.megatron.expert_tensor_parallel_size=$RM_ETP \ + reward_model.megatron.param_offload=${RM_PARAM_OFFLOAD} \ + reward_model.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ + reward_model.megatron.dist_checkpointing_path=${DIST_CKPT_PATH} \ + reward_model.profiler.enable=$PROFILE_ENABLE \ + reward_model.profiler.ranks=$PROFILE_RANKS \ + reward_model.profiler.all_ranks=$PROFILE_RANKS_ALL \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_penalty=kl \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl-test' \ + trainer.experiment_name="${exp_name}" \ + trainer.nnodes=1 \ + trainer.n_gpus_per_node=${NUM_GPUS} \ + trainer.val_before_train="${VAL_BEFORE_TRAIN}" \ + trainer.test_freq="${TEST_FREQ}" \ + trainer.save_freq="${SAVE_FREQ}" \ + trainer.resume_mode="${RESUME_MODE}" \ + trainer.total_epochs=2 \ + trainer.total_training_steps="${TOTAL_TRAIN_STEPS}" \ + global_profiler.profile_continuous_steps=True \ + global_profiler.tool=nsys \ + global_profiler.steps=$PROFILE_STEPS \ + global_profiler.global_tool_config.nsys.discrete=$DISCRETE $@ diff --git a/e2c/verl/tests/special_e2e/run_prime.sh b/e2c/verl/tests/special_e2e/run_prime.sh new file mode 100644 index 0000000000000000000000000000000000000000..cfd72101c49e502f00c149ce76b77b6e949c5135 --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_prime.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +NUM_GPUS=${NUM_GPUS:-8} + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +TRAIN_FILES=${TRAIN_FILES:-${HOME}/data/gsm8k/train.parquet} +VAL_FILES=${VAL_FILES:-${HOME}/data/gsm8k/test.parquet} + +train_traj_micro_bsz_per_gpu=2 # b +n_resp_per_prompt=4 # g + +train_traj_micro_bsz=$((train_traj_micro_bsz_per_gpu * NUM_GPUS)) # b * n +train_traj_mini_bsz=$((train_traj_micro_bsz * 2)) # 2 * b * n +train_prompt_mini_bsz=$((train_traj_mini_bsz * n_resp_per_prompt)) # 2 * b * n / g +train_prompt_bsz=$((train_prompt_mini_bsz * 2)) # 4 * b * n / g + +exp_name="$(basename "${MODEL_ID,,}")-prime-minimal" + +python3 -m recipe.prime.main_prime \ + data.train_files="${TRAIN_FILES}" \ + data.val_files="${VAL_FILES}" \ + data.train_batch_size=${train_prompt_bsz} \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.filter_accuracy=True \ + data.accuracy_lower_bound=0.2 \ + data.accuracy_upper_bound=0.8 \ + data.oversample_factor=4 \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=5e-7 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.model.enable_gradient_checkpointing=False \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.adv_estimator=rloo \ + algorithm.use_kl_in_reward=True \ + algorithm.kl_penalty=kl \ + algorithm.kl_ctrl.kl_coef=0.001 \ + reward_model.model.path="${MODEL_PATH}" \ + reward_model.micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + reward_model.model.update=before \ + reward_model.model.beta_train=0.05 \ + reward_model.model.optim.lr=1e-6 \ + reward_model.model.optim.grad_clip=10.0 \ + reward_model.model.input_tokenizer=null \ + reward_model.mini_batch_size=${train_prompt_bsz} \ + reward_model.reward_manager=prime \ + trainer.val_before_train=False \ + trainer.logger=console \ + trainer.project_name='verl-test' \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=${NUM_GPUS} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.total_training_steps=1 $@ diff --git a/e2c/verl/tests/special_e2e/run_r1_distill_qwen_aime24_eval.sh b/e2c/verl/tests/special_e2e/run_r1_distill_qwen_aime24_eval.sh new file mode 100644 index 0000000000000000000000000000000000000000..c099d9efb2d295bf79c4dddf7f8bd6b464db5b6a --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_r1_distill_qwen_aime24_eval.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +#huggingface-cli download deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \ +# --local-dir $HOME/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B + +python3 -m verl.trainer.main_generation \ + trainer.nnodes=1 \ + trainer.n_gpus_per_node=8 \ + data.path=$HOME/data/r1/test.parquet \ + data.prompt_key=prompt \ + data.batch_size=1024 \ + data.n_samples=1 \ + data.output_path=$HOME/data/r1/test-output-k1.parquet \ + model.path=$HOME/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \ + rollout.temperature=0.6 \ + rollout.top_p=0.95 \ + rollout.prompt_length=1024 \ + rollout.response_length=32768 \ + rollout.tensor_model_parallel_size=1 \ + rollout.gpu_memory_utilization=0.95 \ + rollout.max_num_batched_tokens=65536 \ + rollout.enforce_eager=False \ + rollout.free_cache_engine=True + +python3 -m recipe.r1.main_eval \ + data.path=$HOME/data/r1/test-output-k1.parquet \ + data.prompt_key=prompt \ + data.response_key=responses \ + custom_reward_function.path=recipe/r1/reward_score.py \ + custom_reward_function.name=reward_func \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/run_spin.sh b/e2c/verl/tests/special_e2e/run_spin.sh new file mode 100644 index 0000000000000000000000000000000000000000..0e627ddd29fc4a7298799a6a35376313d56d65c0 --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_spin.sh @@ -0,0 +1,35 @@ +set -e +set -x +NUM_GPUS=${NUM_GPUS:-8} + +exp_name="Qwen2.5-0.5B-Instruct-spin-minimal" + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +CUDA_VISIBLE_DEVICES=${VISIBLE_DEVICES} python3 -m recipe.spin.main_spin \ + data.train_files="${HOME}/data/gsm8k/train.parquet" \ + data.val_files="${HOME}/data/gsm8k/test.parquet" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size=8 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=64 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=64 \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.logger=console \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=1 \ + +trainer.log_freq=1 \ + trainer.ref_update_freq=1 \ + trainer.total_training_steps=1 \ + trainer.total_epochs=1000 2>&1 | tee verl_demo.log \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/run_sppo.sh b/e2c/verl/tests/special_e2e/run_sppo.sh new file mode 100644 index 0000000000000000000000000000000000000000..7b40af157168777ee06229a386e3656ff39b88d0 --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_sppo.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +# in e2e_sppo.yml, we set NUM_GPUS=8 L20 + +NUM_GPUS=${NUM_GPUS:-8} + +exp_name="Qwen2.5-0.5B-Instruct-sppo-minimal" + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +python3 -m recipe.sppo.main_sppo \ + data.train_files="${HOME}/data/math/train.parquet" \ + data.val_files="${HOME}/data/math/test.parquet" \ + data.train_batch_size=1024 \ + data.max_prompt_length=1024 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path="$MODEL_PATH" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=$NUM_GPUS \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.total_training_steps=1 \ + trainer.total_epochs=2 $@ diff --git a/e2c/verl/tests/special_e2e/run_test.sh b/e2c/verl/tests/special_e2e/run_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..74686b41abb99c8944f526a312599bba5e2c1b1e --- /dev/null +++ b/e2c/verl/tests/special_e2e/run_test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -xeuo pipefail + +# Get the configuration name and engine name from arguments +CONFIG_NAME="$1" +ENGINE="${2:-vllm}" + +# Download model if needed +#huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir "$HOME/models/Qwen/Qwen2.5-0.5B" + +# Run the training with the specified configuration +python3 -m verl.trainer.main_ppo \ + --config-name "$CONFIG_NAME" "$@" \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/sft/compare_sft_engine_results.py b/e2c/verl/tests/special_e2e/sft/compare_sft_engine_results.py new file mode 100644 index 0000000000000000000000000000000000000000..b39e133ee5e99d33e51e8113839be724b5d06851 --- /dev/null +++ b/e2c/verl/tests/special_e2e/sft/compare_sft_engine_results.py @@ -0,0 +1,57 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import torch + + +def get_result(file): + file = os.path.expanduser(file) + result = [] + with open(file) as f: + lines = f.readlines() + for line in lines: + result.append(json.loads(line)) + return result + + +def compare_results(golden_results, other_result): + golden_loss = golden_results[0]["data"]["train/loss"] + golden_grad_norm = golden_results[0]["data"]["train/grad_norm"] + + loss = other_result[0]["data"]["train/loss"] + grad_norm = other_result[0]["data"]["train/grad_norm"] + + torch.testing.assert_close(golden_loss, loss, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(golden_grad_norm, grad_norm, atol=1e-4, rtol=1e-2) + + +if __name__ == "__main__": + golden_results = get_result("~/verl/test/log/golden.jsonl") + + # get all other results + other_results = {} + # walk through all files in ~/verl/test/log + for file in os.listdir(os.path.expanduser("~/verl/test/log/verl_sft_test")): + if file.endswith(".jsonl"): + other_results[file] = get_result(os.path.join(os.path.expanduser("~/verl/test/log/verl_sft_test"), file)) + + # # compare results + for file, other_result in other_results.items(): + print(f"compare results {file}") + compare_results(golden_results, other_result) + + print("All results are close to golden results") diff --git a/e2c/verl/tests/special_e2e/sft/run_sft.sh b/e2c/verl/tests/special_e2e/sft/run_sft.sh new file mode 100644 index 0000000000000000000000000000000000000000..ae4fea388d96ca22d91a3bcf347e7551d2ddea94 --- /dev/null +++ b/e2c/verl/tests/special_e2e/sft/run_sft.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +ENTRYPOINT=${ENTRYPOINT:-"-m verl.trainer.fsdp_sft_trainer"} + +NUM_GPUS=${NUM_GPUS:-8} + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +TRAIN_FILES=${TRAIN_FILES:-$HOME/data/gsm8k/train.parquet} +VAL_FILES=${VAL_FILES:-$HOME/data/gsm8k/test.parquet} + +SP_SIZE=${SP_SIZE:-1} +LIGER=${LIGER:-False} +MULTITURN=${MULTITURN:-False} +LORA_RANK=${LORA_RANK:-0} +RM_PAD=${RM_PAD:-True} + +TOTAL_TRAIN_STEP=${TOTAL_TRAIN_STEP:-1} +RESUME_MODE=${RESUME_MODE:-disable} +SAVE_FREQ=${SAVE_FREQ:-1} + +micro_bsz=2 +NUM_GPUS=8 + +project_name="verl-test" +exp_name="$(basename "${MODEL_ID,,}")-sft-minimal" +ckpts_home=${ckpts_home:-$HOME/${project_name}/${exp_name}} + +mkdir -p "${ckpts_home}" + +torchrun --standalone --nnodes=1 --nproc_per_node=${NUM_GPUS} ${ENTRYPOINT} \ + data.train_files="${TRAIN_FILES}" \ + data.val_files="${VAL_FILES}" \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + data.prompt_dict_keys=['question'] \ + data.response_dict_keys=['answer'] \ + data.multiturn.enable="${MULTITURN}" \ + data.multiturn.messages_key=messages \ + optim.lr=1e-4 \ + data.micro_batch_size_per_gpu=${micro_bsz} \ + model.strategy=fsdp \ + model.partial_pretrain="${MODEL_PATH}" \ + model.lora_rank="${LORA_RANK}" \ + model.lora_alpha=16 \ + model.target_modules=all-linear \ + model.use_liger="${LIGER}" \ + ulysses_sequence_parallel_size="${SP_SIZE}" \ + use_remove_padding="${RM_PAD}" \ + trainer.default_local_dir="${ckpts_home}" \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.total_training_steps=${TOTAL_TRAIN_STEP} \ + trainer.save_freq=${SAVE_FREQ} \ + trainer.checkpoint.save_contents=[model,optimizer,extra,hf_model] \ + trainer.max_ckpt_to_keep=1 \ + trainer.resume_mode=${RESUME_MODE} \ + trainer.logger=['console'] $@ + +rm -rf "${ckpts_home:?}/*" \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/sft/run_sft_engine_gsm8k.sh b/e2c/verl/tests/special_e2e/sft/run_sft_engine_gsm8k.sh new file mode 100644 index 0000000000000000000000000000000000000000..90f8d80358881ed307bc573f8850dcf5452afce9 --- /dev/null +++ b/e2c/verl/tests/special_e2e/sft/run_sft_engine_gsm8k.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +ENTRYPOINT=${ENTRYPOINT:-"-m verl.trainer.sft_trainer"} + +NUM_GPUS=${NUM_GPUS:-1} + +TRAIN_FILES=~/data/gsm8k_sft/train.parquet +VAL_FILES=~/data/gsm8k_sft/test.parquet + +backend=${BACKEND:-fsdp} + +project_name=verl_sft_test + +RESUME_MODE=disable + +ckpts_home=${ckpts_home:-~/verl/test/gsm8k-sft-${backend}} + +MODEL_ID=${MODEL_ID:-Qwen/Qwen3-0.6B} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} +#huggingface-cli download "${MODEL_ID}" --local-dir "${MODEL_PATH}" + +SP_SIZE=${SP_SIZE:-1} +FSDP_SIZE=${FSDP_SIZE:-${NUM_GPUS}} +FSDP_STRATEGY=${FSDP_STRATEGY:-"fsdp"} + +TP_SIZE=${TP_SIZE:-1} +PP_SIZE=${PP_SIZE:-1} +VPP_SIZE=${VPP_SIZE:-null} +CP_SIZE=${CP_SIZE:-1} + +PAD_MODE=${PAD_MODE:-left_right} + +USE_REMOVE_PADDING=${USE_REMOVE_PADDING:-True} + +FSDP_ENGINE_CONFIG="\ + engine=${backend} \ + optim=${backend} \ + optim.lr=1e-5 \ + optim.lr_warmup_steps_ratio=0.2 \ + optim.weight_decay=0.1 \ + optim.betas="[0.9,0.95]" \ + optim.clip_grad=1.0 \ + optim.min_lr_ratio=0.1 \ + optim.warmup_style=cosine \ + engine.ulysses_sequence_parallel_size=${SP_SIZE} \ + engine.strategy=${FSDP_STRATEGY} \ + engine.fsdp_size=${FSDP_SIZE}" + + +MEGATRON_ENGINE_CONFIG="\ + engine=${backend} \ + optim=${backend} \ + optim.lr=1e-5 \ + optim.lr_warmup_steps_ratio=0.2 \ + optim.weight_decay=0.1 \ + optim.betas="[0.9,0.95]" \ + optim.clip_grad=1.0 \ + optim.lr_warmup_init=0 \ + optim.lr_decay_style=cosine \ + optim.min_lr=1e-6 \ + engine.tensor_model_parallel_size=${TP_SIZE} \ + engine.pipeline_model_parallel_size=${PP_SIZE} \ + engine.virtual_pipeline_model_parallel_size=${VPP_SIZE} \ + engine.context_parallel_size=${CP_SIZE}" + +if [ "$backend" = "fsdp" ]; then + ENGINE_CONFIG="$FSDP_ENGINE_CONFIG" + echo "Using fsdp engine" + exp_name=gsm8k-${backend}-${FSDP_STRATEGY}-sp${SP_SIZE}-fsdp${FSDP_SIZE}-pad-${PAD_MODE}-use_remove_padding-${USE_REMOVE_PADDING} +else + ENGINE_CONFIG="$MEGATRON_ENGINE_CONFIG" + echo "Using megatron engine" + exp_name=gsm8k-${backend}-tp${TP_SIZE}-pp${PP_SIZE}-vpp${VPP_SIZE}-cp${CP_SIZE}-pad-${PAD_MODE}-use_remove_padding-${USE_REMOVE_PADDING} +fi + +mkdir -p "${ckpts_home}" + +torchrun --standalone --nnodes=1 --nproc_per_node=${NUM_GPUS} ${ENTRYPOINT} \ + data.train_files="${TRAIN_FILES}" \ + data.val_files="${VAL_FILES}" \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.pad_mode=${PAD_MODE} \ + data.truncation=error \ + data.use_dynamic_bsz=True \ + data.max_token_len_per_gpu=8192 \ + data.messages_key=messages \ + model.path=$MODEL_PATH \ + model.use_remove_padding=${USE_REMOVE_PADDING} \ + ${ENGINE_CONFIG} \ + trainer.test_freq=after_each_epoch \ + trainer.save_freq=-1 \ + trainer.logger=['console','file'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.total_epochs=2 \ + trainer.total_training_steps=2 \ + trainer.default_local_dir="${ckpts_home}" \ + trainer.resume_mode=${RESUME_MODE} \ + + # trainer.total_training_steps=${TOTAL_TRAIN_STEP} \ + # trainer.checkpoint.save_contents=[model,optimizer,extra,hf_model] \ + # trainer.max_ckpt_to_keep=1 \ + +rm -rf "${ckpts_home:?}/*" \ No newline at end of file diff --git a/e2c/verl/tests/special_e2e/sft/test_sft_engine_all.sh b/e2c/verl/tests/special_e2e/sft/test_sft_engine_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..8fed89fe0ed4fae739e54f1e2d10b24720e48ad0 --- /dev/null +++ b/e2c/verl/tests/special_e2e/sft/test_sft_engine_all.sh @@ -0,0 +1,61 @@ + +rm -rf ~/verl/test/log +mkdir -p ~/verl/test/log + +export VERL_FILE_LOGGER_ROOT=~/verl/test/log + +# test with single gpu as golden +echo "run with single gpu as golden" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=1 NUM_GPUS=1 FSDP_STRATEGY=fsdp VERL_FILE_LOGGER_PATH=~/verl/test/log/golden.jsonl bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh + +# test with fsdp 1 +echo "run with sp1 fsdp_size2 num_gpus8 fsdp_strategy fsdp pad_mode left_right" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=2 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=left_right bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp1 fsdp_size-1 num_gpus8 fsdp_strategy fsdp pad_mode left_right" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=left_right bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp2 fsdp_size-1 num_gpus8 fsdp_strategy fsdp pad_mode left_right" +BACKEND=fsdp SP_SIZE=2 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=left_right bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp4 fsdp_size4 num_gpus8 fsdp_strategy fsdp pad_mode left_right" +BACKEND=fsdp SP_SIZE=4 FSDP_SIZE=4 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=left_right bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh + +echo "run with sp1 fsdp_size2 num_gpus8 fsdp_strategy fsdp pad_mode no_padding" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=2 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp1 fsdp_size-1 num_gpus8 fsdp_strategy fsdp pad_mode no_padding" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp2 fsdp_size-1 num_gpus8 fsdp_strategy fsdp pad_mode no_padding" +BACKEND=fsdp SP_SIZE=2 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp4 fsdp_size4 num_gpus8 fsdp_strategy fsdp pad_mode no_padding" +BACKEND=fsdp SP_SIZE=4 FSDP_SIZE=4 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh + +# test use_remove_padding and pad_mode left_right/no_padding +echo "run with sp4 fsdp_size4 num_gpus8 fsdp_strategy fsdp pad_mode left_right use_remove_padding False" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=left_right USE_REMOVE_PADDING=False bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp4 fsdp_size4 num_gpus8 fsdp_strategy fsdp pad_mode no_padding use_remove_padding False" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding USE_REMOVE_PADDING=False bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh + + +# test with fsdp 2 +echo "run with sp1 fsdp_size1 num_gpus1 fsdp_strategy fsdp2 pad_mode left_right" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=1 NUM_GPUS=1 FSDP_STRATEGY=fsdp2 PAD_MODE=left_right bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp1 fsdp_size1 num_gpus1 fsdp_strategy fsdp2 pad_mode no_padding" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=1 NUM_GPUS=1 FSDP_STRATEGY=fsdp2 PAD_MODE=no_padding bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh + +echo "run with sp1 fsdp_size-1 num_gpus8 fsdp_strategy fsdp2" +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp2 bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with sp2 fsdp_size-1 num_gpus8 fsdp_strategy fsdp2" +BACKEND=fsdp SP_SIZE=2 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp2 bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=2 NUM_GPUS=8 FSDP_STRATEGY=fsdp2 bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +BACKEND=fsdp SP_SIZE=4 FSDP_SIZE=4 NUM_GPUS=8 FSDP_STRATEGY=fsdp2 bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh + +# test with megatron +echo "run with tp1 pp1 cp1 num_gpus1" +BACKEND=megatron TP_SIZE=1 PP_SIZE=1 CP_SIZE=1 NUM_GPUS=1 bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh +echo "run with tp2 pp2 vpp2 cp1 num_gpus8" +BACKEND=megatron TP_SIZE=2 PP_SIZE=2 VPP_SIZE=2 CP_SIZE=1 NUM_GPUS=8 bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh + +# TODO: toggle with following test when cp is fixed +# BACKEND=megatron TP_SIZE=2 PP_SIZE=2 VPP_SIZE=2 CP_SIZE=1 NUM_GPUS=8 bash tests/special_e2e/sft/run_sft_engine_gsm8k.sh >& ~/verl/test/log/gsm8k-tp2_pp2_vpp2_cp1_num_gpus8.log + +python3 tests/special_e2e/sft/compare_sft_engine_results.py + +rm -rf ~/verl/test/log diff --git a/e2c/verl/tests/special_e2e/sft/test_sp_loss_match.py b/e2c/verl/tests/special_e2e/sft/test_sp_loss_match.py new file mode 100644 index 0000000000000000000000000000000000000000..4dc0cbdae5acf33b4c769d35ca80b2dbb7b8458c --- /dev/null +++ b/e2c/verl/tests/special_e2e/sft/test_sp_loss_match.py @@ -0,0 +1,146 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.distributed +from tensordict import TensorDict +from torch.distributed.device_mesh import init_device_mesh + +from verl.trainer.fsdp_sft_trainer import FSDPSFTTrainer +from verl.utils.distributed import initialize_global_process_group + + +def test_trainer_forward_consistency(trainer: FSDPSFTTrainer, total_steps: int = 4): + """Test consistency between original forward pass and SP+rmpad forward passes. + + Args: + trainer: The FSDPSFTTrainer instance to test + total_steps: Number of steps to test (default: 4) + """ + if trainer.device_mesh.get_rank() == 0: + print("\nStarting debug comparison between original and SP+rmpad forward passes...") + print(f"Sequence parallel size: {trainer.config.ulysses_sequence_parallel_size}") + print(f"Remove padding: {trainer.use_remove_padding}\n") + + steps_remaining = total_steps + + for epoch in range(1): # Just one epoch for testing + trainer.train_sampler.set_epoch(epoch=epoch) + for data in trainer.train_dataloader: + data = TensorDict(data, batch_size=trainer.config.data.train_batch_size).cuda() + trainer.fsdp_model.train() + micro_batches = data.split(trainer.config.data.micro_batch_size_per_gpu) + + for idx, micro_batch in enumerate(micro_batches): + if trainer.device_mesh.get_rank() == 0: + print(f"\nProcessing micro batch {idx + 1}/{len(micro_batches)}") + + # Compute losses using both methods + # Disable SP and rmpad + trainer.use_remove_padding = False + old_sp = trainer.config.ulysses_sequence_parallel_size + trainer.config.ulysses_sequence_parallel_size = 1 + loss_ref = trainer._compute_loss_and_backward(micro_batch.copy(), do_backward=False) + + # Do SP and rmpad + trainer.config.ulysses_sequence_parallel_size = old_sp + trainer.use_remove_padding = True + loss_sp = trainer._compute_loss_and_backward(micro_batch.copy(), do_backward=False) + + # Collect losses across all ranks + loss_ref_all = loss_ref.clone() + loss_sp_all = loss_sp.clone() + torch.distributed.all_reduce(loss_ref_all, op=torch.distributed.ReduceOp.AVG) + torch.distributed.all_reduce(loss_sp_all, op=torch.distributed.ReduceOp.AVG) + + # Calculate relative difference of averaged losses + rel_diff = torch.abs(loss_ref_all - loss_sp_all) / (torch.abs(loss_ref_all) + 1e-8) + + if trainer.device_mesh.get_rank() == 0: + print("\nComparison Results (Averaged across ranks):") + print(f"Reference Loss: {loss_ref_all.item():.6f}") + print(f"SP+rmpad Loss: {loss_sp_all.item():.6f}") + print(f"Relative Difference: {rel_diff.item():.6f}") + + assert rel_diff.item() < 1e-2, "Significant difference detected between averaged losses!" + print("Loss difference is within the acceptable range.") + + steps_remaining -= 1 + if steps_remaining == 0: + break + if steps_remaining == 0: + break + break + + if trainer.device_mesh.get_rank() == 0: + print("\nDebug comparison completed successfully.") + + +def create_trainer(config): + """Create and initialize a trainer instance with the given config. + + Args: + config: Configuration object with training parameters + + Returns: + FSDPSFTTrainer: Initialized trainer instance + """ + local_rank, rank, world_size = initialize_global_process_group() + + device_mesh = init_device_mesh(device_type="cuda", mesh_shape=(world_size,), mesh_dim_names=("fsdp",)) + + dp_size = world_size // config.ulysses_sequence_parallel_size + ulysses_device_mesh = init_device_mesh( + device_type="cuda", mesh_shape=(dp_size, config.ulysses_sequence_parallel_size), mesh_dim_names=("dp", "sp") + ) + + # build tokenizer and datasets first + from verl.trainer.fsdp_sft_trainer import create_sft_dataset + from verl.utils import hf_tokenizer + from verl.utils.fs import copy_to_local + + local_model_path = copy_to_local(src=config.model.partial_pretrain, verbose=True) + tokenizer = hf_tokenizer(local_model_path, trust_remote_code=config.model.trust_remote_code) + train_dataset = create_sft_dataset(config.data.train_files, config.data, tokenizer) + val_dataset = create_sft_dataset(config.data.val_files, config.data, tokenizer) + + return FSDPSFTTrainer( + config=config, + device_mesh=device_mesh, + ulysses_device_mesh=ulysses_device_mesh, + tokenizer=tokenizer, + train_dataset=train_dataset, + val_dataset=val_dataset, + ) + + +def main(config): + """Main function to run trainer tests. + + Args: + config: Configuration object with training parameters + """ + trainer = create_trainer(config) + test_trainer_forward_consistency(trainer) + + +if __name__ == "__main__": + import hydra + from omegaconf import DictConfig + + @hydra.main(config_path="../../../verl/trainer/config", config_name="sft_trainer") + def hydra_entry(cfg: DictConfig) -> None: + main(cfg) + + hydra_entry() diff --git a/e2c/verl/tests/special_npu/run_qwen2_5_05b_dapo.sh b/e2c/verl/tests/special_npu/run_qwen2_5_05b_dapo.sh new file mode 100644 index 0000000000000000000000000000000000000000..cd3524e13359bc7f817af15614521e484643ea35 --- /dev/null +++ b/e2c/verl/tests/special_npu/run_qwen2_5_05b_dapo.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +NUM_GPUS=${NUM_GPUS:-16} + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} + +adv_estimator=grpo + +kl_coef=0.0 +use_kl_in_reward=False +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=1024 +max_response_length=2048 +enable_overlong_buffer=True +overlong_buffer_len=128 +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=True +filter_groups_metric=seq_reward +max_num_gen_batches=10 + +train_traj_micro_bsz_per_gpu=2 # b +n_resp_per_prompt=4 # g + +train_traj_micro_bsz=$((train_traj_micro_bsz_per_gpu * NUM_GPUS)) # b * n +train_traj_mini_bsz=$((train_traj_micro_bsz * 2)) # 2 * b * n +train_prompt_mini_bsz=$((train_traj_mini_bsz * n_resp_per_prompt)) # 2 * b * n / g +train_prompt_bsz=$((train_prompt_mini_bsz * 2)) # 4 * b * n / g + +gen_prompt_bsz=$((train_prompt_bsz * 4)) + +exp_name="$(basename "${MODEL_ID,,}")-dapo-minimal" + +python3 -m recipe.dapo.main_dapo \ + data.train_files="${HOME}/data/gsm8k/train.parquet" \ + data.val_files="${HOME}/data/gsm8k/test.parquet" \ + reward_model.reward_manager=dapo \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ + reward_model.overlong_buffer.len=${overlong_buffer_len} \ + reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + data.train_batch_size=${train_prompt_bsz} \ + data.gen_batch_size=${gen_prompt_bsz} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${train_traj_micro_bsz_per_gpu} \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.ref.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.actor.entropy_checkpointing=True \ + actor_rollout_ref.ref.entropy_checkpointing=True \ + actor_rollout_ref.actor.entropy_from_logits_with_chunking=True \ + actor_rollout_ref.ref.entropy_from_logits_with_chunking=True \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + trainer.logger=console \ + trainer.project_name='verl-test' \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=${NUM_GPUS} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=1 \ + trainer.resume_mode=disable \ + trainer.val_before_train=False \ + trainer.total_training_steps=2 \ + trainer.device=npu $@ diff --git a/e2c/verl/tests/special_npu/run_qwen2_5_05b_grpo.sh b/e2c/verl/tests/special_npu/run_qwen2_5_05b_grpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..25c265eb031cfd11eb29ceb14199a0b01b934038 --- /dev/null +++ b/e2c/verl/tests/special_npu/run_qwen2_5_05b_grpo.sh @@ -0,0 +1,47 @@ +set -x + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=512 \ + data.max_response_length=128 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=5e-7 \ + actor_rollout_ref.model.use_remove_padding=False \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=1 \ + trainer.total_training_steps=2 \ + trainer.device=npu $@ diff --git a/e2c/verl/tests/special_npu/run_qwen2_5_05b_grpo_mindspeed.sh b/e2c/verl/tests/special_npu/run_qwen2_5_05b_grpo_mindspeed.sh new file mode 100644 index 0000000000000000000000000000000000000000..972a3d658fad19166cccc249a0a75ffc092080ce --- /dev/null +++ b/e2c/verl/tests/special_npu/run_qwen2_5_05b_grpo_mindspeed.sh @@ -0,0 +1,68 @@ +set -x + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} + +USE_DIST_CKPT=${USE_DIST_CKPT:-False} +DIST_CKPT_PATH=${DIST_CKPT_PATH:-${HOME}/dist_ckpt/qwen2_5_05b_grpo_mindspeed} +if [ "$USE_DIST_CKPT" = "True" ]; then + if [ "$USE_DUMMY_MODEL" = "True" ]; then + DIST_CKPT_PATH=${HOME}/dist_ckpt_dummy/${MODEL_ID} + fi + python scripts/converter_hf_to_mcore.py \ + --hf_model_path "${MODEL_PATH}" \ + --output_path "${DIST_CKPT_PATH}" +fi + + +python3 -m verl.trainer.main_ppo --config-path=config \ + --config-name='ppo_megatron_trainer.yaml' \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=512 \ + data.max_response_length=128 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=${MODEL_ID} \ + actor_rollout_ref.actor.optim.lr=5e-7 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=20 \ + actor_rollout_ref.actor.strategy=megatron \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=1 \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.actor.megatron.dist_checkpointing_path=${DIST_CKPT_PATH} \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ + actor_rollout_ref.ref.strategy=megatron \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=1 \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \ + actor_rollout_ref.ref.megatron.dist_checkpointing_path=${DIST_CKPT_PATH} \ + actor_rollout_ref.ref.use_torch_compile=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_gsm8k' \ + trainer.experiment_name='qwen2_7b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=1 \ + trainer.total_training_steps=2 \ + trainer.device=npu \ + +actor_rollout_ref.actor.megatron.override_transformer_config.use_flash_attn=True $@ diff --git a/e2c/verl/tests/special_npu/run_qwen2_5_05b_sft_peft_sp2.sh b/e2c/verl/tests/special_npu/run_qwen2_5_05b_sft_peft_sp2.sh new file mode 100644 index 0000000000000000000000000000000000000000..19c3ac033b6a0ed8fe395fa72c50dcc3a4353827 --- /dev/null +++ b/e2c/verl/tests/special_npu/run_qwen2_5_05b_sft_peft_sp2.sh @@ -0,0 +1,33 @@ +set -x + +mkdir -p ./save_ckpts + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} + +torchrun --standalone --nnodes=1 --nproc_per_node=8 \ + -m verl.trainer.fsdp_sft_trainer \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.prompt_key=extra_info \ + data.response_key=extra_info \ + optim.lr=1e-4 \ + data.prompt_dict_keys=['question'] \ + +data.response_dict_keys=['answer'] \ + data.micro_batch_size_per_gpu=32 \ + model.partial_pretrain="${MODEL_PATH}" \ + trainer.default_local_dir=./save_ckpts \ + trainer.project_name=gsm8k-sft \ + trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct \ + trainer.logger=console \ + trainer.total_epochs=1 \ + trainer.total_training_steps=2 \ + model.lora_rank=32 \ + model.lora_alpha=16 \ + model.target_modules=all-linear \ + model.strategy=fsdp \ + ulysses_sequence_parallel_size=2 \ + use_remove_padding=true \ + trainer.device=npu + +rm -rf ./outputs ./save_ckpts diff --git a/e2c/verl/tests/special_npu/run_qwen2_5_vl_3b_npu.sh b/e2c/verl/tests/special_npu/run_qwen2_5_vl_3b_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..c3235b2c5d6d2d83dd541b35cc98c58359b45e9d --- /dev/null +++ b/e2c/verl/tests/special_npu/run_qwen2_5_vl_3b_npu.sh @@ -0,0 +1,56 @@ +set -x +ENGINE=${1:-vllm} + +# Some models are optimized by vllm ascend. While in some case, e.g. rlhf training, +# the optimized model may not be suitable. In this case, set this value to 0 to disable the optimized model. +export USE_OPTIMIZED_MODEL=0 + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-VL-3B-Instruct} +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$HOME/data/geo3k/train.parquet \ + data.val_files=$HOME/data/geo3k/test.parquet \ + data.train_batch_size=512 \ + data.max_prompt_length=1024 \ + data.max_response_length=2048 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.image_key=images \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.01 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.ref.use_torch_compile=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.name=$ENGINE \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enable_chunked_prefill=False \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.n=5 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_grpo_example_geo3k' \ + trainer.experiment_name='qwen2_5_vl_3b_function_rm' \ + trainer.n_gpus_per_node=16 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=1 \ + trainer.total_training_steps=2 \ + trainer.device=npu $@ \ No newline at end of file diff --git a/e2c/verl/tests/special_npu/run_qwen3_06b_ppo.sh b/e2c/verl/tests/special_npu/run_qwen3_06b_ppo.sh new file mode 100644 index 0000000000000000000000000000000000000000..f9dca50c8223065eafacc0ef7aef644801e8a180 --- /dev/null +++ b/e2c/verl/tests/special_npu/run_qwen3_06b_ppo.sh @@ -0,0 +1,55 @@ +set -x + +# TODO (FightingZhen) Env VLLM_USE_V1=1 is not supported in vllm==0.7.3 +# export VLLM_USE_V1=1 + +MODEL_ID=${MODEL_ID:-Qwen/Qwen2.5-0.5B-Instruct} # TODO: change to Qwen3-0.6B when CI server is ready +MODEL_PATH=${MODEL_PATH:-${HOME}/models/${MODEL_ID}} + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=512 \ + data.max_response_length=128 \ + data.shuffle=False \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.enforce_eager=False \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path="${MODEL_PATH}" \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=8 \ + critic.ulysses_sequence_parallel_size=2 \ + critic.model.fsdp_config.param_offload=True \ + critic.model.fsdp_config.optimizer_offload=True \ + critic.use_dynamic_bsz=True \ + trainer.critic_warmup=0 \ + trainer.logger='["console"]' \ + trainer.project_name='verl_ppo_example_gsm8k_qwen3' \ + trainer.experiment_name='qwen3_06b_fsdp' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=1 \ + trainer.total_training_steps=2 \ + trainer.device=npu $@ diff --git a/e2c/verl/tests/special_sanity/check_api_docs.py b/e2c/verl/tests/special_sanity/check_api_docs.py new file mode 100644 index 0000000000000000000000000000000000000000..afc598c831a71c9d10e8e78db2ee9cceab92d9e1 --- /dev/null +++ b/e2c/verl/tests/special_sanity/check_api_docs.py @@ -0,0 +1,142 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Fail CI if any function or class that is publicly exported via +``__all__`` lacks a docstring. + +Usage +----- + # Check specific modules or packages + python check_docstrings.py mypkg.core mypkg.utils + + # Check an entire source tree (all top-level packages under cwd) + python check_docstrings.py +""" + +from __future__ import annotations + +import argparse +import importlib +import inspect +import pkgutil +import sys +from pathlib import Path +from types import ModuleType +from typing import Iterable + +_ALLOW_LIST = [ + "verl.third_party.vllm.LLM", + "verl.third_party.vllm.parallel_state", + "verl.utils.profiler.WorkerProfiler", + "verl.utils.profiler.WorkerProfilerExtension", + "verl.utils.profiler.log_gpu_memory_usage", + "verl.utils.profiler.log_print", + "verl.utils.profiler.mark_annotate", + "verl.utils.profiler.mark_end_range", + "verl.utils.profiler.mark_start_range", + "verl.models.mcore.qwen2_5_vl.get_vision_model_config", + "verl.models.mcore.qwen2_5_vl.get_vision_projection_config", + "verl.models.mcore.mbridge.freeze_moe_router", + "verl.models.mcore.mbridge.make_value_model", + "verl.utils.transformers_compat.flash_attn_supports_top_left_mask", +] + + +def iter_submodules(root: ModuleType) -> Iterable[ModuleType]: + """Yield *root* and every sub-module inside it.""" + yield root + + def print_pkg_error(pkg_name): + print(f"[warn] Skipping {pkg_name!r}", file=sys.stderr) + + if getattr(root, "__path__", None): # only packages have __path__ + for mod_info in pkgutil.walk_packages(root.__path__, prefix=f"{root.__name__}.", onerror=print_pkg_error): + try: + yield importlib.import_module(mod_info.name) + except Exception as exc: # noqa: BLE001 + print(f"[warn] Skipping {mod_info.name!r}: {exc}", file=sys.stderr) + + +def names_missing_doc(mod: ModuleType) -> list[str]: + """Return fully-qualified names that need docstrings.""" + missing: list[str] = [] + public = getattr(mod, "__all__", []) + for name in public: + obj = getattr(mod, name, None) + if f"{mod.__name__}.{name}" in _ALLOW_LIST: + continue + if obj is None: + # Exported but not found in the module: flag it anyway. + missing.append(f"{mod.__name__}.{name} (not found)") + continue + + if inspect.isfunction(obj) or inspect.isclass(obj): + doc = inspect.getdoc(obj) + if not doc or not doc.strip(): + missing.append(f"{mod.__name__}.{name}") + return missing + + +def check_module(qualname: str) -> list[str]: + """Import *qualname* and check it (and sub-modules).""" + try: + module = importlib.import_module(qualname) + except ModuleNotFoundError as exc: + print(f"[error] Cannot import '{qualname}': {exc}", file=sys.stderr) + return [qualname] + + missing: list[str] = [] + for submod in iter_submodules(module): + missing.extend(names_missing_doc(submod)) + return missing + + +def autodiscover_packages() -> list[str]: + """Detect top-level packages under CWD when no argument is given.""" + pkgs: list[str] = [] + for p in Path.cwd().iterdir(): + if p.is_dir() and (p / "__init__.py").exists(): + pkgs.append(p.name) + return pkgs + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "modules", + nargs="*", + help="Fully-qualified module or package names (defaults to every top-level package found in CWD).", + ) + args = parser.parse_args() + + targets = args.modules or autodiscover_packages() + if not targets: + raise ValueError("[error] No modules specified and none detected automatically.") + + all_missing: list[str] = [] + for modname in targets: + all_missing.extend(check_module(modname)) + + if all_missing: + print("\nMissing docstrings:") + for name in sorted(all_missing): + print(f" - {name}") + raise ValueError("Missing docstrings detected. Please enhance them with docs accordingly.") + + print("✅ All exported functions/classes have docstrings.") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/tests/special_sanity/check_dataproto_usage.py b/e2c/verl/tests/special_sanity/check_dataproto_usage.py new file mode 100644 index 0000000000000000000000000000000000000000..7c8521ab12e0fc2f39dd965d3aefbb4f303c12c9 --- /dev/null +++ b/e2c/verl/tests/special_sanity/check_dataproto_usage.py @@ -0,0 +1,69 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This CI test is used for checking whether DataProto is used in the code of some directory +""" + +import os +from argparse import ArgumentParser +from pathlib import Path + +SEARCH_WHITELIST = [] + +SEARCH_KEYWORDS = ["DataProto"] + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--directory", "-d", required=True, type=str) + args = parser.parse_args() + directory_in_str = args.directory + + pathlist = Path(directory_in_str).glob("**/*.py") + for path in pathlist: + path_in_str = str(path.absolute()) + + # judge whether current path is in pre-defined search whitelist or not. + path_in_whitelist = False + + for sw in SEARCH_WHITELIST: + # for easy debugging in non-linux system + sw = sw.replace("/", os.sep) + if sw in path_in_str: + print(f"[SKIP] File {path_in_str} is in device api usage check whitelist, checking is skipped.") + path_in_whitelist = True + break + + if path_in_whitelist: + continue + + with open(path_in_str, encoding="utf-8") as f: + file_content = f.read() + + find_invalid_device_management = False + + for sk in SEARCH_KEYWORDS: + if sk in file_content: + find_invalid_device_management = True + break + + print( + f"[CHECK] File {path_in_str} is detected for DataProto usage check, check result: " + f"{'success' if not find_invalid_device_management else f'failed, because detect {sk}'}." + ) + + assert not find_invalid_device_management, ( + f"file {path_in_str} contains DataProto usage, please use TensorDict directly!" + ) diff --git a/e2c/verl/tests/special_sanity/check_device_api_usage.py b/e2c/verl/tests/special_sanity/check_device_api_usage.py new file mode 100644 index 0000000000000000000000000000000000000000..dae5ac4b43dac2195241eff5df2ee9543ef4d2e3 --- /dev/null +++ b/e2c/verl/tests/special_sanity/check_device_api_usage.py @@ -0,0 +1,100 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This CI test is used for checking whether device api usage is irregular, suggest using api in `verl/utils/device.py`. +Search targets include .py files in verl/recipe and verl/verl. +Some files that must contain ".cuda", "cuda" or "nccl" keyword is pre-defined in whitelist below. +""" + +import os +from argparse import ArgumentParser +from pathlib import Path + +# directory or file path must contain keyword ".cuda" or "cuda" +CUDA_KEYWORD_CHECK_WHITELIST = [ + "verl/utils/device.py", + "recipe/prime/prime_ray_trainer.py", # appear in default device_name + "recipe/spin/spin_trainer.py", # appear in default device_name + "recipe/sppo/sppo_ray_trainer.py", # appear in default device_name + "recipe/one_step_off_policy/ray_trainer.py", # appear in default device_name + "verl/utils/profiler/nvtx_profile.py", # appear in NsightSystemsProfiler + "verl/utils/kernel/linear_cross_entropy.py", # appear in nvidia nvtx + "verl/utils/rendezvous/ray_backend.py", # appear in cupy importance + "verl/single_controller/ray/base.py", # appear in default device_name + "verl/trainer/ppo/ray_trainer.py", # appear in default device_name + "verl/utils/reward_score/sandbox_fusion/utils.py", # appear in sandbox language type + "verl/workers/reward_model/megatron/reward_model.py", # appear in default device_name + "verl/third_party/torch/distributed/_state_dict_utils.py", # torch monkey patch fixes + "verl/third_party/torch/distributed/checkpoint/state_dict.py", # torch monkey patch fixes + "verl/workers/engine/base.py", # appear in default device_name + "verl/workers/engine/fsdp/transformer_impl.py", # appear in default device_name + "verl/workers/rollout/vllm_rollout/vllm_async_server.py", # appear in config.cudagraph_capture_sizes + "verl/workers/rollout/sglang_rollout/async_sglang_server.py", # manually set CUDA_VISIBLE_DEVICES +] + +# directory or file path must contain keyword "nccl" +NCCL_KEYWORD_CHECK_WHITELIST = [ + "verl/utils/device.py", + "verl/third_party/sglang/parallel_state.py", # appear in default backend +] + +SEARCH_WHITELIST = CUDA_KEYWORD_CHECK_WHITELIST + NCCL_KEYWORD_CHECK_WHITELIST + +SEARCH_KEYWORDS = [".cuda", '"cuda"', '"nccl"'] + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--directory", "-d", required=True, type=str) + args = parser.parse_args() + directory_in_str = args.directory + + pathlist = Path(directory_in_str).glob("**/*.py") + for path in pathlist: + path_in_str = str(path.absolute()) + + # judge whether current path is in pre-defined search whitelist or not. + path_in_whitelist = False + + for sw in SEARCH_WHITELIST: + # for easy debugging in non-linux system + sw = sw.replace("/", os.sep) + if sw in path_in_str: + print(f"[SKIP] File {path_in_str} is in device api usage check whitelist, checking is skipped.") + path_in_whitelist = True + break + + if path_in_whitelist: + continue + + with open(path_in_str, encoding="utf-8") as f: + file_content = f.read() + + find_invalid_device_management = False + + for sk in SEARCH_KEYWORDS: + if sk in file_content: + find_invalid_device_management = True + break + + print( + f"[CHECK] File {path_in_str} is detected for device api usage check, check result: " + f"{'success' if not find_invalid_device_management else f'failed, because detect {sk}'}." + ) + + assert not find_invalid_device_management, ( + f'file {path_in_str} contains .cuda/"cuda"/"nccl" usage, please use api in ' + f"verl/utils/device.py directly." + ) diff --git a/e2c/verl/tests/special_sanity/check_docs_time_info.py b/e2c/verl/tests/special_sanity/check_docs_time_info.py new file mode 100644 index 0000000000000000000000000000000000000000..a54d1d50a7e9d21202387e2c9c8e3c6c73a5d807 --- /dev/null +++ b/e2c/verl/tests/special_sanity/check_docs_time_info.py @@ -0,0 +1,84 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Check that every .md and .rst file under docs/ contains the substring "Last updated", +with an allow-list for exceptions. +""" + +import sys +from pathlib import Path + +# === CONFIGURATION === + +# Relative paths (to docs/) or glob patterns to skip checking +ALLOW_LIST = { + "docs/README.md", # you can list individual files + "docs/legacy/*.rst", # or glob patterns + "docs/index.rst", + "docs/start/install.rst", + "docs/start/quickstart.rst", + "docs/README_vllm0.7.md", +} + +# The folder to scan +DOCS_DIR = Path("docs") + +# === SCRIPT === + + +def is_allowed(path: Path) -> bool: + """ + Return True if `path` matches any entry in ALLOW_LIST. + """ + rel = str(path) + for pattern in ALLOW_LIST: + if Path(rel).match(pattern): + return True + return False + + +def main(): + if not DOCS_DIR.exists(): + print(f"Error: Documentation directory '{DOCS_DIR}' does not exist.", file=sys.stderr) + sys.exit(1) + + missing = [] + + # Gather all .md and .rst files under docs/ + for ext in ("*.md", "*.rst"): + for path in DOCS_DIR.rglob(ext): + if is_allowed(path): + continue + + text = path.read_text(encoding="utf-8", errors="ignore") + if "Last updated" not in text: + missing.append(path) + + # Report + if missing: + print("\nThe following files are missing the 'Last updated' string:\n") + for p in missing: + print(f" - {p}") + print(f"\nTotal missing: {len(missing)}\n", file=sys.stderr) + raise AssertionError( + "Some documentation files lack a 'Last updated' line. Please include info such as " + "'Last updated: mm/dd/yyyy' to indicate the last update time of the document." + ) + else: + print("✅ All checked files contain 'Last updated'.") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/tests/special_sanity/check_docstrings.py b/e2c/verl/tests/special_sanity/check_docstrings.py new file mode 100644 index 0000000000000000000000000000000000000000..7c5d8ed714b21d0a6bea422586bd0db660b17398 --- /dev/null +++ b/e2c/verl/tests/special_sanity/check_docstrings.py @@ -0,0 +1,156 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Python script to check docstrings for functions and classes in specified files. +Checks that every public function and class has proper docstring documentation. +""" + +import ast +import os +import sys + + +class DocstringChecker(ast.NodeVisitor): + """AST visitor to check for missing docstrings in functions and classes.""" + + def __init__(self, filename: str): + self.filename = filename + self.missing_docstrings: list[tuple[str, str, int]] = [] + self.current_class = None + self.function_nesting_level = 0 + + def visit_FunctionDef(self, node: ast.FunctionDef): + """Visit function definitions and check for docstrings.""" + if not node.name.startswith("_") and self.function_nesting_level == 0: + if not self._has_docstring(node): + func_name = f"{self.current_class}.{node.name}" if self.current_class else node.name + self.missing_docstrings.append((func_name, self.filename, node.lineno)) + + self.function_nesting_level += 1 + self.generic_visit(node) + self.function_nesting_level -= 1 + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef): + """Visit async function definitions and check for docstrings.""" + if not node.name.startswith("_") and self.function_nesting_level == 0: + if not self._has_docstring(node): + func_name = f"{self.current_class}.{node.name}" if self.current_class else node.name + self.missing_docstrings.append((func_name, self.filename, node.lineno)) + + self.function_nesting_level += 1 + self.generic_visit(node) + self.function_nesting_level -= 1 + + def visit_ClassDef(self, node: ast.ClassDef): + """Visit class definitions and check for docstrings.""" + if not node.name.startswith("_"): + if not self._has_docstring(node): + self.missing_docstrings.append((node.name, self.filename, node.lineno)) + + old_class = self.current_class + self.current_class = node.name + self.generic_visit(node) + self.current_class = old_class + + def _has_docstring(self, node) -> bool: + """Check if a node has a docstring.""" + return ast.get_docstring(node) is not None + + +def check_file_docstrings(filepath: str) -> list[tuple[str, str, int]]: + """Check docstrings in a single file.""" + try: + with open(filepath, encoding="utf-8") as f: + content = f.read() + + tree = ast.parse(content, filename=filepath) + checker = DocstringChecker(filepath) + checker.visit(tree) + return checker.missing_docstrings + + except Exception as e: + print(f"Error processing {filepath}: {e}") + return [] + + +def main(): + """Main function to check docstrings in specified files.""" + + files_to_check = [ + "verl/trainer/ppo/ray_trainer.py", + "verl/trainer/main_ppo.py", + "verl/trainer/ppo/reward.py", + "verl/utils/reward_score/__init__.py", + "verl/trainer/ppo/core_algos.py", + "verl/experimental/agent_loop/agent_loop.py", + "verl/workers/sharding_manager/fsdp_vllm.py", + "verl/workers/sharding_manager/fsdp_ulysses.py", + ] + + script_dir = os.path.dirname(os.path.abspath(__file__)) + repo_path = os.path.dirname(os.path.dirname(script_dir)) + + if not os.path.exists(repo_path): + print(f"Repository path {repo_path} does not exist!") + sys.exit(1) + + os.chdir(repo_path) + + all_missing_docstrings = [] + + print("Checking docstrings in specified files...") + print("=" * 60) + + for file_path in files_to_check: + if not os.path.exists(file_path): + print(f"Warning: File {file_path} does not exist!") + continue + + print(f"Checking {file_path}...") + missing = check_file_docstrings(file_path) + all_missing_docstrings.extend(missing) + + if missing: + print(f" Found {len(missing)} missing docstrings") + else: + print(" All functions and classes have docstrings ✓") + + print("=" * 60) + + if all_missing_docstrings: + print(f"\nSUMMARY: Found {len(all_missing_docstrings)} functions/classes missing docstrings:") + print("-" * 60) + + by_file = {} + for name, filepath, lineno in all_missing_docstrings: + if filepath not in by_file: + by_file[filepath] = [] + by_file[filepath].append((name, lineno)) + + for filepath in sorted(by_file.keys()): + print(f"\n{filepath}:") + for name, lineno in sorted(by_file[filepath], key=lambda x: x[1]): + print(f" - {name} (line {lineno})") + + print(f"\nTotal missing docstrings: {len(all_missing_docstrings)}") + + raise Exception(f"Found {len(all_missing_docstrings)} functions/classes without proper docstrings!") + + else: + print("\n✅ All functions and classes have proper docstrings!") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/tests/special_sanity/check_license.py b/e2c/verl/tests/special_sanity/check_license.py new file mode 100644 index 0000000000000000000000000000000000000000..a4ade02443338eb2808e88d8e77bc2457d4f09ff --- /dev/null +++ b/e2c/verl/tests/special_sanity/check_license.py @@ -0,0 +1,82 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from argparse import ArgumentParser +from pathlib import Path +from typing import Iterable + +license_head_bytedance = "Copyright 2024 Bytedance Ltd. and/or its affiliates" +license_head_bytedance_25 = "Copyright 2025 Bytedance Ltd. and/or its affiliates" +# Add custom license headers below +license_head_prime = "Copyright 2024 PRIME team and/or its affiliates" +license_head_individual = "Copyright 2025 Individual Contributor:" +license_head_sglang = "Copyright 2023-2024 SGLang Team" +license_head_modelbest = "Copyright 2025 ModelBest Inc. and/or its affiliates" +license_head_amazon = "Copyright 2025 Amazon.com Inc and/or its affiliates" +license_head_facebook = "Copyright (c) 2016- Facebook, Inc" +license_headers = [ + license_head_bytedance, + license_head_bytedance_25, + license_head_prime, + license_head_individual, + license_head_sglang, + license_head_modelbest, + license_head_amazon, + license_head_facebook, +] + + +def get_py_files(path_arg: Path) -> Iterable[Path]: + """get py files under a dir. if already py file return it + + Args: + path_arg (Path): path to scan for py files + + Returns: + Iterable[Path]: list of py files + """ + if path_arg.is_dir(): + return path_arg.glob("**/*.py") + elif path_arg.is_file() and path_arg.suffix == ".py": + return [path_arg] + return [] + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument( + "--directories", + "-d", + required=True, + type=Path, + nargs="+", + help="List of directories to check for license headers", + ) + args = parser.parse_args() + + # Collect all Python files from specified directories + pathlist = set(path for path_arg in args.directories for path in get_py_files(path_arg)) + + for path in pathlist: + # because path is object not string + path_in_str = str(path.absolute()) + print(path_in_str) + with open(path_in_str, encoding="utf-8") as f: + file_content = f.read() + + has_license = False + for lh in license_headers: + if lh in file_content: + has_license = True + break + assert has_license, f"file {path_in_str} does not contain license" diff --git a/e2c/verl/tests/special_sanity/check_pr_description.py b/e2c/verl/tests/special_sanity/check_pr_description.py new file mode 100644 index 0000000000000000000000000000000000000000..4ed4563db6088e8562273cebd08116e375bc8bb2 --- /dev/null +++ b/e2c/verl/tests/special_sanity/check_pr_description.py @@ -0,0 +1,97 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 +import json +import os + +# Number of lines to check +NUM_LINES = 5 + + +# Custom exception types for clear error handling +class TemplateFileError(Exception): + pass + + +class PRBodyLoadError(Exception): + pass + + +class PRDescriptionError(Exception): + pass + + +# Path to the PR template file +template_file = os.path.join(os.getenv("GITHUB_WORKSPACE", "."), ".github", "PULL_REQUEST_TEMPLATE.md") + + +def load_template(path): + """ + Load only the first NUM_LINES of the PR template file as a list of lines, + without stripping any characters. + """ + lines = [] + try: + with open(path, encoding="utf-8") as f: + for _ in range(NUM_LINES): + line = f.readline() + if not line: + break + lines.append(line.strip()) + return lines + except Exception as e: + raise TemplateFileError(f"Failed to read PR template (first {NUM_LINES} lines) at {path}: {e}") from e + + +def load_pr_body(event_path): + try: + with open(event_path, encoding="utf-8") as f: + payload = json.load(f) + return payload.get("pull_request", {}).get("body", "") or "" + except Exception as e: + raise PRBodyLoadError(f"Failed to read PR body from {event_path}: {e}") from e + + +def check_pr_description(body, template_lines): + """ + Compare the first NUM_LINES lines of the PR body to the template lines. + If they match exactly, the placeholder was not modified. + """ + pr_lines = body.splitlines(keepends=True) + pr_first = [x.strip() for x in pr_lines[:NUM_LINES]] + if pr_first == template_lines: + raise PRDescriptionError( + "It looks like you haven't updated the '### What does this PR do?' section. Please replace " + "the placeholder text with a concise description of what your PR does." + ) + else: + print(pr_first) + print(template_lines) + + +def main(): + event_path = os.getenv("GITHUB_EVENT_PATH") + if not event_path: + raise OSError("GITHUB_EVENT_PATH is not set.") + + template_lines = load_template(template_file) + pr_body = load_pr_body(event_path) + check_pr_description(pr_body, template_lines) + + print("✅ '### What does this PR do?' section has been filled out.") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/tests/special_sanity/check_pr_title.py b/e2c/verl/tests/special_sanity/check_pr_title.py new file mode 100644 index 0000000000000000000000000000000000000000..cabc2f50d85fc795db468c4ffd47172a41894a5a --- /dev/null +++ b/e2c/verl/tests/special_sanity/check_pr_title.py @@ -0,0 +1,72 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +# Get PR title from environment +pr_title = os.environ.get("PR_TITLE", "").strip() + +# Define rules +allowed_modules = ["fsdp", "megatron", "sglang", "vllm", "rollout", "trainer"] +allowed_modules += ["tests", "training_utils", "recipe", "hardware", "deployment"] +allowed_modules += ["ray", "worker", "single_controller", "misc", "docker", "ci"] +allowed_modules += ["perf", "model", "algo", "env", "tool", "ckpt", "doc", "data", "cfg"] +allowed_types = ["feat", "fix", "refactor", "chore", "test"] + +# Check for [1/N] prefix and extract the rest of the title +progress_match = re.match(r"^\[\d/[\dNn]\]\s*(.+)$", pr_title, re.IGNORECASE) +if progress_match: + pr_title = progress_match.group(1).strip() + +# Check for [BREAKING] prefix and extract the rest of the title +breaking_match = re.match(r"^\[BREAKING\]\s*(.+)$", pr_title, re.IGNORECASE) +if breaking_match: + core_pr_title = breaking_match.group(1).strip() + is_breaking = True +else: + core_pr_title = pr_title + is_breaking = False + +# Build dynamic regex pattern for modules (now working on core_pr_title) +re_modules_pattern = re.compile(r"^\[([a-z_,\s]+)\]", re.IGNORECASE) +re_modules = re_modules_pattern.match(core_pr_title) +if not re_modules: + print(f"❌ Invalid PR title: '{pr_title}'") + print("Expected format: [BREAKING][module] type: description") + print(f"Allowed modules: {', '.join(allowed_modules)}") + raise Exception("Invalid PR title") +else: + modules = re.findall(r"[a-z_]+", re_modules.group(1).lower()) + if not all(module in allowed_modules for module in modules): + invalid_modules = [module for module in modules if module not in allowed_modules] + print(f"❌ Invalid modules: {', '.join(invalid_modules)}") + print(f"Allowed modules: {', '.join(allowed_modules)}") + raise Exception("Invalid PR title") + +types_pattern = "|".join(re.escape(t) for t in allowed_types) +re_types_pattern = re.compile(rf"^\[[a-z_,\s]+\]\s+({types_pattern}):\s+.+$", re.IGNORECASE) +match = re_types_pattern.match(core_pr_title) + +if not match: + print(f"❌ Invalid PR title: '{pr_title}'") + print("Expected format: [BREAKING][module] type: description") + print(f"Allowed types: {', '.join(allowed_types)}") + raise Exception("Invalid PR title") + +change_type = match.group(1).lower() + +# Build the success message +breaking_info = " (BREAKING CHANGE)" if is_breaking else "" +print(f"✅ PR title is valid: {pr_title}, modules: {modules}, type: {change_type}{breaking_info}") diff --git a/e2c/verl/tests/special_sanity/test_config_docs.py b/e2c/verl/tests/special_sanity/test_config_docs.py new file mode 100644 index 0000000000000000000000000000000000000000..b8dc74762450fe41a42db6ca09972851e8dcbdc2 --- /dev/null +++ b/e2c/verl/tests/special_sanity/test_config_docs.py @@ -0,0 +1,88 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + + +def validate_yaml_format(yaml_lines): + errors = [] + i = 0 + + while i < len(yaml_lines): + line = yaml_lines[i] + stripped = line.strip() + + # Skip empty lines + if stripped == "": + i += 1 + continue + + # Match YAML keys like "field:" or "field: value" + key_match = re.match(r"^(\s*)([a-zA-Z0-9_]+):", line) + if key_match: + # Check if there's a comment above + if i == 0 or not yaml_lines[i - 1].strip().startswith("#"): + errors.append(f"Missing comment above line {i + 1}: {line.strip()}") + + # Check for inline comment + if "#" in line and not stripped.startswith("#"): + comment_index = line.index("#") + colon_index = line.index(":") + if comment_index > colon_index: + errors.append(f"Inline comment found on line {i + 1}: {line.strip()}") + + # Check for blank line after this key line (unless next is a deeper indent) + if i + 1 < len(yaml_lines): + next_line = yaml_lines[i + 1] + next_stripped = next_line.strip() + + # If next is not empty and not a deeper nested line, enforce blank line + if next_stripped != "": + errors.append(f"Missing blank line after line {i + 1}: {line.strip()}") + + i += 1 + + return errors + + +def test_trainer_config_doc(): + yamls_to_inspect = [ + "verl/trainer/config/ppo_trainer.yaml", + "verl/trainer/config/actor/actor.yaml", + "verl/trainer/config/actor/dp_actor.yaml", + "verl/trainer/config/critic/critic.yaml", + "verl/trainer/config/critic/dp_critic.yaml", + "verl/trainer/config/ref/ref.yaml", + "verl/trainer/config/ref/dp_ref.yaml", + "verl/trainer/config/rollout/rollout.yaml", + ] + success = True + for yaml_to_inspect in yamls_to_inspect: + yaml_path = Path(yaml_to_inspect) # path to your YAML file + with open(yaml_path) as f: + lines = f.readlines() + + validation_errors = validate_yaml_format(lines) + if validation_errors: + success = False + print("YAML documentation format check failed:") + print(f"Please read the top block of {yaml_to_inspect} to see format rules:\n") + for err in validation_errors: + print(" -", err) + + if not success: + raise Exception("Please fix documentation format.") + else: + print("YAML format check passed ✅") diff --git a/e2c/verl/tests/special_sanity/test_import.py b/e2c/verl/tests/special_sanity/test_import.py new file mode 100644 index 0000000000000000000000000000000000000000..4f8a918fe65679c353e8055c2c2b0a428fdf8f7a --- /dev/null +++ b/e2c/verl/tests/special_sanity/test_import.py @@ -0,0 +1,25 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_import(): + import verl + + print(verl.__version__) + + +def test_single_controller_import(): + import verl.single_controller + + print(verl.single_controller.__version__) diff --git a/e2c/verl/tests/special_sanity/type_coverage_check.py b/e2c/verl/tests/special_sanity/type_coverage_check.py new file mode 100644 index 0000000000000000000000000000000000000000..dc6dc7caf483bf192da47e28d8fe6bec26444965 --- /dev/null +++ b/e2c/verl/tests/special_sanity/type_coverage_check.py @@ -0,0 +1,180 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Custom type annotation check tool. +To inspect the type annotation for functions in the entire codebase, please run: +find verl -type f -name "*.py" | xargs -n 1 python3 tests/special_sanity/type_coverage_check.py --all-lines +--debug --target-file +""" + +import argparse +import ast +import linecache +import subprocess +from pathlib import Path + + +def get_changed_files() -> list[Path]: + result = subprocess.run( + ["git", "diff", "--name-only", "--diff-filter=AM", "origin/main...HEAD"], stdout=subprocess.PIPE, text=True + ) + return [Path(f) for f in result.stdout.splitlines() if f.endswith(".py")] + + +def get_changed_lines(file_path: Path) -> set[int]: + result = subprocess.run( + ["git", "diff", "-U0", "origin/main...HEAD", "--", str(file_path)], + stdout=subprocess.PIPE, + text=True, + ) + lines: set[int] = set() + for line in result.stdout.splitlines(): + if line.startswith("@@"): + for part in line.split(): + try: + if part.startswith("+") and "," in part: + start, count = map(int, part[1:].split(",")) + lines.update(range(start, start + count)) + elif part.startswith("+") and "," not in part: + lines.add(int(part[1:])) + except Exception: + # (vermouth1992) There are many edge cases here because + can be in the changed program + pass + return lines + + +CHECK_SUCCESS = 0 +CHECK_WARNING = 1 +CHECK_FAILURE = -1 + + +def should_check_type(arg_name: str) -> bool: + if arg_name in ("self", "cls"): + return False + if arg_name.startswith("*"): + return False + return True + + +def has_type_annotations(node: ast.AST, debug: bool = False) -> int: + if isinstance(node, ast.FunctionDef): + is_private = node.name.startswith("_") + has_ann = ( + all(arg.annotation is not None for arg in node.args.args if should_check_type(arg.arg)) + and node.returns is not None + ) + if has_ann or is_private: + return CHECK_SUCCESS + else: + if debug: + print(node, [(arg.annotation, arg.arg) for arg in node.args.args if should_check_type(arg.arg)]) + return CHECK_FAILURE + return CHECK_SUCCESS + + +def check_file( + file_path: Path, changed_lines: set[int], debug: bool = False +) -> tuple[int, int, list[tuple[Path, int, str]], list[tuple[Path, int, str]]]: + with open(file_path) as f: + source: str = f.read() + tree = ast.parse(source, filename=str(file_path)) + annotated = 0 + total = 0 + warning_lines: list[tuple[Path, int, str]] = [] + failure_lines: list[tuple[Path, int, str]] = [] + + for node in ast.walk(tree): + if hasattr(node, "lineno") and node.lineno in changed_lines: + if isinstance(node, ast.FunctionDef | ast.Assign | ast.AnnAssign): + total += 1 + result = has_type_annotations(node, debug) + if result == CHECK_SUCCESS or result == CHECK_WARNING: + annotated += 1 + if result == CHECK_WARNING: + warning_lines.append( + (file_path, node.lineno, linecache.getline(str(file_path), node.lineno).strip()) + ) + else: + source_line = linecache.getline(str(file_path), node.lineno).strip() + failure_lines.append((file_path, node.lineno, source_line)) + + return annotated, total, warning_lines, failure_lines + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--threshold", type=float, default=0.3, help="Minimum ratio of annotated lines required (0.0 - 1.0)" + ) + parser.add_argument("--target-file", type=str, default=None, help="Path to the Python source file to analyse") + parser.add_argument( + "--all-lines", + action="store_true", + help="Check all lines in the file instead of only changed lines based on git", + ) + parser.add_argument("--debug", action="store_true", help="Add debugging logs") + args = parser.parse_args() + + total_changed = 0 + total_annotated = 0 + all_warnings: list[tuple[Path, int, str]] = [] + all_failures: list[tuple[Path, int, str]] = [] + + target_files = [args.target_file] if args.target_file is not None else get_changed_files() + for fpath in target_files: + if "tests/" in str(fpath): + continue + if args.all_lines: + changed_lines = [i + 1 for i in range(len(open(fpath).readlines()))] + else: + changed_lines = get_changed_lines(fpath) + annotated, total, warning_lines, failure_lines = check_file(fpath, changed_lines, args.debug) + total_annotated += annotated + total_changed += total + all_warnings.extend(warning_lines) + all_failures.extend(failure_lines) + + ratio = (total_annotated / total_changed) if total_changed else 1.0 + + print( + f"🔍 Type coverage on {'all' if args.all_lines else 'changed'} lines: " + f"{total_annotated}/{total_changed} = {ratio:.2%}. Files inspected: {target_files}" + ) + + if all_warnings: + print("\n⚠️ Suggest Improve: Lines missing type annotations for inputs and outputs:\n") + for fname, lineno, line in all_warnings: + print(f"{fname}:{lineno}: {line}") + + if all_failures: + print("⚠️ [ERROR] Lines missing type annotations for inputs and outputs:\n") + for fname, lineno, line in all_failures: + print(f"{fname}:{lineno}: {line}") + + if ratio < args.threshold: + print( + f"Please add type annotations for inputs and outputs to meet threshold {args.threshold}. " + f"Cases exempt from checking:" + ) + print("1. Private methods.") + print("2. Args with name in ('self', 'cls'), or *args / **kwargs") + print("3. Files under tests/") + raise Exception(f"\n❌ Type coverage below threshold ({args.threshold:.0%}).") + else: + if all_warnings or all_failures: + print("") + print("✅ Type annotation coverage acceptable.\n") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/tests/special_sanity/validate_imported_docs.py b/e2c/verl/tests/special_sanity/validate_imported_docs.py new file mode 100644 index 0000000000000000000000000000000000000000..b36a407be77a777cd72a4abf8ce4727d375eb548 --- /dev/null +++ b/e2c/verl/tests/special_sanity/validate_imported_docs.py @@ -0,0 +1,130 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +verify_imported_docs.py + +Assert that every function or class *explicitly imported* (via +`from import `) in a given Python file has a docstring. +""" + +from __future__ import annotations + +import argparse +import ast +import importlib +import inspect +import pathlib +import sys + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Verify that imported functions/classes have docstrings.") + p.add_argument( + "--target-file", + default="verl/trainer/ppo/ray_trainer.py", + help="Path to the Python source file to analyse (e.g. verl/trainer/ppo/ray_trainer.py)", + ) + p.add_argument( + "--allow-list", + default=["omegaconf.open_dict"], + help="a list of third_party dependencies that do not have proper docs :(", + ) + p.add_argument( + "--project-root", + default=".", + help="Directory to prepend to PYTHONPATH so local packages resolve (default: .)", + ) + p.add_argument( + "--quiet", + action="store_true", + help="Suppress success message (still prints errors).", + ) + return p.parse_args() + + +def _import_attr(module_name: str, attr_name: str): + """Import `module_name` then return `getattr(module, attr_name)`.""" + module = importlib.import_module(module_name) + return getattr(module, attr_name) + + +def _check_file(py_file: pathlib.Path, project_root: pathlib.Path, allow_list: list[str]) -> list[str]: + """Return a list of error strings (empty == success).""" + # Ensure local packages resolve + sys.path.insert(0, str(project_root.resolve())) + + tree = ast.parse(py_file.read_text(), filename=str(py_file)) + problems: list[str] = [] + + for node in ast.walk(tree): + if not isinstance(node, ast.ImportFrom): + continue + + # Relative imports (level > 0) get the leading dots stripped + module_name = "." * node.level + (node.module or "") + for alias in node.names: + if alias.name == "*": + problems.append( + f"{py_file}:{node.lineno} - wildcard import `from {module_name} import *` cannot be verified." + ) + continue + + imported_name = alias.name + + try: + obj = _import_attr(module_name, imported_name) + except Exception: # pragma: no cover – wide net for import quirks + pass + # For some reason the module cannot be imported, skip for now + # problems.append( + # f"{py_file}:{node.lineno} - could not resolve " + # f"`{imported_name}` from `{module_name}` ({exc})" + # ) + continue + + if f"{module_name}.{imported_name}" in allow_list: + continue + if inspect.isfunction(obj) or inspect.isclass(obj): + doc = inspect.getdoc(obj) + if not (doc and doc.strip()): + kind = "class" if inspect.isclass(obj) else "function" + problems.append( + f"{py_file}:{node.lineno} - {kind} `{module_name}.{imported_name}` is missing a docstring." + ) + + return problems + + +def main() -> None: + args = _parse_args() + target_path = pathlib.Path(args.target_file).resolve() + project_root = pathlib.Path(args.project_root).resolve() + + if not target_path.is_file(): + raise Exception(f"❌ Target file not found: {target_path}") + + errors = _check_file(target_path, project_root, args.allow_list) + + if errors: + print("Docstring verification failed:\n") + print("\n".join(f" • {e}" for e in errors)) + raise Exception("❌ Docstring verification failed.") + + if not args.quiet: + print(f"✅ All explicitly imported functions/classes in {target_path} have docstrings.") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/tests/special_sanity/validate_structure.py b/e2c/verl/tests/special_sanity/validate_structure.py new file mode 100644 index 0000000000000000000000000000000000000000..56136b206374ceff9c566aa1cd88d5be30f8c73b --- /dev/null +++ b/e2c/verl/tests/special_sanity/validate_structure.py @@ -0,0 +1,122 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 +""" +Validate that test file subfolders mirror the top-level package layout. + +Usage examples +-------------- + +# Typical run (defaults: impl_root=my_project, tests_root=tests) +python check_tests_structure.py + +# Custom layout and extra allowed folders +python check_tests_structure.py \ + --impl-root verl \ + --tests-root tests \ + --allow-dirs special_e2e special_sanity special_standalone special_distributed +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +def discover_allowed_modules(impl_root: Path, extra: list[str]) -> set[str]: + """Return the set of first-level directories that tests may live under.""" + allowed = {p.name for p in impl_root.iterdir() if p.is_dir()} + allowed.update(extra) + return allowed + + +def find_violations(tests_root: Path, allowed: set[str], allowed_files: list[str]) -> list[str]: + """Return a list of error strings for test files in the wrong place.""" + errors: list[str] = [] + for test_file in tests_root.rglob("test*.py"): + if str(test_file) in allowed_files: + continue + rel_parts = test_file.relative_to(tests_root).parts + if len(rel_parts) < 2: + errors.append(f"{test_file}: must be inside one of {sorted(allowed)} (not at tests root)") + continue + + first_folder = rel_parts[0] + if first_folder not in allowed: + errors.append( + f"{test_file}: subfolder '{first_folder}' under tests/ is not an allowed module. " + f"The valid ones are: {sorted(allowed)}" + ) + return errors + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check that test files follow tests//… layout.") + parser.add_argument( + "--impl-root", + type=Path, + default="verl", + help="Implementation root (default: my_project)", + ) + parser.add_argument( + "--tests-root", + type=Path, + default="tests", + help="Root of test tree (default: tests)", + ) + parser.add_argument( + "--allow-dirs", + nargs="*", + default=["special_e2e", "special_sanity", "special_standalone", "special_distributed"], + help="Extra top-level test folders that are exempt from the rule", + ) + parser.add_argument( + "--allow-files", + nargs="*", + default=[ + "tests/test_protocol_on_cpu.py", + "tests/test_base_config_on_cpu.py", + "tests/test_protocol_v2_on_cpu.py", + ], + help="Extra top-level test folders that are exempt from the rule", + ) + args = parser.parse_args() + + if not args.impl_root.is_dir(): + raise Exception(f"Implementation root '{args.impl_root}' does not exist.") + if not args.tests_root.is_dir(): + raise Exception(f"Tests root '{args.tests_root}' does not exist.") + + allowed = discover_allowed_modules(args.impl_root, args.allow_dirs) + violations = find_violations(args.tests_root, allowed, args.allow_files) + + if violations: + print("❌ Test layout violations found:\n", file=sys.stderr) + for err in violations: + print(" -", err, file=sys.stderr) + + print( + f"\nGuideline:\n Place each test file under tests//…\n where is " + f"one of the top-level packages inside '{args.impl_root}', or is explicitly listed via --allow-dirs.\n", + file=sys.stderr, + ) + raise Exception("❌ Test layout violations found.") + + print("✅ Tests folder structure looks good.") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/tests/special_standalone/README.md b/e2c/verl/tests/special_standalone/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0e3596e1afa9a507c67b6949479d1c254b30aec3 --- /dev/null +++ b/e2c/verl/tests/special_standalone/README.md @@ -0,0 +1 @@ +The standalone test folder is reserved for tests that require dedicated environment (e.g. memory stress tests) diff --git a/e2c/verl/tests/special_standalone/test_memory_buffers.py b/e2c/verl/tests/special_standalone/test_memory_buffers.py new file mode 100644 index 0000000000000000000000000000000000000000..77851534782c7d0f5b9ec93231fde8d4d5e60bb6 --- /dev/null +++ b/e2c/verl/tests/special_standalone/test_memory_buffers.py @@ -0,0 +1,66 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test memory buffers +- We start with two models with the same weights +- We use Memory buffer to make one of the models and then compare the parameters +""" + +import gc + +import torch +from transformers import LlamaConfig, LlamaModel + + +def test_memory_buffers(): + llama_config = LlamaConfig( + vocab_size=256, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=2, + num_attention_heads=16, + num_key_value_heads=16, + ) + + model = LlamaModel(config=llama_config).cuda() + model_copy = LlamaModel(config=llama_config).cuda() + model_copy.load_state_dict(model.state_dict()) + + norm_factor = 1024**3 + + t_before = torch.cuda.get_device_properties(0).total_memory / norm_factor + r_before = torch.cuda.memory_reserved(0) / norm_factor + a_before = torch.cuda.memory_allocated(0) / norm_factor + + print(f"Before Total memory: {t_before} GB, reserved: {r_before} GB, allocated: {a_before} GB") + + t = torch.cuda.get_device_properties(0).total_memory / norm_factor + r = torch.cuda.memory_reserved(0) / norm_factor + a = torch.cuda.memory_allocated(0) / norm_factor + + gc.collect() + torch.cuda.empty_cache() + + print(f"After Total memory: {t} GB, reserved: {r} GB, allocated: {a} GB") + + change_ratio = (a - a_before) / a_before + assert change_ratio < 0.01, f"make sure the allocated change is less than 1%, Got {change_ratio}" + + for (name1, param1), (name2, param2) in zip(model.named_parameters(), model_copy.named_parameters(), strict=True): + assert name1 == name2 + assert torch.eq(param1.data, param2.data).all(), f"{param1.data}, {param2.data}, {name1}" + + +if __name__ == "__main__": + test_memory_buffers() diff --git a/e2c/verl/tests/test_base_config_on_cpu.py b/e2c/verl/tests/test_base_config_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..9a50235c8ffa736551781d50cf5c937ce21afce0 --- /dev/null +++ b/e2c/verl/tests/test_base_config_on_cpu.py @@ -0,0 +1,42 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from verl.base_config import BaseConfig + + +@pytest.fixture +def base_config_mock(): + """Fixture to create a mock BaseConfig instance with test attributes.""" + mock_config = BaseConfig() + mock_config.test_attr = "test_value" + return mock_config + + +def test_getitem_success(base_config_mock): + """Test __getitem__ with existing attribute (happy path).""" + assert base_config_mock["test_attr"] == "test_value" + + +def test_getitem_nonexistent_attribute(base_config_mock): + """Test __getitem__ with non-existent attribute (exception path 1).""" + with pytest.raises(AttributeError): + _ = base_config_mock["nonexistent_attr"] + + +def test_getitem_invalid_key_type(base_config_mock): + """Test __getitem__ with invalid key type (exception path 2).""" + with pytest.raises(TypeError): + _ = base_config_mock[123] # type: ignore diff --git a/e2c/verl/tests/test_protocol_on_cpu.py b/e2c/verl/tests/test_protocol_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..a1b9253bbe7b932458c4810a68bcf72cc022dac1 --- /dev/null +++ b/e2c/verl/tests/test_protocol_on_cpu.py @@ -0,0 +1,838 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +import numpy as np +import pytest +import tensordict +import torch +from packaging.version import parse as parse_version +from tensordict import TensorDict + +from verl import DataProto +from verl.protocol import ( + deserialize_single_tensor, + deserialize_tensordict, + serialize_single_tensor, + serialize_tensordict, + union_numpy_dict, + union_tensor_dict, +) + + +def test_union_tensor_dict(): + obs = torch.randn(100, 10) + + data1 = TensorDict({"obs": obs, "act": torch.randn(100, 3)}, batch_size=[100]) + data2 = TensorDict({"obs": obs, "next_obs": torch.randn(100, 10), "rew": torch.randn(100)}, batch_size=[100]) + + data_with_copied_obs = TensorDict( + {"obs": obs.clone(), "next_obs": torch.randn(100, 10), "rew": torch.randn(100)}, batch_size=[100] + ) + + union_tensor_dict(data1, data2) + with pytest.raises(AssertionError): + union_tensor_dict(data1, data_with_copied_obs) + + +def test_union_numpy_dict(): + """ + A comprehensive test suite for union_numpy_dict, covering standard use + cases, N-dimensional arrays, object-dtype arrays, and NaN value handling. + """ + arr_3d = np.arange(8).reshape((2, 2, 2)) + union_numpy_dict({"a": arr_3d}, {"a": arr_3d}) + arr1 = np.array([1, "hello", np.array([2, 3])], dtype=object) + arr2 = np.array([1, "hello", np.array([2, 3])], dtype=object) + union_numpy_dict({"a": arr1}, {"a": arr2}) + # --- Test Case 1: The original test with mixed object/float types --- + # This test case from the original test file is preserved. + data = np.random.random(100) + # This array intentionally mixes float('nan') and the string 'nan' + nan_data = [float("nan") for _ in range(99)] + nan_data.append("nan") + nan_data_arr = np.array(nan_data, dtype=object) + + dict1 = {"a": data, "b": nan_data_arr} + dict2_same = {"a": data.copy(), "b": nan_data_arr.copy()} + dict3_different = {"a": np.random.random(100)} + + union_numpy_dict(dict1, dict2_same) # Should pass + with pytest.raises(AssertionError): + union_numpy_dict(dict1, dict3_different) + + # --- Test Case 2: Standard 3D arrays (fixes the core bug) --- + arr_3d = np.arange(24, dtype=np.int32).reshape((2, 3, 4)) + dict_3d_1 = {"nd_array": arr_3d} + dict_3d_2_same = {"nd_array": arr_3d.copy()} + dict_3d_3_different = {"nd_array": arr_3d + 1} + + union_numpy_dict(dict_3d_1, dict_3d_2_same) # Should pass + with pytest.raises(AssertionError, match="`nd_array` in tensor_dict1 and tensor_dict2 are not the same object."): + union_numpy_dict(dict_3d_1, dict_3d_3_different) + + # --- Test Case 3: Nested 2D and 4D object-dtype arrays --- + sub_arr1 = np.array([1, 2]) + sub_arr2 = np.array([3.0, 4.0]) + # 2D object array + arr_2d_obj = np.array([[sub_arr1, "text"], [sub_arr2, None]], dtype=object) + arr_2d_obj_diff = np.array([[sub_arr1, "text"], [sub_arr2, "other"]], dtype=object) + + union_numpy_dict({"data": arr_2d_obj}, {"data": arr_2d_obj.copy()}) # Should pass + with pytest.raises(AssertionError): + union_numpy_dict({"data": arr_2d_obj}, {"data": arr_2d_obj_diff}) + + # 4D object array to ensure deep recursion is robust + arr_4d_obj = np.array([[[[sub_arr1]]], [[[sub_arr2]]]], dtype=object) + arr_4d_obj_diff = np.array([[[[sub_arr1]]], [[[np.array([9, 9])]]]], dtype=object) + + union_numpy_dict({"data": arr_4d_obj}, {"data": arr_4d_obj.copy()}) # Should pass + with pytest.raises(AssertionError): + union_numpy_dict({"data": arr_4d_obj}, {"data": arr_4d_obj_diff}) + + # --- Test Case 4: Explicit NaN value comparison --- + # This verifies that our new _deep_equal logic correctly handles NaNs. + nan_arr = np.array([1.0, np.nan, 3.0]) + dict_nan_1 = {"data": nan_arr} + dict_nan_2_same = {"data": np.array([1.0, np.nan, 3.0])} # A new array with same values + dict_nan_3_different_val = {"data": np.array([1.0, 2.0, 3.0])} + dict_nan_4_different_pos = {"data": np.array([np.nan, 1.0, 3.0])} + + # NaNs in the same position should be considered equal for merging. + union_numpy_dict(dict_nan_1, dict_nan_2_same) # Should pass + + with pytest.raises(AssertionError): + union_numpy_dict(dict_nan_1, dict_nan_3_different_val) + with pytest.raises(AssertionError): + union_numpy_dict(dict_nan_1, dict_nan_4_different_pos) + + # --- Test Case 5: Circular reference handling --- + # Create two separate, but structurally identical, circular references. + # This should pass without a RecursionError. + circ_arr_1 = np.array([None], dtype=object) + circ_arr_1[0] = circ_arr_1 + + circ_arr_2 = np.array([None], dtype=object) + circ_arr_2[0] = circ_arr_2 + + union_numpy_dict({"data": circ_arr_1}, {"data": circ_arr_2}) # Should pass + + # Create a circular reference and a non-circular one. + # This should fail with an AssertionError because they are different. + non_circ_arr = np.array([None], dtype=object) + + with pytest.raises(AssertionError): + union_numpy_dict({"data": circ_arr_1}, {"data": non_circ_arr}) + + +def test_tensor_dict_constructor(): + obs = torch.randn(100, 10) + act = torch.randn(100, 10, 3) + data = DataProto.from_dict(tensors={"obs": obs, "act": act}) + + assert data.batch.batch_size == torch.Size([100]) + + with pytest.raises(AssertionError): + data = DataProto.from_dict(tensors={"obs": obs, "act": act}, num_batch_dims=2) + + with pytest.raises(AssertionError): + data = DataProto.from_dict(tensors={"obs": obs, "act": act}, num_batch_dims=3) + + +def test_tensor_dict_make_iterator(): + obs = torch.randn(100, 10) + labels = [random.choice(["abc", "cde"]) for _ in range(100)] + dataset = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}) + + data_iter_1 = dataset.make_iterator(mini_batch_size=10, epochs=2, seed=1) + data_list_1 = [] + for data in data_iter_1: + data_list_1.append(data) + + data_iter_2 = dataset.make_iterator(mini_batch_size=10, epochs=2, seed=1) + data_list_2 = [] + for data in data_iter_2: + data_list_2.append(data) + + for data1, data2 in zip(data_list_1, data_list_2, strict=True): + assert isinstance(data1, DataProto) + assert isinstance(data2, DataProto) + result = torch.all(torch.eq(data1.batch["obs"], data2.batch["obs"])) + if not result.item(): + print(data1.batch["obs"]) + print(data2.batch["obs"]) + raise AssertionError() + non_tensor_result = np.all(np.equal(data1.non_tensor_batch["labels"], data2.non_tensor_batch["labels"])) + if not non_tensor_result.item(): + print(data1.non_tensor_batch["labels"]) + print(data2.non_tensor_batch["labels"]) + + +def test_reorder(): + obs = torch.tensor([1, 2, 3, 4, 5, 6]) + labels = ["a", "b", "c", "d", "e", "f"] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"name": "abdce"}) + data.reorder(torch.tensor([3, 4, 2, 0, 1, 5])) + + assert torch.all(torch.eq(data.batch["obs"], torch.tensor([4, 5, 3, 1, 2, 6]))) + assert np.all(data.non_tensor_batch["labels"] == np.array(["d", "e", "c", "a", "b", "f"])) + assert data.meta_info == {"name": "abdce"} + + +def test_chunk_concat(): + obs = torch.tensor([1, 2, 3, 4, 5, 6]) + labels = ["a", "b", "c", "d", "e", "f"] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"name": "abdce"}) + + with pytest.raises(AssertionError): + data.chunk(5) + + data_split = data.chunk(2) + assert len(data_split) == 2 + assert torch.all(torch.eq(data_split[0].batch["obs"], torch.tensor([1, 2, 3]))) + assert np.all(data_split[0].non_tensor_batch["labels"] == np.array(["a", "b", "c"])) + assert data_split[0].meta_info == {"name": "abdce"} + + assert torch.all(torch.eq(data_split[1].batch["obs"], torch.tensor([4, 5, 6]))) + assert np.all(data_split[1].non_tensor_batch["labels"] == np.array(["d", "e", "f"])) + assert data_split[1].meta_info == {"name": "abdce"} + + concat_data = DataProto.concat(data_split) + assert torch.all(torch.eq(concat_data.batch["obs"], data.batch["obs"])) + assert np.all(concat_data.non_tensor_batch["labels"] == data.non_tensor_batch["labels"]) + assert concat_data.meta_info == data.meta_info + + +def test_pop(): + obs = torch.randn(100, 10) + act = torch.randn(100, 3) + dataset = DataProto.from_dict({"obs": obs, "act": act}, meta_info={"2": 2, "1": 1}) + poped_dataset = dataset.pop(batch_keys=["obs"], meta_info_keys=["2"]) + + assert poped_dataset.batch.keys() == {"obs"} + assert poped_dataset.meta_info.keys() == {"2"} + + assert dataset.batch.keys() == {"act"} + assert dataset.meta_info.keys() == {"1"} + + +def test_repeat(): + # Create a DataProto object with some batch and non-tensor data + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"info": "test_info"}) + + # Test interleave=True + repeated_data_interleave = data.repeat(repeat_times=2, interleave=True) + expected_obs_interleave = torch.tensor([[1, 2], [1, 2], [3, 4], [3, 4], [5, 6], [5, 6]]) + expected_labels_interleave = ["a", "a", "b", "b", "c", "c"] + + assert torch.all(torch.eq(repeated_data_interleave.batch["obs"], expected_obs_interleave)) + assert (repeated_data_interleave.non_tensor_batch["labels"] == expected_labels_interleave).all() + assert repeated_data_interleave.meta_info == {"info": "test_info"} + + # Test interleave=False + repeated_data_no_interleave = data.repeat(repeat_times=2, interleave=False) + expected_obs_no_interleave = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2], [3, 4], [5, 6]]) + expected_labels_no_interleave = ["a", "b", "c", "a", "b", "c"] + + assert torch.all(torch.eq(repeated_data_no_interleave.batch["obs"], expected_obs_no_interleave)) + assert (repeated_data_no_interleave.non_tensor_batch["labels"] == expected_labels_no_interleave).all() + assert repeated_data_no_interleave.meta_info == {"info": "test_info"} + + +def test_dataproto_pad_unpad(): + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"info": "test_info"}) + + from verl.protocol import pad_dataproto_to_divisor, unpad_dataproto + + padded_data, pad_size = pad_dataproto_to_divisor(data, size_divisor=2) + assert pad_size == 1 + + expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2]]) + expected_labels = ["a", "b", "c", "a"] + + assert torch.all(torch.eq(padded_data.batch["obs"], expected_obs)) + assert (padded_data.non_tensor_batch["labels"] == expected_labels).all() + assert padded_data.meta_info == {"info": "test_info"} + + unpadd_data = unpad_dataproto(padded_data, pad_size=pad_size) + assert torch.all(torch.eq(unpadd_data.batch["obs"], obs)) + assert (unpadd_data.non_tensor_batch["labels"] == labels).all() + assert unpadd_data.meta_info == {"info": "test_info"} + + padded_data, pad_size = pad_dataproto_to_divisor(data, size_divisor=3) + assert pad_size == 0 + + expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + expected_labels = ["a", "b", "c"] + + assert torch.all(torch.eq(padded_data.batch["obs"], expected_obs)) + assert (padded_data.non_tensor_batch["labels"] == expected_labels).all() + assert padded_data.meta_info == {"info": "test_info"} + + unpadd_data = unpad_dataproto(padded_data, pad_size=pad_size) + assert torch.all(torch.eq(unpadd_data.batch["obs"], obs)) + assert (unpadd_data.non_tensor_batch["labels"] == labels).all() + assert unpadd_data.meta_info == {"info": "test_info"} + + padded_data, pad_size = pad_dataproto_to_divisor(data, size_divisor=7) + assert pad_size == 4 + + expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2], [3, 4], [5, 6], [1, 2]]) + expected_labels = ["a", "b", "c", "a", "b", "c", "a"] + assert torch.all(torch.eq(padded_data.batch["obs"], expected_obs)) + assert (padded_data.non_tensor_batch["labels"] == expected_labels).all() + assert padded_data.meta_info == {"info": "test_info"} + + unpadd_data = unpad_dataproto(padded_data, pad_size=pad_size) + assert torch.all(torch.eq(unpadd_data.batch["obs"], obs)) + assert (unpadd_data.non_tensor_batch["labels"] == labels).all() + assert unpadd_data.meta_info == {"info": "test_info"} + + +def test_dataproto_fold_unfold(): + from verl.protocol import DataProto, fold_batch_dim, unfold_batch_dim + + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"info": "test_info"}) + + data1 = data.repeat(repeat_times=2, interleave=True) + + data2 = fold_batch_dim(data1, new_batch_size=3) + + torch.testing.assert_close(data2.batch["obs"], torch.tensor([[[1, 2], [1, 2]], [[3, 4], [3, 4]], [[5, 6], [5, 6]]])) + assert (data2.non_tensor_batch["labels"] == [["a", "a"], ["b", "b"], ["c", "c"]]).all() + + data2.reorder(indices=torch.tensor([1, 2, 0])) + + data3 = unfold_batch_dim(data2, batch_dims=2) + + torch.testing.assert_close(data3.batch["obs"], torch.tensor([[3, 4], [3, 4], [5, 6], [5, 6], [1, 2], [1, 2]])) + assert (data3.non_tensor_batch["labels"] == ["b", "b", "c", "c", "a", "a"]).all() + assert data3.meta_info == {"info": "test_info"} + + +def test_torch_save_data_proto(): + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"info": "test_info"}) + data.save_to_disk("test_data.pt") + loaded_data = DataProto.load_from_disk("test_data.pt") + + assert torch.all(torch.eq(loaded_data.batch["obs"], data.batch["obs"])) + assert (loaded_data.non_tensor_batch["labels"] == data.non_tensor_batch["labels"]).all() + assert loaded_data.meta_info == data.meta_info + + import os + + os.remove("test_data.pt") + + +def test_len(): + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = np.array(["a", "b", "c"], dtype=object) + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"info": "test_info"}) + + assert len(data) == 3 + + data = DataProto(batch=None, non_tensor_batch={"labels": labels}, meta_info={"info": "test_info"}) + + assert len(data) == 3 + + data = DataProto(batch=None, non_tensor_batch={}, meta_info={"info": "test_info"}) + + assert len(data) == 0 + + data = DataProto(batch=None, non_tensor_batch=None, meta_info={"info": "test_info"}) + + assert len(data) == 0 + + +def test_dataproto_index(): + data_len = 100 + idx_num = 10 + + obs = torch.randn(data_len, 10) + labels = [random.choice(["abc", "cde"]) for _ in range(data_len)] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}) + labels_np = np.array(labels) + + idx_np_int = np.random.randint(0, data_len, size=(idx_num,)) + result_np_int = data[idx_np_int] + assert result_np_int.batch.keys() == data.batch.keys() + assert result_np_int.non_tensor_batch.keys() == data.non_tensor_batch.keys() + assert result_np_int.batch["obs"].shape[0] == idx_num + assert result_np_int.non_tensor_batch["labels"].shape[0] == idx_num + assert np.array_equal(result_np_int.batch["obs"].cpu().numpy(), obs[idx_np_int].numpy()) + assert np.array_equal(result_np_int.non_tensor_batch["labels"], labels_np[idx_np_int]) + + idx_torch_int = torch.randint(0, data_len, size=(idx_num,)) + result_torch_int = data[idx_torch_int] + assert result_torch_int.batch.keys() == data.batch.keys() + assert result_torch_int.non_tensor_batch.keys() == data.non_tensor_batch.keys() + assert result_torch_int.batch["obs"].shape[0] == idx_num + assert result_torch_int.non_tensor_batch["labels"].shape[0] == idx_num + assert np.array_equal(result_torch_int.batch["obs"].cpu().numpy(), obs[idx_torch_int].cpu().numpy()) + assert np.array_equal(result_torch_int.non_tensor_batch["labels"], labels_np[idx_torch_int.cpu().numpy()]) + + idx_list_int = [np.random.randint(0, data_len) for _ in range(idx_num)] + result_list_int = data[idx_list_int] + assert result_list_int.batch.keys() == data.batch.keys() + assert result_list_int.non_tensor_batch.keys() == data.non_tensor_batch.keys() + assert result_list_int.batch["obs"].shape[0] == idx_num + assert result_list_int.non_tensor_batch["labels"].shape[0] == idx_num + assert np.array_equal(result_list_int.batch["obs"].cpu().numpy(), obs[idx_list_int].cpu().numpy()) + assert np.array_equal(result_list_int.non_tensor_batch["labels"], labels_np[idx_list_int]) + + idx_np_bool = np.random.randint(0, 2, size=(data_len,), dtype=bool) + result_np_bool = data[idx_np_bool] + assert result_np_bool.batch.keys() == data.batch.keys() + assert result_np_bool.non_tensor_batch.keys() == data.non_tensor_batch.keys() + assert result_np_bool.batch["obs"].shape[0] == idx_np_bool.sum() + assert result_np_bool.non_tensor_batch["labels"].shape[0] == idx_np_bool.sum() + assert np.array_equal(result_np_bool.batch["obs"].cpu().numpy(), obs[idx_np_bool].cpu().numpy()) + assert np.array_equal(result_np_bool.non_tensor_batch["labels"], labels_np[idx_np_bool]) + + idx_torch_bool = torch.randint(0, 2, size=(data_len,), dtype=torch.bool) + result_torch_bool = data[idx_torch_bool] + assert result_torch_bool.batch.keys() == data.batch.keys() + assert result_torch_bool.non_tensor_batch.keys() == data.non_tensor_batch.keys() + assert result_torch_bool.batch["obs"].shape[0] == idx_torch_bool.sum().item() + assert result_torch_bool.non_tensor_batch["labels"].shape[0] == idx_torch_bool.sum().item() + assert np.array_equal(result_torch_bool.batch["obs"].cpu().numpy(), obs[idx_torch_bool].cpu().numpy()) + assert np.array_equal(result_torch_bool.non_tensor_batch["labels"], labels_np[idx_torch_bool]) + + idx_list_bool = [np.random.randint(0, 2, dtype=bool) for _ in range(data_len)] + result_list_bool = data[idx_list_bool] + assert result_list_bool.batch.keys() == data.batch.keys() + assert result_list_bool.non_tensor_batch.keys() == data.non_tensor_batch.keys() + assert result_list_bool.batch["obs"].shape[0] == sum(idx_list_bool) + assert result_list_bool.non_tensor_batch["labels"].shape[0] == sum(idx_list_bool) + assert np.array_equal(result_list_bool.batch["obs"].cpu().numpy(), obs[idx_list_bool].cpu().numpy()) + assert np.array_equal(result_list_bool.non_tensor_batch["labels"], labels_np[idx_list_bool]) + + +def test_old_vs_new_from_single_dict(): + class CustomProto(DataProto): + """Uses the new, fixed from_single_dict.""" + + pass + + class OriginProto(DataProto): + """Mimics the *old* from_single_dict (always returns a DataProto).""" + + @classmethod + def from_single_dict(cls, data, meta_info=None, auto_padding=False): + tensors, non_tensors = {}, {} + for k, v in data.items(): + if torch.is_tensor(v): + tensors[k] = v + else: + non_tensors[k] = v + # always calls DataProto.from_dict, ignoring `cls` + return DataProto.from_dict( + tensors=tensors, + non_tensors=non_tensors, + meta_info=meta_info, + auto_padding=auto_padding, + ) + + sample = {"x": torch.tensor([0])} + + orig = OriginProto.from_single_dict(sample) + # old behavior: always DataProto, not a CustomOriginProto + assert type(orig) is DataProto + assert type(orig) is not OriginProto + + cust = CustomProto.from_single_dict(sample) + # new behavior: respects subclass + assert type(cust) is CustomProto + + +def test_dataproto_no_batch(): + labels = ["a", "b", "c"] + data = DataProto.from_dict(non_tensors={"labels": labels}, meta_info={"info": "test_info"}) + selected = data.select(non_tensor_batch_keys=["labels"]) + assert (selected.non_tensor_batch["labels"] == labels).all() + pop_data = data.pop(non_tensor_batch_keys=["labels"]) + assert (pop_data.non_tensor_batch["labels"] == labels).all() + assert data.non_tensor_batch == {} + + +def test_sample_level_repeat(): + # Create a DataProto object with some batch and non-tensor data + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"info": "test_info"}) + + # list + repeated_data_interleave = data.sample_level_repeat(repeat_times=[3, 1, 2]) + expected_obs_interleave = torch.tensor([[1, 2], [1, 2], [1, 2], [3, 4], [5, 6], [5, 6]]) + expected_labels_interleave = ["a", "a", "a", "b", "c", "c"] + + assert torch.all(torch.eq(repeated_data_interleave.batch["obs"], expected_obs_interleave)) + assert (repeated_data_interleave.non_tensor_batch["labels"] == expected_labels_interleave).all() + assert repeated_data_interleave.meta_info == {"info": "test_info"} + + # torch.tensor + repeated_data_no_interleave = data.sample_level_repeat(repeat_times=torch.tensor([1, 2, 3])) + expected_obs_no_interleave = torch.tensor([[1, 2], [3, 4], [3, 4], [5, 6], [5, 6], [5, 6]]) + expected_labels_no_interleave = ["a", "b", "b", "c", "c", "c"] + + assert torch.all(torch.eq(repeated_data_no_interleave.batch["obs"], expected_obs_no_interleave)) + assert (repeated_data_no_interleave.non_tensor_batch["labels"] == expected_labels_no_interleave).all() + assert repeated_data_no_interleave.meta_info == {"info": "test_info"} + + +def test_dataproto_unfold_column_chunks(): + obs1 = torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) + obs2 = torch.tensor([[1, 2], [5, 6], [9, 10]]) + + labels = ["a", "b", "c"] + data = DataProto.from_dict( + tensors={"obs1": obs1, "obs2": obs2}, non_tensors={"labels": labels}, meta_info={"name": "abc"} + ) + ret = data.unfold_column_chunks(2, split_keys=["obs1"]) + + expect_obs1 = torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12]]) + expect_obs2 = torch.tensor([[1, 2], [1, 2], [5, 6], [5, 6], [9, 10], [9, 10]]) + expect_labels = ["a", "a", "b", "b", "c", "c"] + assert torch.all(torch.eq(ret.batch["obs1"], expect_obs1)) + assert torch.all(torch.eq(ret.batch["obs2"], expect_obs2)) + assert (ret.non_tensor_batch["labels"] == expect_labels).all() + assert ret.meta_info == {"name": "abc"} + + obs1 = torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) + obs2 = torch.tensor([[1, 2], [5, 6], [9, 10]]) + + labels = [["a1", "a2"], ["b1", "b2"], ["c1", "c2"]] + data = DataProto.from_dict( + tensors={"obs1": obs1, "obs2": obs2}, non_tensors={"labels": labels}, meta_info={"name": "abc"} + ) + ret = data.unfold_column_chunks(2, split_keys=["obs1", "labels"]) + + expect_obs1 = torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12]]) + expect_obs2 = torch.tensor([[1, 2], [1, 2], [5, 6], [5, 6], [9, 10], [9, 10]]) + expect_labels = [["a1"], ["a2"], ["b1"], ["b2"], ["c1"], ["c2"]] + assert torch.all(torch.eq(ret.batch["obs1"], expect_obs1)) + assert torch.all(torch.eq(ret.batch["obs2"], expect_obs2)) + assert (ret.non_tensor_batch["labels"] == expect_labels).all() + assert ret.meta_info == {"name": "abc"} + + obs1 = torch.tensor( + [[[1, 1], [2, 2], [3, 3], [4, 4]], [[5, 5], [6, 6], [7, 7], [8, 8]], [[9, 9], [10, 10], [11, 11], [12, 12]]] + ) + obs2 = torch.tensor([[[1, 1], [2, 2]], [[5, 5], [6, 6]], [[9, 9], [10, 10]]]) + + labels = ["a", "b", "c"] + data = DataProto.from_dict( + tensors={"obs1": obs1, "obs2": obs2}, non_tensors={"labels": labels}, meta_info={"name": "abc"} + ) + ret = data.unfold_column_chunks(2, split_keys=["obs1"]) + + expect_obs1 = torch.tensor( + [ + [[1, 1], [2, 2]], + [[3, 3], [4, 4]], + [[5, 5], [6, 6]], + [[7, 7], [8, 8]], + [[9, 9], [10, 10]], + [[11, 11], [12, 12]], + ] + ) + expect_obs2 = torch.tensor( + [[[1, 1], [2, 2]], [[1, 1], [2, 2]], [[5, 5], [6, 6]], [[5, 5], [6, 6]], [[9, 9], [10, 10]], [[9, 9], [10, 10]]] + ) + expect_labels = ["a", "a", "b", "b", "c", "c"] + assert torch.all(torch.eq(ret.batch["obs1"], expect_obs1)) + assert torch.all(torch.eq(ret.batch["obs2"], expect_obs2)) + assert (ret.non_tensor_batch["labels"] == expect_labels).all() + assert ret.meta_info == {"name": "abc"} + + +def test_dataproto_chunk_after_index(): + data_len = 4 + obs = torch.randn(data_len, 4) + labels = [f"label_{i}" for i in range(data_len)] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"name": "abc"}) + + # Test with boolean numpy array + bool_mask = np.array([True, False, True, False]) + selected = data[bool_mask] + assert isinstance(selected.batch.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch.batch_size) # int or List[int] + + # Test with integer numpy array + int_mask = np.array([0, 2]) + selected = data[int_mask] + assert isinstance(selected.batch.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch.batch_size) + + # Test with boolean list + list_mask = [True, False, True, False] + selected = data[list_mask] + assert isinstance(selected.batch.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch.batch_size) + + # Test with list + list_mask = [0, 2] + selected = data[list_mask] + assert isinstance(selected.batch.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch.batch_size) + + # Test with torch tensor (bool) + torch_bool_mask = torch.tensor([True, False, True, False]) + selected = data[torch_bool_mask] + assert isinstance(selected.batch.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch.batch_size) + + # Test with torch tensor (int) + torch_int_mask = torch.tensor([0, 2]) + selected = data[torch_int_mask] + assert isinstance(selected.batch.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch.batch_size) + + +@pytest.mark.skipif( + parse_version(tensordict.__version__) < parse_version("0.10"), reason="requires at least tensordict 0.10" +) +def test_to_tensordict(): + obs = torch.tensor([1, 2, 3, 4, 5, 6]) + labels = ["a", "b", "c", "d", "e", "f"] + data = DataProto.from_dict(tensors={"obs": obs}, non_tensors={"labels": labels}, meta_info={"name": "abdce"}) + output = data.to_tensordict() + + assert torch.all(torch.eq(output["obs"], obs)).item() + assert output["labels"] == labels + assert output["name"] == "abdce" + + +def test_serialize_deserialize_single_tensor(): + """Test serialization and deserialization of a single tensor""" + # Create test tensor + original_tensor = torch.randn(3, 4, 5) + + # Serialize + dtype, shape, data = serialize_single_tensor(original_tensor) + + # Deserialize + reconstructed_tensor = deserialize_single_tensor((dtype, shape, data)) + + # Verify results + assert torch.allclose(original_tensor, reconstructed_tensor) + assert original_tensor.shape == reconstructed_tensor.shape + assert original_tensor.dtype == reconstructed_tensor.dtype + + +def test_serialize_deserialize_tensordict_regular_tensors(): + """Test serialization and deserialization of TensorDict with regular tensors""" + # Create test data + batch_size = (5, 3) + tensor1 = torch.randn(*batch_size, 4) + tensor2 = torch.randint(0, 10, (*batch_size, 2)) + + # Create TensorDict + original_tensordict = TensorDict({"tensor1": tensor1, "tensor2": tensor2}, batch_size=batch_size) + + # Serialize + batch_size_serialized, device, encoded_items = serialize_tensordict(original_tensordict) + + # Deserialize + reconstructed_tensordict = deserialize_tensordict((batch_size_serialized, device, encoded_items)) + + # Verify results + assert original_tensordict.batch_size == reconstructed_tensordict.batch_size + assert set(original_tensordict.keys()) == set(reconstructed_tensordict.keys()) + + for key in original_tensordict.keys(): + original_tensor = original_tensordict[key] + reconstructed_tensor = reconstructed_tensordict[key] + + assert torch.allclose(original_tensor, reconstructed_tensor) + assert original_tensor.shape == reconstructed_tensor.shape + assert original_tensor.dtype == reconstructed_tensor.dtype + + +def test_serialize_deserialize_tensordict_nested_tensors(): + """Test serialization and deserialization of TensorDict with nested tensors""" + # Create nested tensor + tensor_list = [torch.randn(2, 3), torch.randn(3, 4), torch.randn(1, 5)] + nested_tensor = torch.nested.as_nested_tensor(tensor_list) + + # Create regular tensor for comparison + regular_tensor = torch.randn(3, 4, 5) + + # Create TensorDict + original_tensordict = TensorDict({"nested": nested_tensor, "regular": regular_tensor}, batch_size=(3,)) + + # Serialize + batch_size_serialized, device, encoded_items = serialize_tensordict(original_tensordict) + + # Deserialize + reconstructed_tensordict = deserialize_tensordict((batch_size_serialized, device, encoded_items)) + + # Verify results + assert original_tensordict.batch_size == reconstructed_tensordict.batch_size + assert set(original_tensordict.keys()) == set(reconstructed_tensordict.keys()) + + # Verify regular tensor + original_regular = original_tensordict["regular"] + reconstructed_regular = reconstructed_tensordict["regular"] + + assert torch.allclose(original_regular, reconstructed_regular) + assert original_regular.shape == reconstructed_regular.shape + assert original_regular.dtype == reconstructed_regular.dtype + + # Verify nested tensor + original_nested = original_tensordict["nested"] + reconstructed_nested = reconstructed_tensordict["nested"] + + # Check if it's a nested tensor + assert original_nested.is_nested + assert reconstructed_nested.is_nested + + # Check layout + assert original_nested.layout == reconstructed_nested.layout + + # Check each tensor after unbinding + original_unbind = original_nested.unbind() + reconstructed_unbind = reconstructed_nested.unbind() + + assert len(original_unbind) == len(reconstructed_unbind) + + for orig, recon in zip(original_unbind, reconstructed_unbind, strict=False): + assert torch.allclose(orig, recon) + assert orig.shape == recon.shape + assert orig.dtype == recon.dtype + + +def test_serialize_deserialize_tensordict_mixed_types(): + """Test serialization and deserialization of TensorDict with mixed tensor types""" + # Create tensors with different data types + float_tensor = torch.randn(2, 3).float() + double_tensor = torch.randn(2, 3).double() + int_tensor = torch.randint(0, 10, (2, 3)).int() + long_tensor = torch.randint(0, 10, (2, 3)).long() + bool_tensor = torch.tensor([[True, False], [False, True]]) + bfloat16_tensor = torch.randn(2, 3).bfloat16() + + # Add fp8 tensor (if available) + # Note: FP8 is not natively supported in all PyTorch versions + # We'll check if it's available and conditionally include it + has_fp8 = hasattr(torch, "float8_e5m2") or hasattr(torch, "float8_e4m3fn") + if has_fp8: + try: + # Try to create an FP8 tensor (implementation may vary) + # This is a placeholder - actual FP8 support might require specific hardware + fp8_tensor = torch.randn(2, 3) + if hasattr(torch, "float8_e5m2"): + fp8_tensor = fp8_tensor.to(torch.float8_e5m2) + elif hasattr(torch, "float8_e4m3fn"): + fp8_tensor = fp8_tensor.to(torch.float8_e4m3fn) + except Exception: + has_fp8 = False + + # Create nested tensor + tensor_list = [ + torch.randn(2, 3), + torch.randn(3, 4), + ] + nested_tensor = torch.nested.as_nested_tensor(tensor_list) + + # Create TensorDict with all available types + tensordict_data = { + "float": float_tensor, + "double": double_tensor, + "int": int_tensor, + "long": long_tensor, + "bool": bool_tensor, + "bfloat16": bfloat16_tensor, + "nested": nested_tensor, + } + + # Conditionally add fp8 tensor if available + if has_fp8: + tensordict_data["fp8"] = fp8_tensor + + original_tensordict = TensorDict( + tensordict_data, + batch_size=(2,), + ) + + # Serialize + batch_size_serialized, device, encoded_items = serialize_tensordict(original_tensordict) + + # Deserialize + reconstructed_tensordict = deserialize_tensordict((batch_size_serialized, device, encoded_items)) + + # Verify results + assert original_tensordict.batch_size == reconstructed_tensordict.batch_size + assert set(original_tensordict.keys()) == set(reconstructed_tensordict.keys()) + + for key in original_tensordict.keys(): + original_tensor = original_tensordict[key] + reconstructed_tensor = reconstructed_tensordict[key] + + if original_tensor.is_nested: + # For nested tensors, check each tensor after unbinding + original_unbind = original_tensor.unbind() + reconstructed_unbind = reconstructed_tensor.unbind() + + assert len(original_unbind) == len(reconstructed_unbind) + + for orig, recon in zip(original_unbind, reconstructed_unbind, strict=False): + assert torch.allclose(orig, recon, equal_nan=True) + assert orig.shape == recon.shape + assert orig.dtype == recon.dtype + else: + # For regular tensors, compare directly + assert torch.all(original_tensor == reconstructed_tensor) + assert original_tensor.shape == reconstructed_tensor.shape + assert original_tensor.dtype == reconstructed_tensor.dtype + + +def test_serialize_deserialize_tensordict_with_device(): + """Test serialization and deserialization of TensorDict with device information""" + # Create test data + batch_size = (2, 3) + tensor1 = torch.randn(*batch_size, 4) + tensor2 = torch.randint(0, 10, (*batch_size, 2)) + + # Create TensorDict with device information + device = "cuda" if torch.cuda.is_available() else "cpu" + original_tensordict = TensorDict({"tensor1": tensor1, "tensor2": tensor2}, batch_size=batch_size, device=device) + + # Serialize + batch_size_serialized, device_serialized, encoded_items = serialize_tensordict(original_tensordict) + + # Deserialize + reconstructed_tensordict = deserialize_tensordict((batch_size_serialized, device_serialized, encoded_items)) + + # Verify results + assert original_tensordict.batch_size == reconstructed_tensordict.batch_size + assert str(original_tensordict.device) == str(reconstructed_tensordict.device) + assert set(original_tensordict.keys()) == set(reconstructed_tensordict.keys()) + + for key in original_tensordict.keys(): + original_tensor = original_tensordict[key] + reconstructed_tensor = reconstructed_tensordict[key] + + assert torch.allclose(original_tensor.cpu(), reconstructed_tensor.cpu()) + assert original_tensor.shape == reconstructed_tensor.shape + assert original_tensor.dtype == reconstructed_tensor.dtype diff --git a/e2c/verl/tests/test_protocol_v2_on_cpu.py b/e2c/verl/tests/test_protocol_v2_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..20307e7e0e051297beecef2dc28834ad78c8f22d --- /dev/null +++ b/e2c/verl/tests/test_protocol_v2_on_cpu.py @@ -0,0 +1,595 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Replace DataProto with raw TensorDict +""" + +import copy +import random + +import numpy as np +import pytest +import torch + +from verl.utils import tensordict_utils as tu + + +def test_union_tensor_dict(): + obs = torch.randn(100, 10) + + meta_info1 = {"top_p": 0.8} + meta_info2 = {"top_p": 0.9} + data1 = {"obs": obs, "act": torch.randn(100, 3), "data_sources": ["gsm8k"] * 100} + data2 = {"obs": obs, "next_obs": torch.randn(100, 10), "rew": torch.randn(100), "data_sources": ["gsm8k"] * 100} + + data_with_copied_obs = {"obs": obs.clone(), "next_obs": torch.randn(100, 10), "rew": torch.randn(100)} + + data1 = tu.get_tensordict(tensor_dict=data1) + data2 = tu.get_tensordict(tensor_dict=data2) + data_with_copied_obs = tu.get_tensordict(data_with_copied_obs) + + tu.union_tensor_dict(data1, data2) + with pytest.raises(AssertionError): + # conflict in tensor values + tu.union_tensor_dict(data1, data_with_copied_obs) + + data1 = tu.assign_non_tensor_dict(data1, meta_info1) + tu.union_tensor_dict(data1, data2) # works ok + + data2 = tu.assign_non_tensor_dict(data2, meta_info2) + + with pytest.raises(AssertionError): + # conflict in NonTensorData + tu.union_tensor_dict(data1, data2) + + data1.pop("top_p") + data2.pop("top_p") + + data2["data_sources"][0] = "math" + with pytest.raises(AssertionError): + # conflict in NonTensorData + tu.union_tensor_dict(data1, data2) + + +def test_tensor_dict_constructor(): + obs = torch.ones(100, 10) + act = torch.zeros(100, 10, 3) + data_source = ["gsm8k"] * 100 + non_tensor_dict = {"name": "abdce"} + + data = tu.get_tensordict( + tensor_dict={"obs": obs, "act": act, "data_source": data_source}, non_tensor_dict=non_tensor_dict + ) + + assert data.batch_size == torch.Size([100]) + + # test slicing + assert torch.all(torch.eq(data[0]["obs"], torch.ones(10))).item() + assert torch.all(torch.eq(data[0]["act"], torch.zeros(10, 3))).item() + assert data[0]["data_source"] == "gsm8k" + + assert torch.all(torch.eq(data[0:2]["obs"], torch.ones(2, 10))).item() + assert torch.all(torch.eq(data[0:2]["act"], torch.zeros(2, 10, 3))).item() + assert data[0:2]["data_source"] == ["gsm8k"] * 2 + + # test non tensor data + assert data["name"] == "abdce" + + +def test_index_select_tensor_dict(): + vocab_size = 128 + a = torch.randint(low=0, high=vocab_size, size=(11,)) + b = torch.randint(low=0, high=vocab_size, size=(13,)) + c = torch.randint(low=0, high=vocab_size, size=(12,)) + d = torch.randint(low=0, high=vocab_size, size=(15,)) + input_ids = [a, b, c, d] + input_ids = torch.nested.as_nested_tensor(input_ids, layout=torch.jagged) + + padded_tensor = torch.randn(4, 10) + non_tensor_dict = {"global_batch_size": "4"} + + data = tu.get_tensordict( + tensor_dict={ + "input_ids": input_ids, + "padded_tensor": padded_tensor, + }, + non_tensor_dict=non_tensor_dict, + ) + + assert data.batch_size == torch.Size([4]) + + # test index select + indices = torch.tensor([1, 3]) + selected_data = tu.index_select_tensor_dict(data, indices) + + assert selected_data.batch_size == torch.Size([2]) + + target_input_ids = torch.nested.as_nested_tensor([input_ids[idx] for idx in indices], layout=torch.jagged) + target_select_data = tu.get_tensordict( + tensor_dict={ + "input_ids": target_input_ids, + "padded_tensor": padded_tensor[indices], + }, + non_tensor_dict=non_tensor_dict, + ) + tu.assert_tensordict_eq(selected_data, target_select_data) + + +def test_tensordict_with_images(): + # each sample contains a sequence with multiple images of different sizes + vocab_size = 128 + a = torch.randint(low=0, high=vocab_size, size=(11,)) + b = torch.randint(low=0, high=vocab_size, size=(13,)) + input_ids = [a, b] + input_ids = torch.nested.as_nested_tensor(input_ids, layout=torch.jagged) + + # must be numpy + # TODO(vermouth1992). We may use nested tensor too. But this requires nested over nested + a_images = [ + torch.randint(low=0, high=255, size=(3, 256, 256), dtype=torch.uint8).numpy(), + torch.randint(low=0, high=255, size=(3, 128, 128), dtype=torch.uint8).numpy(), + ] + b_images = [ + torch.randint(low=0, high=255, size=(3, 256, 256), dtype=torch.uint8).numpy(), + torch.randint(low=0, high=255, size=(3, 128, 128), dtype=torch.uint8).numpy(), + torch.randint(low=0, high=255, size=(3, 64, 64), dtype=torch.uint8).numpy(), + ] + + images = [a_images, b_images] + + data = tu.get_tensordict({"input_ids": input_ids, "images": images}) + + assert np.all(np.equal(data[0]["images"][0], a_images[0])) + assert torch.all(torch.eq(data[0]["input_ids"], a)) + + +def test_tensordict_with_packing(): + vocab_size = 128 + a = torch.randint(low=0, high=vocab_size, size=(11,)) + b = torch.randint(low=0, high=vocab_size, size=(13,)) + input_ids = [a, b] + input_ids = torch.nested.as_nested_tensor(input_ids, layout=torch.jagged) + + data = tu.get_tensordict({"input_ids": input_ids}) + + # test cu_seqlens + cu_seqlens = torch.tensor([0, 11, 24]) + assert torch.all(torch.eq(cu_seqlens, data["input_ids"].offsets())) + + # test index + assert torch.all(torch.eq(data["input_ids"][0], a)) + assert torch.all(torch.eq(data["input_ids"][1], b)) + + assert torch.all(torch.eq(data[0]["input_ids"], a)) + assert torch.all(torch.eq(data[1]["input_ids"], b)) + + data_lst = data.chunk(2) + + assert torch.all(torch.eq(data_lst[0]["input_ids"][0], a)) + assert torch.all(torch.eq(data_lst[1]["input_ids"][0], b)) + + +def test_tensordict_eq(): + obs = torch.tensor([1, 2, 3, 4, 5, 6]) + data_sources = ["abc", "def", "abc", "def", "pol", "klj"] + non_tensor_dict = {"train_sample_kwargs": {"top_p": 1.0}, "val_sample_kwargs": {"top_p": 0.7}} + data = tu.get_tensordict({"obs": obs, "data_sources": data_sources}, non_tensor_dict=non_tensor_dict) + + obs = torch.tensor([1, 2, 3, 4, 5, 6]) + data_sources = ["abc", "def", "abc", "def", "pol", "klj"] + non_tensor_dict = {"train_sample_kwargs": {"top_p": 1.0}, "val_sample_kwargs": {"top_p": 0.7}} + data1 = tu.get_tensordict({"obs": obs, "data_sources": data_sources}, non_tensor_dict=non_tensor_dict) + + tu.assert_tensordict_eq(data, data1) + + data2 = copy.deepcopy(data1) + data2["obs"][0] += 1 + + with pytest.raises(AssertionError): + tu.assert_tensordict_eq(data, data2) + + data2 = copy.deepcopy(data1) + data2["data_sources"][0] = "math" + + with pytest.raises(AssertionError): + tu.assert_tensordict_eq(data, data2) + + data2 = copy.deepcopy(data1) + data2["train_sample_kwargs"]["top_p"] = 0.9 + + with pytest.raises(AssertionError): + tu.assert_tensordict_eq(data, data2) + + tensor_list = [ + torch.tensor([1, 2, 3, 3, 2]), + torch.tensor([4, 5]), + torch.tensor([7, 8, 10, 14]), + torch.tensor([10, 11, 12]), + torch.tensor([13, 14, 15, 18]), + torch.tensor([16, 17]), + ] + obs = torch.nested.as_nested_tensor(tensor_list, layout=torch.jagged) + data_sources = ["abc", "def", "abc", "def", "pol", "klj"] + non_tensor_dict = {"train_sample_kwargs": {"top_p": 1.0}, "val_sample_kwargs": {"top_p": 0.7}} + data3 = tu.get_tensordict({"obs": obs, "data_sources": data_sources}, non_tensor_dict=non_tensor_dict) + + tensor_list[0] = torch.tensor([1, 2, 3, 3, 2]) + obs = torch.nested.as_nested_tensor(tensor_list, layout=torch.jagged) + data4 = tu.get_tensordict({"obs": obs, "data_sources": data_sources}, non_tensor_dict=non_tensor_dict) + tu.assert_tensordict_eq(data3, data4) + + tensor_list[0] = torch.tensor([1, 2, 4]) + obs = torch.nested.as_nested_tensor(tensor_list, layout=torch.jagged) + data5 = tu.get_tensordict({"obs": obs, "data_sources": data_sources}, non_tensor_dict=non_tensor_dict) + with pytest.raises(AssertionError): + tu.assert_tensordict_eq(data3, data5) + + tensor_list[0] = torch.tensor([4, 5]) + tensor_list[1] = torch.tensor([1, 2, 3, 3, 2]) + obs = torch.nested.as_nested_tensor(tensor_list, layout=torch.jagged) + data6 = tu.get_tensordict({"obs": obs, "data_sources": data_sources}, non_tensor_dict=non_tensor_dict) + with pytest.raises(AssertionError): + tu.assert_tensordict_eq(data3, data6) + + +def test_tensor_dict_make_iterator(): + obs = torch.tensor([1, 2, 3, 4, 5, 6]) + data_sources = ["abc", "def", "abc", "def", "pol", "klj"] + non_tensor_dict = {"train_sample_kwargs": {"top_p": 1.0}, "val_sample_kwargs": {"top_p": 0.7}} + dataset = tu.get_tensordict({"obs": obs, "data_sources": data_sources}, non_tensor_dict=non_tensor_dict) + + dataloader = tu.make_iterator( + dataset, mini_batch_size=2, epochs=2, seed=0, dataloader_kwargs={"shuffle": False, "drop_last": False} + ) + + expected_tensor_dict = [dataset[0:2], dataset[2:4], dataset[4:6], dataset[0:2], dataset[2:4], dataset[4:6]] + + i = 0 + + for d in dataloader: + tu.assert_tensordict_eq(d, expected_tensor_dict[i]) + i += 1 + + data_iter_1 = tu.make_iterator(dataset, mini_batch_size=3, epochs=1, seed=1, dataloader_kwargs={"shuffle": True}) + data_list_1 = [] + for data in data_iter_1: + data_list_1.append(data) + + data_iter_2 = tu.make_iterator(dataset, mini_batch_size=3, epochs=1, seed=1, dataloader_kwargs={"shuffle": True}) + data_list_2 = [] + for data in data_iter_2: + data_list_2.append(data) + + for data1, data2 in zip(data_list_1, data_list_2, strict=True): + tu.assert_tensordict_eq(data1, data2) + + +def test_reorder(): + obs = torch.tensor([1, 2, 3, 4, 5, 6]) + labels = ["a", "b", "c", "d", "e", "f"] + non_tensor_dict = {"name": "abdce"} + + data = tu.get_tensordict(tensor_dict={"obs": obs, "labels": labels}, non_tensor_dict=non_tensor_dict) + data = data[torch.tensor([3, 4, 2, 0, 1, 5])] + + assert torch.all(torch.eq(data["obs"], torch.tensor([4, 5, 3, 1, 2, 6]))) + assert np.all(data["labels"] == np.array(["d", "e", "c", "a", "b", "f"])) + assert data["name"] == "abdce" + + +def test_chunk_concat(): + obs = torch.tensor([1, 2, 3, 4, 5, 6]) + labels = ["a", "b", "c", "d", "e", "f"] + data = tu.get_tensordict({"obs": obs, "labels": labels}, non_tensor_dict={"name": "abcde"}) + + data_split = data.tensor_split(indices_or_sections=5, dim=0) + + expected_idx_lst = [[0, 1], [2], [3], [4], [5]] + + for d, expected_idx in zip(data_split, expected_idx_lst, strict=False): + tu.assert_tensordict_eq(d, data[expected_idx]) + + data_split = data.chunk(2) + assert len(data_split) == 2 + assert torch.all(torch.eq(data_split[0]["obs"], torch.tensor([1, 2, 3]))) + assert np.all(data_split[0]["labels"] == np.array(["a", "b", "c"])) + assert data_split[0]["name"] == "abcde" + + assert torch.all(torch.eq(data_split[1]["obs"], torch.tensor([4, 5, 6]))) + assert np.all(data_split[1]["labels"] == np.array(["d", "e", "f"])) + assert data_split[1]["name"] == "abcde" + + concat_data = torch.cat(data_split, dim=0) + assert torch.all(torch.eq(concat_data["obs"], data["obs"])) + assert np.all(concat_data["labels"] == data["labels"]) + assert concat_data["name"] == data["name"] + + +def test_pop(): + obs = torch.randn(100, 10) + act = torch.randn(100, 3) + dataset = tu.get_tensordict({"obs": obs, "act": act}, non_tensor_dict={"2": 2, "1": 1}) + + poped_dataset = tu.pop(dataset, keys=["obs", "2"]) + + assert poped_dataset.batch_size[0] == 100 + + assert poped_dataset.keys() == {"obs", "2"} + + assert dataset.keys() == {"act", "1"} + + +def test_repeat(): + # Create a DataProto object with some batch and non-tensor data + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + data = tu.get_tensordict({"obs": obs, "labels": labels}, non_tensor_dict={"info": "test_info"}) + + # Test interleave=True + repeated_data_interleave = data.repeat_interleave(repeats=2) + expected_obs_interleave = torch.tensor([[1, 2], [1, 2], [3, 4], [3, 4], [5, 6], [5, 6]]) + expected_labels_interleave = ["a", "a", "b", "b", "c", "c"] + + assert torch.all(torch.eq(repeated_data_interleave["obs"], expected_obs_interleave)) + assert repeated_data_interleave["labels"] == expected_labels_interleave + assert repeated_data_interleave["info"] == "test_info" + + # Test interleave=False + repeated_data_no_interleave = data.repeat(2) + expected_obs_no_interleave = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2], [3, 4], [5, 6]]) + expected_labels_no_interleave = ["a", "b", "c", "a", "b", "c"] + + assert torch.all(torch.eq(repeated_data_no_interleave["obs"], expected_obs_no_interleave)) + assert repeated_data_no_interleave["labels"] == expected_labels_no_interleave + assert repeated_data_no_interleave["info"] == "test_info" + + +def test_dataproto_pad_unpad(): + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + data = tu.get_tensordict(tensor_dict={"obs": obs, "labels": labels}, non_tensor_dict={"info": "test_info"}) + + padded_data, pad_size = tu.pad_to_divisor(data, size_divisor=2) + + assert pad_size == 1 + + expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2]]) + expected_labels = ["a", "b", "c", "a"] + + assert torch.all(torch.eq(padded_data["obs"], expected_obs)) + assert padded_data["labels"] == expected_labels + assert padded_data["info"] == "test_info" + + unpadd_data = tu.unpad(padded_data, pad_size=pad_size) + assert torch.all(torch.eq(unpadd_data["obs"], obs)) + assert unpadd_data["labels"] == labels + assert unpadd_data["info"] == "test_info" + + padded_data, pad_size = tu.pad_to_divisor(data, size_divisor=3) + assert pad_size == 0 + + expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + expected_labels = ["a", "b", "c"] + + assert torch.all(torch.eq(padded_data["obs"], expected_obs)) + assert padded_data["labels"] == expected_labels + assert padded_data["info"] == "test_info" + + unpadd_data = tu.unpad(padded_data, pad_size=pad_size) + assert torch.all(torch.eq(unpadd_data["obs"], obs)) + assert unpadd_data["labels"] == labels + assert unpadd_data["info"] == "test_info" + + padded_data, pad_size = tu.pad_to_divisor(data, size_divisor=7) + assert pad_size == 4 + + expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2], [3, 4], [5, 6], [1, 2]]) + expected_labels = ["a", "b", "c", "a", "b", "c", "a"] + assert torch.all(torch.eq(padded_data["obs"], expected_obs)) + assert padded_data["labels"] == expected_labels + assert padded_data["info"] == "test_info" + + unpadd_data = tu.unpad(padded_data, pad_size=pad_size) + assert torch.all(torch.eq(unpadd_data["obs"], obs)) + assert unpadd_data["labels"] == labels + assert unpadd_data["info"] == "test_info" + + +def test_torch_save_data_proto(): + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + data = tu.get_tensordict({"obs": obs, "labels": labels}, non_tensor_dict={"info": "test_info"}) + + filename = "test_data.pt" + torch.save(data, filename) + loaded_data = torch.load(filename, weights_only=False) + + assert torch.all(torch.eq(loaded_data["obs"], data["obs"])) + assert loaded_data["labels"] == data["labels"] + assert loaded_data["info"] == data["info"] + + import os + + os.remove(filename) + + +def test_len(): + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = np.array(["a", "b", "c"], dtype=object) + + data = tu.get_tensordict({"obs": obs, "labels": labels.tolist()}, non_tensor_dict={"info": "test_info"}) + assert len(data) == 3 + + data = tu.get_tensordict({"labels": labels.tolist()}, non_tensor_dict={"info": "test_info"}) + assert len(data) == 3 + + data_item = data[0] + assert len(data_item) == 0 + + data = tu.get_tensordict({}, non_tensor_dict={"info": "test_info"}) + assert len(data) == 0 + + +def test_dataproto_index(): + data_len = 100 + idx_num = 10 + + obs = torch.randn(data_len, 10) + labels = [random.choice(["abc", "cde"]) for _ in range(data_len)] + + data = tu.get_tensordict({"obs": obs, "labels": labels}) + + labels_np = np.array(labels) + + idx_np_int = np.random.randint(0, data_len, size=(idx_num,)) + result_np_int = data[idx_np_int] + assert result_np_int.keys() == data.keys() + assert result_np_int["obs"].shape[0] == idx_num + assert len(result_np_int["labels"]) == idx_num + assert np.array_equal(result_np_int["obs"].cpu().numpy(), obs[idx_np_int].numpy()) + assert np.array_equal(result_np_int["labels"], labels_np[idx_np_int]) + + idx_torch_int = torch.randint(0, data_len, size=(idx_num,)) + result_torch_int = data[idx_torch_int] + assert result_torch_int.keys() == data.keys() + assert result_torch_int["obs"].shape[0] == idx_num + assert len(result_torch_int["labels"]) == idx_num + assert np.array_equal(result_torch_int["obs"].cpu().numpy(), obs[idx_torch_int].cpu().numpy()) + assert np.array_equal(result_torch_int["labels"], labels_np[idx_torch_int.cpu().numpy()]) + + idx_list_int = [np.random.randint(0, data_len) for _ in range(idx_num)] + result_list_int = data[idx_list_int] + assert result_list_int.keys() == data.keys() + assert result_list_int["obs"].shape[0] == idx_num + assert len(result_list_int["labels"]) == idx_num + assert np.array_equal(result_list_int["obs"].cpu().numpy(), obs[idx_list_int].cpu().numpy()) + assert np.array_equal(result_list_int["labels"], labels_np[idx_list_int]) + + # idx_np_bool = np.random.randint(0, 2, size=(data_len,), dtype=bool) + # result_np_bool = data[idx_np_bool] + # assert result_np_bool.keys() == data.keys() + # assert result_np_bool["obs"].shape[0] == idx_np_bool.sum() + # assert len(result_np_bool["labels"]) == idx_np_bool.sum() + # assert np.array_equal(result_np_bool["obs"].cpu().numpy(), obs[idx_np_bool].cpu().numpy()) + # assert np.array_equal(result_np_bool["labels"], labels_np[idx_np_bool]) + + idx_torch_bool = torch.randint(0, 2, size=(data_len,), dtype=torch.bool) + result_torch_bool = data[idx_torch_bool] + assert result_torch_bool.keys() == data.keys() + assert result_torch_bool["obs"].shape[0] == idx_torch_bool.sum().item() + assert len(result_torch_bool["labels"]) == idx_torch_bool.sum().item() + assert np.array_equal(result_torch_bool["obs"].cpu().numpy(), obs[idx_torch_bool].cpu().numpy()) + assert np.array_equal(result_torch_bool["labels"], labels_np[idx_torch_bool]) + + # idx_list_bool = [np.random.randint(0, 2, dtype=bool) for _ in range(data_len)] + # result_list_bool = data[idx_list_bool] + # assert result_list_bool.keys() == data.keys() + # assert result_list_bool["obs"].shape[0] == sum(idx_list_bool) + # assert len(result_list_bool["labels"]) == sum(idx_list_bool) + # assert np.array_equal(result_list_bool["obs"].cpu().numpy(), obs[idx_list_bool].cpu().numpy()) + # assert np.array_equal(result_list_bool["labels"], labels_np[idx_list_bool]) + + +def test_select(): + obs = torch.randn(100, 10) + act = torch.randn(100, 3) + dataset = tu.get_tensordict({"obs": obs, "act": act}, non_tensor_dict={"2": 2, "1": 1}) + + subset = dataset.select("obs", "2") + + assert torch.all(torch.eq(subset["obs"], dataset["obs"])) + assert subset["2"] == dataset["2"] + assert "act" not in subset.keys() + assert "1" not in subset.keys() + + +def test_dataproto_no_batch(): + labels = ["a", "b", "c"] + data = tu.get_tensordict(tensor_dict={"labels": labels}, non_tensor_dict={"info": "test_info"}) + selected = data.select("labels") + + assert selected["labels"] == labels + pop_data = tu.pop(data, keys=["labels"]) + assert pop_data["labels"] == labels + assert "labels" not in data + + +def test_sample_level_repeat(): + # Create a DataProto object with some batch and non-tensor data + obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) + labels = ["a", "b", "c"] + + data = tu.get_tensordict({"obs": obs, "labels": labels}, non_tensor_dict={"info": "test_info"}) + + # list + repeated_data_interleave = data.repeat_interleave(repeats=torch.tensor([3, 1, 2])) + expected_obs_interleave = torch.tensor([[1, 2], [1, 2], [1, 2], [3, 4], [5, 6], [5, 6]]) + expected_labels_interleave = ["a", "a", "a", "b", "c", "c"] + + assert torch.all(torch.eq(repeated_data_interleave["obs"], expected_obs_interleave)) + assert repeated_data_interleave["labels"] == expected_labels_interleave + assert repeated_data_interleave["info"] == "test_info" + + # torch.tensor + repeated_data_no_interleave = data.repeat_interleave(repeats=torch.tensor([1, 2, 3])) + expected_obs_no_interleave = torch.tensor([[1, 2], [3, 4], [3, 4], [5, 6], [5, 6], [5, 6]]) + expected_labels_no_interleave = ["a", "b", "b", "c", "c", "c"] + + assert torch.all(torch.eq(repeated_data_no_interleave["obs"], expected_obs_no_interleave)) + assert repeated_data_no_interleave["labels"] == expected_labels_no_interleave + assert repeated_data_no_interleave["info"] == "test_info" + + +def test_dataproto_chunk_after_index(): + data_len = 4 + obs = torch.randn(data_len, 4) + labels = [f"label_{i}" for i in range(data_len)] + + data = tu.get_tensordict(tensor_dict={"obs": obs, "labels": labels}, non_tensor_dict={"name": "abc"}) + # Test with boolean numpy array + bool_mask = torch.tensor([True, False, True, False]) + selected = data[bool_mask] + assert isinstance(selected.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch_size) # int or List[int] + + # Test with integer numpy array + int_mask = torch.tensor([0, 2]) + selected = data[int_mask] + assert isinstance(selected.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch_size) + + # Test with boolean list + list_mask = [True, False, True, False] + selected = data[list_mask] + assert isinstance(selected.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch_size) + + # Test with list + list_mask = [0, 2] + selected = data[list_mask] + assert isinstance(selected.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch_size) + + # Test with torch tensor (bool) + torch_bool_mask = torch.tensor([True, False, True, False]) + selected = data[torch_bool_mask] + assert isinstance(selected.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch_size) + + # Test with torch tensor (int) + torch_int_mask = torch.tensor([0, 2]) + selected = data[torch_int_mask] + assert isinstance(selected.batch_size, torch.Size) + assert all(isinstance(d, int) for d in selected.batch_size) diff --git a/e2c/verl/tests/trainer/__init__.py b/e2c/verl/tests/trainer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6f79d474d156e16ae54bb3d0c8f9ae7d0e16946e --- /dev/null +++ b/e2c/verl/tests/trainer/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Tests for the trainer module. +""" diff --git a/e2c/verl/tests/trainer/config/__init__.py b/e2c/verl/tests/trainer/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/tests/trainer/config/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/tests/trainer/config/legacy_ppo_megatron_trainer.yaml b/e2c/verl/tests/trainer/config/legacy_ppo_megatron_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1463ba4c2395e223d0cca180a1a5e56f07f13ff9 --- /dev/null +++ b/e2c/verl/tests/trainer/config/legacy_ppo_megatron_trainer.yaml @@ -0,0 +1,463 @@ +data: + tokenizer: null + train_files: ~/data/rlhf/gsm8k/train.parquet + val_files: ~/data/rlhf/gsm8k/test.parquet + prompt_key: prompt + reward_fn_key: data_source + max_prompt_length: 512 + max_response_length: 512 + train_batch_size: 1024 + val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves + return_raw_input_ids: False # This should be set to true when the tokenizer between policy and rm differs + return_raw_chat: False + return_full_prompt: False + shuffle: True + filter_overlong_prompts: False # for large-scale dataset, filtering overlong prompts could be timeconsuming. You cat set the filter_overlong_prompts_workers to use multiprocessing to speed up. + filter_overlong_prompts_workers: 1 + truncation: error + trust_remote_code: False # main_ppo will check this config to determine whether to use remote code for tokenizer + custom_cls: + path: null + name: null + sampler: + class_path: null + class_name: null + dataloader_num_workers: 8 + return_multi_modal_inputs: True + +actor_rollout_ref: + hybrid_engine: True + nccl_timeout: 600 # seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using megatron + model: + path: ~/models/deepseek-llm-7b-chat + custom_chat_template: null + external_lib: null + override_config: + model_config: {} + moe_config: + freeze_moe_router: False + enable_gradient_checkpointing: False + gradient_checkpointing_kwargs: + ## Activation Checkpointing + activations_checkpoint_method: null # 'uniform', 'block'; not used with 'selective' + # 'uniform' divides the total number of transformer layers and checkpoints the input activation of each chunk + # 'block' checkpoints the specified number of layers per pipeline stage at the specified granularity + activations_checkpoint_granularity: null # 'selective' or 'full' + # 'full' will checkpoint the entire transformer layer and 'selective' only checkpoints memory intensive part of attention + activations_checkpoint_num_layers: null # not used with 'selective' + trust_remote_code: False + actor: + strategy: megatron # This is for backward-compatibility + ppo_mini_batch_size: 256 + ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu + ppo_micro_batch_size_per_gpu: null + use_dynamic_bsz: False + ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length} + use_torch_compile: True # False to disable torch compile + # pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high) + clip_ratio: 0.2 # default value if clip_ratio_low and clip_ratio_high are not specified + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + clip_ratio_c: 3.0 # lower bound of the value for Dual-clip PPO from https://arxiv.org/pdf/1912.09729 + loss_agg_mode: "token-mean" # / "seq-mean-token-sum" / "seq-mean-token-mean" + # NOTE: "token-mean" is the default behavior + entropy_coeff: 0 + use_kl_loss: False # True for GRPO + kl_loss_coef: 0.001 # for grpo + kl_loss_type: low_var_kl # for grpo + ppo_epochs: 1 + data_loader_seed: null + shuffle: False + policy_loss: # policy loss config + loss_mode: "vanilla" # Loss function mode: vanilla / clip-cov / kl-cov / gpg from https://arxiv.org/abs/2505.22617, + clip_cov_ratio: 0.0002 # Ratio of tokens to be clipped for clip-cov loss + clip_cov_lb: 1.0 # Lower bound for clip-cov loss + clip_cov_ub: 5.0 # Upper bound for clip-cov loss + kl_cov_ratio: 0.0002 # Ratio of tokens to be applied kl penalty for kl-cov loss + ppo_kl_coef: 0.1 # KL divergence penalty coefficient + optim: + optimizer: adam + lr: 1e-6 + clip_grad: 1.0 + total_training_steps: -1 # must be override by program + lr_warmup_init: 0.0 # initial learning rate for warmup, default to 0.0 + lr_warmup_steps: null # Prioritized. None, 0 or Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime + lr_decay_steps: null + lr_decay_style: constant # select from constant/linear/cosine/inverse_square_root + min_lr: 0.0 # minimum learning rate, default to 0.0 + weight_decay: 0.01 + weight_decay_incr_style: constant # select from constant/linear/cosine + lr_wsd_decay_style: exponential # select from constant/exponential/cosine + lr_wsd_decay_steps: null + use_checkpoint_opt_param_scheduler: False # use checkpoint optimizer parameter scheduler + megatron: + param_offload: False + grad_offload: False + optimizer_offload: False + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: null + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests + context_parallel_size: 1 + sequence_parallel: True + use_distributed_optimizer: True + use_dist_checkpointing: False + dist_checkpointing_path: null + seed: 42 + override_transformer_config: {} # additional transformer config like: num_layers_in_first(/last)_pipeline_stage + use_mbridge: False + profile: # profile the actor model in `update_policy` + use_profile: False # open it when you want to profile the actor model + profile_ranks: null # list, you can specify the ranks to profile + step_start: -1 # start step in update_policy + step_end: -1 # end step + save_path: null # the path to save the profile result + load_weight: True + checkpoint: + async_save: False # save checkpoint asynchronously + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + # For more flexibility, you can specify the contents to load from the checkpoint. + load_contents: ${actor_rollout_ref.actor.checkpoint.save_contents} + ref: + strategy: ${actor_rollout_ref.actor.strategy} + use_torch_compile: ${actor_rollout_ref.actor.use_torch_compile} + megatron: + param_offload: False + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: null + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests + context_parallel_size: 1 + sequence_parallel: True + use_distributed_optimizer: True + use_dist_checkpointing: False + dist_checkpointing_path: null + seed: ${actor_rollout_ref.actor.megatron.seed} + override_transformer_config: ${actor_rollout_ref.actor.megatron.override_transformer_config} + use_mbridge: ${actor_rollout_ref.actor.megatron.use_mbridge} + profile: + use_profile: False + profile_ranks: null + step_start: -1 + step_end: -1 + save_path: null + load_weight: True + log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} + rollout: + name: vllm + mode: sync # sync: LLM, async: AsyncLLM + temperature: 1.0 + top_k: -1 # 0 for hf rollout, -1 for vllm rollout + top_p: 1 + prompt_length: ${data.max_prompt_length} # for xperf_gpt + response_length: ${data.max_response_length} + # for vllm rollout + dtype: bfloat16 # should align with FSDP + gpu_memory_utilization: 0.5 + ignore_eos: False + enforce_eager: False + free_cache_engine: True + load_format: dummy + tensor_model_parallel_size: 2 + max_num_batched_tokens: 8192 + max_model_len: null + max_num_seqs: 1024 + log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} + disable_log_stats: True + enable_chunked_prefill: True # could get higher throughput + # for hf rollout + do_sample: True + layer_name_map: + qkv_layer_name: qkv + gate_proj_layer_name: gate_up + # number of responses (i.e. num sample times) + n: 1 + engine_kwargs: # inference engine parameters, please refer vllm/sglang official doc for detail + vllm: {} + sglang: {} + val_kwargs: + # sampling parameters for validation + top_k: -1 # 0 for hf rollout, -1 for vllm rollout + top_p: 1.0 + temperature: 0 + n: 1 + do_sample: False # default eager for validation + + # Multi-turn interaction config for tools or chat. + multi_turn: + # set to True for multi-turn tool interaction tasks; should set rollout.name to sglang as well + enable: False + + # null for no limit (default max_length // 3) + max_assistant_turns: null + + # null for no tool + tool_config_path: null + + # null for no limit (default max_length // 3) + max_user_turns: null + + # max parallel call for tools in single turn + max_parallel_calls: 1 + + # max length of tool response + max_tool_response_length: 256 + + # truncate side of tool response: left, middle, right + tool_response_truncate_side: middle + + # null for no interaction + interaction_config_path: null + + # - When set to True, the model's default chat template is used for multi-turn rollout, which typically matches production behavior. + # - When set to False, the token ids recorded for training are used instead; unlike the default chat template, these always include the model's full output, + # which may contain additional content such as reasoning content. This maintains the consistency between training and rollout, but it will lead to longer prompts. + use_inference_chat_template: False + + # Tokenization is performed turn by turn and the resulting token ids are concatenated to form the full conversation. + # To ensure this matches the result of tokenizing the entire conversation at once, a sanity check is run at the end of each multi-turn rollout to compare the two sets of token ids. + # Some models are known to produce different tokenization results when tokenizing turn by turn vs. all at once. aThis behavior has already been validated for them. + # To reduce excessive warnings, you can turn off the sanity check for these models if you are using their default chat template: + # Qwen/QwQ-32B, Qwen/Qwen3-xxB + # - disable: disable tokenization sanity check + # - strict: enable strict tokenization sanity check (default) + # - ignore_strippable: ignore strippable tokens when checking tokenization sanity + tokenization_sanity_check_mode: strict + + # Format of the multi-turn interaction. Options: hermes, llama3_json, ... + format: hermes + + # [Experimental] agent loop based rollout configs + agent: + + # Number of agent loop workers + num_workers: 8 + + custom_async_server: + path: null + name: null + + # support logging rollout prob for debugging purpose + calculate_log_probs: False + # Nsight system profiler configs + profiler: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + discrete: False + all_ranks: False + ranks: [] + +critic: + rollout_n: ${actor_rollout_ref.rollout.n} + strategy: ${actor_rollout_ref.actor.strategy} + nccl_timeout: 600 # seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using megatron + optim: + optimizer: adam + lr: 1e-6 + clip_grad: 1.0 + total_training_steps: -1 # must be override by program + lr_warmup_init: 0.0 # initial learning rate for warmup, default to 0.0 + lr_warmup_steps: null # Prioritized. None, 0 or Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime + lr_decay_steps: null + lr_decay_style: constant # select from constant/linear/cosine/inverse_square_root + min_lr: 0.0 # minimum learning rate, default to 0.0 + weight_decay: 0.01 + weight_decay_incr_style: constant # select from constant/linear/cosine + lr_wsd_decay_style: exponential # select from constant/exponential/cosine + lr_wsd_decay_steps: null + use_checkpoint_opt_param_scheduler: False # use checkpoint optimizer parameter scheduler + model: + path: ~/models/deepseek-llm-7b-chat + tokenizer_path: ${actor_rollout_ref.model.path} + override_config: + model_config: {} + moe_config: + freeze_moe_router: False + external_lib: ${actor_rollout_ref.model.external_lib} + trust_remote_code: False + enable_gradient_checkpointing: False + gradient_checkpointing_kwargs: + ## Activation Checkpointing + activations_checkpoint_method: null + activations_checkpoint_granularity: null + activations_checkpoint_num_layers: null + megatron: + param_offload: False + grad_offload: False + optimizer_offload: False + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: null + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests + context_parallel_size: 1 + sequence_parallel: True + use_distributed_optimizer: True + use_dist_checkpointing: False + dist_checkpointing_path: null + seed: ${actor_rollout_ref.actor.megatron.seed} + override_transformer_config: ${actor_rollout_ref.actor.megatron.override_transformer_config} + use_mbridge: ${actor_rollout_ref.actor.megatron.use_mbridge} + load_weight: True + ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} + ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu + ppo_micro_batch_size_per_gpu: null + use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + ppo_max_token_len_per_gpu: 32768 # (${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}) * 2 + forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu} + ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs} + data_loader_seed: ${actor_rollout_ref.actor.data_loader_seed} + shuffle: ${actor_rollout_ref.actor.shuffle} + cliprange_value: 0.5 + loss_agg_mode: ${actor_rollout_ref.actor.loss_agg_mode} + checkpoint: + async_save: False # save checkpoint asynchronously + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + load_contents: ${critic.checkpoint.save_contents} + # Nsight system profiler configs + profiler: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + discrete: False + all_ranks: False + ranks: [] +reward_model: + enable: False + strategy: ${actor_rollout_ref.actor.strategy} + nccl_timeout: 600 # seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using megatron + megatron: + param_offload: False + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: null + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests + context_parallel_size: 1 + sequence_parallel: True + use_distributed_optimizer: False + use_dist_checkpointing: False + dist_checkpointing_path: null + seed: ${actor_rollout_ref.actor.megatron.seed} + override_transformer_config: {} + use_mbridge: ${actor_rollout_ref.actor.megatron.use_mbridge} + model: + input_tokenizer: ${actor_rollout_ref.model.path} # set this to null if the chat template is identical + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + trust_remote_code: False + external_lib: ${actor_rollout_ref.model.external_lib} + load_weight: True + micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu + micro_batch_size_per_gpu: null + use_dynamic_bsz: ${critic.use_dynamic_bsz} + forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + max_length: null + reward_manager: naive + launch_reward_fn_async: False # custom reward function executed async on CPU, during log_prob + sandbox_fusion: + url: null # faas url to run code in cloud sandbox + max_concurrent: 64 # max concurrent requests to sandbox + memory_limit_mb: 1024 # Max memory limit for each sandbox process in MB + # Nsight system profiler configs + profiler: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + discrete: False + all_ranks: False + ranks: [] + +custom_reward_function: + path: null + name: compute_score + +algorithm: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.AlgoConfig + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + norm_adv_by_std_in_grpo: True + use_kl_in_reward: False + kl_penalty: kl # how to estimate kl divergence + kl_ctrl: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.KLControlConfig + type: fixed + kl_coef: 0.001 + horizon: 10000 + target_kl: 0.1 + use_pf_ppo: False + pf_ppo: + reweight_method: pow # ["pow", "max_min", "max_random"] + weight_pow: 2.0 + +trainer: + balance_batch: True + total_epochs: 30 + total_training_steps: null + profile_steps: null # [1,2,5] or [] or null + project_name: verl_examples + experiment_name: gsm8k + logger: ['console', 'wandb'] + log_val_generations: 0 + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + esi_redundant_time: 0 + + # auto: find the last ckpt to resume. If can't find, start from scratch + resume_mode: auto # or disable or resume_path if resume_from_path is set + resume_from_path: null + del_local_ckpt_after_load: False + val_before_train: True + test_freq: -1 + critic_warmup: 0 + default_hdfs_dir: null + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + max_actor_ckpt_to_keep: null + max_critic_ckpt_to_keep: null + # The timeout for ray worker group to wait for the register center to be ready + ray_wait_register_center_timeout: 300 + device: cuda + # see ppo_trainer.yaml for more details + controller_nsight_options: + trace: "cuda,nvtx,cublas,ucx" + cuda-memory-usage: "true" + cuda-graph-trace: "graph" + worker_nsight_options: + trace: "cuda,nvtx,cublas,ucx" + cuda-memory-usage: "true" + cuda-graph-trace: "graph" + capture-range: "cudaProfilerApi" + capture-range-end: null + kill: none + npu_profile: + options: + save_path: ./profiler_data + roles: ["all"] + level: level1 + with_memory: False + record_shapes: False + with_npu: True + with_cpu: True + with_module: False + with_stack: False + analysis: True + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. + timeline_json_file: null diff --git a/e2c/verl/tests/trainer/config/legacy_ppo_trainer.yaml b/e2c/verl/tests/trainer/config/legacy_ppo_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85ef98bb8e335722eb80bc9bb51986d4683b48ca --- /dev/null +++ b/e2c/verl/tests/trainer/config/legacy_ppo_trainer.yaml @@ -0,0 +1,1108 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# dataset config +data: + + # Tokenizer class or path. If null, it will be inferred from the model. + tokenizer: null + + # Whether to use shared memory for data loading. + use_shm: False + + # Training set parquet. Can be a list or a single file. + # The program will read all files into memory, so it can't be too large (< 100GB). + # The path can be either a local path or an HDFS path. + # For HDFS path, we provide utils to download it to DRAM and convert it to a local path. + train_files: ~/data/rlhf/gsm8k/train.parquet + + # Validation parquet. Can be a list or a single file. + val_files: ~/data/rlhf/gsm8k/test.parquet + + # The field in the dataset where the prompt is located. Default is 'prompt'. + prompt_key: prompt + + # The field used to select the reward function (if using different ones per example). + reward_fn_key: data_source + + # Maximum prompt length. All prompts will be left-padded to this length. + # An error will be reported if the length is too long. + max_prompt_length: 512 + + # Maximum response length. Rollout in RL algorithms (e.g. PPO) generates up to this length. + max_response_length: 512 + + # Batch size sampled for one training iteration of different RL algorithms. + train_batch_size: 1024 + + # Batch size used during validation. Can be null. + val_batch_size: null + + # Whether to return the original input_ids without adding chat template. + # This is used when the reward model's chat template differs from the policy. + # If using a model-based RM with different templates, this should be True. + return_raw_input_ids: False + + # Whether to return the original chat (prompt) without applying chat template. + return_raw_chat: False + + # Whether to return the full prompt with chat template. + return_full_prompt: False + + # Whether to shuffle the data in the dataloader. + shuffle: True + + # num dataloader workers + dataloader_num_workers: 8 + + # Whether to shuffle the validation set. + validation_shuffle: False + + # Whether to filter overlong prompts. + filter_overlong_prompts: False + + # Number of workers for filtering overlong prompts. + # For large-scale datasets, filtering can be time-consuming. + # Use multiprocessing to speed up. Default is 1. + filter_overlong_prompts_workers: 1 + + # Truncate the input_ids or prompt if they exceed max_prompt_length. + # Options: 'error', 'left', or 'right'. Default is 'error'. + truncation: error + + # The field in the multi-modal dataset where the image is located. Default is 'images'. + image_key: images + + # The field in the multi-modal dataset where the video is located. + video_key: videos + + # If the remote tokenizer has a Python file, this flag determines whether to allow using it. + trust_remote_code: False + + # Optional: specify a custom dataset class path and name if overriding default loading behavior. + custom_cls: + + # The path to the file containing your customized dataset class. If not specified, pre-implemented dataset will be used. + path: null + + # The name of the dataset class within the specified file. + name: null + + # Whether to return multi-modal inputs in the dataset. Set to False if rollout generates new multi-modal inputs. + return_multi_modal_inputs: True + + # Data generation configuration for augmenting the dataset. + datagen: + + # The path to the file containing your customized data generation class. + # E.g. 'pkg://verl.experimental.dynamic_dataset.dynamicgen_dataset' + path: null + + # The class name of the data generation class within the specified file. + # E.g. 'MockDataGenerator' + name: null + + # settings related to data sampler + sampler: + + # the path to the module containing a curriculum class which implements the + # AbstractSampler interface + class_path: null + + # the name of the curriculum class like `MySampler` + class_name: null + + # Additional kwargs when calling tokenizer.apply_chat_template + apply_chat_template_kwargs: {} + +# config for actor, rollout and reference model +actor_rollout_ref: + + # Whether it's a hybrid engine, currently only supports hybrid engine + hybrid_engine: true + + # common configs for the model + model: + + _target_: verl.workers.config.HFModelConfig + + # Huggingface model path. This can be either local path or HDFS path. + path: ~/models/deepseek-llm-7b-chat + + # Custom chat template for the model. + custom_chat_template: null + + # Whether to use shared memory (SHM) for accelerating the loading of model weights + use_shm: false + + # Additional Python packages to register huggingface models/tokenizers. + external_lib: null + + # Used to override model's original configurations, mainly dropout + override_config: {} + + # Enable gradient checkpointing for actor + enable_gradient_checkpointing: true + + # Enable activation offloading for actor + enable_activation_offload: false + + # Whether to remove padding tokens in inputs during training + use_remove_padding: false + + # Set to positive value to enable LoRA (e.g., 32) + lora_rank: 0 + + # LoRA scaling factor + lora_alpha: 16 + + # Target modules to apply LoRA. Options: "all-linear" (not recommended for VLMs) or + # [q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj] + target_modules: all-linear + + # Exclude modules from applying Lora. Similar usage to target_modules and Peft. + # Example: '.*visual.*' for excluding the ViT in Qwen2.5-VL, as currently vllm does not support ViT Lora. + exclude_modules: null + + # Whether to use Liger for linear layer fusion + use_liger: false + + # Whether to use custom fused kernels (e.g., FlashAttention, fused MLP) + use_fused_kernels: false + + # Options for fused kernels. If use_fused_kernels is true, this will be used. + fused_kernel_options: + + # Implementation backend for fused kernels. Options: "triton" or "torch". + impl_backend: torch + + # Whether to enable loading a remote code model + trust_remote_code: false + + # actor configs + actor: + + # fsdp, fsdp2 or megatron. fsdp backend used here. + strategy: fsdp + + # Split each sample into sub-batches of this size for PPO + ppo_mini_batch_size: 256 + + # [Deprecated] Global micro batch size + ppo_micro_batch_size: null + + # Local per-GPU micro batch size + ppo_micro_batch_size_per_gpu: null + + # Whether to automatically adjust batch size at runtime + use_dynamic_bsz: false + + # Max tokens per GPU in one PPO batch; affects gradient accumulation + # Typically it should be: n * ${data.max_prompt_length} + ${data.max_response_length} + ppo_max_token_len_per_gpu: 16384 + + # Gradient clipping for actor updates + grad_clip: 1.0 + + # PPO clip ratio + clip_ratio: 0.2 + + # Lower bound for asymmetric clipping (used in dual-clip PPO) + clip_ratio_low: 0.2 + + # Upper bound for asymmetric clipping (used in dual-clip PPO) + clip_ratio_high: 0.2 + + # policy loss config + policy_loss: + + # Loss function mode: vanilla / clip-cov / kl-cov /gpg from https://arxiv.org/abs/2505.22617 + loss_mode: "vanilla" + + # Ratio of tokens to be clipped for clip-cov loss + clip_cov_ratio: 0.0002 + + # Lower bound for clip-cov loss + clip_cov_lb: 1.0 + + # Upper bound for clip-cov loss + clip_cov_ub: 5.0 + + # Ratio of tokens to be applied kl penalty for kl-cov loss + kl_cov_ratio: 0.0002 + + # KL divergence penalty coefficient + ppo_kl_coef: 0.1 + + # Constant C in Dual-clip PPO; clips when advantage < 0 and ratio > C + clip_ratio_c: 3.0 + + # Loss aggregation mode: "token-mean", "seq-mean-token-sum", or "seq-mean-token-mean" + loss_agg_mode: token-mean + + # Entropy regularization coefficient in PPO loss + entropy_coeff: 0 + + # Whether to use KL loss instead of KL reward penalty. True for GRPO + use_kl_loss: false + + # Whether to use torch.compile() + use_torch_compile: true + + # KL loss coefficient when use_kl_loss is enabled. For GRPO + kl_loss_coef: 0.001 + + # Type of KL divergence loss. Options: "kl"(k1), "abs", "mse"(k2), "low_var_kl"(k3), "full" + kl_loss_type: low_var_kl + + # Number of PPO epochs per batch + ppo_epochs: 1 + + # Shuffle training data across PPO epochs + shuffle: false + + # Sequence parallelism size for Ulysses-style model parallelism + ulysses_sequence_parallel_size: 1 + + # calculate entropy with chunking to reduce memory peak + entropy_from_logits_with_chunking: False + + # recompute entropy + entropy_checkpointing: False + + # checkpoint configs + checkpoint: + + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + + # For more flexibility, you can specify the contents to load from the checkpoint. + load_contents: ${actor_rollout_ref.actor.checkpoint.save_contents} + + # optimizer configs + optim: + + # Learning rate + lr: 1e-6 + + # Warmup steps; negative value delegates to lr_warmup_steps_ratio + lr_warmup_steps: -1 + + # Warmup steps ratio (used if lr_warmup_steps is negative) + lr_warmup_steps_ratio: 0.0 + + # Minimum LR ratio for cosine schedule + min_lr_ratio: 0.0 + + # Number of cosine cycles in LR schedule + num_cycles: 0.5 + + # LR warmup style: "constant" or "cosine" + warmup_style: constant + + # Total training steps (must be overridden at runtime) + total_training_steps: -1 + + # Weight decay + weight_decay: 0.01 + + # configs for FSDP + fsdp_config: + + # policy for wrapping the model + wrap_policy: + + # Minimum number of parameters to trigger wrapping a layer with FSDP + min_num_params: 0 + + # Whether to offload model parameters to CPU (trades speed for memory) + param_offload: false + + # Whether to offload optimizer state to CPU + optimizer_offload: false + + # Only for FSDP2: offload param/grad/optimizer during train + offload_policy: false + + # Only for FSDP2: Reshard after forward pass to reduce memory footprint + reshard_after_forward: true + + # Number of GPUs in each FSDP shard group; -1 means auto + fsdp_size: -1 + + # Only for FSDP1: FSDP1 configuration, prefetch the next forward-pass all-gather + # before the current forward computation. + forward_prefetch: False + + # Reference model config. + # Reference model will be enabled when actor.use_kl_loss or/and algorithm.use_kl_in_reward is/are True. + ref: + + # actor_rollout_ref.ref: FSDP config same as actor. For models larger than 7B, it’s recommended to turn on offload for ref by default + strategy: ${actor_rollout_ref.actor.strategy} + + # config for FSDP strategy + fsdp_config: + + # whether to offload parameters in FSDP + param_offload: False + + # whether to perform reshard after model forward to save memory. + # only for fsdp2, [True, False, int between 1 and fsdp_size] + reshard_after_forward: True + + # Only for FSDP1: FSDP1 configuration, prefetch the next forward-pass all-gather + # before the current forward computation. + forward_prefetch: False + + # the wrap policy for FSDP model + wrap_policy: + + # minimum number of params in a wrapped module + min_num_params: 0 + + # whether to enable torch.compile + use_torch_compile: ${actor_rollout_ref.actor.use_torch_compile} + + # [Will be deprecated, use log_prob_micro_batch_size_per_gpu] + # The batch size for one forward pass in the computation of log_prob. Global batch size. + log_prob_micro_batch_size: null + + # The batch size for one forward pass in the computation of log_prob. Local batch size per GPU. + log_prob_micro_batch_size_per_gpu: null + + # enable dynamic batch size (sequence packing) for log_prob computation + log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + + # the max token length per GPU + log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} + + # sequence parallel size + ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size} + + # calculate entropy with chunking to reduce memory peak + entropy_from_logits_with_chunking: False + + # recompute entropy + entropy_checkpointing: False + + # Rollout model config. + rollout: + + # actor_rollout_ref.rollout.name: hf/vllm/sglang. + name: vllm + + # sync: LLM, async: AsyncLLM + mode: sync + + # Sampling temperature for rollout. + temperature: 1.0 + + # Top-k sampling parameter. -1 for vLLM rollout, 0 for HF rollout. + top_k: -1 + + # Top-p sampling parameter. Default 1.0. + top_p: 1 + + + # typically the same as data max prompt length + prompt_length: ${data.max_prompt_length} + + # typically the same as data max response length + response_length: ${data.max_response_length} + + # for vllm rollout + # Rollout model parameters type. Align with actor model's FSDP/Megatron type. + dtype: bfloat16 + + # Fraction of GPU memory used by vLLM/SGLang for KV cache. + gpu_memory_utilization: 0.5 + + # Whether to ignore EOS and continue generating after EOS is hit. + ignore_eos: False + + # Whether to disable CUDA graph. Default True to allow cache freeing. + enforce_eager: False + + # Whether to free engine KVCache after generation. Set enforce_eager=True when enabled. + free_cache_engine: True + + # Which loader to use for rollout model weights: dummy_dtensor, hf, megatron, etc. + # safetensors (for huge model, and set use_shm=True); dummy_dtensor: randomly init model weight + load_format: dummy + + # for huge model, layered summon can save memory (prevent OOM) but make it slower + layered_summon: False + + # TP size for rollout. Only effective for vLLM. + tensor_model_parallel_size: 2 + + # max number of tokens in a batch + max_num_batched_tokens: 8192 + + # max length for rollout + max_model_len: null + + # max length of sequences + max_num_seqs: 1024 + + # [Will be deprecated, use log_prob_micro_batch_size_per_gpu] The batch size for one forward pass in the computation of log_prob. Global batch size. + log_prob_micro_batch_size: null + + # The batch size for one forward pass in the computation of log_prob. Local batch size per GPU. + log_prob_micro_batch_size_per_gpu: null + + # enable dynamic batch size (sequence packing) for log_prob computation + log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + + # max token length for log_prob computation + log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} + + # disable logging statistics + disable_log_stats: True + + # may get higher throughput when set to True. When activated, Please increase max_num_batched_tokens or decrease max_model_len. + enable_chunked_prefill: True + + # for hf rollout + # Whether to sample during training rollout. False uses greedy sampling. + do_sample: True + + # number of responses (i.e. num sample times). > 1 for grpo + n: 1 + + # Whether to wake up inference engine in multi-stage to reduce peak memory during training-rollout transition. + multi_stage_wake_up: false + + # Extra inference engine arguments, please refer vllm/sglang official doc for detail + engine_kwargs: + + # vllm engine config + vllm: {} + + # sglang engine config + sglang: {} + + # Sampling parameters used during validation. + val_kwargs: + + # sampling parameters for validation + # Top-k sampling parameter. -1 for vLLM rollout, 0 for HF rollout. + top_k: -1 + + # Top-p sampling parameter. Default 1.0. + top_p: 1.0 + + # Sampling temperature for rollout. + temperature: 0 + + # whether to repeat n times for validation + n: 1 + + # Whether to sample during training rollout. False uses greedy sampling. + do_sample: False + + # Multi-turn interaction config for tools or chat. + multi_turn: + + # set to True for multi-turn tool interaction tasks; should set rollout.name to sglang as well + enable: False + + # null for no limit (default max_length // 3) + max_assistant_turns: null + + # null for no tool + tool_config_path: null + + # null for no limit (default max_length // 3) + max_user_turns: null + + # max parallel call for tools in single turn + max_parallel_calls: 1 + + # max length of tool response + max_tool_response_length: 256 + + # truncate side of tool response: left, middle, right + tool_response_truncate_side: middle + + # null for no interaction + interaction_config_path: null + + # - When set to True, the model's default chat template is used for multi-turn rollout, which typically matches production behavior. + # - When set to False, the token ids recorded for training are used instead; unlike the default chat template, these always include the model's full output, + # which may contain additional content such as reasoning content. This maintains the consistency between training and rollout, but it will lead to longer prompts. + use_inference_chat_template: False + + # Tokenization is performed turn by turn and the resulting token ids are concatenated to form the full conversation. + # To ensure this matches the result of tokenizing the entire conversation at once, a sanity check is run at the end of each multi-turn rollout to compare the two sets of token ids. + # Some models are known to produce different tokenization results when tokenizing turn by turn vs. all at once. aThis behavior has already been validated for them. + # To reduce excessive warnings, you can turn off the sanity check for these models if you are using their default chat template: + # Qwen/QwQ-32B, Qwen/Qwen3-xxB + # - disable: disable tokenization sanity check + # - strict: enable strict tokenization sanity check (default) + # - ignore_strippable: ignore strippable tokens when checking tokenization sanity + tokenization_sanity_check_mode: strict + + # Format of the multi-turn interaction. Options: hermes, llama3_json, ... + format: hermes + + # support logging rollout prob for debugging purpose + calculate_log_probs: False + + # [Experimental] agent loop based rollout configs + agent: + + # Number of agent loop workers + num_workers: 8 + + # custom async server configs + custom_async_server: + + # Path to the custom async server implementation + path: null + + # Class name of the custom async server class (e.g. AsyncvLLMServer) + name: null + + # profiler configs + profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + +# configs for the critic +critic: + + # Number of rollouts per update (mirrors actor rollout_n) + rollout_n: ${actor_rollout_ref.rollout.n} + + # fsdp or fsdp2 strategy used for critic model training + strategy: ${actor_rollout_ref.actor.strategy} + + # optimizer configs + optim: + + # Learning rate + lr: 1e-5 + + # Warmup steps ratio; total steps will be injected at runtime + lr_warmup_steps_ratio: 0. + + # Minimum LR ratio for cosine schedule + min_lr_ratio: 0.0 + + # LR warmup style: "constant" or "cosine" + warmup_style: constant + + # Total training steps (must be overridden at runtime) + total_training_steps: -1 + + # Weight decay + weight_decay: 0.01 + + # model config for the critic + model: + + # Path to pretrained model weights + path: ~/models/deepseek-llm-7b-chat + + # Whether to use shared memory for loading the model + use_shm: False + + # Tokenizer path (defaults to actor's model path) + tokenizer_path: ${actor_rollout_ref.model.path} + + # Hugging Face config override + override_config: { } + + # External model implementation (optional) + external_lib: ${actor_rollout_ref.model.external_lib} + + # Enable gradient checkpointing to save memory + enable_gradient_checkpointing: True + + # Offload activations to CPU to reduce GPU memory usage + enable_activation_offload: False + + # Use remove padding optimization (saves compute) + use_remove_padding: False + + # Whether to trust remote code from Hugging Face models + trust_remote_code: ${actor_rollout_ref.model.trust_remote_code} + + # FSDP-specific config + fsdp_config: + + # Whether to offload model parameters to CPU + param_offload: False + + # Whether to offload optimizer state to CPU + optimizer_offload: False + + # Only for FSDP2: offload param/grad/optimizer during train + offload_policy: False + + # Only for FSDP2: Reshard after forward pass to reduce memory footprint + reshard_after_forward: True + + # Policy for wrapping layers with FSDP + wrap_policy: + + # Minimum number of parameters to trigger wrapping + min_num_params: 0 + + # Number of GPUs in each FSDP shard group; -1 means auto + fsdp_size: -1 + + # Only for FSDP1: FSDP1 configuration, prefetch the next forward-pass all-gather + # before the current forward computation. + forward_prefetch: False + + # Set to positive value to enable LoRA (e.g., 32) + lora_rank: 0 + + # LoRA scaling factor + lora_alpha: 16 + + # LoRA target modules: "all-linear" or list of linear projection layers + target_modules: all-linear + + # PPO mini-batch size per update + ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} + + # [Deprecated] Global micro batch size + ppo_micro_batch_size: null + + # Local per-GPU micro batch size + ppo_micro_batch_size_per_gpu: null + + # Forward-only batch size (global) + forward_micro_batch_size: ${critic.ppo_micro_batch_size} + + # Forward-only batch size (per GPU) + forward_micro_batch_size_per_gpu: ${critic.ppo_micro_batch_size_per_gpu} + + # Whether to automatically adjust batch size at runtime + use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} + + # Max tokens per GPU in one PPO batch (doubled for critic) + ppo_max_token_len_per_gpu: 32768 + + # Max token length per GPU in forward pass + forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu} + + # Sequence parallelism size for Ulysses-style model parallelism + ulysses_sequence_parallel_size: 1 + + # Number of PPO epochs per batch + ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs} + + # Shuffle training data across PPO epochs + shuffle: ${actor_rollout_ref.actor.shuffle} + + # Gradient clipping for critic updates + grad_clip: 1.0 + + # PPO value function clipping range + cliprange_value: 0.5 + + # Loss aggregation mode: "token-mean", "seq-mean-token-sum", or "seq-mean-token-mean" + loss_agg_mode: ${actor_rollout_ref.actor.loss_agg_mode} + + # checkpoint configs + checkpoint: + + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + + # What to include when loading checkpoints + load_contents: ${critic.checkpoint.save_contents} + + # profiler configs + # the corresponding dataclass is verl.utils.profiler.ProfilerConfig. + profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + +# configs for the reward model +reward_model: + + # Whether to enable reward model. If False, we compute the reward only with the user-defined reward functions. + # In GSM8K and Math examples, we disable reward model. + # For RLHF alignment example using full_hh_rlhf, we utilize reward model to assess the responses. + # If False, the following parameters are not effective + enable: False + + # FSDP strategy: "fsdp" or "fsdp2" + strategy: ${actor_rollout_ref.actor.strategy} + + # model config for reward scoring + model: + + # Input tokenizer. If the reward model’s chat template is inconsistent with the policy, + # we need to first decode to plaintext, then apply the rm’s chat_template. + # Then score with RM. If chat_templates are consistent, it can be set to null. + input_tokenizer: ${actor_rollout_ref.model.path} + + # RM’s HDFS path or local path. Note that RM only supports AutoModelForSequenceClassification. + # Other model types need to define their own RewardModelWorker and pass it from the code. + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + + # Whether to use shared memory for loading the model + use_shm: False + + # External model implementation (optional) + external_lib: ${actor_rollout_ref.model.external_lib} + + # Use remove padding optimization (saves compute) + use_remove_padding: False + + # Whether to use fused reward kernels for speedup + use_fused_kernels: ${actor_rollout_ref.model.use_fused_kernels} + + # Whether to enable loading a remote code model, default to False + trust_remote_code: False + + # FSDP-specific config + fsdp_config: + + # Policy for wrapping layers with FSDP + wrap_policy: + + # Minimum number of parameters to trigger wrapping + min_num_params: 0 + + # Whether to offload model parameters to CPU + param_offload: False + + # Only for FSDP2: Reshard after forward pass to reduce memory footprint + reshard_after_forward: True + + # Number of GPUs in each FSDP shard group; -1 means auto + fsdp_size: -1 + + # Only for FSDP1: FSDP1 configuration, prefetch the next forward-pass all-gather + # before the current forward computation. + forward_prefetch: False + + # [Deprecated] Global micro batch size + micro_batch_size: null + + # Local per-GPU micro batch size + micro_batch_size_per_gpu: null + + # Maximum sequence length to process for scoring + max_length: null + + # Sequence parallelism size for Ulysses-style model parallelism + ulysses_sequence_parallel_size: 1 + + # Whether to dynamically adjust batch size at runtime + use_dynamic_bsz: ${critic.use_dynamic_bsz} + + # Maximum number of tokens per GPU in one forward pass + forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + + # Reward Manager. This defines the mechanism of computing rule-based reward and handling different reward sources. + # Default is naive. If all verification functions are multiprocessing-safe, + # the reward manager can be set to prime for parallel verification. + reward_manager: naive + + # Whether to launch custom reward function asynchronously during log_prob + launch_reward_fn_async: False + + # Cloud/local sandbox fusion configuration for custom reward logic + sandbox_fusion: + + # Cloud/local function URL for sandbox execution + url: null + + # Max concurrent requests allowed to sandbox + max_concurrent: 64 + + # Max memory limit for each sandbox process in MB + memory_limit_mb: 1024 + + # profiler configs + profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + +# custom reward function definition +custom_reward_function: + + # The path to the file containing your customized reward function. + # If not specified, pre-implemented reward functions will be used. + path: null + + # The name of the reward function within the specified file. Default is 'compute_score'. + name: compute_score + +# config for the algorithm +algorithm: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.AlgoConfig + + # Discount factor for future rewards + gamma: 1.0 + + # Trade-off between bias and variance in the GAE estimator + lam: 1.0 + + # Advantage estimator type: "gae", "grpo", "reinforce_plus_plus", etc. + adv_estimator: gae + + # Whether to normalize advantages by std (specific to GRPO) + norm_adv_by_std_in_grpo: True + + # Whether to enable in-reward KL penalty + use_kl_in_reward: False + + # How to estimate KL divergence: "kl", "abs", "mse", "low_var_kl", or "full" + kl_penalty: kl + + # KL control configuration + kl_ctrl: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.KLControlConfig + + # KL control type: "fixed" or "adaptive" + type: fixed + + # Initial coefficient for KL penalty + kl_coef: 0.001 + + # Horizon value for adaptive controller (if enabled) + horizon: 10000 + + # Target KL divergence (used for adaptive controller) + target_kl: 0.1 + + # Whether to enable preference feedback PPO + use_pf_ppo: False + + # Preference feedback PPO settings + pf_ppo: + + # Method for reweighting samples: "pow", "max_min", or "max_random" + reweight_method: pow + + # Power used for weight scaling in "pow" method + weight_pow: 2.0 + +# config for the trainer +trainer: + + # Whether to balance batch sizes across distributed workers + balance_batch: True + + # Number of epochs in training + total_epochs: 30 + + # Total training steps (can be set explicitly or derived from epochs) + total_training_steps: null + + # The steps that will be profiled. null means no profiling. null or [1,2,5,...] + profile_steps: null + + # controller Nvidia Nsight Systems Options. Must set when profile_steps is not None. + ## reference https://docs.nvidia.com/nsight-systems/UserGuide/index.html + ## reference https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html + controller_nsight_options: + + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # worker Nvidia Nsight Systems Options. Must set when profile_steps is not None. + worker_nsight_options: + + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # Profiling only in a range of torch.cuda.profiler.start and stop. Do not change this config. + capture-range: "cudaProfilerApi" + + # Specify the desired behavior when a capture range ends. + # In verl we need the orch.cuda.profiler.start/stop pair to repeats n times. + # valid values are "repeat-shutdown:n" or null. + # For normal whole step profiling, n = len(profile_steps); + # but for discrete profiling, n = len(profile_steps) * Number(subtasks). + # Or you can just leave it null and the program will use n = len(profile_steps) * 6; + capture-range-end: null + + # Send signal to the target application's process group. We let the program to exit by itself. + kill: none + + # Config for npu profiler. Must set when profile_steps is not None and torch_npu is available. + npu_profile: + + # Options for the npu profiler + options: + + # Storage path of collected data. + save_path: ./profiler_data + + # The roles that will be profiled. Only takes effect in discrete mode. + # optional values: all, rollout_generate, actor_compute_log_prob, actor_update and ref_compute_log_prob. + # "all" means all roles will be profiled. + roles: ["all"] + + # Collection level, optional values: level_none, level0, level1, level2. + level: level1 + + # Whether to enable memory analysis. + with_memory: False + + # Whether to record tensor shape. + record_shapes: False + + # Whether to record Device-side performance data. + with_npu: True + + # Whether to record Host-side performance data. + with_cpu: True + + # Whether to record Python call stack information. + with_module: False + + # Whether to record operator call stack information. + with_stack: False + + # Whether to automatically parse the data. + analysis: True + + # Project name for experiment tracking (e.g., wandb) + project_name: verl_examples + + # Experiment name for run identification in tracking tools + experiment_name: gsm8k + + # Logging backends to use: "console", "wandb", etc. + logger: [ 'console', 'wandb' ] + + # Number of generations to log during validation + log_val_generations: 0 + + # Directory for logging rollout data; no dump if null + rollout_data_dir: null + + # Directory for logging validation data; no dump if null + validation_data_dir: null + + # Number of nodes used in the training + nnodes: 1 + + # Number of GPUs per node + n_gpus_per_node: 8 + + # Save frequency (by iteration) for model checkpoints + save_freq: -1 + + # ESI refers to the elastic server instance used during training, similar to the training plan. For example, + # if you purchase 10 hours of computing power, the ESI will automatically shut down after 10 hours of training. + # To ensure a checkpoint is saved before ESI shuts down, the system will start saving a checkpoint in advance. + # The advance time is calculated as: Advance Time = Longest historical step duration + Checkpoint save duration + esi_redundant_time. + # Here, esi_redundant_time is a user-defined value that further extends the advance time for added safety. + esi_redundant_time: 0 + + # Resume mode: "auto", "disable", or "resume_path" + # "auto": resume from last checkpoint if available + # "disable": start from scratch + # "resume_path": resume from a user-defined path + resume_mode: auto + + # Path to resume training from (only used when resume_mode is "resume_path") + resume_from_path: null + + # Whether to run validation before training begins + val_before_train: True + + # Whether to run validation only + val_only: False + + # Validation frequency (in training iterations) + test_freq: -1 + + # Number of iterations to warm up the critic before updating policy + critic_warmup: 0 + + # Default path to distributed filesystem for saving checkpoints + default_hdfs_dir: null + + # Whether to delete local checkpoints after loading + del_local_ckpt_after_load: False + + # Default local directory for saving checkpoints + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + + # Maximum number of actor checkpoints to keep + max_actor_ckpt_to_keep: null + + # Maximum number of critic checkpoints to keep + max_critic_ckpt_to_keep: null + + # Timeout (in seconds) for Ray worker to wait for registration + ray_wait_register_center_timeout: 300 + + # Device to run training on (e.g., "cuda", "cpu") + device: cuda + +# configs related to ray +ray_kwargs: + # configs related to ray initialization + ray_init: + + # Number of CPUs for Ray. Use a fixed number instead of null when using SLURM. + num_cpus: null + + # Path to save Ray timeline JSON for performance profiling + timeline_json_file: null diff --git a/e2c/verl/tests/trainer/config/test_algo_config_on_cpu.py b/e2c/verl/tests/trainer/config/test_algo_config_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..d08c949ee48a3b6fc045f43b0fc455a4f4ac4708 --- /dev/null +++ b/e2c/verl/tests/trainer/config/test_algo_config_on_cpu.py @@ -0,0 +1,204 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from omegaconf import OmegaConf + +from verl.trainer.config import AlgoConfig, KLControlConfig +from verl.trainer.ppo.core_algos import ( + compute_gae_advantage_return, + compute_grpo_outcome_advantage, + get_adv_estimator_fn, +) +from verl.utils.config import omega_conf_to_dataclass + + +class TestAlgoConfig(unittest.TestCase): + """Test the AlgoConfig dataclass and its integration with core algorithms.""" + + def setUp(self): + """Set up test fixtures.""" + # Create a sample algorithm config as DictConfig (similar to what comes from YAML) + self.config_dict = { + "_target_": "verl.trainer.config.AlgoConfig", + "gamma": 0.99, + "lam": 0.95, + "adv_estimator": "gae", + "norm_adv_by_std_in_grpo": True, + "use_kl_in_reward": True, + "kl_penalty": "kl", + "kl_ctrl": { + "_target_": "verl.trainer.config.KLControlConfig", + "type": "adaptive", + "kl_coef": 0.002, + "horizon": 5000, + "target_kl": 0.05, + }, + "use_pf_ppo": True, + "pf_ppo": {"reweight_method": "max_min", "weight_pow": 3.0}, + } + self.omega_config = OmegaConf.create(self.config_dict) + + def test_dataclass_creation_from_dict(self): + """Test creating AlgoConfig from dictionary.""" + config = omega_conf_to_dataclass(self.config_dict) + + self.assertIsInstance(config, AlgoConfig) + self.assertEqual(config.gamma, 0.99) + self.assertEqual(config.lam, 0.95) + self.assertEqual(config.adv_estimator, "gae") + self.assertTrue(config.norm_adv_by_std_in_grpo) + self.assertTrue(config.use_kl_in_reward) + self.assertEqual(config.kl_penalty, "kl") + self.assertTrue(config.use_pf_ppo) + + def test_dataclass_creation_from_omega_config(self): + """Test creating AlgoConfig from OmegaConf DictConfig.""" + config = omega_conf_to_dataclass(self.omega_config) + + self.assertIsInstance(config, AlgoConfig) + self.assertEqual(config.gamma, 0.99) + self.assertEqual(config.lam, 0.95) + + def test_nested_configs(self): + """Test that nested configurations are properly converted.""" + config = omega_conf_to_dataclass(self.omega_config) + + # Test KL control config + self.assertIsInstance(config.kl_ctrl, KLControlConfig) + self.assertEqual(config.kl_ctrl.type, "adaptive") + self.assertEqual(config.kl_ctrl.kl_coef, 0.002) + self.assertEqual(config.kl_ctrl.horizon, 5000) + self.assertEqual(config.kl_ctrl.target_kl, 0.05) + + # Test PF PPO config + self.assertEqual(config.pf_ppo.get("reweight_method"), "max_min") + self.assertEqual(config.pf_ppo.get("weight_pow"), 3.0) + + def test_default_values(self): + """Test that default values are properly set.""" + minimal_config = {"gamma": 0.8} + config = omega_conf_to_dataclass(minimal_config, AlgoConfig) + + self.assertEqual(config.gamma, 0.8) + self.assertEqual(config.lam, 1.0) # default value + self.assertEqual(config.adv_estimator, "gae") # default value + self.assertTrue(config.norm_adv_by_std_in_grpo) # default value + self.assertFalse(config.use_kl_in_reward) # default value + self.assertEqual(config.kl_penalty, "kl") # default value + self.assertFalse(config.use_pf_ppo) # default value + + def test_get_method_backward_compatibility(self): + """Test the get method for backward compatibility.""" + config = omega_conf_to_dataclass(self.omega_config) + + # Test existing attribute + self.assertEqual(config.get("gamma"), 0.99) + self.assertEqual(config.get("gamma", 1.0), 0.99) + + # Test non-existing attribute + self.assertIsNone(config.get("non_existing")) + self.assertEqual(config.get("non_existing", "default"), "default") + + def test_post_init_nested_configs(self): + """Test that __post_init__ properly initializes nested configs when None.""" + # Create config without nested configs + minimal_config = AlgoConfig(gamma=0.9) + + # Check that nested configs are initialized + self.assertIsNotNone(minimal_config.kl_ctrl) + self.assertIsInstance(minimal_config.kl_ctrl, KLControlConfig) + assert not minimal_config.pf_ppo + + def test_config_init_from_yaml(self): + import os + + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + cfg = compose(config_name="ppo_trainer") + algo_config = omega_conf_to_dataclass(cfg.algorithm) + from verl.trainer.config import AlgoConfig + + assert isinstance(algo_config, AlgoConfig) + + +class TestAlgoCompute(unittest.TestCase): + """Test the AlgoConfig dataclass and its integration with core algorithms.""" + + def setUp(self): + """Set up test fixtures.""" + self.algo_config = AlgoConfig( + gamma=0.99, + lam=0.95, + adv_estimator="gae", + norm_adv_by_std_in_grpo=True, + use_kl_in_reward=True, + kl_penalty="kl", + kl_ctrl=KLControlConfig(type="adaptive", kl_coef=0.002, horizon=5000, target_kl=0.05), + use_pf_ppo=True, + pf_ppo={"reweight_method": "max_min", "weight_pow": 3.0}, + ) + + def test_advantage_estimator_with_cfg(self): + """Test integration with advantage estimators from core_algos.""" + config = self.algo_config + + # Test GAE advantage estimator + adv_fn = get_adv_estimator_fn(config.adv_estimator) + self.assertIsNotNone(adv_fn) + + # Test with actual GAE computation + batch_size, seq_len = 2, 5 + token_level_rewards = torch.randn(batch_size, seq_len) + values = torch.randn(batch_size, seq_len) + response_mask = torch.ones(batch_size, seq_len) + + advantages, returns = compute_gae_advantage_return( + token_level_rewards=token_level_rewards, + values=values, + response_mask=response_mask, + gamma=config.gamma, + lam=config.lam, + ) + + self.assertEqual(advantages.shape, (batch_size, seq_len)) + self.assertEqual(returns.shape, (batch_size, seq_len)) + + def test_grpo_advantage_estimator_with_cfg(self): + """Test integration with GRPO advantage estimator.""" + grpo_config = AlgoConfig(adv_estimator="grpo", norm_adv_by_std_in_grpo=True) + + # Test GRPO advantage computation + batch_size, seq_len = 4, 3 + token_level_rewards = torch.tensor([[1.0, 0.5, 0.0], [2.0, 1.0, 0.0], [0.5, 0.2, 0.0], [1.5, 0.8, 0.0]]) + response_mask = torch.ones(batch_size, seq_len) + index = np.array([0, 0, 1, 1]) # Two groups + + advantages, returns = compute_grpo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + norm_adv_by_std_in_grpo=grpo_config.norm_adv_by_std_in_grpo, + ) + + self.assertEqual(advantages.shape, (batch_size, seq_len)) + self.assertEqual(returns.shape, (batch_size, seq_len)) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/trainer/config/test_legacy_config_on_cpu.py b/e2c/verl/tests/trainer/config/test_legacy_config_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..7117e27d80a175e0beb884d5bfeef951775aab9b --- /dev/null +++ b/e2c/verl/tests/trainer/config/test_legacy_config_on_cpu.py @@ -0,0 +1,176 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest +import warnings + +from hydra import compose, initialize_config_dir +from hydra.core.global_hydra import GlobalHydra +from omegaconf import OmegaConf + +_BREAKING_CHANGES = [ + "critic.optim.lr", # mcore critic lr init value 1e-6 -> 1e-5 + "actor_rollout_ref.actor.optim.lr_warmup_steps", # None -> -1 + "critic.optim.lr_warmup_steps", # None -> -1 + "actor_rollout_ref.rollout.name", # vllm -> ??? + "actor_rollout_ref.actor.megatron.expert_tensor_parallel_size", + "actor_rollout_ref.ref.megatron.expert_tensor_parallel_size", + "critic.megatron.expert_tensor_parallel_size", + "reward_model.megatron.expert_tensor_parallel_size", +] + + +class TestConfigComparison(unittest.TestCase): + """Test that current configs match their legacy counterparts exactly.""" + + ignored_keys = [ + "enable_gradient_checkpointing", + "gradient_checkpointing_kwargs", + "activations_checkpoint_method", + "activations_checkpoint_granularity", + "activations_checkpoint_num_layers", + "discrete", + "profiler", + "profile", + "use_profile", + "npu_profile", + "profile_steps", + "worker_nsight_options", + "controller_nsight_options", + ] + + def _compare_configs_recursively( + self, current_config, legacy_config, path="", legacy_allow_missing=True, current_allow_missing=False + ): + """Recursively compare two OmegaConf configs and assert they are identical. + + Args: + legacy_allow_missing (bool): sometimes the legacy megatron config contains fewer keys and + we allow that to happen + """ + if isinstance(current_config, dict) and isinstance(legacy_config, dict): + current_keys = set(current_config.keys()) + legacy_keys = set(legacy_config.keys()) + + missing_in_current = legacy_keys - current_keys + missing_in_legacy = current_keys - legacy_keys + + # Ignore specific keys that are allowed to be missing + for key in self.ignored_keys: + if key in missing_in_current: + missing_in_current.remove(key) + if key in missing_in_legacy: + missing_in_legacy.remove(key) + + if missing_in_current: + msg = f"Keys missing in current config at {path}: {missing_in_current}" + if current_allow_missing: + warnings.warn(msg, stacklevel=1) + else: + self.fail(f"Keys missing in current config at {path}: {missing_in_current}") + if missing_in_legacy: + # if the legacy + msg = f"Keys missing in legacy config at {path}: {missing_in_legacy}" + if legacy_allow_missing: + warnings.warn(msg, stacklevel=1) + else: + self.fail(msg) + + for key in current_keys: + current_path = f"{path}.{key}" if path else key + if key in legacy_config: + self._compare_configs_recursively(current_config[key], legacy_config[key], current_path) + elif isinstance(current_config, list) and isinstance(legacy_config, list): + self.assertEqual( + len(current_config), + len(legacy_config), + f"List lengths differ at {path}: current={len(current_config)}, legacy={len(legacy_config)}", + ) + for i, (current_item, legacy_item) in enumerate(zip(current_config, legacy_config, strict=True)): + self._compare_configs_recursively(current_item, legacy_item, f"{path}[{i}]") + elif path not in _BREAKING_CHANGES: + self.assertEqual( + current_config, + legacy_config, + f"Values differ at {path}: current={current_config}, legacy={legacy_config}", + ) + + def test_ppo_trainer_config_matches_legacy(self): + """Test that ppo_trainer.yaml matches legacy_ppo_trainer.yaml exactly.""" + import os + + from hydra import compose, initialize_config_dir + from hydra.core.global_hydra import GlobalHydra + + GlobalHydra.instance().clear() + + try: + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + current_config = compose(config_name="ppo_trainer") + + legacy_config = OmegaConf.load("tests/trainer/config/legacy_ppo_trainer.yaml") + current_dict = OmegaConf.to_container(current_config, resolve=True) + legacy_dict = OmegaConf.to_container(legacy_config, resolve=True) + + if "defaults" in current_dict: + del current_dict["defaults"] + + self._compare_configs_recursively(current_dict, legacy_dict) + finally: + GlobalHydra.instance().clear() + + def test_ppo_megatron_trainer_config_matches_legacy(self): + """Test that ppo_megatron_trainer.yaml matches legacy_ppo_megatron_trainer.yaml exactly.""" + + GlobalHydra.instance().clear() + + try: + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + current_config = compose(config_name="ppo_megatron_trainer") + + legacy_config = OmegaConf.load("tests/trainer/config/legacy_ppo_megatron_trainer.yaml") + current_dict = OmegaConf.to_container(current_config, resolve=True) + legacy_dict = OmegaConf.to_container(legacy_config, resolve=True) + + if "defaults" in current_dict: + del current_dict["defaults"] + + self._compare_configs_recursively( + current_dict, legacy_dict, legacy_allow_missing=True, current_allow_missing=False + ) + finally: + GlobalHydra.instance().clear() + + def test_load_component(self): + """Test that ppo_megatron_trainer.yaml matches legacy_ppo_megatron_trainer.yaml exactly.""" + + GlobalHydra.instance().clear() + configs_to_load = [ + ("verl/trainer/config/actor", "dp_actor"), + ("verl/trainer/config/actor", "megatron_actor"), + ("verl/trainer/config/ref", "dp_ref"), + ("verl/trainer/config/ref", "megatron_ref"), + ("verl/trainer/config/rollout", "rollout"), + ] + for config_dir, config_file in configs_to_load: + try: + with initialize_config_dir(config_dir=os.path.abspath(config_dir)): + compose(config_name=config_file) + finally: + GlobalHydra.instance().clear() + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/trainer/ppo/__init__.py b/e2c/verl/tests/trainer/ppo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..26d7c04fc335c873ef77f8989e82e4239be7dba1 --- /dev/null +++ b/e2c/verl/tests/trainer/ppo/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Tests for the PPO trainer module. +""" diff --git a/e2c/verl/tests/trainer/ppo/test_core_algos_on_cpu.py b/e2c/verl/tests/trainer/ppo/test_core_algos_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..288f28e63989df8a05e53f43d75aad43b86662bd --- /dev/null +++ b/e2c/verl/tests/trainer/ppo/test_core_algos_on_cpu.py @@ -0,0 +1,317 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import pytest +import torch + +import verl.trainer.ppo.core_algos +from verl.trainer.ppo.core_algos import ( + compute_gae_advantage_return, + compute_grpo_outcome_advantage, + compute_grpo_vectorized_outcome_advantage, + compute_rloo_outcome_advantage, + compute_rloo_vectorized_outcome_advantage, + get_adv_estimator_fn, + register_adv_est, +) + + +def mock_test_fn(): + pass + + +class TestRegisterAdvEst(unittest.TestCase): + def setUp(self): + """Clear the registry before each test""" + verl.trainer.ppo.core_algos.ADV_ESTIMATOR_REGISTRY.clear() + verl.trainer.ppo.core_algos.ADV_ESTIMATOR_REGISTRY = { + "gae": lambda x: x * 2, + "vtrace": lambda x: x + 1, + } + self.ADV_ESTIMATOR_REGISTRY = verl.trainer.ppo.core_algos.ADV_ESTIMATOR_REGISTRY + + def tearDown(self) -> None: + verl.trainer.ppo.core_algos.ADV_ESTIMATOR_REGISTRY.clear() + return super().tearDown() + + def test_register_new_function(self): + """Test registering a new function with a string name""" + + @register_adv_est("test_estimator") + def test_fn(): + pass + + self.assertIn("test_estimator", self.ADV_ESTIMATOR_REGISTRY) + self.assertEqual(self.ADV_ESTIMATOR_REGISTRY["test_estimator"], test_fn) + + def test_register_with_enum(self): + """Test registering with an enum value (assuming AdvantageEstimator exists)""" + from enum import Enum + + class AdvantageEstimator(Enum): + TEST = "test_enum_estimator" + + @register_adv_est(AdvantageEstimator.TEST) + def test_fn(): + pass + + self.assertIn("test_enum_estimator", self.ADV_ESTIMATOR_REGISTRY) + self.assertEqual(self.ADV_ESTIMATOR_REGISTRY["test_enum_estimator"], test_fn) + + def test_duplicate_registration_same_function(self): + """Test that registering the same function twice doesn't raise an error""" + register_adv_est("duplicate_test")(mock_test_fn) + register_adv_est("duplicate_test")(mock_test_fn) + + self.assertEqual(self.ADV_ESTIMATOR_REGISTRY["duplicate_test"], mock_test_fn) + + def test_duplicate_registration_different_function(self): + """Test that registering different functions with same name raises ValueError""" + + @register_adv_est("conflict_test") + def test_fn1(): + pass + + with self.assertRaises(ValueError): + + @register_adv_est("conflict_test") + def test_fn2(): + pass + + def test_decorator_preserves_function(self): + """Test that the decorator returns the original function""" + + def test_fn(): + return "original" + + decorated = register_adv_est("preserve_test")(test_fn) + self.assertEqual(decorated(), "original") + + def test_multiple_registrations(self): + """Test registering multiple different functions""" + init_adv_count = len(self.ADV_ESTIMATOR_REGISTRY) + + @register_adv_est("estimator1") + def fn1(): + pass + + @register_adv_est("estimator2") + def fn2(): + pass + + self.assertEqual(len(self.ADV_ESTIMATOR_REGISTRY), 2 + init_adv_count) + self.assertEqual(self.ADV_ESTIMATOR_REGISTRY["estimator1"], fn1) + self.assertEqual(self.ADV_ESTIMATOR_REGISTRY["estimator2"], fn2) + + def test_get_adv_estimator_fn_valid_names(self): + """Test that valid names return the correct function from registry.""" + # Test GAE + gae_fn = get_adv_estimator_fn("gae") + assert gae_fn(5) == 10 # 5 * 2 = 10 + + # Test Vtrace + vtrace_fn = get_adv_estimator_fn("vtrace") + assert vtrace_fn(5) == 6 # 5 + 1 = 6 + + def test_get_adv_estimator_fn_invalid_name(self): + """Test that invalid names raise ValueError.""" + with pytest.raises(ValueError) as excinfo: + get_adv_estimator_fn("invalid_name") + assert "Unknown advantage estimator simply: invalid_name" in str(excinfo.value) + + def test_get_adv_estimator_fn_case_sensitive(self): + """Test that name lookup is case-sensitive.""" + with pytest.raises(ValueError): + get_adv_estimator_fn("GAE") # Different case + + +def test_multi_turn_compute_gae_advantage_return(): + """Test multi-turn GAE skip observation tokens.""" + gamma = random.uniform(0.0, 1.0) + lam = random.uniform(0.0, 1.0) + + rewards = torch.tensor([[0.0, 0.0, 0.1, 0.1, 0.1, 0.0, 0.0, 0.1, 1.0, 0.0, 0.0]], dtype=torch.float) + + values1 = torch.tensor( + [ + [ + random.uniform(-100.0, 100.0), + random.random(), + 4.0, + 5.0, + 6.0, + random.uniform(-100.0, 0), + random.random(), + 7.0, + 9.0, + 0.0, + 0.0, + ] + ], + dtype=torch.float, + ) + + values2 = torch.tensor( + [ + [ + random.random(), + random.uniform(-100.0, 100.0), + 4.0, + 5.0, + 6.0, + random.random(), + random.uniform(0.0, 100.0), + 7.0, + 9.0, + 0.0, + 0.0, + ] + ], + dtype=torch.float, + ) + + response_mask = torch.tensor([[0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0]], dtype=torch.float) + + adv1, ret1 = compute_gae_advantage_return(rewards, values1, response_mask, gamma, lam) + adv2, ret2 = compute_gae_advantage_return(rewards, values2, response_mask, gamma, lam) + + ret1 *= response_mask + ret2 *= response_mask + assert torch.equal(adv1, adv2), f"{adv1=}, {adv2=}" + assert torch.equal(ret1, ret2), f"{ret1=}, {ret2=}" + print(f" [CORRECT] \n\n{adv1=}, \n\n{ret1=}") + + +def _make_group_index(batch_size: int, num_groups: int) -> np.ndarray: + """Create a numpy index array ensuring each group has at least 2 samples.""" + assert num_groups * 2 <= batch_size, "batch_size must allow >=2 samples per group" + counts: list[int] = [2] * num_groups + remaining = batch_size - 2 * num_groups + for _ in range(remaining): + counts[random.randrange(num_groups)] += 1 + index = [] + for gid, c in enumerate(counts): + index.extend([gid] * c) + random.shuffle(index) + return np.asarray(index, dtype=np.int64) + + +def _rand_mask(batch_size: int, seq_len: int) -> torch.Tensor: + mask = torch.randint(0, 2, (batch_size, seq_len), dtype=torch.int64).float() + rows_without_one = (mask.sum(dim=-1) == 0).nonzero(as_tuple=True)[0] + if len(rows_without_one) > 0: + mask[rows_without_one, -1] = 1.0 + return mask + + +@pytest.mark.parametrize( + "batch_size,seq_len,num_groups,seed", + [ + (64, 128, 5, 0), + (128, 256, 8, 1), + (512, 512, 10, 2), + ], +) +def test_rloo_and_vectorized_equivalence(batch_size: int, seq_len: int, num_groups: int, seed: int): + torch.manual_seed(seed) + random.seed(seed) + np.random.seed(seed) + index = _make_group_index(batch_size, num_groups) + response_mask = _rand_mask(batch_size, seq_len) + base_rewards = torch.randn(batch_size, seq_len, dtype=torch.float32) + token_level_rewards = base_rewards * response_mask + adv1, ret1 = compute_rloo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + ) + adv2, ret2 = compute_rloo_vectorized_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + ) + # Print concise diagnostics for visibility during test runs + adv_max_diff = (adv1 - adv2).abs().max().item() + ret_max_diff = (ret1 - ret2).abs().max().item() + total_mask_tokens = int(response_mask.sum().item()) + print( + f"[RLOO] seed={seed} groups={num_groups} shape={adv1.shape} " + f"mask_tokens={total_mask_tokens} adv_max_diff={adv_max_diff:.3e} ret_max_diff={ret_max_diff:.3e}" + ) + assert adv1.shape == adv2.shape == (batch_size, seq_len) + assert ret1.shape == ret2.shape == (batch_size, seq_len) + assert torch.allclose(adv1, adv2, rtol=1e-5, atol=1e-6) + assert torch.allclose(ret1, ret2, rtol=1e-5, atol=1e-6) + + +@pytest.mark.parametrize( + "batch_size,seq_len,num_groups,seed", + [ + (64, 128, 5, 0), + (128, 256, 8, 1), + (512, 512, 10, 2), + ], +) +def test_grpo_and_vectorized_equivalence(batch_size: int, seq_len: int, num_groups: int, seed: int): + # Set seeds for reproducibility + torch.manual_seed(seed) + random.seed(seed) + np.random.seed(seed) + + # Generate group indices (numpy array of shape [batch_size]) + index = _make_group_index(batch_size, num_groups) + + # Generate binary response mask (at least one valid token per row) + response_mask = _rand_mask(batch_size, seq_len) + + # Generate token-level rewards and apply mask + base_rewards = torch.randn(batch_size, seq_len, dtype=torch.float32) + token_level_rewards = base_rewards * response_mask + + # Compute GRPO outcome advantage (original implementation) + adv1, ret1 = compute_grpo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + ) + + # Compute GRPO outcome advantage (vectorized implementation) + adv2, ret2 = compute_grpo_vectorized_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + ) + + # Diagnostic info for visibility (same style as RLOO test) + adv_max_diff = (adv1 - adv2).abs().max().item() + ret_max_diff = (ret1 - ret2).abs().max().item() + total_mask_tokens = int(response_mask.sum().item()) + print( + f"[GRPO] seed={seed} groups={num_groups} shape={adv1.shape} " + f"mask_tokens={total_mask_tokens} adv_max_diff={adv_max_diff:.3e} ret_max_diff={ret_max_diff:.3e}" + ) + + # Assert shape and numerical equivalence + assert adv1.shape == adv2.shape == (batch_size, seq_len) + assert ret1.shape == ret2.shape == (batch_size, seq_len) + assert torch.allclose(adv1, adv2, rtol=1e-5, atol=1e-6) + assert torch.allclose(ret1, ret2, rtol=1e-5, atol=1e-6) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/trainer/ppo/test_metric_utils_on_cpu.py b/e2c/verl/tests/trainer/ppo/test_metric_utils_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..50fe952c03bdd7eaf143e4678318fc4b45f8b373 --- /dev/null +++ b/e2c/verl/tests/trainer/ppo/test_metric_utils_on_cpu.py @@ -0,0 +1,324 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Tests for the metric utilities in verl.trainer.ppo.metric_utils. +""" + +import unittest +from unittest.mock import MagicMock, patch + +import numpy as np +import torch + +from verl.trainer.ppo.metric_utils import ( + bootstrap_metric, + calc_maj_val, + compute_data_metrics, + compute_throughout_metrics, + compute_timing_metrics, + process_validation_metrics, +) +from verl.utils.metric import ( + reduce_metrics, +) + + +class TestReduceMetrics(unittest.TestCase): + """Tests for the reduce_metrics function.""" + + def test_reduce_metrics_basic(self): + """Test that reduce_metrics correctly computes means.""" + metrics = { + "loss": [1.0, 2.0, 3.0], + "accuracy": [0.0, 0.5, 1.0], + } + result = reduce_metrics(metrics) + + self.assertEqual(result["loss"], 2.0) + self.assertEqual(result["accuracy"], 0.5) + + def test_reduce_metrics_empty(self): + """Test that reduce_metrics handles empty lists.""" + metrics = { + "empty": [], + } + result = reduce_metrics(metrics) + + self.assertTrue(np.isnan(result["empty"])) + + def test_reduce_metrics_single_value(self): + """Test that reduce_metrics works with single values.""" + metrics = { + "single": [5.0], + } + result = reduce_metrics(metrics) + + self.assertEqual(result["single"], 5.0) + + +class TestComputeDataMetrics(unittest.TestCase): + """Tests for the compute_data_metrics function.""" + + def setUp(self): + """Set up common test data.""" + # Create a mock DataProto object + self.batch = MagicMock() + self.batch.batch = { + "token_level_scores": torch.tensor([[1.0, 2.0], [3.0, 4.0]]), + "token_level_rewards": torch.tensor([[0.5, 1.0], [1.5, 2.0]]), + "advantages": torch.tensor([[0.1, 0.2], [0.3, 0.4]]), + "returns": torch.tensor([[1.1, 1.2], [1.3, 1.4]]), + "responses": torch.zeros((2, 2)), # 2 samples, 2 tokens each + "attention_mask": torch.tensor( + [ + [1, 1, 1, 1], # 2 prompt tokens, 2 response tokens + [1, 1, 1, 1], + ] + ), + "response_mask": torch.tensor( + [ + [1, 1], # 2 response tokens + [1, 1], + ] + ), + "values": torch.tensor([[0.9, 1.0], [1.1, 1.2]]), + } + + def test_compute_data_metrics_with_critic(self): + """Test compute_data_metrics with critic enabled.""" + metrics = compute_data_metrics(self.batch, use_critic=True) + + # Check that all expected metrics are present + self.assertIn("critic/score/mean", metrics) + self.assertIn("critic/rewards/mean", metrics) + self.assertIn("critic/advantages/mean", metrics) + self.assertIn("critic/returns/mean", metrics) + self.assertIn("critic/values/mean", metrics) + self.assertIn("critic/vf_explained_var", metrics) + self.assertIn("response_length/mean", metrics) + self.assertIn("prompt_length/mean", metrics) + + # Check some specific values + self.assertAlmostEqual(metrics["critic/score/mean"], 5.0) # Sum of token_level_scores + self.assertAlmostEqual(metrics["critic/rewards/mean"], 2.5) # Sum of token_level_rewards + + def test_compute_data_metrics_without_critic(self): + """Test compute_data_metrics with critic disabled.""" + metrics = compute_data_metrics(self.batch, use_critic=False) + + # Check that critic-specific metrics are not present + self.assertNotIn("critic/values/mean", metrics) + self.assertNotIn("critic/vf_explained_var", metrics) + + # Check that other metrics are still present + self.assertIn("critic/score/mean", metrics) + self.assertIn("critic/rewards/mean", metrics) + self.assertIn("response_length/mean", metrics) + + +class TestComputeTimingMetrics(unittest.TestCase): + """Tests for the compute_timing_metrics function.""" + + def setUp(self): + """Set up common test data.""" + # Create a mock DataProto object + self.batch = MagicMock() + self.batch.batch = { + "responses": torch.zeros((2, 3)), # 2 samples, 3 response tokens each + "attention_mask": torch.tensor( + [ + [1, 1, 1, 1, 1, 1], # 3 prompt tokens, 3 response tokens + [1, 1, 1, 1, 1, 1], + ] + ), + } + + # Mock the _compute_response_info function to return known values + self.response_info = { + "prompt_length": torch.tensor([3.0, 3.0]), + "response_length": torch.tensor([3.0, 3.0]), + "response_mask": torch.ones((2, 3)), + } + + @patch("verl.trainer.ppo.metric_utils._compute_response_info") + def test_compute_timing_metrics(self, mock_compute_response_info): + """Test compute_timing_metrics with various timing data.""" + mock_compute_response_info.return_value = self.response_info + + timing_raw = { + "gen": 0.5, # 500ms + "ref": 0.3, # 300ms + "values": 0.2, # 200ms + } + + metrics = compute_timing_metrics(self.batch, timing_raw) + + # Check raw timing metrics + self.assertEqual(metrics["timing_s/gen"], 0.5) + self.assertEqual(metrics["timing_s/ref"], 0.3) + self.assertEqual(metrics["timing_s/values"], 0.2) + + # Check per-token timing metrics + # gen uses only response tokens (6 tokens) + self.assertAlmostEqual(metrics["timing_per_token_ms/gen"], 0.5 * 1000 / 6, places=5) + + # ref and values use all tokens (12 tokens) + self.assertAlmostEqual(metrics["timing_per_token_ms/ref"], 0.3 * 1000 / 12, places=5) + self.assertAlmostEqual(metrics["timing_per_token_ms/values"], 0.2 * 1000 / 12, places=5) + + +class TestComputeThroughputMetrics(unittest.TestCase): + """Tests for the compute_throughout_metrics function.""" + + def setUp(self): + """Set up common test data.""" + # Create a mock DataProto object + self.batch = MagicMock() + self.batch.meta_info = { + "global_token_num": [100, 200, 300], # 600 tokens total + } + + def test_compute_throughout_metrics(self): + """Test compute_throughout_metrics with various timing data.""" + timing_raw = { + "step": 2.0, # 2 seconds per step + } + + # Test with 1 GPU + metrics = compute_throughout_metrics(self.batch, timing_raw, n_gpus=1) + + self.assertEqual(metrics["perf/total_num_tokens"], 600) + self.assertEqual(metrics["perf/time_per_step"], 2.0) + self.assertEqual(metrics["perf/throughput"], 600 / 2.0) # 300 tokens/sec + + # Test with 2 GPUs + metrics = compute_throughout_metrics(self.batch, timing_raw, n_gpus=2) + + self.assertEqual(metrics["perf/total_num_tokens"], 600) + self.assertEqual(metrics["perf/time_per_step"], 2.0) + self.assertEqual(metrics["perf/throughput"], 600 / (2.0 * 2)) # 150 tokens/sec/GPU + + +class TestBootstrapMetric(unittest.TestCase): + """Tests for the bootstrap_metric function.""" + + def test_bootstrap_metric_basic(self): + """Test bootstrap_metric with simple data and functions.""" + data = [1, 2, 3, 4, 5] + reduce_fns = [np.mean, np.max] + + # Use a fixed seed for reproducibility + result = bootstrap_metric(data, subset_size=3, reduce_fns=reduce_fns, n_bootstrap=100, seed=42) + + # Check that we get two results (one for each reduce_fn) + self.assertEqual(len(result), 2) + + # Each result should be a tuple of (mean, std) + mean_result, max_result = result + self.assertEqual(len(mean_result), 2) + self.assertEqual(len(max_result), 2) + + # The mean of means should be close to the true mean (3.0) + self.assertAlmostEqual(mean_result[0], 3.0, delta=0.3) + + # The mean of maxes should be close to the expected value for samples of size 3 + # For samples of size 3 from [1,2,3,4,5], the expected max is around 4.0-4.5 + self.assertGreater(max_result[0], 3.5) + self.assertLess(max_result[0], 5.0) + + def test_bootstrap_metric_empty(self): + """Test bootstrap_metric with empty data.""" + with self.assertRaises(ValueError): + bootstrap_metric([], subset_size=1, reduce_fns=[np.mean]) + + +class TestCalcMajVal(unittest.TestCase): + """Tests for the calc_maj_val function.""" + + def test_calc_maj_val_basic(self): + """Test calc_maj_val with simple data.""" + data = [ + {"pred": "A", "val": 0.9}, + {"pred": "B", "val": 0.8}, + {"pred": "A", "val": 0.7}, + ] + + result = calc_maj_val(data, vote_key="pred", val_key="val") + + # "A" is the majority vote, so we should get the first "val" for "A" + self.assertEqual(result, 0.9) + + def test_calc_maj_val_tie(self): + """Test calc_maj_val with tied votes.""" + data = [ + {"pred": "A", "val": 0.9}, + {"pred": "B", "val": 0.8}, + {"pred": "B", "val": 0.7}, + {"pred": "A", "val": 0.6}, + ] + + # In case of a tie, the first key in sorted order wins + # This depends on Python's dict implementation, but for this test + # we just verify that one of the valid values is returned + result = calc_maj_val(data, vote_key="pred", val_key="val") + + self.assertTrue(result in [0.9, 0.8]) + + +class TestProcessValidationMetrics(unittest.TestCase): + """Tests for the process_validation_metrics function.""" + + def test_process_validation_metrics_basic(self): + """Test process_validation_metrics with simple data.""" + data_sources = ["source1", "source1", "source2"] + sample_inputs = ["prompt1", "prompt1", "prompt2"] + infos_dict = { + "score": [0.8, 0.9, 0.7], + } + + result = process_validation_metrics(data_sources, sample_inputs, infos_dict, seed=42) + + # Check the structure of the result + self.assertIn("source1", result) + self.assertIn("source2", result) + + # Check that source1 has metrics for score + self.assertIn("score", result["source1"]) + + # Check that mean@2 is present for source1/score + self.assertIn("mean@2", result["source1"]["score"]) + + # Check the value of mean@2 for source1/score + self.assertAlmostEqual(result["source1"]["score"]["mean@2"], 0.85) + + def test_process_validation_metrics_with_pred(self): + """Test process_validation_metrics with prediction data.""" + data_sources = ["source1", "source1", "source1"] + sample_inputs = ["prompt1", "prompt1", "prompt1"] + infos_dict = { + "score": [0.8, 0.9, 0.7], + "pred": ["A", "B", "A"], + } + + result = process_validation_metrics(data_sources, sample_inputs, infos_dict, seed=42) + + # Check that majority voting metrics are present + self.assertIn("maj@2/mean", result["source1"]["score"]) + + # For bootstrap with n=2, the majority vote could be either A or B + # depending on the random sampling, so we don't check the exact value + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/utils/_test_module.py b/e2c/verl/tests/utils/_test_module.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3d5fb6596c2bf7adf1ccec47f191b192398872 --- /dev/null +++ b/e2c/verl/tests/utils/_test_module.py @@ -0,0 +1,31 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Test module for import_utils.load_extern_type testing +class TestClass: + """A test class to be imported by load_extern_type""" + + def __init__(self, value=None): + self.value = value or "default" + + def get_value(self): + return self.value + + +TEST_CONSTANT = "test_constant_value" + + +def test_function(): + return "test_function_result" diff --git a/e2c/verl/tests/utils/ckpt/test_esi_save_ckpt_on_cpu.py b/e2c/verl/tests/utils/ckpt/test_esi_save_ckpt_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..203494bd90bd9676fd615f5db5576e94c0219ee9 --- /dev/null +++ b/e2c/verl/tests/utils/ckpt/test_esi_save_ckpt_on_cpu.py @@ -0,0 +1,70 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import time +from datetime import datetime, timedelta +from unittest import TestCase + +from verl.utils.checkpoint.checkpoint_manager import should_save_ckpt_esi + + +class TestShouldSaveCkptEsi(TestCase): + def test_no_expiration_timestamp(self): + """Test case when no expiration timestamp is set""" + os.environ.pop("MLP_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP", None) + os.environ.pop("SAGEMAKER_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP", None) + self.assertFalse(should_save_ckpt_esi(100)) + + def test_mlp_expiration_valid(self): + """Test valid MLP expiration timestamp requiring save""" + current_time = time.time() + os.environ["MLP_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP"] = str(current_time + 90) + self.assertTrue(should_save_ckpt_esi(30)) # max_steps_duration=30 seconds + + def test_mlp_expiration_passed(self): + """Test expired MLP timestamp""" + current_time = time.time() + os.environ["MLP_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP"] = str(current_time - 10) + self.assertFalse(should_save_ckpt_esi(30)) + + def test_mlp_invalid_timestamp(self): + """Test invalid MLP timestamp format""" + os.environ["MLP_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP"] = "invalid" + self.assertFalse(should_save_ckpt_esi(30)) + + def test_mlp_expiration_not_reached(self): + """Test MLP expiration timestamp with insufficient remaining time""" + current_time = time.time() + os.environ["MLP_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP"] = str(current_time + 200) + self.assertFalse(should_save_ckpt_esi(30)) # max_steps_duration=30 + + def test_aws_expiration_not_reached(self): + """Test AWS expiration timestamp with sufficient remaining time""" + now = datetime.now() + expiration = now + timedelta(minutes=100) # Exceeds 90-minute threshold + os.environ["SAGEMAKER_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP"] = str(int(expiration.timestamp())) + self.assertFalse(should_save_ckpt_esi(30 * 60)) + + def test_redundant_time(self): + """Test redundant_time parameter effect""" + current_time = time.time() + # Total required: 60+30+30=120 seconds + os.environ["MLP_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP"] = str(current_time + 120) + self.assertTrue(should_save_ckpt_esi(30, redundant_time=30)) + + def test_zero_max_steps_duration(self): + """Test zero max_steps_duration""" + current_time = time.time() + os.environ["MLP_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP"] = str(current_time + 60) + self.assertFalse(should_save_ckpt_esi(0)) diff --git a/e2c/verl/tests/utils/dataset/test_create_rl_sampler_on_cpu.py b/e2c/verl/tests/utils/dataset/test_create_rl_sampler_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..35bf5a3ab5bd32544b2eec487e96ef61312766b9 --- /dev/null +++ b/e2c/verl/tests/utils/dataset/test_create_rl_sampler_on_cpu.py @@ -0,0 +1,108 @@ +# Copyright 2025 Amazon.com Inc and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +test create_rl_sampler +""" + +from collections.abc import Sized + +import pytest +import torch +from omegaconf import DictConfig, OmegaConf +from torch.utils.data import Dataset, RandomSampler + +from verl.experimental.dataset.sampler import AbstractCurriculumSampler +from verl.trainer.main_ppo import create_rl_sampler + + +class RandomCurriculumSampler(AbstractCurriculumSampler): + def __init__( + self, + data_source: Sized, + data_config: DictConfig, + ): + train_dataloader_generator = torch.Generator() + train_dataloader_generator.manual_seed(1) + sampler = RandomSampler(data_source=data_source) + self.sampler = sampler + + def __iter__(self): + return self.sampler.__iter__() + + def __len__(self) -> int: + return len(self.sampler) + + def update(self, batch) -> None: + return + + +class MockIncorrectSampler: + """A fake sampler class that does not adhere to the AbstractCurriculumSampler interface.""" + + def __init__(self, data_source, data_config): + pass + + +class MockChatDataset(Dataset): + def __init__(self): + self.data = [ + {"prompt": "What's your name?", "response": "My name is Assistant."}, + {"prompt": "How are you?", "response": "I'm doing well, thank you."}, + {"prompt": "What is the capital of France?", "response": "Paris."}, + { + "prompt": "Tell me a joke.", + "response": "Why did the chicken cross the road? To get to the other side!", + }, + {"prompt": "What is 2+2?", "response": "4"}, + ] + + def __getitem__(self, index): + return self.data[index] + + def __len__(self): + return len(self.data) + + +def test_create_custom_curriculum_samper(): + data_config = OmegaConf.create( + { + "dataloader_num_workers": 0, + "sampler": { + "class_path": "pkg://tests.utils.dataset.test_create_rl_sampler_on_cpu", + "class_name": "RandomCurriculumSampler", + }, + } + ) + + dataset = MockChatDataset() + + # doesn't raise + create_rl_sampler(data_config, dataset) + + +def test_create_custom_curriculum_samper_wrong_class(): + data_config = OmegaConf.create( + { + "sampler": { + "class_path": "pkg://tests.utils.dataset.test_create_rl_sampler_on_cpu", + "class_name": "MockIncorrectSampler", + } + } + ) + + dataset = MockChatDataset() + + # MockIncorrectSampler is not an instance of AbstractCurriculumSampler, so raises + with pytest.raises(AssertionError): + create_rl_sampler(data_config, dataset) diff --git a/e2c/verl/tests/utils/dataset/test_multiturn_sft_dataset_on_cpu.py b/e2c/verl/tests/utils/dataset/test_multiturn_sft_dataset_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..6e7960a45e90775a8b93adac63d45fecd11f7925 --- /dev/null +++ b/e2c/verl/tests/utils/dataset/test_multiturn_sft_dataset_on_cpu.py @@ -0,0 +1,204 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test the MultiTurnSFTDataset implementation +""" + +import os + +import pandas as pd +import torch +from transformers import AutoTokenizer + +from verl.utils.dataset.multiturn_sft_dataset import MultiTurnSFTDataset + + +def test_multiturn_sft_dataset(): + print("Starting test...") + # Create a temporary parquet file with test data + test_data = { + "messages": [ + [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": "2+2 equals 4."}, + {"role": "user", "content": "And what is 4+4?"}, + {"role": "assistant", "content": "4+4 equals 8."}, + ], + [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Tell me a joke."}, + {"role": "assistant", "content": "Why did the chicken cross the road?"}, + {"role": "user", "content": "Why?"}, + {"role": "assistant", "content": "To get to the other side!"}, + ], + ] + } + + # Create test directory if it doesn't exist + os.makedirs("test_data", exist_ok=True) + test_file = "test_data/test.parquet" + + # Save test data to parquet + df = pd.DataFrame(test_data) + df.to_parquet(test_file) + + # Initialize tokenizer and dataset + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-7B-Instruct") + config = {"max_length": 512, "truncation": "error", "multiturn": {"messages_key": "messages"}} + dataset = MultiTurnSFTDataset(parquet_files=test_file, tokenizer=tokenizer, config=config) + + # Test 1: Dataset Length + assert len(dataset) == 2, f"Expected dataset length 2, got {len(dataset)}" + + # Get items for testing + item0 = dataset[0] # Math conversation + item1 = dataset[1] # Joke conversation + + # Test 2: Required Keys and Types + required_keys = ["input_ids", "attention_mask", "position_ids", "loss_mask"] + for key in required_keys: + assert key in item0, f"Missing key {key} in dataset item" + assert isinstance(item0[key], torch.Tensor), f"Expected torch.Tensor for {key}" + assert item0[key].dtype == torch.long, f"Expected torch.long for {key}, got {item0[key].dtype}" + + # Test 3: Shape Consistency + assert item0["loss_mask"].shape == item0["input_ids"].shape, "Loss mask shape doesn't match input_ids shape" + assert item0["attention_mask"].shape == item0["input_ids"].shape, ( + "Attention mask shape doesn't match input_ids shape" + ) + assert item0["position_ids"].shape == item0["input_ids"].shape, "Position IDs shape doesn't match input_ids shape" + + # Test 4: Loss Mask Pattern - Math Conversation + loss_mask0 = item0["loss_mask"] + input_ids0 = item0["input_ids"] + + # Find assistant response positions + assistant_positions0 = torch.where(loss_mask0 == 1)[0] + assert len(assistant_positions0) > 0, "No assistant positions found in loss mask" + + # Decode and verify assistant responses + assistant_text0 = tokenizer.decode(input_ids0[loss_mask0 == 1]) + print(f"Math conversation assistant text: {assistant_text0}") + assert "2+2 equals 4" in assistant_text0, "First assistant response not found" + assert "4+4 equals 8" in assistant_text0, "Second assistant response not found" + + # Test 5: Loss Mask Pattern - Joke Conversation + loss_mask1 = item1["loss_mask"] + input_ids1 = item1["input_ids"] + + # Find assistant response positions + assistant_positions1 = torch.where(loss_mask1 == 1)[0] + assert len(assistant_positions1) > 0, "No assistant positions found in loss mask" + + # Decode and verify assistant responses + assistant_text1 = tokenizer.decode(input_ids1[loss_mask1 == 1]) + print(f"Joke conversation assistant text: {assistant_text1}") + assert "chicken cross the road" in assistant_text1, "First assistant response not found" + assert "other side" in assistant_text1, "Second assistant response not found" + + # Test 6: Attention Mask Pattern + attention_mask0 = item0["attention_mask"] + sequence_length = torch.sum(attention_mask0) + assert sequence_length > 0, "No tokens marked as attended in attention mask" + assert torch.all(attention_mask0[:sequence_length] == 1), "Incorrect attention mask pattern" + if sequence_length < len(attention_mask0): + assert torch.all(attention_mask0[sequence_length:] == 0), "Padding not properly masked" + + # Test 7: Position IDs Pattern + position_ids0 = item0["position_ids"] + assert torch.equal(position_ids0[:sequence_length], torch.arange(sequence_length)), ( + "Position IDs not sequential for non-padded tokens" + ) + if sequence_length < len(position_ids0): + assert torch.all(position_ids0[sequence_length:] == 0), "Padding position IDs not zero" + + # Test 8: Verify loss mask for assistant responses + # Get the full conversation text + full_text = tokenizer.decode(input_ids0) + print(f"\nFull conversation text:\n{full_text}") + + # Get the assistant responses + assistant_text = tokenizer.decode(input_ids0[loss_mask0 == 1]) + print(f"\nAssistant responses (from loss mask):\n{assistant_text}") + + # Verify that loss mask is set for all assistant responses + for msg in test_data["messages"][0]: # First conversation + if msg["role"] == "assistant": + # The content should appear in the masked text + assert msg["content"] in assistant_text, f"Assistant message '{msg['content']}' not found in masked text" + + # The content should NOT appear in the non-masked text + non_assistant_text = tokenizer.decode(input_ids0[loss_mask0 == 0]) + assert msg["content"] not in non_assistant_text, ( + f"Assistant message '{msg['content']}' found in non-assistant text" + ) + + # Test 9: Verify non-assistant parts have loss_mask=0 + # Get non-assistant text + non_assistant_text = tokenizer.decode(input_ids0[loss_mask0 == 0]) + print(f"\nNon-assistant text (from loss mask):\n{non_assistant_text}") + + # Verify that system and user messages are in the non-assistant text + for msg in test_data["messages"][0]: # First conversation + if msg["role"] in ["system", "user"]: + assert msg["content"] in non_assistant_text, ( + f"{msg['role'].title()} message '{msg['content']}' not found in non-assistant text" + ) + + # And verify they're NOT in the assistant text + assert msg["content"] not in assistant_text, ( + f"{msg['role'].title()} message '{msg['content']}' found in assistant text" + ) + + # Test 10: Verify padding behavior + padding_config = {"max_length": 1024, "truncation": "error", "multiturn": {"messages_key": "messages"}} + small_dataset = MultiTurnSFTDataset(parquet_files=test_file, tokenizer=tokenizer, config=padding_config) + padded_item = small_dataset[0] + + # Get actual sequence length (before padding) + actual_length = torch.sum(padded_item["attention_mask"]) + + # Verify padding tokens + assert torch.all(padded_item["input_ids"][actual_length:] == tokenizer.pad_token_id), ( + "Padding tokens not set correctly" + ) + assert torch.all(padded_item["attention_mask"][actual_length:] == 0), "Attention mask not set correctly for padding" + assert torch.all(padded_item["loss_mask"][actual_length:] == 0), "Loss mask not set correctly for padding" + + # test left right padding + config = { + "max_length": 512, + "truncation": "error", + "multiturn": {"messages_key": "messages"}, + "pad_mode": "left_right", + "max_prompt_length": 64, + "max_response_length": 64, + } + dataset = MultiTurnSFTDataset(parquet_files=test_file, tokenizer=tokenizer, config=config) + + item0 = dataset[0] + + # make sure all the input_ids with attention_mask == 0 are all padding + assert torch.all(item0["input_ids"][item0["attention_mask"] == 0] == tokenizer.pad_token_id) + + # make sure assistant_text matches with expected + assistant_text = tokenizer.decode(item0["responses"][item0["response_mask"] == 1]) + assert assistant_text == "2+2 equals 4.<|im_end|>\n4+4 equals 8.<|im_end|>\n" + + # make sure responses are part of input_ids + assert torch.all(item0["input_ids"][-item0["responses"].shape[0] :] == item0["responses"]) + + print("All tests passed!") + print("Starting test...") diff --git a/e2c/verl/tests/utils/dataset/test_rl_collate_fn_on_cpu.py b/e2c/verl/tests/utils/dataset/test_rl_collate_fn_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..415595295e7fde5d4de648284091bc87c53b4a10 --- /dev/null +++ b/e2c/verl/tests/utils/dataset/test_rl_collate_fn_on_cpu.py @@ -0,0 +1,72 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + + +def test_rl_collate_fn(): + from verl.utils.dataset.rl_dataset import collate_fn + + max_prompt_length = 5 + + test_data = [ + { + # test tensor + "input_ids": torch.randint(0, 10, (max_prompt_length,)), + # test fixed length (1) list within a batch + "messages": [{"role": "user", "content": "Hi."}], + # test variable length list within a batch + "raw_prompt_ids": [1, 2, 3, 4], + # test string + "ability": "math", + # test dict + "reward_model": {"ground_truth": 5, "style": "rule"}, + # test empty dict + "tools_kwargs": {}, + }, + { + "input_ids": torch.randint(0, 10, (max_prompt_length,)), + "messages": [{"role": "user", "content": "Hello."}], + "raw_prompt_ids": [1, 2, 3], + "ability": "toolcall", + "reward_model": { + "ground_truth": '[{"name": "rgb_to_cmyk", "arguments": {"r": 0, "g": 0, "b": 255}}]', + "style": "rule", + }, + "tools_kwargs": {}, + }, + ] + + batch_size = len(test_data) + batch = collate_fn(test_data) + + # Tensor part + assert batch["input_ids"].shape == (batch_size, max_prompt_length) + assert isinstance(batch["input_ids"], torch.Tensor) + + # Non-tensor parts + expected_types = { + "messages": list, + "raw_prompt_ids": list, + "ability": str, + "reward_model": dict, + "tools_kwargs": dict, + } + + for key, dtype in expected_types.items(): + assert batch[key].shape == (batch_size,), ( + f"Expected shape {(batch_size,)} for '{key}', but got {batch[key].shape}" + ) + assert isinstance(batch[key][0], dtype), ( + f"'{key}' should contain elements of type {dtype}, but got {type(batch[key][0])}" + ) diff --git a/e2c/verl/tests/utils/dataset/test_rl_dataset_on_cpu.py b/e2c/verl/tests/utils/dataset/test_rl_dataset_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..391e89a94d57f1e4f63e1c6fe61737881469146d --- /dev/null +++ b/e2c/verl/tests/utils/dataset/test_rl_dataset_on_cpu.py @@ -0,0 +1,113 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import torch +from omegaconf import OmegaConf +from torch.utils.data import DataLoader + + +def get_gsm8k_data(): + # prepare test dataset + local_folder = os.path.expanduser("~/verl-data/gsm8k/") + local_path = os.path.join(local_folder, "train.parquet") + os.makedirs(local_folder, exist_ok=True) + return local_path + + +def test_rl_dataset(): + from verl.utils import hf_tokenizer + from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn + + tokenizer = hf_tokenizer("deepseek-ai/deepseek-coder-1.3b-instruct") + local_path = get_gsm8k_data() + config = OmegaConf.create( + { + "prompt_key": "prompt", + "max_prompt_length": 256, + "filter_overlong_prompts": True, + "filter_overlong_prompts_workers": 2, + } + ) + dataset = RLHFDataset(data_files=local_path, tokenizer=tokenizer, config=config) + + dataloader = DataLoader(dataset=dataset, batch_size=16, shuffle=True, drop_last=True, collate_fn=collate_fn) + + a = next(iter(dataloader)) + + from verl import DataProto + + tensors = {} + non_tensors = {} + + for key, val in a.items(): + if isinstance(val, torch.Tensor): + tensors[key] = val + else: + non_tensors[key] = val + + data_proto = DataProto.from_dict(tensors=tensors, non_tensors=non_tensors) + assert "input_ids" in data_proto.batch + + data = dataset[0]["input_ids"] + output = tokenizer.batch_decode([data])[0] + print(f"type: type{output}") + print(f"\n\noutput: {output}") + + +def test_image_rl_data(): + from verl.utils import hf_processor, hf_tokenizer + from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn + + tokenizer = hf_tokenizer("Qwen/Qwen2-VL-2B-Instruct") + processor = hf_processor("Qwen/Qwen2-VL-2B-Instruct") + config = OmegaConf.create( + { + "prompt_key": "prompt", + "max_prompt_length": 1024, + "filter_overlong_prompts": True, + "filter_overlong_prompts_workers": 1, + } + ) + dataset = RLHFDataset( + data_files=os.path.expanduser("~/data/geo3k/train.parquet"), + tokenizer=tokenizer, + config=config, + processor=processor, + ) + + dataloader = DataLoader(dataset=dataset, batch_size=16, shuffle=True, drop_last=True, collate_fn=collate_fn) + + a = next(iter(dataloader)) + + from verl import DataProto + + tensors = {} + non_tensors = {} + + for key, val in a.items(): + if isinstance(val, torch.Tensor): + tensors[key] = val + else: + non_tensors[key] = val + + data_proto = DataProto.from_dict(tensors=tensors, non_tensors=non_tensors) + + assert "multi_modal_data" in data_proto.non_tensor_batch, data_proto + assert "multi_modal_inputs" in data_proto.non_tensor_batch, data_proto + + data = dataset[0]["input_ids"] + output = tokenizer.batch_decode([data])[0] + print(f"type: type{output}") + print(f"\n\noutput: {output}") diff --git a/e2c/verl/tests/utils/dataset/test_sft_dataset_on_cpu.py b/e2c/verl/tests/utils/dataset/test_sft_dataset_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..680fce45a2a433d05544faba1cd76969587cc37c --- /dev/null +++ b/e2c/verl/tests/utils/dataset/test_sft_dataset_on_cpu.py @@ -0,0 +1,74 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from verl.utils import hf_tokenizer +from verl.utils.dataset.sft_dataset import SFTDataset + + +def get_gsm8k_data(): + # prepare test dataset + local_folder = os.path.expanduser("~/verl-data/gsm8k/") + local_path = os.path.join(local_folder, "train.parquet") + return local_path + + +def test_sft_cot_dataset(): + tokenizer = hf_tokenizer("deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct") + local_path = get_gsm8k_data() + from omegaconf import OmegaConf + + dataset = SFTDataset( + parquet_files=local_path, + tokenizer=tokenizer, + config=OmegaConf.create( + { + "prompt_key": "prompt", + "prompt_dict_keys": ["content"], + "response_key": "extra_info", + "response_dict_keys": ["answer"], + "max_length": 512, + } + ), + ) + + data = dataset[0]["input_ids"] + output = tokenizer.batch_decode([data])[0] + assert len(output) > 1 + assert isinstance(output, str) + + +def test_sft_dataset(): + tokenizer = hf_tokenizer("deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct") + local_path = get_gsm8k_data() + from omegaconf import OmegaConf + + dataset = SFTDataset( + parquet_files=local_path, + tokenizer=tokenizer, + config=OmegaConf.create( + { + "prompt_key": "extra_info", + "prompt_dict_keys": ["question"], + "response_key": "extra_info", + "response_dict_keys": ["answer"], + "max_length": 512, + } + ), + ) + + data = dataset[0]["input_ids"] + output = tokenizer.batch_decode([data])[0] + assert len(output) > 1 + assert isinstance(output, str) diff --git a/e2c/verl/tests/utils/debug/test_metrics.py b/e2c/verl/tests/utils/debug/test_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..1b2f7f8faa17dd024df4b92c3a3b1b81d48923e0 --- /dev/null +++ b/e2c/verl/tests/utils/debug/test_metrics.py @@ -0,0 +1,48 @@ +# Copyright 2025 Individual Contributor: TomQunChaoA +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import torch + +from verl.protocol import DataProto +from verl.utils.debug.metrics import calculate_debug_metrics + + +class TestMetrics(unittest.TestCase): + def test_calculate_debug_metrics(self): + data = DataProto.from_dict( + { + "rollout_log_probs": torch.tensor( + [ + [-1.5085, -0.1200, -0.6650, -0.4823, -0.1426, -1.5557, -2.8532, -0.3919, -0.4294, -0.4700], + [-0.0585, -0.0573, -0.4681, -0.5187, -0.7451, -1.2737, -0.0682, -0.4284, -0.5754, -0.0611], + ] + ), + "old_log_probs": torch.tensor( + [ + [-1.8636, -0.7863, -0.2136, -0.4376, -2.0257, -0.2579, -1.1547, -0.5203, -0.3802, -0.9872], + [-0.3507, -0.5426, -0.2725, -0.4637, -0.3577, -0.3733, -1.7560, -1.9542, -0.4229, -1.3098], + ] + ), + "loss_mask": torch.tensor([[1, 0, 0, 0, 1, 1, 0, 1, 1, 0], [1, 0, 1, 0, 1, 1, 1, 0, 1, 1]]), + "responses": torch.zeros((2, 10)), + } + ) + metrics = calculate_debug_metrics(data) + print(metrics) + assert metrics["training/rollout_probs_diff_valid"] == 1 + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/utils/megatron/test_pipeline_parallel.py b/e2c/verl/tests/utils/megatron/test_pipeline_parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..24a416987dae68089a3d26d18f34d5defbd14245 --- /dev/null +++ b/e2c/verl/tests/utils/megatron/test_pipeline_parallel.py @@ -0,0 +1,70 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from verl.model_merger.megatron_model_merger import get_dynamic_pipeline_shards +from verl.utils.megatron.pipeline_parallel import make_batch_generator + + +def test_make_batch_generator_no_vpp(): + batches = [1, 2, 3] + vpp_size = 1 + generator = make_batch_generator(batches, vpp_size) + assert list(generator) == batches + + +def test_make_batch_generator_with_vpp(): + batches = [{"data": 1}, {"data": 2}] + vpp_size = 2 + generators = make_batch_generator(batches, vpp_size) + assert isinstance(generators, list) + assert len(generators) == vpp_size + + # Check each generator yields the original batches + for gen in generators: + assert list(gen) == batches + + +def test_make_batch_generator_empty(): + batches = [] + vpp_size = 1 + generator = make_batch_generator(batches, vpp_size) + assert list(generator) == [] + + vpp_size = 3 + generators = make_batch_generator(batches, vpp_size) + assert len(generators) == vpp_size + for gen in generators: + assert list(gen) == [] + + +@pytest.mark.parametrize( + "layer_num,pp_size,gt", + [ + (61, 8, [6, 8, 8, 8, 8, 8, 8, 7]), + (61, 7, [8, 9, 9, 9, 9, 9, 8]), + (61, 1, [61]), + (61, 0, ValueError), + (10, 16, ValueError), + ], +) +def test_get_dynamic_pipeline_shards(layer_num, pp_size, gt): + if isinstance(gt, list): + shards = get_dynamic_pipeline_shards(layer_num, pp_size) + assert len(shards) == len(gt) == pp_size, f"Expected {pp_size} shards, got {len(shards)}" + assert all([shard == gt[i] for i, shard in enumerate(shards)]), f"Expected shards {gt}, got {shards}" + elif issubclass(gt, Exception): + with pytest.raises(gt): + shards = get_dynamic_pipeline_shards(layer_num, pp_size) diff --git a/e2c/verl/tests/utils/reward_score/reward_score/test_sandbox_fusion_on_cpu.py b/e2c/verl/tests/utils/reward_score/reward_score/test_sandbox_fusion_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..83aed24d054ddce33bc8fd311de2705fcca24776 --- /dev/null +++ b/e2c/verl/tests/utils/reward_score/reward_score/test_sandbox_fusion_on_cpu.py @@ -0,0 +1,747 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import multiprocessing +import os +import time +from concurrent.futures import ProcessPoolExecutor +from unittest.mock import patch + +import pytest + +# Import the function to be tested +from verl.utils.reward_score.sandbox_fusion.utils import check_correctness + +# Get SANDBOX_URL from environment variable +SANDBOX_URL = os.environ.get("SANDBOX_FUSION_URL") +# Define skip condition and reason +skip_reason = "SANDBOX_FUSION_URL environment variable not set" +skip_condition = not SANDBOX_URL + +# --- Test code (for real API calls) --- +CODE_SUCCESS = """ +import sys +data = sys.stdin.read() +if data == 'input1': + print('output1\\n', end='') +elif data == 'input2': + print('output2\\n', end='') +else: + print('unexpected input', end='') +""" + +CODE_WRONG_OUTPUT = """ +print('wrong_output\\n', end='') +""" + +CODE_COMPILE_ERROR = """ +a=b +""" + +CODE_RUNTIME_ERROR = """ +import sys +print("About to raise error", file=sys.stderr) +raise ValueError("This is a runtime error") +""" + +CODE_TIMEOUT = """ +import time +import sys +print("Sleeping...", file=sys.stderr) +time.sleep(10) # Sleep time should be longer than the timeout set in the test +print("Finished sleeping", file=sys.stderr) +""" + +# --- Test input/output data --- +INPUT_OUTPUT_VALID = {"inputs": ["input1", "input2"], "outputs": ["output1\n", "output2\n"]} + +INPUT_OUTPUT_SINGLE = {"inputs": ["input1"], "outputs": ["output1\n"]} + +INPUT_OUTPUT_MISMATCH = {"inputs": ["input1"], "outputs": ["output1\n", "output2\n"]} + +INPUT_OUTPUT_INVALID_MISSING_KEY = {"inputs": ["input1"]} + +# --- Integration test cases (calling real API) --- + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_integration_success_correct(): + """Integration test: Code is correct, output is correct""" + results, metadata_list = check_correctness(SANDBOX_URL, INPUT_OUTPUT_VALID, CODE_SUCCESS) + assert results == [True, True] + assert metadata_list[0]["status"] == "success" + assert metadata_list[0]["stdout"] == "output1\n" + assert metadata_list[1]["status"] == "success" + assert metadata_list[1]["stdout"] == "output2\n" + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_integration_success_wrong_output(): + """Integration test: Code runs successfully, but output is wrong""" + results, metadata_list = check_correctness(SANDBOX_URL, INPUT_OUTPUT_VALID, CODE_WRONG_OUTPUT) + assert results == [False, False] + assert metadata_list[0]["status"] == "wrong_answer" + assert metadata_list[0]["stdout"] == "wrong_output\n" + assert metadata_list[1]["status"] == "wrong_answer" + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_integration_compile_error(): + """Integration test: Code causes compile error""" + results, metadata_list = check_correctness(SANDBOX_URL, INPUT_OUTPUT_VALID, CODE_COMPILE_ERROR, language="cpp") + assert results == [-4, -4] + assert metadata_list[0]["status"] == "compile_error" + assert metadata_list[1]["status"] == "compile_error" + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_integration_runtime_error(): + """Integration test: Code causes runtime error""" + results, metadata_list = check_correctness(SANDBOX_URL, INPUT_OUTPUT_SINGLE, CODE_RUNTIME_ERROR) + assert results == [-2] + assert metadata_list[0]["status"] == "runtime_error" + # More assertions can be added based on the actual API response, e.g., exit_code, stderr + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_integration_runtime_timeout(): + """Integration test: Code causes runtime timeout""" + test_timeout = 5 # Set a timeout shorter than the sleep time in CODE_TIMEOUT + results, metadata_list = check_correctness(SANDBOX_URL, INPUT_OUTPUT_SINGLE, CODE_TIMEOUT, timeout=test_timeout) + assert results == [-3] + assert metadata_list[0]["status"] == "timeout" + # More assertions can be added based on the actual API response, e.g., run_status + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_integration_concurrency_high_load(): + """Integration test: High concurrency (100 cases) against real API with mixed results (success, wrong + answer, timeout)""" + concurrency_level = 100 + # Indices for different expected outcomes + wrong_answer_indices = {10, 25, 50} + timeout_indices = {5, 30, 60, 90} # Indices where we expect a timeout + + # Generate 100 input/output pairs and code + high_load_inputs = [] + high_load_outputs = [] + expected_results_map = {} # Store expected result for each index + + for i in range(concurrency_level): + if i in timeout_indices: + # Use a special input to trigger timeout in the code + high_load_inputs.append(f"input_timeout_{i}") + # Output doesn't matter for timeout, but keep it consistent + high_load_outputs.append(f"output_{i}\n") + expected_results_map[i] = -3 # Expect timeout + elif i in wrong_answer_indices: + high_load_inputs.append(f"input_{i}") + # Intentionally set wrong expected output + high_load_outputs.append(f"wrong_output_{i}\n") + expected_results_map[i] = False # Expect wrong answer + else: + high_load_inputs.append(f"input_{i}") + # Correct expected output + high_load_outputs.append(f"output_{i}\n") + expected_results_map[i] = True # Expect success + + high_load_in_outs = {"inputs": high_load_inputs, "outputs": high_load_outputs} + + # Code that handles normal inputs, and sleeps on specific "timeout" inputs + code_mixed_concurrent = """ +import sys +import time +data = sys.stdin.read() +if data.startswith('input_timeout_'): + time.sleep(20) # Sleep longer than the test timeout + print(f"output_{data.split('_')[-1]}\\n", end='') # Still print something in case it finishes early +elif data.startswith('input_'): + print(f"output_{data.split('_')[-1]}\\n", end='') +else: + print("unknown_input\\n", end='') +""" + # Set a reasonable timeout per case (must be less than the sleep time in the code) + test_timeout = 15 # Allow slightly more time due to potential API load, but less than 20s sleep + + start_time = time.time() + results, metadata_list = check_correctness( + SANDBOX_URL, + high_load_in_outs, + code_mixed_concurrent, # Use the new code + timeout=test_timeout, + ) + end_time = time.time() + duration = end_time - start_time + print( + f"\nHigh concurrency test ({concurrency_level} cases with {len(wrong_answer_indices)} wrong answers, " + f"{len(timeout_indices)} timeouts) duration: {duration:.2f} seconds" + ) + + # Verify results against the expected map + assert len(results) == concurrency_level, f"Expected {concurrency_level} results, got {len(results)}" + + correct_count = 0 + wrong_count = 0 + timeout_count = 0 + unexpected_results = [] + for i, r in enumerate(results): + expected = expected_results_map[i] + if r == expected: + if expected is True: + correct_count += 1 + elif expected is False: + wrong_count += 1 + elif expected == -3: + timeout_count += 1 + else: + unexpected_results.append((i, r, f"Expected {expected}")) + + print( + f"Correct results (True): {correct_count}/" + f"{concurrency_level - len(wrong_answer_indices) - len(timeout_indices)}" + ) + print(f"Expected wrong answers (False, correctly identified): {wrong_count}/{len(wrong_answer_indices)}") + print(f"Expected timeouts (-3, correctly identified): {timeout_count}/{len(timeout_indices)}") + + if unexpected_results: + print("Unexpected results found:") + for idx, res, expected_str in unexpected_results[:10]: # Print first 10 unexpected + print(f" Index {idx}: Got {res}, {expected_str}. Metadata: {metadata_list[idx]}") + raise AssertionError(f"Found {len(unexpected_results)} unexpected results.") + + assert correct_count == concurrency_level - len(wrong_answer_indices) - len(timeout_indices), ( + "Incorrect number of successful results" + ) + assert wrong_count == len(wrong_answer_indices), "Incorrect number of identified wrong answers" + assert timeout_count == len(timeout_indices), "Incorrect number of identified timeouts" + + # Verify metadata count and basic status of one of each type + assert len(metadata_list) == concurrency_level + # Find the first correct index + first_correct_index = next( + i for i in range(concurrency_level) if i not in wrong_answer_indices and i not in timeout_indices + ) + assert metadata_list[first_correct_index]["status"] == "success" + assert metadata_list[first_correct_index]["stdout"] == f"output_{first_correct_index}\n" + + # Check the status of the first intentionally wrong case + first_wrong_index = min(wrong_answer_indices) + assert metadata_list[first_wrong_index]["status"] == "wrong_answer" + assert metadata_list[first_wrong_index]["stdout"] == f"output_{first_wrong_index}\n" + assert metadata_list[first_wrong_index]["expected_output"] == f"wrong_output_{first_wrong_index}\n" + + # Check the status of the first intentionally timeout case + first_timeout_index = min(timeout_indices) + assert metadata_list[first_timeout_index]["status"] == "timeout" + # For timeout, stdout might be None or empty depending on when the timeout occurred + # assert metadata_list[first_timeout_index]["stdout"] is None or metadata_list[first_timeout_index]["stdout"] == "" + + +# --- Unit test cases (using mock) --- + + +@patch("verl.utils.reward_score.sandbox_fusion.utils.call_sandbox_api") +def test_unit_concurrency_order(mock_call_sandbox_api): + sandbox_url = "mock_url" + generation = "print(input())" + language = "python" + timeout = 5 + in_outs = {"inputs": ["input1", "input2", "input3"], "outputs": ["output1", "output2", "output3"]} + + def side_effect(*args, **kwargs): + stdin = kwargs.get("stdin") + if stdin == "input1": + return ( + {"status": "Success", "run_result": {"status": "Finished", "stdout": "output1", "return_code": 0}}, + None, + ) + elif stdin == "input2": + time.sleep(0.1) + return ( + {"status": "Success", "run_result": {"status": "Finished", "stdout": "output2", "return_code": 0}}, + None, + ) + elif stdin == "input3": + return ( + {"status": "Success", "run_result": {"status": "Finished", "stdout": "output3", "return_code": 0}}, + None, + ) + else: + return (None, "Unknown input in mock") + + mock_call_sandbox_api.side_effect = side_effect + + results, metadata_list = check_correctness(sandbox_url, in_outs, generation, timeout, language) + + assert results == [True, True, True] + assert len(metadata_list) == 3 + assert metadata_list[0]["case_index"] == 0 + assert metadata_list[0]["status"] == "success" + assert metadata_list[1]["case_index"] == 1 + assert metadata_list[1]["status"] == "success" + assert metadata_list[2]["case_index"] == 2 + assert metadata_list[2]["status"] == "success" + assert mock_call_sandbox_api.call_count == 3 + + +@patch("verl.utils.reward_score.sandbox_fusion.utils.call_sandbox_api") +def test_unit_api_timeout_error_concurrent(mock_call_sandbox_api): + sandbox_url = "mock_url" + generation = "print(input())" + language = "python" + timeout = 5 + in_outs = {"inputs": ["input1", "input2_timeout", "input3"], "outputs": ["output1", "output2", "output3"]} + + api_error_message = "API Call Failed: Gateway Timeout (504) on attempt 3/3" + + def side_effect(*args, **kwargs): + stdin = kwargs.get("stdin") + if stdin == "input1": + return ( + {"status": "Success", "run_result": {"status": "Finished", "stdout": "output1", "return_code": 0}}, + None, + ) + elif stdin == "input2_timeout": + return (None, api_error_message) + elif stdin == "input3": + return ( + {"status": "Success", "run_result": {"status": "Finished", "stdout": "output3", "return_code": 0}}, + None, + ) + else: + return (None, "Unknown input in mock") + + mock_call_sandbox_api.side_effect = side_effect + + results, metadata_list = check_correctness(sandbox_url, in_outs, generation, timeout, language) + + assert results == [True, -1, True] + assert len(metadata_list) == 3 + assert metadata_list[0]["status"] == "success" + assert metadata_list[1]["status"] == "api_error" + assert metadata_list[1]["api_request_error"] == api_error_message + assert metadata_list[2]["status"] == "success" + assert mock_call_sandbox_api.call_count == 3 + + +# --- Constants for the new concurrency test --- +# Define a low global concurrency limit to test the semaphore's effect +MAX_GLOBAL_CONCURRENCY_LIMIT_TEST = 5 +# Define the number of processes used in the test +NUM_PROCESSES_TEST = 4 +# Define the number of tasks processed by check_correctness in each process (i.e., internal +# ThreadPoolExecutor's concurrency potential) +NUM_TASKS_PER_PROCESS_TEST = 3 +# Simulate API call duration to ensure calls can overlap +SIMULATED_API_CALL_DURATION_TEST = 0.2 # seconds + + +# --- Mock API call function for concurrency tracking --- +# This function will replace the real call_sandbox_api and use shared variables to track concurrency +def _mock_api_call_for_concurrency_tracking( + active_calls_counter, # multiprocessing.Value + max_calls_tracker, # multiprocessing.Value + call_lock, # multiprocessing.Lock + # Standard call_sandbox_api parameters + sandbox_fusion_url, + code, + stdin, + compile_timeout, + run_timeout, + memory_limit_mb, + language, +): + # entry_time = time.time() # For detailed logging + with call_lock: + active_calls_counter.value += 1 + if active_calls_counter.value > max_calls_tracker.value: + max_calls_tracker.value = active_calls_counter.value + # Optional debug log: + # print(f"[PID:{os.getpid()}-TID:{threading.get_ident()}] API Call Start. Active: " + # f"{active_calls_counter.value}, Max Observed: {max_calls_tracker.value}, Input: {stdin}") + + time.sleep(SIMULATED_API_CALL_DURATION_TEST) # Simulate actual work duration + + # exit_time = time.time() # For detailed logging + with call_lock: + active_calls_counter.value -= 1 + # Optional debug log: + # print(f"[PID:{os.getpid()}-TID:{threading.get_ident()}] API Call End. Active: " + # f"{active_calls_counter.value}, Input: {stdin}, Duration: {exit_time - entry_time:.2f}s") + + # Return a simulated successful API response + return { + "status": "Success", + "run_result": {"status": "Finished", "stdout": f"mock_output_for_{stdin}", "return_code": 0}, + }, None + + +# --- Worker function for ProcessPoolExecutor --- +# This function runs in each child process of ProcessPoolExecutor +def _process_pool_worker_for_concurrency_test( + sandbox_url, + in_outs, + generation, + memory_limit_mb, + language, + timeout, + mp_semaphore_for_check_correctness, + active_calls_counter, + max_calls_tracker, + call_lock, +): + # Corrected lambda to accept keyword arguments matching call_sandbox_api's usage + curried_mock_api_call = ( + lambda sandbox_fusion_url, code, stdin, compile_timeout, run_timeout, memory_limit_mb, language: ( + _mock_api_call_for_concurrency_tracking( + active_calls_counter, + max_calls_tracker, + call_lock, + sandbox_fusion_url, + code, + stdin, + compile_timeout, + run_timeout, + memory_limit_mb, + language, + ) + ) + ) + + # ---- START DEBUG PRINTS ---- + import os + + import verl.utils.reward_score.sandbox_fusion.utils + + print( + f"[Worker PID:{os.getpid()}] Original call_sandbox_api: " + f"{verl.utils.reward_score.sandbox_fusion.utils.call_sandbox_api}", + flush=True, + ) + # ---- END DEBUG PRINTS ---- + + with patch( + "verl.utils.reward_score.sandbox_fusion.utils.call_sandbox_api", side_effect=curried_mock_api_call + ) as mock_obj: + # ---- START DEBUG PRINTS ---- + print( + f"[Worker PID:{os.getpid()}] Patched call_sandbox_api: " + f"{verl.utils.reward_score.sandbox_fusion.utils.call_sandbox_api}", + flush=True, + ) + print(f"[Worker PID:{os.getpid()}] Mock object: {mock_obj}", flush=True) + # ---- END DEBUG PRINTS ---- + results, metadata_list = check_correctness( + sandbox_fusion_url=sandbox_url, + in_outs=in_outs, + generation=generation, + timeout=timeout, + memory_limit_mb=memory_limit_mb, + language=language, + concurrent_semaphore=mp_semaphore_for_check_correctness, # Pass multiprocessing.Semaphore + ) + # print(f"Process {os.getpid()} finished check_correctness. Processed {len(results)} tasks.") + return len(results) # Return the number of processed tasks for basic validation + + +# --- The actual test case for multiprocess concurrency control --- +def test_multiprocess_global_concurrency_limit_with_semaphore(): + """ + Tests that the global concurrent_semaphore (multiprocessing.Semaphore) + correctly limits the number of concurrent calls to call_sandbox_api + across multiple processes, each potentially running multiple threads + via check_correctness's internal ThreadPoolExecutor. + """ + manager = multiprocessing.Manager() + active_calls_counter = manager.Value("i", 0) # Current active mock API calls + max_calls_tracker = manager.Value("i", 0) # Observed maximum concurrent mock API calls + call_lock = manager.Lock() # Lock to protect counters + + # Create a multiprocessing.Semaphore instance, this is the global semaphore we are testing. + # It will be passed to check_correctness and used by _process_single_case to limit calls to call_sandbox_api. + global_mp_semaphore = manager.Semaphore(MAX_GLOBAL_CONCURRENCY_LIMIT_TEST) + + mock_sandbox_url = "mock_url_for_concurrency_test" + mock_generation = "pass" # Specific code content is not important as API call is mocked + mock_memory_limit_mb = 1024 + mock_language = "python" + mock_timeout = 5 # Timeout setting, not critical for mock calls + + # Input/output data for each process + # NUM_TASKS_PER_PROCESS_TEST tasks will be handled by check_correctness's internal ThreadPoolExecutor + process_in_outs = { + "inputs": [f"task_input_{i}" for i in range(NUM_TASKS_PER_PROCESS_TEST)], + "outputs": [f"task_output_{i}" for i in range(NUM_TASKS_PER_PROCESS_TEST)], + } + + futures = [] + total_tasks_expected_to_run = NUM_PROCESSES_TEST * NUM_TASKS_PER_PROCESS_TEST + + test_start_time = time.time() + + with ProcessPoolExecutor(max_workers=NUM_PROCESSES_TEST) as executor: + for i in range(NUM_PROCESSES_TEST): + future = executor.submit( + _process_pool_worker_for_concurrency_test, # Worker function + mock_sandbox_url, + process_in_outs, + mock_generation, + mock_memory_limit_mb, + mock_language, + mock_timeout, + global_mp_semaphore, # Global semaphore to test + active_calls_counter, # Shared variables for tracking + max_calls_tracker, + call_lock, + ) + futures.append(future) + + # Wait for all processes to complete and collect results + num_tasks_processed_per_worker = [f.result() for f in futures] + test_end_time = time.time() + total_execution_time = test_end_time - test_start_time + + # Print some test statistics for debugging and validation + print("\n--- Global Concurrency Test Stats ---") + print(f"Semaphore Limit (MAX_GLOBAL_CONCURRENCY_LIMIT_TEST): {MAX_GLOBAL_CONCURRENCY_LIMIT_TEST}") + print(f"Number of Processes (NUM_PROCESSES_TEST): {NUM_PROCESSES_TEST}") + print(f"Tasks per Process (NUM_TASKS_PER_PROCESS_TEST): {NUM_TASKS_PER_PROCESS_TEST}") + print(f"Total Tasks Submitted: {total_tasks_expected_to_run}") + print(f"Simulated API Call Duration: {SIMULATED_API_CALL_DURATION_TEST}s") + print(f"Total Test Execution Time: {total_execution_time:.2f}s") + print(f"Max Concurrent Mock API Calls Observed: {max_calls_tracker.value}") + # print(f"Tasks processed per worker: {num_tasks_processed_per_worker}") + + # Verify that all submitted tasks have been processed + assert sum(num_tasks_processed_per_worker) == total_tasks_expected_to_run, ( + "Mismatch in the number of tasks processed." + ) + + # Verify that the mock API was called at least once + assert max_calls_tracker.value > 0, "The mocked API call_sandbox_api was not called." + + # Core assertion: Observed maximum concurrent calls should not exceed the semaphore's limit + assert max_calls_tracker.value <= MAX_GLOBAL_CONCURRENCY_LIMIT_TEST, ( + f"Observed concurrency ({max_calls_tracker.value}) exceeded semaphore limit " + f"({MAX_GLOBAL_CONCURRENCY_LIMIT_TEST})." + ) + + # Optional: Rough check on execution time to verify semaphore is working to limit concurrency + # Theoretical minimum execution time = (Total tasks / Concurrency limit) * Single task duration + # Actual time will be longer due to various overheads + min_expected_duration = ( + total_tasks_expected_to_run * SIMULATED_API_CALL_DURATION_TEST + ) / MAX_GLOBAL_CONCURRENCY_LIMIT_TEST + # print(f"Minimum Expected Execution Time (approx): {min_expected_duration:.2f}s") + # Allow some margin, e.g., 80% of theoretical minimum time + assert total_execution_time >= min_expected_duration * 0.8, ( + f"Total execution time ({total_execution_time:.2f}s) was unexpectedly short, suggesting the " + f"semaphore might not be effectively limiting concurrency as expected " + f"(min expected: {min_expected_duration * 0.8:.2f}s)." + ) + + +# Ensure there is no more code after this point if these were the last functions. +# If there was other code, it would follow here. +def test_unit_invalid_input_format(): + """Unit test: Invalid in_outs format passed""" + results, metadata_list = check_correctness(SANDBOX_URL, None, CODE_SUCCESS) + assert results == [-1] + assert metadata_list[0]["error"] == "Invalid input/output data" + + results, metadata_list = check_correctness(SANDBOX_URL, {}, CODE_SUCCESS) + assert results == [-1] + assert metadata_list[0]["error"] == "Invalid input/output data" + + results, metadata_list = check_correctness(SANDBOX_URL, INPUT_OUTPUT_INVALID_MISSING_KEY, CODE_SUCCESS) + assert results == [-1] + assert metadata_list[0]["error"] == "Invalid input/output data" + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_unit_input_output_mismatch(): + """Unit test: Mismatch between the number of inputs and outputs""" + results, metadata_list = check_correctness(SANDBOX_URL, INPUT_OUTPUT_MISMATCH, CODE_SUCCESS) + assert results == [-1] + assert len(metadata_list) == 1 + assert metadata_list[0]["error"] == "Input/output count mismatch" + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_integration_concurrency_all_timeout(): + """Integration test: High concurrency (100 cases) against real API, all causing timeout""" + concurrency_level = 100 + code_infinite_loop = """ +def knight_moves(X, Y): + MOD = 10**9 + 7 + dp = [[0] * (Y + 1) for _ in range(X + 1)] + dp[0][0] = 1 + for i in range(1, X + 1): + for j in range(1, Y + 1): + dp[i][j] = (dp[i - 1][j] + dp[i][j - 1]) % MOD + return dp[X][Y] + +def solve(): + X, Y = map(int, input().split()) + print(knight_moves(X, Y)) + +if __name__ == "__main__": + solve() + """ + + # Generate 100 simple input/output pairs (content doesn't matter) + timeout_inputs = ["324 384429" for i in range(concurrency_level)] + timeout_outputs = [f"output_{i}\n" for i in range(concurrency_level)] + timeout_in_outs = {"inputs": timeout_inputs, "outputs": timeout_outputs} + + # Set a timeout for the test cases + test_timeout = 10 # Set a timeout value + + start_time = time.time() + results, metadata_list = check_correctness(SANDBOX_URL, timeout_in_outs, code_infinite_loop, timeout=test_timeout) + end_time = time.time() + duration = end_time - start_time + print(f"\nHigh concurrency all timeout test ({concurrency_level} cases) duration: {duration:.2f} seconds") + + # Verify all results are -3 (timeout) + assert len(results) == concurrency_level, f"Expected {concurrency_level} results, got {len(results)}" + all_timed_out = all(r == -3 for r in results) + if not all_timed_out: + non_timeout_indices = [i for i, r in enumerate(results) if r != -3] + print(f"Indices that did not time out: {non_timeout_indices}") + # Print metadata for the first few non-timeout cases for debugging + for i in non_timeout_indices[:5]: + print(f"Metadata for non-timeout case {i}: {metadata_list[i]}") + assert all_timed_out, f"Not all {concurrency_level} concurrent tests resulted in timeout (-3). Results: {results}" + + # Verify metadata count and status of the first case + assert len(metadata_list) == concurrency_level + assert metadata_list[0]["status"] == "timeout" + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_fn_name_success_single_case(): + """Tests successful execution for a single test case with fn_name. + from livecodebench/code_generation_lite test 510 + """ + generation_code = """ +class Solution: + def occurrencesOfElement(self, nums: List[int], queries: List[int], x: int) -> List[int]: + positions = defaultdict(list) + for idx, num in enumerate(nums): + positions[num].append(idx) + + x_positions = positions[x] + answer = [] + for k in queries: + if k > len(x_positions): + answer.append(-1) + else: + answer.append(x_positions[k-1]) + return answer +""" + in_outs = { + "fn_name": "occurrencesOfElement", + "inputs": ["[1, 3, 1, 7]\n[1, 3, 2, 4]\n1", "[1, 2, 3]\n[10]\n5"], + "outputs": ["[0, -1, 2, -1]", "[-1]"], + } + + # Use a short timeout for fast tests + results, metadata_list = check_correctness(SANDBOX_URL, in_outs, generation_code, timeout=5) + # from verl.utils.reward_score.prime_code import apps_check_correctness + # results, metadata_list = apps_check_correctness(in_outs=in_outs, generation=generation_code, + # timeout=50000, debug=True) + + assert results == [True, True] + assert "error" not in metadata_list[0] + assert metadata_list[0].get("status") != "compile_error" + assert metadata_list[0].get("status") != "runtime_error" + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_none_and_empty_stdin_passed_correctly(): + """ + Tests that when stdin data is set to an empty string or None, it is still + is passed correctly to Sandbox Fusion as an empty string. + """ + echo_code = """ +import sys +print(f"You said '{sys.stdin.readline().strip()}'") +""" + in_outs = { + "inputs": [None, "", "hello"], + "outputs": ["You said ''", "You said ''", "You said 'hello'"], + } + + # Use a short timeout for fast tests + results, metadata_list = check_correctness(SANDBOX_URL, in_outs, echo_code, timeout=5) + + assert results == [True, True, True] + assert "error" not in metadata_list[0] + assert metadata_list[0].get("status") != "compile_error" + assert metadata_list[0].get("status") != "runtime_error" + + +@pytest.mark.skipif(skip_condition, reason=skip_reason) +def test_assert_case_success(): + """Tests successful execution for assert case. + from KodCode + """ + generation_code = """ +from typing import List, Tuple + +def merge_intervals(intervals: List[Tuple[int, int]]) -> List[Tuple[int, int]]: + if not intervals: + return [] + + # Sort intervals by the start time + intervals.sort(key=lambda x: x[0]) + + merged = [intervals[0]] + + for current in intervals[1:]: + last = merged[-1] + # If intervals overlap, merge them + if current[0] <= last[1]: + merged[-1] = (last[0], max(last[1], current[1])) + else: + merged.append(current) + + return merged +""" + test_cases = { + "fn_name": "merge_intervals", + "assert_case": [ + "assert merge_intervals([(0, 1), (3, 5), (4, 7), (6, 8), (10, 12)," + " (12, 14)]) == [(0, 1), (3, 8), (10, 14)]", + "assert merge_intervals([(1, 2), (2, 3), (3, 4)]) == [(1, 4)]", + "assert merge_intervals([(1, 2), (3, 4), (5, 6)]) == [(1, 2), (3, 4), (5, 5)]", + ], + } + + assert_cases = test_cases.get("assert_case") + test_cases.setdefault("inputs", ["" for _ in assert_cases]) + test_cases.setdefault("outputs", [None for _ in assert_cases]) + + # Use a short timeout for fast tests + results, metadata_list = check_correctness(SANDBOX_URL, test_cases, generation_code, timeout=5) + assert results == [True, True, -2] + for i in range(2): + assert "error" not in metadata_list[i] + assert metadata_list[i].get("status") == "success" + assert metadata_list[i].get("expected_output") is None + assert metadata_list[i].get("status") != "runtime_error" + assert "error" not in metadata_list[2] + assert metadata_list[2].get("status") != "success" + assert metadata_list[2].get("expected_output") is None + assert metadata_list[2].get("status") == "runtime_error" diff --git a/e2c/verl/tests/utils/reward_score/test_sandbox_on_cpu.py b/e2c/verl/tests/utils/reward_score/test_sandbox_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..ff4073232f54ecd14ff63cf57c75f443e1a878fe --- /dev/null +++ b/e2c/verl/tests/utils/reward_score/test_sandbox_on_cpu.py @@ -0,0 +1,185 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json +import os + +import pytest + +from verl.utils.reward_score import default_compute_score, prime_code, sandbox_fusion +from verl.utils.reward_score.prime_code import apps_check_correctness +from verl.workers.reward_manager.prime import parallel_compute_score_async + +prime_math_answers = [ + """\\begin{bmatrix}\n -7 & 6 & -8 \\\\\n 11 & -9 & 12 \\\\\n 15 & -16 & 19 \n \\end{bmatrix}""", + """\\frac{\\sqrt{505}}{7}""", + """x^2 + y^2 + 4x - 6y + 13""", +] +prime_math_gts = [ + """\\begin{pmatrix}\n -7 & 6 & -8 \\\\\n 11 & -9 & 12 \\\\\n 15 & -16 & 19\n \\end{pmatrix}""", # mat test + """\\frac{\\sqrt{505}}{7}""", # frac test + """(x + 2)^2 + (y - 3)^2 """, # symbolic test +] + +prime_code_answers = [ + """import sys +from collections import deque + +def main(): + data = sys.stdin.read().split() + it = iter(data) + + # Read start and target positions + x0, y0, x1, y1 = int(next(it)), int(next(it)), int(next(it)), int(next(it)) + + n = int(next(it)) + allowed = set() + # The total number of allowed cells is at most 10^5. + for _ in range(n): + r = int(next(it)) + a = int(next(it)) + b = int(next(it)) + for c in range(a, b + 1): + allowed.add((r, c)) + + # Directions for the king (8 neighboring cells) + directions = [(-1, -1), (-1, 0), (-1, 1), + (0, -1), (0, 1), + (1, -1), (1, 0), (1, 1)] + + start = (x0, y0) + target = (x1, y1) + + # BFS initialization + queue = deque() + queue.append((x0, y0, 0)) + # Mark the starting cell as visited by removing it from allowed set. + allowed.discard(start) + + while queue: + x, y, moves = queue.popleft() + if (x, y) == target: + print(moves) + return + for dx, dy in directions: + nx, ny = x + dx, y + dy + if (nx, ny) in allowed: + allowed.remove((nx, ny)) + queue.append((nx, ny, moves + 1)) + + print(-1) + +if __name__ == '__main__': + main() +""" +] * 2 +prime_code_gts = [ + """{\n \"inputs\": [\n \"5 7 6 11\\n3\\n5 3 8\\n6 7 11\\n5 2 5\\n\",\n \"3 4 3 10\\n3\\n3 1 4\\n4 5 9\\n3 10 10\\n\",\n \"1 1 2 10\\n2\\n1 1 3\\n2 6 10\\n\",\n \"9 8 7 8\\n9\\n10 6 6\\n10 6 6\\n7 7 8\\n9 5 6\\n8 9 9\\n9 5 5\\n9 8 8\\n8 5 6\\n9 10 10\\n\",\n \"6 15 7 15\\n9\\n6 15 15\\n7 14 14\\n6 15 15\\n9 14 14\\n7 14 16\\n6 15 15\\n6 15 15\\n7 14 14\\n8 15 15\\n\",\n \"13 16 20 10\\n18\\n13 16 16\\n20 10 10\\n19 10 10\\n12 15 15\\n20 10 10\\n18 11 11\\n19 10 10\\n19 10 10\\n20 10 10\\n19 10 10\\n20 10 10\\n20 10 10\\n19 10 10\\n18 11 11\\n13 16 16\\n12 15 15\\n19 10 10\\n19 10 10\\n\",\n \"89 29 88 30\\n16\\n87 31 31\\n14 95 95\\n98 88 89\\n96 88 88\\n14 97 97\\n13 97 98\\n100 88 88\\n88 32 32\\n99 88 89\\n90 29 29\\n87 31 31\\n15 94 96\\n89 29 29\\n88 32 32\\n97 89 89\\n88 29 30\\n\",\n \"30 14 39 19\\n31\\n35 7 11\\n37 11 12\\n32 13 13\\n37 5 6\\n46 13 13\\n37 14 14\\n31 13 13\\n43 13 19\\n45 15 19\\n46 13 13\\n32 17 17\\n41 14 19\\n30 14 14\\n43 13 17\\n34 16 18\\n44 11 19\\n38 13 13\\n40 12 20\\n37 16 18\\n46 16 18\\n34 10 14\\n36 9 10\\n36 15 19\\n38 15 19\\n42 13 19\\n33 14 15\\n35 15 19\\n33 17 18\\n39 12 20\\n36 5 7\\n45 12 12\\n\",\n \"2 1 1 1\\n2\\n1 1 2\\n2 1 2\\n\",\n \"1 1 1 2\\n5\\n1000000000 1 10000\\n19920401 1188 5566\\n1000000000 1 10000\\n1 1 10000\\n5 100 200\\n\",\n \"1 1 1000000000 2\\n5\\n1000000000 1 10000\\n19920401 1188 5566\\n1000000000 1 10000\\n1 1 10000\\n5 100 200\\n\"\n ],\n \"outputs\": [\n \"4\\n\",\n \"6\\n\",\n \"-1\\n\",\n \"2\\n\",\n \"1\\n\",\n \"-1\\n\",\n \"1\\n\",\n \"9\\n\",\n \"1\\n\",\n \"1\\n\",\n \"-1\\n\"\n ]\n}""", # A correct sample # noqa: E501 + """{\n \"inputs\": [\n \"5 7 6 11\\n3\\n5 3 8\\n6 7 11\\n5 2 5\\n\",\n \"3 4 3 10\\n3\\n3 1 4\\n4 5 9\\n3 10 10\\n\",\n \"1 1 2 10\\n2\\n1 1 3\\n2 6 10\\n\",\n \"9 8 7 8\\n9\\n10 6 6\\n10 6 6\\n7 7 8\\n9 5 6\\n8 9 9\\n9 5 5\\n9 8 8\\n8 5 6\\n9 10 10\\n\",\n \"6 15 7 15\\n9\\n6 15 15\\n7 14 14\\n6 15 15\\n9 14 14\\n7 14 16\\n6 15 15\\n6 15 15\\n7 14 14\\n8 15 15\\n\",\n \"13 16 20 10\\n18\\n13 16 16\\n20 10 10\\n19 10 10\\n12 15 15\\n20 10 10\\n18 11 11\\n19 10 10\\n19 10 10\\n20 10 10\\n19 10 10\\n20 10 10\\n20 10 10\\n19 10 10\\n18 11 11\\n13 16 16\\n12 15 15\\n19 10 10\\n19 10 10\\n\",\n \"89 29 88 30\\n16\\n87 31 31\\n14 95 95\\n98 88 89\\n96 88 88\\n14 97 97\\n13 97 98\\n100 88 88\\n88 32 32\\n99 88 89\\n90 29 29\\n87 31 31\\n15 94 96\\n89 29 29\\n88 32 32\\n97 89 89\\n88 29 30\\n\",\n \"30 14 39 19\\n31\\n35 7 11\\n37 11 12\\n32 13 13\\n37 5 6\\n46 13 13\\n37 14 14\\n31 13 13\\n43 13 19\\n45 15 19\\n46 13 13\\n32 17 17\\n41 14 19\\n30 14 14\\n43 13 17\\n34 16 18\\n44 11 19\\n38 13 13\\n40 12 20\\n37 16 18\\n46 16 18\\n34 10 14\\n36 9 10\\n36 15 19\\n38 15 19\\n42 13 19\\n33 14 15\\n35 15 19\\n33 17 18\\n39 12 20\\n36 5 7\\n45 12 12\\n\",\n \"2 1 1 1\\n2\\n1 1 2\\n2 1 2\\n\",\n \"1 1 1 2\\n5\\n1000000000 1 10000\\n19920401 1188 5566\\n1000000000 1 10000\\n1 1 10000\\n5 100 200\\n\",\n \"1 1 1000000000 2\\n5\\n1000000000 1 10000\\n19920401 1188 5566\\n1000000000 1 10000\\n1 1 10000\\n5 100 200\\n\"\n ],\n \"outputs\": [\n \"4\\n\",\n \"6\\n\",\n \"-1\\n\",\n \"-1\\n\",\n \"1\\n\",\n \"-1\\n\",\n \"1\\n\",\n \"9\\n\",\n \"1\\n\",\n \"1\\n\",\n \"-1\\n\"\n ]\n}""", # noqa: E501 +] # A failed sample with first several in-out passed + +prime_code_scores = [1.0, 0.9] + + +def test_parallelism(): + """ + Test if process pool works properly + """ + sequences_str = [] + ground_truth = [] + data_sources = [] + while len(sequences_str) < 32: + sequences_str.extend(prime_code_answers) + ground_truth.extend(prime_code_gts) + data_sources.extend(["codecontests"] * len(prime_code_answers)) + + sequences_str.extend(prime_math_answers) + ground_truth.extend(prime_math_gts) + data_sources.extend(["numina_aops_forum"] * len(prime_math_answers)) + + scores = asyncio.run( + parallel_compute_score_async(default_compute_score, sequences_str, ground_truth, data_sources, num_processes=16) + ) + print(scores) + + +def test_prime_code(): + """ + Test PRIME code sandbox. + """ + data_source = "codecontests" + for completion, ground_truth, score_ in zip(prime_code_answers, prime_code_gts, prime_code_scores, strict=True): + score = default_compute_score(data_source, completion, ground_truth) + assert float(score) == score_ + + +# Use the pytest.mark.skipif decorator to skip the test +@pytest.mark.skipif(not os.environ.get("SANDBOX_FUSION_URL"), reason="SANDBOX_FUSION_URL environment variable not set") +def test_prime_code_sandbox_fusion(): + """ + Test PRIME code on sandbox fusion. Skips if SANDBOX_FUSION_URL is not set. + """ + data_source = "codecontests" + # Get the URL from the environment variable, as skipif ensures it is set at this point + sandbox_fusion_url = os.environ.get("SANDBOX_FUSION_URL") + # Removed the previous 'if not sandbox_url' check block + + for completion, ground_truth, score_ in zip(prime_code_answers, prime_code_gts, prime_code_scores, strict=True): + score = default_compute_score( + data_source, completion, ground_truth, extra_info={"sandbox_fusion_url": sandbox_fusion_url} + ) # <-- Use the URL obtained from the environment variable + assert float(score) == score_ + + +@pytest.mark.skipif(not os.environ.get("SANDBOX_FUSION_URL"), reason="SANDBOX_FUSION_URL environment variable not set") +def test_continuous_score_consistency(): + """ + Verify that continuous score calculation is consistent between prime_code and sandbox_fusion. + Uses a test case where the first 9 out of 11 sub-cases pass (expected score 0.9). + """ + completion = prime_code_answers[1] # Use the second sample + ground_truth = prime_code_gts[1] # Use the second sample (9/11 pass, first 9 pass) + expected_continuous_score = 0.9 + + # 1. Calculate score using prime_code (default) with continuous=True + prime_score, _ = sandbox_fusion.compute_score( + os.environ.get("SANDBOX_FUSION_URL"), None, completion, ground_truth, continuous=True + ) + + # 2. Calculate score using sandbox_fusion with continuous=True + # Ensure the extra_info key triggers the sandbox_fusion path in default_compute_score + fusion_score, _ = prime_code.compute_score(completion, ground_truth, continuous=True) + + # 3. Assert scores are equal (using pytest.approx for float comparison) + assert float(prime_score) == pytest.approx(expected_continuous_score) + assert float(fusion_score) == pytest.approx(expected_continuous_score) + assert float(prime_score) == pytest.approx(float(fusion_score)) + print(f"Continuous Score (Prime Code): {prime_score}") + print(f"Continuous Score (Sandbox Fusion): {fusion_score}") + + +def test_check_correctness(): + completion = prime_code_answers[0] + ground_truth = json.loads(prime_code_gts[0]) + ground_truth_single = {"inputs": ground_truth["inputs"][:1], "outputs": ground_truth["outputs"][:1]} + res, meta = apps_check_correctness(in_outs=ground_truth_single, generation=completion, timeout=5, debug=False) + print(res, meta) + + +def test_prime_math(): + data_source = "numina_aops_forum" + for completion, ground_truth in zip(prime_math_answers, prime_math_gts, strict=True): + score = default_compute_score(data_source, completion, ground_truth) + assert float(score) == 1.0 diff --git a/e2c/verl/tests/utils/test_activation_offload.py b/e2c/verl/tests/utils/test_activation_offload.py new file mode 100644 index 0000000000000000000000000000000000000000..25bc23c40acf521c175077fce94efdeab2f9b52c --- /dev/null +++ b/e2c/verl/tests/utils/test_activation_offload.py @@ -0,0 +1,172 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +import tempfile + +import pytest +import torch +import torch.distributed +import torch.multiprocessing as mp +from torch.distributed import init_device_mesh +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import MixedPrecision, ShardingStrategy +from transformers import AutoModelForCausalLM, AutoTokenizer, Qwen2Config + +from verl.utils.activation_offload import enable_activation_offloading +from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager +from verl.utils.fsdp_utils import MixedPrecisionPolicy, apply_fsdp2, get_fsdp_wrap_policy + + +def create_random_input_ids(batch_size, seq_len, vocab_size): + from flash_attn.bert_padding import unpad_input + + from verl.utils.model import compute_position_id_with_mask, create_random_mask + + input_ids = torch.randint(0, vocab_size, (batch_size, seq_len), device="cuda") + + attention_mask = create_random_mask( + input_ids, max_ratio_of_left_padding=0.1, min_ratio_of_valid_token=0.5, max_ratio_of_valid_token=0.7 + ) + position_ids = compute_position_id_with_mask(attention_mask) + + input_ids = unpad_input(input_ids.unsqueeze(-1), attention_mask)[0].transpose(0, 1) + position_ids = unpad_input(position_ids.unsqueeze(-1), attention_mask)[0].transpose(0, 1) + return input_ids, position_ids + + +def _fsdp_activation_offloading_test(rank, world_size, rendezvous_file, strategy="fsdp"): + torch.cuda.set_device(rank) + torch.distributed.init_process_group( + backend="nccl", + init_method=f"file://{rendezvous_file}", + rank=rank, + world_size=world_size, + ) + device_mesh = init_device_mesh("cuda", mesh_shape=(world_size,), mesh_dim_names=("dp",)) + + model_name = "Qwen/Qwen2.5-0.5B-Instruct" + config = Qwen2Config(num_hidden_layers=4) + + with torch.device("cuda"): + model = AutoModelForCausalLM.from_config( + config=config, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + ) + model = model.to(device="cuda") + + # Wrap model with FSDP + mixed_precision = MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32) + + if strategy == "fsdp": + model = FSDP( + model, + use_orig_params=False, + device_id=torch.cuda.current_device(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + device_mesh=device_mesh, + auto_wrap_policy=get_fsdp_wrap_policy(module=model), + ) + else: + mp_policy = MixedPrecisionPolicy( + param_dtype=torch.bfloat16, reduce_dtype=torch.float32, cast_forward_inputs=True + ) + fsdp_kwargs = { + "mesh": device_mesh, + "mp_policy": mp_policy, + } + apply_fsdp2(model, fsdp_kwargs, {}) + + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4) + lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.9) + + # Create checkpoint manager + tokenizer = AutoTokenizer.from_pretrained(model_name) + checkpoint_manager = FSDPCheckpointManager( + model=model, optimizer=optimizer, lr_scheduler=lr_scheduler, tokenizer=tokenizer + ) + + # Generate sample input + batch_size = 2 + seq_len = 32 + vocab_size = 32000 + # First input for initial update + input_ids1, position_ids1 = create_random_input_ids(batch_size, seq_len, vocab_size) + + # Second input for verification + input_ids2, position_ids2 = create_random_input_ids(batch_size, seq_len, vocab_size) + + # Step 1: Initial update and save checkpoint + outputs1 = model(input_ids=input_ids1, position_ids=position_ids1) + loss1 = outputs1.logits.mean() + loss1.backward() + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Save checkpoint after first update + temp_dir = tempfile.mkdtemp() + checkpoint_path = os.path.join(temp_dir, "checkpoint") + checkpoint_manager.save_checkpoint(local_path=checkpoint_path, hdfs_path=None, global_step=0) + + # Step 2: Second update and forward pass + outputs2 = model(input_ids=input_ids2, position_ids=position_ids2) + loss2 = outputs2.logits.mean() + loss2.backward() + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Record logits after second update + with torch.no_grad(): + logits_without_offloading = model(input_ids=input_ids2, position_ids=position_ids2).logits + + # Step 3: wrap module with activation offloading and load checkpoint + enable_activation_offloading(model, strategy=strategy) + checkpoint_manager.load_checkpoint(checkpoint_path) + + # Step 4: Repeat the second update with same input + outputs3 = model(input_ids=input_ids2, position_ids=position_ids2) + loss3 = outputs3.logits.mean() + loss3.backward() + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Record logits after loaded checkpoint and update + with torch.no_grad(): + logits_with_offloading = model(input_ids=input_ids2, position_ids=position_ids2).logits + + # Step 4: Verify outputs match + torch.testing.assert_close(logits_without_offloading, logits_with_offloading, atol=0.0, rtol=0.0) + print(f"Activaiton offloading for {strategy} test passed on {world_size} GPUs!") + + # Cleanup + shutil.rmtree(temp_dir) + torch.distributed.barrier() + torch.distributed.destroy_process_group() + + +@pytest.mark.parametrize("world_size", (2, 4)) +@pytest.mark.parametrize("strategy", ("fsdp", "fsdp2")) +def test_activation_offloading(world_size, strategy, tmp_path): + rendezvous_file = str(tmp_path / "rdzv_file") + os.makedirs(os.path.dirname(rendezvous_file), exist_ok=True) + + mp.spawn( + fn=_fsdp_activation_offloading_test, + args=(world_size, rendezvous_file, strategy), + nprocs=world_size, + join=True, + ) diff --git a/e2c/verl/tests/utils/test_config_on_cpu.py b/e2c/verl/tests/utils/test_config_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..f55e7d682913cbd284e2666895c4bbb5da25387c --- /dev/null +++ b/e2c/verl/tests/utils/test_config_on_cpu.py @@ -0,0 +1,97 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from dataclasses import dataclass, field + +from omegaconf import OmegaConf + +from verl.base_config import BaseConfig +from verl.utils import omega_conf_to_dataclass + + +@dataclass +class TestDataclass(BaseConfig): + hidden_size: int = 0 + activation: str = "relu" + + +@dataclass +class TestTrainConfig(BaseConfig): + batch_size: int = 0 + model: TestDataclass = field(default_factory=TestDataclass) + override_config: dict = field(default_factory=dict) + + +_cfg_str = """train_config: + _target_: tests.utils.test_config_on_cpu.TestTrainConfig + batch_size: 32 + model: + hidden_size: 768 + activation: relu + override_config: {}""" + + +class TestConfigOnCPU(unittest.TestCase): + """Test cases for configuration utilities on CPU. + + Test Plan: + 1. Test basic OmegaConf to dataclass conversion for simple nested structures + 2. Test nested OmegaConf to dataclass conversion for complex hierarchical configurations + 3. Verify all configuration values are correctly converted and accessible + """ + + def setUp(self): + self.config = OmegaConf.create(_cfg_str) + + def test_omega_conf_to_dataclass(self): + sub_cfg = self.config.train_config.model + cfg = omega_conf_to_dataclass(sub_cfg, TestDataclass) + self.assertEqual(cfg.hidden_size, 768) + self.assertEqual(cfg.activation, "relu") + assert isinstance(cfg, TestDataclass) + + def test_nested_omega_conf_to_dataclass(self): + cfg = omega_conf_to_dataclass(self.config.train_config, TestTrainConfig) + self.assertEqual(cfg.batch_size, 32) + self.assertEqual(cfg.model.hidden_size, 768) + self.assertEqual(cfg.model.activation, "relu") + assert isinstance(cfg, TestTrainConfig) + assert isinstance(cfg.model, TestDataclass) + + +class TestPrintCfgCommand(unittest.TestCase): + """Test suite for the print_cfg.py command-line tool.""" + + def test_command_with_override(self): + """Test that the command runs without error when overriding config values.""" + import subprocess + + # Run the command + result = subprocess.run( + ["python3", "scripts/print_cfg.py"], + capture_output=True, + text=True, + ) + + # Verify the command exited successfully + self.assertEqual(result.returncode, 0, f"Command failed with stderr: {result.stderr}") + + # Verify the output contains expected config information + self.assertIn("critic", result.stdout) + self.assertIn("profiler", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/utils/test_flops_counter.py b/e2c/verl/tests/utils/test_flops_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..da6178c9d854e51cd2e7fcb3b9235973dbdb0e2e --- /dev/null +++ b/e2c/verl/tests/utils/test_flops_counter.py @@ -0,0 +1,246 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import pytest + +from verl.utils.flops_counter import FlopsCounter + +VALID_CONFIG_TYPE = {"llama", "qwen2", "qwen3", "qwen3_moe", "deepseek_v3", "mistral", "gemma3_text", "apertus"} + + +class Config: + def __init__(self, config_dict): + for key, value in config_dict.items(): + setattr(self, key, value) + + +CONFIG = { + "llama": { + "config": { # llama2-7B + "model_type": "llama", + "vocab_size": 32000, + "hidden_size": 4096, + "intermediate_size": 11008, + "num_hidden_layers": 32, + "num_attention_heads": 32, + "num_key_value_heads": 32, + }, + "batch_seqlens_tuple": ([512, 1024, 2048], [4096, 4096, 4096]), + # 6*(vocab*hidden*2+layer*(hidden*(q+k+v+head*head_dim)+ hidden*inter*3))*token_sum + + # 12*sum(seqlen^2)*layer*head*head_dim + # 6*(32000*4096*2+32*(4096*4096*4+4096*11008*3))*(512+1024+2048) + + # 12*(512*512+1024*1024+2048*2048)*32*4096 + # 6*(32000*4096*2+32*(4096*4096*4+4096*11008*3))*(4096+4096+4096) + + # 12*(4096*4096+4096*4096+4096*4096)*32*4096 + "expected_flops_tuple": (153555818250240 / 1e12, 575955114393600 / 1e12), + }, + "qwen2": { + "config": { # Qwen/Qwen2.5-7B-Instruct + "model_type": "qwen2", + "vocab_size": 152064, + "hidden_size": 3584, + "intermediate_size": 18944, + "num_hidden_layers": 28, + "num_attention_heads": 28, + "num_key_value_heads": 4, + }, + "batch_seqlens_tuple": ([512, 1024, 2048], [4096, 4096, 4096]), + # 6*(vocab*hidden*2+layer*(hidden*(q+k+v+head*head_dim)+ hidden*inter*3))*token_sum + + # 12*sum(seqlen^2)*layer*head*head_dim + # 6*(152064*3584*2+28*(3584*(3584+512+512+3584)+3584*18944*3))*(512+1024+2048) + + # 12*(512*512+1024*1024+2048*2048)*28*3584 + # 6*(152064*3584*2+28*(3584*(3584+512+512+3584)+3584*18944*3))*(4096+4096+4096) + + # 12*(4096*4096+4096*4096+4096*4096)*28*3584 + "expected_flops_tuple": (170388331954176 / 1e12, 622070178250752 / 1e12), + }, + "qwen3": { + "config": { # Qwen/Qwen3-8B + "model_type": "qwen3", + "vocab_size": 151936, + "hidden_size": 4096, + "intermediate_size": 12288, + "num_hidden_layers": 36, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 128, + }, + "batch_seqlens_tuple": ([512, 1024, 2048], [4096, 4096, 4096]), + # 6*(vocab*hidden*2+layer*(hidden*(q+k+v+head*head_dim)+ hidden*inter*3))*token_sum + + # 12*sum(seqlen^2)*layer*head*head_dim + # 6*(151936*4096*2+36*(4096*(128*32+128*8*2+128*32)+4096*12288*3))*(512+1024+2048) + + # 12*(512*512+1024*1024+2048*2048)*36*128*32 + # 6*(151936*4096*2+36*(4096*(128*32+128*8*2+128*32)+4096*12288*3))*(4096+4096+4096) + + # 12*(4096*4096+4096*4096+4096*4096)*36*128*32 + "expected_flops_tuple": (185867930959872 / 1e12, 692924253732864 / 1e12), + }, + "qwen3_moe": { + "config": { # Qwen/Qwen3-30B-A3B-Base + "model_type": "qwen3_moe", + "hidden_size": 2048, + "vocab_size": 151936, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_attention_heads": 32, + "head_dim": 128, + "moe_intermediate_size": 768, + "num_experts_per_tok": 8, + "num_experts": 128, + }, + "batch_seqlens_tuple": ([512, 1024, 2048], [4096, 4096, 4096]), + # 6*(vocab*hidden*2+layer*(hidden*(q+k+v+head*head_dim)+hidden*inter*top_k_exp*3 + + # hidden*num_experts))*token_sum + 12*sum(seqlen^2)*layer*head*head_dim + # 6*(151936*2048*2+48*(2048*(128*32+128*4*2+128*32)+2048*768*8*3+2048*128))*(512+1024+2048) + + # 12*(512*512+1024*1024+2048*2048)*48*128*32 + # 6*(151936*2048*2+48*(2048*(128*32+128*4*2+128*32)+2048*768*8*3+2048*128))*(4096+4096+4096) + + # 12*(4096*4096+4096*4096+4096*4096)*48*128*32 + "expected_flops_tuple": (85087060230144 / 1e12, 365944098521088 / 1e12), + }, + "deepseek_v3": { + "config": { # deepseek-ai/DeepSeek-Prover-V2-671B + "model_type": "deepseek_v3", + "hidden_size": 7168, + "vocab_size": 129280, + "moe_intermediate_size": 2048, + "num_hidden_layers": 61, + "first_k_dense_replace": 3, + "num_attention_heads": 128, + "n_routed_experts": 256, + "num_experts_per_tok": 8, + "n_shared_experts": 1, + "kv_lora_rank": 512, + "qk_rope_head_dim": 64, + "v_head_dim": 128, + "intermediate_size": 18432, + "qk_nope_head_dim": 128, + "q_lora_rank": 1536, + }, + "batch_seqlens_tuple": ([512, 1024, 2048], [4096, 4096, 4096]), + # (1536*7168+128*192*1536+7168*(512+64)+128*(128+128)*512+128*128*7168) = 187105280 + # 6*(129280*7168*2+ 3*(7168*18432*3+187105280)+ 58*(187105280+7168*256+7168*2048*9*3))*(512+1024+2048) + + # 12*(512*512+1024*1024+2048*2048)*61*192*128 + # 6*(129280*7168*2+ 3*(7168*18432*3+187105280)+ 58*(187105280+7168*256+7168*2048*9*3))*(4096+4096+4096) + + # 12*(4096*4096+4096*4096+4096*4096)*61*192*128 + "expected_flops_tuple": (906535995703296 / 1e12, 3674028304760832 / 1e12), + }, + "mistral": { + "config": { # mistralai/Mistral-Small-24B-Instruct-2501 + "model_type": "mistral", + "vocab_size": 131072, + "hidden_size": 5120, + "intermediate_size": 32768, + "num_hidden_layers": 40, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 128, + }, + "batch_seqlens_tuple": ([512, 1024, 2048], [4096, 4096, 4096]), + # Mistral uses same architecture as Llama, with GQA + # 6*(vocab*hidden*2+layer*(hidden*(q+k+v+head*head_dim)+ hidden*inter*3))*token_sum + + # 12*sum(seqlen^2)*layer*head*head_dim + # vocab part: 131072*5120*2 = 1342177280 + # attn part per layer: 5120*(128*32+128*8+128*8+128*32) = 5120*10240 = 52428800 + # mlp part per layer: 5120*32768*3 = 503316480 + # total per layer: 52428800 + 503316480 = 555745280 + # all layers: 1342177280 + 40*555745280 = 23571988480 + # For batch [512, 1024, 2048], tokens_sum = 3584: + # dense flops: 6 * 23571988480 * 3584 = 506892040273920 + # attn flops: 12 * 5505024 * 40 * 128 * 32 = 10823317585920 + # total: 517715357859840 / 1e12 = 517.71535785984 + # For batch [4096, 4096, 4096], tokens_sum = 12288: + # dense flops: 6 * 23571988480 * 12288 = 1737915566653440 + # attn flops: 12 * 50331648 * 40 * 128 * 32 = 98956046499840 + # total: 1836871613153280 / 1e12 = 1836.87161315328 + "expected_flops_tuple": (517715357859840 / 1e12, 1836871613153280 / 1e12), + }, + "gemma3_text": { + "config": { # Gemma3-12B-IT-TextOnly + "model_type": "gemma3_text", + "vocab_size": 262208, + "hidden_size": 3840, + "intermediate_size": 15360, + "num_hidden_layers": 48, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "head_dim": 256, + "sliding_window": 1024, + "layer_types": None, + # Will be auto-generated based on sliding_window_pattern + "sliding_window_pattern": 6, + # Every 6th layer is full attention + }, + "batch_seqlens_tuple": ([512, 1024, 2048], [4096, 4096, 4096]), + # Gemma3 has alternating sliding window attention + # With sliding_window_pattern=6: layers 5,11,17,23,29,35,41,47 use full attention (8 layers) + # Other 40 layers use sliding window attention with window_size=1024 + # + # Non-attention FLOPs: + # vocab part: 262208*3840*2 = 2013757440 + # attn part per layer: 3840*(256*16+256*8+256*8+256*16) = 3840*12288 = 47185920 + # mlp part per layer: 3840*15360*3 = 176947200 + # total per layer: 47185920 + 176947200 = 224133120 + # all layers: 2013757440 + 48*224133120 = 12772147200 + # + # For batch [512, 1024, 2048], tokens_sum = 3584: + # dense flops: 6 * 12772147200 * 3584 = 274652253388800 + # seqlen_square_sum: 180355072 (calculated with sliding window logic) + # attn flops: 12 * 180355072 * 256 * 16 = 8864812498944 + # total: 283517065887744 / 1e12 = 283.517065887744 + # + # For batch [4096, 4096, 4096], tokens_sum = 12288: + # dense flops: 6 * 12772147200 * 12288 = 941664868761600 + # seqlen_square_sum: 905969664 (calculated with sliding window logic) + # attn flops: 12 * 905969664 * 256 * 16 = 44530220924928 + # total: 986195089686528 / 1e12 = 986.195089686528 + "expected_flops_tuple": (283517065887744 / 1e12, 986195089686528 / 1e12), + }, + "apertus": { + "config": { # swiss-ai/Apertus-8B + "model_type": "apertus", + "vocab_size": 131072, + "hidden_size": 4096, + "intermediate_size": 21504, + "num_hidden_layers": 32, + "num_attention_heads": 32, + "num_key_value_heads": 32, + "hidden_act": "xielu", + # head_dim will be derived as 4096 / 32 = 128 + }, + "batch_seqlens_tuple": ([512, 1024, 2048], [4096, 4096, 4096]), + # Calculation for Apertus (hidden_act="xielu" -> MLP uses [k_mlp=2]*H*I params; qk_norm=True -> [k_qkn=2]*H): + # V=131072, H=4096, I=21504, L=32, k_mlp=2 (XIELU), k_qkn=2 (QK norm), S=6 + # S*(2*V*H + L*(4*H**2 + k_mlp*H*I + k_qkn*H)) * (SUM[seqlen]) + 12*SUM[seqlen**2]*L*H + "expected_flops_tuple": (199154680725504 / 1e12, 732294071451648 / 1e12), + }, +} + + +@pytest.mark.parametrize( + "config_type", + ["llama", "qwen2", "qwen3", "qwen3_moe", "deepseek_v3", "mistral", "gemma3_text", "apertus"], +) +def test_flops_counter(config_type: str): + test_config = CONFIG[config_type] + config = Config(test_config["config"]) + flops_counter = FlopsCounter(config) + for batch_seqlens, expected_flops in zip( + test_config["batch_seqlens_tuple"], test_config["expected_flops_tuple"], strict=True + ): + # set delta time to 1 to get the flops + counted_flops, _ = flops_counter.estimate_flops(batch_seqlens, 1) + print(f"Expect flops for {test_config['config']} is {expected_flops}, but get {counted_flops}") + assert math.isclose(counted_flops, expected_flops), ( + f"Expect flops for {test_config['config']} is {expected_flops}, but get {counted_flops}" + ) diff --git a/e2c/verl/tests/utils/test_fs_on_cpu.py b/e2c/verl/tests/utils/test_fs_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..7ae85e01aeccf25bdd906e3860a45338ed2406b3 --- /dev/null +++ b/e2c/verl/tests/utils/test_fs_on_cpu.py @@ -0,0 +1,94 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path + +import verl.utils.fs as fs + + +def test_record_and_check_directory_structure(tmp_path): + # Create test directory structure + test_dir = tmp_path / "test_dir" + test_dir.mkdir() + (test_dir / "file1.txt").write_text("test") + (test_dir / "subdir").mkdir() + (test_dir / "subdir" / "file2.txt").write_text("test") + + # Create structure record + record_file = fs._record_directory_structure(test_dir) + + # Verify record file exists + assert os.path.exists(record_file) + + # Initial check should pass + assert fs._check_directory_structure(test_dir, record_file) is True + + # Modify structure and verify check fails + (test_dir / "new_file.txt").write_text("test") + assert fs._check_directory_structure(test_dir, record_file) is False + + +def test_copy_from_hdfs_with_mocks(tmp_path, monkeypatch): + # Mock HDFS dependencies + monkeypatch.setattr(fs, "is_non_local", lambda path: True) + + # side_effect will simulate the copy by creating parent dirs + empty file + def fake_copy(src: str, dst: str, *args, **kwargs): + dst_path = Path(dst) + dst_path.parent.mkdir(parents=True, exist_ok=True) + dst_path.write_bytes(b"") # touch an empty file + + monkeypatch.setattr(fs, "copy", fake_copy) # Mock actual HDFS copy + + # Test parameters + test_cache = tmp_path / "cache" + hdfs_path = "hdfs://test/path/file.txt" + + # Test initial copy + local_path = fs.copy_to_local(hdfs_path, cache_dir=test_cache) + expected_path = os.path.join(test_cache, fs.md5_encode(hdfs_path), os.path.basename(hdfs_path)) + assert local_path == expected_path + assert os.path.exists(local_path) + + +def test_always_recopy_flag(tmp_path, monkeypatch): + # Mock HDFS dependencies + monkeypatch.setattr(fs, "is_non_local", lambda path: True) + + copy_call_count = 0 + + def fake_copy(src: str, dst: str, *args, **kwargs): + nonlocal copy_call_count + copy_call_count += 1 + dst_path = Path(dst) + dst_path.parent.mkdir(parents=True, exist_ok=True) + dst_path.write_bytes(b"") + + monkeypatch.setattr(fs, "copy", fake_copy) # Mock actual HDFS copy + + test_cache = tmp_path / "cache" + hdfs_path = "hdfs://test/path/file.txt" + + # Initial copy (always_recopy=False) + fs.copy_to_local(hdfs_path, cache_dir=test_cache) + assert copy_call_count == 1 + + # Force recopy (always_recopy=True) + fs.copy_to_local(hdfs_path, cache_dir=test_cache, always_recopy=True) + assert copy_call_count == 2 + + # Subsequent normal call (always_recopy=False) + fs.copy_to_local(hdfs_path, cache_dir=test_cache) + assert copy_call_count == 2 # Should not increment diff --git a/e2c/verl/tests/utils/test_groupwise.py b/e2c/verl/tests/utils/test_groupwise.py new file mode 100644 index 0000000000000000000000000000000000000000..59ba1f33e4afef91dc0246a22a9305df8f5a068c --- /dev/null +++ b/e2c/verl/tests/utils/test_groupwise.py @@ -0,0 +1,72 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +os.environ.setdefault("VERL_FORCE_DEVICE", "cpu") # ensure CPU for tests + +import numpy as np +import pytest +import torch + +from verl.utils import as_torch_index, group_mean_std + + +def test_as_torch_index_basic_integers(): + g = as_torch_index([2, 2, 5, 7, 5, 2]) + assert g.dtype == torch.long + assert g.device.type == "cpu" + # Values should be contiguous 0..G-1, keeping equal labels equal + assert g.tolist()[0] == g.tolist()[1] + assert len(torch.unique(g)) == 3 # {2,5,7} -> 3 groups + + +def test_as_torch_index_near_integer_floats(): + arr = np.array([1.0000001, 2.0, 1.0, 3.0000000001], dtype=np.float64) + g = as_torch_index(arr) # should round to integers then factorize + assert g.dtype == torch.long + assert len(torch.unique(g)) == 3 # {1,2,3} + + +def test_as_torch_index_factorization_mixed(): + labels = ["a", "b", "a", "c", "0042", 42] + g = as_torch_index(labels) + # "0042" and 42 should NOT be the same group (strings are not coerced here) + assert g.tolist()[4] != g.tolist()[5] + assert len(torch.unique(g)) == 5 + + +def test_group_mean_std_simple(): + # groups: 0 -> [1, 3], 1 -> [2] + scores = torch.tensor([1.0, 2.0, 3.0], dtype=torch.float32) + gidx = as_torch_index([0, 1, 0]) + + mean_g, std_g, cnt_g = group_mean_std(scores, gidx) + # group 0: mean = (1+3)/2 = 2 + # sample std (unbiased) = sqrt( (sum(x^2) - (sum(x)^2)/n) / (n-1) ) + # = sqrt( (1^2+3^2) - (1+3)^2/2 ) / (2-1) = sqrt(10 - 16/2) = sqrt(2) + assert torch.allclose(mean_g, torch.tensor([2.0, 0.0])) + assert torch.allclose(cnt_g, torch.tensor([2.0, 1.0])) + # singleton group -> std = 1.0 + assert mean_g[1].item() == 0.0 + assert std_g[1].item() == 1.0 + assert pytest.approx(std_g[0].item(), rel=1e-6) == (2.0**0.5) + + +def test_group_mean_std_empty(): + scores = torch.tensor([], dtype=torch.float32) + gidx = torch.tensor([], dtype=torch.long) + mean_g, std_g, cnt_g = group_mean_std(scores, gidx) + assert mean_g.numel() == 0 and std_g.numel() == 0 and cnt_g.numel() == 0 diff --git a/e2c/verl/tests/utils/test_import_utils_on_cpu.py b/e2c/verl/tests/utils/test_import_utils_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..59709b876bce5893c0c2d42a26af3ec1c6848d8e --- /dev/null +++ b/e2c/verl/tests/utils/test_import_utils_on_cpu.py @@ -0,0 +1,97 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from verl.utils.import_utils import load_extern_type + +# Path to the test module +TEST_MODULE_PATH = os.path.join(os.path.dirname(__file__), "_test_module.py") + + +def test_load_extern_type_class(): + """Test loading a class from an external file""" + TestClass = load_extern_type(TEST_MODULE_PATH, "TestClass") + + # Verify the class was loaded correctly + assert TestClass is not None + assert TestClass.__name__ == "TestClass" + + # Test instantiation and functionality + instance = TestClass() + assert instance.value == "default" + + # Test with a custom value + custom_instance = TestClass("custom") + assert custom_instance.get_value() == "custom" + + +def test_load_extern_type_function(): + """Test loading a function from an external file""" + test_function = load_extern_type(TEST_MODULE_PATH, "test_function") + + # Verify the function was loaded correctly + assert test_function is not None + assert callable(test_function) + + # Test function execution + result = test_function() + assert result == "test_function_result" + + +def test_load_extern_type_constant(): + """Test loading a constant from an external file""" + constant = load_extern_type(TEST_MODULE_PATH, "TEST_CONSTANT") + + # Verify the constant was loaded correctly + assert constant is not None + assert constant == "test_constant_value" + + +def test_load_extern_type_nonexistent_file(): + """Test behavior when file doesn't exist""" + with pytest.raises(FileNotFoundError): + load_extern_type("/nonexistent/path.py", "SomeType") + + +def test_load_extern_type_nonexistent_type(): + """Test behavior when type doesn't exist in the file""" + with pytest.raises(AttributeError): + load_extern_type(TEST_MODULE_PATH, "NonExistentType") + + +def test_load_extern_type_none_path(): + """Test behavior when file path is None""" + result = load_extern_type(None, "SomeType") + assert result is None + + +def test_load_extern_type_invalid_module(): + """Test behavior when module has syntax errors""" + # Create a temporary file with syntax errors + import tempfile + + with tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False) as temp_file: + temp_file.write("This is not valid Python syntax :") + temp_path = temp_file.name + + try: + with pytest.raises(RuntimeError): + load_extern_type(temp_path, "SomeType") + finally: + # Clean up the temporary file + if os.path.exists(temp_path): + os.remove(temp_path) diff --git a/e2c/verl/tests/utils/test_linear_cross_entropy.py b/e2c/verl/tests/utils/test_linear_cross_entropy.py new file mode 100644 index 0000000000000000000000000000000000000000..0512d1376de07d32cfa5862e72acf826a9588433 --- /dev/null +++ b/e2c/verl/tests/utils/test_linear_cross_entropy.py @@ -0,0 +1,361 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch + +import verl.utils.torch_functional as verl_F +from verl.utils.experimental.torch_functional import FusedLinearForPPO +from verl.utils.kernel.linear_cross_entropy import linear_cross_entropy +from verl.utils.torch_functional import logprobs_from_logits + +compute_entropy_from_logits = torch.compile(verl_F.entropy_from_logits, dynamic=True) +fused_linear_for_ppo = FusedLinearForPPO() +fused_linear_for_ppo.compile(dynamic=True) + +MAX_TEST_CASES = os.environ.get("MAX_TEST_CASES", 5) + + +def run_torch_entropy( + hidden: torch.Tensor, weight: torch.Tensor, labels: torch.Tensor, temperature: float, reduction="none" +) -> list[torch.Tensor]: + hidden = hidden.squeeze(0).to(torch.float32) + weight = weight.transpose(0, 1).to(torch.float32) + logits = torch.matmul(hidden, weight) # [num_tokens, vocab_size] + logits /= temperature + pd = torch.nn.functional.softmax(logits, dim=-1) # [num_tokens, vocab_size] + entropy_a = torch.logsumexp(logits, dim=-1) # [num_tokens] + entropy_b = torch.sum(pd * logits, dim=-1) # [num_tokens] + entropy = entropy_a - entropy_b + logprobs = torch.nn.functional.cross_entropy(logits, labels.squeeze(0), reduction=reduction) # [num_tokens] + logprobs = torch.neg(logprobs) + return logprobs, entropy + + +def run_verl_original_entropy( + hidden: torch.Tensor, + weight: torch.Tensor, + labels: torch.Tensor, + temperature: float, +) -> list[torch.Tensor]: + hidden = hidden.squeeze(0).to(torch.float32) + weight = weight.transpose(0, 1).to(torch.float32) + logits = torch.matmul(hidden, weight) # [num_tokens, vocab_size] + logits /= temperature + # compute entropy + entropy = compute_entropy_from_logits(logits) # ((total_nnz / sp) + pad) + # if use_sp: ((total_nnz / sp) + pad) ; if not use_sp: (batch, seqlen) + logprobs = logprobs_from_logits(logits=logits, labels=labels, inplace_backward=False) + return logprobs, entropy + + +# To be tested +def run_verl_torch_fused_entropy( + hidden: torch.Tensor, + weight: torch.Tensor, + labels: torch.Tensor, + temperature: float, +): + hidden = hidden.to(torch.float32) + weight = weight.to(torch.float32) + logprobs, entropy = fused_linear_for_ppo( + hidden, + weight, + labels, + temperature=temperature, + ) + return logprobs.squeeze(0), entropy.squeeze(0) + + +class TestLinearCrossEntropy: + def __init__(self, test_case_idx: int, temperature: float = 1.5) -> None: + self.test_case_idx = test_case_idx + self.temperature = temperature + + def cleanup(self): + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats() + import gc + + gc.collect() + torch.cuda.synchronize() + + def generate_hyper(self): + global MAX_TEST_CASES + + self.dtype = torch.bfloat16 + if self.test_case_idx == 0: + self.batch_size = 1 + self.num_tokens = 1937 + self.hidden_size = 3584 + self.vocab_size = 152064 + elif self.test_case_idx == 1: + self.batch_size = 1 + self.num_tokens = 2169 + self.hidden_size = 896 + self.vocab_size = 151936 + elif self.test_case_idx == 2: + self.batch_size = 1 + self.num_tokens = 1530 + self.hidden_size = 2048 + self.vocab_size = 32256 + elif self.test_case_idx == 3: + self.batch_size = 1 + self.num_tokens = 1388 + self.hidden_size = 4096 + self.vocab_size = 102400 + elif self.test_case_idx == 4: + self.batch_size = 1 + self.num_tokens = 8192 + self.hidden_size = 4096 + self.vocab_size = 102400 + else: + raise ValueError(f"Invalid test case index: {self.test_case_idx}") + assert MAX_TEST_CASES <= 5, "MAX_TEST_CASES should be less than or equal to 5." + + def generate_forward_inputs(self): + hidden = ( + torch.empty((self.batch_size, self.num_tokens, self.hidden_size), dtype=self.dtype, device="cuda") + .uniform_(-0.5, 0.5) + .requires_grad_() + ) + weight = ( + torch.empty((self.vocab_size, self.hidden_size), dtype=self.dtype, device="cuda") + .uniform_(-0.5, 0.5) + .requires_grad_() + ) + labels = torch.randint(0, self.vocab_size, (self.batch_size, self.num_tokens), device="cuda") + return hidden, weight, labels + + def generate_backward_inputs(self): + g_entropy = torch.empty((self.num_tokens,), dtype=self.dtype, device="cuda").uniform_(-0.5, 0.5) + g_logprobs = torch.empty((self.num_tokens,), dtype=self.dtype, device="cuda").uniform_(-1, 1) + return g_entropy, g_logprobs + + def verify_correctness(self, iterations=5): + self.cleanup() + self.generate_hyper() + + torch_forward_latency = list() + torch_backward_latency = list() + verl_forward_latency = list() + verl_backward_latency = list() + verl_fused_forward_latency = list() + verl_fused_backward_latency = list() + kernel_forward_latency = list() + kernel_backward_latency = list() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + for i in range(iterations): + print(f"[INFO]: Iteration {i + 1} / {iterations}...", end="\r") + hidden, weight, labels = self.generate_forward_inputs() + + start_event.record() + (torch_logprobs, torch_entropy) = run_torch_entropy(hidden, weight, labels, self.temperature) + end_event.record() + torch.cuda.synchronize() + torch_forward_latency.append(start_event.elapsed_time(end_event)) + + start_event.record() + (verl_logprobs, verl_entropy) = run_verl_original_entropy(hidden, weight, labels, self.temperature) + end_event.record() + torch.cuda.synchronize() + verl_forward_latency.append(start_event.elapsed_time(end_event)) + + start_event.record() + (verl_fused_logprobs, verl_fused_entropy) = run_verl_torch_fused_entropy( + hidden, weight, labels, self.temperature + ) + end_event.record() + torch.cuda.synchronize() + verl_fused_forward_latency.append(start_event.elapsed_time(end_event)) + + start_event.record() + (kernel_logprobs, kernel_entropy) = linear_cross_entropy(hidden, weight, labels, self.temperature) + end_event.record() + torch.cuda.synchronize() + kernel_forward_latency.append(start_event.elapsed_time(end_event)) + + torch.testing.assert_close(torch_logprobs, verl_logprobs, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(torch_entropy, verl_entropy, atol=1e-4, rtol=1e-4) + + torch.testing.assert_close(torch_logprobs, verl_fused_logprobs, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(torch_entropy, verl_fused_entropy, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(verl_logprobs, verl_fused_logprobs, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(verl_entropy, verl_fused_entropy, atol=1e-4, rtol=1e-4) + + torch.testing.assert_close(torch_logprobs, kernel_logprobs, atol=1e-3, rtol=2e-4) + torch.testing.assert_close(torch_entropy, kernel_entropy, atol=5e-3, rtol=5e-4) + torch.testing.assert_close(verl_logprobs, kernel_logprobs, atol=1e-3, rtol=2e-4) + torch.testing.assert_close(verl_entropy, kernel_entropy, atol=5e-3, rtol=5e-4) + torch.testing.assert_close(verl_fused_logprobs, kernel_logprobs, atol=1e-3, rtol=2e-4) + torch.testing.assert_close(verl_fused_entropy, kernel_entropy, atol=5e-3, rtol=5e-4) + + # backward + g_entropy, g_logprobs = self.generate_backward_inputs() + + start_event.record() + (d_torch_hidden, d_torch_weight) = torch.autograd.grad( + (torch_entropy, torch_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + end_event.record() + torch.cuda.synchronize() + torch_backward_latency.append(start_event.elapsed_time(end_event)) + + start_event.record() + (d_verl_hidden, d_verl_weight) = torch.autograd.grad( + (verl_entropy, verl_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + end_event.record() + torch.cuda.synchronize() + verl_backward_latency.append(start_event.elapsed_time(end_event)) + + start_event.record() + (d_verl_fused_hidden, d_verl_fused_weight) = torch.autograd.grad( + (verl_fused_entropy, verl_fused_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + end_event.record() + torch.cuda.synchronize() + verl_fused_backward_latency.append(start_event.elapsed_time(end_event)) + + start_event.record() + (d_kernel_hidden, d_kernel_weight) = torch.autograd.grad( + (kernel_entropy, kernel_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + end_event.record() + torch.cuda.synchronize() + kernel_backward_latency.append(start_event.elapsed_time(end_event)) + + torch.testing.assert_close(d_torch_hidden, d_verl_hidden, atol=1e-2, rtol=1e-4) + torch.testing.assert_close(d_torch_weight, d_verl_weight, atol=1e-2, rtol=1e-4) + + torch.testing.assert_close(d_torch_hidden, d_verl_fused_hidden, atol=1e-2, rtol=1e-4) + torch.testing.assert_close(d_torch_weight, d_verl_fused_weight, atol=1e-2, rtol=1e-4) + torch.testing.assert_close(d_verl_hidden, d_verl_fused_hidden, atol=1e-2, rtol=1e-4) + torch.testing.assert_close(d_verl_weight, d_verl_fused_weight, atol=1e-2, rtol=1e-4) + torch.testing.assert_close(d_torch_hidden, d_verl_hidden, atol=1e-2, rtol=1e-4) + torch.testing.assert_close(d_torch_weight, d_verl_weight, atol=1e-2, rtol=1e-4) + + torch.testing.assert_close(d_torch_hidden, d_kernel_hidden, atol=2e-2, rtol=4e-2) + torch.testing.assert_close(d_torch_weight, d_kernel_weight, atol=2e-2, rtol=4e-2) + torch.testing.assert_close(d_verl_hidden, d_kernel_hidden, atol=2e-2, rtol=4e-2) + torch.testing.assert_close(d_verl_weight, d_kernel_weight, atol=2e-2, rtol=4e-2) + torch.testing.assert_close(d_verl_fused_hidden, d_kernel_hidden, atol=2e-2, rtol=4e-2) + torch.testing.assert_close(d_verl_fused_weight, d_kernel_weight, atol=2e-2, rtol=4e-2) + + # remove first latency + torch_forward_latency = torch_forward_latency[1:] + torch_backward_latency = torch_backward_latency[1:] + verl_forward_latency = verl_forward_latency[1:] + verl_backward_latency = verl_backward_latency[1:] + verl_fused_forward_latency = verl_fused_forward_latency[1:] + verl_fused_backward_latency = verl_fused_backward_latency[1:] + kernel_forward_latency = kernel_forward_latency[1:] + kernel_backward_latency = kernel_backward_latency[1:] + + print("\n[INFO]: Verified forward & backward correctness.") + + print( + f"[INFO]: Forward pass: Torch implementation average time: " + f"{sum(torch_forward_latency) / len(torch_forward_latency):.2f} ms" + ) + print( + f"[INFO]: Backward pass: torch implementation average time: " + f"{sum(torch_backward_latency) / len(torch_backward_latency):.2f} ms" + ) + print( + f"[INFO]: Forward pass: VeRL implementation average time: " + f"{sum(verl_forward_latency) / len(verl_forward_latency):.2f} ms" + ) + print( + f"[INFO]: Backward pass: VeRL implementation average time: " + f"{sum(verl_backward_latency) / len(verl_backward_latency):.2f} ms" + ) + print( + f"[INFO]: Forward pass: VeRL Fused Entropy implementation average time: " + f"{sum(verl_fused_forward_latency) / len(verl_fused_forward_latency):.2f} ms" + ) + print( + f"[INFO]: Backward pass: VeRL Fused Entropy implementation average time: " + f"{sum(verl_fused_backward_latency) / len(verl_fused_backward_latency):.2f} ms" + ) + print( + f"[INFO]: Forward pass: Kernel implementation average time: " + f"{sum(kernel_forward_latency) / len(kernel_forward_latency):.2f} ms" + ) + print( + f"[INFO]: Backward pass: kernel implementation average time: " + f"{sum(kernel_backward_latency) / len(kernel_backward_latency):.2f} ms" + ) + + def check_storage(self, method_name, run_forward): + self.cleanup() + self.generate_hyper() + + hidden, weight, labels = self.generate_forward_inputs() + + torch.cuda.reset_peak_memory_stats() + (logprobs, entropy) = run_forward(hidden, weight, labels, self.temperature) + torch.cuda.synchronize() + torch_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024 + print(f"[INFO]: {method_name} Forward pass peak memory: {torch_max_memory:.2f} MB") + + g_entropy, g_logprobs = self.generate_backward_inputs() + + torch.cuda.reset_peak_memory_stats() + (d_torch_hidden, d_torch_weight) = torch.autograd.grad( + (entropy, logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + torch.cuda.synchronize() + torch_backward_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024 + print(f"[INFO]: {method_name} Backward pass peak memory: {torch_backward_max_memory:.2f} MB") + + def check_storage_all(self): + self.check_storage("Torch", run_torch_entropy) + self.check_storage("VeRL", run_verl_original_entropy) + self.check_storage("VeRL Torch Fused", run_verl_torch_fused_entropy) + self.check_storage("Kernel", linear_cross_entropy) + + +if __name__ == "__main__": + # torch.cuda.memory._record_memory_history() + + for test_case_idx in range(MAX_TEST_CASES): + print(f"[INFO] Running test case {test_case_idx}") + test = TestLinearCrossEntropy(test_case_idx) + + test.verify_correctness() + test.check_storage_all() + + # torch.cuda.memory._dump_snapshot("test_linear_cross_entropy.pkl") diff --git a/e2c/verl/tests/utils/test_model_on_cpu.py b/e2c/verl/tests/utils/test_model_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..8b1416c8a03a7607cd54f92ccadfa41af11ece4e --- /dev/null +++ b/e2c/verl/tests/utils/test_model_on_cpu.py @@ -0,0 +1,52 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import SimpleNamespace # Or use a mock object library + +import pytest + +from verl.utils.model import update_model_config + + +# Parametrize with different override scenarios +@pytest.mark.parametrize( + "override_kwargs", + [ + {"param_a": 5, "new_param": "plain_added"}, + {"param_a": 2, "nested_params": {"sub_param_x": "updated_x", "sub_param_z": True}}, + ], +) +def test_update_model_config(override_kwargs): + """ + Tests that update_model_config correctly updates attributes, + handling both plain and nested overrides via parametrization. + """ + # Create a fresh mock config object for each test case + mock_config = SimpleNamespace( + param_a=1, nested_params=SimpleNamespace(sub_param_x="original_x", sub_param_y=100), other_param="keep_me" + ) + # Apply the updates using the parametrized override_kwargs + update_model_config(mock_config, override_kwargs) + + # Assertions to check if the config was updated correctly + if "nested_params" in override_kwargs: # Case 2: Nested override + override_nested = override_kwargs["nested_params"] + assert mock_config.nested_params.sub_param_x == override_nested["sub_param_x"], "Nested sub_param_x mismatch" + assert mock_config.nested_params.sub_param_y == 100, "Nested sub_param_y should be unchanged" + assert hasattr(mock_config.nested_params, "sub_param_z"), "Expected nested sub_param_z to be added" + assert mock_config.nested_params.sub_param_z == override_nested["sub_param_z"], "Value of sub_param_z mismatch" + else: # Case 1: Plain override (nested params untouched) + assert mock_config.nested_params.sub_param_x == "original_x", "Nested sub_param_x should be unchanged" + assert mock_config.nested_params.sub_param_y == 100, "Nested sub_param_y should be unchanged" + assert not hasattr(mock_config.nested_params, "sub_param_z"), "Nested sub_param_z should not exist" diff --git a/e2c/verl/tests/utils/test_nvtx_profile.py b/e2c/verl/tests/utils/test_nvtx_profile.py new file mode 100644 index 0000000000000000000000000000000000000000..645da153d0a89c14d26f3da7bd68c0dd1797225f --- /dev/null +++ b/e2c/verl/tests/utils/test_nvtx_profile.py @@ -0,0 +1,168 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest.mock import MagicMock, patch + +from verl.utils import omega_conf_to_dataclass +from verl.utils.profiler.config import NsightToolConfig, ProfilerConfig +from verl.utils.profiler.nvtx_profile import NsightSystemsProfiler + + +class TestProfilerConfig(unittest.TestCase): + def test_config_init(self): + import os + + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + cfg = compose(config_name="ppo_trainer") + for config in [ + cfg.actor_rollout_ref.actor.profiler, + cfg.actor_rollout_ref.rollout.profiler, + cfg.actor_rollout_ref.ref.profiler, + cfg.critic.profiler, + cfg.reward_model.profiler, + ]: + profiler_config = omega_conf_to_dataclass(config) + self.assertEqual(profiler_config.tool, config.tool) + self.assertEqual(profiler_config.enable, config.enable) + self.assertEqual(profiler_config.all_ranks, config.all_ranks) + self.assertEqual(profiler_config.ranks, config.ranks) + self.assertEqual(profiler_config.save_path, config.save_path) + self.assertEqual(profiler_config.ranks, config.ranks) + assert isinstance(profiler_config, ProfilerConfig) + with self.assertRaises(AttributeError): + _ = profiler_config.non_existing_key + assert config.get("non_existing_key") == profiler_config.get("non_existing_key") + assert config.get("non_existing_key", 1) == profiler_config.get("non_existing_key", 1) + + def test_frozen_config(self): + """Test that modifying frozen keys in ProfilerConfig raises exceptions.""" + from dataclasses import FrozenInstanceError + + from verl.utils.profiler.config import ProfilerConfig + + # Create a new ProfilerConfig instance + config = ProfilerConfig(all_ranks=False, ranks=[0]) + + with self.assertRaises(FrozenInstanceError): + config.all_ranks = True + + with self.assertRaises(FrozenInstanceError): + config.ranks = [1, 2, 3] + + with self.assertRaises(TypeError): + config["all_ranks"] = True + + with self.assertRaises(TypeError): + config["ranks"] = [1, 2, 3] + + +class TestNsightSystemsProfiler(unittest.TestCase): + """Test suite for NsightSystemsProfiler functionality. + + Test Plan: + 1. Initialization: Verify profiler state after creation + 2. Basic Profiling: Test start/stop functionality + 3. Discrete Mode: TODO: Test discrete profiling behavior + 4. Annotation: Test the annotate decorator in both normal and discrete modes + 5. Config Validation: Verify proper config initialization from OmegaConf + """ + + def setUp(self): + self.config = ProfilerConfig(enable=True, all_ranks=True) + self.rank = 0 + self.profiler = NsightSystemsProfiler(self.rank, self.config, tool_config=NsightToolConfig(discrete=False)) + + def test_initialization(self): + self.assertEqual(self.profiler.this_rank, True) + self.assertEqual(self.profiler.this_step, False) + + def test_start_stop_profiling(self): + with patch("torch.cuda.profiler.start") as mock_start, patch("torch.cuda.profiler.stop") as mock_stop: + # Test start + self.profiler.start() + self.assertTrue(self.profiler.this_step) + mock_start.assert_called_once() + + # Test stop + self.profiler.stop() + self.assertFalse(self.profiler.this_step) + mock_stop.assert_called_once() + + # def test_discrete_profiling(self): + # discrete_config = ProfilerConfig(discrete=True, all_ranks=True) + # profiler = NsightSystemsProfiler(self.rank, discrete_config) + + # with patch("torch.cuda.profiler.start") as mock_start, patch("torch.cuda.profiler.stop") as mock_stop: + # profiler.start() + # self.assertTrue(profiler.this_step) + # mock_start.assert_not_called() # Shouldn't start immediately in discrete mode + + # profiler.stop() + # self.assertFalse(profiler.this_step) + # mock_stop.assert_not_called() # Shouldn't stop immediately in discrete mode + + def test_annotate_decorator(self): + mock_self = MagicMock() + mock_self.profiler = self.profiler + mock_self.profiler.this_step = True + decorator = mock_self.profiler.annotate(message="test") + + @decorator + def test_func(self, *args, **kwargs): + return "result" + + with ( + patch("torch.cuda.profiler.start") as mock_start, + patch("torch.cuda.profiler.stop") as mock_stop, + patch("verl.utils.profiler.nvtx_profile.mark_start_range") as mock_start_range, + patch("verl.utils.profiler.nvtx_profile.mark_end_range") as mock_end_range, + ): + result = test_func(mock_self) + self.assertEqual(result, "result") + mock_start_range.assert_called_once() + mock_end_range.assert_called_once() + mock_start.assert_not_called() # Not discrete mode + mock_stop.assert_not_called() # Not discrete mode + + # def test_annotate_discrete_mode(self): + # discrete_config = ProfilerConfig(discrete=True, all_ranks=True) + # profiler = NsightSystemsProfiler(self.rank, discrete_config) + # mock_self = MagicMock() + # mock_self.profiler = profiler + # mock_self.profiler.this_step = True + + # @NsightSystemsProfiler.annotate(message="test") + # def test_func(self, *args, **kwargs): + # return "result" + + # with ( + # patch("torch.cuda.profiler.start") as mock_start, + # patch("torch.cuda.profiler.stop") as mock_stop, + # patch("verl.utils.profiler.nvtx_profile.mark_start_range") as mock_start_range, + # patch("verl.utils.profiler.nvtx_profile.mark_end_range") as mock_end_range, + # ): + # result = test_func(mock_self) + # self.assertEqual(result, "result") + # mock_start_range.assert_called_once() + # mock_end_range.assert_called_once() + # mock_start.assert_called_once() # Should start in discrete mode + # mock_stop.assert_called_once() # Should stop in discrete mode + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/utils/test_rollout_skip_on_cpu.py b/e2c/verl/tests/utils/test_rollout_skip_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..5b8b31e641d03d82ca06deb608db86d83a35ed33 --- /dev/null +++ b/e2c/verl/tests/utils/test_rollout_skip_on_cpu.py @@ -0,0 +1,142 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import shutil +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import torch + +from verl.utils.rollout_skip import DataProto, RolloutSkip + +len_prompt = 50 +len_response = 100 + + +def temp_dir(): + # Create a temporary directory + temp_dir = Path(tempfile.mkdtemp()) + yield temp_dir + # Cleanup + shutil.rmtree(temp_dir) + + +def build_generate_fn(gen_bs, n): + len_tokenizer = 1024 + + def iterate(): + while True: + prompt = torch.randint(len_tokenizer, size=(gen_bs, len_prompt)).repeat_interleave(n, dim=0) + generate = torch.randint(len_tokenizer, size=(gen_bs * n, len_response)) + data = DataProto.from_dict(tensors={"prompt": prompt, "response": generate}) + yield data + + mock_infer_engine = iterate() + + def fn(batch, **kwargs): + # Simulate the inference engine returning the next batch + return next(mock_infer_engine) + + return fn + + +@pytest.fixture(params=[(32, 4), (64, 4), (64, 8)]) +def mock_rollout_wg(request): + gen_bs, n = request.param + rollout_wg = MagicMock() + + config = MagicMock() + config.actor_rollout_ref.rollout = { + "n": n, + "skip_dump_dir": next(temp_dir()), + } + config.data = {"gen_batch_size": gen_bs} + + rollout_wg.generate_sequences = build_generate_fn(gen_bs, n) + + yield config, rollout_wg + # Cleanup + shutil.rmtree(next(temp_dir())) + + +class TestRolloutSkip: + def test_initialization(self, capsys): + """Test that RolloutSkip initializes correctly""" + config = MagicMock() + config.actor_rollout_ref.rollout = { + "n": 16, + "skip_dump_dir": "tmp/rollout_dump", + } + config.data = {"gen_batch_size": 128} + mock_rollout_wg = MagicMock() + skip = RolloutSkip(config, mock_rollout_wg) + + assert skip.n == 16 + assert skip.gbs == 128 + assert str(skip.dumped_dir) == "tmp/rollout_dump" + + assert skip._rollout_wg == mock_rollout_wg + skip.wrap_generate_sequences() + captured = capsys.readouterr() + assert "Successfully patched" in captured.out + + def test_generate_without_wrap(self, mock_rollout_wg): + """Test that generate_sequences works without wrapping""" + + config, rollout_wg = mock_rollout_wg + _ = RolloutSkip(config, rollout_wg) + + _result = rollout_wg.generate_sequences(MagicMock()) + for _ in range(10): + result = rollout_wg.generate_sequences(MagicMock()) + assert isinstance(result, DataProto) + # * make sure the data is different + assert torch.abs(_result.batch["prompt"] - result.batch["prompt"]).sum() > 0 + assert torch.abs(_result.batch["response"] - result.batch["response"]).sum() > 0 + _result = result + + def test_dump(self, mock_rollout_wg, capsys): + config, rollout_wg = mock_rollout_wg + skip = RolloutSkip(config, rollout_wg) + skip.wrap_generate_sequences() + + result = rollout_wg.generate_sequences(MagicMock()) + # * check if dump is OK + assert skip.curr_path_dump.exists() + captured = capsys.readouterr() + assert "Successfully dump data in" in captured.out + # * get file size, estimate file size + file_size = skip.curr_path_dump.stat().st_size + est_file_size = (len_prompt + len_response) * skip.gbs * skip.n * result.batch["prompt"].dtype.itemsize + assert file_size >= est_file_size, "Dumped file size is smaller than expected" + + def test_generate_with_wrap(self, mock_rollout_wg, capsys): + """Test that generate_sequences works without wrapping""" + + config, rollout_wg = mock_rollout_wg + skip = RolloutSkip(config, rollout_wg) + skip.wrap_generate_sequences() + + _result = rollout_wg.generate_sequences(MagicMock()) + + for _ in range(10): + result = rollout_wg.generate_sequences(MagicMock()) + assert isinstance(result, DataProto) + # * make sure the data is different + assert torch.abs(_result.batch["prompt"] - result.batch["prompt"]).sum() == 0 + assert torch.abs(_result.batch["response"] - result.batch["response"]).sum() == 0 + captured = capsys.readouterr() + assert "Successfully load pre-generated data from" in captured.out + _result = result diff --git a/e2c/verl/tests/utils/test_rollout_trace_on_cpu.py b/e2c/verl/tests/utils/test_rollout_trace_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..04dfbeef8d949650d6c3be17f2b74f937d329836 --- /dev/null +++ b/e2c/verl/tests/utils/test_rollout_trace_on_cpu.py @@ -0,0 +1,170 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from verl.utils.rollout_trace import RolloutTraceConfig, rollout_trace_attr, rollout_trace_op + + +@pytest.fixture(autouse=True) +def reset_rollout_trace_config_singleton(): + """Fixture to reset the RolloutTraceConfig singleton before each test.""" + RolloutTraceConfig.reset() + + +@pytest.fixture +def mock_weave_client(): + """Mocks the weave module and its client, yielding the mock client.""" + mock_weave = MagicMock() + mock_client = MagicMock() + mock_call = MagicMock() + mock_client.create_call.return_value = mock_call + mock_weave.init.return_value = mock_client + + # Also mock the call_context if it's used internally by the decorator + mock_weave.trace.context.call_context.return_value = MagicMock() + + with patch.dict(sys.modules, {"weave": mock_weave, "weave.trace.context": mock_weave.trace.context}): + yield mock_client + + +class TracedClass: + @rollout_trace_op + # @weave.op + # @mlflow.trace + async def my_method(self, a, b="default"): + return f"result: {a}, {b}" + + @rollout_trace_op + # @weave.op + # @mlflow.trace + async def middle_method(self, a, b="default"): + await self.my_method("test_a1", b="test_b1") + return f"result: {a}, {b}" + + @rollout_trace_op + # @mlflow.trace + async def my_method_with_exception(self): + raise ValueError("Test Exception") + + async def upper_method(self): + await self.my_method("test_a0", b="test_b0") + await self.middle_method("test_a2", b="test_b2") + return True + + +class UntracedClass: + @rollout_trace_op + async def my_method(self, x): + return x * 2 + + +async def test_rollout_trace_on_untraced_class(): + """Tests that the decorator works correctly when no backend is configured.""" + instance = UntracedClass() + assert await instance.my_method(10) == 20 + + +async def test_rollout_trace_with_tracer(mock_weave_client): + """Tests that the decorator calls the tracer's methods correctly.""" + RolloutTraceConfig.init(project_name="my-project", experiment_name="my-experiment", backend="weave") + instance = TracedClass() + assert RolloutTraceConfig.get_client() is mock_weave_client + + result = await instance.my_method("test_a", b="test_b") + + assert result == "result: test_a, test_b" + mock_weave_client.create_call.assert_called_once() + call_kwargs = mock_weave_client.create_call.call_args.kwargs + assert call_kwargs["op"] == "TracedClass.my_method" + expected_inputs = {"a": "test_a", "b": "test_b"} + assert call_kwargs["inputs"] == expected_inputs + + mock_call = mock_weave_client.create_call.return_value + mock_weave_client.finish_call.assert_called_once_with(mock_call, output=result) + + +async def test_rollout_trace_with_exception(mock_weave_client): + """Tests that `finish` is called with the exception when one is raised.""" + RolloutTraceConfig.init(project_name="my-project", experiment_name="my-experiment", backend="weave") + instance = TracedClass() + + with pytest.raises(ValueError, match="Test Exception"): + await instance.my_method_with_exception() + + mock_weave_client.create_call.assert_called_once() + mock_call = mock_weave_client.create_call.return_value + mock_weave_client.finish_call.assert_called_once() + + # Check that finish_call was called with the exception + args, kwargs = mock_weave_client.finish_call.call_args + assert args[0] == mock_call + assert "exception" in kwargs + assert isinstance(kwargs["exception"], ValueError) + + +async def test_rollout_trace_with_dummy_backend(mock_weave_client): + """Tests that the tracer is not called when the backend is 'dummy'.""" + RolloutTraceConfig.init(project_name="my-project", experiment_name="my-experiment", backend="dummy") + instance = TracedClass() + + await instance.my_method("test_a") + + mock_weave_client.create_call.assert_not_called() + + +@pytest.mark.skipif( + os.environ.get("RUN_WEAVE_INTEGRATION_TESTS", "false").lower() != "true", + reason="Skipping weave integration test. Set RUN_WEAVE_INTEGRATION_TESTS=true to run.", +) +async def test_rollout_trace_with_real_weave_backend(): + """Integration test with a real weave backend.""" + + # This assumes that the weave environment (e.g., project) is configured + RolloutTraceConfig.init(project_name="my-project", experiment_name="my-experiment", backend="weave") + + instance = TracedClass() + + with rollout_trace_attr(step=1, sample_index=2, rollout_n=3): + await instance.upper_method() + + with pytest.raises(ValueError, match="Test Exception"): + await instance.my_method_with_exception() + + print("\nWeave integration test ran successfully. Check your weave project for the trace.") + + +@pytest.mark.skipif( + os.environ.get("RUN_MLFLOW_INTEGRATION_TESTS", "false").lower() != "true", + reason="Skipping mlflow integration test. Set RUN_MLFLOW_INTEGRATION_TESTS=true to run.", +) +async def test_rollout_trace_with_real_mlflow_backend(): + """Integration test with a real mlflow backend.""" + + # This assumes that the mlflow environment (e.g., project) is configured + RolloutTraceConfig.init(project_name="my-project", experiment_name="my-experiment", backend="mlflow") + + instance = TracedClass() + + with rollout_trace_attr(step=1, sample_index=2, rollout_n=3, name="agent_run"): + assert await instance.upper_method() + + # with pytest.raises(ValueError, match="Test Exception"): + # await instance.my_method_with_exception() + + print("\nWeave integration test ran successfully. Check your weave project for the trace.") diff --git a/e2c/verl/tests/utils/test_seqlen_balancing.py b/e2c/verl/tests/utils/test_seqlen_balancing.py new file mode 100644 index 0000000000000000000000000000000000000000..9de777f1c9e9ba0939c04df810e98a2d67b6708c --- /dev/null +++ b/e2c/verl/tests/utils/test_seqlen_balancing.py @@ -0,0 +1,201 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from verl import DataProto +from verl.utils.model import create_random_mask +from verl.utils.seqlen_balancing import ( + ceildiv, + get_reverse_idx, + prepare_dynamic_batch, + rearrange_micro_batches, + restore_dynamic_batch, +) + + +def test_seqlen_balancing(): + input_ids = torch.randint(low=0, high=10, size=(20, 100)) + + attention_mask = create_random_mask( + input_ids=input_ids, max_ratio_of_left_padding=0.1, max_ratio_of_valid_token=0.9, min_ratio_of_valid_token=0.5 + ) + data = {"input_ids": input_ids, "attention_mask": attention_mask} + dataproto = DataProto.from_single_dict(data) + micro_batches, micro_bsz_idx_lst = rearrange_micro_batches(dataproto.batch, max_token_len=300) + batch = torch.cat(micro_batches) + micro_bsz_idx = [] + for idx in micro_bsz_idx_lst: + micro_bsz_idx.extend(idx) + reverse_idx_map = get_reverse_idx(micro_bsz_idx) + reverse_idx_map = torch.tensor(reverse_idx_map) + new_batch = batch[reverse_idx_map] + torch.testing.assert_close(new_batch, dataproto.batch) + + +def test_dynamic_batch(): + input_ids = torch.randint(low=0, high=10, size=(20, 100)) + + attention_mask = create_random_mask( + input_ids=input_ids, max_ratio_of_left_padding=0.1, max_ratio_of_valid_token=0.9, min_ratio_of_valid_token=0.5 + ) + data = {"input_ids": input_ids, "attention_mask": attention_mask} + dataproto = DataProto.from_single_dict(data) + micro_batches, micro_bsz_idx_lst = prepare_dynamic_batch(dataproto, max_token_len=300) + input_ids = torch.cat([micro_batch.batch["input_ids"] for micro_batch in micro_batches], dim=0) + input_ids = restore_dynamic_batch(input_ids, micro_bsz_idx_lst) + torch.testing.assert_close(input_ids, dataproto.batch["input_ids"]) + + +def _worker(rank, world_size, init_method, max_token_len, use_same_dp, min_mb): + # 1) init process group & CUDA + torch.cuda.set_device(rank) + dist.init_process_group( + backend="nccl", + init_method=init_method, + world_size=world_size, + rank=rank, + ) + + # 2) build a small random batch (each rank different length to force mismatch) + torch.manual_seed(42 + rank) + input_ids = torch.randint(0, 10, (20 + rank * 5, 100), device=f"cuda:{rank}") + attention_mask = create_random_mask( + input_ids=input_ids, + max_ratio_of_left_padding=0.1, + max_ratio_of_valid_token=0.9, + min_ratio_of_valid_token=0.5, + ) + dp = {"input_ids": input_ids, "attention_mask": attention_mask} + proto = DataProto.from_single_dict(dp) + batch = proto.batch + + # 3) call rearrange_micro_batches with one of the two params under test + micros, idx_lst = rearrange_micro_batches( + batch, + max_token_len=max_token_len, + dp_group=dist.group.WORLD, + same_micro_num_in_dp=use_same_dp, + min_num_micro_batch=min_mb, + ) + + # 4) check the enforced counts + seq_len_effective: torch.Tensor = batch["attention_mask"].sum(dim=1) + total_seqlen = seq_len_effective.sum().item() + local = min(len(seq_len_effective), ceildiv(total_seqlen, max_token_len)) + + if min_mb is not None: + expected = max(local, min_mb) + assert len(micros) == expected + if use_same_dp: + # gather all local_counts + counts = [torch.zeros(1, device=f"cuda:{rank}") for _ in range(world_size)] + counts[rank].fill_(local) + dist.all_gather(counts, counts[rank]) + expected = max(int(c.item()) for c in counts) + assert len(micros) == expected + else: + # if neither, we get the local natural count + assert len(micros) == local + + # 5) reconstruction sanity: concat→reverse_idx→orig + flat = torch.cat(micros, dim=0) + idx = [] + for sub in idx_lst: + idx.extend(sub) + inv = get_reverse_idx(idx) + inv = torch.tensor(inv, device=flat.device) + reconstructed = flat[inv] + torch.testing.assert_close(reconstructed, batch) + + dist.destroy_process_group() + + +def test_dataproto_split_uneven(): + """Test DataProto.split with uneven splits""" + # Create test data with 10 items + input_ids = torch.randint(low=0, high=10, size=(10, 5)) + attention_mask = torch.ones(10, 5) + data = {"input_ids": input_ids, "attention_mask": attention_mask} + dataproto = DataProto.from_single_dict(data) + + # Test split with size 3 (should create chunks of [3, 3, 3, 1]) + splits = dataproto.split(3) + assert len(splits) == 4 + assert len(splits[0]) == 3 + assert len(splits[1]) == 3 + assert len(splits[2]) == 3 + assert len(splits[3]) == 1 + + reconstructed = DataProto.concat(splits) + torch.testing.assert_close(reconstructed.batch["input_ids"], dataproto.batch["input_ids"]) + torch.testing.assert_close(reconstructed.batch["attention_mask"], dataproto.batch["attention_mask"]) + + # Test split with size equal to length (should create one chunk) + splits = dataproto.split(10) + assert len(splits) == 1 + assert len(splits[0]) == 10 + + # Test split with size larger than length (should create one chunk with all data) + splits = dataproto.split(15) + assert len(splits) == 1 + assert len(splits[0]) == 10 + + # Test with non-tensor batch data + import numpy as np + + data_with_non_tensor = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "labels": np.array([f"label_{i}" for i in range(10)], dtype=object), + } + dataproto_with_non_tensor = DataProto.from_single_dict(data_with_non_tensor) + + splits = dataproto_with_non_tensor.split(3) + assert len(splits) == 4 + assert len(splits[0]) == 3 + assert len(splits[1]) == 3 + assert len(splits[2]) == 3 + assert len(splits[3]) == 1 + + # Verify non-tensor data integrity + reconstructed = DataProto.concat(splits) + np.testing.assert_array_equal( + reconstructed.non_tensor_batch["labels"], dataproto_with_non_tensor.non_tensor_batch["labels"] + ) + + +def test_seqlen_balancing_distributed_params(tmp_path): + world_size = 2 + init_file = tmp_path / "dist_init" + init_file.write_text("") # empty file + init_method = f"file://{init_file}" + + # test min_num_micro_batch only + mp.spawn( + _worker, + args=(world_size, init_method, 300, False, 4), + nprocs=world_size, + join=True, + ) + + # test same_micro_num_in_dp only + mp.spawn( + _worker, + args=(world_size, init_method, 300, True, None), + nprocs=world_size, + join=True, + ) diff --git a/e2c/verl/tests/utils/test_special_linear_cross_entropy_tp.py b/e2c/verl/tests/utils/test_special_linear_cross_entropy_tp.py new file mode 100644 index 0000000000000000000000000000000000000000..9c1f868a93ea44ccb2eb4c2538b50e207e9eea64 --- /dev/null +++ b/e2c/verl/tests/utils/test_special_linear_cross_entropy_tp.py @@ -0,0 +1,514 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +import torch.distributed as dist + +try: + from verl.utils.kernel.linear_cross_entropy import linear_cross_entropy +except ImportError: + # FIXME: remove these manually included paths + import sys + + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../"))) +finally: + from verl.utils.kernel.linear_cross_entropy import linear_cross_entropy + +import verl.utils.torch_functional as verl_F + +compute_entropy_from_logits = torch.compile(verl_F.entropy_from_logits, dynamic=True) + +MAX_TEST_CASES = os.environ.get("MAX_TEST_CASES", 5) +VERIFY_TORCH_SELF = os.environ.get("VERIFY_TORCH_SELF", False) +LOW_MEMORY = os.environ.get("LOW_MEMORY", False) +LOW_MEMORY_DIV_FACTOR = os.environ.get("LOW_MEMORY_DIV_FACTOR", 16) + + +def run_torch_entropy( + hidden: torch.Tensor, weight: torch.Tensor, labels: torch.Tensor, temperature: float, reduction="none" +) -> list[torch.Tensor]: + # [num_tokens, vocab_size] + if len(hidden.shape) > 2: + hidden = hidden.view(-1, hidden.shape[-1]) # [num_tokens, hidden_size] + if len(labels.shape) > 1: + labels = labels.view(-1) + logits = torch.matmul( + hidden.to(torch.float32), + weight.to(torch.float32) if weight.size(0) == hidden.size(1) else weight.T.to(torch.float32), + ) + logits /= temperature + pd = torch.nn.functional.softmax(logits, dim=-1) # [num_tokens, vocab_size] + entropy_a = torch.logsumexp(logits, dim=-1) # [num_tokens] + entropy_b = torch.sum(pd * logits, dim=-1) # [num_tokens] + entropy = entropy_a - entropy_b + logprobs = torch.nn.functional.cross_entropy(logits, labels, reduction=reduction) # [num_tokens] + logprobs = torch.neg(logprobs) + return logprobs, entropy + + +class TorchEntropyTP(torch.autograd.Function): + """ + it is used for testing the correctness of the kernel + it is not efficient and is not recommended to use in practice + """ + + @staticmethod + def forward( + ctx, + hidden: torch.Tensor, + weight: torch.Tensor, + labels: torch.Tensor, + temperature: float, + dist_process_group: torch.distributed.ProcessGroup, + ): + # weight has shape [vocab_size, hidden_size], hidden has shape [num_tokens, hidden_size] + ctx.original_hidden_shape = hidden.shape + if len(hidden.shape) > 2: + hidden = hidden.view(-1, hidden.shape[-1]) # [num_tokens, hidden_size] + if len(labels.shape) > 1: + labels = labels.view(-1) + + logits = torch.matmul(hidden.to(torch.float32), weight.to(torch.float32).T) # [num_tokens, vocab_size] + logits /= temperature + whole_logits = torch.empty( + (logits.shape[0], logits.shape[1] * dist.get_world_size(dist_process_group)), + dtype=logits.dtype, + device=logits.device, + ) + whole_logits_ref = [ + whole_logits[:, i * logits.shape[1] : (i + 1) * logits.shape[1]] + for i in range(dist.get_world_size(dist_process_group)) + ] + dist.all_gather(whole_logits_ref, logits, group=dist_process_group) + + pd = torch.nn.functional.softmax(whole_logits, dim=-1) + entropy_a = torch.logsumexp(whole_logits, dim=-1) # [num_tokens] + entropy_b = torch.sum(pd * whole_logits, dim=-1) # [num_tokens] + entropy = entropy_a - entropy_b + + logprobs = torch.nn.functional.cross_entropy(whole_logits, labels, reduction="none") + logprobs = torch.neg(logprobs) + + ctx.save_for_backward(hidden, weight, labels, whole_logits, entropy_b) + ctx.dist_process_group = dist_process_group + ctx.temperature = temperature + return logprobs, entropy + + @staticmethod + def backward(ctx, g_logprobs: torch.Tensor, g_entropy: torch.Tensor): + hidden, weight, labels, whole_logits, entropy_b = ctx.saved_tensors + dist_process_group = ctx.dist_process_group + temperature = ctx.temperature + batch_size, hidden_size = hidden.shape + vocab_size, hidden_size = weight.shape + rank = dist.get_rank(dist_process_group) + + # Compute softmax probabilities + maximum, _ = torch.max(whole_logits, dim=-1, keepdim=True) + exp_logits = torch.exp(whole_logits - maximum) + accumulate = exp_logits.sum(dim=-1, keepdim=True) + pd = exp_logits / accumulate + + # Gradient for entropy + # entropy = entropy_a - entropy_b + # entropy_a = log(sum(exp(logits))) + # entropy_b = sum(pd * logits) + # d_entropy_a/d_logits = pd + # d_entropy_b/d_logits = pd * (logits - b.unsqueeze(1) + 1) + # d_entropy/d_logits = d_entropy_a - d_entropy_b + # d_entropy/d_logits = pd - pd * (logits - b.unsqueeze(1) + 1) + # d_entropy/d_logits = -pd * (logits - b.unsqueeze(1)) + d_logits_entropy = g_entropy.unsqueeze(1) * (-pd * (whole_logits - entropy_b.unsqueeze(1))) + + # Gradient for logprobs + # logprobs = -cross_entropy = -log(pd[labels]) + # d_logprobs/d_logits = (pd - one_hot(labels)) + one_hot = torch.zeros_like(whole_logits) + one_hot.scatter_(1, labels.unsqueeze(1), 1) + g_logprobs = torch.neg(g_logprobs) + d_logits_logprobs = g_logprobs.unsqueeze(1) * (pd - one_hot) + # NOTE: This will lead to wrong result + # d_logits_logprobs = g_logprobs.unsqueeze(1) * (pd - 1) * one_hot + + # Combine gradients + d_logits = d_logits_entropy + d_logits_logprobs + d_logits /= temperature + + # Get local slice of gradients + local_d_logits = d_logits[:, rank * vocab_size : (rank + 1) * vocab_size] + + # Compute gradients for hidden and weight + d_hidden = torch.matmul(local_d_logits, weight.to(torch.float32)) + d_weight = torch.matmul(local_d_logits.T, hidden.to(torch.float32)) + d_hidden = d_hidden.view(ctx.original_hidden_shape) + + return d_hidden, d_weight, None, None, None + + +run_torch_entropy_tp = TorchEntropyTP.apply + + +class TestLinearCrossEntropy_TensorParallel: + def __init__(self): + dist.init_process_group(backend="nccl") + self.group = dist.group.WORLD + + self.local_rank = dist.get_rank(self.group) + self.world_size = dist.get_world_size(self.group) + device = torch.device(f"cuda:{self.local_rank}") + torch.cuda.set_device(device) + print(f"[INFO]: Local rank: {self.local_rank}, World size: {self.world_size}") + + def initialize(self, test_case_idx: int, temperature: float = 1.5): + self.test_case_idx = test_case_idx + self.temperature = temperature + + def shutdown(self): + dist.destroy_process_group() + + def cleanup(self): + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats() + import gc + + gc.collect() + torch.cuda.synchronize() + + def generate_hyper(self): + global LOW_MEMORY, LOW_MEMORY_DIV_FACTOR, MAX_TEST_CASES + + self.dtype = torch.bfloat16 + if self.test_case_idx == 0: + self.batch_size = 1 + self.num_tokens = 1937 + self.hidden_size = 3584 + self.vocab_size = 152064 + elif self.test_case_idx == 1: + self.batch_size = 1 + self.num_tokens = 2169 + self.hidden_size = 896 + self.vocab_size = 151936 + elif self.test_case_idx == 2: + self.batch_size = 1 + self.num_tokens = 1530 + self.hidden_size = 2048 + self.vocab_size = 32256 + elif self.test_case_idx == 3: + self.batch_size = 1 + self.num_tokens = 1388 + self.hidden_size = 4096 + self.vocab_size = 102400 + elif self.test_case_idx == 4: + self.batch_size = 1 + self.num_tokens = 8192 + self.hidden_size = 4096 + self.vocab_size = 102400 + else: + raise ValueError(f"Invalid test case index: {self.test_case_idx}") + if LOW_MEMORY: + self.vocab_size = int(self.vocab_size / LOW_MEMORY_DIV_FACTOR) + assert MAX_TEST_CASES <= 5, "MAX_TEST_CASES should be less than or equal to 5." + + def generate_forward_inputs(self): + hidden = ( + torch.empty((self.batch_size, self.num_tokens, self.hidden_size), dtype=self.dtype, device="cuda") + .uniform_(-0.5, 0.5) + .requires_grad_() + ) + weight = ( + torch.empty((self.vocab_size, self.hidden_size), dtype=self.dtype, device="cuda") + .uniform_(-0.5, 0.5) + .requires_grad_() + ) + labels = torch.randint(0, self.vocab_size, (self.batch_size, self.num_tokens), device="cuda") + return hidden, weight, labels + + def generate_backward_inputs(self): + g_entropy = torch.empty((self.num_tokens,), dtype=self.dtype, device="cuda").uniform_(-0.5, 0.5) + g_logprobs = torch.empty((self.num_tokens,), dtype=self.dtype, device="cuda").uniform_(-1, 1) + return g_entropy, g_logprobs + + def verify_torch_itself(self, iterations: int = 5): + self.cleanup() + self.generate_hyper() + + for i in range(iterations): + hidden, weight, labels = self.generate_forward_inputs() + + # NOTE: we need to manually synchronize hidden and labels among Process Group + dist.broadcast(hidden, src=0, group=self.group) + dist.broadcast(labels, src=0, group=self.group) + + # forward pass + # Create a tensor to hold the gathered weights from all ranks + # weight has shape [vocab_size, hidden_size] + # We want to gather along the first dimension to get [vocab_size * world_size, hidden_size] + + # Create a single contiguous tensor to hold all gathered weights + whole_weight = torch.empty( + (self.vocab_size * self.world_size, self.hidden_size), dtype=weight.dtype, device=weight.device + ) + + # Create views into the tensor for each rank's portion + whole_weight_views = [ + whole_weight[i * self.vocab_size : (i + 1) * self.vocab_size] for i in range(self.world_size) + ] + + # Perform all_gather operation using the views + dist.all_gather(whole_weight_views, weight, group=self.group) + + # Set requires_grad for autograd + whole_weight.requires_grad_() + + (single_logprobs, single_entropy) = run_torch_entropy(hidden, whole_weight, labels, self.temperature) + + (tp_logprobs, tp_entropy) = run_torch_entropy_tp(hidden, weight, labels, self.temperature, self.group) + + torch.testing.assert_close(single_logprobs, tp_logprobs, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(single_entropy, tp_entropy, atol=1e-4, rtol=1e-4) + + # backward pass + g_entropy, g_logprobs = self.generate_backward_inputs() + # NOTE: we need to manually synchronize g_entropy and g_logprobs among Process Group + dist.broadcast(g_entropy, src=0, group=self.group) + dist.broadcast(g_logprobs, src=0, group=self.group) + + (single_d_hidden, single_d_weight) = torch.autograd.grad( + (single_entropy, single_logprobs), (hidden, whole_weight), (g_entropy, g_logprobs), retain_graph=False + ) + + (tp_d_hidden, tp_d_weight) = torch.autograd.grad( + (tp_entropy, tp_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + # NOTE: all-reduce on hidden is conducted outside the kernel + dist.all_reduce(tp_d_hidden, op=dist.ReduceOp.SUM, group=self.group) + + torch.testing.assert_close(tp_d_hidden, single_d_hidden, atol=1e-2, rtol=1e-4) + # Extract the corresponding slice from single_d_weight for comparison + # tp_d_weight has shape [vocab_size, hidden_size] + # single_d_weight has shape [vocab_size * world_size, hidden_size] + torch.testing.assert_close( + tp_d_weight, + single_d_weight[self.local_rank * self.vocab_size : (self.local_rank + 1) * self.vocab_size], + atol=1e-2, + rtol=1e-4, + ) + + # atol=1e-3, rtol=1e-4) + if self.local_rank == 0: + print("[PASS] torch TP correctness is verified") + + def check_torch_storage(self): + self.cleanup() + self.generate_hyper() + + hidden, weight, labels = self.generate_forward_inputs() + + # NOTE: we need to manually synchronize hidden and labels among Process Group + dist.broadcast(hidden, src=0, group=self.group) + dist.broadcast(labels, src=0, group=self.group) + + torch.cuda.reset_peak_memory_stats() + (tp_logprobs, tp_entropy) = run_torch_entropy_tp(hidden, weight, labels, self.temperature, self.group) + torch.cuda.synchronize() + forward_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024 + + g_entropy, g_logprobs = self.generate_backward_inputs() + # NOTE: we need to manually synchronize g_entropy and g_logprobs among Process Group + dist.broadcast(g_entropy, src=0, group=self.group) + dist.broadcast(g_logprobs, src=0, group=self.group) + + torch.cuda.reset_peak_memory_stats() + (d_tp_hidden, d_tp_weight) = torch.autograd.grad( + (tp_entropy, tp_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + torch.cuda.synchronize() + backward_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024 + # NOTE: all-reduce on hidden is conducted outside the kernel + dist.all_reduce(d_tp_hidden, op=dist.ReduceOp.SUM, group=self.group) + + if self.local_rank == 0: + print(f"[INFO]: Torch Forward pass peak memory: {forward_max_memory:.2f} MB") + print(f"[INFO]: Torch Backward pass peak memory: {backward_max_memory:.2f} MB") + + def verify_kernel_correctness(self, iterations: int = 5): + self.cleanup() + self.generate_hyper() + + torch_forward_latency = list() + torch_backward_latency = list() + kernel_forward_latency = list() + kernel_backward_latency = list() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + for i in range(iterations): + hidden, weight, labels = self.generate_forward_inputs() + + # NOTE: we need to manually synchronize hidden and labels among Process Group + dist.broadcast(hidden, src=0, group=self.group) + dist.broadcast(labels, src=0, group=self.group) + + start_event.record() + (torch_logprobs, torch_entropy) = run_torch_entropy_tp(hidden, weight, labels, self.temperature, self.group) + end_event.record() + torch.cuda.synchronize() + torch_forward_latency.append(start_event.elapsed_time(end_event)) + + start_event.record() + (kernel_logprobs, kernel_entropy) = linear_cross_entropy( + hidden, weight, labels, self.temperature, "none", self.group + ) + end_event.record() + torch.cuda.synchronize() + kernel_forward_latency.append(start_event.elapsed_time(end_event)) + + torch.testing.assert_close(torch_logprobs, kernel_logprobs, atol=1e-1, rtol=1e-2) + torch.testing.assert_close(torch_entropy, kernel_entropy, atol=1e-1, rtol=1e-2) + + # backward pass + g_entropy, g_logprobs = self.generate_backward_inputs() + # NOTE: we need to manually synchronize g_entropy and g_logprobs among Process Group + dist.broadcast(g_entropy, src=0, group=self.group) + dist.broadcast(g_logprobs, src=0, group=self.group) + + start_event.record() + (torch_d_hidden, torch_d_weight) = torch.autograd.grad( + (torch_entropy, torch_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + end_event.record() + torch.cuda.synchronize() + torch_backward_latency.append(start_event.elapsed_time(end_event)) + # NOTE: all-reduce on hidden is conducted outside the kernel + dist.all_reduce(torch_d_hidden, op=dist.ReduceOp.SUM, group=self.group) + + start_event.record() + (kernel_d_hidden, kernel_d_weight) = torch.autograd.grad( + (kernel_entropy, kernel_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + end_event.record() + torch.cuda.synchronize() + kernel_backward_latency.append(start_event.elapsed_time(end_event)) + # NOTE: all-reduce on hidden is conducted outside the kernel + dist.all_reduce(kernel_d_hidden, op=dist.ReduceOp.SUM, group=self.group) + + torch.testing.assert_close(torch_d_hidden, kernel_d_hidden, atol=2e-2, rtol=4e-2) + torch.testing.assert_close(torch_d_weight, kernel_d_weight, atol=2e-2, rtol=4e-2) + + # remove first latency + torch_forward_latency = torch_forward_latency[1:] + torch_backward_latency = torch_backward_latency[1:] + kernel_forward_latency = kernel_forward_latency[1:] + kernel_backward_latency = kernel_backward_latency[1:] + + if self.local_rank == 0: + print("\n[PASS]: Verified kernel forward & backward correctness.") + + print( + f"[INFO]: Forward pass: Torch implementation average time: " + f"{sum(torch_forward_latency) / len(torch_forward_latency):.2f} ms" + ) + print( + f"[INFO]: Backward pass: torch implementation average time: " + f"{sum(torch_backward_latency) / len(torch_backward_latency):.2f} ms" + ) + print( + f"[INFO]: Forward pass: Kernel implementation average time: " + f"{sum(kernel_forward_latency) / len(kernel_forward_latency):.2f} ms" + ) + print( + f"[INFO]: Backward pass: kernel implementation average time: " + f"{sum(kernel_backward_latency) / len(kernel_backward_latency):.2f} ms" + ) + + def check_kernel_storage(self): + self.cleanup() + self.generate_hyper() + + hidden, weight, labels = self.generate_forward_inputs() + + # NOTE: we need to manually synchronize hidden and labels among Process Group + dist.broadcast(hidden, src=0, group=self.group) + dist.broadcast(labels, src=0, group=self.group) + + torch.cuda.reset_peak_memory_stats() + (kernel_logprobs, kernel_entropy) = linear_cross_entropy( + hidden, weight, labels, self.temperature, "none", self.group + ) + torch.cuda.synchronize() + kernel_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024 + + g_entropy, g_logprobs = self.generate_backward_inputs() + # NOTE: we need to manually synchronize g_entropy and g_logprobs among Process Group + dist.broadcast(g_entropy, src=0, group=self.group) + dist.broadcast(g_logprobs, src=0, group=self.group) + + torch.cuda.reset_peak_memory_stats() + (d_kernel_hidden, d_kernel_weight) = torch.autograd.grad( + (kernel_entropy, kernel_logprobs), (hidden, weight), (g_entropy, g_logprobs), retain_graph=False + ) + torch.cuda.synchronize() + kernel_backward_max_memory = torch.cuda.max_memory_allocated() / 1024 / 1024 + # NOTE: all-reduce on hidden is conducted outside the kernel + dist.all_reduce(d_kernel_hidden, op=dist.ReduceOp.SUM, group=self.group) + + if self.local_rank == 0: + print(f"[INFO]: Kernel Forward pass peak memory: {kernel_max_memory:.2f} MB") + print(f"[INFO]: Kernel Backward pass peak memory: {kernel_backward_max_memory:.2f} MB") + + +if __name__ == "__main__": + # TP command: torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/kernels/test_linear_cross_entropy_tp.py + + # Check if running with torchrun (distributed mode) + assert int(os.environ["WORLD_SIZE"]) > 1, ( + "[ERROR]: This test is designed to run in distributed mode with torchrun. Please use torchrun to " + "execute this script." + ) + torch.manual_seed(233376 + int(os.environ.get("RANK", 0))) + + # set_backward_method(BackwardEnum._Total_Fuse_MN) + # set_backward_method(BackwardEnum._Split_Dlogits_N) + + test = TestLinearCrossEntropy_TensorParallel() + for test_case_idx in range(MAX_TEST_CASES): + print(f"[INFO] Running test case {test_case_idx}") + test.initialize(test_case_idx) + if VERIFY_TORCH_SELF: + test.verify_torch_itself() + test.check_torch_storage() + test.verify_kernel_correctness() + test.check_kernel_storage() + + test.shutdown() diff --git a/e2c/verl/tests/utils/test_special_mstx_profile.py b/e2c/verl/tests/utils/test_special_mstx_profile.py new file mode 100644 index 0000000000000000000000000000000000000000..a80cabfa49c6a40428eed077a32839ff113ccec8 --- /dev/null +++ b/e2c/verl/tests/utils/test_special_mstx_profile.py @@ -0,0 +1,273 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest.mock import MagicMock, patch + +from verl.utils.profiler.config import NPUToolConfig, ProfilerConfig +from verl.utils.profiler.mstx_profile import NPUProfiler + + +class TestNPUProfilerInitialization(unittest.TestCase): + def setUp(self): + NPUProfiler._define_count = 0 + + def test_init_with_default_config(self): + tool_config = NPUToolConfig() + profiler = NPUProfiler(rank=0, config=None, tool_config=tool_config) + self.assertFalse(profiler.enable) + self.assertFalse(hasattr(profiler, "profile_npu")) + + def test_init_with_disabled_config(self): + config = ProfilerConfig(enable=False) + tool_config = NPUToolConfig() + profiler = NPUProfiler(rank=0, config=config, tool_config=tool_config) + self.assertFalse(profiler.enable) + self.assertFalse(hasattr(profiler, "profile_npu")) + + def test_init_with_all_ranks_true(self): + config = ProfilerConfig(enable=True, all_ranks=True) + tool_config = NPUToolConfig() + profiler = NPUProfiler(rank=0, config=config, tool_config=tool_config) + self.assertTrue(profiler.this_rank) + + def test_init_with_ranks_list(self): + config = ProfilerConfig(enable=True, ranks=[1, 2]) + tool_config = NPUToolConfig() + profiler = NPUProfiler(rank=1, config=config, tool_config=tool_config) + self.assertTrue(profiler.this_rank) + + def test_init_with_rank_not_in_ranks(self): + config = ProfilerConfig(enable=True, ranks=[1, 2]) + tool_config = NPUToolConfig() + profiler = NPUProfiler(rank=3, config=config, tool_config=tool_config) + self.assertFalse(profiler.this_rank) + + +class TestNPUProfilerStart(unittest.TestCase): + def setUp(self): + NPUProfiler._define_count = 0 + self.config = ProfilerConfig(enable=True, ranks=[0]) + self.tool_config = NPUToolConfig(discrete=False) + + @patch("verl.utils.profiler.mstx_profile.get_npu_profiler") + def test_start_when_enabled_and_this_rank(self, mock_get_profiler): + profiler = NPUProfiler(rank=0, config=self.config, tool_config=self.tool_config) + profiler.start(role="worker", profile_step="1") + self.assertTrue(profiler.this_step) + self.assertEqual(NPUProfiler._define_count, 1) + mock_get_profiler.assert_called_once() + + @patch("verl.utils.profiler.mstx_profile.get_npu_profiler") + def test_start_when_not_this_rank(self, mock_get_profiler): + profiler = NPUProfiler(rank=1, config=self.config, tool_config=self.tool_config) + profiler.start() + self.assertFalse(profiler.this_step) + self.assertEqual(NPUProfiler._define_count, 0) + mock_get_profiler.assert_not_called() + + @patch("verl.utils.profiler.mstx_profile.get_npu_profiler") + def test_start_discrete_mode_does_not_increase_count(self, mock_get_profiler): + tool_config = NPUToolConfig(discrete=True) + profiler = NPUProfiler(rank=0, config=self.config, tool_config=tool_config) + profiler.start() + self.assertEqual(NPUProfiler._define_count, 0) + mock_get_profiler.assert_not_called() + + @patch("verl.utils.profiler.mstx_profile.get_npu_profiler") + def test_multiple_start_calls_do_not_increase_count(self, mock_get_profiler): + profiler = NPUProfiler(rank=0, config=self.config, tool_config=self.tool_config) + profiler.start() + profiler.start() + self.assertEqual(NPUProfiler._define_count, 1) + mock_get_profiler.assert_called_once() + + +class TestNPUProfilerStartStopInteraction(unittest.TestCase): + def setUp(self): + NPUProfiler._define_count = 0 + self.config = ProfilerConfig(enable=True, ranks=[0]) + self.tool_config = NPUToolConfig(discrete=False) + + @patch("verl.utils.profiler.mstx_profile.get_npu_profiler") + def test_start_stop_cycle(self, mock_get_profiler): + mock_profile_npu = MagicMock() + mock_get_profiler.return_value = mock_profile_npu + + profiler = NPUProfiler(rank=0, config=self.config, tool_config=self.tool_config) + profiler.start() + self.assertEqual(NPUProfiler._define_count, 1) + self.assertEqual(mock_profile_npu.start.call_count, 1) + profiler.stop() + self.assertEqual(NPUProfiler._define_count, 0) + self.assertEqual(mock_profile_npu.step.call_count, 1) + self.assertEqual(mock_profile_npu.stop.call_count, 1) + + @patch("verl.utils.profiler.mstx_profile.get_npu_profiler") + def test_multiple_instances_share_define_count(self, mock_get_profiler): + mock_profile_npu = MagicMock() + mock_get_profiler.return_value = mock_profile_npu + + profiler1 = NPUProfiler(rank=0, config=self.config, tool_config=self.tool_config) + profiler2 = NPUProfiler(rank=0, config=self.config, tool_config=self.tool_config) + profiler1.start() + profiler2.start() + self.assertEqual(NPUProfiler._define_count, 1) + self.assertEqual(mock_profile_npu.start.call_count, 1) + profiler1.stop() + self.assertEqual(NPUProfiler._define_count, 0) + + +class TestNPUProfilerAnnotate(unittest.TestCase): + def setUp(self): + self.config = ProfilerConfig(enable=True, all_ranks=True) + self.tool_config = NPUToolConfig(discrete=False) + self.rank = 0 + + def test_annotate_decorator_applied_correctly(self): + mock_worker = MagicMock() + mock_worker.profiler = NPUProfiler(rank=self.rank, config=self.config, tool_config=self.tool_config) + mock_worker.profiler.this_step = True + + mock_mark_range = "mocked_range_handle" + + with ( + patch("verl.utils.profiler.mstx_profile.mark_start_range") as mock_start_patch, + patch("verl.utils.profiler.mstx_profile.mark_end_range") as mock_end_patch, + ): + mock_start_patch.return_value = mock_mark_range + + with patch("verl.utils.profiler.mstx_profile.get_npu_profiler") as mock_get_profiler: + decorator = mock_worker.profiler.annotate(message="test") + + @decorator + def test_func(self, *args, **kwargs): + return "result" + + result = test_func(mock_worker) + + self.assertEqual(result, "result") + mock_start_patch.assert_called_once_with(message="test") + mock_end_patch.assert_called_once_with(mock_mark_range) + mock_get_profiler.assert_not_called() + + def test_annotate_when_profiler_disabled(self): + disabled_config = ProfilerConfig(enable=False) + mock_worker = MagicMock() + mock_worker.profiler = NPUProfiler(rank=self.rank, config=disabled_config, tool_config=self.tool_config) + + with ( + patch("verl.utils.profiler.mstx_profile.mark_start_range") as mock_start_patch, + patch("verl.utils.profiler.mstx_profile.mark_end_range") as mock_end_patch, + patch("verl.utils.profiler.mstx_profile.get_npu_profiler") as mock_get_profiler, + ): + decorator = mock_worker.profiler.annotate(message="test") + + @decorator + def test_func(self, *args, **kwargs): + return "result" + + result = test_func(mock_worker) + + self.assertEqual(result, "result") + mock_start_patch.assert_not_called() + mock_end_patch.assert_not_called() + mock_get_profiler.assert_not_called() + + def test_annotate_when_this_step_disabled(self): + mock_worker = MagicMock() + mock_worker.profiler = NPUProfiler(rank=self.rank, config=self.config, tool_config=self.tool_config) + mock_worker.profiler.this_step = False + + with ( + patch("verl.utils.profiler.mstx_profile.mark_start_range") as mock_start_patch, + patch("verl.utils.profiler.mstx_profile.mark_end_range") as mock_end_patch, + patch("verl.utils.profiler.mstx_profile.get_npu_profiler") as mock_get_profiler, + ): + decorator = mock_worker.profiler.annotate(message="test") + + @decorator + def test_func(self, *args, **kwargs): + return "result" + + result = test_func(mock_worker) + + self.assertEqual(result, "result") + mock_start_patch.assert_not_called() + mock_end_patch.assert_not_called() + mock_get_profiler.assert_not_called() + + def test_annotate_discrete_mode_enabled(self): + discrete_tool_config = NPUToolConfig(discrete=True) + mock_worker = MagicMock() + mock_worker.profiler = NPUProfiler(rank=self.rank, config=self.config, tool_config=discrete_tool_config) + mock_worker.profiler.this_step = True + + mock_mark_range = "mocked_range_handle" + mock_profile_npu = MagicMock() + + with ( + patch("verl.utils.profiler.mstx_profile.mark_start_range") as mock_start_patch, + patch("verl.utils.profiler.mstx_profile.mark_end_range") as mock_end_patch, + patch("verl.utils.profiler.mstx_profile.get_npu_profiler") as mock_get_profiler, + ): + mock_start_patch.return_value = mock_mark_range + mock_get_profiler.return_value = mock_profile_npu + decorator = mock_worker.profiler.annotate(message="test", role="test_role") + + @decorator + def test_func(self, *args, **kwargs): + return "result" + + result = test_func(mock_worker) + + self.assertEqual(result, "result") + mock_start_patch.assert_called_once_with(message="test") + mock_end_patch.assert_called_once_with(mock_mark_range) + mock_get_profiler.assert_called_once_with( + contents=mock_worker.profiler.profile_contents, + profile_level=mock_worker.profiler.profile_level, + profile_save_path=mock_worker.profiler.profile_save_path, + analysis=mock_worker.profiler.analysis, + role="test_role", + ) + mock_profile_npu.start.assert_called_once() + mock_profile_npu.step.assert_called_once() + mock_profile_npu.stop.assert_called_once() + + def test_annotate_with_default_message(self): + mock_worker = MagicMock() + mock_worker.profiler = NPUProfiler(rank=self.rank, config=self.config, tool_config=self.tool_config) + mock_worker.profiler.this_step = True + + mock_mark_range = "mocked_range_handle" + with ( + patch("verl.utils.profiler.mstx_profile.mark_start_range") as mock_start_patch, + patch("verl.utils.profiler.mstx_profile.mark_end_range") as mock_end_patch, + ): + mock_start_patch.return_value = mock_mark_range + decorator = mock_worker.profiler.annotate() + + @decorator + def test_func(self, *args, **kwargs): + return "result" + + test_func(mock_worker) + + mock_start_patch.assert_called_once_with(message="test_func") + mock_end_patch.assert_called_once_with(mock_mark_range) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/utils/test_temp_env_on_cpu.py b/e2c/verl/tests/utils/test_temp_env_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..851e4cbe43263c2c16ed4b5db73706aa1ef325c3 --- /dev/null +++ b/e2c/verl/tests/utils/test_temp_env_on_cpu.py @@ -0,0 +1,143 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from verl.utils.py_functional import temp_env_var + + +@pytest.fixture(autouse=True) +def clean_env(): + """Fixture to clean up environment variables before and after each test.""" + # Store original environment state + original_env = dict(os.environ) + + # Clean up any test variables that might exist + test_vars = ["TEST_VAR", "TEST_VAR_2", "EXISTING_VAR"] + for var in test_vars: + if var in os.environ: + del os.environ[var] + + # Yield control to the test function + yield + + # Restore original environment state after test + os.environ.clear() + os.environ.update(original_env) + + +def test_set_new_env_var(): + """Test setting a new environment variable that didn't exist before.""" + # Ensure variable doesn't exist + assert "TEST_VAR" not in os.environ + + with temp_env_var("TEST_VAR", "test_value"): + # Variable should be set inside context + assert os.environ["TEST_VAR"] == "test_value" + assert "TEST_VAR" in os.environ + + # Variable should be removed after context + assert "TEST_VAR" not in os.environ + + +def test_restore_existing_env_var(): + """Test restoring an environment variable that already existed.""" + # Set up existing variable + os.environ["EXISTING_VAR"] = "original_value" + + with temp_env_var("EXISTING_VAR", "temporary_value"): + # Variable should be temporarily changed + assert os.environ["EXISTING_VAR"] == "temporary_value" + + # Variable should be restored to original value + assert os.environ["EXISTING_VAR"] == "original_value" + + +def test_env_var_restored_on_exception(): + """Test that environment variables are restored even when exceptions occur.""" + # Set up existing variable + os.environ["EXISTING_VAR"] = "original_value" + + with pytest.raises(ValueError): + with temp_env_var("EXISTING_VAR", "temporary_value"): + # Verify variable is set + assert os.environ["EXISTING_VAR"] == "temporary_value" + # Raise exception + raise ValueError("Test exception") + + # Variable should still be restored despite exception + assert os.environ["EXISTING_VAR"] == "original_value" + + +def test_nested_context_managers(): + """Test nested temp_env_var context managers.""" + # Set up original variable + os.environ["TEST_VAR"] = "original" + + with temp_env_var("TEST_VAR", "level1"): + assert os.environ["TEST_VAR"] == "level1" + + with temp_env_var("TEST_VAR", "level2"): + assert os.environ["TEST_VAR"] == "level2" + + # Should restore to level1 + assert os.environ["TEST_VAR"] == "level1" + + # Should restore to original + assert os.environ["TEST_VAR"] == "original" + + +def test_multiple_different_vars(): + """Test setting multiple different environment variables.""" + # Set up one existing variable + os.environ["EXISTING_VAR"] = "existing_value" + + with temp_env_var("EXISTING_VAR", "modified"): + with temp_env_var("TEST_VAR", "new_value"): + assert os.environ["EXISTING_VAR"] == "modified" + assert os.environ["TEST_VAR"] == "new_value" + + # Check restoration + assert os.environ["EXISTING_VAR"] == "existing_value" + assert "TEST_VAR" not in os.environ + + +def test_empty_string_value(): + """Test setting environment variable to empty string.""" + with temp_env_var("TEST_VAR", ""): + assert os.environ["TEST_VAR"] == "" + assert "TEST_VAR" in os.environ + + # Should be removed after context + assert "TEST_VAR" not in os.environ + + +def test_overwrite_with_empty_string(): + """Test overwriting existing variable with empty string.""" + os.environ["EXISTING_VAR"] = "original" + + with temp_env_var("EXISTING_VAR", ""): + assert os.environ["EXISTING_VAR"] == "" + + # Should restore original value + assert os.environ["EXISTING_VAR"] == "original" + + +def test_context_manager_returns_none(): + """Test that context manager yields None.""" + with temp_env_var("TEST_VAR", "value") as result: + assert result is None + assert os.environ["TEST_VAR"] == "value" diff --git a/e2c/verl/tests/utils/test_timeout_decorator_cpu.py b/e2c/verl/tests/utils/test_timeout_decorator_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..3417469db22a12f355f3b20e8c97a73ad84de4a8 --- /dev/null +++ b/e2c/verl/tests/utils/test_timeout_decorator_cpu.py @@ -0,0 +1,238 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing +import sys +import threading +import time + +import pytest # Import pytest + +from verl.utils.py_functional import timeout_limit as timeout + +# --- Test Task Functions --- +TEST_TIMEOUT_SECONDS = 1.5 # Timeout duration for tests +LONG_TASK_DURATION = TEST_TIMEOUT_SECONDS + 0.5 # Duration slightly longer than timeout + + +@timeout(seconds=TEST_TIMEOUT_SECONDS) # Keep global decorator for mp tests +def quick_task(x): + """A task that completes quickly.""" + time.sleep(0.1) + return "quick_ok" + + +@timeout(seconds=TEST_TIMEOUT_SECONDS) # Keep global decorator for mp tests +def slow_task(x): + """A task that takes longer than the timeout.""" + time.sleep(LONG_TASK_DURATION) + return "slow_finished" # This return value indicates it didn't time out + + +# REMOVE global decorator here +def task_raises_value_error(): # Now truly not globally decorated + """A task that intentionally raises a ValueError.""" + raise ValueError("Specific value error from task") + + +# --- Top-level function for signal test in subprocess --- +# Keep this decorated globally for the specific subprocess test case +@timeout(seconds=TEST_TIMEOUT_SECONDS, use_signals=True) +def top_level_decorated_quick_task_signal(): + """A pickleable top-level function decorated with signal timeout.""" + # Assuming this calls the logic of quick_task directly for the test purpose + time.sleep(0.1) + return "quick_ok_signal_subprocess" # Different return for clarity if needed + + +# --- Top-level function for signal test in subprocess --- +# Keep this decorated globally for the specific subprocess test case +@timeout(seconds=TEST_TIMEOUT_SECONDS, use_signals=True) +def top_level_decorated_slow_task_signal(): + """A pickleable top-level function decorated with signal timeout.""" + time.sleep(LONG_TASK_DURATION) + return "slow_finished" + + +# --- NEW: Top-level helper function to run target in process --- +def run_target_and_put_in_queue(target_func, q): + """ + Top-level helper function to run a target function and put its result or exception into a queue. + This function is pickleable and can be used as the target for multiprocessing.Process. + """ + try: + result = target_func() + q.put(("success", result)) + except Exception as e: + q.put(("error", e)) + + +# Use a module-level fixture to set the start method on macOS +@pytest.fixture(scope="module", autouse=True) # Changed scope to module +def set_macos_start_method(): + if sys.platform == "darwin": + # Force fork method on macOS to avoid pickling issues with globally decorated functions + # when running tests via pytest discovery. + current_method = multiprocessing.get_start_method(allow_none=True) + # Only set if not already set or if set to something else (less likely in test run) + if current_method is None or current_method != "fork": + try: + multiprocessing.set_start_method("fork", force=True) + except RuntimeError: + # Might fail if context is already started, ignore in that case. + pass + + +def test_quick_task(): # Renamed from test_multiprocessing_quick_task + """Tests timeout handles a quick task correctly.""" + # Call the globally decorated function directly + result = quick_task(1) + assert result == "quick_ok" # Use pytest assert + + +def test_slow_task_timeout(): # Renamed from test_multiprocessing_slow_task_timeout + """Tests timeout correctly raises TimeoutError for a slow task.""" + # Call the globally decorated function directly within pytest.raises + with pytest.raises(TimeoutError) as excinfo: # Use pytest.raises + slow_task(1) + # Check the error message from the multiprocessing implementation + assert f"timed out after {TEST_TIMEOUT_SECONDS} seconds" in str(excinfo.value) # Use pytest assert + + +def test_internal_exception(): # Renamed from test_multiprocessing_internal_exception + """Tests timeout correctly propagates internal exceptions.""" + # Apply the default timeout decorator dynamically to the undecorated function + decorated_task = timeout(seconds=TEST_TIMEOUT_SECONDS)(task_raises_value_error) # Apply decorator dynamically + with pytest.raises(ValueError) as excinfo: # Use pytest.raises + decorated_task() # Call the dynamically decorated function + assert str(excinfo.value) == "Specific value error from task" # Use pytest assert + + +# --- Test the signal implementation (use_signals=True) --- +# Note: As per py_functional.py, use_signals=True currently falls back to +# multiprocessing on POSIX. These tests verify that behavior. + + +def test_signal_quick_task_main_process(): # Removed self + """Tests signal timeout handles a quick task correctly in the main process.""" + + # Apply the signal decorator dynamically + def plain_quick_task_logic(): + time.sleep(0.1) + return "quick_ok_signal" + + decorated_task = timeout(seconds=TEST_TIMEOUT_SECONDS, use_signals=True)(plain_quick_task_logic) + assert decorated_task() == "quick_ok_signal" # Use pytest assert + + +def test_signal_slow_task_main_process_timeout(): # Removed self + """Tests signal timeout correctly raises TimeoutError for a slow task in the main process.""" + + # Apply the signal decorator dynamically + def plain_slow_task_logic(): + time.sleep(LONG_TASK_DURATION) + return "slow_finished_signal" + + decorated_task = timeout(seconds=TEST_TIMEOUT_SECONDS, use_signals=True)(plain_slow_task_logic) + with pytest.raises(TimeoutError) as excinfo: # Use pytest.raises + decorated_task() + # Check the error message (falls back to multiprocessing message on POSIX) + assert f"timed out after {TEST_TIMEOUT_SECONDS} seconds" in str(excinfo.value) # Use pytest assert + + +@pytest.mark.skip(reason="this test won't pass. Just to show why use_signals should not be used") +def test_signal_in_thread_does_not_timeout(): + """ + Tests that signal-based timeout does NOT work reliably in a child thread. + The TimeoutError from the signal handler is not expected to be raised. + """ + result_container = [] # Use a list to store result from thread + exception_container = [] # Use a list to store exception from thread + + @timeout(seconds=TEST_TIMEOUT_SECONDS, use_signals=True) + def slow_task_in_thread(): + try: + print("Thread: Starting slow task...") + time.sleep(LONG_TASK_DURATION) + print("Thread: Slow task finished.") + return "slow_finished_in_thread" + except Exception as e: + # Catch any exception within the thread's target function + print(f"Thread: Caught exception: {e}") + exception_container.append(e) + return None # Indicate failure + + def thread_target(): + try: + # Run the decorated function inside the thread + res = slow_task_in_thread() + if res is not None: + result_container.append(res) + except Exception as e: + # This might catch exceptions happening *outside* the decorated function + # but still within the thread target, though less likely here. + print(f"Thread Target: Caught exception: {e}") + exception_container.append(e) + + thread = threading.Thread(target=thread_target) + print("Main: Starting thread...") + thread.start() + # Wait longer than the timeout + task duration to ensure the thread finishes + # regardless of whether timeout worked or not. + thread.join(timeout=LONG_TASK_DURATION + 1) + + assert len(exception_container) == 1 + assert isinstance(exception_container[0], TimeoutError) + assert not result_container + + +def test_in_thread_timeout(): + result_container = [] # Use a list to store result from thread + exception_container = [] # Use a list to store exception from thread + + @timeout(seconds=TEST_TIMEOUT_SECONDS, use_signals=False) + def slow_task_in_thread(): + try: + print("Thread: Starting slow task...") + time.sleep(LONG_TASK_DURATION) + print("Thread: Slow task finished.") + return "slow_finished_in_thread" + except Exception as e: + # Catch any exception within the thread's target function + print(f"Thread: Caught exception: {e}") + exception_container.append(e) + return None # Indicate failure + + def thread_target(): + try: + # Run the decorated function inside the thread + res = slow_task_in_thread() + if res is not None: + result_container.append(res) + except Exception as e: + # This might catch exceptions happening *outside* the decorated function + # but still within the thread target, though less likely here. + print(f"Thread Target: Caught exception: {e}") + exception_container.append(e) + + thread = threading.Thread(target=thread_target) + print("Main: Starting thread...") + thread.start() + # Wait longer than the timeout + task duration to ensure the thread finishes + # regardless of whether timeout worked or not. + thread.join(timeout=LONG_TASK_DURATION + 1) + + assert len(exception_container) == 1 + assert isinstance(exception_container[0], TimeoutError) + assert not result_container diff --git a/e2c/verl/tests/utils/test_torch_functional.py b/e2c/verl/tests/utils/test_torch_functional.py new file mode 100644 index 0000000000000000000000000000000000000000..900cb5d542fae2aef462d8b0efed88f12b4e63a0 --- /dev/null +++ b/e2c/verl/tests/utils/test_torch_functional.py @@ -0,0 +1,117 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from verl.utils.torch_functional import distributed_masked_mean, distributed_mean_max_min_std, masked_mean + + +def _worker_mean(rank: int, world_size: int, rendezvous_file: str): + # 1) set GPU and init NCCL + torch.cuda.set_device(rank) + dist.init_process_group( + backend="nccl", + init_method=f"file://{rendezvous_file}", + rank=rank, + world_size=world_size, + ) + + # each rank holds tensor [rank+1] + local = torch.tensor([float(rank + 1)], device=f"cuda:{rank}") + mean, gmax, gmin, gstd = distributed_mean_max_min_std(local, True, True, True) + + values = [float(i + 1) for i in range(world_size)] + exp_mean = sum(values) / len(values) + exp_max = max(values) + exp_min = min(values) + var = sum((x - exp_mean) ** 2 for x in values) / (len(values) - 1) + exp_std = var**0.5 + + # all ranks should see the same result + assert torch.allclose(mean.cpu(), torch.tensor(exp_mean)), f"mean@{rank}" + assert torch.allclose(gmax.cpu(), torch.tensor(exp_max)), f"max@{rank}" + assert torch.allclose(gmin.cpu(), torch.tensor(exp_min)), f"min@{rank}" + assert torch.allclose(gstd.cpu(), torch.tensor(exp_std)), f"std@{rank}" + + dist.destroy_process_group() + + +@pytest.mark.parametrize( + "value,mask,gt", + [ + ([1.0, 2.0, 3.0, 4.0], [1, 0, 0, 1], 2.5), + ([1.0, 2.0, float("nan"), 4.0], [1, 0, 0, 1], 2.5), + ([1.0, 2.0, float("nan"), 4.0], [1, 0, 1, 0], float("nan")), + ], +) +def test_masked_mean(value, mask, gt): + res = masked_mean(torch.tensor(value), torch.tensor(mask)) + gt = torch.tensor(gt) + assert torch.allclose(res, gt) or (torch.isnan(res) and torch.isnan(gt)) + + +@pytest.mark.parametrize("world_size", [2, 4]) +def test_distributed_mean_max_min_std(world_size, tmp_path): + rendezvous_file = str(tmp_path / "rdzv_mean") + os.makedirs(os.path.dirname(rendezvous_file), exist_ok=True) + + mp.spawn( + fn=_worker_mean, + args=(world_size, rendezvous_file), + nprocs=world_size, + join=True, + ) + + +def _worker_mask(rank: int, world_size: int, rendezvous_file: str): + torch.cuda.set_device(rank) + dist.init_process_group( + backend="nccl", + init_method=f"file://{rendezvous_file}", + rank=rank, + world_size=world_size, + ) + + # build per‐rank tensor and mask + local_tensor = torch.tensor([rank * 2 + 1.0, rank * 2 + 2.0], device=f"cuda:{rank}") + if rank == 0: + mask = torch.tensor([1, 0], device=f"cuda:{rank}", dtype=torch.float32) + else: + mask = torch.tensor([0, 1], device=f"cuda:{rank}", dtype=torch.float32) + + gmean = distributed_masked_mean(local_tensor, mask) + + valid_values = [1.0] + [2 * i + 2.0 for i in range(1, world_size)] + expected_mean = sum(valid_values) / len(valid_values) + assert torch.allclose(gmean.cpu(), torch.tensor(expected_mean)), f"masked_mean@{rank}" + + dist.destroy_process_group() + + +@pytest.mark.parametrize("world_size", [2, 4]) +def test_distributed_masked_mean(world_size, tmp_path): + rendezvous_file = str(tmp_path / "rdzv_mask") + os.makedirs(os.path.dirname(rendezvous_file), exist_ok=True) + + mp.spawn( + fn=_worker_mask, + args=(world_size, rendezvous_file), + nprocs=world_size, + join=True, + ) diff --git a/e2c/verl/tests/workers/actor/test_special_dp_actor.py b/e2c/verl/tests/workers/actor/test_special_dp_actor.py new file mode 100644 index 0000000000000000000000000000000000000000..33ed9cd679f3ef5cc47dc9b2a8ef78550f829e0c --- /dev/null +++ b/e2c/verl/tests/workers/actor/test_special_dp_actor.py @@ -0,0 +1,289 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import torch +import torch.nn as nn +from tensordict import TensorDict +from transformers import AutoModelForCausalLM, Qwen3Config + +from verl import DataProto +from verl.workers.actor.dp_actor import DataParallelPPOActor +from verl.workers.config import FSDPActorConfig, OptimizerConfig + + +class MockTransformerModel(nn.Module): + """Mock transformer model for testing DataParallelPPOActor""" + + def __init__(self, vocab_size=1000, hidden_size=64): + super().__init__() + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.embedding = nn.Embedding(vocab_size, hidden_size) + self.transformer = nn.TransformerEncoder( + nn.TransformerEncoderLayer(d_model=hidden_size, nhead=4, batch_first=True), num_layers=2 + ) + self.lm_head = nn.Linear(hidden_size, vocab_size) + + def forward(self, input_ids, attention_mask=None, position_ids=None, use_cache=False, **kwargs): + batch_size, seq_len = input_ids.shape + + embeddings = self.embedding(input_ids) + hidden_states = self.transformer(embeddings) + logits = self.lm_head(hidden_states) + + class MockOutput: + def __init__(self, logits): + self.logits = logits + + return MockOutput(logits) + + +class TestDataParallelPPOActor(unittest.TestCase): + """Test DataParallelPPOActor compute_log_prob and update_policy methods""" + + @classmethod + def setUpClass(cls): + """Set up distributed environment""" + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group( + backend="nccl" if torch.cuda.is_available() else "gloo", init_method="env://" + ) + + cls.rank = torch.distributed.get_rank() + cls.world_size = torch.distributed.get_world_size() + + if torch.cuda.is_available(): + torch.cuda.set_device(cls.rank) + cls.device = torch.device(f"cuda:{cls.rank}") + else: + cls.device = torch.device("cpu") + + def setUp(self): + """Set up test fixtures""" + self.config = FSDPActorConfig( + strategy="fsdp2", + ppo_mini_batch_size=4, + ppo_micro_batch_size_per_gpu=2, + ppo_epochs=1, + clip_ratio=0.2, + entropy_coeff=0.01, + grad_clip=1.0, + use_dynamic_bsz=False, + use_torch_compile=False, # Disable torch.compile for testing + ulysses_sequence_parallel_size=1, + optim=OptimizerConfig(lr=1e-6), + ) + + self.mock_model = MockTransformerModel(vocab_size=1000, hidden_size=64).to(self.device) + self.mock_optimizer = torch.optim.Adam(self.mock_model.parameters(), lr=1e-4) + + self.actor = DataParallelPPOActor( + config=self.config, actor_module=self.mock_model, actor_optimizer=self.mock_optimizer + ) + + @classmethod + def tearDownClass(cls): + """Clean up distributed environment""" + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() + + def _create_test_data_for_compute_log_prob(self): + """Create test DataProto for compute_log_prob method""" + batch_size = 2 + prompt_length = 8 + response_length = 4 + total_length = prompt_length + response_length + vocab_size = 1000 + + input_ids = torch.randint(0, vocab_size, (batch_size, total_length)).to(self.device) + attention_mask = torch.ones(batch_size, total_length).to(self.device) + position_ids = torch.arange(total_length).unsqueeze(0).expand(batch_size, -1).to(self.device) + responses = input_ids[:, -response_length:] # Last part is the response + + tensor_dict = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "responses": responses, + }, + batch_size=[batch_size], + ) + + meta_info = {"micro_batch_size": batch_size, "temperature": 1.0, "use_dynamic_bsz": False} + + return DataProto(batch=tensor_dict, meta_info=meta_info) + + def _create_test_data_for_update_policy(self): + """Create test DataProto for update_policy method""" + batch_size = 4 # Must match ppo_mini_batch_size + prompt_length = 8 + response_length = 4 + total_length = prompt_length + response_length + vocab_size = 1000 + + input_ids = torch.randint(0, vocab_size, (batch_size, total_length)).to(self.device) + attention_mask = torch.ones(batch_size, total_length).to(self.device) + position_ids = torch.arange(total_length).unsqueeze(0).expand(batch_size, -1).to(self.device) + responses = input_ids[:, -response_length:] + response_mask = torch.ones(batch_size, response_length).to(self.device) + old_log_probs = torch.randn(batch_size, response_length).to(self.device) * 0.1 # Small values + advantages = torch.randn(batch_size, response_length).to(self.device) * 0.5 + + tensor_dict = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "responses": responses, + "response_mask": response_mask, + "old_log_probs": old_log_probs, + "advantages": advantages, + }, + batch_size=[batch_size], + ) + + meta_info = {"temperature": 1.0} + + return DataProto(batch=tensor_dict, meta_info=meta_info) + + def test_compute_log_prob(self): + """Test compute_log_prob method""" + data = self._create_test_data_for_compute_log_prob() + + log_probs, entropies = self.actor.compute_log_prob(data, calculate_entropy=True) + + batch_size = data.batch["responses"].shape[0] + response_length = data.batch["responses"].shape[1] + + self.assertIsInstance(log_probs, torch.Tensor) + self.assertEqual(log_probs.shape, (batch_size, response_length)) + self.assertTrue(torch.all(torch.isfinite(log_probs))) + + self.assertIsInstance(entropies, torch.Tensor) + self.assertEqual(entropies.shape, (batch_size, response_length)) + self.assertTrue(torch.all(torch.isfinite(entropies))) + self.assertTrue(torch.all(entropies >= 0)) # Entropy should be non-negative + + def test_compute_log_prob_without_entropy(self): + """Test compute_log_prob method without entropy calculation""" + data = self._create_test_data_for_compute_log_prob() + + log_probs, entropies = self.actor.compute_log_prob(data, calculate_entropy=False) + + batch_size = data.batch["responses"].shape[0] + response_length = data.batch["responses"].shape[1] + + self.assertIsInstance(log_probs, torch.Tensor) + self.assertEqual(log_probs.shape, (batch_size, response_length)) + self.assertTrue(torch.all(torch.isfinite(log_probs))) + + self.assertIsNone(entropies) + + def test_update_policy(self): + """Test update_policy method""" + data = self._create_test_data_for_update_policy() + + metrics = self.actor.update_policy(data) + + self.assertIsInstance(metrics, dict) + + expected_metric_keys = [ + "actor/pg_loss", + "actor/pg_clipfrac", + "actor/ppo_kl", + "actor/pg_clipfrac_lower", + "actor/grad_norm", + ] + + for key in expected_metric_keys: + self.assertIn(key, metrics) + if isinstance(metrics[key], list): + self.assertTrue(all(torch.isfinite(torch.tensor(v)) for v in metrics[key])) + else: + self.assertIsInstance(metrics[key], (float, int)) + self.assertTrue(torch.isfinite(torch.tensor(metrics[key]))) + + def test_dataparallelppoactor_initialization(self): + """Test DataParallelPPOActor initialization""" + self.assertIsNotNone(self.actor.actor_module) + self.assertIsNotNone(self.actor.actor_optimizer) + self.assertEqual(self.actor.config, self.config) + + self.assertEqual(self.actor.config.strategy, "fsdp2") + self.assertEqual(self.actor.config.ppo_mini_batch_size, 4) + self.assertEqual(self.actor.config.clip_ratio, 0.2) + + def test_dataparallelppoactor_with_qwen3_model(self): + """Test DataParallelPPOActor with real Qwen3ForCausalLM model""" + qwen_config = Qwen3Config( + vocab_size=1000, + hidden_size=64, + intermediate_size=128, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + max_position_embeddings=512, + torch_dtype=torch.float32, + use_cache=False, + ) + + with torch.device(self.device): + qwen_model = AutoModelForCausalLM.from_config(config=qwen_config, torch_dtype=torch.float32).to(self.device) + + qwen_optimizer = torch.optim.Adam(qwen_model.parameters(), lr=1e-4) + + qwen_actor = DataParallelPPOActor(config=self.config, actor_module=qwen_model, actor_optimizer=qwen_optimizer) + + data = self._create_test_data_for_compute_log_prob() + log_probs, entropies = qwen_actor.compute_log_prob(data, calculate_entropy=True) + + batch_size = data.batch["responses"].shape[0] + response_length = data.batch["responses"].shape[1] + + self.assertIsInstance(log_probs, torch.Tensor) + self.assertEqual(log_probs.shape, (batch_size, response_length)) + self.assertTrue(torch.all(torch.isfinite(log_probs))) + + self.assertIsInstance(entropies, torch.Tensor) + self.assertEqual(entropies.shape, (batch_size, response_length)) + self.assertTrue(torch.all(torch.isfinite(entropies))) + self.assertTrue(torch.all(entropies >= 0)) + + policy_data = self._create_test_data_for_update_policy() + metrics = qwen_actor.update_policy(policy_data) + + self.assertIsInstance(metrics, dict) + + expected_metric_keys = [ + "actor/pg_loss", + "actor/pg_clipfrac", + "actor/ppo_kl", + "actor/pg_clipfrac_lower", + "actor/grad_norm", + ] + + for key in expected_metric_keys: + self.assertIn(key, metrics) + if isinstance(metrics[key], list): + self.assertTrue(all(torch.isfinite(torch.tensor(v)) for v in metrics[key])) + else: + self.assertIsInstance(metrics[key], (float, int)) + self.assertTrue(torch.isfinite(torch.tensor(metrics[key]))) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/workers/config/test_actor_config_on_cpu.py b/e2c/verl/tests/workers/config/test_actor_config_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..61468e1f7a21160afa6ede205401109a5ae0ce32 --- /dev/null +++ b/e2c/verl/tests/workers/config/test_actor_config_on_cpu.py @@ -0,0 +1,240 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest + +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import ActorConfig, FSDPActorConfig, McoreActorConfig, OptimizerConfig + + +class TestActorConfig(unittest.TestCase): + """Test the ActorConfig dataclass and its variants.""" + + def test_config_inheritance(self): + """Test that the inheritance hierarchy works correctly.""" + megatron_dict = { + "_target_": "verl.workers.config.McoreActorConfig", + "strategy": "megatron", + "ppo_mini_batch_size": 256, + "ppo_micro_batch_size_per_gpu": 256, + "clip_ratio": 0.2, + "optim": { + "_target_": "verl.workers.config.OptimizerConfig", + "lr": 0.1, + }, + } + fsdp_dict = { + "_target_": "verl.workers.config.FSDPActorConfig", + "strategy": "fsdp", + "ppo_mini_batch_size": 256, + "ppo_micro_batch_size_per_gpu": 256, + "clip_ratio": 0.2, + "optim": { + "_target_": "verl.workers.config.OptimizerConfig", + "lr": 0.1, + }, + } + + megatron_config = omega_conf_to_dataclass(megatron_dict) + fsdp_config = omega_conf_to_dataclass(fsdp_dict) + + self.assertIsInstance(megatron_config, ActorConfig) + self.assertIsInstance(fsdp_config, ActorConfig) + + self.assertEqual(megatron_config.ppo_mini_batch_size, fsdp_config.ppo_mini_batch_size) + self.assertEqual(megatron_config.clip_ratio, fsdp_config.clip_ratio) + + def test_actor_config_from_yaml(self): + """Test creating ActorConfig from YAML file.""" + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config/actor")): + cfg = compose(config_name="actor", overrides=["strategy=fsdp", "ppo_micro_batch_size_per_gpu=128"]) + + config = omega_conf_to_dataclass(cfg) + + self.assertIsInstance(config, ActorConfig) + self.assertEqual(config.strategy, "fsdp") + + def test_fsdp_actor_config_from_yaml(self): + """Test creating FSDPActorConfig from YAML file.""" + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config/actor")): + cfg = compose(config_name="dp_actor", overrides=["strategy=fsdp2", "ppo_micro_batch_size_per_gpu=128"]) + + config = omega_conf_to_dataclass(cfg) + + self.assertIsInstance(config, FSDPActorConfig) + self.assertEqual(config.strategy, "fsdp2") + + def test_megatron_actor_config_from_yaml(self): + """Test creating McoreActorConfig from YAML file.""" + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config/actor")): + cfg = compose(config_name="megatron_actor", overrides=["ppo_micro_batch_size_per_gpu=128"]) + + config = omega_conf_to_dataclass(cfg) + + self.assertIsInstance(config, McoreActorConfig) + self.assertEqual(config.strategy, "megatron") + + def test_config_get_method(self): + """Test the get method for backward compatibility.""" + config_dict = { + "_target_": "verl.workers.config.ActorConfig", + "strategy": "fsdp", + "ppo_mini_batch_size": 256, + "ppo_micro_batch_size_per_gpu": 256, + "optim": { + "_target_": "verl.workers.config.OptimizerConfig", + "lr": 0.1, + }, + } + config = omega_conf_to_dataclass(config_dict) + + self.assertEqual(config.get("strategy"), "fsdp") + self.assertEqual(config.get("ppo_mini_batch_size"), 256) + + self.assertIsNone(config.get("non_existing")) + self.assertEqual(config.get("non_existing", "default"), "default") + + def test_config_dict_like_access(self): + """Test dictionary-like access to config fields.""" + config_dict = { + "_target_": "verl.workers.config.ActorConfig", + "strategy": "fsdp", + "ppo_mini_batch_size": 256, + "ppo_micro_batch_size_per_gpu": 256, + "optim": { + "_target_": "verl.workers.config.OptimizerConfig", + "lr": 0.1, + }, + } + config = omega_conf_to_dataclass(config_dict) + + self.assertEqual(config["strategy"], "fsdp") + self.assertEqual(config["ppo_mini_batch_size"], 256) + + field_names = list(config) + self.assertIn("strategy", field_names) + self.assertIn("ppo_mini_batch_size", field_names) + + self.assertGreater(len(config), 0) + + def test_frozen_fields_modification_raises_exception(self): + """Test that modifying frozen fields raises an exception.""" + config_dict = { + "_target_": "verl.workers.config.ActorConfig", + "strategy": "fsdp", + "ppo_mini_batch_size": 256, + "ppo_micro_batch_size_per_gpu": 256, + "optim": { + "_target_": "verl.workers.config.OptimizerConfig", + "lr": 0.1, + }, + } + config = omega_conf_to_dataclass(config_dict) + + with self.assertRaises(AttributeError): + config.strategy = "megatron" + + with self.assertRaises(AttributeError): + config.clip_ratio = 0.5 + + config.ppo_mini_batch_size = 512 # This should work since it's not in frozen fields anymore + self.assertEqual(config.ppo_mini_batch_size, 512) + + def test_actor_config_validation_exceptions(self): + """Test that ActorConfig.__post_init__ raises appropriate validation exceptions.""" + optim = OptimizerConfig(lr=0.1) + with self.assertRaises((ValueError, AssertionError)) as cm: + ActorConfig( + strategy="fsdp", + loss_agg_mode="invalid-mode", + use_dynamic_bsz=True, + optim=optim, + ppo_micro_batch_size_per_gpu=4, + ) + self.assertIn("Invalid loss_agg_mode", str(cm.exception)) + + with self.assertRaises((ValueError, AssertionError)) as cm: + ActorConfig( + strategy="fsdp", + use_dynamic_bsz=False, + ppo_micro_batch_size=4, + ppo_micro_batch_size_per_gpu=2, + optim=optim, + ) + self.assertIn("You have set both", str(cm.exception)) + + with self.assertRaises((ValueError, AssertionError)) as cm: + ActorConfig( + strategy="fsdp", + use_dynamic_bsz=False, + ppo_micro_batch_size=None, + ppo_micro_batch_size_per_gpu=None, + optim=optim, + ) + self.assertIn("Please set at least one", str(cm.exception)) + + config = ActorConfig( + strategy="fsdp", + use_dynamic_bsz=True, + ppo_micro_batch_size=None, + ppo_micro_batch_size_per_gpu=None, + optim=optim, + ) + self.assertIsNotNone(config) # Should not raise an exception + + def test_fsdp_actor_config_validation_exceptions(self): + """Test that FSDPActorConfig.validate() raises appropriate validation exceptions.""" + optim = OptimizerConfig(lr=0.1) + config = FSDPActorConfig( + strategy="fsdp", + ulysses_sequence_parallel_size=2, + use_dynamic_bsz=True, # Skip batch size validation to focus on FSDP validation + optim=optim, + ) + + model_config = {"use_remove_padding": False} + with self.assertRaises(ValueError) as cm: + config.validate(n_gpus=8, train_batch_size=256, model_config=model_config) + self.assertIn("you must enable `use_remove_padding`", str(cm.exception)) + + def test_actor_config_validate_method_exceptions(self): + """Test that ActorConfig.validate() raises appropriate validation exceptions.""" + optim = OptimizerConfig(lr=0.1) + config = ActorConfig( + strategy="fsdp", + use_dynamic_bsz=False, + ppo_mini_batch_size=256, + ppo_micro_batch_size=8, + ppo_micro_batch_size_per_gpu=None, # Ensure only one batch size setting is used + optim=optim, + ) + + with self.assertRaises(ValueError) as cm: + config.validate(n_gpus=8, train_batch_size=128) + self.assertIn("train_batch_size", str(cm.exception)) + + with self.assertRaises(ValueError) as cm: + config.validate(n_gpus=16, train_batch_size=512) + self.assertIn("must be >= n_gpus", str(cm.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/workers/config/test_critic_config_on_cpu.py b/e2c/verl/tests/workers/config/test_critic_config_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..025038f44ea4d03930e73a35401b3577fd3a52a7 --- /dev/null +++ b/e2c/verl/tests/workers/config/test_critic_config_on_cpu.py @@ -0,0 +1,305 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path + +import pytest +from hydra import compose, initialize_config_dir + +from verl.utils.config import omega_conf_to_dataclass +from verl.utils.profiler import ProfilerConfig +from verl.workers.config import ( + CriticConfig, + FSDPCriticConfig, + McoreCriticConfig, + OptimizerConfig, +) + + +class TestCriticConfig: + """Test suite for critic configuration dataclasses.""" + + @pytest.fixture + def config_dir(self): + """Get the path to the config directory.""" + return Path(__file__).parent.parent.parent.parent / "verl" / "trainer" / "config" / "critic" + + def test_megatron_critic_config_instantiation_from_yaml(self, config_dir): + """Test that McoreCriticConfig can be instantiated from megatron_critic.yaml.""" + yaml_path = config_dir / "megatron_critic.yaml" + assert yaml_path.exists(), f"Config file not found: {yaml_path}" + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config/critic")): + test_config = compose(config_name="megatron_critic", overrides=["ppo_micro_batch_size_per_gpu=1"]) + + megatron_config_obj = omega_conf_to_dataclass(test_config) + + assert isinstance(megatron_config_obj, McoreCriticConfig) + assert isinstance(megatron_config_obj, CriticConfig) + + expected_attrs = [ + "strategy", + "rollout_n", + "optim", + "model", + "ppo_mini_batch_size", + "ppo_max_token_len_per_gpu", + "cliprange_value", + "get", + "nccl_timeout", + "megatron", + "load_weight", + ] + for attr in expected_attrs: + assert hasattr(megatron_config_obj, attr), f"Missing attribute: {attr}" + + assert callable(megatron_config_obj.get) + assert megatron_config_obj.strategy == "megatron" + + def test_fsdp_critic_config_instantiation_from_yaml(self, config_dir): + """Test that FSDPCriticConfig can be instantiated from dp_critic.yaml.""" + yaml_path = config_dir / "dp_critic.yaml" + assert yaml_path.exists(), f"Config file not found: {yaml_path}" + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config/critic")): + test_config = compose(config_name="dp_critic", overrides=["ppo_micro_batch_size_per_gpu=1"]) + + fsdp_config_obj = omega_conf_to_dataclass(test_config) + + assert isinstance(fsdp_config_obj, FSDPCriticConfig) + assert isinstance(fsdp_config_obj, CriticConfig) + + expected_attrs = [ + "strategy", + "rollout_n", + "optim", + "model", + "ppo_mini_batch_size", + "ppo_max_token_len_per_gpu", + "cliprange_value", + "get", + "forward_micro_batch_size", + "forward_micro_batch_size_per_gpu", + "ulysses_sequence_parallel_size", + "grad_clip", + ] + for attr in expected_attrs: + assert hasattr(fsdp_config_obj, attr), f"Missing attribute: {attr}" + + assert callable(fsdp_config_obj.get) + assert fsdp_config_obj.strategy == "fsdp" + + def test_config_inheritance_hierarchy(self): + """Test that the inheritance hierarchy is correct.""" + optim = OptimizerConfig(lr=0.1) + megatron_config = McoreCriticConfig(ppo_micro_batch_size_per_gpu=1, optim=optim) + assert isinstance(megatron_config, CriticConfig) + assert isinstance(megatron_config, McoreCriticConfig) + + fsdp_config = FSDPCriticConfig(ppo_micro_batch_size_per_gpu=1, optim=optim) + assert isinstance(fsdp_config, CriticConfig) + assert isinstance(fsdp_config, FSDPCriticConfig) + + critic_config = CriticConfig(ppo_micro_batch_size_per_gpu=1, strategy="fsdp2", optim=optim) + assert isinstance(critic_config, CriticConfig) + assert not isinstance(critic_config, McoreCriticConfig) + assert not isinstance(critic_config, FSDPCriticConfig) + + def test_config_dict_interface(self): + """Test that configs provide dict-like interface from BaseConfig.""" + optim = OptimizerConfig(lr=0.1) + config = CriticConfig(ppo_micro_batch_size_per_gpu=1, strategy="fsdp2", optim=optim) + + assert "strategy" in config + assert config["strategy"] == "fsdp2" + + assert config.get("strategy") == "fsdp2" + assert config.get("nonexistent_key", "default") == "default" + + keys = list(config) + assert "strategy" in keys + assert "rollout_n" in keys + + assert len(config) > 0 + + def test_frozen_fields_immutability(self): + """Test that frozen fields raise exceptions when modified after creation.""" + optim = OptimizerConfig(lr=0.1) + critic_config = CriticConfig(ppo_micro_batch_size_per_gpu=1, strategy="fsdp2", optim=optim) + frozen_fields = ["rollout_n", "strategy", "cliprange_value"] + + for field_name in frozen_fields: + with pytest.raises((AttributeError, TypeError, ValueError)): + setattr(critic_config, field_name, "modified_value") + + megatron_config = McoreCriticConfig(ppo_micro_batch_size_per_gpu=1, optim=optim) + megatron_frozen_fields = ["nccl_timeout", "load_weight", "data_loader_seed"] + + for field_name in megatron_frozen_fields: + with pytest.raises((AttributeError, TypeError, ValueError)): + setattr(megatron_config, field_name, "modified_value") + + fsdp_config = FSDPCriticConfig(ppo_micro_batch_size_per_gpu=1, optim=optim) + fsdp_frozen_fields = ["ulysses_sequence_parallel_size", "grad_clip"] + + for field_name in fsdp_frozen_fields: + with pytest.raises((AttributeError, TypeError, ValueError)): + setattr(fsdp_config, field_name, "modified_value") + + def test_batch_size_fields_modifiable(self): + """Test that batch size fields can be modified after creation.""" + optim = OptimizerConfig(lr=0.1) + critic_config = CriticConfig(ppo_micro_batch_size_per_gpu=1, strategy="fsdp2", optim=optim) + + critic_config.ppo_mini_batch_size = 8 + critic_config.ppo_micro_batch_size = 4 + critic_config.ppo_micro_batch_size_per_gpu = 2 + + assert critic_config.ppo_mini_batch_size == 8 + assert critic_config.ppo_micro_batch_size == 4 + assert critic_config.ppo_micro_batch_size_per_gpu == 2 + + fsdp_config = FSDPCriticConfig(ppo_micro_batch_size_per_gpu=1, optim=optim) + + fsdp_config.forward_micro_batch_size = 16 + fsdp_config.forward_micro_batch_size_per_gpu = 8 + + assert fsdp_config.forward_micro_batch_size == 16 + assert fsdp_config.forward_micro_batch_size_per_gpu == 8 + + def test_profiler_config_type_validation(self): + """Test that profiler field has correct type and validation.""" + optim = OptimizerConfig(lr=0.1) + critic_config = CriticConfig(ppo_micro_batch_size_per_gpu=1, strategy="fsdp2", optim=optim) + assert isinstance(critic_config.profiler, ProfilerConfig) + assert critic_config.profiler.all_ranks is False + assert critic_config.profiler.ranks == [] + + custom_profiler = ProfilerConfig(all_ranks=True, ranks=[0, 1]) + critic_config_custom = CriticConfig( + profiler=custom_profiler, ppo_micro_batch_size_per_gpu=1, strategy="fsdp2", optim=optim + ) + assert isinstance(critic_config_custom.profiler, ProfilerConfig) + assert critic_config_custom.profiler.all_ranks is True + assert critic_config_custom.profiler.ranks == [0, 1] + + profiler1 = ProfilerConfig(enable=True, ranks=[0, 1]) + profiler2 = ProfilerConfig(all_ranks=True, ranks=[1, 2]) + + union_result = profiler1.union(profiler2) + assert union_result.enable is True + assert union_result.all_ranks is True + assert set(union_result.ranks) == {0, 1, 2} + + intersect_result = profiler1.intersect(profiler2) + assert intersect_result.all_ranks is False + assert intersect_result.ranks == [1] + + def test_critic_config_validation_logic(self): + """Test the __post_init__ validation logic for CriticConfig.""" + optim = OptimizerConfig(lr=0.1) + valid_config = CriticConfig( + strategy="fsdp2", ppo_micro_batch_size_per_gpu=2, use_dynamic_bsz=False, optim=optim + ) + assert valid_config.ppo_micro_batch_size_per_gpu == 2 + + valid_config2 = CriticConfig( + strategy="fsdp2", + ppo_micro_batch_size_per_gpu=None, + ppo_micro_batch_size=4, + ppo_mini_batch_size=8, + use_dynamic_bsz=False, + optim=optim, + ) + assert valid_config2.ppo_micro_batch_size == 4 + + dynamic_config = CriticConfig( + strategy="fsdp2", ppo_micro_batch_size_per_gpu=2, use_dynamic_bsz=True, optim=optim + ) + assert dynamic_config.use_dynamic_bsz is True + + with pytest.raises(ValueError, match="You have set both.*micro_batch_size.*AND.*micro_batch_size_per_gpu"): + CriticConfig( + strategy="fsdp2", + ppo_micro_batch_size=4, + ppo_micro_batch_size_per_gpu=2, + use_dynamic_bsz=False, + optim=optim, + ) + + with pytest.raises( + ValueError, match="Please set at least one of.*micro_batch_size.*or.*micro_batch_size_per_gpu" + ): + CriticConfig( + strategy="fsdp2", + ppo_micro_batch_size=None, + ppo_micro_batch_size_per_gpu=None, + use_dynamic_bsz=False, + optim=optim, + ) + + def test_micro_batch_size_divisibility_validation(self): + """Test micro batch size divisibility validation in __post_init__.""" + optim = OptimizerConfig(lr=0.1) + valid_config = CriticConfig( + strategy="fsdp2", ppo_micro_batch_size_per_gpu=2, ppo_mini_batch_size=8, use_dynamic_bsz=False, optim=optim + ) + assert valid_config.ppo_mini_batch_size == 8 + assert valid_config.ppo_micro_batch_size_per_gpu == 2 + + valid_config_with_mbs = CriticConfig( + strategy="fsdp2", ppo_mini_batch_size=8, ppo_micro_batch_size=4, use_dynamic_bsz=False, optim=optim + ) + assert valid_config_with_mbs.ppo_mini_batch_size == 8 + assert valid_config_with_mbs.ppo_micro_batch_size == 4 + + with pytest.raises(ValueError, match="ppo_mini_batch_size.*must be divisible by.*ppo_micro_batch_size"): + CriticConfig( + strategy="fsdp2", ppo_mini_batch_size=7, ppo_micro_batch_size=4, use_dynamic_bsz=False, optim=optim + ) + + dynamic_config = CriticConfig( + strategy="fsdp2", ppo_mini_batch_size=7, ppo_micro_batch_size=4, use_dynamic_bsz=True, optim=optim + ) + assert dynamic_config.use_dynamic_bsz is True + + def test_fsdp_sequence_parallelism_validation(self): + """Test FSDP sequence parallelism validation in FSDPCriticConfig.__post_init__.""" + optim = OptimizerConfig(lr=0.1) + valid_config = FSDPCriticConfig( + ppo_micro_batch_size_per_gpu=2, + ulysses_sequence_parallel_size=2, + model={"use_remove_padding": True}, + optim=optim, + ) + assert valid_config.ulysses_sequence_parallel_size == 2 + + with pytest.raises( + ValueError, match="When using sequence parallelism for critic, you must enable.*use_remove_padding" + ): + FSDPCriticConfig( + ppo_micro_batch_size_per_gpu=2, + ulysses_sequence_parallel_size=2, + model={"use_remove_padding": False}, + optim=optim, + ) + + valid_config_no_sp = FSDPCriticConfig( + ppo_micro_batch_size_per_gpu=2, + ulysses_sequence_parallel_size=1, + model={"use_remove_padding": False}, + optim=optim, + ) + assert valid_config_no_sp.ulysses_sequence_parallel_size == 1 diff --git a/e2c/verl/tests/workers/config/test_engine_config_on_cpu.py b/e2c/verl/tests/workers/config/test_engine_config_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..1253f5c9ab9943df3c187a3c8458b35f78fe6994 --- /dev/null +++ b/e2c/verl/tests/workers/config/test_engine_config_on_cpu.py @@ -0,0 +1,67 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from verl.workers.config.engine import FSDPEngineConfig, McoreEngineConfig + + +class TestMcoreEngineConfig: + def test_default_values(self): + config = McoreEngineConfig() + assert config.tensor_model_parallel_size == 1 + assert config.sequence_parallel is False # Should be auto-corrected + assert config.seed == 42 + + def test_post_init_validation(self): + # Test TP size 1 forces sequence_parallel=False + config = McoreEngineConfig(tensor_model_parallel_size=1) + assert config.sequence_parallel is False + + # Test TP >1 keeps sequence_parallel=True + config = McoreEngineConfig(tensor_model_parallel_size=2) + assert config.sequence_parallel is True + + def test_mutable_fields(self): + config = McoreEngineConfig() + config.sequence_parallel = True # Should be mutable + with pytest.raises(AttributeError): + config.tensor_model_parallel_size = 2 # Frozen field + + @pytest.mark.parametrize("offload_field", ["param_offload", "grad_offload", "optimizer_offload"]) + def test_offload_flags(self, offload_field): + config = McoreEngineConfig(**{offload_field: True}) + assert getattr(config, offload_field) is True + + +class TestFSDPEngineConfigCPU: + def test_default_values(self): + config = FSDPEngineConfig() + assert config.param_offload is False + assert config.optimizer_offload is False + assert config.fsdp_size == -1 + + @pytest.mark.parametrize( + "offload_params", + [{"param_offload": True}, {"optimizer_offload": True}, {"param_offload": True, "optimizer_offload": True}], + ) + def test_offload_combinations(self, offload_params): + config = FSDPEngineConfig(**offload_params) + assert config.param_offload == offload_params.get("param_offload", False) + assert config.optimizer_offload == offload_params.get("optimizer_offload", False) + + def test_wrap_policy_configuration(self): + test_policy = {"layer_class": "TransformerBlock"} + config = FSDPEngineConfig(wrap_policy=test_policy) + assert config.wrap_policy == test_policy diff --git a/e2c/verl/tests/workers/config/test_optim_config_on_cpu.py b/e2c/verl/tests/workers/config/test_optim_config_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..4e9d9f86b56d95844c6303e129706f4f7f87f4fb --- /dev/null +++ b/e2c/verl/tests/workers/config/test_optim_config_on_cpu.py @@ -0,0 +1,39 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from verl.workers.config.optimizer import FSDPOptimizerConfig + + +class TestFSDPOptimizerConfigCPU: + def test_default_configuration(self): + config = FSDPOptimizerConfig(lr=0.1) + assert config.min_lr_ratio is None + assert config.warmup_style == "constant" + assert config.num_cycles == 0.5 + + @pytest.mark.parametrize("warmup_style", ["constant", "cosine"]) + def test_valid_warmup_styles(self, warmup_style): + config = FSDPOptimizerConfig(warmup_style=warmup_style, lr=0.1) + assert config.warmup_style == warmup_style + + def test_invalid_warmup_style(self): + with pytest.raises((ValueError, AssertionError)): + FSDPOptimizerConfig(warmup_style="invalid_style", lr=0.1) + + @pytest.mark.parametrize("num_cycles", [0.1, 1.0, 2.5]) + def test_num_cycles_configuration(self, num_cycles): + config = FSDPOptimizerConfig(num_cycles=num_cycles, lr=0.1) + assert config.num_cycles == num_cycles diff --git a/e2c/verl/tests/workers/critic/test_special_dp_critic.py b/e2c/verl/tests/workers/critic/test_special_dp_critic.py new file mode 100644 index 0000000000000000000000000000000000000000..3f471a978abe64ac1fcad7c821f5a7c1a84e1c1f --- /dev/null +++ b/e2c/verl/tests/workers/critic/test_special_dp_critic.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest + +import torch +import torch.distributed +from tensordict import TensorDict +from transformers import AutoConfig + +from verl import DataProto +from verl.workers.config import FSDPCriticConfig, OptimizerConfig +from verl.workers.config.critic import FSDPCriticModelCfg +from verl.workers.config.engine import FSDPEngineConfig +from verl.workers.fsdp_workers import CriticWorker + + +class TestCriticWorker(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Set up distributed environment""" + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group( + backend="nccl" if torch.cuda.is_available() else "gloo", init_method="env://" + ) + + cls.rank = torch.distributed.get_rank() + cls.world_size = torch.distributed.get_world_size() + + if torch.cuda.is_available(): + torch.cuda.set_device(cls.rank) + cls.device = torch.device(f"cuda:{cls.rank}") + else: + cls.device = torch.device("cpu") + + @classmethod + def tearDownClass(cls): + """Clean up distributed environment""" + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() + + def setUp(self): + """Set up test fixtures""" + + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.temp_dir = tempfile.mkdtemp() + + config = AutoConfig.from_pretrained("Qwen/Qwen2.5-0.5B-Instruct") + config.save_pretrained(self.temp_dir) + + self.config = FSDPCriticConfig( + strategy="fsdp2", + ppo_mini_batch_size=4, + ppo_micro_batch_size_per_gpu=2, + forward_micro_batch_size_per_gpu=2, + ppo_epochs=1, + cliprange_value=0.5, + grad_clip=1.0, + use_dynamic_bsz=False, + ulysses_sequence_parallel_size=1, + rollout_n=1, + optim=OptimizerConfig(lr=1e-6), + model=FSDPCriticModelCfg( + path="Qwen/Qwen2.5-0.5B-Instruct", + tokenizer_path="Qwen/Qwen2.5-0.5B-Instruct", + fsdp_config=FSDPEngineConfig(fsdp_size=-1), + use_remove_padding=False, + ), + ) + assert self.world_size <= 4 // 2 + + def tearDown(self): + """Clean up test fixtures""" + import shutil + + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _create_test_data_for_compute_values(self, batch_size=2, seq_len=10, response_len=5): + """Create test data for compute_values method""" + input_ids = torch.randint(0, 1000, (batch_size, seq_len), dtype=torch.long) + attention_mask = torch.ones(batch_size, seq_len, dtype=torch.long) + position_ids = torch.arange(seq_len).unsqueeze(0).expand(batch_size, -1) + responses = torch.randint(0, 1000, (batch_size, response_len), dtype=torch.long) + response_mask = torch.ones(batch_size, response_len, dtype=torch.float) + + batch = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "responses": responses, + "response_mask": response_mask, + }, + batch_size=[batch_size], + ) + + data = DataProto( + batch=batch, meta_info={"micro_batch_size": 2, "max_token_len": seq_len, "use_dynamic_bsz": False} + ) + + return data + + def _create_test_data_for_update_critic(self, batch_size=2, seq_len=10, response_len=5): + """Create test data for update_critic method""" + input_ids = torch.randint(0, 1000, (batch_size, seq_len), dtype=torch.long) + attention_mask = torch.ones(batch_size, seq_len, dtype=torch.long) + position_ids = torch.arange(seq_len).unsqueeze(0).expand(batch_size, -1) + responses = torch.randint(0, 1000, (batch_size, response_len), dtype=torch.long) + response_mask = torch.ones(batch_size, response_len, dtype=torch.float) + values = torch.randn(batch_size, response_len, dtype=torch.float) + returns = torch.randn(batch_size, response_len, dtype=torch.float) + + batch = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "responses": responses, + "response_mask": response_mask, + "values": values, + "returns": returns, + }, + batch_size=[batch_size], + ) + + data = DataProto( + batch=batch, + meta_info={"global_token_num": [response_len] * batch_size, "batch_seqlens": [response_len] * batch_size}, + ) + + return data + + def test_init_model(self): + """Test CriticWorker.init_model() method""" + worker = CriticWorker(self.config) + worker.init_model() + + self.assertIsNotNone(worker.critic_module) + self.assertIsNotNone(worker.critic_optimizer) + self.assertIsNotNone(worker.critic) + self.assertIsNotNone(worker.checkpoint_manager) + + def test_compute_values(self): + """Test CriticWorker.compute_values() method""" + worker = CriticWorker(self.config) + worker.init_model() + + data = self._create_test_data_for_compute_values() + + result = worker.compute_values(data) + + self.assertIsInstance(result, DataProto) + self.assertIn("values", result.batch) + values = result.batch["values"] + + batch_size, response_len = 2, 5 + self.assertEqual(values.shape, (batch_size, response_len)) + + self.assertTrue(torch.isfinite(values).all()) + + def test_update_critic(self): + """Test CriticWorker.update_critic() method""" + worker = CriticWorker(self.config) + worker.init_model() + + data = self._create_test_data_for_update_critic() + + result = worker.update_critic(data) + + self.assertIsInstance(result, DataProto) + self.assertIn("metrics", result.meta_info) + metrics = result.meta_info["metrics"] + + expected_keys = ["critic/vf_loss", "critic/vf_clipfrac", "critic/vpred_mean", "critic/grad_norm"] + for key in expected_keys: + self.assertIn(key, metrics) + + for key, value in metrics.items(): + if isinstance(value, list | tuple): + for v in value: + self.assertTrue(torch.isfinite(torch.tensor(v)).all()) + else: + self.assertTrue(torch.isfinite(torch.tensor(value)).all()) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2c/verl/tests/workers/reward_manager/test_registry_on_cpu.py b/e2c/verl/tests/workers/reward_manager/test_registry_on_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..9932ae8917805e3c92bbc0e11abd398463e8e87a --- /dev/null +++ b/e2c/verl/tests/workers/reward_manager/test_registry_on_cpu.py @@ -0,0 +1,94 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +# Assuming REWARD_MANAGER_REGISTRY is defined somewhere in the module +from verl.workers.reward_manager.registry import REWARD_MANAGER_REGISTRY, get_reward_manager_cls, register + + +@pytest.fixture +def setup(): + """Setup test cases with a mock registry.""" + REWARD_MANAGER_REGISTRY.clear() + REWARD_MANAGER_REGISTRY.update({"manager1": "Manager1Class", "manager2": "Manager2Class"}) + return REWARD_MANAGER_REGISTRY + + +def test_get_existing_manager(setup): + """Test getting an existing reward manager class.""" + assert get_reward_manager_cls("manager1") == "Manager1Class" + assert get_reward_manager_cls("manager2") == "Manager2Class" + + +def test_get_nonexistent_manager(setup): + """Test getting a non-existent reward manager raises ValueError.""" + with pytest.raises(ValueError) as excinfo: + get_reward_manager_cls("unknown_manager") + assert "Unknown reward manager: unknown_manager" in str(excinfo.value) + + +def test_case_sensitivity(setup): + """Test that manager names are case-sensitive.""" + with pytest.raises(ValueError): + get_reward_manager_cls("MANAGER1") + with pytest.raises(ValueError): + get_reward_manager_cls("Manager1") + + +def test_empty_registry(setup): + """Test behavior when registry is empty.""" + REWARD_MANAGER_REGISTRY.clear() + with pytest.raises(ValueError) as excinfo: + get_reward_manager_cls("any_manager") + assert "Unknown reward manager: any_manager" in str(excinfo.value) + + +def test_register_new_class(setup): + """Test registering a new class with the decorator.""" + + @register("test_manager") + class TestManager: + pass + + assert "test_manager" in REWARD_MANAGER_REGISTRY + assert REWARD_MANAGER_REGISTRY["test_manager"] == TestManager + + +def test_register_different_classes_same_name(setup): + """Test that registering different classes with same name raises ValueError.""" + + @register("conflict_manager") + class Manager1: + pass + + with pytest.raises(ValueError): + + @register("conflict_manager") + class Manager2: + pass + + assert REWARD_MANAGER_REGISTRY["conflict_manager"] == Manager1 + + +def test_decorator_returns_original_class(setup): + """Test that the decorator returns the original class unchanged.""" + + @register("return_test") + class OriginalClass: + def method(setup): + return 42 + + assert OriginalClass().method() == 42 + assert REWARD_MANAGER_REGISTRY["return_test"] == OriginalClass diff --git a/e2c/verl/tests/workers/reward_model/agent_utils.py b/e2c/verl/tests/workers/reward_model/agent_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0d73bd60772b0f9e1da72d470bbf35d0abdb6497 --- /dev/null +++ b/e2c/verl/tests/workers/reward_model/agent_utils.py @@ -0,0 +1,104 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import ray +from omegaconf import DictConfig + +from verl.experimental.agent_loop import AgentLoopManager +from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup +from verl.single_controller.ray.base import create_colocated_worker_cls +from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role +from verl.workers.config import RewardModelConfig +from verl.workers.fsdp_workers import ActorRolloutRefWorker, AsyncActorRolloutRefWorker + +if os.environ["LEGACY_IMPL_RM"] == "disable": + from verl.workers.roles import RewardModelWorker +else: + from verl.workers.fsdp_workers import RewardModelWorker + + +def init_agent_loop_manager( + config: DictConfig, reward_model_config: RewardModelConfig = None +) -> AgentLoopManager | RayWorkerGroup: + # =========================== 1. Create hybrid ActorRollout workers =========================== + actor_rollout_cls = ( + AsyncActorRolloutRefWorker if config.actor_rollout_ref.rollout.mode == "async" else ActorRolloutRefWorker + ) + role_worker_mapping = { + Role.ActorRollout: ray.remote(actor_rollout_cls), + } + reward_model_config = reward_model_config or config.reward_model + if reward_model_config.enable: + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + } + if reward_model_config.enable_resource_pool: + mapping[Role.RewardModel] = "reward_pool" + if reward_model_config.n_gpus_per_node <= 0: + raise ValueError("reward_model_config.n_gpus_per_node must be greater than 0") + if reward_model_config.nnodes <= 0: + raise ValueError("reward_model_config.nnodes must be greater than 0") + + reward_pool = [reward_model_config.n_gpus_per_node] * reward_model_config.nnodes + resource_pool_spec["reward_pool"] = reward_pool + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + resource_pool_manager.create_resource_pool() + resource_pool_to_cls = {pool: {} for pool in resource_pool_manager.resource_pool_dict.values()} + + # create actor and rollout + resource_pool = resource_pool_manager.get_resource_pool(Role.ActorRollout) + actor_rollout_cls = RayClassWithInitArgs( + cls=role_worker_mapping[Role.ActorRollout], config=config.actor_rollout_ref, role="actor_rollout" + ) + resource_pool_to_cls[resource_pool]["actor_rollout"] = actor_rollout_cls + + if reward_model_config.enable: + # we create a RM here + resource_pool = resource_pool_manager.get_resource_pool(Role.RewardModel) + rm_cls = RayClassWithInitArgs(role_worker_mapping[Role.RewardModel], config=reward_model_config) + resource_pool_to_cls[resource_pool]["rm"] = rm_cls + + all_wg = {} + for resource_pool, class_dict in resource_pool_to_cls.items(): + worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + wg_dict = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=worker_dict_cls) + spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + all_wg.update(spawn_wg) + actor_rollout_wg = all_wg["actor_rollout"] + actor_rollout_wg.init_model() + + if config.actor_rollout_ref.rollout.mode == "sync": + return actor_rollout_wg + + if reward_model_config.enable_resource_pool and reward_model_config.enable: + rm_wg = all_wg["rm"] + rm_wg.init_model() + else: + rm_wg = None + # =========================== 2. Create AgentLoopManager =========================== + agent_loop_manager = AgentLoopManager( + config=config, + worker_group=actor_rollout_wg, + rm_wg=rm_wg, + ) + + return agent_loop_manager diff --git a/e2c/verl/tests/workers/reward_model/process_fn.py b/e2c/verl/tests/workers/reward_model/process_fn.py new file mode 100644 index 0000000000000000000000000000000000000000..1d5348c76d4d7a50a30e6023b3a52e2ea5a5ad5c --- /dev/null +++ b/e2c/verl/tests/workers/reward_model/process_fn.py @@ -0,0 +1,52 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from verl.utils.reward_score.math_reward import last_boxed_only_string, remove_boxed + +GENRM_PROMPT_TEMPLATE = """ +The following is a math problem and an AI solution: + +[Math Problem] + +{problem} + +[AI Solution] + +{solution} + +Your task is to review and critique the solution step by step, and output whether the AI solution is correct. + +Please put your final answer (i.e., 'True' or 'False') in \\boxed{{}}. +""".strip() + + +def construct_genrm_inputs_from_rollouts(rollout_question, rollout_response, ground_truth=None) -> str: + prompt = GENRM_PROMPT_TEMPLATE.format(problem=rollout_question, solution=rollout_response) + return prompt + + +def _compute_reward_score(response: str) -> float: + reward_score = 0.0 + try: + boxed_result = last_boxed_only_string(response) + if boxed_result is not None: + result = remove_boxed(boxed_result) + reward_score = float(result == "True") + except Exception as e: + print(e) + return reward_score + + +def convert_genrm_responses_to_rewards(output: str) -> float: + return _compute_reward_score(output) diff --git a/e2c/verl/tests/workers/reward_model/test_agent_loop_reward_model.py b/e2c/verl/tests/workers/reward_model/test_agent_loop_reward_model.py new file mode 100644 index 0000000000000000000000000000000000000000..c35e8ed55ea57e879e8243722b12458334ea475b --- /dev/null +++ b/e2c/verl/tests/workers/reward_model/test_agent_loop_reward_model.py @@ -0,0 +1,116 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import ray +from hydra import compose, initialize_config_dir +from torchdata.stateful_dataloader import StatefulDataLoader +from transformers import AutoTokenizer + +from tests.workers.reward_model.agent_utils import init_agent_loop_manager +from verl.protocol import DataProto +from verl.trainer.main_ppo import create_rl_sampler +from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn + + +def test_agent_loop_compute_score_with_model(): + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + "VLLM_USE_V1": "1", + } + } + ) + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + config = compose("ppo_trainer") + + model_path = "meta-llama/Llama-3.2-3B-Instruct" + rm_path = "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2" + config.data.return_raw_chat = True + config.actor_rollout_ref.model.path = model_path + config.actor_rollout_ref.actor.use_dynamic_bsz = True + config.actor_rollout_ref.rollout.name = "vllm" + config.actor_rollout_ref.rollout.mode = "async" + config.actor_rollout_ref.rollout.temperature = 0.0 + config.actor_rollout_ref.rollout.prompt_length = 1024 + config.actor_rollout_ref.rollout.response_length = 4096 + + if os.environ["LEGACY_IMPL_RM"] == "disable": + from verl.workers.config import HFModelConfig, RewardModelConfig + + model_config = HFModelConfig(path=rm_path) + reward_model_config = RewardModelConfig( + enable=True, + enable_resource_pool=True, + n_gpus_per_node=4, + nnodes=1, + model_config=model_config, + input_model_config=None, + tensor_model_parallel_size=2, + gpu_memory_utilization=0.8, + ) + else: + config.reward_model.enable = True + config.reward_model.model.path = rm_path + config.reward_model.use_dynamic_bsz = True + config.reward_model.forward_max_token_len_per_gpu = 6000 + config.reward_model.micro_batch_size_per_gpu = 40 + config.reward_model.enable_resource_pool = True + config.reward_model.n_gpus_per_node = 4 + config.reward_model.nnodes = 1 + config.reward_model.model.trust_remote_code = True + config.reward_model.model.input_tokenizer = None + reward_model_config = None + + config.trainer.n_gpus_per_node = 4 + config.trainer.nnodes = 1 + # 1. init agent loop manager + agent_loop_manager = init_agent_loop_manager(config, reward_model_config) + + # 2. init dataset and dataloader + local_folder = os.path.expanduser("~/verl-data/gsm8k/") + data_files = [os.path.join(local_folder, "train.parquet")] + tokenizer = AutoTokenizer.from_pretrained(model_path) + + dataset = RLHFDataset( + data_files=data_files, + tokenizer=tokenizer, + config=config.data, + processor=None, + ) + + batch_size = 128 + sampler = create_rl_sampler(config.data, dataset) + dataloader = StatefulDataLoader( + dataset=dataset, + batch_size=batch_size, + num_workers=config.data.dataloader_num_workers, + drop_last=True, + collate_fn=collate_fn, + sampler=sampler, + ) + + # 3. generate_sequences with agent loop + batch_dict = next(iter(dataloader)) + batch = DataProto.from_single_dict(batch_dict) + gen_batch = agent_loop_manager.generate_sequences(prompts=batch) + + rm_scores = gen_batch.batch["rm_scores"] + sample_scores = rm_scores.sum(dim=1) + print(sample_scores) + ray.shutdown() diff --git a/e2c/verl/tests/workers/reward_model/test_discriminative_reward_model.py b/e2c/verl/tests/workers/reward_model/test_discriminative_reward_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f45c6205779c25426dc0e85adb40b4818a934353 --- /dev/null +++ b/e2c/verl/tests/workers/reward_model/test_discriminative_reward_model.py @@ -0,0 +1,138 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import ray +import torch +from transformers import AutoModelForSequenceClassification + +from verl import DataProto +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup +from verl.utils.model import compute_position_id_with_mask +from verl.workers.config import HFModelConfig, RewardModelConfig +from verl.workers.roles import RewardModelWorker + + +def create_data_samples(tokenizer) -> DataProto: + convs = [ + [ + { + "role": "user", + "content": "What is the range of the numeric output of a sigmoid node in a neural network?", + }, + {"role": "assistant", "content": "Between -1 and 1."}, + ], + [ + { + "role": "user", + "content": "What is the range of the numeric output of a sigmoid node in a neural network?", + }, + {"role": "assistant", "content": "Between 0 and 1."}, + ], + [ + {"role": "user", "content": "What is the capital of Australia?"}, + { + "role": "assistant", + "content": "Canberra is the capital city of Australia.", + }, + ], + [ + {"role": "user", "content": "What is the capital of Australia?"}, + { + "role": "assistant", + "content": "Sydney is the capital of Australia.", + }, + ], + ] + + prompt_length, response_length = 1024, 4096 + pad_token_id = tokenizer.pad_token_id + prompts, responses, input_ids, attention_masks = [], [], [], [] + for conv in convs: + prompt_tokens = tokenizer.apply_chat_template(conv[:1], tokenize=True) + response_tokens = tokenizer.apply_chat_template(conv, tokenize=True)[len(prompt_tokens) :] + + padded_prompt = [pad_token_id] * (prompt_length - len(prompt_tokens)) + prompt_tokens + padded_response = response_tokens + [pad_token_id] * (response_length - len(response_tokens)) + attention_mask = ( + [0] * (prompt_length - len(prompt_tokens)) + + [1] * len(prompt_tokens) + + [1] * len(response_tokens) + + [0] * (response_length - len(response_tokens)) + ) + prompts.append(torch.tensor(padded_prompt)) + responses.append(torch.tensor(padded_response)) + input_ids.append(torch.tensor(padded_prompt + padded_response)) + attention_masks.append(torch.tensor(attention_mask)) + + prompts = torch.stack(prompts) + responses = torch.stack(responses) + input_ids = torch.stack(input_ids) + attention_masks = torch.stack(attention_masks) + position_ids = compute_position_id_with_mask(attention_masks) + + return DataProto.from_dict( + tensors={ + "prompts": prompts, + "responses": responses, + "input_ids": input_ids, + "attention_mask": attention_masks, + "position_ids": position_ids, + }, + ) + + +def test_reward_model(): + ray.init() + + rm_path = os.path.expanduser("~/models/Skywork/Skywork-Reward-V2-Llama-3.2-1B") + model_config = HFModelConfig(path=rm_path) + config = RewardModelConfig( + enable=True, + name="sglang", + model_type="discriminative", + dtype="bfloat16", + model_config=model_config, + input_model_config=None, + tensor_model_parallel_size=2, + ) + ray_cls_with_init = RayClassWithInitArgs(cls=ray.remote(RewardModelWorker), config=config) + resource_pool = RayResourcePool(process_on_nodes=[8]) + rm_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) + # init model + rm_wg.init_model() + + # create data samples + tokenizer = model_config.get_processor() + data = create_data_samples(tokenizer) + + gen_batch = rm_wg.compute_rm_score(data) + server_rm_scores = gen_batch.batch["rm_scores"].sum(dim=-1) + print(f"{server_rm_scores=}") + server_rm_scores_mean = torch.mean(server_rm_scores) + + hf_model = AutoModelForSequenceClassification.from_pretrained(rm_path, torch_dtype=torch.bfloat16) + hf_model.pad_token_id = tokenizer.pad_token_id + hf_output = hf_model( + input_ids=data.batch["input_ids"], + attention_mask=data.batch["attention_mask"], + ) + hf_rm_scores = hf_output.logits.squeeze().detach().to("cpu") + print(f"{hf_rm_scores=}") + hf_rm_scores_mean = torch.mean(hf_rm_scores).to(server_rm_scores.dtype) + + torch.testing.assert_close(server_rm_scores_mean, hf_rm_scores_mean, atol=2e-2, rtol=1e-2) + + ray.shutdown() diff --git a/e2c/verl/tests/workers/reward_model/test_generative_reward_model.py b/e2c/verl/tests/workers/reward_model/test_generative_reward_model.py new file mode 100644 index 0000000000000000000000000000000000000000..ded4bf04b1e939d1ed60f503b0f8967873d9ad0c --- /dev/null +++ b/e2c/verl/tests/workers/reward_model/test_generative_reward_model.py @@ -0,0 +1,134 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import ray +import torch + +from verl import DataProto +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup +from verl.utils.model import compute_position_id_with_mask +from verl.workers.config import HFModelConfig, RewardModelConfig, RewardModelDataProcessorConfig, SamplingConfig +from verl.workers.roles import RewardModelWorker + + +def create_data_samples(tokenizer) -> DataProto: + convs = [ + [ + { + "role": "user", + "content": "What is the range of the numeric output of a sigmoid node in a neural network?", + }, + {"role": "assistant", "content": "Between -1 and 1."}, + ], + [ + { + "role": "user", + "content": "What is the range of the numeric output of a sigmoid node in a neural network?", + }, + {"role": "assistant", "content": "Between 0 and 1."}, + ], + [ + {"role": "user", "content": "What is the capital of Australia?"}, + { + "role": "assistant", + "content": "Canberra is the capital city of Australia.", + }, + ], + [ + {"role": "user", "content": "What is the capital of Australia?"}, + { + "role": "assistant", + "content": "Sydney is the capital of Australia.", + }, + ], + ] + + prompt_length, response_length = 1024, 4096 + pad_token_id = tokenizer.pad_token_id + prompts, responses, input_ids, attention_masks = [], [], [], [] + for conv in convs: + prompt_tokens = tokenizer.apply_chat_template(conv[:1], tokenize=True) + response_tokens = tokenizer.apply_chat_template(conv, tokenize=True)[len(prompt_tokens) :] + + padded_prompt = [pad_token_id] * (prompt_length - len(prompt_tokens)) + prompt_tokens + padded_response = response_tokens + [pad_token_id] * (response_length - len(response_tokens)) + attention_mask = ( + [0] * (prompt_length - len(prompt_tokens)) + + [1] * len(prompt_tokens) + + [1] * len(response_tokens) + + [0] * (response_length - len(response_tokens)) + ) + prompts.append(torch.tensor(padded_prompt)) + responses.append(torch.tensor(padded_response)) + input_ids.append(torch.tensor(padded_prompt + padded_response)) + attention_masks.append(torch.tensor(attention_mask)) + + prompts = torch.stack(prompts) + responses = torch.stack(responses) + input_ids = torch.stack(input_ids) + attention_masks = torch.stack(attention_masks) + position_ids = compute_position_id_with_mask(attention_masks) + + return DataProto.from_dict( + tensors={ + "prompts": prompts, + "responses": responses, + "input_ids": input_ids, + "attention_mask": attention_masks, + "position_ids": position_ids, + }, + ) + + +def test_reward_model(): + ray.init() + + rm_path = os.path.expanduser("~/models/verl-team/GenRM-CI-Test-1.5B") + model_config = HFModelConfig(path=rm_path) + sampling_config = SamplingConfig(temperature=0.0) + data_processor_config = RewardModelDataProcessorConfig( + path="tests/workers/reward_model/process_fn.py", + preprocess_fn_name="construct_genrm_inputs_from_rollouts", + postprocess_fn_name="convert_genrm_responses_to_rewards", + ) + config = RewardModelConfig( + enable=True, + name="sglang", + model_type="generative", + prompt_length=2048, + response_length=4096, + dtype="bfloat16", + model_config=model_config, + input_model_config=None, + tensor_model_parallel_size=2, + sampling_config=sampling_config, + data_processor_config=data_processor_config, + ) + ray_cls_with_init = RayClassWithInitArgs(cls=ray.remote(RewardModelWorker), config=config) + resource_pool = RayResourcePool(process_on_nodes=[8]) + rm_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) + # init model + rm_wg.init_model() + + # create data samples + tokenizer = model_config.get_processor() + data = create_data_samples(tokenizer) + + gen_batch = rm_wg.compute_rm_score(data) + server_rm_scores = gen_batch.batch["rm_scores"].sum(dim=-1) + print(f"{server_rm_scores=}") + + ray.shutdown() diff --git a/e2c/verl/tests/workers/rollout/perf/vllm_async_rollout.py b/e2c/verl/tests/workers/rollout/perf/vllm_async_rollout.py new file mode 100644 index 0000000000000000000000000000000000000000..d7239ea88dd14f6b7fc4927388ff47273c02a34e --- /dev/null +++ b/e2c/verl/tests/workers/rollout/perf/vllm_async_rollout.py @@ -0,0 +1,138 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Compare vLLM AsyncLLM backend: ExternalRayDistributedExecutor(remote call) vs RayDistributedExecutor(compiled graph) + +1. Prepare openai/gsm8k dataset +python3 examples/data_preprocess/gsm8k.py + +2. Run perf test +python3 tests/workers/rollout/perf/vllm_async_rollout.py >perf.log 2>&1 + +hardware: Nvidia 8*H20 +packages: +- torch==2.6.0 +- vllm==0.8.5 + +[DEBUG] backend: sync, n_gpus_per_node: 8, batch_size: 2048, step: 0, step_time: 21.27 secs +[DEBUG] backend: zeromq, n_gpus_per_node: 8, batch_size: 2048, step: 0, step_time: 23.40 secs +[DEBUG] backend: ray, n_gpus_per_node: 8, batch_size: 2048, step: 0, step_time: 25.33 secs +""" + +import os +import time + +import ray +from omegaconf import DictConfig +from torch.utils.data import SequentialSampler +from torchdata.stateful_dataloader import StatefulDataLoader + +from tests.experimental.agent_loop.agent_utils import AgentLoopManager, RayWorkerGroup, init_agent_loop_manager +from verl.protocol import DataProto +from verl.utils import hf_tokenizer +from verl.utils.dataset import RLHFDataset +from verl.utils.dataset.rl_dataset import collate_fn as default_collate_fn + + +def init_config(n_gpus_per_node) -> DictConfig: + import os + + from hydra import compose, initialize_config_dir + + with initialize_config_dir(config_dir=os.path.abspath("verl/trainer/config")): + config = compose( + config_name="ppo_trainer", + overrides=[ + "actor_rollout_ref.actor.use_dynamic_bsz=true", + "actor_rollout_ref.actor.fsdp_config.param_offload=True", + "actor_rollout_ref.actor.fsdp_config.optimizer_offload=True", + ], + ) + config.trainer.n_gpus_per_node = n_gpus_per_node + config.data.train_batch_size = 128 + config.data.return_raw_chat = True + config.actor_rollout_ref.model.path = "Qwen/Qwen2.5-7B-Instruct" + config.actor_rollout_ref.rollout.mode = "async" + config.actor_rollout_ref.rollout.tensor_model_parallel_size = 2 + config.actor_rollout_ref.rollout.gpu_memory_utilization = 0.9 + config.actor_rollout_ref.rollout.multi_turn.format = "hermes" + config.actor_rollout_ref.rollout.prompt_length = 4096 + config.actor_rollout_ref.rollout.response_length = 4096 + config.actor_rollout_ref.rollout.n = 16 + + return config + + +def initialize(config, backend) -> tuple[AgentLoopManager | RayWorkerGroup, StatefulDataLoader]: + env_vars = { + "NCCL_DEBUG": "WARN", + "VLLM_USE_V1": "1", + "VERL_VLLM_DISTRIBUTED_BACKEND": backend, + } + ray.init(runtime_env={"env_vars": env_vars}) + + # STEP 1: init async llm server + server = init_agent_loop_manager(config) + + # STEP 2: create dataloader + tokenizer = hf_tokenizer(config.actor_rollout_ref.model.path) + dataset = RLHFDataset( + data_files=os.path.expanduser("~/data/gsm8k/train.parquet"), + tokenizer=tokenizer, + config=config.data, + ) + dataloader = StatefulDataLoader( + dataset=dataset, + batch_size=config.data.get("gen_batch_size", config.data.train_batch_size), + num_workers=config.data.get("dataloader_num_workers", 8), + drop_last=True, + collate_fn=default_collate_fn, + sampler=SequentialSampler(dataset), + ) + + return server, dataloader + + +def perf_rollout(mode, backend, n_gpus_per_node, num_steps): + config = init_config(n_gpus_per_node) + config.actor_rollout_ref.rollout.mode = mode + agent_loop_manager, dataloader = initialize(config, backend) + + for step, batch in enumerate(dataloader): + batch: DataProto = DataProto.from_single_dict(batch) + batch = batch.pop( + batch_keys=["input_ids", "attention_mask", "position_ids"], + non_tensor_batch_keys=["raw_prompt_ids", "raw_prompt"], + ) + t_start = time.time() + gen_batch = agent_loop_manager.generate_sequences(batch) + t_end = time.time() + print( + f"[DEBUG] backend: {backend}, n_gpus_per_node: {n_gpus_per_node}, batch_size: {len(gen_batch)}, " + f"step: {step}, step_time: {t_end - t_start:.2f} secs" + ) + if step + 1 >= num_steps: + break + + ray.shutdown() + + +if __name__ == "__main__": + num_steps = 1 + n_gpus_per_node = 8 + + # test_cases = [("sync", "sync"), ("async", "zeromq"), ("async", "ray")] + test_cases = [("async", "zeromq"), ("async", "ray")] + for mode, backend in test_cases: + perf_rollout(mode=mode, backend=backend, n_gpus_per_node=n_gpus_per_node, num_steps=num_steps) diff --git a/e2c/verl/tests/workers/rollout/resource/tool_configs/mcp_server.json b/e2c/verl/tests/workers/rollout/resource/tool_configs/mcp_server.json new file mode 100644 index 0000000000000000000000000000000000000000..9ed41f10bc00784d6c4935bf882900aee748723f --- /dev/null +++ b/e2c/verl/tests/workers/rollout/resource/tool_configs/mcp_server.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "Tavily Expert": { + "url": "https://tavily.api.tadata.com/mcp/tavily/your_expert", + "auth_token": "your_tavily_token" + } + } +} \ No newline at end of file diff --git a/e2c/verl/tests/workers/rollout/resource/tool_configs/mcp_tool_config b/e2c/verl/tests/workers/rollout/resource/tool_configs/mcp_tool_config new file mode 100644 index 0000000000000000000000000000000000000000..a9a45bd0bc2fdc7b0805f7af2fa56521a1544a47 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/resource/tool_configs/mcp_tool_config @@ -0,0 +1,11 @@ +tools: + - class_name: verl.tools.mcp_search_tool.MCPSearchTool + config: + rate_limit: 120 + timeout: 120 + type: mcp + mcp: + mcp_servers_config_path: ./resource/tool_configs/mcp_server.json + # optional + tool_selected_list: + - tavily_search_tool \ No newline at end of file diff --git a/e2c/verl/tests/workers/rollout/resource/tool_configs/sandbox_fusion_tool_config b/e2c/verl/tests/workers/rollout/resource/tool_configs/sandbox_fusion_tool_config new file mode 100644 index 0000000000000000000000000000000000000000..aa3f1eec5af8477543a487bacd602ab0d2f7390b --- /dev/null +++ b/e2c/verl/tests/workers/rollout/resource/tool_configs/sandbox_fusion_tool_config @@ -0,0 +1,17 @@ +tools: + - class_name: "verl.tools.sandbox_fusion_tools.SandboxFusionTool" + config: + sandbox_fusion_url: "https://xxx.apigateway-cn-beijing.volceapi.com/run_code" + type: native + tool_schema: + type: "function" + function: + name: "code_interpreter" + description: "A tool for executing code." + parameters: + type: "object" + properties: + code: + type: "string" + description: "The code to execute." + required: ["code"] \ No newline at end of file diff --git a/e2c/verl/tests/workers/rollout/resource/tool_configs/search_tool_config b/e2c/verl/tests/workers/rollout/resource/tool_configs/search_tool_config new file mode 100644 index 0000000000000000000000000000000000000000..926b6b832f283175f92cc86b6cc4a1964096a8d3 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/resource/tool_configs/search_tool_config @@ -0,0 +1,23 @@ +tools: + - class_name: verl.tools.search_tool.SearchTool + config: + retrieval_service_url: http://127.0.0.1:8000/retrieve + num_workers: 120 + rate_limit: 120 + timeout: 30 + type: native + tool_schema: + type: function + function: + name: search + description: Searches the web for relevant information based on the given query. + parameters: + type: object + properties: + query_list: + type: array + item: + type: string + description: A list of fully-formed semantic queries. The tool will return search results for each query. + required: + - query_list \ No newline at end of file diff --git a/e2c/verl/tests/workers/rollout/rollout_sglang/test_http_server_engine.py b/e2c/verl/tests/workers/rollout/rollout_sglang/test_http_server_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..e89607705fef92b7ea728cceee7275fa8054c1d0 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/rollout_sglang/test_http_server_engine.py @@ -0,0 +1,978 @@ +# Copyright 2025 z.ai +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file is adapted from multiple sources: +# 1. THUDM/slime project +# Original source: https://github.com/THUDM/slime/blob/main/slime/backends/sglang_utils/http_server_engine.py +# Copyright 2025 z.ai +# Licensed under the Apache License, Version 2.0 +# 2. SGLang project +# Original source: https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/http_server_engine.py +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 +# +# Modifications made by z.ai and ModelBest Inc. include but are not limited to: +# - Enhanced error handling and retry logic +# - Added async support with connection pooling +# - Extended functionality for distributed weight updates +# - Improved logging and monitoring capabilities +# - Additional configuration options and optimizations + +"""Complete unit tests for HTTP Server Engine Adapters. + +This module contains comprehensive unit tests for both HttpServerEngineAdapter +and AsyncHttpServerEngineAdapter classes, covering all public methods, +error handling scenarios, edge cases, and boundary conditions using pytest and mock frameworks. + +Tests use real SGLang modules for integration testing while mocking external dependencies. +""" + +import asyncio +from unittest.mock import AsyncMock, Mock, patch + +import aiohttp +import pytest +import requests +from sglang.srt.managers.io_struct import ( + UpdateWeightsFromTensorReqInput, +) +from sglang.srt.utils import MultiprocessingSerializer + +# Import the module under test +from verl.workers.rollout.sglang_rollout.http_server_engine import ( + AsyncHttpServerAdapter, + HttpServerAdapter, + launch_server_process, +) + + +@pytest.fixture(scope="session") +def event_loop(): + """Create an event loop for the entire test session.""" + loop = asyncio.new_event_loop() + yield loop + loop.close() + + +@pytest.fixture +def basic_adapter_kwargs(): + """Provide basic kwargs for creating HTTP server adapters.""" + return { + "host": "localhost", + "port": 8000, + "node_rank": 0, + "model_path": "/tmp/test_model", + } + + +@pytest.fixture +def router_adapter_kwargs(): + """Provide kwargs for creating adapters with router configuration.""" + return { + "router_ip": "192.168.1.1", + "router_port": 8080, + "host": "localhost", + "port": 8000, + "node_rank": 0, + "model_path": "/tmp/test_model", + } + + +@pytest.fixture +def non_master_adapter_kwargs(): + """Provide kwargs for creating non-master node adapters.""" + return { + "host": "localhost", + "port": 8000, + "node_rank": 1, # Non-master + "model_path": "/tmp/test_model", + } + + +@pytest.fixture +def mock_launch_server_process(): + """Mock the launch_server_process function for testing without actual server startup.""" + from unittest.mock import patch + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.launch_server_process") as mock_launch: + mock_process = Mock() + mock_process.is_alive.return_value = True + mock_process.pid = 12345 + mock_launch.return_value = mock_process + yield mock_launch + + +@pytest.fixture +def mock_multiprocessing_process(): + """Create mock multiprocessing.Process for testing without actual process creation.""" + from unittest.mock import patch + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.multiprocessing.Process") as mock_process_class: + mock_process = Mock() + mock_process.is_alive.return_value = True + mock_process.pid = 12345 + mock_process_class.return_value = mock_process + yield mock_process + + +@pytest.fixture +def mock_requests_session(): + """Create mock requests.Session for testing HTTP interactions.""" + from unittest.mock import patch + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.Session") as mock_session_class: + mock_session = Mock() + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"status": "success"} + mock_session.get.return_value = mock_response + mock_session.post.return_value = mock_response + mock_session_class.return_value.__enter__.return_value = mock_session + yield mock_session + + +@pytest.fixture +def mock_requests_post(): + """Mock requests.post for testing HTTP POST requests.""" + from unittest.mock import patch + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"status": "success"} + mock_post.return_value = mock_response + yield mock_post + + +@pytest.fixture +def mock_requests_get(): + """Mock requests.get for testing HTTP GET requests.""" + from unittest.mock import patch + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.get") as mock_get: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"status": "success"} + mock_get.return_value = mock_response + yield mock_get + + +@pytest.fixture +def mock_aiohttp_session(): + """Create mock aiohttp.ClientSession for testing async HTTP interactions.""" + mock_session = AsyncMock() + mock_session.closed = False + + # Mock response + mock_response = AsyncMock() + mock_response.status = 200 + mock_response.json = AsyncMock(return_value={"status": "success"}) + mock_response.raise_for_status = Mock() + + # Mock context managers + mock_session.get.return_value.__aenter__.return_value = mock_response + mock_session.post.return_value.__aenter__.return_value = mock_response + + return mock_session + + +@pytest.fixture +def mock_kill_process_tree(): + """Mock kill_process_tree function for testing cleanup without actual process termination.""" + from unittest.mock import patch + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.kill_process_tree") as mock_kill: + yield mock_kill + + +# Test environment fixtures for real SGLang testing +@pytest.fixture(scope="session") +def sglang_test_model_path(): + """Provide a test model path for SGLang tests. + + This can be overridden by environment variable SGLANG_TEST_MODEL_PATH + for tests that need a real model. + """ + import os + + return os.getenv("SGLANG_TEST_MODEL_PATH", "/tmp/test_model") + + +@pytest.fixture +def real_adapter_kwargs(sglang_test_model_path): + """Provide kwargs for creating adapters with real SGLang integration.""" + return { + "host": "localhost", + "port": 8000, + "node_rank": 0, + "model_path": sglang_test_model_path, + } + + +@pytest.fixture(autouse=True) +def mock_server_args_post_init(): + """Mock ServerArgs.__post_init__ to skip model path validation.""" + from unittest.mock import patch + + with patch( + "verl.workers.rollout.sglang_rollout.http_server_engine.ServerArgs.__post_init__", return_value=None + ) as mock_post_init: + yield mock_post_init + + +class TestLaunchServerProcess: + """Test cases for launch_server_process function.""" + + def test_launch_server_process_success( + self, mock_multiprocessing_process, mock_requests_session, real_adapter_kwargs + ): + """Test successful server process launch and health check.""" + # Import real SGLang ServerArgs + from sglang.srt.server_args import ServerArgs + + # Create server args using real ServerArgs + server_args = ServerArgs(**real_adapter_kwargs) + + # Test + with patch( + "verl.workers.rollout.sglang_rollout.http_server_engine.multiprocessing.Process" + ) as mock_process_class: + mock_process_class.return_value = mock_multiprocessing_process + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.Session") as mock_session_class: + mock_session_class.return_value.__enter__.return_value = mock_requests_session + + result = launch_server_process(server_args, first_rank_in_node=True) + + # Assertions + assert result == mock_multiprocessing_process + mock_multiprocessing_process.start.assert_called_once() + assert mock_requests_session.get.call_count >= 2 # health_generate and flush_cache + + def test_launch_server_process_non_master(self, mock_multiprocessing_process, non_master_adapter_kwargs): + """Test server launch for non-master nodes (should return immediately).""" + from sglang.srt.server_args import ServerArgs + + server_args = ServerArgs(**non_master_adapter_kwargs) + + with patch( + "verl.workers.rollout.sglang_rollout.http_server_engine.multiprocessing.Process" + ) as mock_process_class: + mock_process_class.return_value = mock_multiprocessing_process + result = launch_server_process(server_args, first_rank_in_node=True) + + assert result == mock_multiprocessing_process + mock_multiprocessing_process.start.assert_not_called() + + def test_launch_server_process_timeout(self, mock_multiprocessing_process, real_adapter_kwargs): + """Test timeout during server health check.""" + from sglang.srt.server_args import ServerArgs + + server_args = ServerArgs(**real_adapter_kwargs) + + with patch( + "verl.workers.rollout.sglang_rollout.http_server_engine.multiprocessing.Process" + ) as mock_process_class: + mock_process_class.return_value = mock_multiprocessing_process + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.Session") as mock_session_class: + mock_session = Mock() + mock_session.get.side_effect = requests.RequestException("Connection failed") + mock_session_class.return_value.__enter__.return_value = mock_session + + import itertools + + with patch( + "verl.workers.rollout.sglang_rollout.http_server_engine.time.time", + side_effect=itertools.chain([0], itertools.repeat(400)), # 第一次返回0,之后一直返回400 + ): + with pytest.raises(TimeoutError): + launch_server_process(server_args, first_rank_in_node=True) + + mock_multiprocessing_process.terminate.assert_called_once() + + def test_launch_server_process_died(self, real_adapter_kwargs): + """Test server process dies during startup.""" + from sglang.srt.server_args import ServerArgs + + server_args = ServerArgs(**real_adapter_kwargs) + + with patch( + "verl.workers.rollout.sglang_rollout.http_server_engine.multiprocessing.Process" + ) as mock_process_class: + mock_process = Mock() + mock_process.is_alive.return_value = False + mock_process_class.return_value = mock_process + + with pytest.raises(RuntimeError, match="Server process terminated unexpectedly"): + launch_server_process(server_args, first_rank_in_node=True) + + +class TestHttpServerEngineAdapter: + """Test cases for HttpServerEngineAdapter class.""" + + def test_init_with_router_registration(self, mock_launch_server_process, mock_requests_post, router_adapter_kwargs): + """Test initialization with router registration.""" + adapter = HttpServerAdapter(**router_adapter_kwargs) + + assert adapter.router_ip == "192.168.1.1" + assert adapter.router_port == 8080 + assert adapter.process == mock_launch_server_process.return_value + mock_requests_post.assert_called_once() + + def test_init_without_router(self, mock_launch_server_process, basic_adapter_kwargs): + """Test initialization without router registration.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + assert adapter.router_ip is None + assert adapter.router_port is None + assert adapter.process == mock_launch_server_process.return_value + + def test_register_with_router_failure(self, mock_launch_server_process, router_adapter_kwargs): + """Test router registration failure handling.""" + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + mock_post.side_effect = requests.RequestException("Connection failed") + + # Should not raise exception, just log error + adapter = HttpServerAdapter(**router_adapter_kwargs) + + assert adapter.router_ip == "192.168.1.1" + mock_post.assert_called_once() + + def test_make_request_success(self, mock_launch_server_process, basic_adapter_kwargs): + """Test successful HTTP request.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"status": "success"} + mock_post.return_value = mock_response + + result = adapter._make_request("test_endpoint", {"param": "value"}) + + assert result == {"status": "success"} + mock_post.assert_called_with( + "http://localhost:8000/test_endpoint", + json={"param": "value"}, + timeout=adapter.timeout, + ) + + def test_make_request_get_method(self, mock_launch_server_process, basic_adapter_kwargs): + """Test HTTP GET request.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.get") as mock_get: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"data": "test"} + mock_get.return_value = mock_response + + result = adapter._make_request("test_endpoint", method="GET") + + assert result == {"data": "test"} + mock_get.assert_called_with("http://localhost:8000/test_endpoint", timeout=adapter.timeout) + + def test_make_request_non_master(self, mock_launch_server_process): + """Test request from non-master node returns empty dict.""" + kwargs = {"host": "localhost", "port": 8000, "node_rank": 1, "model_path": "/tmp/test_model"} + adapter = HttpServerAdapter(**kwargs) + result = adapter._make_request("test_endpoint") + + assert result == {} + + def test_make_request_retry_logic(self, mock_launch_server_process, basic_adapter_kwargs): + """Test retry logic for failed requests.""" + adapter = HttpServerAdapter(max_attempts=3, **basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + with patch("time.sleep") as mock_sleep: + # First two calls fail, third succeeds + mock_post.side_effect = [ + requests.exceptions.Timeout(), + requests.exceptions.ConnectionError(), + Mock(status_code=200, json=lambda: {"success": True}), + ] + + result = adapter._make_request("test_endpoint") + + assert result == {"success": True} + assert mock_post.call_count == 3 + assert mock_sleep.call_count == 2 + + def test_make_request_http_error(self, mock_launch_server_process, basic_adapter_kwargs): + """Test HTTP error handling.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + mock_response = Mock() + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("404 Not Found") + mock_post.return_value = mock_response + + with pytest.raises(requests.exceptions.HTTPError): + adapter._make_request("test_endpoint") + + def test_make_request_max_attempts_exceeded(self, mock_launch_server_process, basic_adapter_kwargs): + """Test max retries exceeded.""" + adapter = HttpServerAdapter(max_attempts=1, **basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + with patch("time.sleep"): + mock_post.side_effect = requests.exceptions.Timeout() + + with pytest.raises(RuntimeError, match="Failed to complete request"): + adapter._make_request("test_endpoint") + + assert mock_post.call_count == 1 # Initial retry + + def test_update_weights_from_tensor_strict(self, mock_launch_server_process, basic_adapter_kwargs): + import base64 + + from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput + + from verl.workers.rollout.sglang_rollout.http_server_engine import HttpServerAdapter + + basic_adapter_kwargs.setdefault("node_rank", 0) + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"status": "updated"} + + req = UpdateWeightsFromTensorReqInput( + serialized_named_tensors=[b"tensor1", b"tensor2"], + load_format="safetensors", + flush_cache=True, + ) + result = adapter.update_weights_from_tensor(req) + + assert result == {"status": "updated"} + + expected_b64_1 = base64.b64encode(b"tensor1").decode("utf-8") + expected_b64_2 = base64.b64encode(b"tensor2").decode("utf-8") + + mock_request.assert_called_once_with( + "update_weights_from_tensor", + { + "serialized_named_tensors": [expected_b64_1, expected_b64_2], + "load_format": "safetensors", + "flush_cache": True, + }, + ) + + def test_update_weights_from_tensor_empty(self, mock_launch_server_process, basic_adapter_kwargs): + from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput + + from verl.workers.rollout.sglang_rollout.http_server_engine import HttpServerAdapter + + basic_adapter_kwargs.setdefault("node_rank", 0) + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"status": "updated"} + + req = UpdateWeightsFromTensorReqInput( + serialized_named_tensors=[], + load_format="safetensors", + flush_cache=True, + ) + result = adapter.update_weights_from_tensor(req) + + assert result == {"status": "updated"} + + mock_request.assert_called_once_with( + "update_weights_from_tensor", + { + "serialized_named_tensors": [], + "load_format": "safetensors", + "flush_cache": True, + }, + ) + + def test_update_weights_from_tensor_none(self, mock_launch_server_process, basic_adapter_kwargs): + from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput + + from verl.workers.rollout.sglang_rollout.http_server_engine import HttpServerAdapter + + basic_adapter_kwargs.setdefault("node_rank", 0) + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"status": "updated"} + + req = UpdateWeightsFromTensorReqInput( + serialized_named_tensors=None, + load_format="safetensors", + flush_cache=True, + ) + result = adapter.update_weights_from_tensor(req) + + assert result == {"status": "updated"} + + mock_request.assert_called_once_with( + "update_weights_from_tensor", + { + "serialized_named_tensors": [], + "load_format": "safetensors", + "flush_cache": True, + }, + ) + + def test_generate(self, mock_launch_server_process, basic_adapter_kwargs): + """Test generate method.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"text": "Generated text"} + + result = adapter.generate( + prompt="Hello world", + sampling_params={"temperature": 0.7}, + return_logprob=True, + ) + + assert result == {"text": "Generated text"} + mock_request.assert_called_once_with( + "generate", + { + "text": "Hello world", + "sampling_params": {"temperature": 0.7}, + "return_logprob": True, + }, + only_master=False, + ) + + def test_flush_cache(self, mock_launch_server_process, basic_adapter_kwargs): + """Test flush_cache method.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.get") as mock_get: + with patch("time.sleep") as mock_sleep: + # First call fails, second succeeds + mock_responses = [ + Mock(status_code=503), # Service unavailable + Mock(status_code=200, json=lambda: {"cache_flushed": True}), + ] + mock_get.side_effect = mock_responses + + result = adapter.flush_cache() + + assert result == {"cache_flushed": True} + assert mock_get.call_count == 2 + mock_sleep.assert_called_once() + + def test_flush_cache_non_master(self, mock_launch_server_process): + """Test flush_cache for non-master node.""" + kwargs = {"host": "localhost", "port": 8000, "node_rank": 1, "model_path": "/tmp/test_model"} + adapter = HttpServerAdapter(**kwargs) + result = adapter.flush_cache() + + assert result == {} + + def test_memory_management_methods(self, mock_launch_server_process, basic_adapter_kwargs): + """Test memory release and resume methods.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"status": "success"} + + # Test release_memory_occupation + result = adapter.release_memory_occupation(["weights", "kv_cache"]) + assert result == {"status": "success"} + mock_request.assert_called_with("release_memory_occupation", {"tags": ["weights", "kv_cache"]}) + + # Test resume_memory_occupation + result = adapter.resume_memory_occupation(["weights"]) + assert result == {"status": "success"} + mock_request.assert_called_with("resume_memory_occupation", {"tags": ["weights"]}) + + def test_generation_control_methods(self, mock_launch_server_process, basic_adapter_kwargs): + """Test generation control methods.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"status": "success"} + + def test_shutdown(self, mock_launch_server_process, mock_kill_process_tree, router_adapter_kwargs): + """Test shutdown method.""" + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + adapter = HttpServerAdapter(**router_adapter_kwargs) + + adapter.shutdown() + + # Should unregister from router + assert mock_post.call_count == 2 # Once for registration, once for unregistration + # Should kill process + mock_kill_process_tree.assert_called_once_with(mock_launch_server_process.return_value.pid) + + def test_shutdown_with_errors(self, mock_launch_server_process, mock_kill_process_tree, router_adapter_kwargs): + """Test shutdown method with errors.""" + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + # Mock registration success but unregistration failure + mock_post.side_effect = [ + Mock(status_code=200), # Registration success + requests.RequestException("Unregistration failed"), # Unregistration failure + ] + + # Mock process kill failure + mock_kill_process_tree.side_effect = Exception("Kill failed") + + adapter = HttpServerAdapter(**router_adapter_kwargs) + + # Should not raise exceptions + adapter.shutdown() + + assert mock_post.call_count == 2 + mock_kill_process_tree.assert_called_once_with(mock_launch_server_process.return_value.pid) + + # Edge cases for HttpServerEngineAdapter + def test_empty_and_none_parameters(self, mock_launch_server_process, basic_adapter_kwargs): + """Test handling of empty and None parameters.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"status": "success"} + req = UpdateWeightsFromTensorReqInput( + serialized_named_tensors=None, + load_format=None, + flush_cache=None, + ) + + # Test generate with all None parameters + result = adapter.generate() + assert result == {"status": "success"} + + # Test with empty lists + result = adapter.update_weights_from_tensor(req) + assert result == {"status": "success"} + + # Test with empty tags + result = adapter.release_memory_occupation(req) + assert result == {"status": "success"} + + def test_large_payload_handling(self, mock_launch_server_process, basic_adapter_kwargs): + """Test handling of large payloads.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"status": "success"} + + # Test with large tensor list + large_tensor_list = [MultiprocessingSerializer.serialize(f"tensor_{i}") for i in range(1000)] + + req = UpdateWeightsFromTensorReqInput( + serialized_named_tensors=large_tensor_list, + load_format="safetensors", + flush_cache=True, + ) + result = adapter.update_weights_from_tensor(req) + assert result == {"status": "success"} + + # Test with large prompt + large_prompt = "A" * 10000 + result = adapter.generate(prompt=large_prompt) + assert result == {"status": "success"} + + def test_timeout_edge_cases(self, mock_launch_server_process): + """Test various timeout scenarios.""" + # Test with very small timeout + kwargs = {"host": "localhost", "port": 8000, "node_rank": 0, "model_path": "/tmp/test_model", "timeout": 0.001} + adapter = HttpServerAdapter(**kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + mock_post.side_effect = requests.exceptions.Timeout() + + with pytest.raises(RuntimeError, match="Failed to complete request"): + adapter._make_request("test_endpoint") + + def test_extreme_configuration_values(self, mock_launch_server_process): + """Test extreme configuration values.""" + # Test with extreme values + kwargs = { + "host": "localhost", + "port": 8000, + "node_rank": 0, + "model_path": "/tmp/test_model", + "timeout": 0.001, # Very small + "max_attempts": 100, # Very large + "retry_delay": 0.001, # Very small + } + adapter = HttpServerAdapter(**kwargs) + + assert adapter.timeout == 0.001 + assert adapter.max_attempts == 100 + assert adapter.retry_delay == 0.001 + + +class TestAsyncHttpServerEngineAdapter: + """Test cases for AsyncHttpServerEngineAdapter class.""" + + def test_init(self, mock_launch_server_process, basic_adapter_kwargs): + """Test async adapter initialization.""" + adapter = AsyncHttpServerAdapter(max_connections=50, **basic_adapter_kwargs) + + assert adapter.max_connections == 50 + + @pytest.mark.asyncio + async def test_make_async_request_success(self, mock_launch_server_process, basic_adapter_kwargs): + """Test successful async HTTP request.""" + + # Instantiate adapter + adapter = AsyncHttpServerAdapter(**basic_adapter_kwargs) + + mock_response = AsyncMock() + mock_response.status = 200 + mock_response.json = AsyncMock(return_value={"status": "success"}) + mock_response.raise_for_status = Mock() + + mock_post_context_manager = AsyncMock() + mock_post_context_manager.__aenter__.return_value = mock_response + + mock_session = AsyncMock(spec=aiohttp.ClientSession) + mock_session.closed = False + mock_session.post.return_value = mock_post_context_manager + + mock_session_cm = AsyncMock() + mock_session_cm.__aenter__.return_value = mock_session + + with patch.object(adapter, "_get_session", return_value=mock_session_cm): + result = await adapter._make_async_request("test_endpoint", {"param": "value"}) + + # Assert result is correct + assert result == {"status": "success"} + + # Verify post was called + mock_session.post.assert_called_once_with( + "http://localhost:8000/test_endpoint", json={"param": "value"}, timeout=adapter.timeout + ) + + @pytest.mark.asyncio + async def test_make_async_request_get_method(self, mock_launch_server_process, basic_adapter_kwargs): + """Test async GET request using aiohttp and proper context mocking.""" + + # Instantiate the async adapter + adapter = AsyncHttpServerAdapter(**basic_adapter_kwargs) + + mock_response = AsyncMock() + mock_response.status = 200 + mock_response.json = AsyncMock(return_value={"data": "test"}) + mock_response.raise_for_status = Mock() + + mock_get_context_manager = AsyncMock() + mock_get_context_manager.__aenter__.return_value = mock_response + + mock_session = AsyncMock(spec=aiohttp.ClientSession) + mock_session.closed = False + mock_session.get.return_value = mock_get_context_manager + + mock_session_cm = AsyncMock() + mock_session_cm.__aenter__.return_value = mock_session + + with patch.object(adapter, "_get_session", return_value=mock_session_cm): + result = await adapter._make_async_request("test_endpoint", method="GET") + + # Validate + assert result == {"data": "test"} + mock_session.get.assert_called_once_with("http://localhost:8000/test_endpoint", timeout=adapter.timeout) + + @pytest.mark.asyncio + async def test_make_async_request_non_master(self, mock_launch_server_process): + """Test async request from non-master node.""" + kwargs = {"host": "localhost", "port": 8000, "node_rank": 1, "model_path": "/tmp/test_model"} + adapter = AsyncHttpServerAdapter(**kwargs) + result = await adapter._make_async_request("test_endpoint") + + assert result == {} + + @pytest.mark.asyncio + async def test_async_generate(self, mock_launch_server_process, basic_adapter_kwargs): + """Test async generate method.""" + adapter = AsyncHttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_async_request", new_callable=AsyncMock) as mock_request: + mock_request.return_value = {"text": "Generated text"} + + result = await adapter.generate( + prompt="Hello world", + sampling_params={"temperature": 0.7}, + return_logprob=True, + ) + + assert result == {"text": "Generated text"} + mock_request.assert_called_once() + + @pytest.mark.asyncio + async def test_async_memory_management(self, mock_launch_server_process, basic_adapter_kwargs): + """Test async memory management methods.""" + adapter = AsyncHttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_async_request", new_callable=AsyncMock) as mock_request: + mock_request.return_value = {"status": "success"} + + # Test release_memory_occupation + result = await adapter.release_memory_occupation(["weights"]) + assert result == {"status": "success"} + mock_request.assert_called_with("release_memory_occupation", {"tags": ["weights"]}) + + # Test resume_memory_occupation + result = await adapter.resume_memory_occupation(["weights"]) + assert result == {"status": "success"} + mock_request.assert_called_with("resume_memory_occupation", {"tags": ["weights"]}) + assert ( + mock_request.call_count == 2 + ) # resume memory occupation will also call release memory occupation once + + +class TestErrorRecovery: + """Test error recovery mechanisms.""" + + def test_flush_cache_recovery(self, mock_launch_server_process, basic_adapter_kwargs): + """Test flush cache recovery from failures.""" + adapter = HttpServerAdapter(max_attempts=2, **basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.get") as mock_get: + # Simulate multiple failures then success + mock_get.side_effect = [ + requests.exceptions.ConnectionError(), + requests.exceptions.Timeout(), + Mock(status_code=503), # Service unavailable + Mock(status_code=200, json=lambda: {"cache_flushed": True}), + ] + + with patch("time.sleep"): + result = adapter.flush_cache() + assert result == {"cache_flushed": True} + + def test_flush_cache_max_attempts(self, mock_launch_server_process, basic_adapter_kwargs): + """Test flush cache max retries exceeded.""" + adapter = HttpServerAdapter(max_attempts=1, **basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.get") as mock_get: + # All attempts fail + mock_get.side_effect = requests.exceptions.ConnectionError() + + with patch("time.sleep"): + result = adapter.flush_cache() + assert result == {} # Should return empty dict on failure + + def test_network_partition_recovery(self, mock_launch_server_process, basic_adapter_kwargs): + """Test recovery from network partition scenarios.""" + adapter = HttpServerAdapter(max_attempts=3, **basic_adapter_kwargs) + + with patch("verl.workers.rollout.sglang_rollout.http_server_engine.requests.post") as mock_post: + # Simulate network partition then recovery + mock_post.side_effect = [ + requests.exceptions.ConnectionError("Network unreachable"), + requests.exceptions.ConnectionError("Network unreachable"), + Mock(status_code=200, json=lambda: {"recovered": True}), + ] + + with patch("time.sleep"): + result = adapter._make_request("test_endpoint") + assert result == {"recovered": True} + + +class TestResourceManagement: + """Test resource management and cleanup.""" + + def test_resource_cleanup_on_exception( + self, mock_launch_server_process, mock_kill_process_tree, basic_adapter_kwargs + ): + """Test resource cleanup when exceptions occur.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + # Simulate exception during operation + with patch.object(adapter, "_make_request", side_effect=Exception("Test error")): + try: + adapter.generate(prompt="test") + except Exception: + pass + + # Cleanup should still work + adapter.shutdown() + mock_kill_process_tree.assert_called_once_with(mock_launch_server_process.return_value.pid) + + def test_multiple_shutdown_calls(self, mock_launch_server_process, basic_adapter_kwargs): + """Test multiple shutdown calls are safe.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + # Multiple shutdown calls should be safe + adapter.shutdown() + adapter.shutdown() + adapter.shutdown() + + +class TestDataTypeHandling: + """Test handling of various data types.""" + + def test_complex_data_structures(self, mock_launch_server_process, basic_adapter_kwargs): + """Test handling of complex data structures.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {"status": "success"} + + # Test with complex sampling params + complex_sampling_params = { + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50, + "repetition_penalty": 1.1, + "stop_sequences": ["
", "\n\n"], + "max_tokens": 100, + "logit_bias": {"token_123": 0.5, "token_456": -0.5}, + "nested_config": { + "beam_search": True, + "num_beams": 4, + "early_stopping": True, + }, + } + + result = adapter.generate( + prompt="Test prompt", + sampling_params=complex_sampling_params, + ) + + assert result == {"status": "success"} + # Verify the complex structure was passed through + call_args = mock_request.call_args[0][1] + assert call_args["sampling_params"] == complex_sampling_params + + +class TestIntegration: + """Integration tests for both adapters.""" + + def test_error_scenarios(self, mock_launch_server_process, basic_adapter_kwargs): + """Test various error scenarios.""" + adapter = HttpServerAdapter(**basic_adapter_kwargs) + + # Test with None payload + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {} + result = adapter.generate() + assert result == {} + + # Test with empty parameters + with patch.object(adapter, "_make_request") as mock_request: + mock_request.return_value = {} + req = UpdateWeightsFromTensorReqInput( + serialized_named_tensors=None, + load_format=None, + flush_cache=None, + ) + result = adapter.update_weights_from_tensor(req) + assert result == {} diff --git a/e2c/verl/tests/workers/rollout/rollout_vllm/run_fsdp_vllm.py b/e2c/verl/tests/workers/rollout/rollout_vllm/run_fsdp_vllm.py new file mode 100644 index 0000000000000000000000000000000000000000..69223890d7a618c282af6e2e917718b8f66f1a0c --- /dev/null +++ b/e2c/verl/tests/workers/rollout/rollout_vllm/run_fsdp_vllm.py @@ -0,0 +1,162 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import torch +import torch.distributed as dist +from torch.distributed.fsdp import CPUOffload, MixedPrecision +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp.api import ShardedStateDictConfig, ShardingStrategy, StateDictType +from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer +from vllm import SamplingParams + +from verl.third_party.vllm import LLM +from verl.utils.distributed import initialize_global_process_group + + +def main(): + assert torch.cuda.is_available(), "CUDA must be present to run FSDP vLLM example" + local_rank, rank, world_size = initialize_global_process_group() + + local_cache_path = "~/.cache/verl/rlhf" + local_cache_path = os.path.expanduser(local_cache_path) + hdfs_path = "Qwen/Qwen2-7B-Instruct" + + from verl.utils.fs import copy_to_local + + local_model_path = copy_to_local(src=hdfs_path, cache_dir=local_cache_path) + tokenizer = AutoTokenizer.from_pretrained(local_model_path, trust_remote_code=True) + actor_model_config = AutoConfig.from_pretrained(local_model_path, trust_remote_code=True) + with torch.device("cuda"): + actor_model = AutoModelForCausalLM.from_pretrained(local_model_path, trust_remote_code=True) + actor_model.to(torch.bfloat16) + + max_prompt_length = 16 + response_length = 32 + preencode_prompts = [ + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + tokenizer.pad_token = tokenizer.eos_token + prompts = tokenizer(preencode_prompts, return_tensors="pt", padding=True) + input_ids = prompts["input_ids"] + attention_mask = prompts["attention_mask"] + from verl.utils.torch_functional import pad_sequence_to_length + + input_ids = pad_sequence_to_length(input_ids, max_prompt_length, tokenizer.pad_token_id, left_pad=True).cuda() + attention_mask = pad_sequence_to_length(attention_mask, max_prompt_length, 0, left_pad=True).cuda() + + from transformers import GenerationConfig + + generation_config = GenerationConfig(do_sample=False) + actor_model.cuda() + output = actor_model.generate( + input_ids=input_ids, + attention_mask=attention_mask, + max_new_tokens=32, + # max_length=max_length, + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.pad_token_id, + generation_config=generation_config, + # renormalize_logits=True, + output_scores=False, # this is potentially very large + return_dict_in_generate=True, + use_cache=False, + ) # may OOM when use_cache = True + seq = output.sequences + response = seq[:, max_prompt_length:] + + print(f"hf response: {tokenizer.batch_decode(response)}") + + tensor_model_parallel_size = 4 + from torch.distributed.device_mesh import init_device_mesh + + device_mesh = init_device_mesh("cuda", mesh_shape=(world_size,), mesh_dim_names=["fsdp"]) + + mixed_precision = MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32) + fsdp_model = FSDP( + actor_model, + use_orig_params=True, + auto_wrap_policy=None, + device_id=torch.cuda.current_device(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + cpu_offload=CPUOffload(offload_params=False), + sync_module_states=False, + device_mesh=device_mesh, + ) + + FSDP.set_state_dict_type( + fsdp_model, state_dict_type=StateDictType.SHARDED_STATE_DICT, state_dict_config=ShardedStateDictConfig() + ) + + state_dict = fsdp_model.state_dict() + + sampling_params = SamplingParams( + temperature=0, top_p=1, n=1, max_tokens=response_length, logprobs=1, ignore_eos=True, detokenize=False + ) + + print(actor_model_config) + llm = LLM( + model=None, + tokenizer=tokenizer, + model_hf_config=actor_model_config, + tensor_parallel_size=tensor_model_parallel_size, + enforce_eager=True, + dtype="bfloat16", + load_format="dummy_dtensor", + gpu_memory_utilization=0.8, + trust_remote_code=True, + ) + + # Warmup iterations + for _ in range(10): + torch.cuda.synchronize() + llm.sync_model_weights(actor_weights=state_dict, load_format="dtensor") + torch.cuda.synchronize() + dist.barrier() + + start_time = time.time() + llm.sync_model_weights(actor_weights=state_dict, load_format="dtensor") + torch.cuda.synchronize() + dist.barrier() + end_time = time.time() + + # Calculate elapsed time + elapsed_time = end_time - start_time + print(f"Time taken: {elapsed_time:.6f} seconds") + + input_ids = input_ids.cuda() + attention_mask = attention_mask.cuda() + idx_list = [] + batch_size = input_ids.shape[0] + + pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id + from verl.workers.rollout.vllm_rollout.vllm_rollout_spmd import _pre_process_inputs + + for i in range(batch_size): + idx_list.append(_pre_process_inputs(pad_token_id, input_ids[i])) + print("start generation") + outputs = llm.generate(prompt_token_ids=idx_list, sampling_params=sampling_params, use_tqdm=False) + vllm_output = outputs[0].cuda() + if torch.distributed.get_rank() == 0: + print(f"hf response: {tokenizer.batch_decode(response)}") + print(f"vllm response: {tokenizer.batch_decode(vllm_output)}") + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/tests/workers/rollout/rollout_vllm/test_vllm_model_rope_scaling.py b/e2c/verl/tests/workers/rollout/rollout_vllm/test_vllm_model_rope_scaling.py new file mode 100644 index 0000000000000000000000000000000000000000..a8a63be23e91227f1fbfd8ff782818558c0ca8ae --- /dev/null +++ b/e2c/verl/tests/workers/rollout/rollout_vllm/test_vllm_model_rope_scaling.py @@ -0,0 +1,137 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import os + +import torch +import torch.distributed +import torch.distributed as dist +from omegaconf import OmegaConf +from transformers import AutoTokenizer + +from verl import DataProto +from verl.utils.config import omega_conf_to_dataclass +from verl.utils.distributed import initialize_global_process_group +from verl.utils.model import compute_position_id_with_mask +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.vllm_rollout.vllm_rollout_spmd import vLLMRollout + + +def test_vllm_rollout_with_yarn_position_embeddings(): + """ + Test the vLLM rollout with yarn position embeddings. + """ + + local_rank, rank, world_size = initialize_global_process_group() + model_path = os.path.expanduser("~/models/OldKingMeister/Qwen2.5-1.5B-Instruct-YaRN") + config = OmegaConf.create( + { + "name": "vllm", + "prompt_length": 35000, + "response_length": 512, + "dtype": "bfloat16", + "enforce_eager": True, + "gpu_memory_utilization": 0.4, + "enable_chunked_prefill": False, + "free_cache_engine": False, + "disable_log_stats": True, + "max_model_len": 35000 + 512, + "max_num_seqs": 1024, + "load_format": "auto", + "val_kwargs": { + "top_k": -1, + "top_p": 1.0, + "temperature": 0, + "n": 1, + "do_sample": False, + }, + "tensor_model_parallel_size": 4, + "calculate_log_probs": False, + "do_sample": False, + "temperature": 0.0, + "max_num_batched_tokens": 35000 + 512, + } + ) + + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + + # do_sample=False for temperate=0 deterministic + input_dataproto = prepare_input_dataproto(tokenizer, config, validate=True, do_sample=False) + + rollout_config: RolloutConfig = omega_conf_to_dataclass(config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=model_path) + model_config.tokenizer.pad_token = tokenizer.eos_token + + vllm_rollout = vLLMRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + # rollout + rollout_response = vllm_rollout.generate_sequences( + prompts=input_dataproto, + ) + if rank == 0: + print("VLLM Rollout Outputs:") + print(tokenizer.batch_decode(rollout_response.batch["responses"][:], skip_special_tokens=False)) + for response in rollout_response.batch["responses"]: + assert "<|im_end|>" in tokenizer.decode(response, skip_special_tokens=False), ( + "Response should contain <|im_end|> token" + ) + print("Checks passed.") + + del vllm_rollout + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + dist.barrier() + torch.distributed.destroy_process_group() + + +def prepare_input_dataproto(tokenizer, config, validate, do_sample=False): + base_phrase = "Roses are red, sky is blue. " * 4096 + preencode_prompts = [ + # 32810 tokens > 32768 tokens + [{"role": "user", "content": base_phrase + "Who won the Champions League in 2019?"}], + [{"role": "user", "content": base_phrase + "The founder of Apple is"}], + [{"role": "user", "content": base_phrase + "What's your name"}], + ] + formatted_prompts = [ + tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True) + for conversation in preencode_prompts + ] + prompts = tokenizer(formatted_prompts, return_tensors="pt", padding="max_length", max_length=config.prompt_length) + input_dataproto = DataProto.from_dict( + { + "input_ids": prompts["input_ids"], + "attention_mask": prompts["attention_mask"], + "position_ids": compute_position_id_with_mask(prompts["attention_mask"]), + }, + meta_info={ + "bos_token_id": tokenizer.bos_token_id, + "eos_token_id": tokenizer.eos_token_id, + "pad_token_id": tokenizer.pad_token_id, + "validate": validate, + "do_sample": do_sample, + "response_length": config.response_length, + "temperature": config.temperature, + }, + ) + return input_dataproto + + +if __name__ == "__main__": + test_vllm_rollout_with_yarn_position_embeddings() diff --git a/e2c/verl/tests/workers/rollout/rollout_vllm/test_vllm_spmd.py b/e2c/verl/tests/workers/rollout/rollout_vllm/test_vllm_spmd.py new file mode 100644 index 0000000000000000000000000000000000000000..84505734bc3dfa72fa2c0cb7911dac5976b06ab0 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/rollout_vllm/test_vllm_spmd.py @@ -0,0 +1,183 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest +import torch +from torch.distributed.fsdp import CPUOffload, MixedPrecision +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp.api import ShardedStateDictConfig, ShardingStrategy, StateDictType +from transformers import AutoModelForCausalLM, AutoTokenizer +from vllm import LLM, SamplingParams + +from verl.utils.distributed import initialize_global_process_group +from verl.utils.torch_functional import pad_sequence_to_length + + +def levenshtein(s1, s2): + m, n = len(s1), len(s2) + # Initialize matrix of zeros + dp = [[0] * (n + 1) for _ in range(m + 1)] + # Initialize first column and first row of the matrix + for i in range(m + 1): + dp[i][0] = i # Deletion from s1 to empty string + for j in range(n + 1): + dp[0][j] = j # Insertion to s1 from empty string + # Compute the Levenshtein distance matrix + for i in range(1, m + 1): + for j in range(1, n + 1): + cost = 0 if s1[i - 1] == s2[j - 1] else 1 # No cost if characters match + dp[i][j] = min( + dp[i - 1][j] + 1, # Deletion + dp[i][j - 1] + 1, # Insertion + dp[i - 1][j - 1] + cost, # Substitution + ) + return dp[m][n] + + +def are_lists_similar(a, b): + if len(a) != len(b): + print("The lists are of different lengths.") + return False + + total_length = 0 + total_diff = 0 + + for s1, s2 in zip(a, b, strict=True): + max_len = max(len(s1), len(s2)) + total_length += max_len + diff = levenshtein(s1, s2) + total_diff += diff + print(f"Comparing strings:\n{s1}\n{s2}\nDifference: {diff} characters\n") + + percentage_difference = (total_diff / total_length) * 100 + print(f"Total difference: {percentage_difference:.2f}%") + + return percentage_difference <= 15 + + +@pytest.mark.skip("https://github.com/vllm-project/vllm/issues/16993") +def test_vllm_spmd(): + assert torch.cuda.device_count() >= 2, "At least 2 GPUs is required to run tp+dp tests." + local_rank, rank, world_size = initialize_global_process_group() + + # Initialize model and token + local_cache_path = "~/.cache/verl/rlhf" + local_cache_path = os.path.expanduser(local_cache_path) + hdfs_path = "Qwen/Qwen2.5-1.5B-Instruct" + from verl.utils.fs import copy_to_local + + local_model_path = copy_to_local(src=hdfs_path, cache_dir=local_cache_path) + tokenizer = AutoTokenizer.from_pretrained(local_model_path, padding_side="left", trust_remote_code=True) + + actor_model = AutoModelForCausalLM.from_pretrained(local_model_path, trust_remote_code=True) + actor_model.to(torch.bfloat16) + + # fill rollout config + max_prompt_length = 16 + max_response_length = 32 + preencode_prompts = [ + "Who won the Champions League in 2019?", + "The founder of Apple is", + "What's your name?", + ] + tokenizer.pad_token = tokenizer.eos_token + prompts = tokenizer(preencode_prompts, return_tensors="pt", padding=True) + input_ids = prompts["input_ids"] + attention_mask = prompts["attention_mask"] + + input_ids = pad_sequence_to_length(input_ids, max_prompt_length, tokenizer.pad_token_id, left_pad=True) + attention_mask = pad_sequence_to_length(attention_mask, max_prompt_length, 0, left_pad=True) + + print("start generation") + input_ids = input_ids.cuda() + attention_mask = attention_mask.cuda() + + temperature = 0 + top_p = 1 + kwargs = dict( + n=1, temperature=temperature, top_p=top_p, max_tokens=max_response_length, logprobs=1, ignore_eos=True + ) + + tensor_parallel_size = 4 + + from torch.distributed.device_mesh import init_device_mesh + + device_mesh = init_device_mesh("cuda", mesh_shape=(world_size,), mesh_dim_names=["fsdp"]) + + mixed_precision = MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32) + + fsdp_model = FSDP( + actor_model, + use_orig_params=True, + auto_wrap_policy=None, + device_id=torch.cuda.current_device(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + cpu_offload=CPUOffload(offload_params=False), + sync_module_states=False, + device_mesh=device_mesh, + ) + + FSDP.set_state_dict_type( + fsdp_model, state_dict_type=StateDictType.SHARDED_STATE_DICT, state_dict_config=ShardedStateDictConfig() + ) + + state_dict = fsdp_model.state_dict() + + sampling_params = SamplingParams(**kwargs) + llm = LLM( + model=local_model_path, + enable_sleep_mode=True, + tensor_parallel_size=tensor_parallel_size, + distributed_executor_backend="external_launcher", + dtype="bfloat16", + enforce_eager=True, + gpu_memory_utilization=0.8, + disable_custom_all_reduce=True, + skip_tokenizer_init=False, + enable_prefix_caching=True, + trust_remote_code=True, + seed=1, + ) + + outputs = llm.generate(preencode_prompts, sampling_params=sampling_params, use_tqdm=False) + vllm_response_tokens = [] + for output in outputs: + generated_text = output.outputs[0].text + vllm_response_tokens.append(generated_text) + + world_size = torch.distributed.get_world_size() + model = llm.llm_engine.model_executor.driver_worker.worker.model_runner.model + model.load_weights( + ((name, param.full_tensor() if world_size != 1 else param) for name, param in state_dict.items()) + ) + + outputs = llm.generate(preencode_prompts, sampling_params=sampling_params, use_tqdm=False) + verl_vllm_response_tokens = [] + for output in outputs: + generated_text = output.outputs[0].text + verl_vllm_response_tokens.append(generated_text) + + if torch.distributed.get_rank() == 0: + print(f"vllm response: {vllm_response_tokens}") + print(f"verl-vllm response: {verl_vllm_response_tokens}") + assert are_lists_similar(vllm_response_tokens, verl_vllm_response_tokens), "Strings differ more than 10%:\n" + print("Check Pass") + torch.distributed.destroy_process_group() + + +if __name__ == "__main__": + test_vllm_spmd() diff --git a/e2c/verl/tests/workers/rollout/test_hf_rollout.py b/e2c/verl/tests/workers/rollout/test_hf_rollout.py new file mode 100644 index 0000000000000000000000000000000000000000..3eb6f4bb2ff3f04a6127304828793151c7b24052 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_hf_rollout.py @@ -0,0 +1,180 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +from omegaconf import OmegaConf +from torch.distributed.fsdp import CPUOffload, MixedPrecision +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp.api import ShardedStateDictConfig, ShardingStrategy, StateDictType +from transformers import AutoModelForCausalLM, AutoTokenizer + +from verl import DataProto +from verl.utils.distributed import initialize_global_process_group +from verl.utils.fs import copy_to_local +from verl.utils.model import compute_position_id_with_mask +from verl.workers.rollout.hf_rollout import HFRollout + +BASE_HF_ROLLOUT_CONFIG = { + "temperature": 1.0, + "top_k": -1, + "top_p": 1, + "prompt_length": 64, + "response_length": 64, + "do_sample": True, + "n": 1, + "val_kwargs": { + "top_k": -1, + "top_p": 1.0, + "temperature": 0, + "n": 1, + "do_sample": False, + }, +} + + +def prepare_input_dataproto(tokenizer, config, validate): + preencode_prompts = [ + [{"role": "user", "content": "Who won the Champions League in 2019?"}], + [{"role": "user", "content": "The founder of Apple is"}], + [{"role": "user", "content": "What's your name"}], + ] + formatted_prompts = [ + tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True) + for conversation in preencode_prompts + ] + prompts = tokenizer(formatted_prompts, return_tensors="pt", padding="max_length", max_length=config.prompt_length) + input_dataproto = DataProto.from_dict( + { + "input_ids": prompts["input_ids"], + "attention_mask": prompts["attention_mask"], + "position_ids": compute_position_id_with_mask(prompts["attention_mask"]), + }, + meta_info={ + "bos_token_id": tokenizer.bos_token_id, + "eos_token_id": tokenizer.eos_token_id, + "pad_token_id": tokenizer.pad_token_id, + "validate": validate, + }, + ) + return input_dataproto + + +def prepare_fsdp_model(model, world_size): + from torch.distributed.device_mesh import init_device_mesh + + device_mesh = init_device_mesh("cuda", mesh_shape=(world_size,), mesh_dim_names=["fsdp"]) + + mixed_precision = MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32) + + fsdp_model = FSDP( + model, + use_orig_params=True, + auto_wrap_policy=None, + device_id=torch.cuda.current_device(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + cpu_offload=CPUOffload(offload_params=False), + sync_module_states=False, + device_mesh=device_mesh, + ) + + FSDP.set_state_dict_type( + fsdp_model, state_dict_type=StateDictType.SHARDED_STATE_DICT, state_dict_config=ShardedStateDictConfig() + ) + return fsdp_model + + +def test_hf_rollout(n: int = 1, do_sample: bool = True, validate: bool = False): + config = OmegaConf.create(BASE_HF_ROLLOUT_CONFIG) + config.update({"n": n, "do_sample": do_sample}) + + assert torch.cuda.device_count() >= 2, "At least 2 GPUs is required to run tp+dp tests." + local_rank, rank, world_size = initialize_global_process_group() + + # Initialize model and tokenizer + local_cache_path = "~/.cache/verl/rlhf" + local_cache_path = os.path.expanduser(local_cache_path) + hdfs_path = "Qwen/Qwen2-7B-Instruct" + local_model_path = copy_to_local(src=hdfs_path, cache_dir=local_cache_path) + tokenizer = AutoTokenizer.from_pretrained(local_model_path, padding_side="left", trust_remote_code=True) + tokenizer.pad_token = tokenizer.eos_token + + # Initialize FSDP model + actor_model = AutoModelForCausalLM.from_pretrained(local_model_path, trust_remote_code=True) + actor_model.to(torch.bfloat16) + fsdp_model = prepare_fsdp_model(actor_model, world_size) + + # Initialize HFRollout and start generate + hf_rollout = HFRollout(fsdp_model, OmegaConf.create(config)) + input = prepare_input_dataproto(tokenizer, config, validate).to(torch.cuda.current_device()) + outputs = hf_rollout.generate_sequences(input) + + # check generated batch size is expected + generated_batch_size = outputs.batch.batch_size[0] + assert generated_batch_size == input.batch.batch_size[0] * config.n + + for i in range(generated_batch_size): + prompt_tokens = outputs.batch["prompts"][i] + prompt_mask = prompt_tokens != tokenizer.pad_token_id + prompt_tokens = prompt_tokens[prompt_mask] + decoded_prompt = tokenizer.decode(prompt_tokens, skip_special_tokens=False) + + response_tokens = outputs.batch["responses"][i] + response_mask = response_tokens != tokenizer.pad_token_id + response_tokens = response_tokens[response_mask] + decoded_response = tokenizer.decode(response_tokens, skip_special_tokens=False) + + attention_mask = outputs.batch["attention_mask"][i] + position_ids = outputs.batch["position_ids"][i] + prompt_length = outputs.batch["prompts"].size(1) + response_length = outputs.batch["responses"].size(1) + + assert attention_mask.size(0) == prompt_length + response_length + assert position_ids.size(0) == prompt_length + response_length + + # check response attention mask is expected + response_attention = attention_mask[prompt_length:] + eos_positions = (outputs.batch["responses"][i] == tokenizer.pad_token_id).nonzero(as_tuple=True)[0] + if len(eos_positions) > 0: + first_eos_pos = eos_positions[0].item() + assert response_attention[: first_eos_pos + 1].all(), "Response attention mask should be 1 until EOS" + if first_eos_pos + 1 < response_length: + assert not response_attention[first_eos_pos + 1 :].any(), ( + "Response attention mask should be 0 after EOS" + ) + else: + assert response_attention.all(), "Response attention mask should be all 1 if no EOS token" + + # check response position ids is expected + prompt_positions = position_ids[:prompt_length] + response_positions = position_ids[prompt_length:] + valid_response_length = min(len(response_tokens), response_length) + if valid_response_length > 0: + assert response_positions[0] == prompt_positions[-1] + 1 + for j in range(1, valid_response_length): + assert response_positions[j] == response_positions[j - 1] + 1 + + # print generated text for inspection + if torch.distributed.get_rank() == 0: + print(f"prompt: {decoded_prompt}") + print(f"response: {decoded_response}") + print("=" * 30) + + +if __name__ == "__main__": + test_hf_rollout(n=2, do_sample=True, validate=False) + # test_hf_rollout(n=1, do_sample=False, validate=True) + # test_hf_rollout(n=1, do_sample=True, validate=False) diff --git a/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_mcp_tools.py b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_mcp_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..1ec7a6eabfcea5a6a27c30af108d7fcf4e6824bf --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_mcp_tools.py @@ -0,0 +1,466 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from tests/workers/rollout/test_sglang_async_rollout_sf_tools.py + + +import asyncio +import os +from copy import deepcopy +from unittest.mock import AsyncMock, MagicMock, patch + +import numpy as np +import pytest +from tensordict import TensorDict +from transformers import AutoConfig, AutoTokenizer +from utils_sglang import get_rollout_config, prepare_inputs + +from verl.protocol import DataProto +from verl.tools.mcp_search_tool import MCPSearchTool +from verl.tools.schemas import ToolResponse +from verl.tools.utils.mcp_clients.McpClientManager import MCPClientManager +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.schemas import AsyncRolloutRequest, AsyncRolloutRequestStateEnum, Message +from verl.workers.rollout.sglang_rollout.sglang_rollout import SGLangRollout + +DEFAULT_USER_CONTENT_PREFIX = ( + "Answer the given question. You must conduct reasoning inside and " + "first every time you get new information. After reasoning, if you find you lack " + "some knowledge, you can call a search engine by query " + "and it will return the top searched results between and " + ". You can search as many times as your want. If you find no " + "further external knowledge needed, you can directly provide the answer inside " + " and , without detailed illustrations. For example, " + " Beijing . Question: " +) +user_content = DEFAULT_USER_CONTENT_PREFIX.rstrip("\n") + "How's the weather lately?" + + +def get_search_messages(): + user_prompt = { + "role": "user", + "content": user_content, + } + + expect_turn_0_msg = { + "role": "assistant", + "content": "Let me search the web.", + "tool_calls": [ + { + "id": "10", + "type": "function", + "function": { + "name": "tavily_search_tool", + "arguments": { + "what_is_your_intent": "Search for the weather lately", + "query": "the weather in Beijing today", + "search_depth": "basic", + "time_range": "day", + "include_domains": ["google.com", "baidu.com"], + "max_results": 2, + }, + }, + } + ], + } + + expect_turn_1_msg = { + "role": "assistant", + "content": "Let me search again.", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "tavily_search_tool", + "arguments": { + "what_is_your_intent": "Search for the weather lately", + "query": "the weather in Beijing tomorrow", + "search_depth": "basic", + "time_range": "day", + "include_domains": ["google.com", "baidu.com"], + "max_results": 2, + }, + }, + } + ], + } + + expect_turn_2_msg = { + "role": "assistant", + "content": "Today is sunny and tomorrow will be cloudy in Beijing.", + } + + # Mock search tool responses + tool_return_0_msg = {"role": "tool", "content": [{"type": "text", "text": "Today's weather in Beijing is sunny."}]} + tool_return_1_msg = { + "role": "tool", + "content": [{"type": "text", "text": "Tomorrow's weather in Beijing is cloudy."}], + } + + user_prompts = [user_prompt] + expect_turn_array = [expect_turn_0_msg, expect_turn_1_msg, expect_turn_2_msg] + tool_return_array = [tool_return_0_msg, tool_return_1_msg] + + return user_prompts, expect_turn_array, tool_return_array + + +class TestRolloutWithMCPSearchTools: + local_model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B") + + @pytest.fixture + def qwen_tokenizer(self): + tokenizer = AutoTokenizer.from_pretrained(self.local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + return tokenizer + + # we only need this for tokenizer + @pytest.fixture + def qwen_model_config(self): + config = AutoConfig.from_pretrained(self.local_model_path) + return config + + @pytest.fixture + def search_data(self, qwen_tokenizer): + user_prompt, expect_turn_array, tool_return_array = get_search_messages() + prompts = [[message] for message in user_prompt] + preencode_turn_array = [ + qwen_tokenizer.apply_chat_template([turn], tokenize=False, add_generation_prompt=False) + for turn in expect_turn_array + ] + preencode_tool_return_array = [ + ToolResponse(text=qwen_tokenizer.apply_chat_template([turn], tokenize=False, add_generation_prompt=True)) + for turn in tool_return_array + ] + return prompts, preencode_turn_array, preencode_tool_return_array + + @pytest.fixture + def search_rollout_config(self): + max_prompt_length = 4096 + max_response_length = 3000 + dtype = "bfloat16" + tensor_parallel_size = 1 + tool_path = "./resource/tool_configs/mcp_tool_config" + rollout_config = get_rollout_config( + max_response_length, max_prompt_length, dtype, tensor_parallel_size, tool_path + ) + return rollout_config + + @pytest.fixture + def search_data_proto(self, search_data, qwen_tokenizer): + preencode_prompts, _, _ = search_data + prompts = [ + qwen_tokenizer.apply_chat_template(message, tokenize=False, add_generation_prompt=True) + for message in preencode_prompts + ] + input_ids, attention_mask, position_ids = prepare_inputs(qwen_tokenizer, prompts, 1000) + prompt_dict = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=input_ids.shape[0], + ) + messages = np.asarray(preencode_prompts) + + tools_kwargs = np.array( + [ + { + "tavily_search_tool": { + "create_kwargs": {"ground_truth": "Today is sunny and tomorrow will be cloudy in Beijing."}, + }, + } + ], + dtype=object, + ) + index = np.array([0], dtype=object) + prompts = DataProto( + batch=prompt_dict, non_tensor_batch={"raw_prompt": messages, "tools_kwargs": tools_kwargs, "index": index} + ) + return prompts + + @pytest.fixture + def mock_rollout(self, search_rollout_config, qwen_tokenizer, qwen_model_config): + """Mock the rollout instance with sampling_params initialized.""" + tool_schema = [ + { + "type": "function", + "function": { + "name": "tavily_search_tool", + "description": "A powerful web search tool...", + "parameters": { + "type": "object", + "properties": { + "what_is_your_intent": { + "type": "string", + "description": "Describe your intent for using Tavily", + }, + "query": {"type": "string", "description": "Search query"}, + "search_depth": { + "type": "string", + "description": "The depth of the search ('basic' or 'advanced')", + }, + "topic": { + "type": "string", + "description": "The category of the search ('general' or 'news')", + }, + "days": { + "type": "integer", + "description": "Number of days back to include in search results (only for " + "'news' topic)", + }, + "time_range": { + "type": "string", + "description": "Time range for results ('day', 'week', 'month', 'year', 'd', " + "'w', 'm', 'y')", + }, + "include_domains": { + "type": "array", + "description": "List of domains to specifically include in search results", + }, + "exclude_domains": { + "type": "array", + "description": "List of domains to specifically exclude from search results", + }, + "include_answer": { + "type": "boolean", + "description": "Whether to include an answer summary generated by an LLM", + }, + "include_raw_content": { + "type": "boolean", + "description": "Whether to include the cleaned and parsed HTML content of each result", + }, + "include_images": { + "type": "boolean", + "description": "Whether to include images from search results", + }, + "include_image_descriptions": { + "type": "boolean", + "description": "Whether to include descriptions with images", + }, + "max_results": { + "type": "integer", + "description": "Maximum number of results to return (5-20)", + }, + "async_search": { + "type": "boolean", + "description": "Whether to perform the search asynchronously", + }, + }, + "required": ["what_is_your_intent", "query"], + }, + "strict": False, + }, + } + ] + with ( + patch.object(MCPClientManager, "fetch_tool_schemas", return_value=tool_schema), + patch.object(SGLangRollout, "_init_distributed_env", return_value=None), + patch.object(SGLangRollout, "_init_inference_engine", return_value=None), + patch.object(SGLangRollout, "_init_sampling_params", return_value=None), + ): + rollout_config: RolloutConfig = omega_conf_to_dataclass(search_rollout_config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + rollout.sampling_params = { + "n": 1, + "max_new_tokens": search_rollout_config.response_length, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "repetition_penalty": 1.0, + } + return rollout + + def test_tools_registration(self, mock_rollout): + assert len(mock_rollout._tool_schemas) != 0 + assert "tavily_search_tool" in mock_rollout._tool_map.keys() + from verl.tools.mcp_search_tool import MCPSearchTool + + assert isinstance(mock_rollout._tool_map["tavily_search_tool"], MCPSearchTool) + # depend on the tokenizer + assert mock_rollout._tool_call_parser_type == "qwen25" + + def test_rollout_req_creation(self, mock_rollout, search_data_proto): + req_list = mock_rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1) + assert len(req_list) == 1 + assert req_list[0].state == AsyncRolloutRequestStateEnum.PENDING + assert len(req_list[0].tool_schemas) == 1 + + def test_over_size_case(self, mock_rollout, search_data_proto, search_data): + mock_rollout.config.multi_turn.max_assistant_turns = 1 + req = mock_rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1)[0] + req = MagicMock(wraps=req, spec=AsyncRolloutRequest) + req.finalize = MagicMock() + req_list = [req] + + _, expect_turn_array, _ = search_data + # here we mock a meta info with 'length'. indicate the response is truncate + mock_rollout._handle_engine_call = MagicMock() + future = asyncio.Future() + future.set_result( + { + "text": expect_turn_array[0], + "meta_info": { + "id": "d1188d81cba840359df5b352b344bc8e", + "finish_reason": {"type": "length", "length": 3000}, + "prompt_tokens": 132, + "completion_tokens": 100, + "cached_tokens": 0, + "e2e_latency": 2.23543, + }, + } + ) + mock_rollout._handle_engine_call.return_value = future + mock_rollout._tp_rank = 0 + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather( + *[mock_rollout._async_rollout_a_request(req, True, False) for req in req_list], + ) + ) + assert len(output_req_list) == 1 + output_req = output_req_list[0] + assert output_req.state == AsyncRolloutRequestStateEnum.COMPLETED + assert output_req.reward_scores.get("tavily_search_tool") == [] + # we should only have two message, one for prompt, second for response. + assert len(output_req.messages) == 2 + assert output_req.messages[1] == Message( + role="assistant", + content=expect_turn_array[0], + tool_calls=None, + ) + + @patch.object(MCPSearchTool, "execute", new_callable=AsyncMock) + def test_tool_call_basic_case(self, mock_execute, mock_rollout, search_data_proto, search_data): + _, expect_turn_array, tool_return_array = search_data + # Mock search tool execution to return predefined responses + mock_execute.side_effect = [(msg, 0.0, {"status": "success"}) for msg in tool_return_array] + + mock_rollout.config.multi_turn.max_assistant_turns = 10 + req = mock_rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1)[0] + req = MagicMock(wraps=req, spec=AsyncRolloutRequest) + req.finalize = MagicMock() + req_list = [req] + + mock_rollout._handle_engine_call = MagicMock() + futures = [asyncio.Future() for i in expect_turn_array] + for idx, (i, turn) in enumerate(zip(futures, expect_turn_array, strict=True)): + i.set_result( + { + "text": turn, + "meta_info": { + "id": "d1188d81cba840359df5b352b344bc8e", + "finish_reason": {"type": "tool_calls" if idx < len(expect_turn_array) - 1 else "stop"}, + "prompt_tokens": len(turn), + "completion_tokens": 100, + "cached_tokens": 0, + "e2e_latency": 2.23543, + }, + } + ) + if idx < len(expect_turn_array) - 1: + assert mock_rollout._function_call_parser.has_tool_call(turn) + assert mock_rollout._function_call_parser.parse_non_stream(turn) + + mock_rollout._handle_engine_call.side_effect = futures + mock_rollout._tp_rank = 0 + + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather(*[mock_rollout._async_rollout_a_request(req, True, False) for req in req_list]) + ) + + # Verify conversation completed successfully with proper tool usage + output_req = output_req_list[0] + assert output_req.state == AsyncRolloutRequestStateEnum.COMPLETED + assert "tavily_search_tool" in output_req.metrics + assert output_req.metrics["tavily_search_tool"][0]["status"] == "success" + assert mock_execute.await_count == 2 + assert len(output_req.messages) == 6 + # Verify tool response messages contain expected content + search_counter = 0 + for msg in output_req.messages: + if msg.role == "tool": + assert msg.content == tool_return_array[search_counter].text + search_counter += 1 + assert search_counter == 2 + + @patch.object(MCPSearchTool, "execute", new_callable=AsyncMock) + def test_tool_call_batch_case(self, mock_execute, mock_rollout, search_data_proto, search_data): + _, expect_turn_array, tool_return_array = search_data + # Mock tool execution for large batch (100 requests * 2 calls each) + mock_execute.side_effect = [ + (tool_return_array[0], 0.0, {"status": "success"}), + (tool_return_array[1], 0.0, {"status": "success"}), + ] * 100 + + mock_rollout.config.multi_turn.max_assistant_turns = 10 + base_req = mock_rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1)[0] + + req_nums = 100 + req_list = [] + req_turns_map = {} + req_turns_counter = {} + + for i in range(req_nums): + tmp_req = deepcopy(base_req) + tmp_req.batch_data_id = i + tmp_req.request_id = i + req_list.append(MagicMock(wraps=tmp_req, spec=AsyncRolloutRequest)) + + futures = [asyncio.Future() for _ in expect_turn_array] + for idx, (fut, turn) in enumerate(zip(futures, expect_turn_array, strict=True)): + fut.set_result( + { + "text": turn, + "meta_info": { + "id": "dummy", + "finish_reason": {"type": "tool_calls" if idx < len(expect_turn_array) - 1 else "stop"}, + "prompt_tokens": len(turn), + "completion_tokens": 100, + }, + } + ) + req_turns_map[i] = futures + req_turns_counter[i] = 0 + + async def hacked_handle_engine_call(self, _req: AsyncRolloutRequest, *_args, **_kwargs): + fut = req_turns_map[_req.batch_data_id][req_turns_counter[_req.batch_data_id]] + req_turns_counter[_req.batch_data_id] += 1 + return await fut + + with patch.object(SGLangRollout, "_handle_engine_call", new=hacked_handle_engine_call): + mock_rollout._tp_rank = 0 + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather(*[mock_rollout._async_rollout_a_request(r, True, False) for r in req_list]) + ) + + # Verify all requests completed successfully + assert len(output_req_list) == req_nums + for out_req in output_req_list: + assert out_req.state == AsyncRolloutRequestStateEnum.COMPLETED + assert "tavily_search_tool" in out_req.metrics + for metric in out_req.metrics["tavily_search_tool"]: + assert metric["status"] == "success" + assert len(out_req.messages) == 6 + assert sum(1 for m in out_req.messages if m.role == "tool") == 2 + + assert mock_execute.await_count == 2 * req_nums diff --git a/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_multimodal_delta.py b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_multimodal_delta.py new file mode 100644 index 0000000000000000000000000000000000000000..dea1b14eaf6bf13e09f4653ff02a0b7208160794 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_multimodal_delta.py @@ -0,0 +1,194 @@ +# Copyright 2025 Amazon.com, Inc. or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pytest + +from verl.tools.schemas import ToolResponse +from verl.utils.dataset.vision_utils import process_image +from verl.utils.tokenizer import hf_processor +from verl.workers.rollout.schemas import ( + AsyncRolloutRequest, + AsyncRolloutRequestStateEnum, + TokenizationSanityCheckModeEnum, +) + + +def _test_add_tool_response_messages_image_delta(processor, image_list, description_list, resize_image=False): + assert len(image_list) == len(description_list) + # Get the smallest dimensions across all images + processed_images = [] + for img_url in image_list: + img = process_image(img_url) + processed_images.append(img) + + min_width = min(img.size[0] for img in processed_images) + min_height = min(img.size[1] for img in processed_images) + min_size = (min_width, min_height) + + if resize_image: + processed_images_resized = [] + for img in processed_images: + img = img.resize(min_size) + processed_images_resized.append(img) + processed_images = processed_images_resized + + # Initial message history + system_prompt = ( + "You will be provided with an image. Describe this image and then generate a new image for the next round" + ) + messages = [ + { + "role": "system", + "content": system_prompt, + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Here is the first image provided: "}, + {"type": "image", "image": [processed_images[0]]}, + ], + }, + ] + + # Initial multi_modal_data with one image + multi_modal_data = {"image": [processed_images[0]], "video": []} + # Minimal required fields for AsyncRolloutRequest + + req = AsyncRolloutRequest( + batch_data_id=0, + request_id="test-req-1", + state=AsyncRolloutRequestStateEnum.PENDING, + messages=messages, + multi_modal_keys=["image", "video"], + multi_modal_data=multi_modal_data.copy(), + tool_schemas=[], + tools_kwargs={}, + interaction_kwargs={}, + input_ids=None, + prompt_ids=None, + response_ids=None, + attention_mask=None, + prompt_attention_mask=None, + response_attention_mask=None, + position_ids=None, + prompt_position_ids=None, + response_position_ids=None, + loss_mask=None, + prompt_loss_mask=None, + response_loss_mask=None, + reward_scores={}, + max_prompt_len=8192, + max_response_len=8192, + max_model_len=16384, + metrics={}, + use_inference_chat_template=True, + tokenization_sanity_check_mode=TokenizationSanityCheckModeEnum.STRICT, + generation_prompt_ids=None, + base_conv_wo_gen_prompt_end_pos=0, + base_conv_with_gen_prompt_end_pos=0, + processing_class=processor, + ) + + prev_generated_len = 0 + # Add First Assistant Message and first tool response message(image) + for idx, img in enumerate(processed_images): + if idx == 0: + continue + _ = req.get_generation_prompt_ids(processor) + req.add_assistant_message(processor, content=description_list[idx - 1]) + before_tool_call_len = req.input_ids.shape[-1] + req.add_tool_response_messages( + processor, [ToolResponse(image=[img], text="Here is the new image you requested: ")] + ) + after_tool_call_len = req.input_ids.shape[-1] + if prev_generated_len == 0: + prev_generated_len = after_tool_call_len - before_tool_call_len + else: + if resize_image: + assert after_tool_call_len - before_tool_call_len == prev_generated_len + assert req.multi_modal_data["image"] == processed_images[: idx + 1] + + _ = req.get_generation_prompt_ids(processor) + req.add_assistant_message(processor, content=description_list[-1]) + + messages = [msg.model_dump() for msg in req.messages] + tools = [tool.model_dump() for tool in req.tool_schemas] if req.tool_schemas else None + full_prompt_info = req._handle_apply_chat_template( + processor, + messages, + multi_modal_data=req.multi_modal_data, + tools=tools, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + ) + full_prompt_ids = full_prompt_info["input_ids"] + assert full_prompt_ids.eq(req.input_ids).all() + + # We must use dict(full_prompt_info) to convert BatchFeature values to a new dict + # because np.array() only keeps the keys for BatchFeature. + full_prompt_multi_modal_inputs = full_prompt_info.copy() + full_prompt_multi_modal_inputs.pop("input_ids", None) + full_prompt_multi_modal_inputs.pop("attention_mask", None) + + for key in full_prompt_multi_modal_inputs: + assert full_prompt_multi_modal_inputs[key].eq(req.multi_modal_inputs[key]).all() + + +@pytest.mark.skipif( + hf_processor(os.path.expanduser("~/models/Qwen/Qwen2.5-VL-3B-Instruct")) is None, + reason="Processor not available for Qwen/Qwen2.5-VL-B-Instruct", +) +def test_add_tool_response_messages_image_delta(): + processor = hf_processor(os.path.expanduser("~/models/Qwen/Qwen2.5-VL-3B-Instruct")) + + # From Qwen2.5-VL-3B-Instruct HF example + img_1_url = {"image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"} + img_1_description = "A woman sits on the beach at sunset, smiling as she shares a high five with her large dog." + # GitHub Logo + img_2_url = {"image": "https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png"} + img_2_description = "A GitHub Logo image" + # Octocat + img_3_url = {"image": "https://octodex.github.com/images/orderedlistocat.png"} + img_3_description = "An Octocat image" + + image_list = [img_1_url, img_2_url, img_3_url] + description_list = [img_1_description, img_2_description, img_3_description] + _test_add_tool_response_messages_image_delta(processor, image_list, description_list, resize_image=False) + + +@pytest.mark.skipif( + hf_processor(os.path.expanduser("~/models/Qwen/Qwen2.5-VL-3B-Instruct")) is None, + reason="Processor not available for Qwen/Qwen2.5-VL-B-Instruct", +) +def test_add_tool_response_messages_image_delta_resize_image(): + processor = hf_processor(os.path.expanduser("~/models/Qwen/Qwen2.5-VL-3B-Instruct")) + + # From Qwen2.5-VL-3B-Instruct HF example + img_1_url = {"image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"} + img_1_description = "A woman sits on the beach at sunset, smiling as she shares a high five with her large dog." + # GitHub Logo + img_2_url = {"image": "https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png"} + img_2_description = "A GitHub Logo image" + # Octocat + img_3_url = {"image": "https://octodex.github.com/images/orderedlistocat.png"} + img_3_description = "An Octocat image" + + image_list = [img_1_url, img_2_url, img_3_url] + description_list = [img_1_description, img_2_description, img_3_description] + _test_add_tool_response_messages_image_delta(processor, image_list, description_list, resize_image=True) diff --git a/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_search_tools.py b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_search_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..c807624f7413a2d57e040d99c6d5c56e6c3a9909 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_search_tools.py @@ -0,0 +1,425 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from tests/workers/rollout/test_sglang_async_rollout_sf_tools.py + + +import asyncio +import os +from copy import deepcopy +from unittest.mock import AsyncMock, MagicMock, patch + +import numpy as np +import pytest +from tensordict import TensorDict +from transformers import AutoConfig, AutoTokenizer +from utils_sglang import get_rollout_config, prepare_inputs + +from verl.protocol import DataProto +from verl.tools.schemas import ( + OpenAIFunctionParametersSchema, + OpenAIFunctionPropertySchema, + OpenAIFunctionSchema, + OpenAIFunctionToolSchema, + ToolResponse, +) +from verl.tools.search_tool import SearchTool +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.schemas import AsyncRolloutRequest, AsyncRolloutRequestStateEnum, Message +from verl.workers.rollout.sglang_rollout.sglang_rollout import SGLangRollout + +DEFAULT_USER_CONTENT_PREFIX = ( + "Answer the given question. You must conduct reasoning inside and " + "first every time you get new information. After reasoning, if you find you lack " + "some knowledge, you can call a search engine by query " + "and it will return the top searched results between and " + ". You can search as many times as your want. If you find no " + "further external knowledge needed, you can directly provide the answer inside " + " and , without detailed illustrations. For example, " + " Beijing . Question: " +) +user_content = DEFAULT_USER_CONTENT_PREFIX.rstrip("\n") + "How's the weather lately?" + + +def get_search_messages(): + user_prompt = { + "role": "user", + "content": user_content, + } + + expect_turn_0_msg = { + "role": "assistant", + "content": "Let me search the web.", + "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "today's weather"}}}], + } + + expect_turn_1_msg = { + "role": "assistant", + "content": "Let me search again.", + "tool_calls": [ + {"type": "function", "function": {"name": "search", "arguments": {"query": "tomorrow's weather"}}} + ], + } + + expect_turn_2_msg = { + "role": "assistant", + "content": "Today is sunny and tomorrow will be cloudy in Beijing.", + } + + # Mock search tool responses + tool_return_0_msg = {"role": "tool", "content": "Today's weather in Beijing is sunny."} + tool_return_1_msg = {"role": "tool", "content": "Tomorrow's weather in Beijing is cloudy."} + + user_prompts = [user_prompt] + expect_turn_array = [expect_turn_0_msg, expect_turn_1_msg, expect_turn_2_msg] + tool_return_array = [tool_return_0_msg, tool_return_1_msg] + + return user_prompts, expect_turn_array, tool_return_array + + +class TestRolloutWithSearchTools: + local_model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B") + + @pytest.fixture + def qwen_tokenizer(self): + tokenizer = AutoTokenizer.from_pretrained(self.local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + return tokenizer + + # we only need this for tokenizer + @pytest.fixture + def qwen_model_config(self): + config = AutoConfig.from_pretrained(self.local_model_path) + return config + + @pytest.fixture + def search_data(self, qwen_tokenizer): + user_prompt, expect_turn_array, tool_return_array = get_search_messages() + prompts = [[message] for message in user_prompt] + preencode_turn_array = [ + qwen_tokenizer.apply_chat_template([turn], tokenize=False, add_generation_prompt=False) + for turn in expect_turn_array + ] + preencode_tool_return_array = [ + ToolResponse(text=qwen_tokenizer.apply_chat_template([turn], tokenize=False, add_generation_prompt=True)) + for turn in tool_return_array + ] + return prompts, preencode_turn_array, preencode_tool_return_array + + @pytest.fixture + def search_rollout_config(self): + max_prompt_length = 4096 + max_response_length = 3000 + dtype = "bfloat16" + tensor_parallel_size = 1 + tool_path = "./resource/tool_configs/search_tool_config" + rollout_config = get_rollout_config( + max_response_length, max_prompt_length, dtype, tensor_parallel_size, tool_path + ) + return rollout_config + + @pytest.fixture + def search_data_proto(self, search_data, qwen_tokenizer): + preencode_prompts, _, _ = search_data + prompts = [ + qwen_tokenizer.apply_chat_template(message, tokenize=False, add_generation_prompt=True) + for message in preencode_prompts + ] + input_ids, attention_mask, position_ids = prepare_inputs(qwen_tokenizer, prompts, 1000) + prompt_dict = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=input_ids.shape[0], + ) + messages = np.asarray(preencode_prompts) + + tools_kwargs = np.array( + [ + { + "search": { + "create_kwargs": { + "ground_truth": "Today is sunny and tomorrow will be cloudy in Beijing.", + "data_source": "searchR1_nq", + }, + }, + } + ], + dtype=object, + ) + index = np.array([0], dtype=object) + prompts = DataProto( + batch=prompt_dict, non_tensor_batch={"raw_prompt": messages, "tools_kwargs": tools_kwargs, "index": index} + ) + return prompts + + @pytest.fixture + def mock_rollout(self, search_rollout_config, qwen_tokenizer, qwen_model_config): + """Mock the rollout instance with sampling_params initialized.""" + with ( + patch.object(SGLangRollout, "_init_distributed_env", return_value=None), + patch.object(SGLangRollout, "_init_inference_engine", return_value=None), + patch.object(SGLangRollout, "_init_sampling_params", return_value=None), + ): + rollout_config: RolloutConfig = omega_conf_to_dataclass(search_rollout_config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + rollout.sampling_params = { + "n": 1, + "max_new_tokens": search_rollout_config.response_length, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "repetition_penalty": 1.0, + } + return rollout + + @patch.object(SGLangRollout, "_init_distributed_env", return_value=None) + @patch.object(SGLangRollout, "_init_inference_engine", return_value=None) + @patch.object(SGLangRollout, "_init_sampling_params", return_value=None) + def test_tools_registration( + self, mock_env, mock_engine, mock_sampling, search_rollout_config, qwen_tokenizer, qwen_model_config + ): + rollout_config: RolloutConfig = omega_conf_to_dataclass(search_rollout_config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + assert len(rollout._tool_schemas) == 1 + assert "search" in rollout._tool_map.keys() + from verl.tools.search_tool import SearchTool + + assert isinstance(rollout._tool_map["search"], SearchTool) + # depend on the tokenizer + assert rollout._tool_call_parser_type == "qwen25" + + @patch.object(SGLangRollout, "_init_distributed_env", return_value=None) + @patch.object(SGLangRollout, "_init_inference_engine", return_value=None) + @patch.object(SGLangRollout, "_init_sampling_params", return_value=None) + def test_rollout_req_creation( + self, + mock_env, + mock_engine, + mock_sampling, + search_rollout_config, + qwen_tokenizer, + qwen_model_config, + search_data_proto, + ): + rollout_config: RolloutConfig = omega_conf_to_dataclass(search_rollout_config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + req_list = rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1) + assert len(req_list) == 1 + assert req_list[0].state == AsyncRolloutRequestStateEnum.PENDING + assert len(req_list[0].tool_schemas) == 1 + print(type(req_list[0].tool_schemas[0])) + assert req_list[0].tool_schemas[0] == OpenAIFunctionToolSchema( + type="function", + function=OpenAIFunctionSchema( + name="search", + description="Searches the web for relevant information based on the given query.", + parameters=OpenAIFunctionParametersSchema( + type="object", + properties={ + "query_list": OpenAIFunctionPropertySchema( + type="array", + description="A list of fully-formed semantic queries. The tool will return search " + "results for each query.", + items={"type": "string"}, + ) + }, + required=["query_list"], + ), + strict=False, + ), + ) + + def test_over_size_case(self, mock_rollout, search_data_proto, search_data): + mock_rollout.config.multi_turn.max_assistant_turns = 1 + req = mock_rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1)[0] + req = MagicMock(wraps=req, spec=AsyncRolloutRequest) + req.finalize = MagicMock() + req_list = [req] + + _, expect_turn_array, _ = search_data + mock_rollout._handle_engine_call = MagicMock() + future = asyncio.Future() + future.set_result( + { + "text": expect_turn_array[0], + "meta_info": { + "id": "d1188d81cba840359df5b352b344bc8e", + "finish_reason": {"type": "length", "length": 3000}, + "prompt_tokens": 132, + "completion_tokens": 100, + "cached_tokens": 0, + "e2e_latency": 2.23543, + }, + } + ) + mock_rollout._handle_engine_call.return_value = future + mock_rollout._tp_rank = 0 + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather( + *[mock_rollout._async_rollout_a_request(req, True, False) for req in req_list], + ) + ) + assert len(output_req_list) == 1 + output_req = output_req_list[0] + assert output_req.state == AsyncRolloutRequestStateEnum.COMPLETED + assert output_req.reward_scores.get("search") == [] + assert len(output_req.messages) == 2 + assert output_req.messages[1] == Message( + role="assistant", + content=expect_turn_array[0], + tool_calls=None, + ) + + @patch.object(SearchTool, "execute", new_callable=AsyncMock) + def test_tool_call_basic_case(self, mock_execute, mock_rollout, search_data_proto, search_data): + _, expect_turn_array, tool_return_array = search_data + + # Mock search tool execution to return predefined responses + mock_execute.side_effect = [(msg, 0.0, {"status": "success"}) for msg in tool_return_array] + + mock_rollout.config.multi_turn.max_assistant_turns = 10 + mock_rollout._tool_map["search"].retrieval_service_url = "mock://dummy" + + req = mock_rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1)[0] + req = MagicMock(wraps=req, spec=AsyncRolloutRequest) + req.finalize = MagicMock() + req_list = [req] + + mock_rollout._handle_engine_call = MagicMock() + futures = [asyncio.Future() for i in expect_turn_array] + for idx, (i, turn) in enumerate(zip(futures, expect_turn_array, strict=True)): + i.set_result( + { + "text": turn, + "meta_info": { + "id": "d1188d81cba840359df5b352b344bc8e", + "finish_reason": {"type": "tool_calls" if idx < len(expect_turn_array) - 1 else "stop"}, + "prompt_tokens": len(turn), + "completion_tokens": 100, + "cached_tokens": 0, + "e2e_latency": 2.23543, + }, + } + ) + if idx < len(expect_turn_array) - 1: + assert mock_rollout._function_call_parser.has_tool_call(turn) + assert mock_rollout._function_call_parser.parse_non_stream(turn) + + mock_rollout._handle_engine_call.side_effect = futures + mock_rollout._tp_rank = 0 + + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather(*[mock_rollout._async_rollout_a_request(req, True, False) for req in req_list]) + ) + + # Verify conversation completed successfully with proper tool usage + output_req = output_req_list[0] + assert output_req.state == AsyncRolloutRequestStateEnum.COMPLETED + assert "search" in output_req.metrics + assert output_req.metrics["search"][0]["status"] == "success" + assert mock_execute.await_count == 2 + assert len(output_req.messages) == 6 # user + 3*assistant + 2*tool_call + # Verify tool response messages contain expected content + search_counter = 0 + for msg in output_req.messages: + if msg.role == "tool": + assert msg.content == tool_return_array[search_counter].text + search_counter += 1 + assert search_counter == 2 + + @patch.object(SearchTool, "execute", new_callable=AsyncMock) + def test_tool_call_batch_case(self, mock_execute, mock_rollout, search_data_proto, search_data): + _, expect_turn_array, tool_return_array = search_data + + # Mock tool execution for large batch (100 requests * 2 calls each) + mock_execute.side_effect = [ + (tool_return_array[0], 0.0, {"status": "success"}), + (tool_return_array[1], 0.0, {"status": "success"}), + ] * 100 + + mock_rollout.config.multi_turn.max_assistant_turns = 10 + mock_rollout._tool_map["search"].retrieval_service_url = "mock://dummy" + + base_req = mock_rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1)[0] + + req_nums = 100 + req_list = [] + req_turns_map = {} + req_turns_counter = {} + + for i in range(req_nums): + tmp_req = deepcopy(base_req) + tmp_req.batch_data_id = i + tmp_req.request_id = i + req_list.append(MagicMock(wraps=tmp_req, spec=AsyncRolloutRequest)) + + futures = [asyncio.Future() for _ in expect_turn_array] + for idx, (fut, turn) in enumerate(zip(futures, expect_turn_array, strict=True)): + fut.set_result( + { + "text": turn, + "meta_info": { + "id": "dummy", + "finish_reason": {"type": "tool_calls" if idx < len(expect_turn_array) - 1 else "stop"}, + "prompt_tokens": len(turn), + "completion_tokens": 100, + }, + } + ) + req_turns_map[i] = futures + req_turns_counter[i] = 0 + + async def hacked_handle_engine_call(self, _req: AsyncRolloutRequest, *_args, **_kwargs): + fut = req_turns_map[_req.batch_data_id][req_turns_counter[_req.batch_data_id]] + req_turns_counter[_req.batch_data_id] += 1 + return await fut + + with patch.object(SGLangRollout, "_handle_engine_call", new=hacked_handle_engine_call): + mock_rollout._tp_rank = 0 + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather(*[mock_rollout._async_rollout_a_request(r, True, False) for r in req_list]) + ) + + # Verify all requests completed successfully + assert len(output_req_list) == req_nums + for out_req in output_req_list: + assert out_req.state == AsyncRolloutRequestStateEnum.COMPLETED + assert "search" in out_req.metrics + for metric in out_req.metrics["search"]: + assert metric["status"] == "success" + assert len(out_req.messages) == 6 # user + 3 assistant + 2 tool + assert sum(1 for m in out_req.messages if m.role == "tool") == 2 + + assert mock_execute.await_count == 2 * req_nums diff --git a/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_sf_tools.py b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_sf_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..00ab8e6eaa4e43623cf0b4b7fb1847fb9840c388 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_sf_tools.py @@ -0,0 +1,665 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# noqa + +import os +import asyncio +import time +from copy import deepcopy +from functools import wraps +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest +import ray +from tensordict import TensorDict +from torch.testing._internal.common_distributed import MultiProcessTestCase +from transformers import AutoConfig, AutoTokenizer +from utils_sglang import ( + get_rollout_config, + prepare_inputs, +) + +from verl.protocol import DataProto +from verl.tools.sandbox_fusion_tools import TokenBucketWorker +from verl.tools.schemas import ( + OpenAIFunctionParametersSchema, + OpenAIFunctionPropertySchema, + OpenAIFunctionSchema, + OpenAIFunctionToolSchema, + ToolResponse, +) +from verl.workers.rollout.schemas import AsyncRolloutRequest, AsyncRolloutRequestStateEnum, Message +from verl.workers.rollout.sglang_rollout.sglang_rollout import SGLangRollout +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RolloutConfig + +sandbox_url = "" + + +def get_sandbox_fusion_messages(): + user_prompt = { + "role": "user", + "content": """ + Solve the following problem step by step. You now have the ability to selectively + write executable Python code to enhance your reasoning process. \n\n**user question:**\nThere + are 152 students at Dala High School. Assume the following: \n- 100 students take a Math class \n- 94 + students take a Science class \n- 57 students take an English class \n- 73 students take a Math class + and a Science class \n- 24 students take a Math class and an English class \n- 27 students take a Science + class and an English class \n- 22 students take a Math class and a Science class and an English class\n \nHow + many students take neither a Math class nor a Science class nor an Eglish class?\n\nRemember to place the final + answer in the last part using the format: \n\n\boxed{'The final answer goes here.'}\n + """, + } + expect_turn_0_msg = { + "role": "assistant", + "content": """ + Okay, so I need to find out how many students at Dala High School are not taking any of the three classes: Math, + Science, or English. The total number of students is 152. Let me see... I remember this is a problem about sets + and maybe using the principle of inclusion-exclusion. Let me recall how that works.\n\nFirst, the inclusion-exclusion + principle for three sets says that the total number of students taking at least one of the classes is equal to the + sum of the numbers in each individual class, minus the sum of the numbers in each pair of classes, plus the number in + all three classes. Then, subtract that total from the overall number of students to get those not taking any of the + three. \n\nLet me write that down step by step. Let M be the set of students taking Math, S for Science, and E for English. + Then:\n\nTotal in at least one class = |M ∪ S ∪ E| = |M| + |S| + |E| - |M ∩ S| - |M ∩ E| - |S ∩ E| + |M ∩ S ∩ E|\n\nGiven the + numbers:\n\n|M| = 100\n\n|S| = 94\n\n|E| = 57\n\n|M ∩ S| = 73\n\n|M ∩ E| = 24\n\n|S ∩ E| = 27\n\n|M ∩ S ∩ E| = 22\n\nSo plugging + these into the formula:\n\nTotal = 100 + 94 + 57 - 73 - 24 - 27 + 22\n\nLet me compute that step by step using code to ensure + accuracy.\n + """, + "tool_calls": [ + { + "type": "function", + "function": { + "name": "code_interpreter", + "arguments": { + "code": """M = 100\\nS = 94\\nE = 57\\nM_S = 73\\nM_E = 24\\nS_E = 27\\nM_S_E = + 22\\n\\ntotal_in_any = M + S + E - M_S - M_E - S_E + M_S_E\\nstudents_neither = 152 - total_in_any\\nprint(students_neither)""", + }, + }, + } + ], + } + + tool_return_0_msg = { + "role": "tool", + "content": """3""", + } + + expect_turn_1_msg = { + "role": "assistant", + "content": """ + The code returns 3, indicating that 3 students take none of the classes. To verify, I'll break down each + category to ensure the total matches:\n\n- Only Math and Science: 73 - 22 = 51\n- Only Math and English: + 24 - 22 = 2\n- Only Science and English: 27 - 22 = 5\n- Only Math: 100 - 51 - 2 - 22 = 25\n- Only Science: + 94 - 51 - 5 - 22 = 16\n- Only English: 57 - 2 - 5 - 22 = 28\n\nSumming all categories:\n + """, + "tool_calls": [ + { + "type": "function", + "function": { + "name": "code_interpreter", + "arguments": { + "code": """only_M_S = 73 - 22\\nonly_M_E = 24 - 22\\nonly_S_E = 27 - 22\\n\\nonly_M = 100 - only_M_S - + only_M_E - 22\\nonly_S = 94 - only_M_S - only_S_E - 22\\nonly_E = 57 - only_M_E - only_S_E - 22\\n\\ntotal_verify + = only_M + only_S + only_E + only_M_S + only_M_E + only_S_E + 22\\nprint(total_verify)""", + }, + }, + } + ], + } + + tool_return_1_msg = { + "role": "tool", + "content": """149""", + } + expect_turn_2_msg = { + "role": "assistant", + "content": """ + The verification total is 149, so students not taking any classes are 152 - 149 = 3, confirming the initial + result.\n\n\n\\boxed{3}\n + """, + } + + user_prompts = [user_prompt] + expect_turn_array = [expect_turn_0_msg, expect_turn_1_msg, expect_turn_2_msg] + tool_return_array = [tool_return_0_msg, tool_return_1_msg] + + return user_prompts, expect_turn_array, tool_return_array + + +def skip_if_valid_sandbox(url): + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + if url == "" or url is None: + pytest.skip("No valid sandbox url provided") + + return wrapper + + return decorator + + +class TestRolloutWithTools: + local_model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B") + + @pytest.fixture + def qwen_tokenizer(self): + tokenizer = AutoTokenizer.from_pretrained(self.local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + return tokenizer + + # we only need this for tokenizer + @pytest.fixture + def qwen_model_config(self): + config = AutoConfig.from_pretrained(self.local_model_path) + return config + + @pytest.fixture + def sandbox_fusion_data(self, qwen_tokenizer): + user_prompt, expect_turn_array, tool_return_array = get_sandbox_fusion_messages() + prompts = [[message] for message in user_prompt] + preencode_turn_array = [ + qwen_tokenizer.apply_chat_template([turn], tokenize=False, add_generation_prompt=False) + for turn in expect_turn_array + ] + preencode_tool_return_array = [ + ToolResponse(text=qwen_tokenizer.apply_chat_template([turn], tokenize=False, add_generation_prompt=True)) + for turn in tool_return_array + ] + return prompts, preencode_turn_array, preencode_tool_return_array + + @pytest.fixture + def sandbox_fusion_rollout_config(self): + max_prompt_length = 1024 + max_response_length = 1024 + dtype = "bfloat16" + tensor_parallel_size = 1 + tool_path = "./resource/tool_configs/sandbox_fusion_tool_config" + rollout_config = get_rollout_config( + max_response_length, max_prompt_length, dtype, tensor_parallel_size, tool_path + ) + return rollout_config + + @pytest.fixture + def sandbox_data_proto(self, sandbox_fusion_data, qwen_tokenizer): + preencode_prompts, _, _ = sandbox_fusion_data + prompts = [ + qwen_tokenizer.apply_chat_template(message, tokenize=False, add_generation_prompt=True) + for message in preencode_prompts + ] + input_ids, attention_mask, position_ids = prepare_inputs(qwen_tokenizer, prompts, 1000) + prompt_dict = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=input_ids.shape[0], + ) + messages = np.asarray(preencode_prompts) + tools_kwargs = np.array( + [ + { + "code_interpreter": { + "create_kwargs": {"ground_truth": "test-solution-str"}, + }, + } + ], + dtype=object, + ) + index = np.array([0], dtype=object) + prompts = DataProto( + batch=prompt_dict, non_tensor_batch={"raw_prompt": messages, "tools_kwargs": tools_kwargs, "index": index} + ) + return prompts + + @pytest.fixture + def mock_rollout(self, sandbox_fusion_rollout_config, qwen_tokenizer, qwen_model_config): + """Mock the rollout instance""" + with patch.object(SGLangRollout, "_init_distributed_env", return_value=None), patch.object( + SGLangRollout, "_init_inference_engine", return_value=None + ), patch.object(SGLangRollout, "_init_sampling_params", return_value=None): + rollout_config: RolloutConfig = omega_conf_to_dataclass(sandbox_fusion_rollout_config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + # set default sampling_params + rollout.sampling_params = { + "n": 1, + "max_new_tokens": sandbox_fusion_rollout_config.response_length, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "repetition_penalty": 1.0, + } + return rollout + + def test_tools_registration(self, mock_rollout): + """Test tool registration functionality""" + assert len(mock_rollout._tool_schemas) == 1 + assert "code_interpreter" in mock_rollout._tool_map.keys() + from verl.tools.sandbox_fusion_tools import SandboxFusionTool + + assert isinstance(mock_rollout._tool_map["code_interpreter"], SandboxFusionTool) + assert mock_rollout._tool_call_parser_type == "qwen25" + + def test_rollout_req_creation(self, mock_rollout, sandbox_data_proto): + """Test request creation functionality""" + req_list = mock_rollout._preprocess_prompt_to_async_rollout_requests(sandbox_data_proto, n=1) + assert len(req_list) == 1 + assert req_list[0].state == AsyncRolloutRequestStateEnum.PENDING + assert len(req_list[0].tool_schemas) == 1 + print(type(req_list[0].tool_schemas[0])) + assert req_list[0].tool_schemas[0] == OpenAIFunctionToolSchema( + type="function", + function=OpenAIFunctionSchema( + name="code_interpreter", + description="A tool for executing code.", + parameters=OpenAIFunctionParametersSchema( + type="object", + properties={ + "code": OpenAIFunctionPropertySchema( + type="string", + description="The code to execute.", + enum=None, + ) + }, + required=["code"], + ), + strict=False, + ), + ) + + def test_over_size_case(self, mock_rollout, sandbox_data_proto, sandbox_fusion_data): + """Test over-size response truncation case""" + mock_rollout.config.multi_turn.max_assistant_turns = 1 + req = mock_rollout._preprocess_prompt_to_async_rollout_requests(sandbox_data_proto, n=1)[0] + req = MagicMock(wraps=req, spec=AsyncRolloutRequest) + req.finalize = MagicMock() + req_list = [req] + + _, expect_turn_array, tool_return_array = sandbox_fusion_data + # here we mock a meta info with 'length'. indicate the response is truncate + mock_rollout._handle_engine_call = MagicMock() + future = asyncio.Future() + future.set_result( + { + "text": expect_turn_array[0], + "meta_info": { + "id": "d1188d81cba840359df5b352b344bc8e", + "finish_reason": {"type": "length", "length": 1024}, + "prompt_tokens": 132, + "completion_tokens": 100, + "cached_tokens": 0, + "e2e_latency": 9.9304039478302, + }, + } + ) + mock_rollout._handle_engine_call.return_value = future + mock_rollout._tp_rank = 0 + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather( + *[mock_rollout._async_rollout_a_request(req, True, False) for req in req_list], + ) + ) + assert len(output_req_list) == 1 + output_req = output_req_list[0] + assert output_req.state == AsyncRolloutRequestStateEnum.COMPLETED + assert output_req.reward_scores.get("code_interpreter") == [] + # we should only have two message, one for prompt, second for response. + assert len(output_req.messages) == 2 + assert output_req.messages[1] == Message( + role="assistant", + content=expect_turn_array[0], + tool_calls=None, + ) + + @skip_if_valid_sandbox(sandbox_url) + def test_tool_call_basic_case(self, mock_rollout, sandbox_data_proto, sandbox_fusion_data): + """Test basic tool call case""" + mock_rollout.config.multi_turn.max_assistant_turns = 10 + mock_rollout._tool_map["code_interpreter"].sandbox_fusion_url = sandbox_url + req = mock_rollout._preprocess_prompt_to_async_rollout_requests(sandbox_data_proto, n=1)[0] + req = MagicMock(wraps=req, spec=AsyncRolloutRequest) + req.finalize = MagicMock() + req_list = [req] + _, expect_turn_array, tool_return_array = sandbox_fusion_data + # here we mock a meta info with 'length'. indicate the response is truncate + mock_rollout._handle_engine_call = MagicMock() + futures = [asyncio.Future() for i in expect_turn_array] + for idx, (i, turn) in enumerate(zip(futures, expect_turn_array)): + i.set_result( + { + "text": turn, + "meta_info": { + "id": "d1188d81cba840359df5b352b344bc8e", + "finish_reason": {"type": "tool_calls" if idx < len(expect_turn_array) - 1 else "stop"}, + "prompt_tokens": len(turn), + "completion_tokens": 100, + "cached_tokens": 0, + "e2e_latency": 9.9304039478302, + }, + } + ) + if idx < len(expect_turn_array) - 1: + assert mock_rollout._function_call_parser.has_tool_call(turn) + assert mock_rollout._function_call_parser.parse_non_stream(turn) + + mock_rollout._handle_engine_call.side_effect = futures + mock_rollout._tp_rank = 0 + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather( + *[mock_rollout._async_rollout_a_request(req, True, False) for req in req_list], + ) + ) + assert len(output_req_list) == 1 + output_req = output_req_list[0] + assert output_req.state == AsyncRolloutRequestStateEnum.COMPLETED + # here we verify whether the code sandbox is executed correctly + assert output_req.metrics == {"code_interpreter": ["3", "149"]} + assert mock_rollout._handle_engine_call.call_count == 3 + assert len(output_req.messages) == 6 # user + 3*assistant + 2*tool_call + code_counter = 0 + for msg in output_req.messages: + if msg.role == "tool": + code_counter += 1 + assert msg.content == tool_return_array[code_counter] + assert code_counter == 2 + + @skip_if_valid_sandbox(sandbox_url) + def test_tool_call_batch_case(self, mock_rollout, sandbox_data_proto, sandbox_fusion_data): + """Test batch tool call case""" + mock_rollout.config.multi_turn.max_assistant_turns = 10 + mock_rollout._tool_map["code_interpreter"].sandbox_fusion_url = sandbox_url + req = mock_rollout._preprocess_prompt_to_async_rollout_requests(sandbox_data_proto, n=1)[0] + req_nums = 100 + req_list = [] + req_turns_counter = {} + # this map should a Map[id:List[Futures]] + req_turns_map = {} + _, expect_turn_array, tool_return_array = sandbox_fusion_data + for i in range(req_nums): + _temp_req = deepcopy(req) + _temp_req.batch_data_id = i + _temp_req.request_id = i + req_list.append(MagicMock(wraps=_temp_req, spec=AsyncRolloutRequest)) + futures = [asyncio.Future() for i in expect_turn_array] + for idx, (i, turn) in enumerate(zip(futures, expect_turn_array)): + i.set_result( + { + "text": turn, + "meta_info": { + "id": "d1188d81cba840359df5b352b344bc8e", + "finish_reason": {"type": "tool_calls" if idx < len(expect_turn_array) - 1 else "stop"}, + "prompt_tokens": len(turn), + "completion_tokens": 100, + "cached_tokens": 0, + "e2e_latency": 9.9304039478302, + }, + } + ) + if idx < len(expect_turn_array) - 1: + assert mock_rollout._function_call_parser.has_tool_call(turn) + assert mock_rollout._function_call_parser.parse_non_stream(turn) + req_turns_map[_temp_req.batch_data_id] = futures + req_turns_counter[_temp_req.batch_data_id] = 0 + + async def hacked_handle_engine_call( + self, _req: AsyncRolloutRequest, do_sample: bool, is_validate: bool, **kwargs + ): + result = req_turns_map[_req.batch_data_id][req_turns_counter[_req.batch_data_id]] + req_turns_counter[_req.batch_data_id] += 1 + re = await result + return re + + with patch.object(SGLangRollout, "_handle_engine_call", new=hacked_handle_engine_call): + mock_rollout._tp_rank = 0 + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather( + *[mock_rollout._async_rollout_a_request(req, True, False) for req in req_list], + ) + ) + assert len(output_req_list) == req_nums + # FIGUER out how to count this + # assert rollout._handle_engine_call.call_count == 3 * req_nums + for output_req in output_req_list: + assert output_req.state == AsyncRolloutRequestStateEnum.COMPLETED + # here we verify whether the code sandbox is executed correctly + assert output_req.metrics == {"code_interpreter": ["3", "149"]} + assert len(output_req.messages) == 6 # user + 3*assistant + 2*tool_call + code_counter = 0 + for msg in output_req.messages: + if msg.role == "tool": + code_counter += 1 + assert code_counter == 2 + + def test_sampling_params_functionality(self, mock_rollout): + """Test sampling_params functionality""" + # test basic copy functionality + copied_params = mock_rollout.sampling_params.copy() + assert copied_params == mock_rollout.sampling_params + assert copied_params is not mock_rollout.sampling_params + + # test parameter update + copied_params.update({"temperature": 0.8, "top_p": 0.9}) + assert copied_params["temperature"] == 0.8 + assert copied_params["top_p"] == 0.9 + + # ensure original parameters are not modified + assert "temperature" not in mock_rollout.sampling_params + assert "top_p" not in mock_rollout.sampling_params + + +class RayMultiProcessTestCase(MultiProcessTestCase): + def setUp(self): + super().setUp() + ray.init(ignore_reinit_error=True) + print("init_single cluster") + self._spawn_processes() + + def tearDown(self): + print("tearDown_single cluster") + ray.shutdown() + + +@ray.remote +class TestActor: + def __init__(self, rank, world_size): + self._world_size = world_size + self._rank = rank + self.rank_list = [] + self.time_list = [] + + def record_rank(self, rank): + self.rank_list.append(rank) + + def get_rank(self): + return self._rank + + def ping(self): + return True + + def record_execution_time(self, time): + self.time_list.append(time) + + def get_time(self, timeout): + import time + + now = time.time() + while time.time() - now < timeout: + # for start and end time + if len(self.time_list) == self._world_size * 2: + self.time_list.sort() + return self.time_list[-1] - self.time_list[0] + else: + time.sleep(1) + continue + return False + + def verify_rank(self): + import time + + now = time.time() + while time.time() - now < 10: + if len(self.rank_list) == self._world_size: + print(self.rank_list) + self.rank_list.sort() + for i in range(self._world_size): + if self.rank_list[i] != i: + return False + return True + else: + time.sleep(1) + continue + return False + + +class TestRayGlobalActorCase(RayMultiProcessTestCase): + @property + def world_size(self) -> int: + # for DP = 8 + return 2 + + def test_basic_multi_process_init(self): + ray.init("auto", namespace="test", ignore_reinit_error=True) + handle = TestActor.remote(self.rank, self.world_size) + re = ray.get(handle.get_rank.remote()) + assert re == self.rank, f"rank not match: {re} != {self.rank}" + + # def test_global_actor(self): + # ray.init("auto",namespace="test",ignore_reinit_error=True) + # handle = TestActor.options(get_if_exists=True,name="test-actor").remote(self.rank,self.world_size) + # handle.record_rank.remote(self.rank) + # # since test actor's concurrency is 1, we need to wait for all processes to finish + # time.sleep(5) + # assert ray.get(handle.ping.remote()) == True # make sure actor handle is valid + # if self.rank == 0: + # assert ray.get(handle.verify_rank.remote()) == True + # else: + # # get_actor use weak_ref, so we need to make sure the actor is not garbage collected + # time.sleep(10) + + +class TestSingleNodeRateLimiterCase(RayMultiProcessTestCase): + @property + def world_size(self) -> int: + return 1 + + def test_rate_limiter(self): + ray.init("auto", namespace="test", ignore_reinit_error=True) + from verl.tools.sandbox_fusion_tools import PoolMode, init_execution_pool + + # exec_worker = ExecutionWorker.options(max_concurrency=10).remote(enable_global_rate_limit=True, rate_limit=3) + exec_worker = init_execution_pool( + num_workers=10, enable_global_rate_limit=True, rate_limit=3, mode=PoolMode.ThreadMode + ) + center = TestActor.options(get_if_exists=True, name="test-actor").remote(self.rank, self.world_size) + ray.get(exec_worker.ping.remote()) + + def fn(i): + import time + + time.sleep(3) + return i + + start = time.time() + tasks = [exec_worker.execute.remote(fn, i) for i in range(6)] + loop = asyncio.get_event_loop() + results = loop.run_until_complete(asyncio.gather(*tasks)) + end = time.time() + duration = end - start + center.record_execution_time.remote(start) + center.record_execution_time.remote(end) + print(f"Total time: {duration:.2f} seconds for rank: {self.rank}") + + assert results == list(range(6)) + # we have 6 task with rate limit of 3, therefore we need at least 2 round: 3*2=6 seconds + assert duration > 6 + assert duration < 10 + + def test_rotten_execution(self): + ray.init("auto", namespace="test", ignore_reinit_error=True) + from verl.tools.sandbox_fusion_tools import PoolMode, init_execution_pool + + # exec_worker = ExecutionWorker.options(max_concurrency=10).remote(enable_global_rate_limit=True, rate_limit=6) + exec_worker = init_execution_pool( + num_workers=10, enable_global_rate_limit=True, rate_limit=6, mode=PoolMode.ThreadMode + ) + ray.get(exec_worker.ping.remote()) + + def fn(i): + if i == 10: + raise Exception("test") + else: + return i + + tasks = [exec_worker.execute.remote(fn, i) for i in range(20)] + loop = asyncio.get_event_loop() + results = loop.run_until_complete(asyncio.gather(*tasks)) + expect_result = [None] + list(range(10)) + list(range(11, 20)) + sorted_data = sorted(results, key=lambda x: (x is not None, x)) + assert sorted_data == expect_result, f"results: {results}, expect_result: {expect_result}" + rate_limiter = TokenBucketWorker.options(name="rate-limiter", get_if_exists=True).remote() + rate = ray.get(rate_limiter.get_current_count.remote()) + assert rate == 0, f"rate: {rate}" + + +class TestMultiNodeRateLimiterCase(RayMultiProcessTestCase): + @property + def world_size(self) -> int: + return 2 + + def test_rate_limiter(self): + ray.init("auto", namespace="test", ignore_reinit_error=True) + from verl.tools.sandbox_fusion_tools import PoolMode, init_execution_pool + + # exec_worker = ExecutionWorker.options(max_concurrency=10).remote(enable_global_rate_limit=True, rate_limit=6) + exec_worker = init_execution_pool( + num_workers=10, enable_global_rate_limit=True, rate_limit=6, mode=PoolMode.ThreadMode + ) + center = TestActor.options(get_if_exists=True, name="test-actor").remote(self.rank, self.world_size) + ray.get(exec_worker.ping.remote()) + + def fn(i): + import time + + time.sleep(2) + return i + + start = time.time() + tasks = [exec_worker.execute.remote(fn, i) for i in range(6)] + loop = asyncio.get_event_loop() + results = loop.run_until_complete(asyncio.gather(*tasks)) + end = time.time() + duration = end - start + center.record_execution_time.remote(start) + center.record_execution_time.remote(end) + print(f"Total time: {duration:.2f} seconds for rank: {self.rank}") + assert results == list(range(6)) + time.sleep(5) + if self.rank == 0: + total_cost = ray.get(center.get_time.remote(10)) + print(f"for total cost: {total_cost}") + # # we have 6 task each node * 2node = 12 task, each task take 2 second. + # with rate limit of 6, + # therefore we need at least 2 round: 12/6*2=4 seconds + assert total_cost > 4, total_cost + else: + time.sleep(10) diff --git a/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_interaction.py b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..fca0cb9448993c3af44e957116d74e2aa75d2e75 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_interaction.py @@ -0,0 +1,149 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +usage: torchrun --standalone --nnodes=1 \ + --nproc_per_node=2 $(which pytest) \ + -s test_sglang_async_rollout_w_interaction.py +""" + +import numpy as np +import torch +from tensordict import TensorDict +from utils_sglang import ( + are_lists_similar, + clean_torchelastic_env, + generate_hf_output, + get_rollout_config, + initialize_global_process_group, + load_tokenizer_and_model, + prepare_inputs, +) + +from verl import DataProto +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.sglang_rollout.sglang_rollout import SGLangRollout + + +def test_async_sglang_rollout_w_interaction(): + import os + + assert torch.cuda.device_count() >= 2 + initialize_global_process_group() + clean_torchelastic_env() + + max_prompt_length = 32 + max_response_length = 16 + dtype = "bfloat16" + tensor_parallel_size = 2 + local_model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B") + + tokenizer, actor_model = load_tokenizer_and_model(local_model_path) + + preencode_prompts = [ + [{"role": "user", "content": prompt, "tool_calls": None}] + for prompt in [ + "Who won the Champions League in 2019?", + "The founder of Apple is", + "What's the best way to learn python?", + ] + ] + interaction_kwargs = [ + {"name": "gsm8k", "query": "Who won the Champions League in 2019?", "ground_truth": "Real Madrid"}, + {"name": "gsm8k", "query": "The founder of Apple is", "ground_truth": "Steve Jobs"}, + {"name": "gsm8k", "query": "What's the best way to learn python?", "ground_truth": "Learn python from scratch"}, + ] + prompts = [ + tokenizer.apply_chat_template(message, tokenize=False, add_generation_prompt=True) + for message in preencode_prompts + ] + input_ids, attention_mask, position_ids = prepare_inputs(tokenizer, prompts, max_prompt_length) + + hf_response_tokens = generate_hf_output(actor_model, input_ids, attention_mask, tokenizer, max_response_length) + + # Create a temporary interaction config file for testing + import tempfile + + from omegaconf import OmegaConf + + interaction_config = { + "interaction": [ + {"name": "gsm8k", "class_name": "verl.interactions.gsm8k_interaction.Gsm8kInteraction", "config": {}} + ] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(interaction_config, f.name) + interaction_config_path = f.name + + rollout_config = get_rollout_config( + max_response_length, max_prompt_length, dtype, tensor_parallel_size, None, interaction_config_path + ) + rollout_config: RolloutConfig = omega_conf_to_dataclass(rollout_config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + + prompt_dict = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=input_ids.shape[0], + ) + print(f"preprocessed {input_ids.shape=}") + + messages = np.asarray(preencode_prompts) + prompts = DataProto( + batch=prompt_dict, + non_tensor_batch={"raw_prompt": messages, "interaction_kwargs": np.asarray(interaction_kwargs)}, + ) + + prompts.meta_info.update( + { + "eos_token_id": tokenizer.eos_token_id, + "pad_token_id": tokenizer.pad_token_id, + } + ) + + # log_gpu_memory_usage("Before generating sequences", logger=None) + output = rollout.generate_sequences(prompts=prompts) + print(f"generated {output.batch['responses'].shape=}") + # log_gpu_memory_usage("After generating sequences", logger=None) + + sglang_output = output.to("cpu") + + sglang_response_tokens = tokenizer.batch_decode(sglang_output.batch["responses"]) + + print(f"hf response: {hf_response_tokens}") + print(f"sglang response: {sglang_response_tokens}") + assert are_lists_similar(hf_response_tokens, sglang_response_tokens) + print("SGLang w interaction Test Passed!") + + # Clean up temporary config file + import os + + os.unlink(interaction_config_path) + + torch.distributed.barrier() + torch.distributed.destroy_process_group() + + +if __name__ == "__main__": + test_async_sglang_rollout_w_interaction() diff --git a/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_tools.py b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..e5f7d22ed78bd70ed07d1618d39f60cb05b729d3 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_tools.py @@ -0,0 +1,131 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +usage: torchrun --standalone --nnodes=1 \ + --nproc_per_node=2 $(which pytest) \ + -s test_sglang_async_rollout_w_tools.py +""" + +import numpy as np +import torch +from tensordict import TensorDict +from utils_sglang import ( + are_lists_similar, + clean_torchelastic_env, + generate_hf_output, + get_rollout_config, + initialize_global_process_group, + load_tokenizer_and_model, + prepare_inputs, +) + +from verl import DataProto +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.sglang_rollout.sglang_rollout import SGLangRollout + + +def test_async_sglang_rollout_w_tool(): + import os + + assert torch.cuda.device_count() >= 2 + initialize_global_process_group() + clean_torchelastic_env() + + max_prompt_length = 32 + max_response_length = 16 + dtype = "bfloat16" + tensor_parallel_size = 2 + local_model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B") + + tokenizer, actor_model = load_tokenizer_and_model(local_model_path) + + preencode_prompts = [ + [{"role": "user", "content": prompt, "tool_calls": None}] + for prompt in [ + "Who won the Champions League in 2019?", + "The founder of Apple is", + "What's the best way to learn python?", + ] + ] + prompts = [ + tokenizer.apply_chat_template(message, tokenize=False, add_generation_prompt=True) + for message in preencode_prompts + ] + input_ids, attention_mask, position_ids = prepare_inputs(tokenizer, prompts, max_prompt_length) + + hf_response_tokens = generate_hf_output(actor_model, input_ids, attention_mask, tokenizer, max_response_length) + + rollout_config = get_rollout_config( + max_response_length, + max_prompt_length, + dtype, + tensor_parallel_size, + "./resource/tool_configs/sandbox_fusion_tool_config", + ) + rollout_config: RolloutConfig = omega_conf_to_dataclass(rollout_config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + + prompt_dict = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=input_ids.shape[0], + ) + print(f"preprocessed {input_ids.shape=}") + + messages = np.asarray(preencode_prompts) + prompts = DataProto( + batch=prompt_dict, + non_tensor_batch={ + "raw_prompt": messages, + "tools_kwargs": np.array([{}] * input_ids.shape[0], dtype=object), + }, + ) + + prompts.meta_info.update( + { + "eos_token_id": tokenizer.eos_token_id, + "pad_token_id": tokenizer.pad_token_id, + } + ) + + # log_gpu_memory_usage("Before generating sequences", logger=None) + output = rollout.generate_sequences(prompts=prompts) + print(f"generated {output.batch['responses'].shape=}") + # log_gpu_memory_usage("After generating sequences", logger=None) + + sglang_output = output.to("cpu") + + sglang_response_tokens = tokenizer.batch_decode(sglang_output.batch["responses"]) + + print(f"hf response: {hf_response_tokens}") + print(f"sglang response: {sglang_response_tokens}") + assert are_lists_similar(hf_response_tokens, sglang_response_tokens) + print("SGLang w tool Test Passed!") + + torch.distributed.barrier() + torch.distributed.destroy_process_group() + + +if __name__ == "__main__": + test_async_sglang_rollout_w_tool() diff --git a/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_tools_token_out.py b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_tools_token_out.py new file mode 100644 index 0000000000000000000000000000000000000000..e6a3efd450754f9caa2c49a561cf1f932fc1edeb --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_async_rollout_w_tools_token_out.py @@ -0,0 +1,133 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +usage: torchrun --standalone --nnodes=1 \ + --nproc_per_node=2 $(which pytest) \ + -s test_sglang_async_rollout_w_tools.py +""" + +import numpy as np +import torch +from tensordict import TensorDict +from utils_sglang import ( + are_lists_similar, + clean_torchelastic_env, + generate_hf_output, + get_rollout_config, + initialize_global_process_group, + load_tokenizer_and_model, + prepare_inputs, +) + +from verl import DataProto +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.sglang_rollout.sglang_rollout import SGLangRollout + + +def test_async_sglang_rollout_w_tool(): + import os + + assert torch.cuda.device_count() >= 2 + initialize_global_process_group() + clean_torchelastic_env() + + max_prompt_length = 32 + max_response_length = 16 + dtype = "bfloat16" + tensor_parallel_size = 2 + skip_tokenizer_init = True + local_model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B") + + tokenizer, actor_model = load_tokenizer_and_model(local_model_path) + + preencode_prompts = [ + [{"role": "user", "content": prompt, "tool_calls": None}] + for prompt in [ + "Who won the Champions League in 2019?", + "The founder of Apple is", + "What's the best way to learn python?", + ] + ] + prompts = [ + tokenizer.apply_chat_template(message, tokenize=False, add_generation_prompt=True) + for message in preencode_prompts + ] + input_ids, attention_mask, position_ids = prepare_inputs(tokenizer, prompts, max_prompt_length) + + hf_response_tokens = generate_hf_output(actor_model, input_ids, attention_mask, tokenizer, max_response_length) + + rollout_config = get_rollout_config( + max_response_length, + max_prompt_length, + dtype, + tensor_parallel_size, + tool_config_path="./resource/tool_configs/sandbox_fusion_tool_config", + skip_tokenizer_init=skip_tokenizer_init, + ) + rollout_config: RolloutConfig = omega_conf_to_dataclass(rollout_config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + + prompt_dict = TensorDict( + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=input_ids.shape[0], + ) + print(f"preprocessed {input_ids.shape=}") + + messages = np.asarray(preencode_prompts) + prompts = DataProto( + batch=prompt_dict, + non_tensor_batch={ + "raw_prompt": messages, + "tools_kwargs": np.array([{}] * input_ids.shape[0], dtype=object), + }, + ) + + prompts.meta_info.update( + { + "eos_token_id": tokenizer.eos_token_id, + "pad_token_id": tokenizer.pad_token_id, + } + ) + + # log_gpu_memory_usage("Before generating sequences", logger=None) + output = rollout.generate_sequences(prompts=prompts) + print(f"generated {output.batch['responses'].shape=}") + # log_gpu_memory_usage("After generating sequences", logger=None) + + sglang_output = output.to("cpu") + + sglang_response_tokens = tokenizer.batch_decode(sglang_output.batch["responses"]) + + print(f"hf response: {hf_response_tokens}") + print(f"sglang response: {sglang_response_tokens}") + assert are_lists_similar(hf_response_tokens, sglang_response_tokens) + print("SGLang w tool Test Passed!") + + torch.distributed.barrier() + torch.distributed.destroy_process_group() + + +if __name__ == "__main__": + test_async_sglang_rollout_w_tool() diff --git a/e2c/verl/tests/workers/rollout/test_sglang_multi_interaction.py b/e2c/verl/tests/workers/rollout/test_sglang_multi_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..82826a72377f7234c0147f66dbc1319709b2f8e2 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_multi_interaction.py @@ -0,0 +1,422 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Test for multi-interaction support in SGLangRollout. +usage: torchrun --standalone --nnodes=1 \ + --nproc_per_node=2 $(which pytest) \ + -s test_sglang_multi_interaction.py +""" + +import os +import tempfile +from unittest.mock import MagicMock, patch + +import torch +import torch.distributed as dist +from omegaconf import DictConfig, OmegaConf +from transformers import AutoTokenizer + +from verl.interactions.base import BaseInteraction +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.sglang_rollout.sglang_rollout import SGLangRollout + + +class MockInteraction(BaseInteraction): + """Mock interaction for testing.""" + + def __init__(self, config): + super().__init__(config) + self.started_instances = set() + + async def start_interaction(self, instance_id=None, **kwargs): + if instance_id is None: + instance_id = "mock_instance" + self.started_instances.add(instance_id) + return instance_id + + async def generate_response(self, instance_id, messages, **kwargs): + return False, f"Mock response from {self.name}", 1.0, {} + + +def create_mock_config_with_multi_interactions(): + """Create a mock configuration with multiple interactions.""" + # Create temporary interaction config file + interaction_config = { + "interaction": [ + { + "name": "mock_agent1", + "class_name": "tests.workers.rollout.test_sglang_multi_interaction.MockInteraction", + "config": {"param1": "value1"}, + }, + { + "name": "mock_agent2", + "class_name": "tests.workers.rollout.test_sglang_multi_interaction.MockInteraction", + "config": {"param2": "value2"}, + }, + ] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(interaction_config, f.name) + interaction_config_path = f.name + + # Create mock SGLangRollout config + config = DictConfig( + { + "name": "sglang", + "multi_turn": { + "interaction_config_path": interaction_config_path, + "tool_config_path": None, + "enable": True, + "max_assistant_turns": 5, + "max_user_turns": 3, + "use_inference_chat_template": True, + "tokenization_sanity_check_mode": "off", + }, + "prompt_length": 32, + "response_length": 16, + "max_model_len": 512, + "dtype": "bfloat16", + "gpu_memory_utilization": 0.8, + "load_format": "dummy", + "enforce_eager": True, + "free_cache_engine": False, + "calculate_log_probs": False, + "tensor_model_parallel_size": 1, + "n": 1, + "val_kwargs": {"top_k": 1, "top_p": 1.0, "temperature": 0.0}, + } + ) + + return config, interaction_config_path + + +def setup_distributed(): + """Initialize distributed environment if not already initialized.""" + if not dist.is_initialized(): + dist.init_process_group(backend="nccl" if torch.cuda.is_available() else "gloo") + + +class TestSGLangMultiInteraction: + local_model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B") + + def test_initialize_multiple_interactions(self): + """Test that SGLangRollout can initialize multiple interactions.""" + setup_distributed() + config, temp_config_path = create_mock_config_with_multi_interactions() + + try: + # Mock SGLang engine and initialization methods like the reference test + with ( + patch.object(SGLangRollout, "_init_distributed_env", return_value=None), + patch.object(SGLangRollout, "_init_inference_engine", return_value=None), + patch.object(SGLangRollout, "_init_sampling_params", return_value=None), + ): + # Create a real tokenizer like the reference test + tokenizer = AutoTokenizer.from_pretrained(self.local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + + # Mock model config + mock_model_config = MagicMock() + mock_model_config.max_position_embeddings = 2048 + # since this is a mock, we can set any rope scaling config + # to test the rope_scaling logic at the same time of this test + mock_model_config.rope_scaling = { + "factor": 4.0, + "original_max_position_embeddings": 32768, + "type": "yarn", + } + + rollout_config: RolloutConfig = omega_conf_to_dataclass(config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + + # Check that interactions were initialized + assert len(rollout.interaction_map) == 2 + assert "mock_agent1" in rollout.interaction_map + assert "mock_agent2" in rollout.interaction_map + + # Use class name comparison instead of isinstance for multi-process compatibility + assert rollout.interaction_map["mock_agent1"].__class__.__name__ == "MockInteraction" + assert rollout.interaction_map["mock_agent2"].__class__.__name__ == "MockInteraction" + + # Also check that they are instances of BaseInteraction (which should work across processes) + assert isinstance(rollout.interaction_map["mock_agent1"], BaseInteraction) + assert isinstance(rollout.interaction_map["mock_agent2"], BaseInteraction) + + # Check that names were set correctly + assert rollout.interaction_map["mock_agent1"].name == "mock_agent1" + assert rollout.interaction_map["mock_agent2"].name == "mock_agent2" + + finally: + os.unlink(temp_config_path) + + def test_interaction_selection_by_name(self): + """Test that interactions are selected by name from interaction_kwargs.""" + setup_distributed() + config, temp_config_path = create_mock_config_with_multi_interactions() + + try: + with ( + patch.object(SGLangRollout, "_init_distributed_env", return_value=None), + patch.object(SGLangRollout, "_init_inference_engine", return_value=None), + patch.object(SGLangRollout, "_init_sampling_params", return_value=None), + ): + tokenizer = AutoTokenizer.from_pretrained(self.local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + + mock_model_config = MagicMock() + mock_model_config.max_position_embeddings = 2048 + mock_model_config.rope_scaling = { + "factor": 4.0, + "original_max_position_embeddings": 32768, + "type": "yarn", + } + + rollout_config: RolloutConfig = omega_conf_to_dataclass(config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + + # Test interaction selection logic + from verl.workers.rollout.schemas import AsyncRolloutRequest, AsyncRolloutRequestStateEnum, Message + + # Create a mock request with specific interaction name + req = AsyncRolloutRequest( + request_id="test_req", + state=AsyncRolloutRequestStateEnum.INTERACTING, + messages=[Message(role="user", content="test message")], + interaction_kwargs={"name": "mock_agent2", "test_param": "value"}, + input_ids=None, + prompt_ids=None, + response_ids=None, + attention_mask=None, + prompt_attention_mask=None, + response_attention_mask=None, + position_ids=None, + prompt_position_ids=None, + response_position_ids=None, + loss_mask=None, + prompt_loss_mask=None, + response_loss_mask=None, + reward_scores={}, + max_prompt_len=32, + max_response_len=16, + max_model_len=512, + use_inference_chat_template=True, + tokenization_sanity_check_mode="disable", + processing_class=tokenizer, + ) + + # Test that the correct interaction is selected + interaction_name = req.interaction_kwargs.get("name", "gsm8k") + assert interaction_name == "mock_agent2" + assert interaction_name in rollout.interaction_map + + selected_interaction = rollout.interaction_map[interaction_name] + assert selected_interaction.name == "mock_agent2" + + finally: + os.unlink(temp_config_path) + + def test_fallback_to_default_interaction(self): + """Test fallback to default interaction when name is not specified.""" + setup_distributed() + # Create config with gsm8k interaction + interaction_config = { + "interaction": [ + { + "name": "gsm8k", + "class_name": "tests.workers.rollout.test_sglang_multi_interaction.MockInteraction", + "config": {}, + } + ] + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + OmegaConf.save(interaction_config, f.name) + interaction_config_path = f.name + + config = DictConfig( + { + "name": "sglang", + "multi_turn": { + "interaction_config_path": interaction_config_path, + "tool_config_path": None, + "enable": True, + "max_assistant_turns": 5, + "max_user_turns": 3, + "use_inference_chat_template": True, + "tokenization_sanity_check_mode": "disable", + }, + "prompt_length": 32, + "response_length": 16, + "max_model_len": 512, + "dtype": "bfloat16", + "gpu_memory_utilization": 0.8, + "load_format": "dummy", + "enforce_eager": True, + "free_cache_engine": False, + "calculate_log_probs": False, + "tensor_model_parallel_size": 1, + "n": 1, + "val_kwargs": {"top_k": 1, "top_p": 1.0, "temperature": 0.0}, + } + ) + + try: + with ( + patch.object(SGLangRollout, "_init_distributed_env", return_value=None), + patch.object(SGLangRollout, "_init_inference_engine", return_value=None), + patch.object(SGLangRollout, "_init_sampling_params", return_value=None), + ): + tokenizer = AutoTokenizer.from_pretrained(self.local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + + mock_model_config = MagicMock() + mock_model_config.max_position_embeddings = 2048 + mock_model_config.rope_scaling = { + "factor": 4.0, + "original_max_position_embeddings": 32768, + "type": "yarn", + } + + rollout_config: RolloutConfig = omega_conf_to_dataclass(config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + + # Test that default interaction name works + interaction_kwargs_without_name = {"test_param": "value"} + default_name = interaction_kwargs_without_name.get("name", "gsm8k") + assert default_name == "gsm8k" + assert default_name in rollout.interaction_map + + finally: + os.unlink(interaction_config_path) + + def test_error_on_missing_interaction(self): + """Test that error is raised when requested interaction is not found.""" + setup_distributed() + config, temp_config_path = create_mock_config_with_multi_interactions() + + try: + with ( + patch.object(SGLangRollout, "_init_distributed_env", return_value=None), + patch.object(SGLangRollout, "_init_inference_engine", return_value=None), + patch.object(SGLangRollout, "_init_sampling_params", return_value=None), + ): + tokenizer = AutoTokenizer.from_pretrained(self.local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + + mock_model_config = MagicMock() + mock_model_config.max_position_embeddings = 2048 + mock_model_config.rope_scaling = { + "factor": 4.0, + "original_max_position_embeddings": 32768, + "type": "yarn", + } + + rollout_config: RolloutConfig = omega_conf_to_dataclass(config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + + # Test error when requesting non-existent interaction + non_existent_name = "non_existent_interaction" + assert non_existent_name not in rollout.interaction_map + + # This should raise ValueError in actual usage + available_interactions = list(rollout.interaction_map.keys()) + assert "mock_agent1" in available_interactions + assert "mock_agent2" in available_interactions + assert non_existent_name not in available_interactions + + finally: + os.unlink(temp_config_path) + + def test_backward_compatibility_no_interaction_config(self): + """Test backward compatibility when no interaction config is provided.""" + setup_distributed() + # Create config without interaction config + config = DictConfig( + { + "name": "sglang", + "multi_turn": { + "interaction_config_path": None, + "tool_config_path": None, + "enable": True, + "max_assistant_turns": 5, + "max_user_turns": 3, + "use_inference_chat_template": True, + "tokenization_sanity_check_mode": "disable", + }, + "prompt_length": 32, + "response_length": 16, + "max_model_len": 512, + "dtype": "bfloat16", + "gpu_memory_utilization": 0.8, + "load_format": "dummy", + "enforce_eager": True, + "free_cache_engine": False, + "calculate_log_probs": False, + "tensor_model_parallel_size": 1, + "n": 1, + "val_kwargs": {"top_k": 1, "top_p": 1.0, "temperature": 0.0}, + } + ) + + with ( + patch.object(SGLangRollout, "_init_distributed_env", return_value=None), + patch.object(SGLangRollout, "_init_inference_engine", return_value=None), + patch.object(SGLangRollout, "_init_sampling_params", return_value=None), + ): + tokenizer = AutoTokenizer.from_pretrained(self.local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + + mock_model_config = MagicMock() + mock_model_config.max_position_embeddings = 2048 + mock_model_config.rope_scaling = { + "factor": 4.0, + "original_max_position_embeddings": 32768, + "type": "yarn", + } + + rollout_config: RolloutConfig = omega_conf_to_dataclass(config, dataclass_type=RolloutConfig) + model_config = HFModelConfig(path=self.local_model_path) + rollout = SGLangRollout( + config=rollout_config, + model_config=model_config, + device_mesh=None, + ) + + # Check that no interactions were initialized + assert len(rollout.interaction_map) == 0 diff --git a/e2c/verl/tests/workers/rollout/test_sglang_rollout_sharding_manager.py b/e2c/verl/tests/workers/rollout/test_sglang_rollout_sharding_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..0d3c7b5da2bea7c5ba757ba2b42cc30f58890eb7 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_rollout_sharding_manager.py @@ -0,0 +1,57 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch + +from verl.workers.rollout.sglang_rollout.utils import get_named_tensor_buckets + +_TENSOR_1MB = torch.zeros(512, 512) +_BYTES_1MB = 1 << 20 + + +@pytest.mark.parametrize( + "named_tensors, bucket_size_mb, gt_groups", + [ + ( + [("a", _TENSOR_1MB), ("b", _TENSOR_1MB)], + 0.5 * _BYTES_1MB, + [["a"], ["b"]], + ), + ( + [("a", _TENSOR_1MB), ("b", _TENSOR_1MB)], + 1 * _BYTES_1MB, + [["a"], ["b"]], + ), + ( + [("a", _TENSOR_1MB), ("b", _TENSOR_1MB)], + 1.5 * _BYTES_1MB, + [["a"], ["b"]], + ), + ( + [("a", _TENSOR_1MB), ("b", _TENSOR_1MB)], + 2 * _BYTES_1MB, + [["a", "b"]], + ), + ], +) +def test_get_named_tensor_buckets(named_tensors, bucket_size_mb, gt_groups: list[list[str]]): + named_tensors_iter = iter(named_tensors) + groups = list(get_named_tensor_buckets(named_tensors_iter, bucket_size_mb)) + assert len(groups) == len(gt_groups) + for group, gt_group in zip(groups, gt_groups, strict=True): + assert len(group) == len(gt_group) + for (name, _), (gt_name) in zip(group, gt_group, strict=True): + assert name == gt_name diff --git a/e2c/verl/tests/workers/rollout/test_sglang_spmd.py b/e2c/verl/tests/workers/rollout/test_sglang_spmd.py new file mode 100644 index 0000000000000000000000000000000000000000..4e4006ca08142ef8cc916292595173e45bfca199 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/test_sglang_spmd.py @@ -0,0 +1,117 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +usage: torchrun --standalone --nnodes=1 \ + --nproc_per_node=2 $(which pytest) \ + -s test_sglang_async_spmd.py +""" + +import asyncio +import os + +import torch +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.utils import broadcast_pyobj +from torch.distributed.device_mesh import init_device_mesh +from utils_sglang import ( + are_lists_similar, + clean_torchelastic_env, + generate_hf_output, + initialize_global_process_group, + load_tokenizer_and_model, + prepare_inputs, +) + + +def _pre_process_inputs(pad_token_id, prompt_token_ids: torch.Tensor): + non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] + token_ids = prompt_token_ids[non_pad_index:].tolist() + return token_ids + + +def test_sglang_spmd(): + assert torch.cuda.device_count() >= 2 + initialize_global_process_group(spmd=True) + clean_torchelastic_env() + + max_prompt_length = 16 + max_response_length = 16 + + local_model_path = os.path.expanduser("~/models/Qwen/Qwen2.5-0.5B") + tokenizer, actor_model = load_tokenizer_and_model(local_model_path) + + preencode_prompts = ["Who won the Champions League in 2019?", "The founder of Apple is", "What's your name?"] + input_ids, attention_mask, _ = prepare_inputs(tokenizer, preencode_prompts, max_prompt_length) + + hf_response_tokens = generate_hf_output(actor_model, input_ids, attention_mask, tokenizer, max_response_length) + + tensor_parallel_size = 2 + inference_device_mesh_cpu = init_device_mesh( + "cpu", mesh_shape=(1, tensor_parallel_size, 1), mesh_dim_names=["dp", "tp", "pp"] + ) + tp_rank = inference_device_mesh_cpu["tp"].get_local_rank() + + if tp_rank == 0: + llm = Engine( + model_path=local_model_path, + dtype="bfloat16", + mem_fraction_static=0.5, + enable_memory_saver=True, + tp_size=inference_device_mesh_cpu["tp"].size(), + attention_backend="fa3", + ) + + input_ids = input_ids.cuda() + idx_list = [] + + pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id + for i in range(input_ids.shape[0]): + idx_list.append(_pre_process_inputs(pad_token_id, input_ids[i])) + + sampling_params = dict( + n=1, + temperature=0, + top_p=1, + top_k=-1, + max_new_tokens=max_response_length, + presence_penalty=0.0, + frequency_penalty=0.0, + repetition_penalty=1.0, + skip_special_tokens=True, + spaces_between_special_tokens=True, + ignore_eos=False, + ) + + loop = asyncio.get_event_loop() + outputs = loop.run_until_complete(llm.async_generate(input_ids=idx_list, sampling_params=sampling_params)) + else: + outputs = None + + [outputs] = broadcast_pyobj( + [outputs], + rank=inference_device_mesh_cpu["tp"].get_local_rank(), + src=inference_device_mesh_cpu["tp"].mesh[0].item(), + dist_group=inference_device_mesh_cpu["tp"].get_group(), + force_cpu_device=False, + ) + + sglang_response_tokens = [output["text"] for output in outputs] + + print(f"sglang response: {sglang_response_tokens}") + assert are_lists_similar(hf_response_tokens, sglang_response_tokens), "Strings differ more than 10%:\n" + print("SPMD Test Passed!") + + torch.distributed.barrier() + torch.distributed.destroy_process_group() diff --git a/e2c/verl/tests/workers/rollout/utils_sglang.py b/e2c/verl/tests/workers/rollout/utils_sglang.py new file mode 100644 index 0000000000000000000000000000000000000000..48e2b5ca1c88a07422acc3729bc8fb00c111d433 --- /dev/null +++ b/e2c/verl/tests/workers/rollout/utils_sglang.py @@ -0,0 +1,175 @@ +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from datetime import timedelta + +import torch +from omegaconf import OmegaConf +from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig + +from verl.utils.model import compute_position_id_with_mask +from verl.utils.torch_functional import pad_sequence_to_length + + +# ====================== utils ====================== +def levenshtein(s1, s2): + m, n = len(s1), len(s2) + dp = [[0] * (n + 1) for _ in range(m + 1)] + for i in range(m + 1): + dp[i][0] = i + for j in range(n + 1): + dp[0][j] = j + for i in range(1, m + 1): + for j in range(1, n + 1): + cost = 0 if s1[i - 1] == s2[j - 1] else 1 + dp[i][j] = min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + cost) + return dp[m][n] + + +def are_lists_similar(a, b, threshold=10): + if len(a) != len(b): + print("The lists are of different lengths.") + return False + total_length = 0 + total_diff = 0 + for s1, s2 in zip(a, b, strict=True): + max_len = max(len(s1), len(s2)) + total_length += max_len + total_diff += levenshtein(s1, s2) + percentage_difference = (total_diff / total_length) * 100 + print(f"Total difference: {percentage_difference:.2f}%") + return percentage_difference <= threshold + + +def initialize_global_process_group(timeout_second=36000, spmd=False): + import torch.distributed + + if not torch.distributed.is_initialized(): # Check if already initialized + print("Initializing process group...") + torch.distributed.init_process_group(timeout=timedelta(seconds=timeout_second)) + else: + print("Process group already initialized.") + + local_rank = int(os.environ["LOCAL_RANK"]) + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + torch.cuda.set_device(local_rank) + + CUDA_VISIBLE_DEVICES = os.environ.get("CUDA_VISIBLE_DEVICES", "") + if not CUDA_VISIBLE_DEVICES: + if spmd: + # CUDA_VISIBLE_DEVICES = ','.join(str(i) for i in range(tensor_parallel_size)) + CUDA_VISIBLE_DEVICES = ",".join(str(i) for i in range(world_size)) + else: + CUDA_VISIBLE_DEVICES = str(local_rank) + os.environ["CUDA_VISIBLE_DEVICES"] = CUDA_VISIBLE_DEVICES + print(f"CUDA_VISIBLE_DEVICES is not set, set to {CUDA_VISIBLE_DEVICES}") + + return local_rank, rank, world_size + + +def clean_torchelastic_env(): + for k in ["TORCHELASTIC_USE_AGENT_STORE"]: + if k in os.environ: + del os.environ[k] + + +def load_tokenizer_and_model(local_model_path, dtype="bfloat16"): + tokenizer = AutoTokenizer.from_pretrained(local_model_path, padding_side="left") + tokenizer.pad_token = tokenizer.eos_token + model = AutoModelForCausalLM.from_pretrained(local_model_path, torch_dtype=getattr(torch, dtype), device_map="cuda") + return tokenizer, model + + +def prepare_inputs(tokenizer, prompts, max_prompt_length): + pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id + tokenized = tokenizer(prompts, return_tensors="pt", padding=True) + input_ids = pad_sequence_to_length(tokenized["input_ids"], max_prompt_length, pad_token_id, left_pad=True) + attention_mask = pad_sequence_to_length( + tokenized["attention_mask"], max_prompt_length, pad_token_id=0, left_pad=True + ) + position_ids = compute_position_id_with_mask(attention_mask) + position_ids = pad_sequence_to_length(position_ids, max_prompt_length, pad_token_id=0, left_pad=True) + return input_ids, attention_mask, position_ids + + +def generate_hf_output(model, input_ids, attention_mask, tokenizer, max_response_length): + generation_config = GenerationConfig(do_sample=False) + output = model.generate( + input_ids=input_ids.cuda(), + attention_mask=attention_mask.cuda(), + max_new_tokens=max_response_length, + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.pad_token_id, + generation_config=generation_config, + output_scores=False, + return_dict_in_generate=True, + use_cache=False, + ) + seq = output.sequences + response = seq[:, input_ids.shape[1] :] + return tokenizer.batch_decode(response) + + +def get_rollout_config( + max_response_length, + max_prompt_length, + dtype, + tensor_parallel_size, + tool_config_path=None, + interaction_config_path=None, + skip_tokenizer_init=False, +): + sampling_params = dict( + n=1, + temperature=0, + top_p=1, + top_k=-1, + ) + + rollout_config = OmegaConf.create( + { + "name": "sglang", + "mode": "sync", + "load_format": "auto", + "enforce_eager": False, + "free_cache_engine": True, + "dtype": dtype, + "gpu_memory_utilization": 0.5, + "ignore_eos": False, + "max_num_batched_tokens": 8192, + "prompt_length": max_prompt_length, + "response_length": max_response_length, + "tensor_model_parallel_size": tensor_parallel_size, + # set to 128MB only for testing + "update_weights_bucket_megabytes": 128, + # do not drop any samples in the test + "over_sample_rate": 0.0, + "multi_turn": { + "max_assistant_turns": 4, + "max_user_turns": 4, + "enable": True, + "tool_config_path": tool_config_path, + "interaction_config_path": interaction_config_path, + "use_inference_chat_template": False, + "tokenization_sanity_check_mode": "strict", + }, + "calculate_log_probs": False, + "max_model_len": None, + "skip_tokenizer_init": skip_tokenizer_init, + **sampling_params, + } + ) + + return rollout_config diff --git a/e2c/verl/tests/workers/test_fsdp_workers.py b/e2c/verl/tests/workers/test_fsdp_workers.py new file mode 100644 index 0000000000000000000000000000000000000000..e8b2eaf6b256e7f587d94e63807b1c6de4cbc56f --- /dev/null +++ b/e2c/verl/tests/workers/test_fsdp_workers.py @@ -0,0 +1,77 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from omegaconf import OmegaConf + +from verl.workers.fsdp_workers import ActorRolloutRefWorker + + +def test_actor_rollout_ref_worker_actor_ref_model(): + """Test specifying different reference/actor model""" + os.environ["RANK"] = "0" + os.environ["WORLD_SIZE"] = "1" + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = "8888" + + config_str = """ + model: + path: Qwen/Qwen2.5-0.5B-Instruct + actor: + _target_: verl.workers.config.FSDPActorConfig + strategy: fsdp + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + fsdp_size: -1 + forward_prefetch: false + profiler: + tool: torch_memory + save_path: ./mem_snapshots + tool_config: + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: 100000 + stack_depth: 32 + ref: + model: + path: Qwen/Qwen2.5-1.5B-Instruct + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + fsdp_size: -1 + profiler: + tool: torch_memory + save_path: ./mem_snapshots + tool_config: + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: 100000 + stack_depth: 32 + log_prob_micro_batch_size: 1 + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + """ + dict_conf = OmegaConf.create(config_str) + actor_rollout_ref_worker = ActorRolloutRefWorker(dict_conf, role="ref") + actor_rollout_ref_worker.init_model() + + model_config = actor_rollout_ref_worker.ref_module_fsdp._fsdp_wrapped_module.config + assert model_config.hidden_size == 1536 + + # set ref.model to null, fallback to default case where actor is the same as reference + dict_conf["ref"]["model"] = None + actor_rollout_ref_worker = ActorRolloutRefWorker(dict_conf, role="ref") + actor_rollout_ref_worker.init_model() + + model_config = actor_rollout_ref_worker.ref_module_fsdp._fsdp_wrapped_module.config + assert model_config.hidden_size == 896 diff --git a/e2c/verl/verl/__init__.py b/e2c/verl/verl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..38f2e7cf90ab86fc94507685aef056cbc1bf087a --- /dev/null +++ b/e2c/verl/verl/__init__.py @@ -0,0 +1,92 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import logging +import os +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as get_version + +from packaging.version import parse as parse_version + +from .protocol import DataProto +from .utils.device import is_npu_available +from .utils.logging_utils import set_basic_config + +version_folder = os.path.dirname(os.path.join(os.path.abspath(__file__))) + +with open(os.path.join(version_folder, "version/version")) as f: + __version__ = f.read().strip() + + +set_basic_config(level=logging.WARNING) + + +__all__ = ["DataProto", "__version__"] + +if os.getenv("VERL_USE_MODELSCOPE", "False").lower() == "true": + if importlib.util.find_spec("modelscope") is None: + raise ImportError("You are using the modelscope hub, please install modelscope by `pip install modelscope -U`") + # Patch hub to download models from modelscope to speed up. + from modelscope.utils.hf_util import patch_hub + + patch_hub() + +if is_npu_available: + from .models.transformers import npu_patch as npu_patch + + package_name = "transformers" + required_version_spec = "4.52.4" + try: + installed_version = get_version(package_name) + installed = parse_version(installed_version) + required = parse_version(required_version_spec) + + if installed < required: + raise ValueError( + f"{package_name} version >= {required_version_spec} is required on ASCEND NPU, current version is " + f"{installed}." + ) + except PackageNotFoundError as e: + raise ImportError( + f"package {package_name} is not installed, please run pip install {package_name}=={required_version_spec}" + ) from e + + # In verl, the driver process aggregates the computation results of workers via Ray. + # Therefore, after a worker completes its computation job, it will package the output + # using tensordict and transfer it to the CPU. Since the `to` operation of tensordict + # is non-blocking, when transferring data from a device to the CPU, it is necessary to + # ensure that a batch of data has been completely transferred before being used on the + # host; otherwise, unexpected precision issues may arise. Tensordict has already noticed + # this problem and fixed it. Ref: https://github.com/pytorch/tensordict/issues/725 + # However, the relevant modifications only cover CUDA and MPS devices and do not take effect + # for third-party devices such as NPUs. This patch fixes this issue, and the relevant + # modifications can be removed once the fix is merged into tensordict. + + import tensordict + + if parse_version(tensordict.__version__) < parse_version("0.10.0"): + from tensordict.base import TensorDictBase + + def _sync_all_patch(self): + from torch._utils import _get_available_device_type, _get_device_module + + device_type = _get_available_device_type() + if device_type is None: + return + + device_module = _get_device_module(device_type) + device_module.synchronize() + + TensorDictBase._sync_all = _sync_all_patch diff --git a/e2c/verl/verl/base_config.py b/e2c/verl/verl/base_config.py new file mode 100644 index 0000000000000000000000000000000000000000..f425dd1464b0f13c83a0944249cd84d55903f120 --- /dev/null +++ b/e2c/verl/verl/base_config.py @@ -0,0 +1,86 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +from dataclasses import FrozenInstanceError, dataclass, fields +from typing import Any + + +# BaseConfig class inherits from collections.abc.Mapping, which means it can act like a dictionary +@dataclass +class BaseConfig(collections.abc.Mapping): + """The BaseConfig provides dict-like interface for a dataclass config. + + By default all fields in the config is not mutable, unless specified in + "_mutable_fields". The BaseConfig class implements the Mapping Abstract Base Class. + This allows instances of this class to be used like dictionaries. + """ + + _mutable_fields = set() + _target_: str = "" + + def __setattr__(self, name: str, value): + """Set the value of an attribute. Check if the attr is mutable before setting the value.""" + # If the field already exists, it's considered frozen unless it's in _mutable_fields + if name in self.__dict__ and name not in getattr(self, "_mutable_fields", set()): + raise FrozenInstanceError(f"Field '{name}' is frozen and cannot be modified") + super().__setattr__(name, value) + + def get(self, key: str, default: Any = None) -> Any: + """Get the value associated with the given key. If the key does not exist, return the default value. + + Args: + key (str): The attribute name to retrieve. + default (Any, optional): The value to return if the attribute does not exist. Defaults to None. + + Returns: + Any: The value of the attribute or the default value. + """ + try: + return getattr(self, key) + except AttributeError: + return default + + def __getitem__(self, key: str): + """Implement the [] operator for the class. Allows accessing attributes like dictionary items. + + Args: + key (str): The attribute name to retrieve. + + Returns: + Any: The value of the attribute. + + Raises: + AttributeError: If the attribute does not exist. + TypeError: If the key type is not string + """ + return getattr(self, key) + + def __iter__(self): + """Implement the iterator protocol. Allows iterating over the attribute names of the instance. + + Yields: + str: The name of each field in the dataclass. + """ + for f in fields(self): + yield f.name + + def __len__(self): + """ + Return the number of fields in the dataclass. + + Returns: + int: The number of fields in the dataclass. + """ + return len(fields(self)) diff --git a/e2c/verl/verl/experimental/__init__.py b/e2c/verl/verl/experimental/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/experimental/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/experimental/agent_loop/__init__.py b/e2c/verl/verl/experimental/agent_loop/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..88d61ee41699019187b50f282faa77189933725c --- /dev/null +++ b/e2c/verl/verl/experimental/agent_loop/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .agent_loop import AgentLoopBase, AgentLoopManager, AsyncLLMServerManager +from .single_turn_agent_loop import SingleTurnAgentLoop +from .tool_agent_loop import ToolAgentLoop + +_ = [SingleTurnAgentLoop, ToolAgentLoop] + +__all__ = ["AgentLoopBase", "AgentLoopManager", "AsyncLLMServerManager"] diff --git a/e2c/verl/verl/experimental/agent_loop/agent_loop.py b/e2c/verl/verl/experimental/agent_loop/agent_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..01ff2bc98b293abd62e5f7e919a12ca840f7c194 --- /dev/null +++ b/e2c/verl/verl/experimental/agent_loop/agent_loop.py @@ -0,0 +1,897 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import heapq +import logging +import os +import queue +import random +import threading +from abc import ABC, abstractmethod +from concurrent.futures import Future +from typing import Any, Optional + +import hydra +import numpy as np +import ray +import torch +from cachetools import LRUCache +from omegaconf import DictConfig, OmegaConf +from pydantic import BaseModel, ConfigDict +from tensordict import TensorDict +from transformers import AutoProcessor, AutoTokenizer + +from verl.protocol import DataProto +from verl.single_controller.ray.base import RayWorkerGroup +from verl.trainer.ppo.reward import load_reward_manager +from verl.utils import hf_processor, hf_tokenizer +from verl.utils.fs import copy_to_local +from verl.utils.model import compute_position_id_with_mask +from verl.utils.rollout_trace import RolloutTraceConfig, rollout_trace_attr, rollout_trace_op +from verl.workers.rollout.replica import TokenOutput, get_rollout_replica_class + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class AsyncLLMServerManager: + """ + A class to manage multiple OpenAI compatible LLM servers. This class provides + - Load balance: least requests load balancing + - Sticky session: send multi-turn chat completions to same server for automatic prefix caching + """ + + def __init__(self, config: DictConfig, server_handles: list[ray.actor.ActorHandle], max_cache_size: int = 10000): + """Initialize the AsyncLLMServerManager. + + Args: + config (DictConfig): YAML config. + server_handles (List[ray.actor.ActorHandle]): OpenAI compatible LLM server actor handles. + max_cache_size (int, optional): max cache size for request_id to server mapping. Defaults to 10000. + """ + self.config = config + self.server_handles = server_handles + random.shuffle(self.server_handles) + + # Least requests load balancing + self.weighted_serveres = [[0, (hash(server), server)] for server in server_handles] + heapq.heapify(self.weighted_serveres) + + # LRU cache to map request_id to server + self.request_id_to_server = LRUCache(maxsize=max_cache_size) + + def _choose_server(self, request_id: str) -> ray.actor.ActorHandle: + # TODO: implement server pressure awareness load balancing + if request_id in self.request_id_to_server: + return self.request_id_to_server[request_id] + + server = self.weighted_serveres[0][1][1] + self.weighted_serveres[0][0] += 1 + heapq.heapreplace(self.weighted_serveres, self.weighted_serveres[0]) + self.request_id_to_server[request_id] = server + return server + + @rollout_trace_op + async def generate( + self, + request_id, + *, + prompt_ids: list[int], + sampling_params: dict[str, Any], + image_data: Optional[list[Any]] = None, + ) -> TokenOutput: + """Generate tokens from prompt ids. + + Args: + request_id (str): request id for sticky session. + prompt_ids (List[int]): List of prompt token ids. + sampling_params (Dict[str, Any]): Sampling parameters for the chat completion. + + Returns: + TokenOutput: token output + """ + server = self._choose_server(request_id) + output = await server.generate.remote( + request_id=request_id, + prompt_ids=prompt_ids, + sampling_params=sampling_params, + image_data=image_data, + ) + return output + + +class AgentLoopMetrics(BaseModel): + """Agent loop performance metrics.""" + + generate_sequences: float = 0.0 + tool_calls: float = 0.0 + + +class AgentLoopOutput(BaseModel): + """Agent loop output.""" + + prompt_ids: list[int] + """Prompt token ids.""" + response_ids: list[int] + """Response token ids including LLM generated token, tool response token.""" + response_mask: list[int] + """Response mask, 1 for LLM generated token, 0 for tool response token.""" + response_logprobs: Optional[list[float]] = None + """Log probabilities for the response tokens.""" + multi_modal_data: Optional[dict[str, Any]] = None + """Multi-modal data for multi-modal tools.""" + reward_score: Optional[float] = None + """Reward score for the trajectory.""" + num_turns: int = 0 + """Number of chat turns, including user, assistant, tool.""" + metrics: AgentLoopMetrics + """Auxiliary performance metrics""" + extra_fields: dict[str, Any] = {} + """Extra fields for dynamic addition.""" + + +class _InternalAgentLoopOutput(AgentLoopOutput): + """Internal agent loop output with padded sequences.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + prompt_ids: torch.Tensor + """Padded prompt token ids.""" + response_ids: torch.Tensor + """Padded response token ids.""" + input_ids: torch.Tensor + """Padded input ids(prompt_ids + response_ids).""" + position_ids: torch.Tensor + """Padded position ids.""" + response_mask: torch.Tensor + """Padded response mask.""" + attention_mask: torch.Tensor + """Padded attention mask.""" + response_logprobs: Optional[torch.Tensor] = None + """Padded log probabilities for the response tokens.""" + multi_modal_inputs: Optional[dict[str, torch.Tensor]] = None + """Multi-modal inputs for processors (e.g., pixel_values, image_grid_thw).""" + extra_fields: dict[str, Any] = {} + """Extra fields for dynamic addition.""" + + +# make hydra.utils.instantiate happy +class _DummyConfig: + def __init__(self, config: DictConfig) -> None: + self.config = config + + +class AgentLoopBase(ABC): + """An agent loop takes a input message, chat with OpenAI compatible LLM server and interact with various + environments.""" + + _class_initialized = False + + def __init__( + self, + trainer_config: _DummyConfig, + server_manager: AsyncLLMServerManager, + tokenizer: AutoTokenizer, + processor: AutoProcessor, + **kwargs, + ): + """Initialize agent loop, each sample will have its own loop instance. + + Args: + trainer_config (_DummyConfig): trainer config. + server_manager (AsyncLLMServerManager): OpenAI compatible LLM server manager. + tokenizer (AutoTokenizer): Tokenizer for tokenize messages. + processor (AutoProcessor): Processor for process messages. + """ + self.init_class(config=trainer_config.config, tokenizer=tokenizer, processor=processor, **kwargs) + self.config = trainer_config.config + self.server_manager = server_manager + self.tokenizer = tokenizer + self.processor = processor + self.loop = asyncio.get_running_loop() + + @classmethod + def init_class(cls, config: DictConfig, tokenizer: AutoTokenizer, processor: AutoProcessor, **kwargs): + """This is used to do heavy initialization work that should shared across all instances. It's only called once. + + Args: + config (DictConfig): trainer config. + tokenizer (AutoTokenizer): Tokenizer for tokenize messages. + processor (AutoProcessor): Processor for process multi_modal data. + **kwargs: extra kwargs from config file passed in by `hydra.utils.instantiate`. + """ + if cls._class_initialized: + return + cls._class_initialized = True + + @abstractmethod + async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput: + """Run agent loop to interact with LLM server and environment. + + Args: + sampling_params (Dict[str, Any]): LLM sampling params. + **kwargs: dataset fields from `verl.utils.dataset.RLHFDataset`. + + Returns: + AgentLoopOutput: Agent loop output. + """ + raise NotImplementedError + + +"""Agent loop registry: key is agent_name, value is a dict of agent loop config +used by hydra.utils.instantiate to initialize agent loop instance. + +https://hydra.cc/docs/advanced/instantiate_objects/overview/ +""" +_agent_loop_registry: dict[str, dict] = {} + + +def register(agent_name: str): + """Register agent loop class.""" + + def decorator(subclass: type[AgentLoopBase]) -> type[AgentLoopBase]: + fqdn = f"{subclass.__module__}.{subclass.__qualname__}" + _agent_loop_registry[agent_name] = {"_target_": fqdn} + return subclass + + return decorator + + +@ray.remote(num_cpus=1) +class BatchExecutor: + """Batch executor is used to collect requests into a batch execution""" + + def __init__(self, batch_func, micro_batch_size=1, max_batch_size=None): + """ + + Args: + batch_func: batch processing function. + micro_batch_size (int, optional): micro batch size. Defaults to 1. + max_batch_size: batch size for batching. + """ + self._q = queue.Queue() + self._batch_func = batch_func + self._max_batch = max_batch_size + self._micro_batch_size = micro_batch_size + + self._worker = threading.Thread(target=self._worker_loop, daemon=True) + self._worker.start() + + async def submit_task(self, item): + """ + Blocking submission, returning Future + Args: + item: function input + + Returns: + fut: function output + """ + fut = Future() + self._q.put((item, fut)) + async_fut = asyncio.wrap_future(fut) + res = await async_fut + return res + + def _worker_loop(self): + while True: + # 1. Fetch a full batch (block until at least one) + first, first_fut = self._q.get() + items = [first] + futs = [first_fut] + + # Take the remaining tasks at once + while True: + try: + next_item, next_fut = self._q.get_nowait() + items.append(next_item) + futs.append(next_fut) + if self._max_batch and len(items) >= self._max_batch: + break + except queue.Empty: + while len(items) % self._micro_batch_size != 0: + next_item, next_fut = self._q.get() + items.append(next_item) + futs.append(next_fut) + if self._max_batch and len(items) >= self._max_batch: + break + break + + try: + results = self._batch_func(items) + except Exception as e: + for f in futs: + f.set_exception(e) + else: + for f, r in zip(futs, results, strict=False): + f.set_result(r) + + +@ray.remote(num_cpus=1) +class RewardManagerWorker: + """Reward manager worker to compute reward score asynchronously to overlap with agent loop.""" + + def __init__(self, config: DictConfig, local_path: str, rm_executor: BatchExecutor = None) -> None: + tokenizer = hf_tokenizer(local_path, trust_remote_code=True) + self.reward_manager = load_reward_manager( + config, tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {}) + ) + self.rm_executor = rm_executor + self.loop = asyncio.get_event_loop() + + async def compute_score( + self, + data: DataProto, + ) -> dict: + """Compute reward score for agent loop output. + + NOTE: Since `reward_manager.__call__` is blocking function, we run it in thread pool to + compute multiple samples in parallel. + + Args: + data: reward function input + + Returns: + dict: Reward score and reward extra info. + """ + result = await self.loop.run_in_executor( + None, + self.reward_wrapper, + data, + True, # return_dict + ) + + reward_score = result["reward_tensor"].sum(dim=-1).item() + reward_extra_info = {k: v[0] for k, v in result.get("reward_extra_info", {}).items()} + return {"reward_score": reward_score, "reward_extra_info": reward_extra_info} + + def reward_wrapper(self, data: DataProto, return_dict=False) -> torch.Tensor: + """Assemble reward functions and reward model into one function and expose it to the event loop + Args: + return_dict: whether return as dict + data: DataProto from compute reward score + Returns: + torch.Tensor: Reward score tensor. + """ + if self.rm_executor is not None: + res = ray.get(self.rm_executor.submit_task.remote(data)) + data = data.union(res) + + return self.reward_manager(data, return_dict) + + +@ray.remote +class AgentLoopWorker: + """Agent loop worker takes a batch of messages and run each message in an agent loop.""" + + def __init__( + self, config: DictConfig, server_handles: list[ray.actor.ActorHandle], rm_executor: BatchExecutor = None + ): + """Initialize agent loop manager. + + Args: + config (DictConfig): YAML config. + server_handles (List[ray.actor.ActorHandle]): OpenAI compatible LLM server actor handles. + """ + self.config = config + self.server_manager = AsyncLLMServerManager(config, server_handles) + self.rm_executor = rm_executor + + model_path = config.actor_rollout_ref.model.path + self.model_name = "/".join(model_path.split("/")[-2:]) + local_path = copy_to_local(config.actor_rollout_ref.model.path) + self.tokenizer = hf_tokenizer(local_path, trust_remote_code=True) + self.processor = hf_processor(local_path, trust_remote_code=True) + + agent_loop_config_path = config.actor_rollout_ref.rollout.agent.agent_loop_config_path + if agent_loop_config_path: + agent_loop_configs = OmegaConf.load(agent_loop_config_path) + for agent_loop_config in agent_loop_configs: + _agent_loop_registry[agent_loop_config.name] = agent_loop_config + if self.config.actor_rollout_ref.model.get("custom_chat_template", None) is not None: + if self.processor is not None: + self.processor.chat_template = self.config.actor_rollout_ref.model.custom_chat_template + self.tokenizer.chat_template = self.config.actor_rollout_ref.model.custom_chat_template + + self.reward_manager_worker = RewardManagerWorker.options( + scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=ray.get_runtime_context().get_node_id(), + soft=False, + ), + ).remote(self.config, local_path, self.rm_executor) + + trace_config = self.config.actor_rollout_ref.rollout.get("trace", {}) + RolloutTraceConfig.init( + self.config.trainer.project_name, + self.config.trainer.experiment_name, + trace_config.get("backend"), + trace_config.get("token2text", False), + ) + + async def generate_sequences(self, batch: DataProto) -> DataProto: + """Generate sequences from agent loop. + + Args: + batch (DataProto): Input batch. + + Returns: + DataProto: Output batch. + - prompts: [bsz, prompt_length], prompt token ids from dataset. + - responses: [bsz, response_length], output token ids include response tokens + from LLM generation and observation tokens from tool_calls. + - response_mask: [bsz, response_length], 1 for LLM generated tokens, 0 for observation/padding tokens. + - input_ids: [bsz, prompt_length + response_length], whole sequence token ids, including prompt tokens + and response tokens. + - attention_mask: [bsz, prompt_length + response_length], 0 for padding tokens, 1 for other tokens. + - position_ids: [bsz, prompt_length + response_length], incremental position ids. + + For multi-turn conversations: + responses: |<- LLM generation ->|<- tool_calls ->|<- LLM generation ->|<- padding ->| + response_mask: | 1, 1, 1, ..., 1, 1 | 0, 0, .., 0, 0 | 1, 1, 1, ..., 1, 1 | 0, 0, ..., 0| + """ + config = self.config.actor_rollout_ref.rollout + sampling_params = dict( + temperature=config.temperature, + top_p=config.top_p, + repetition_penalty=1.0, + logprobs=config.calculate_log_probs, + ) + + # override sampling params for validation + if batch.meta_info.get("validate", False): + sampling_params["top_p"] = config.val_kwargs.top_p + sampling_params["temperature"] = config.val_kwargs.temperature + + # by default, we assume it's a single turn agent + if "agent_name" not in batch.non_tensor_batch: + batch.non_tensor_batch["agent_name"] = np.array(["single_turn_agent"] * len(batch), dtype=object) + + if "index" in batch.non_tensor_batch: + index = batch.non_tensor_batch["index"] + else: + index = np.arange(len(batch)) + + trajectory_info = await get_trajectory_info( + batch.meta_info.get("global_steps", -1), index.tolist(), batch.meta_info.get("validate", False) + ) + + tasks = [] + for i in range(len(batch)): + kwargs = {k: v[i] for k, v in batch.non_tensor_batch.items()} + tasks.append(asyncio.create_task(self._run_agent_loop(sampling_params, trajectory_info[i], **kwargs))) + outputs = await asyncio.gather(*tasks) + + output = self._postprocess(outputs) + return output + + async def _run_agent_loop( + self, + sampling_params: dict[str, Any], + trajectory: dict[str, Any], + *, + agent_name: str, + **kwargs, + ) -> _InternalAgentLoopOutput: + with rollout_trace_attr( + step=trajectory["step"], + sample_index=trajectory["sample_index"], + rollout_n=trajectory["rollout_n"], + validate=trajectory["validate"], + name="agent_loop", + ): + assert agent_name in _agent_loop_registry, ( + f"Agent loop {agent_name} not registered, registered agent loops: {_agent_loop_registry.keys()}" + ) + + agent_loop_config = _agent_loop_registry[agent_name] + agent_loop = hydra.utils.instantiate( + config=agent_loop_config, + trainer_config=_DummyConfig(config=self.config), + server_manager=self.server_manager, + tokenizer=self.tokenizer, + processor=self.processor, + ) + output: AgentLoopOutput = await agent_loop.run(sampling_params, **kwargs) + + # Some AgentLoop may have already computed the reward score, e.g SWE-agent. + + # NOTE: consistent with batch version of generate_sequences in vllm_rollout_spmd.py + # prompt_ids: left padded with zeros (e.g., [0,0,0,0,1,2,3,4]) + # response_ids: right padded with zeros (e.g., [5,6,7,8,0,0,0,0]) + # input_ids: concatenation of prompt + response + # Mask: + # For example, if the prompt is [1,2,3,4] and the response is [5,6,7,(tool start)8,9(tool end),10,11,12] + # - prompt_attention_mask: 0s for padding, 1s for tokens + # e.g., [0,0,0,0,1,1,1,1] + # - response_attention_mask: 0s for padding, 1s for tokens + # e.g., [1,1,1,1,1,1,1,1,1,1,1,0,0,0,0] + # attention_mask: concatenation of prompt_attention_mask and response_attention_mask + # e.g., [0,0,0,0,1,1,1,1(prompt),1,1,1,1,1,1,1,1,1,1,1,0,0,0,0(response)] + # - response_mask: 1s for LLM generated tokens, 0 for tool response/padding tokens + # e.g., [1,1,1,1,1,1,1,(tool start),0,0(tool end),1,1,0,0,0,0] + # - position_ids: sequential positions for tokens, starting at 0 + # e.g., [0,0,0,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0,0,0,0] + + self.tokenizer.padding_side = "left" + prompt_output = self.tokenizer.pad( + {"input_ids": output.prompt_ids}, + padding="max_length", + max_length=self.config.actor_rollout_ref.rollout.prompt_length, + return_tensors="pt", + return_attention_mask=True, + ) + if prompt_output["input_ids"].dim() == 1: + prompt_output["input_ids"] = prompt_output["input_ids"].unsqueeze(0) + prompt_output["attention_mask"] = prompt_output["attention_mask"].unsqueeze(0) + + self.tokenizer.padding_side = "right" + response_output = self.tokenizer.pad( + {"input_ids": output.response_ids}, + padding="max_length", + max_length=self.config.actor_rollout_ref.rollout.response_length, + return_tensors="pt", + return_attention_mask=True, + ) + if response_output["input_ids"].dim() == 1: + response_output["input_ids"] = response_output["input_ids"].unsqueeze(0) + response_output["attention_mask"] = response_output["attention_mask"].unsqueeze(0) + + response_mask_output = self.tokenizer.pad( + {"input_ids": output.response_mask}, + padding="max_length", + max_length=self.config.actor_rollout_ref.rollout.response_length, + return_tensors="pt", + return_attention_mask=False, + ) + if response_mask_output["input_ids"].dim() == 1: + response_mask_output["input_ids"] = response_mask_output["input_ids"].unsqueeze(0) + + response_logprobs = None + if output.response_logprobs is not None: + pad_size = self.config.actor_rollout_ref.rollout.response_length - len(output.response_logprobs) + response_logprobs = torch.tensor(output.response_logprobs + [0.0] * pad_size).unsqueeze(0) + + response_mask = response_mask_output["input_ids"] * response_output["attention_mask"] + attention_mask = torch.cat([prompt_output["attention_mask"], response_output["attention_mask"]], dim=1) + input_ids = torch.cat([prompt_output["input_ids"], response_output["input_ids"]], dim=1) + + # Handle multi-modal inputs and position_ids calculation + # Only support Qwen2VLImageProcessor for multi-modal processing currently + # TODO: support other multi-modal inputs + multi_modal_inputs = None + if ( + self.processor is not None + and "Qwen2VLImageProcessor" in self.processor.image_processor.__class__.__name__ + ): + from verl.models.transformers.qwen2_vl import get_rope_index + + images = output.multi_modal_data.get("image", None) + current_text = self.tokenizer.decode(input_ids.squeeze(0), skip_special_tokens=True) + multi_modal_inputs = self.processor(text=[current_text], images=images, return_tensors="pt") + multi_modal_inputs.pop("input_ids", None) + multi_modal_inputs.pop("attention_mask", None) + + # We must use dict(multi_modal_inputs) to convert BatchFeature values to a new dict + # because np.array() only keeps the keys for BatchFeature. + multi_modal_inputs = dict(multi_modal_inputs) + + image_grid_thw = multi_modal_inputs.get("image_grid_thw") + video_grid_thw = multi_modal_inputs.get("video_grid_thw") + second_per_grid_ts = multi_modal_inputs.get("second_per_grid_ts") + + position_ids = get_rope_index( + self.processor, + input_ids=input_ids.squeeze(0), + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + second_per_grid_ts=second_per_grid_ts, + attention_mask=attention_mask.squeeze(0), + ).unsqueeze(0) # (1, 3, seq_len) + else: + position_ids = compute_position_id_with_mask(attention_mask) # (1, seq_len) + enable_async_reward = ( + self.rm_executor is not None and self.config.reward_model.enable_resource_pool + ) or not self.config.reward_model.enable + if output.reward_score is None and enable_async_reward: + batch = TensorDict( + { + "prompts": prompt_output["input_ids"], # [1, prompt_length] + "responses": response_output["input_ids"], # [1, response_length] + "attention_mask": attention_mask, # [1, prompt_length + response_length] + "input_ids": input_ids, # [1, prompt_length + response_length] + "position_ids": position_ids, + }, + batch_size=1, + ) + non_tensor_batch = { + **{k: np.array([v]) for k, v in kwargs.items()}, + "__num_turns__": np.array([output.num_turns]), + } + extra_fields = {} + for key, val in output.extra_fields.items(): + extra_fields[key] = np.array([val], dtype=object) + + non_tensor_batch.update(extra_fields) + data = DataProto( + batch=batch, + non_tensor_batch=non_tensor_batch, + ) + result = await self.reward_manager_worker.compute_score.remote(data) + output.reward_score = result["reward_score"] + output.extra_fields["reward_extra_info"] = result["reward_extra_info"] + + return _InternalAgentLoopOutput( + prompt_ids=prompt_output["input_ids"], + response_ids=response_output["input_ids"], + input_ids=input_ids, + position_ids=position_ids, + response_mask=response_mask, + attention_mask=attention_mask, + response_logprobs=response_logprobs, + multi_modal_inputs=multi_modal_inputs, + multi_modal_data=output.multi_modal_data, + reward_score=output.reward_score, + num_turns=output.num_turns, + metrics=output.metrics, + extra_fields=output.extra_fields, + ) + + def _postprocess(self, inputs: list[_InternalAgentLoopOutput]) -> DataProto: + """Process the padded outputs from _run_agent_loop and combine them into a batch.""" + # Convert lists back to tensors and stack them to create a batch. + prompt_ids = torch.cat([input.prompt_ids for input in inputs], dim=0) + response_ids = torch.cat([input.response_ids for input in inputs], dim=0) + response_mask = torch.cat([input.response_mask for input in inputs], dim=0) + attention_mask = torch.cat([input.attention_mask for input in inputs], dim=0) + input_ids = torch.cat([input.input_ids for input in inputs], dim=0) + position_ids = torch.cat([input.position_ids for input in inputs], dim=0) + optional_outputs = {} + if inputs[0].response_logprobs is not None: + optional_outputs["rollout_log_probs"] = torch.cat([input.response_logprobs for input in inputs], dim=0) + + batch = TensorDict( + { + "prompts": prompt_ids, # [bsz, prompt_length] + "responses": response_ids, # [bsz, response_length] + "response_mask": response_mask, # [bsz, response_length] + "input_ids": input_ids, # [bsz, prompt_length + response_length] + "attention_mask": attention_mask, # [bsz, prompt_length + response_length] + # position_ids: [bsz, 3, prompt_length + response_length] or [bsz, prompt_length + response_length] + "position_ids": position_ids, + **optional_outputs, + }, + batch_size=len(inputs), + ) + + scores = [input.reward_score for input in inputs] + if all(score is not None for score in scores): + prompt_length = prompt_ids.size(1) + response_length = attention_mask[:, prompt_length:].sum(dim=1) - 1 + rm_scores = torch.zeros_like(response_mask, dtype=torch.float32) + rm_scores[torch.arange(response_mask.size(0)), response_length] = torch.tensor(scores, dtype=torch.float32) + batch["rm_scores"] = rm_scores + + non_tensor_batch = { + "__num_turns__": np.array([input.num_turns for input in inputs], dtype=np.int32), + } + + # add reward_extra_info to non_tensor_batch + reward_extra_infos = [input.extra_fields.get("reward_extra_info", {}) for input in inputs] + reward_extra_keys = list(reward_extra_infos[0].keys()) + for key in reward_extra_keys: + non_tensor_batch[key] = np.array([info[key] for info in reward_extra_infos]) + + # Add multi_modal_inputs to non_tensor_batch if any samples have them + multi_modal_inputs_list = [input.multi_modal_inputs for input in inputs] + if any(mmi is not None for mmi in multi_modal_inputs_list): + non_tensor_batch["multi_modal_inputs"] = np.array(multi_modal_inputs_list, dtype=object) + + metrics = [input.metrics.model_dump() for input in inputs] + # Collect extra fields from all inputs and convert them to np.ndarray + extra_fields = {} + all_keys = set(key for input_item in inputs for key in input_item.extra_fields) + for key in all_keys: + temp_arr = np.empty(len(inputs), dtype=object) + temp_arr[:] = [input.extra_fields.get(key) for input in inputs] + extra_fields[key] = temp_arr + + non_tensor_batch.update(extra_fields) + return DataProto( + batch=batch, + non_tensor_batch=non_tensor_batch, + meta_info={"metrics": metrics, "reward_extra_keys": reward_extra_keys}, + ) + + +async def get_trajectory_info(step, index, validate): + """Get trajectory info. + + Args: + step (int): global steps in the trainer. + index (list): form datastore extra_info.index column. + validate (bool): whether is a validate step. + + Returns: + list: trajectory. + """ + trajectory_info = [] + rollout_n = 0 + for i in range(len(index)): + if i > 0 and index[i - 1] == index[i]: + rollout_n += 1 + else: + rollout_n = 0 + trajectory_info.append({"step": step, "sample_index": index[i], "rollout_n": rollout_n, "validate": validate}) + return trajectory_info + + +class AgentLoopManager: + """Agent loop manager that manages a group of agent loop workers.""" + + def __init__(self, config: DictConfig, worker_group: RayWorkerGroup = None, rm_wg: RayWorkerGroup = None): + """Initialize agent loop manager. + + Args: + config (DictConfig): trainer config. + worker_group (RayWorkerGroup): ActorRolloutRef worker group for hybrid mode; None for standalone mode. + """ + self.config = config + self.worker_group = worker_group + self.rm_executor = None + self.rm_micro_batch_size = None + if rm_wg: + + def batch_fn(data_list: list[DataProto]) -> list[torch.Tensor]: + new_data_list = [] + for data in data_list: + temp_non_tensor_batch = {"__num_turns__": data.non_tensor_batch["__num_turns__"]} + temp_data = DataProto(batch=data.batch, non_tensor_batch=temp_non_tensor_batch) + new_data_list.append(temp_data) + + new_batch = DataProto.concat(new_data_list) + out_data = rm_wg.compute_rm_score(new_batch) + return out_data.split(1) + + self.rm_executor = BatchExecutor.options( + scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=ray.get_runtime_context().get_node_id(), + soft=False, + ), + ).remote(batch_fn, rm_wg.world_size) + + self.rm_micro_batch_size = rm_wg.world_size + + self._initialize_llm_servers() + self._init_agent_loop_workers() + + # Initially we're in sleep mode. + if self.config.actor_rollout_ref.rollout.free_cache_engine: + self.sleep() + + def _initialize_llm_servers(self): + rollout_world_size = ( + self.config.actor_rollout_ref.rollout.tensor_model_parallel_size + * self.config.actor_rollout_ref.rollout.data_parallel_size + ) + world_size = ( + self.worker_group.world_size + if self.worker_group + else self.config.trainer.n_gpus_per_node * self.config.trainer.nnodes + ) + num_replicas = world_size // rollout_world_size + + rollout_replica_class = get_rollout_replica_class(self.config.actor_rollout_ref.rollout.name) + rollout_config = self.config.actor_rollout_ref.rollout + model_config = self.config.actor_rollout_ref.model + self.rollout_replicas = [ + rollout_replica_class( + replica_rank=replica_rank, + config=rollout_config, + model_config=model_config, + gpus_per_node=self.config.trainer.n_gpus_per_node, + ) + for replica_rank in range(num_replicas) + ] + if self.worker_group: + self._run_all([server.init_hybrid(self.worker_group) for server in self.rollout_replicas]) + else: + self._run_all([server.init_standalone() for server in self.rollout_replicas]) + self.server_handles = [server._server_handle for server in self.rollout_replicas] + self.server_addresses = [server._server_address for server in self.rollout_replicas] + + def _init_agent_loop_workers(self): + self.agent_loop_workers = [] + num_workers = self.config.actor_rollout_ref.rollout.agent.num_workers + + node_ids = [node["NodeID"] for node in ray.nodes() if node["Alive"] and node["Resources"].get("CPU", 0) > 0] + for i in range(num_workers): + # Round-robin scheduling over the all nodes + node_id = node_ids[i % len(node_ids)] + self.agent_loop_workers.append( + AgentLoopWorker.options( + name=f"agent_loop_worker_{i}", + scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=node_id, soft=True + ), + ).remote(self.config, self.server_handles, self.rm_executor) + ) + + def generate_sequences(self, prompts: DataProto) -> DataProto: + """Split input batch and dispatch to agent loop workers. + + Args: + prompts (DataProto): Input batch. + + Returns: + DataProto: Output batch. + """ + + if self.rm_micro_batch_size and len(prompts) % self.rm_micro_batch_size != 0: + raise ValueError( + f"The length of prompts {len(prompts)} cannot divide the world size of rm_wg {self.rm_micro_batch_size}" + ) + if self.config.actor_rollout_ref.rollout.free_cache_engine: + self.wake_up() + chunkes = prompts.chunk(len(self.agent_loop_workers)) + outputs = ray.get( + [ + worker.generate_sequences.remote(chunk) + for worker, chunk in zip(self.agent_loop_workers, chunkes, strict=True) + ] + ) + output = DataProto.concat(outputs) + if self.config.actor_rollout_ref.rollout.free_cache_engine: + self.sleep() + + # calculate performance metrics + metrics = [output.meta_info.pop("metrics") for output in outputs] # List[List[Dict[str, str]]] + timing = self._performance_metrics(metrics, output) + + output.meta_info = {"timing": timing, **outputs[0].meta_info} + return output + + def _performance_metrics(self, metrics: list[list[dict[str, str]]], output: DataProto) -> dict[str, float]: + timing = {} + t_generate_sequences = np.array([metric["generate_sequences"] for chunk in metrics for metric in chunk]) + t_tool_calls = np.array([metric["tool_calls"] for chunk in metrics for metric in chunk]) + timing["agent_loop/generate_sequences/min"] = t_generate_sequences.min() + timing["agent_loop/generate_sequences/max"] = t_generate_sequences.max() + timing["agent_loop/generate_sequences/mean"] = t_generate_sequences.mean() + timing["agent_loop/tool_calls/min"] = t_tool_calls.min() + timing["agent_loop/tool_calls/max"] = t_tool_calls.max() + timing["agent_loop/tool_calls/mean"] = t_tool_calls.mean() + + # batch sequence generation is bounded by the slowest sample + slowest = np.argmax(t_generate_sequences + t_tool_calls) + attention_mask = output.batch["attention_mask"][slowest] + prompt_length = output.batch["prompts"].shape[1] + timing["agent_loop/slowest/generate_sequences"] = t_generate_sequences[slowest] + timing["agent_loop/slowest/tool_calls"] = t_tool_calls[slowest] + timing["agent_loop/slowest/prompt_length"] = attention_mask[:prompt_length].sum().item() + timing["agent_loop/slowest/response_length"] = attention_mask[prompt_length:].sum().item() + + return timing + + def wake_up(self): + """Wake up all rollout replica instances.""" + self._run_all([replica.wake_up() for replica in self.rollout_replicas]) + + def sleep(self): + """Sleep all rollout replica instances.""" + self._run_all([replica.sleep() for replica in self.rollout_replicas]) + + def _run_all(self, tasks: list[asyncio.Task]): + async def run_all(): + await asyncio.gather(*tasks) + + asyncio.run(run_all()) diff --git a/e2c/verl/verl/experimental/agent_loop/single_turn_agent_loop.py b/e2c/verl/verl/experimental/agent_loop/single_turn_agent_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..9196f8138b26889d12096e230e81c71b05a324f1 --- /dev/null +++ b/e2c/verl/verl/experimental/agent_loop/single_turn_agent_loop.py @@ -0,0 +1,63 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from typing import Any +from uuid import uuid4 + +from verl.experimental.agent_loop.agent_loop import AgentLoopBase, AgentLoopOutput, register +from verl.utils.profiler import simple_timer + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +@register("single_turn_agent") +class SingleTurnAgentLoop(AgentLoopBase): + """Naive agent loop that only do single turn chat completion.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.prompt_length = self.config.actor_rollout_ref.rollout.prompt_length + self.response_length = self.config.actor_rollout_ref.rollout.response_length + self.apply_chat_template_kwargs = self.config.data.get("apply_chat_template_kwargs", {}) + + async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput: + messages = list(kwargs["raw_prompt"]) + + metrics = {} + request_id = uuid4().hex + prompt_ids = await self.loop.run_in_executor( + None, + lambda: self.tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=True, **self.apply_chat_template_kwargs + ), + ) + + with simple_timer("generate_sequences", metrics): + output = await self.server_manager.generate( + request_id=request_id, prompt_ids=prompt_ids, sampling_params=sampling_params + ) + response_mask = [1] * len(output.token_ids) + + output = AgentLoopOutput( + prompt_ids=prompt_ids, + response_ids=output.token_ids[: self.response_length], + response_mask=response_mask[: self.response_length], + response_logprobs=output.log_probs[: self.response_length] if output.log_probs else None, + multi_modal_data={}, + num_turns=2, + metrics=metrics, + ) + return output diff --git a/e2c/verl/verl/experimental/agent_loop/tool_agent_loop.py b/e2c/verl/verl/experimental/agent_loop/tool_agent_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..c3bd1a6fc553253380a75bf80595007b39e9d5a1 --- /dev/null +++ b/e2c/verl/verl/experimental/agent_loop/tool_agent_loop.py @@ -0,0 +1,469 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import copy +import json +import logging +import os +from enum import Enum +from typing import Any, Optional +from uuid import uuid4 + +from verl.experimental.agent_loop.agent_loop import AgentLoopBase, AgentLoopOutput, register +from verl.experimental.agent_loop.tool_parser import FunctionCall, ToolParser +from verl.interactions.base import BaseInteraction +from verl.interactions.utils.interaction_registry import initialize_interactions_from_config +from verl.tools.schemas import ToolResponse +from verl.tools.utils.tool_registry import initialize_tools_from_config +from verl.utils.profiler import simple_timer +from verl.utils.rollout_trace import rollout_trace_op + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class AgentState(Enum): + PENDING = "pending" + GENERATING = "generating" + PROCESSING_TOOLS = "processing_tools" + TERMINATED = "terminated" + INTERACTING = "interacting" + + +class AgentData: + """Encapsulates all state variables for the agent loop.""" + + def __init__( + self, + messages: list[dict[str, Any]], + image_data: Any, + metrics: dict[str, Any], + request_id: str, + tools_kwargs: dict[str, Any], + interaction: Optional[BaseInteraction] = None, + interaction_kwargs: Optional[dict[str, Any]] = None, + ): + self.messages = messages + self.image_data = image_data + self.metrics = metrics + self.request_id = request_id + self.tools_kwargs = tools_kwargs + self.interaction = interaction + self.interaction_kwargs = interaction_kwargs or {} + + # State variables + self.prompt_ids: list[int] = [] + self.response_ids: list[int] = [] + self.response_mask: list[int] = [] + self.response_logprobs: list[float] = [] + self.turn_scores: list[float] = [] + self.tool_rewards: list[float] = [] + self.user_turns = 0 + self.assistant_turns = 0 + + # Temporary state for tool calls + self.tool_calls: list[FunctionCall] = [] + + +@register("tool_agent") +class ToolAgentLoop(AgentLoopBase): + @classmethod + def init_class(cls, config, tokenizer, processor, **kwargs): + if cls._class_initialized: + return + cls._class_initialized = True + print("Performing class-level ToolAgentLoop initialization") + + # Initialize tools from config file + cls.tokenizer = tokenizer + cls.processor = processor + cls.max_user_turns = config.actor_rollout_ref.rollout.multi_turn.max_user_turns + cls.max_assistant_turns = config.actor_rollout_ref.rollout.multi_turn.max_assistant_turns + cls.max_parallel_calls = config.actor_rollout_ref.rollout.multi_turn.max_parallel_calls + cls.max_tool_response_length = config.actor_rollout_ref.rollout.multi_turn.max_tool_response_length + cls.tool_response_truncate_side = config.actor_rollout_ref.rollout.multi_turn.tool_response_truncate_side + tool_config_path = config.actor_rollout_ref.rollout.multi_turn.tool_config_path + tool_list = initialize_tools_from_config(tool_config_path) if tool_config_path else [] + cls.tools = {tool.name: tool for tool in tool_list} + cls.tool_schemas = [tool.tool_schema.model_dump(exclude_unset=True, exclude_none=True) for tool in tool_list] + cls.tool_parser = ToolParser.get_tool_parser(config.actor_rollout_ref.rollout.multi_turn.format, cls.tokenizer) + print(f"Initialized tools: {cls.tools}") + + cls.apply_chat_template_kwargs = config.data.get("apply_chat_template_kwargs", {}) + cls.prompt_length = config.actor_rollout_ref.rollout.prompt_length + cls.response_length = config.actor_rollout_ref.rollout.response_length + cls.system_prompt = tokenizer.apply_chat_template( + [{}], add_generation_prompt=False, tokenize=True, **cls.apply_chat_template_kwargs + ) + # Initialize interactions from config file + cls.interaction_config_file = config.actor_rollout_ref.rollout.multi_turn.interaction_config_path + if cls.interaction_config_file: + cls.interaction_map: dict[str, BaseInteraction] = cls._initialize_interactions(cls.interaction_config_file) + + @rollout_trace_op + async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput: + messages = list(kwargs["raw_prompt"]) + image_data = copy.deepcopy(kwargs.get("multi_modal_data", {}).get("image", None)) + metrics = {} + request_id = uuid4().hex + tools_kwargs = kwargs.get("tools_kwargs", {}) + + # Initialize interaction if needed + interaction = None + interaction_kwargs = {} + if self.interaction_config_file: + interaction_kwargs = kwargs["extra_info"]["interaction_kwargs"] + if "name" not in interaction_kwargs: + raise ValueError("'name' key is required in interaction_kwargs") + interaction_name = interaction_kwargs["name"] + if interaction_name not in self.interaction_map: + raise ValueError( + f"Interaction '{interaction_name}' not found in interaction_map. Available interactions: " + f"{list(self.interaction_map.keys())}" + ) + interaction = self.interaction_map[interaction_name] + await interaction.start_interaction(request_id, **interaction_kwargs) + # Create AgentData instance to encapsulate all state + agent_data = AgentData( + messages=messages, + image_data=image_data, + metrics=metrics, + request_id=request_id, + tools_kwargs=tools_kwargs, + interaction=interaction, + interaction_kwargs=interaction_kwargs, + ) + + # State machine loop + state = AgentState.PENDING + while state != AgentState.TERMINATED: + if state == AgentState.PENDING: + state = await self._handle_pending_state(agent_data, sampling_params) + elif state == AgentState.GENERATING: + state = await self._handle_generating_state(agent_data, sampling_params) + elif state == AgentState.PROCESSING_TOOLS: + state = await self._handle_processing_tools_state(agent_data) + elif state == AgentState.INTERACTING: + state = await self._handle_interacting_state(agent_data) + else: + logger.error(f"Invalid state: {state}") + state = AgentState.TERMINATED + + # Finalize output + response_ids = agent_data.prompt_ids[-len(agent_data.response_mask) :] + prompt_ids = agent_data.prompt_ids[: len(agent_data.prompt_ids) - len(agent_data.response_mask)] + multi_modal_data = {"image": agent_data.image_data} if agent_data.image_data is not None else {} + output = AgentLoopOutput( + prompt_ids=prompt_ids, + response_ids=response_ids[: self.response_length], + response_mask=agent_data.response_mask[: self.response_length], + multi_modal_data=multi_modal_data, + response_logprobs=agent_data.response_logprobs[: self.response_length] + if agent_data.response_logprobs + else None, + num_turns=agent_data.user_turns + agent_data.assistant_turns + 1, + metrics=agent_data.metrics, + extra_fields={}, + ) + output.extra_fields.update({"turn_scores": agent_data.turn_scores, "tool_rewards": agent_data.tool_rewards}) + return output + + async def _handle_pending_state(self, agent_data: AgentData, sampling_params: dict[str, Any]) -> AgentState: + """Handle the pending state: prepare the prompt and start generation.""" + if self.processor is not None: + raw_prompt = await self.loop.run_in_executor( + None, + lambda: self.processor.apply_chat_template( + agent_data.messages, + tools=self.tool_schemas, + add_generation_prompt=True, + tokenize=False, + **self.apply_chat_template_kwargs, + ), + ) + model_inputs = self.processor(text=[raw_prompt], images=agent_data.image_data, return_tensors="pt") + agent_data.prompt_ids = model_inputs.pop("input_ids").squeeze(0).tolist() + else: + agent_data.prompt_ids = await self.loop.run_in_executor( + None, + lambda: self.tokenizer.apply_chat_template( + agent_data.messages, + tools=self.tool_schemas, + add_generation_prompt=True, + tokenize=True, + **self.apply_chat_template_kwargs, + ), + ) + return AgentState.GENERATING + + async def _handle_generating_state( + self, agent_data: AgentData, sampling_params: dict[str, Any], ignore_termination: bool = False + ) -> AgentState: + """Handle the generating state: generate model response and check for tool calls.""" + add_messages: list[dict[str, Any]] = [] + + with simple_timer("generate_sequences", agent_data.metrics): + output = await self.server_manager.generate( + request_id=agent_data.request_id, + prompt_ids=agent_data.prompt_ids, + sampling_params=sampling_params, + image_data=agent_data.image_data, + ) + + agent_data.assistant_turns += 1 + agent_data.response_ids = output.token_ids + agent_data.prompt_ids += agent_data.response_ids + agent_data.response_mask += [1] * len(agent_data.response_ids) + if output.log_probs: + agent_data.response_logprobs += output.log_probs + + # Check termination conditions + if not ignore_termination and len(agent_data.response_mask) >= self.response_length: + return AgentState.TERMINATED + if self.max_assistant_turns and agent_data.assistant_turns >= self.max_assistant_turns: + return AgentState.TERMINATED + if self.max_user_turns and agent_data.user_turns >= self.max_user_turns: + return AgentState.TERMINATED + + # Extract tool calls + _, agent_data.tool_calls = await self.tool_parser.extract_tool_calls(agent_data.response_ids) + + # Handle interaction if needed + if self.interaction_config_file: + assistant_message = await self.loop.run_in_executor( + None, lambda: self.tokenizer.decode(agent_data.response_ids, skip_special_tokens=True) + ) + add_messages.append({"role": "assistant", "content": assistant_message}) + agent_data.messages.extend(add_messages) + + # Determine next state + if agent_data.tool_calls: + return AgentState.PROCESSING_TOOLS + elif self.interaction_config_file: + return AgentState.INTERACTING + else: + return AgentState.TERMINATED + + async def _handle_processing_tools_state(self, agent_data: AgentData) -> AgentState: + """Handle the processing tools state: execute tool calls and prepare tool responses.""" + add_messages: list[dict[str, Any]] = [] + new_images_this_turn: list[Any] = [] # Local variable instead of agent_data attribute + + tasks = [] + for tool_call in agent_data.tool_calls[: self.max_parallel_calls]: + tasks.append(self._call_tool(tool_call, agent_data.tools_kwargs)) + + with simple_timer("tool_calls", agent_data.metrics): + responses = await asyncio.gather(*tasks) + + # Process tool responses and update multi_modal_data + # Removed: agent_data.new_images_this_turn = [] + for tool_response, tool_reward, _ in responses: + # Create message from tool response + if tool_response.image or tool_response.video: + # Multi-modal content with structured format + if not getattr(self.processor, "image_processor", None): + raise ValueError( + "Multimedia data can only be processed by `processor`, but the processor is None. " + "This error is often caused if you are using a LLM model but your tool returns multimodal " + "data. Plase use a vlm as the base model." + ) + content = [] + if tool_response.image: + content.append({"type": "image"}) + if tool_response.video: + content.append({"type": "video"}) + if tool_response.text: + content.append({"type": "text", "text": tool_response.text}) + message = {"role": "tool", "content": content} + else: + # Text-only content + message = {"role": "tool", "content": tool_response.text or ""} + + add_messages.append(message) + agent_data.messages.extend(add_messages) + + # Handle image data + if tool_response.image: + if agent_data.image_data is None: + agent_data.image_data = [] + elif not isinstance(agent_data.image_data, list): + agent_data.image_data = [agent_data.image_data] + + # Add new image data + if isinstance(tool_response.image, list): + # Ensure all elements in the list are valid image objects + for img in tool_response.image: + if img is not None: # Add a check to ensure the image is not None + agent_data.image_data.append(img) + new_images_this_turn.append(img) # Using local variable + else: + # Ensure the image is not None + if tool_response.image is not None: + agent_data.image_data.append(tool_response.image) + new_images_this_turn.append(tool_response.image) # Using local variable + + # Handle video data + if tool_response.video: + # Currently not supported, raise informative error + logger.warning("Multimedia type 'video' is not currently supported. Only 'image' is supported.") + raise NotImplementedError( + "Multimedia type 'video' is not currently supported. Only 'image' is supported." + ) + + if tool_reward is not None: + agent_data.tool_rewards.append(tool_reward) + + # Update prompt with tool responses + if self.processor is not None: + raw_tool_response = await self.loop.run_in_executor( + None, + lambda: self.processor.apply_chat_template( + add_messages, + add_generation_prompt=True, + tokenize=False, + **self.apply_chat_template_kwargs, + ), + ) + # Use only the new images from this turn for processing tool responses + current_images = new_images_this_turn if new_images_this_turn else None # Using local variable + model_inputs = self.processor(text=[raw_tool_response], images=current_images, return_tensors="pt") + response_ids = model_inputs.pop("input_ids").squeeze(0).tolist() + else: + response_ids = await self.loop.run_in_executor( + None, + lambda: self.tokenizer.apply_chat_template(add_messages, add_generation_prompt=True, tokenize=True), + ) + response_ids = response_ids[len(self.system_prompt) :] + if len(agent_data.response_mask) + len(response_ids) >= self.response_length: + return AgentState.TERMINATED + # Update prompt_ids and response_mask + agent_data.prompt_ids += response_ids + agent_data.response_mask += [0] * len(response_ids) + if agent_data.response_logprobs: + agent_data.response_logprobs += [0.0] * len(response_ids) + agent_data.user_turns += 1 + return AgentState.GENERATING + + async def _handle_interacting_state(self, agent_data: AgentData) -> AgentState: + """Handle the interacting state: get user input from interaction.""" + ( + should_terminate_sequence, + interaction_responses, + reward, + metrics, + ) = await agent_data.interaction.generate_response( + agent_data.request_id, agent_data.messages, **agent_data.interaction_kwargs + ) + agent_data.user_turns += 1 + + add_messages: list[dict[str, Any]] = [{"role": "user", "content": interaction_responses}] + agent_data.messages.extend(add_messages) + + if reward is not None: + agent_data.turn_scores.append(reward) + + # Update prompt with user responses (similar to _handle_processing_tools_state) + if self.processor is not None: + raw_user_response = await self.loop.run_in_executor( + None, + lambda: self.processor.apply_chat_template( + add_messages, + add_generation_prompt=True, + tokenize=False, + **self.apply_chat_template_kwargs, + ), + ) + model_inputs = self.processor(text=[raw_user_response], images=None, return_tensors="pt") + response_ids = model_inputs.pop("input_ids").squeeze(0).tolist() + else: + response_ids = await self.loop.run_in_executor( + None, + lambda: self.tokenizer.apply_chat_template(add_messages, add_generation_prompt=True, tokenize=True), + ) + response_ids = response_ids[len(self.system_prompt) :] + + # Update prompt_ids and response_mask + agent_data.prompt_ids += response_ids + agent_data.response_mask += [0] * len(response_ids) + if agent_data.response_logprobs: + agent_data.response_logprobs += [0.0] * len(response_ids) + + # double check prompt + # Check termination condition + if should_terminate_sequence: + return AgentState.TERMINATED + else: + return AgentState.GENERATING + + async def _call_tool( + self, tool_call: FunctionCall, tools_kwargs: dict[str, Any] + ) -> tuple[ToolResponse, float, dict]: + """Call tool and return tool response.""" + tool, instance_id = None, None + try: + # TODO: append malformed tool_call to the prompt: invalid function name or arguments + tool_name = tool_call.name + tool_args = json.loads(tool_call.arguments) + tool = self.tools[tool_name] + kwargs = tools_kwargs.get(tool_name, {}) + instance_id, _ = await tool.create(create_kwargs=kwargs.get("create_kwargs", {})) + tool_execution_response, tool_reward, res = await tool.execute(instance_id, tool_args) + except Exception as e: + logger.warning(f"Error when executing tool: {e}") + return ( + ToolResponse( + text=f"Error when executing tool: {e}", + ), + 0.0, + {}, + ) + finally: + if tool and instance_id: + await tool.release(instance_id) + + tool_response_text = tool_execution_response.text + if tool_response_text and len(tool_response_text) > self.max_tool_response_length: + if self.tool_response_truncate_side == "left": + tool_response_text = tool_response_text[: self.max_tool_response_length] + "...(truncated)" + elif self.tool_response_truncate_side == "right": + tool_response_text = "(truncated)..." + tool_response_text[-self.max_tool_response_length :] + else: + length = self.max_tool_response_length // 2 + tool_response_text = tool_response_text[:length] + "...(truncated)..." + tool_response_text[-length:] + + # Create ToolResponse from tool execution result + tool_response_kwargs = {"text": tool_response_text} + + # Add multimedia data if present + for attr_name in ["image", "video"]: + if hasattr(tool_execution_response, attr_name): + attr_value = getattr(tool_execution_response, attr_name) + if attr_value is not None: + tool_response_kwargs[attr_name] = attr_value + + return ToolResponse(**tool_response_kwargs), tool_reward, res + + @classmethod + def _initialize_interactions(cls, interaction_config_file): + """Initialize interactions from configuration. + Returns: + dict[str, BaseInteraction]: A dictionary mapping interaction names to interaction instances. + """ + if interaction_config_file is None: + return {} + + interaction_map = initialize_interactions_from_config(interaction_config_file) + logger.info(f"Initialize interactions from configuration: interaction_map: {list(interaction_map.keys())}") + return interaction_map diff --git a/e2c/verl/verl/experimental/agent_loop/tool_parser.py b/e2c/verl/verl/experimental/agent_loop/tool_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..5b4de4a8e75521bcac217aeb0f18f6d1a0b9b5ec --- /dev/null +++ b/e2c/verl/verl/experimental/agent_loop/tool_parser.py @@ -0,0 +1,106 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import json +import logging +import os +from abc import ABC, abstractmethod + +import regex as re +from pydantic import BaseModel + +from verl.utils.rollout_trace import rollout_trace_op + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class FunctionCall(BaseModel): + arguments: str + """ + The arguments to call the function with, as generated by the model in JSON + format. Note that the model does not always generate valid JSON, and may + hallucinate parameters not defined by your function schema. Validate the + arguments in your code before calling your function. + """ + + name: str + """The name of the function to call.""" + + +class ToolParser(ABC): + _registry: dict[str, type["ToolParser"]] = {} + + def __init__(self, tokenizer) -> None: + self.tokenizer = tokenizer + + @abstractmethod + async def extract_tool_calls(self, responses_ids: list[int]) -> tuple[str, list[FunctionCall]]: + """Extract tool calls from the responses. + + Args: + responses_ids (List[int]): The ids of the responses. + + Returns: + Tuple[str, List[FunctionCall]]: Content and extracted tool calls. + """ + raise NotImplementedError + + @classmethod + def get_tool_parser(cls, name: str, tokenizer): + if name not in cls._registry: + raise ValueError(f"Unknown tool parser: {name}") + return cls._registry[name](tokenizer) + + @classmethod + def register(cls, name: str): + def decorator(subclass: type[ToolParser]) -> type[ToolParser]: + cls._registry[name] = subclass + return subclass + + return decorator + + +@ToolParser.register("hermes") +class HermesToolParser(ToolParser): + """Adapted from https://github.com/vllm-project/vllm/blob/v0.9.1/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py""" + + def __init__(self, tokenizer) -> None: + super().__init__(tokenizer) + + self.tool_call_start_token: str = "" + self.tool_call_end_token: str = "" + self.tool_call_regex = re.compile(r"(.*?)", re.DOTALL) + + @rollout_trace_op + async def extract_tool_calls(self, responses_ids: list[int]) -> tuple[str, list[FunctionCall]]: + loop = asyncio.get_running_loop() + text = await loop.run_in_executor(None, self.tokenizer.decode, responses_ids) + if self.tool_call_start_token not in text or self.tool_call_end_token not in text: + return text, [] + + matches = self.tool_call_regex.findall(text) + function_calls = [] + for match in matches: + try: + function_call = json.loads(match) + name, arguments = function_call["name"], function_call["arguments"] + function_calls.append(FunctionCall(name=name, arguments=json.dumps(arguments, ensure_ascii=False))) + except Exception as e: + logger.error(f"Failed to decode tool call: {e}") + + # remaing text exclude tool call tokens + content = self.tool_call_regex.sub("", text) + + return content, function_calls diff --git a/e2c/verl/verl/experimental/dataset/__init__.py b/e2c/verl/verl/experimental/dataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/experimental/dataset/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/experimental/dataset/sampler.py b/e2c/verl/verl/experimental/dataset/sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..b7b15b422c823280c862397dd88c362aac213554 --- /dev/null +++ b/e2c/verl/verl/experimental/dataset/sampler.py @@ -0,0 +1,40 @@ +# Copyright 2025 Amazon.com Inc and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import abstractmethod +from collections.abc import Sized + +from omegaconf import DictConfig +from torch.utils.data import Sampler + +from verl import DataProto + + +class AbstractSampler(Sampler[int]): + """Abstract interface for custom samplers.""" + + @abstractmethod + def __init__( + self, + data_source: Sized, + data_config: DictConfig, + ): + pass + + +class AbstractCurriculumSampler(AbstractSampler): + """Experimental interface for curriculum learning samplers.""" + + @abstractmethod + def update(self, batch: DataProto) -> None: + pass diff --git a/e2c/verl/verl/experimental/dynamic_dataset/__init__.py b/e2c/verl/verl/experimental/dynamic_dataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/experimental/dynamic_dataset/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/experimental/dynamic_dataset/dynamicgen_dataset.py b/e2c/verl/verl/experimental/dynamic_dataset/dynamicgen_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..a9532aa0345218e1b847a9a78d030e88821975bb --- /dev/null +++ b/e2c/verl/verl/experimental/dynamic_dataset/dynamicgen_dataset.py @@ -0,0 +1,112 @@ +# Copyright 2025 Amazon.com Inc and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Dataset class that enables dynamic data generation strategies between iterations of training. +This class extends RLHFDataset and uses an AbstractDataGen instance to generate data. + +This is especially useful in settings where proposer model generates new tasks based +on rollout data. +""" + +import logging +from abc import ABC, abstractmethod +from typing import Optional + +import datasets +from omegaconf import DictConfig +from torch.utils.data import Dataset +from transformers import PreTrainedTokenizer, ProcessorMixin + +from verl import DataProto +from verl.utils.dataset import RLHFDataset +from verl.utils.import_utils import load_extern_type + +logger = logging.getLogger(__name__) + + +class AbstractDataGenerator(ABC): + def __init__(self, config: DictConfig): + self.config = config + + @abstractmethod + def generate(self, dataset: Dataset) -> datasets.Dataset: + """ + Generate method must be implemented by subclasses. + Args: + dataset: The dataset to generate from. + Returns: + Processed data or result as implemented by the subclass. + """ + pass + + +class MockDataGenerator(AbstractDataGenerator): + """ + A noop data gen class that only reappends the first datapoint. + This class is useful as a placeholder and testing. + """ + + def __init__(self, config: DictConfig = None): + super().__init__(config) + + def generate(self, dataset: Dataset) -> datasets.Dataset: + print("MockDataGenerator: No operation performed on the dataset.") + return dataset.dataframe.select([0]) + + +class DynamicGenDataset(RLHFDataset): + """ + A dataset class that uses a data generation strategy to process data. + This class extends RLHFDataset and uses an AbstractDataGen instance to generate data. + """ + + def __init__( + self, + data_files: str | list[str], + tokenizer: PreTrainedTokenizer, + config: DictConfig, + processor: Optional[ProcessorMixin] = None, + ): + super().__init__(data_files, tokenizer, config, processor) + self.datagen: AbstractDataGenerator = config.datagen + assert "datagen" in config and config.datagen.get("path", None) is not None, ( + f"datagen path is not set in config: {config}" + ) + # Dynamically load the custom datagen class + datagen_cls = load_extern_type(config.datagen.path, config.datagen.name) + + # Verify that the custom datagen class inherits from AbstractDataGenerator + abs_cls = AbstractDataGenerator + if not issubclass(datagen_cls, abs_cls): + raise TypeError( + f"The custom datagen class '{config.datagen.name}' from '{config.datagen.path}'" + + " must inherit from {abs_cls}" + ) + + self.data_generator = datagen_cls(config.datagen) + self.on_batch_end() + + def append_dataframe(self, new_dataframe: datasets.Dataset): + new_dataframe = self.maybe_filter_out_long_prompts(new_dataframe) + self.dataframe = datasets.concatenate_datasets([self.dataframe, new_dataframe]) + + logger.info(f"new dataset len: {len(self.dataframe)}") + + def on_batch_end(self, batch: DataProto) -> None: + """ + Generate data using the provided data generation strategy. + Note: This method is intended to change the dataset after each training batch. + """ + new_data = self.data_generator.generate(self) + self.append_dataframe(new_data) diff --git a/e2c/verl/verl/interactions/__init__.py b/e2c/verl/verl/interactions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b6db0fcef70b051ba5975c4a94d2b68b986e1127 --- /dev/null +++ b/e2c/verl/verl/interactions/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/interactions/base.py b/e2c/verl/verl/interactions/base.py new file mode 100644 index 0000000000000000000000000000000000000000..7c5d200abdc65b009ee8e49a8fb9825642c6b67c --- /dev/null +++ b/e2c/verl/verl/interactions/base.py @@ -0,0 +1,72 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Optional +from uuid import uuid4 + + +class BaseInteraction: + def __init__(self, config: dict[str, Any]): + self.config = config + self.name: str = config.get("name", "interaction_agent") # More general agent default role name + + async def start_interaction(self, instance_id: Optional[str] = None, **kwargs) -> str: + """Create a tool instance. + + Args: + instance_id: The instance id of the tool. + + Returns: + The instance id of the tool. + """ + if instance_id is None: + return str(uuid4()) + else: + return instance_id + + async def generate_response( + self, instance_id: str, messages: list[dict[str, Any]], **kwargs + ) -> tuple[bool, str, float, dict[str, Any]]: # More clear response generation method + """ + Generates a response for the current turn of interaction. + Returns a tuple containing: + - should_terminate_sequence (bool): True if the interaction sequence should end. + - response_content (str): The textual content of the response. + - current_turn_score (float): The score for this specific turn/response. + - additional_data (dict): Any extra information or metadata. + """ + should_terminate_sequence: bool = False # if True, end rollout + response_content: str = "Your current result seems acceptable." + current_turn_score: float = 0.8 + additional_data: dict[str, Any] = {} + return should_terminate_sequence, response_content, current_turn_score, additional_data + + async def calculate_score(self) -> float: # More clear score calculation method + """ + Calculates a score for the interaction, + potentially considering aspects like partial exposure & in-context task switching. + should be invoke at turn-level + """ + # ...implement the logic to calculate turn-level score... + score = 0.0 + return score + + async def finalize_interaction(self) -> None: # More clear interaction end and resource release method + """ + Finalizes the interaction session and releases any associated state or resources. + Simulates: release state + """ + # ...implement the logic to release state... + pass diff --git a/e2c/verl/verl/interactions/collabllm_interation.py b/e2c/verl/verl/interactions/collabllm_interation.py new file mode 100644 index 0000000000000000000000000000000000000000..7acfa1b3bebb6ad6113d04feeac1dbfad403cc7d --- /dev/null +++ b/e2c/verl/verl/interactions/collabllm_interation.py @@ -0,0 +1,374 @@ +# Copyright 2024 CollabLLM Ltd. and/or its affiliates +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import copy +import logging +import os +from typing import Any, Optional +from uuid import uuid4 + +from recipe.collabllm.utils import remove_think_block +from verl.utils.rollout_trace import rollout_trace_op + +from .base import BaseInteraction + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +TERMINATION_SIGNAL = "[[TERMINATE CHAT]]" +USER_PROMPT_TEMPLATE = """You are role-playing as a human USER interacting with an AI collaborator to complete a specific task. Your goal is to generate realistic, natural responses that a user might give in this scenario. + +## Input Information: +You will be provided with: +- Task Description: The type of task you are trying to accomplish. +- Complete Prompt or Reference Goal: This field may include the complete user request/query or a reference answer to user's request. Use this field to understand the user's intent, requirements, or what would count as a satisfactory outcome. +- Chat History: The ongoing conversation between you (as the user) and the AI + +Inputs: +<|The Start of Task Description (Not visible to the AI)|> +{task_desc} +<|The End of Task Description|> + +<|The Start of Complete Prompt or Reference Goal (Not visible to the AI)|> +{single_turn_prompt} +<|The End of Complete Prompt or Reference Goal|> + +<|The Start of Chat History|> +{chat_history} +<|The End of Chat History|> + + +## Guidelines: +- Stay in Character: Role-play as a human USER. You are NOT an AI. Maintain a consistent personality throughout the chat. +- Minimize Effort: IMPORTANT! As a user, avoid being too detailed in your responses. Provide vague or incomplete demands in the early stages of the conversation to minimize your effort. Let the AI ask for clarification rather than providing everything upfront. +- Knowledge Background: Reflect the user's knowledge level in the role-playing. If the user is less knowledgeable about a task, they might not notice incorrect statements. Ask questions that demonstrate your current understanding and areas of confusion. +- Occasionally Make Mistakes: Real-world users might misspell words, provide incorrect dates, give wrong information, or ask unclear questions. Simulate this behavior to reflect natural interactions. +- Mention Personal Preferences: Include preferences or constraints that might influence your requests or responses. For example, "I prefer short answers," "I need this done quickly," or "I like detailed comments in code." +- Goal-Oriented: Keep the chat focused on your intent. Avoid small talk or digressions. Redirect the chat back to the main objective if it starts to stray. + +## Output Format: +You should output a JSON object with three entries: +- "current_answer" (str): Briefly summerize the AI's current solution to the task. +- "thought" (str): Output your thought process as a user deciding what to say next. Consider: +1. Have you obtained a satisfactory solution from the AI? If yes, you can terminate this chat. +2. If not, what specific part of the problem or solution are you struggling with? +3. Has the AI asked you to perform a task or answer a question? If so, how should you approach it? +4. Are you noticing any patterns or potential misunderstandings that need clarification? +5. If you're stuck, how can you phrase your question to get the most helpful response while demonstrating your current understanding? +- "response" (str): Based on your thought process, respond to the AI as the user you are role-playing. Stop immediately when the user's response is completed. + +## Important Notes: +- Respond Based on Previous Messages: Your responses should be based on the context of the current chat history. Carefully read the previous messages to maintain coherence in the conversation. +- Conversation Flow: If "Current Chat History" is empty, start the conversation from scratch with an initial request. Otherwise, continue based on the existing conversation. +- Don't Copy Input Directly: Use the provided information for understanding context only. Avoid copying target queries or any provided information directly in your responses. +- Completion Signal: Use "{termination_signal}" as your response when you believe your goal has been solved or if you determine the AI cannot help further. +- Double check if the JSON object is formatted correctly. Ensure that all fields are present and properly structured. + +Remember to stay in character as a user throughout your response, and follow the instructions and guidelines carefully.""" # noqa + + +class CollabLLMInteraction(BaseInteraction): + """A demo interaction for calculating the reward of CollabLLM. + + - `start_interaction`: start a interaction instance for a trajectory. + - `generate_response`: generate the response of the assistant. + - `calculate_score`: calculate the score of the interaction. + - `finalize_interaction`: finalize the interaction instance. + """ + + def __init__(self, config: dict): + super().__init__(config) + _config = copy.deepcopy(config) + + _config.pop("enable_log", None) + + self.name = _config.pop("name") + self.user_model = _config.pop("user_model") + + self.termination_signal = _config.pop("termination_signal", TERMINATION_SIGNAL) + self.num_retries = _config.pop("num_retries", 3) + + self.user_model_kwargs = _config + + self._instance_dict = {} + + async def start_interaction( + self, instance_id: Optional[str] = None, ground_truth: Optional[str] = None, **kwargs + ) -> str: + if instance_id is None: + instance_id = str(uuid4()) + self._instance_dict[instance_id] = { + "response": "", + "ground_truth": ground_truth, + "reward": 0.0, + } + self.interaction_kwargs = kwargs + assert "single_turn_prompt" in kwargs, "single_turn_prompt is required in interaction_kwargs" + return instance_id + + @rollout_trace_op + async def generate_response( + self, instance_id: str, messages: list[dict[str, Any]], **kwargs + ) -> tuple[bool, str, float, dict]: + assert messages[-1]["role"] in ["system", "assistant"], ( + "Last message input to the user model must be from system or assistant role" + ) + + import litellm + + chat_history = self._parse_messages(messages, strip_sys_prompt=True) + prompt = USER_PROMPT_TEMPLATE.format( + task_desc=self.interaction_kwargs.get("task_desc", "general assistance task"), + single_turn_prompt=self.interaction_kwargs["single_turn_prompt"], + chat_history=chat_history, + termination_signal=self.termination_signal, + ) + response = "" + for i in range(self.num_retries): + try: + full_response = ( + ( + await litellm.acompletion( + model=self.user_model, + messages=[{"role": "user", "content": prompt}], + **self.user_model_kwargs, + ) + ) + .choices[0] + .message.content + ) + except litellm.RateLimitError as e: + logger.warning(f"[CollabLLMInteraction] hit RateLimitError: {e}. Retrying...") + await asyncio.sleep(max(2**i, 60)) + continue + except Exception as e: + logger.exception(f"An unexpected error occurred in CollabLLMAgentLoop: {e}") + continue + + try: + if isinstance(full_response, str): + full_response = extract_json(full_response) + except Exception as e: + logger.warning(f"[CollabLLMInteraction] Error extracting JSON: {e}. Retrying...") + continue + + if isinstance(full_response, dict): + keys = full_response.keys() + if {"current_answer", "thought", "response"}.issubset(keys): + response = full_response.pop("response") + if isinstance(response, str): + break + else: + logger.warning( + f"[CollabLLMInteraction] got an invaild response {response} full_response {full_response}. \ + Retrying..." + ) + continue + else: + logger.warning(f"[CollabLLMInteraction] Keys {keys} do not match expected keys. Retrying...") + continue + + self._instance_dict[instance_id]["response"] = response + logger.debug(f"[CollabLLMInteraction] User: {response}") + should_terminate_sequence = self.termination_signal in response + reward = 0.0 + + return should_terminate_sequence, response, reward, {} + + async def finalize_interaction(self, instance_id: str, **kwargs) -> None: + del self._instance_dict[instance_id] + + def _parse_messages(self, messages, strip_sys_prompt=True): + if messages is None: + return "" + + if strip_sys_prompt: + messages = [msg for msg in messages if msg["role"] != "system"] + + messages = [remove_think_block(msg) for msg in messages] + + chat = "\n".join(f"**{m['role'].capitalize()}**: {m['content']}" for m in messages) + + return chat + + +def extract_json(s): + def convert_value(value): + true_values = {"true": True, "false": False, "null": None} + value_lower = value.lower() + if value_lower in true_values: + return true_values[value_lower] + try: + if "." in value or "e" in value.lower(): + return float(value) + else: + return int(value) + except ValueError: + return value # Return as string if not a number + + def parse_number(s, pos): + start = pos + while pos < len(s) and s[pos] in "-+0123456789.eE": + pos += 1 + num_str = s[start:pos] + try: + if "." in num_str or "e" in num_str.lower(): + return float(num_str), pos + else: + return int(num_str), pos + except ValueError: + logger.error(f"Invalid number at position {start}: {num_str}") + raise + + def skip_whitespace(s, pos): + while pos < len(s) and s[pos] in " \t\n\r": + pos += 1 + return pos + + def parse_string(s, pos): + quote_char = s[pos] + assert quote_char in ('"', "'") + pos += 1 + result = "" + while pos < len(s): + c = s[pos] + if c == "\\": + pos += 1 + if pos >= len(s): + raise ValueError("Invalid escape sequence") + c = s[pos] + escape_sequences = {"n": "\n", "t": "\t", "r": "\r", "\\": "\\", quote_char: quote_char} + result += escape_sequences.get(c, c) + elif c == quote_char: + pos += 1 + # Attempt to convert to a number if possible + converted_value = convert_value(result) + return converted_value, pos + else: + result += c + pos += 1 + raise ValueError("Unterminated string") + + def parse_key(s, pos): + pos = skip_whitespace(s, pos) + if s[pos] in ('"', "'"): + key, pos = parse_string(s, pos) + return key, pos + else: + raise ValueError(f"Expected string for key at position {pos}") + + def parse_object(s, pos): + obj = {} + assert s[pos] == "{" + pos += 1 + pos = skip_whitespace(s, pos) + while pos < len(s) and s[pos] != "}": + pos = skip_whitespace(s, pos) + key, pos = parse_key(s, pos) + pos = skip_whitespace(s, pos) + if pos >= len(s) or s[pos] != ":": + raise ValueError(f'Expected ":" at position {pos}') + pos += 1 + pos = skip_whitespace(s, pos) + value, pos = parse_value(s, pos) + obj[key] = value + pos = skip_whitespace(s, pos) + if pos < len(s) and s[pos] == ",": + pos += 1 + pos = skip_whitespace(s, pos) + elif pos < len(s) and s[pos] == "}": + break + elif pos < len(s) and s[pos] != "}": + raise ValueError(f'Expected "," or "}}" at position {pos}') + if pos >= len(s) or s[pos] != "}": + raise ValueError(f'Expected "}}" at position {pos}') + pos += 1 + return obj, pos + + def parse_array(s, pos): + lst = [] + assert s[pos] == "[" + pos += 1 + pos = skip_whitespace(s, pos) + while pos < len(s) and s[pos] != "]": + value, pos = parse_value(s, pos) + lst.append(value) + pos = skip_whitespace(s, pos) + if pos < len(s) and s[pos] == ",": + pos += 1 + pos = skip_whitespace(s, pos) + elif pos < len(s) and s[pos] == "]": + break + elif pos < len(s) and s[pos] != "]": + raise ValueError(f'Expected "," or "]" at position {pos}') + if pos >= len(s) or s[pos] != "]": + raise ValueError(f'Expected "]" at position {pos}') + pos += 1 + return lst, pos + + def parse_triple_quoted_string(s, pos): + if s[pos : pos + 3] == "'''": + quote_str = "'''" + elif s[pos : pos + 3] == '"""': + quote_str = '"""' + else: + raise ValueError(f"Expected triple quotes at position {pos}") + pos += 3 + result = "" + while pos < len(s): + if s[pos : pos + 3] == quote_str: + pos += 3 + # Attempt to convert to a number if possible + converted_value = convert_value(result) + return converted_value, pos + else: + result += s[pos] + pos += 1 + raise ValueError("Unterminated triple-quoted string") + + def parse_value(s, pos): + pos = skip_whitespace(s, pos) + if pos >= len(s): + raise ValueError("Unexpected end of input") + if s[pos] == "{": + return parse_object(s, pos) + elif s[pos] == "[": + return parse_array(s, pos) + elif s[pos : pos + 3] in ("'''", '"""'): + return parse_triple_quoted_string(s, pos) + elif s[pos] in ('"', "'"): + return parse_string(s, pos) + elif s[pos : pos + 4].lower() == "true": + return True, pos + 4 + elif s[pos : pos + 5].lower() == "false": + return False, pos + 5 + elif s[pos : pos + 4].lower() == "null": + return None, pos + 4 + elif s[pos] in "-+0123456789.": + return parse_number(s, pos) + else: + raise ValueError(f"Unexpected character at position {pos}: {s[pos]}") + + json_start = s.index("{") + json_end = s.rfind("}") + s = s[json_start : json_end + 1] + + s = s.strip() + result, pos = parse_value(s, 0) + pos = skip_whitespace(s, pos) + if pos != len(s): + raise ValueError(f"Unexpected content at position {pos}") + return result diff --git a/e2c/verl/verl/interactions/gsm8k_interaction.py b/e2c/verl/verl/interactions/gsm8k_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..67898ad577a0e277bd92df4956c50be3c7004ae8 --- /dev/null +++ b/e2c/verl/verl/interactions/gsm8k_interaction.py @@ -0,0 +1,87 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from typing import Any, Optional +from uuid import uuid4 + +from verl.utils.reward_score import gsm8k + +from .base import BaseInteraction + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class Gsm8kInteraction(BaseInteraction): + """A demo interaction for calculating the reward of gsm8k. + + - `start_interaction`: start a interaction instance for a trajectory. + - `generate_response`: generate the response of the assistant. + - `calculate_score`: calculate the score of the interaction. + - `finalize_interaction`: finalize the interaction instance. + """ + + def __init__(self, config: dict): + super().__init__(config) + self._instance_dict = {} + + async def start_interaction( + self, instance_id: Optional[str] = None, ground_truth: Optional[str] = None, **kwargs + ) -> str: + if instance_id is None: + instance_id = str(uuid4()) + self._instance_dict[instance_id] = { + "response": "", + "ground_truth": ground_truth, + "reward": 0.0, + } + return instance_id + + async def generate_response( + self, instance_id: str, messages: list[dict[str, Any]], **kwargs + ) -> tuple[bool, str, float, dict]: + content = "" + for i in range(len(messages) - 1, -1, -1): + item = messages[i] + if item.get("role") == "assistant": + content = item.get("content") + break + + self._instance_dict[instance_id]["response"] = content + + reward = await self.calculate_score(instance_id) + if reward == 1.0: + response = "Your response is correct!" + should_terminate_sequence = True + else: + response = "Your response is incorrect! You need to reflect on your answer and try again." + should_terminate_sequence = False + + return should_terminate_sequence, response, reward, {} + + async def calculate_score(self, instance_id: str, **kwargs) -> float: + return gsm8k.compute_score( + self._instance_dict[instance_id]["response"], + self._instance_dict[instance_id]["ground_truth"], + method="strict", + format_score=0.0, + score=1.0, + ) + + async def finalize_interaction(self, instance_id: str, **kwargs) -> None: + del self._instance_dict[instance_id] diff --git a/e2c/verl/verl/interactions/utils/__init__.py b/e2c/verl/verl/interactions/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c4b932b1ae7eeeb4c53c98c684cf0ba9b670a86b --- /dev/null +++ b/e2c/verl/verl/interactions/utils/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/interactions/utils/interaction_registry.py b/e2c/verl/verl/interactions/utils/interaction_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..df747af11d0e119360acb0f9ff6c9ba49926e0a3 --- /dev/null +++ b/e2c/verl/verl/interactions/utils/interaction_registry.py @@ -0,0 +1,85 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import logging +import os +import sys + +from omegaconf import OmegaConf + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +def get_interaction_class(cls_name): + """Dynamically import and return the interaction class.""" + module_name, class_name = cls_name.rsplit(".", 1) + if module_name not in sys.modules: + spec = importlib.util.find_spec(module_name) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + else: + module = sys.modules[module_name] + + interaction_cls = getattr(module, class_name) + return interaction_cls + + +def initialize_interactions_from_config(interaction_config_file): + """Initialize interactions from configuration file. + + Args: + interaction_config_file: Path to the interaction configuration file. + + Returns: + dict: A dictionary mapping interaction names to BaseInteraction instances. + """ + interaction_config = OmegaConf.load(interaction_config_file) + interaction_map = {} + + for interaction_item in interaction_config.interaction: + cls_name = interaction_item.class_name + interaction_cls = get_interaction_class(cls_name) + + # Extract config and name + config = OmegaConf.to_container(interaction_item.config, resolve=True) + + # Get the interaction name - either from config or derive from class name + name = interaction_item.get("name", None) + if name is None: + # If no name is specified, use the class name as default + class_simple_name = cls_name.split(".")[-1] + # Remove "Interaction" suffix if present, otherwise use full class name + if class_simple_name.endswith("Interaction"): + name = class_simple_name[:-11].lower() # Remove "Interaction" (11 chars) + else: + name = class_simple_name.lower() + + # Check for duplicate names + if name in interaction_map: + raise ValueError(f"Duplicate interaction name '{name}' found. Each interaction must have a unique name.") + + # Inject the name into the config + config["name"] = name + + # Create the interaction instance + interaction = interaction_cls(config=config) + interaction_map[name] = interaction + + logger.info(f"Initialized interaction '{name}' with class '{cls_name}'") + + return interaction_map diff --git a/e2c/verl/verl/interactions/weather_interaction.py b/e2c/verl/verl/interactions/weather_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..9e4022652e7b024699baf57c03fce56c63ee21c8 --- /dev/null +++ b/e2c/verl/verl/interactions/weather_interaction.py @@ -0,0 +1,79 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from typing import Any, Optional +from uuid import uuid4 + +from .base import BaseInteraction + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class WeatherInteraction(BaseInteraction): + """A demo interaction for handling weather-related queries. + + - `start_interaction`: start a interaction instance for a trajectory. + - `generate_response`: generate the response of the assistant. + - `calculate_score`: calculate the score of the interaction. + - `finalize_interaction`: finalize the interaction instance. + """ + + def __init__(self, config: dict): + super().__init__(config) + self._instance_dict = {} + + async def start_interaction( + self, instance_id: Optional[str] = None, ground_truth: Optional[str] = None, **kwargs + ) -> str: + if instance_id is None: + instance_id = str(uuid4()) + self._instance_dict[instance_id] = { + "response": "", + "ground_truth": ground_truth, + "reward": 0.0, + } + return instance_id + + async def generate_response( + self, instance_id: str, messages: list[dict[str, Any]], **kwargs + ) -> tuple[bool, str, float, dict]: + content = "no tool call" + for i in range(len(messages) - 1, -1, -1): + item = messages[i] + if item.get("role") == "tool": + content = item.get("content") + break + self._instance_dict[instance_id]["response"] = content + + reward = await self.calculate_score(instance_id) + if reward == 1.0: + response = "Thank you for your weather query!" + should_terminate_sequence = True + else: + response = "Please use the weather tool to get the weather information." + should_terminate_sequence = True + return should_terminate_sequence, response, reward, {} + + async def calculate_score(self, instance_id: str, **kwargs) -> float: + # For weather interaction, we can implement a more complex scoring logic + # For now, we'll just return a default score of 1.0 + if self._instance_dict[instance_id]["response"] == "no tool call": + return 0.0 + return 1.0 + + async def finalize_interaction(self, instance_id: str, **kwargs) -> None: + del self._instance_dict[instance_id] diff --git a/e2c/verl/verl/model_merger/__init__.py b/e2c/verl/verl/model_merger/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/model_merger/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/model_merger/__main__.py b/e2c/verl/verl/model_merger/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..f3ab5b9c29b5d5114fc918042ea496848078d38a --- /dev/null +++ b/e2c/verl/verl/model_merger/__main__.py @@ -0,0 +1,73 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module is used to merge huggingface model and test verl checkpoints from FSDP and Megatron backends. + +To merge FSDP checkpoints: +```sh +python -m verl.model_merger merge \ + --backend fsdp \ + --local_dir checkpoints/verl_fsdp_gsm8k_examples/qwen2_5_0b5_fsdp_saveload/global_step_1/actor \ + --target_dir /path/to/merged_hf_model +``` + +To merge Megatron checkpoints: +```sh +python -m verl.model_merger merge \ + --backend megatron \ + --tie-word-embedding \ + --local_dir checkpoints/verl_megatron_gsm8k_examples/qwen2_5_0b5_megatron_saveload/global_step_1/actor \ + --target_dir /path/to/merged_hf_model +``` + +or use distribtued merge for large models like dpskv3 671B + +```sh +torchrun --nproc_per_node 1 --nnodes 8 --node_rank ${RANK} -m verl.model_merger merge\ + --backend megatron \ + --local_dir ./checkpoints/global_step_1/actor \ + --target_dir /path/to/merged_hf_model +``` + + +For more details, please refer to documentation: +https://verl.readthedocs.io/en/latest/advance/checkpoint.html#convert-fsdp-and-megatron-checkpoints-to-huggingface-format-model +""" + +from .base_model_merger import generate_config_from_args, parse_args + + +def main(): + args = parse_args() + config = generate_config_from_args(args) + print(f"config: {config}") + + if config.backend == "fsdp": + from .fsdp_model_merger import FSDPModelMerger + + merger = FSDPModelMerger(config) + elif config.backend == "megatron": + from .megatron_model_merger import MegatronModelMerger + + merger = MegatronModelMerger(config) + else: + raise NotImplementedError(f"Unknown backend: {config.backend}") + + merger.merge_and_save() + merger.cleanup() + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/verl/model_merger/base_model_merger.py b/e2c/verl/verl/model_merger/base_model_merger.py new file mode 100644 index 0000000000000000000000000000000000000000..b46f40f879b6648d3ac2d61ec6e3ba382088eba7 --- /dev/null +++ b/e2c/verl/verl/model_merger/base_model_merger.py @@ -0,0 +1,362 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Optional + +import torch +from accelerate import init_empty_weights +from transformers import ( + AutoConfig, + AutoModelForCausalLM, + AutoModelForTokenClassification, + AutoModelForVision2Seq, + GenerationConfig, +) + +from verl.utils import hf_processor, hf_tokenizer + + +def parse_args(): + parser = argparse.ArgumentParser(description="verl model merger") + subparsers = parser.add_subparsers(dest="operation", required=True, help="Specify 'merge' or 'test' operation.") + + base_op_parser = argparse.ArgumentParser(add_help=False) + base_op_parser.add_argument( + "--backend", type=str, required=True, choices=["fsdp", "megatron"], help="The backend of the model" + ) + base_op_parser.add_argument("--local_dir", type=str, default=None, help="Path to the saved model checkpoints.") + base_op_parser.add_argument( + "--tie-word-embedding", + action="store_true", + help="Whether to tie word embedding weights (currently only Megatron supported)", + ) + base_op_parser.add_argument("--trust-remote-code", action="store_true", help="Whether to trust remote code") + base_op_parser.add_argument( + "--is-value-model", + action="store_true", + help="Whether the model is a value model (currently only Megatron supported)", + ) + base_op_parser.add_argument( + "--use_cpu_initialization", + action="store_true", + help="Whether to use CPU initialization for the model. This is useful for large models that cannot " + "fit into GPU memory during initialization.", + ) + + merge_parser = subparsers.add_parser("merge", parents=[base_op_parser], help="Merge model checkpoints and save.") + merge_parser.add_argument( + "--target_dir", default="tmp", type=str, help="Directory to save the merged huggingface model" + ) + merge_parser.add_argument( + "--hf_upload_path", default=None, type=str, help="Hugging Face repository ID to upload the model" + ) + merge_parser.add_argument( + "--private", action="store_true", help="Whether to upload the model to a private Hugging Face repository" + ) + + test_parser = subparsers.add_parser( + "test", parents=[base_op_parser], help="Test merged model against a reference Hugging Face model" + ) + test_parser.add_argument( + "--test_hf_dir", type=str, required=True, help="Path to the reference Hugging Face model directory for testing" + ) + + args = parser.parse_args() + return args + + +@dataclass +class ModelMergerConfig: + """Configuration for model merger operations. + + Args: + operation (str): Operation type - 'merge' or 'test'. + backend (str): Backend type for the model ('fsdp' or 'megatron'). + target_dir (Optional[str]): Directory to save the merged huggingface model. Defaults to "tmp". + hf_upload_path (Optional[str]): Hugging Face repository ID to upload the model. Defaults to None. + private (bool): Whether to upload the model to a private Hugging Face repository. Defaults to False. + test_hf_dir (Optional[str]): Path to the reference Hugging Face model directory for testing. Defaults to None. + tie_word_embedding (bool): Whether to tie word embedding weights (currently only Megatron + supported). Defaults to False. + trust_remote_code (bool): Whether to trust remote code. Defaults to False. + is_value_model (bool): Whether the model is a value model (currently only Megatron + supported). Defaults to False. + local_dir (Optional[str]): Path to the saved model checkpoints. Defaults to None. + hf_model_config_path (Optional[str]): Path to HuggingFace model configuration files. Defaults to None. + hf_upload (bool): Whether to upload to HuggingFace (computed automatically). Not for initialization. + use_cpu_initialization (bool): Whether to use CPU initialization for large models. Defaults to False. + """ + + operation: str # 'merge' or 'test' + backend: str + target_dir: Optional[str] = "tmp" + hf_upload_path: Optional[str] = None + private: bool = False + test_hf_dir: Optional[str] = None + tie_word_embedding: bool = False + trust_remote_code: bool = False + is_value_model: bool = False + local_dir: Optional[str] = None + hf_model_config_path: Optional[str] = None + hf_upload: bool = field(init=False) + use_cpu_initialization: bool = False + + def __post_init__(self): + self.hf_upload = self.operation == "merge" and bool(self.hf_upload_path) + if self.operation == "test": + self.target_dir = None + self.hf_upload_path = None + self.private = False + + +def generate_config_from_args(args: argparse.Namespace) -> ModelMergerConfig: + common_config_args = { + "operation": args.operation, + "backend": args.backend, + "tie_word_embedding": args.tie_word_embedding, + "trust_remote_code": args.trust_remote_code, + "is_value_model": args.is_value_model, + "local_dir": args.local_dir, + "hf_model_config_path": os.path.join(args.local_dir, "huggingface"), + "use_cpu_initialization": args.use_cpu_initialization, + } + + if args.operation == "merge": + config = ModelMergerConfig( + **common_config_args, + target_dir=args.target_dir, + hf_upload_path=args.hf_upload_path, + private=args.private, + test_hf_dir=None, + ) + os.makedirs(config.target_dir, exist_ok=True) + elif args.operation == "test": + config = ModelMergerConfig( + **common_config_args, + test_hf_dir=args.test_hf_dir, + # the following args are not used by test operation + target_dir=None, + hf_upload_path=None, + private=False, + ) + else: + raise NotImplementedError(f"Unknown operation: {args.operation}") + return config + + +class BaseModelMerger(ABC): + """ + Abstract base class for merging distributed model checkpoints into HuggingFace format. + + This class provides common functionality for converting model checkpoints from different + distributed training backends (FSDP, Megatron) into standard HuggingFace format that + can be easily loaded and used for inference or further training. + + The merger supports two main operations: + - merge: Convert and save checkpoints to HuggingFace format + - test: Validate merged checkpoints against a reference model + + Args: + config (ModelMergerConfig): Configuration object containing paths, backend type, + and operation parameters. + + Attributes: + config (ModelMergerConfig): The configuration object passed during initialization. + hf_model_config_path (str): Path to the HuggingFace model configuration files. + model_config (PretrainedConfig): Loaded HuggingFace model configuration. + """ + + def __init__(self, config: ModelMergerConfig): + self.config = config + self.hf_model_config_path = config.hf_model_config_path + self.model_config = AutoConfig.from_pretrained( + self.hf_model_config_path, trust_remote_code=self.config.trust_remote_code + ) + + def get_transformers_auto_model_class(self): + has_remote_code = hasattr(self.model_config, "auto_map") and any( + self.model_config.architectures[0] in val for val in self.model_config.auto_map.values() + ) + if has_remote_code: + auto_class = next( + k for k, v in self.model_config.auto_map.items() if self.model_config.architectures[0] in v + ) + match auto_class: + case "AutoModelForCausalLM": + return AutoModelForCausalLM + case "AutoModelForTokenClassification": + return AutoModelForTokenClassification + case "AutoModelForVision2Seq": + return AutoModelForVision2Seq + case _: + raise NotImplementedError(f"Unknown auto class {auto_class}") + else: + if "ForTokenClassification" in self.model_config.architectures[0]: + return AutoModelForTokenClassification + elif "ForCausalLM" in self.model_config.architectures[0]: + return AutoModelForCausalLM + elif "ForConditionalGeneration" in self.model_config.architectures[0]: + return AutoModelForVision2Seq + + raise NotImplementedError(f"Unknown architecture {self.model_config.architectures}") + + def patch_model_generation_config(self, model): + """ + The generation_config created from model config may be different to the pretrained model, + this may lead to error when generating: https://github.com/volcengine/verl/issues/1246 + + This function patch the generation_config created from model config to the pretrained model. + """ + if model.can_generate(): + try: + model.generation_config = GenerationConfig.from_pretrained(self.hf_model_config_path) + except OSError: + print( + f"Warning: Generation config file not found in {self.hf_model_config_path}, using a " + f"generation config created from the model config." + ) + return model + + def save_lora_adapter(self, state_dict: dict[str, torch.Tensor]): + """ + Save lora adapter to safetensors. + + Returns: + lora_path: str, the path to the lora adapter. None if no lora adapter found. + + Note: + This function change the 'state_dict' in place. + """ + lora_params_names = [name for name in state_dict.keys() if "lora_" in name] + + if len(lora_params_names) == 0: + return None + + import json + from typing import OrderedDict + + import peft + from safetensors.torch import save_file + + lora_params = OrderedDict() + target_modules = set() + lora_key = None + + for name in lora_params_names: + lora_key = name.replace(".default.weight", ".weight") + target_modules.add(lora_key.split(".")[-3]) + lora_params[lora_key] = state_dict.pop(name) + + lora_rank = min(lora_params[lora_key].shape[0], lora_params[lora_key].shape[1]) + peft_dict = { + "r": lora_rank, + "lora_alpha": 0, # lora_alpha is not set. An error should be raised to inform the user to set it manually. + "target_modules": list(target_modules), + } + peft_config = peft.LoraConfig(**peft_dict).to_dict() + peft_config["task_type"] = peft_config["task_type"].value if peft_config["task_type"] else None + peft_config["peft_type"] = peft_config["peft_type"].value if peft_config["peft_type"] else None + peft_config["target_modules"] = list(peft_config["target_modules"]) + + lora_path = os.path.join(self.config.target_dir, "lora_adapter") + os.makedirs(lora_path, exist_ok=True) + with open(os.path.join(lora_path, "adapter_config.json"), "w", encoding="utf-8") as f: + json.dump(peft_config, f, ensure_ascii=False, indent=4) + save_file(lora_params, os.path.join(lora_path, "adapter_model.safetensors")) + + for name in list(state_dict.keys()): + key = ( + name.replace("base_model.model.", "") + .replace(".base_layer.weight", ".weight") + .replace(".base_layer.bias", ".bias") + ) + state_dict[key] = state_dict.pop(name) + + return lora_path + + def save_hf_model_and_tokenizer(self, state_dict: dict[str, torch.Tensor]): + auto_model_class = self.get_transformers_auto_model_class() + with init_empty_weights(): + model = auto_model_class.from_config( + self.model_config, torch_dtype=torch.bfloat16, trust_remote_code=self.config.trust_remote_code + ) + model.to_empty(device="cpu") + model = self.patch_model_generation_config(model) + + lora_path = self.save_lora_adapter(state_dict) + if lora_path: + print(f"Saving lora adapter to {lora_path}") + + print(f"Saving model to {self.config.target_dir}") + model.save_pretrained(self.config.target_dir, state_dict=state_dict) + del state_dict + del model + + processor = hf_processor(self.hf_model_config_path, trust_remote_code=self.config.trust_remote_code) + tokenizer = hf_tokenizer(self.hf_model_config_path, trust_remote_code=self.config.trust_remote_code) + if processor is not None: + print(f"Saving processor to {self.config.target_dir}") + processor.save_pretrained(self.config.target_dir) + if tokenizer is not None: + print(f"Saving tokenizer to {self.config.target_dir}") + tokenizer.save_pretrained(self.config.target_dir) + + def upload_to_huggingface(self): + import requests + from huggingface_hub import HfApi + from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError + + api = HfApi() + try: + # Attempt to create repository + api.create_repo(repo_id=self.config.hf_upload_path, private=self.config.private, exist_ok=True) + except HfHubHTTPError as e: + # Handle authentication/API errors + if e.response.status_code == 401: + raise PermissionError( + "Hugging Face authentication failed. Verify your token is valid and has write permissions." + ) from e + elif e.response.status_code == 404: + raise RepositoryNotFoundError(f"Repository path not found: {self.config.hf_upload_path}") from e + else: + raise ConnectionError(f"Failed to create repository ({e.response.status_code}): {e}") from e + except requests.exceptions.ConnectionError as e: + raise ConnectionError("Network connection failed. Check your internet connection.") from e + + try: + # Attempt folder upload + api.upload_folder(folder_path=self.config.target_dir, repo_id=self.config.hf_upload_path, repo_type="model") + except HfHubHTTPError as e: + if e.response.status_code == 401: + raise PermissionError("Authentication failed during upload. Token may have expired.") from e + else: + raise RuntimeError(f"Upload failed ({e.response.status_code}): {e}") from e + except requests.exceptions.ConnectionError as e: + raise ConnectionError("Network interruption during upload. Try again with stable connection.") from e + except OSError as e: + raise FileNotFoundError(f"Local folder error: {self.config.target_dir} - {str(e)}") from e + except Exception as e: + raise RuntimeError(f"Unexpected error during upload: {str(e)}") from e + + @abstractmethod + def merge_and_save(self): + raise NotImplementedError("Subclasses should implement this method") + + @abstractmethod + def cleanup(self): + raise NotImplementedError("Subclasses should implement this method to clean up resources if needed") diff --git a/e2c/verl/verl/model_merger/fsdp_model_merger.py b/e2c/verl/verl/model_merger/fsdp_model_merger.py new file mode 100644 index 0000000000000000000000000000000000000000..7853b2b79878a8142153cbc647eafc665ab718f4 --- /dev/null +++ b/e2c/verl/verl/model_merger/fsdp_model_merger.py @@ -0,0 +1,265 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import numpy as np +import torch +from torch.distributed._tensor import Placement, Shard + +try: + # for torch 2.5+ + from torch.distributed.tensor import DTensor +except ImportError: + from torch.distributed._tensor import DTensor + +from tqdm import tqdm + +from .base_model_merger import BaseModelMerger + + +class FSDPModelMerger(BaseModelMerger): + """ + Model merger for FSDP (Fully Sharded Data Parallel) checkpoints. + + This class handles the conversion of FSDP distributed checkpoints into HuggingFace format. + FSDP shards model parameters across multiple processes, and this merger reconstructs + the full model by loading and concatenating the sharded parameters from all ranks. + + The merger supports various FSDP configurations including: + - Pure FSDP (single dimension sharding) + - FSDP + DDP (data parallel + fully sharded data parallel) + - DTensor-based sharding with custom device meshes + + Key features: + - Automatic detection of world size from checkpoint filenames + - Support for DTensor and non-DTensor checkpoints + - Parallel loading of checkpoint shards for efficiency + - Validation against reference HuggingFace models + + Example: + To merge FSDP checkpoints: + ```python + config = ModelMergerConfig( + operation="merge", + backend="fsdp", + local_dir="path/to/fsdp/checkpoints", + target_dir="path/to/output" + ) + merger = FSDPModelMerger(config) + merger.merge_and_save() + ``` + """ + + def _get_world_size(self) -> int: + """_summary_ + From FSDP json config file, extract the world size. + + Returns: + int: world size + """ + config_path = Path(self.config.local_dir) / "fsdp_config.json" + if not config_path.exists(): + raise FileNotFoundError(f"Config file {config_path} does not exist.") + + with open(config_path) as f: + config = json.load(f) + + # Extract world size from the config + world_size = config.get("world_size", None) + if world_size is None: + raise ValueError("World size not found in the config file.") + + return world_size + + def _load_rank_zero_state_dict(self, world_size: int) -> dict: + return torch.load( + Path(self.config.local_dir) / f"model_world_size_{world_size}_rank_0.pt", + map_location="cpu", + weights_only=False, + ) + + def _extract_device_mesh_info(self, state_dict: dict, world_size: int) -> tuple[np.ndarray, tuple[str, ...]]: + """ + Retrieves sharding information (device_mesh, mesh_dim_names) from a DTensor in the state_dict. + If no DTensor is found, infers a simple FSDP mesh based on world_size. + """ + pivot_key = sorted(list(state_dict.keys()))[0] + weight = state_dict[pivot_key] + + if isinstance(weight, DTensor): + # get sharding info + device_mesh = weight.device_mesh + mesh = device_mesh.mesh + mesh_dim_names = device_mesh.mesh_dim_names + else: + # for non-DTensor + mesh = np.array([world_size], dtype=np.int64) + mesh_dim_names = ("fsdp",) + + return mesh, mesh_dim_names + + def _calculate_shard_configuration( + self, mesh: np.ndarray, mesh_dim_names: tuple[str, ...] + ) -> tuple[int, tuple[int, ...]]: + """Calculates the total number of shards and the shape of the device mesh.""" + assert mesh_dim_names in (("fsdp",), ("ddp", "fsdp")), f"Unsupported mesh_dim_names {mesh_dim_names}" + + if "tp" in mesh_dim_names: + # TODO: "tp" is not supported yet due to the above assert + total_shards = mesh.shape[-1] * mesh.shape[-2] + mesh_shape = (mesh.shape[-2], mesh.shape[-1]) + else: + total_shards = mesh.shape[-1] + mesh_shape = (mesh.shape[-1],) + + return total_shards, mesh_shape + + def _merge_by_placement(self, tensors: list[torch.Tensor], placement: Placement) -> torch.Tensor: + """Merges a list of tensors based on their DTensor placement""" + if placement.is_replicate(): + return tensors[0] + elif placement.is_partial(): + raise NotImplementedError("Partial placement is not supported yet") + elif placement.is_shard(): + return torch.cat(tensors, dim=placement.dim).contiguous() + + raise NotImplementedError(f"Unsupported placement: {placement}") + + def _load_and_merge_state_dicts( + self, world_size: int, total_shards: int, mesh_shape: tuple[int, ...], mesh_dim_names: tuple[str, ...] + ) -> dict[str, torch.Tensor]: + model_state_dict_lst = [None] * total_shards + + def process_one_shard(rank: int, model_state_dict_lst: list): + model_path = Path(self.config.local_dir) / f"model_world_size_{world_size}_rank_{rank}.pt" + state_dict = torch.load(model_path, map_location="cpu", weights_only=False) + model_state_dict_lst[rank] = state_dict + return state_dict + + with ThreadPoolExecutor(max_workers=min(32, os.cpu_count())) as executor: + futures = [executor.submit(process_one_shard, rank, model_state_dict_lst) for rank in range(total_shards)] + for future in tqdm(futures, desc=f"Loading {total_shards} FSDP shards", total=total_shards): + future.result() + + # Merge state dicts from all shards + state_dict = {} + param_placements: dict[str, list] = {} + + for key in set(model_state_dict_lst[0].keys()): + state_dict[key] = [] + for model_state_shard in model_state_dict_lst: + # add tensor shard in order of rank to state_dict[key] + tensor = model_state_shard.pop(key) + if isinstance(tensor, DTensor): + state_dict[key].append(tensor._local_tensor.bfloat16()) + + placements = tuple(tensor.placements) + # replicated placement at dp dimension can be discarded + if mesh_dim_names[0] in ("dp", "ddp"): + placements = placements[1:] + + if key not in param_placements: + param_placements[key] = placements + else: + assert param_placements[key] == placements + else: + state_dict[key].append(tensor.bfloat16()) + + del model_state_dict_lst + + # Merge tensors + for key in sorted(state_dict): + if not isinstance(state_dict[key], list): + print(f"No need to merge key {key}") + continue + if key in param_placements: + # merge shards + placements: tuple[Shard] = param_placements[key] + if len(mesh_shape) == 1: + # 1-D list, FSDP without TP + assert len(placements) == 1 + shards = state_dict[key] + state_dict[key] = self._merge_by_placement(shards, placements[0]) + else: + # 2-D list, FSDP + TP + raise NotImplementedError("FSDP + TP is not supported yet") + else: + state_dict[key] = torch.cat(state_dict[key], dim=0) + + return state_dict + + def merge_and_save(self): + world_size = self._get_world_size() + rank_zero_state_dict = self._load_rank_zero_state_dict(world_size) + + mesh, mesh_dim_names = self._extract_device_mesh_info(rank_zero_state_dict, world_size) + print(f"Got device mesh {mesh}, mesh_dim_names {mesh_dim_names}") + + total_shards, mesh_shape = self._calculate_shard_configuration(mesh, mesh_dim_names) + print(f"Processing model shards with {total_shards} {mesh_shape} in total") + + merged_state_dict = self._load_and_merge_state_dicts(world_size, total_shards, mesh_shape, mesh_dim_names) + + if self.config.operation == "test": + if not self.config.test_hf_dir: + raise ValueError("test_hf_dir must be provided for test operation") + self._validate_state_dict(merged_state_dict) + elif self.config.operation == "merge": + self.save_hf_model_and_tokenizer(merged_state_dict) + if self.config.hf_upload: + self.upload_to_huggingface() + else: + raise ValueError(f"Unknown operation: {self.config.operation}") + + def _validate_state_dict(self, state_dict: dict[str, torch.Tensor]): + auto_model_class = self.get_transformers_auto_model_class() + + hf_model = auto_model_class.from_pretrained(self.config.test_hf_dir, torch_dtype=torch.bfloat16) + hf_state_dict = hf_model.state_dict() + del hf_model + + hf_model_keys = set(hf_state_dict.keys()) + collected_keys = set(state_dict.keys()) + + missing_keys = hf_model_keys - collected_keys + assert len(missing_keys) == 0, f"Missing keys in collected state dict: {list(sorted(missing_keys))}" + + extra_keys = collected_keys - hf_model_keys + assert len(extra_keys) == 0, f"Extra keys in collected state dict: {list(sorted(extra_keys))}" + + for key in hf_model_keys: + hf_shape = hf_state_dict[key].shape + collected_shape = state_dict[key].shape + assert hf_shape == collected_shape, ( + f"Shape mismatch for key '{key}': original {hf_shape} vs collected {collected_shape}" + ) + + hf_dtype = hf_state_dict[key].dtype + collected_dtype = state_dict[key].dtype + assert hf_dtype == collected_dtype, ( + f"Dtype mismatch for key '{key}': original {hf_dtype} vs collected {collected_dtype}" + ) + + torch.testing.assert_close(hf_state_dict[key], state_dict[key], atol=1e-6, rtol=1e-6) + + print("FSDP checks passed: The merged state_dict matches the hf model saved by FSDPCheckpointManager.") + + def cleanup(self): + """Cleanup temporary files if needed.""" + # FSDP merger does not create temporary files, so no cleanup is needed. + pass diff --git a/e2c/verl/verl/model_merger/megatron_model_merger.py b/e2c/verl/verl/model_merger/megatron_model_merger.py new file mode 100644 index 0000000000000000000000000000000000000000..522ca93d4fc17d26bf6bb3f54648055c8758bcd5 --- /dev/null +++ b/e2c/verl/verl/model_merger/megatron_model_merger.py @@ -0,0 +1,539 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import warnings +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Callable, ContextManager + +import numpy as np +import torch +import torch.distributed as dist +from accelerate import init_empty_weights +from megatron.core import mpu +from megatron.core.models.gpt.gpt_model import ModelType +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from safetensors.torch import load_file +from transformers import ( + AutoConfig, + PretrainedConfig, +) + +from verl.models.mcore import hf_to_mcore_config +from verl.utils.device import get_device_name, get_nccl_backend, get_torch_device +from verl.utils.distributed import set_numa_affinity +from verl.utils.megatron.dist_checkpointing import load_dist_checkpointing +from verl.utils.megatron_utils import get_model +from verl.utils.tokenizer import hf_processor, hf_tokenizer + +from .base_model_merger import BaseModelMerger, ModelMergerConfig + + +@contextmanager +def noop_context() -> Any: + yield + + +def get_dynamic_pipeline_shards(layer_num: int, pp_size: int) -> list[int]: + """Calculate the pipeline sharding configuration for Megatron-LM. + + Args: + layer_num: Total number of layers in the model. + pp_size: Number of pipeline parallel ranks. + + Returns: + layer number of each pp rank. Make the sharding of the pipeline as uniform as possible. + """ + if layer_num < pp_size: + raise ValueError(f"layer_num {layer_num} must be greater than pp_size {pp_size}.") + + if pp_size < 1: + raise ValueError(f"pp_size must be at least 1, got {pp_size}.") + if pp_size == 1: + return [layer_num] + + if pp_size == 2: + return [ + layer_num // 2, + layer_num - layer_num // 2, + ] + + middle_size = pp_size - 2 + shards_strategy = [] + for middle_layer_num in range(layer_num): + first_last_layer_num = layer_num - middle_layer_num * middle_size + first_layer_num = first_last_layer_num // 2 + last_layer_num = first_last_layer_num - first_last_layer_num // 2 + if 0 < first_layer_num <= middle_layer_num and 0 < last_layer_num <= middle_layer_num: + shards_strategy.append( + ( + [first_layer_num] + [middle_layer_num] * middle_size + [last_layer_num], + abs(first_layer_num - middle_layer_num), + ) + ) + + # sort by diff of layer_num, to make it as uniform as possible + res = sorted(shards_strategy, key=lambda x: x[1])[0][0] + assert sum(res) == layer_num, f"sum(res)={sum(res)} != layer_num={layer_num}, pp_size={pp_size}" + return res + + +class MegatronModelMerger(BaseModelMerger): + """ + Model merger for Megatron-LM distributed checkpoints. + + This class handles the conversion of Megatron-LM distributed checkpoints into HuggingFace format. + Megatron-LM uses tensor parallelism, pipeline parallelism, and data parallelism to distribute + large language models across multiple GPUs. This merger reconstructs the full model by + loading distributed checkpoints and applying the necessary transformations. + + Key features: + - Support for tensor parallel, pipeline parallel, and data parallel configurations + - Automatic parameter name mapping from Megatron to HuggingFace conventions + - Handling of QKV and gate-up tensor splitting/merging + - Support for tied word embeddings and value models + - Integration with Megatron's distributed checkpointing system + + The merger handles various model architectures and configurations: + - Standard transformer models (GPT-style) + - Models with tied word embeddings + - Value models for reinforcement learning + - Multi-layer attention (MLA) architectures + - Mixture of Experts (MoE) models + + Args: + config (ModelMergerConfig): Configuration object with Megatron-specific settings + including tie_word_embedding and is_value_model flags. + + Example: + To merge Megatron checkpoints: + ```python + config = ModelMergerConfig( + operation="merge", + backend="megatron", + local_dir="path/to/megatron/checkpoints", + target_dir="path/to/output", + tie_word_embedding=True + ) + merger = MegatronModelMerger(config) + merger.merge_and_save() + ``` + """ + + def __init__(self, config: ModelMergerConfig): + super().__init__(config) + # Currently we use only 1 rank to merge the dist_ckpt, we will move to multi-process save shortly afterwards + if "WORLD_SIZE" not in os.environ: + os.environ["RANK"] = "0" + os.environ["LOCAL_RANK"] = "0" + os.environ["WORLD_SIZE"] = "1" + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12355" + + set_numa_affinity() + torch.distributed.init_process_group(get_nccl_backend()) + + self.rank = torch.distributed.get_rank() + self.world_size = torch.distributed.get_world_size() + local_rank = os.environ.get("LOCAL_RANK", 0) + get_torch_device().set_device(f"{get_device_name()}:{local_rank}") + + mpu.initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=self.world_size, + virtual_pipeline_model_parallel_size=None, + context_parallel_size=1, + expert_model_parallel_size=1, + ) + model_parallel_cuda_manual_seed(0) + self.hf_config = AutoConfig.from_pretrained( + self.config.hf_model_config_path, trust_remote_code=self.config.trust_remote_code + ) + print(self.hf_config, flush=True) + + self.params_mapping = { + # megatron core gpt model name, huggingface model name + # NOTICE: It's a little bit tricky, when 2 keys have the same prefix, we need to make sure the + # longer key within the containing relationship is processed first. + "embedding.word_embeddings": "model.embed_tokens", + # input layer norm for dpskv3 + "input_layernorm.weight": "input_layernorm.weight", + "input_layernorm.bias": "input_layernorm.bias", + # attn + "self_attention.linear_qkv.layer_norm_weight": "input_layernorm.weight", + "self_attention.linear_qkv.layer_norm_bias": "input_layernorm.bias", + "self_attention.linear_qkv": "self_attn.qkv_proj", + "self_attention.q_layernorm": "self_attn.q_norm", + "self_attention.k_layernorm": "self_attn.k_norm", + "self_attention.linear_proj": "self_attn.o_proj", + # mla + "self_attention.linear_q_proj": "self_attn.q_proj", + "self_attention.linear_q_down_proj": "self_attn.q_a_proj", + "self_attention.linear_q_up_proj.layer_norm_weight": "self_attn.q_a_layernorm.weight", + "self_attention.linear_q_up_proj": "self_attn.q_b_proj", + "self_attention.linear_kv_down_proj": "self_attn.kv_a_proj_with_mqa", + "self_attention.linear_kv_up_proj.layer_norm_weight": "self_attn.kv_a_layernorm.weight", + "self_attention.linear_kv_up_proj": "self_attn.kv_b_proj", + # mlp + "pre_mlp_layernorm": "post_attention_layernorm", + "mlp.linear_fc1.layer_norm_weight": "post_attention_layernorm.weight", + "mlp.linear_fc1.layer_norm_bias": "post_attention_layernorm.bias", + "mlp.linear_fc1": "mlp.gate_up_proj", + "mlp.linear_fc2": "mlp.down_proj", + # moe + "mlp.router.expert_bias": "mlp.gate.e_score_correction_bias", + "mlp.router": "mlp.gate", + "mlp.shared_experts.linear_fc1": "mlp.shared_experts.gate_up_proj", + "mlp.shared_experts.linear_fc2": "mlp.shared_experts.down_proj", + "linear_fc1": "gate_up_proj", + "linear_fc2": "down_proj", + # output + "final_layernorm": "norm", + "output_layer": "lm_head", + } + + if "Qwen2MoeForCausalLM" in self.hf_config.architectures: + self.params_mapping["mlp.shared_experts.linear_fc1"] = "mlp.shared_expert.gate_up_proj" + self.params_mapping["mlp.shared_experts.linear_fc2"] = "mlp.shared_expert.down_proj" + self.params_mapping["mlp.shared_experts.gate_weight"] = "mlp.shared_expert_gate.weight" + + def _load_state_dicts(self, model_ckpt_path: str) -> dict[str, Any]: + """_summary_ + Use Megatron dist_checkpointing to load the model state dicts from the checkpoint directory. + + Args: + model_ckpt_path (str): Path to the model checkpoint directory. + + Returns: + State dict containing the model parameters. + """ + + # init hf config + self.pipeline_shards = get_dynamic_pipeline_shards(self.hf_config.num_hidden_layers, self.world_size) + print(f"Pipeline shards: {self.pipeline_shards}, total layers: {sum(self.pipeline_shards)}") + + tf_config = hf_to_mcore_config( + self.hf_config, + torch.bfloat16, + num_layers_in_first_pipeline_stage=self.pipeline_shards[0] if len(self.pipeline_shards) > 1 else None, + num_layers_in_last_pipeline_stage=self.pipeline_shards[-1] if len(self.pipeline_shards) > 2 else None, + ) + tf_config.use_cpu_initialization = self.config.use_cpu_initialization + tie_word_embeddings = getattr(self.hf_config, "tie_word_embeddings", False) + + # init megatron model + def megatron_model_provider(pre_process, post_process): + from verl.models.mcore import init_mcore_model + + parallel_model = init_mcore_model( + tf_config, + self.hf_config, + pre_process, + post_process, + share_embeddings_and_output_weights=tie_word_embeddings, + value=False, + ) + return parallel_model + + context: Callable[..., ContextManager] = ( + init_empty_weights if self.config.use_cpu_initialization else noop_context + ) + with context(): + whole_model = get_model( + model_provider_func=megatron_model_provider, + model_type=ModelType.encoder_or_decoder, + wrap_with_ddp=False, + transformer_config=tf_config, + ) + + if self.config.use_cpu_initialization: + # convert meta device to empty tensor so it can use `copy_` function + whole_model[0].module = whole_model[0].module.to_empty(device="cpu") + + # load state dicts + sharded_state_dict = {} + for vpp_rank, model in enumerate(whole_model): + key = f"model{vpp_rank}" if len(whole_model) > 1 else "model" + mpu.set_virtual_pipeline_model_parallel_rank(vpp_rank) + sharded_state_dict[key] = model.sharded_state_dict() + model_state_dict = load_dist_checkpointing(sharded_state_dict, model_ckpt_path) + model_state_dict_list = [] + for vpp_rank, model in enumerate(whole_model): + key = f"model{vpp_rank}" if len(whole_model) > 1 else "model" + mpu.set_virtual_pipeline_model_parallel_rank(vpp_rank) + model_state_dict_list.append(model_state_dict[key]) + + return model_state_dict_list + + def _check_megatron_state_key(self, key: str) -> bool: + """ + Checks if the key is a valid Megatron state key. + + Now the model merger only supports keys that start with "decoder/embedding/output_layer" in TransformerLayer. + Shall not use key starts with "model." + """ + if key.startswith("model."): + raise ValueError( + f"Invalid key {key} in Megatron state_dict. Expected keys to start with " + f"'decoder/embedding/output_layer' in TransformerLayer." + ) + + skip_checking_keys = ["embedding.word_embeddings", "output_layer"] + for skip_key in skip_checking_keys: + if skip_key in key: + print(f"skip checking key {key}") + return + + # Exclude extra state keys + if not key.startswith("decoder"): + raise ValueError( + f"Invalid key {key} in Megatron state_dict. Expected keys to start with 'decoder' in TransformerLayer." + ) + + def _split_tensors( + self, key: str, tensor: torch.Tensor, config: PretrainedConfig, is_value_model: bool = False + ) -> list[torch.Tensor]: + """ + Splits a tensor into multiple tensors based on the name. + This is used to handle qkv and gate_up tensors. + """ + if "linear_fc1.weight" in key: + # if the tensor is gate and proj + gate_lst = [] + up_lst = [] + gate, up = tensor.chunk(2) + gate_lst.append(gate) + up_lst.append(up) + gate = torch.cat(gate_lst, dim=0) + up = torch.cat(up_lst, dim=0) + return [gate, up] + elif "self_attention.linear_qkv." in key and "layer_norm" not in key: + # if the tensor is qkv, for each param on tp, split into q, k, v + # concat q, k, v separately. + q_lst, k_lst, v_lst = [], [], [] + assert config.num_attention_heads % config.num_key_value_heads == 0 + num_q_per_kv = config.num_attention_heads // config.num_key_value_heads + assert tensor.shape[0] % (num_q_per_kv + 2) == 0, ( + f"Tensor shape {tensor.shape} is not divisible by {num_q_per_kv + 2}" + ) + kv_size = tensor.shape[0] // (num_q_per_kv + 2) + split_size = [kv_size * num_q_per_kv, kv_size, kv_size] + + num_query_groups_per_partition = config.num_key_value_heads + for chunk in tensor.chunk(num_query_groups_per_partition): + split_size = [ + kv_size * num_q_per_kv // num_query_groups_per_partition, + kv_size // num_query_groups_per_partition, + kv_size // num_query_groups_per_partition, + ] + q, k, v = chunk.split(split_size) + q_lst.append(q) + k_lst.append(k) + v_lst.append(v) + + return [torch.cat(q_lst, dim=0), torch.cat(k_lst, dim=0), torch.cat(v_lst, dim=0)] + else: + return [tensor] + + def _merge_state_dicts(self, model_state_dict_list: list[dict[str, Any]]) -> dict[str, torch.Tensor]: + state_dict = {} + layers_cum = 0 + if self.world_size > 1: + pipeline_cumsum = np.cumsum(self.pipeline_shards) + layers_cum = 0 if self.rank == 0 else pipeline_cumsum[self.rank - 1] + + print(f"{layers_cum=}") + for model_state_dict in model_state_dict_list: + layers_handled = 0 + keys = model_state_dict.keys() + for key in keys: + if "extra_state" in key: + continue + if self.config.tie_word_embedding and ("output_layer" in key): + print("skip lm_head and reward_head loading because of tie_word_embeddings") + continue + + self._check_megatron_state_key(key) + hf_name = self._replace_name(key, self.params_mapping) + assert hf_name is not None, f"Failed to convert layer name [{key}] from megatron to huggingface." + if "model.layers." in hf_name: + local_layer_no = int(hf_name.split(".")[2]) + layers_handled = max(local_layer_no, layers_handled) + global_layer_no = local_layer_no + layers_cum + new_key_list = hf_name.split(".") + new_key_list[2] = str(global_layer_no) + hf_name = ".".join(new_key_list) + else: + warnings.warn(f"hf_name {hf_name} will not be fixed with layer number", stacklevel=2) + + if "mlp.experts." in hf_name and ".weight" in hf_name: + name_prefix, expert_id = hf_name.split(".weight") + for proj in ["gate_up", "down"]: + if f"{proj}_proj" in hf_name: + hf_name = hf_name.replace( + f"mlp.experts.{proj}_proj.weight{expert_id}", + f"mlp.experts.{expert_id}.{proj}_proj.weight", + ) + + tensor = model_state_dict[key] + split_tensor = self._split_tensors( + key, tensor, self.hf_config, is_value_model=self.config.is_value_model + ) + + if len(split_tensor) == 1: + state_dict[hf_name] = split_tensor[0] + elif len(split_tensor) == 3: + # split qkv + for n, d in zip(["q", "k", "v"], split_tensor, strict=True): + state_dict[hf_name.replace("qkv", n)] = d + elif len(split_tensor) == 2: + # split gate up + state_dict[hf_name.replace("gate_up", "gate")] = split_tensor[0] + state_dict[hf_name.replace("gate_up", "up")] = split_tensor[1] + shape_info = ( + split_tensor.shape if isinstance(split_tensor, torch.Tensor) else [t.shape for t in split_tensor] + ) + print(f"converted {key} to {hf_name} with shape {shape_info}") + + layers_cum += layers_handled + 1 # zero based + + return state_dict + + def save_hf_model_and_tokenizer(self, merged_state_dict): + if self.world_size == 1: + return super().save_hf_model_and_tokenizer(merged_state_dict) + + from safetensors.torch import save_file + + layer_num = self.hf_config.num_hidden_layers + + # FIXME: make configurable + saves_per_layer = 1 if layer_num < 30 else 2 + saves_total = saves_per_layer * layer_num + saves_indexes = {} + + # calculate the layer start index and key chunks + layer_this_rank = self.pipeline_shards[self.rank] + pipeline_cumsum = np.cumsum(self.pipeline_shards) + layer_start = 0 if self.rank == 0 else pipeline_cumsum[self.rank - 1] + keys = list(merged_state_dict.keys()) + keys_chunk = np.array_split(np.array(keys), layer_this_rank * saves_per_layer) + numel = 0 + + assert len(keys_chunk) == layer_this_rank * saves_per_layer, ( + f"Expected {len(keys_chunk)} chunks, but got {layer_this_rank * saves_per_layer} for rank {self.rank}." + ) + + # save to model shards manually + target_dir = Path(self.config.target_dir) + for i, keys in enumerate(keys_chunk): + sd_to_save = {k: merged_state_dict[k] for k in keys} + numel += sum([sd_to_save[i].numel() for i in sd_to_save]) + save_idx = layer_start * saves_per_layer + i + save_path = target_dir / f"model-{save_idx + 1:05d}-of-{saves_total:05d}.safetensors" + + save_file(sd_to_save, save_path) + for k in keys: + saves_indexes[k] = str(save_path.name) + + tensor = torch.tensor([numel]).to(get_device_name()) + dist.all_reduce(tensor, op=dist.ReduceOp.SUM) + numel = tensor.cpu().item() + + all_save_indexes = [{} for _ in range(self.world_size)] + dist.all_gather_object(all_save_indexes, saves_indexes) + saves_indexes = {k: v for i in all_save_indexes for k, v in i.items()} + if self.rank == 0: + with open(target_dir / "model.safetensors.index.json", "w") as f: + json.dump( + { + "metadata": { + "total_size": numel, + }, + "weight_map": saves_indexes, + }, + f, + indent=4, + ) + print(f"model saved to {target_dir} with {numel=}") + + self.model_config.save_pretrained(self.config.target_dir) + + processor = hf_processor(self.hf_model_config_path, trust_remote_code=self.config.trust_remote_code) + tokenizer = hf_tokenizer(self.hf_model_config_path, trust_remote_code=self.config.trust_remote_code) + if processor is not None: + print(f"Saving processor to {self.config.target_dir}") + processor.save_pretrained(self.config.target_dir) + if tokenizer is not None: + print(f"Saving tokenizer to {self.config.target_dir}") + tokenizer.save_pretrained(self.config.target_dir) + + def merge_and_save(self): + from verl.utils.megatron_utils import get_dist_checkpoint_path + + model_ckpt_path = get_dist_checkpoint_path(self.config.local_dir) + + model_state_dict = self._load_state_dicts(model_ckpt_path) + merged_state_dict = self._merge_state_dicts(model_state_dict) + del model_state_dict + + if self.config.operation == "test": + if not self.config.test_hf_dir: + raise ValueError("test_hf_dir must be provided for test operation") + self._validate_state_dict(merged_state_dict) + elif self.config.operation == "merge": + self.save_hf_model_and_tokenizer(merged_state_dict) + if self.config.hf_upload: + self.upload_to_huggingface() + else: + raise ValueError(f"Unknown operation: {self.config.operation}") + + def _validate_state_dict(self, state_dict: dict[str, torch.Tensor]): + """ + Compares the merged Megatron state_dict against a reference safetensors model. + Applies necessary name mappings from Megatron to Hugging Face conventions using _replace_name. + """ + ref_state_dict = load_file(Path(self.config.test_hf_dir) / "model.safetensors") + + for name, loaded_weight in state_dict.items(): + # name = self._replace_name(original_name, self.params_mapping) + if not name or name.endswith(".bias") and name not in ref_state_dict: + continue + if "rotary_emb.inv_freq" in name: + continue + if "lm_head.weight" in name: + if self.config.is_value_model or self.config.tie_word_embedding: + continue + if name not in ref_state_dict: + raise RuntimeError(f"key: {name} not exist in state_dict") + param = ref_state_dict[name] + assert loaded_weight.dtype == param.dtype + torch.testing.assert_close(loaded_weight.to("cpu"), param, atol=1e-2, rtol=5e-2) + + def _replace_name(self, megatron_name: str, name_mapping: dict[str, str]) -> str: + for m_name, v_name in name_mapping.items(): + if m_name not in megatron_name: + continue + + megatron_name = megatron_name.replace("decoder", "model") + param_name = megatron_name.replace(m_name, v_name) + + return param_name + + return None # Return None if no mapping found + + def cleanup(self): + torch.distributed.destroy_process_group() diff --git a/e2c/verl/verl/models/README.md b/e2c/verl/verl/models/README.md new file mode 100644 index 0000000000000000000000000000000000000000..677b92f3871aa2f76a7f5bd8c07d1050bab14564 --- /dev/null +++ b/e2c/verl/verl/models/README.md @@ -0,0 +1,35 @@ +# Models +Common modelzoo such as huggingface/transformers stuggles when using Pytorch native model parallelism. Following the design principle of vLLM, we keep a simple, parallelizable, highly-optimized with packed inputs in verl. +## Adding a New Huggingface Model +### Step 1: Copy the model file from HF to verl +- Add a new file under verl/models/hf +- Copy ONLY the model file from huggingface/transformers/models to verl/models/hf + +### Step 2: Modify the model file to use packed inputs +- Remove all the code related to inference (kv cache) +- Modify the inputs to include only + - input_ids (total_nnz,) + - cu_seqlens (total_nnz + 1,) + - max_seqlen_in_batch: int +- Note that this requires using flash attention with causal mask. + +### Step 2.5: Add tests +- Add a test to compare this version and the huggingface version +- Following the infrastructure and add tests to tests/models/hf + +### Step 3: Add a function to apply tensor parallelism +- Please follow + - https://pytorch.org/docs/stable/distributed.tensor.parallel.html + - https://pytorch.org/tutorials/intermediate/TP_tutorial.html +- General comments + - Tensor Parallelism in native Pytorch is NOT auto-parallelism. The way it works is to specify how model parameters and input/output reshards using configs. These configs are then registered as hooks to perform input/output resharding before/after model forward. + +### Step 4: Add a function to apply data parallelism +- Please use FSDP2 APIs +- See demo here https://github.com/pytorch/torchtitan/blob/main/torchtitan/parallelisms/parallelize_llama.py#L413 + +### Step 5: Add a function to apply pipeline parallelism +- Comes in Pytorch 2.4 +- Currently only in alpha in nightly version +- Check torchtitan for more details + diff --git a/e2c/verl/verl/models/__init__.py b/e2c/verl/verl/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/models/llama/__init__.py b/e2c/verl/verl/models/llama/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/models/llama/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/models/llama/megatron/__init__.py b/e2c/verl/verl/models/llama/megatron/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fc851ea435ff43ad31eff24dc729df0e78cf8bee --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/__init__.py @@ -0,0 +1,34 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .modeling_llama_megatron import ( + ParallelLlamaForCausalLM, + # rmpad with megatron + ParallelLlamaForCausalLMRmPad, + # rmpad with megatron and pipeline parallelism + ParallelLlamaForCausalLMRmPadPP, + ParallelLlamaForValueRmPad, + ParallelLlamaForValueRmPadPP, + # original model with megatron + ParallelLlamaModel, +) + +__all__ = [ + "ParallelLlamaForCausalLM", + "ParallelLlamaForCausalLMRmPad", + "ParallelLlamaForCausalLMRmPadPP", + "ParallelLlamaForValueRmPad", + "ParallelLlamaForValueRmPadPP", + "ParallelLlamaModel", +] diff --git a/e2c/verl/verl/models/llama/megatron/checkpoint_utils/__init__.py b/e2c/verl/verl/models/llama/megatron/checkpoint_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/checkpoint_utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_loader.py b/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..dafecfdf084e81d2e72df9151fb3c593770127ac --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_loader.py @@ -0,0 +1,317 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import torch +import torch.distributed as dist + +from verl.utils.device import get_device_id, get_torch_device + + +def _megatron_calc_layer_map(config): + """Calculate the mapping of global layer_idx to local layer_idx + Returns: + layer_map (Dict: int -> tuple(int, int, int)): + mapping from the global layer index to + a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) + """ + from megatron.core import mpu + + print(f"get megatron data parallel size: {mpu.get_data_parallel_world_size()}") + + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + + layer_map = dict() + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + for pp_rank_idx in range(pp_size): + for virtual_pp_rank_idx in range(virtual_pp_size): + layer_offset = ( + virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + pp_rank_idx * num_layers_per_model + ) + for layer_idx in range(num_layers_per_model): + layer_map[layer_offset + layer_idx] = ( + pp_rank_idx, + virtual_pp_rank_idx, + layer_idx, + ) + return layer_map + + +def load_state_dict_to_megatron_llama( + state_dict, wrapped_models, config, params_dtype, is_value_model=False, tie_word_embeddings=False +): + """Load merged state_dict to sharded Megatron module in training.""" + from megatron.core import DistributedDataParallel as LocalDDP + from megatron.core import mpu + from megatron.core.transformer.module import Float16Module + from torch.nn.parallel import DistributedDataParallel as torchDDP + + from verl.utils.logger import print_rank_0 + from verl.utils.megatron_utils import unwrap_model + + start_time = time.time() + + def _get_gpt_model(model): + return model + + def fetch_params(module): + for param in module.parameters(): + torch.distributed.fetch( + param.data, src=mpu.get_data_parallel_src_rank(), group=mpu.get_data_parallel_group() + ) + + dp_rank = mpu.get_data_parallel_rank() + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + mp_group = mpu.get_model_parallel_group() + + if torch.distributed.get_rank() == 0: + assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" + assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" + assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" + + if not isinstance(wrapped_models, list | tuple): + wrapped_models = list(wrapped_models) + + assert len(wrapped_models) == virtual_pp_size + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers, ( + f"num_layers_per_model: {num_layers_per_model} * pp_size: {pp_size} * virtual_pp_size " + f"{virtual_pp_size} != config.num_hidden_layers: {config.num_hidden_layers}" + ) + + models = [None] * len(wrapped_models) + + for i, wrapped_model in enumerate(wrapped_models): + models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) + gpt_model_module = _get_gpt_model(models[i]) + assert len(gpt_model_module.model.layers) == num_layers_per_model + + def _fetch_tensor(tensor, name) -> torch.Tensor: + """fetch tensor""" + nonlocal state_dict + if tensor is not None: + tensor.data.copy_(state_dict[name]) + + def _fetch_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """fetch tensor in tp shards""" + nonlocal state_dict + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + if name in state_dict: + full_weight = state_dict[name] + + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + if tensor is not None: + tensor.data.copy_(tensor_chunk[tp_rank]) + else: + print(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + + def _fetch_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """fetch tensor in tp shards""" + nonlocal state_dict + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + if name in state_dict: + full_weight = state_dict[name] + + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + if tensor is not None: + tensor.data.copy_(tensor_chunk[tp_rank]) + else: + print(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + + def _fetch_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: + """fetch gate_up tensor in tp shards""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + if gate_name in state_dict and up_name in state_dict: + gate_weight = state_dict[gate_name] + up_weight = state_dict[up_name] + new_gate_up_weight = torch.empty( + config.intermediate_size * 2, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + intermediate_size_tp = config.intermediate_size // tp_size + gate_weight_tp = gate_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + up_weight_tp = up_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + new_gate_up_weight[intermediate_size_tp * 2 * i : intermediate_size_tp * 2 * (i + 1)].copy_( + torch.cat([gate_weight_tp, up_weight_tp], dim=0) + ) + + tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) + if tensor is not None: + tensor.data.copy_(tensor_chunk[tp_rank]) + else: + print(f"tp_shard tensor:[{gate_name}, {up_name}] not in state_dict, skip loading") + + def _fetch_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name) -> torch.Tensor: + """fetch tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + assert q_name in state_dict and k_name in state_dict and v_name in state_dict + full_weight_q = state_dict[q_name] + full_weight_k = state_dict[k_name] + full_weight_v = state_dict[v_name] + + hidden_size_per_head = config.hidden_size // config.num_attention_heads + + if config.num_key_value_heads >= tp_size: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size + total_size = q_size_tp + 2 * kv_size_tp + new_weight_qkv = torch.empty( + total_size * tp_size, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + k_part = full_weight_k[i * kv_size_tp : (i + 1) * kv_size_tp] + v_part = full_weight_v[i * kv_size_tp : (i + 1) * kv_size_tp] + new_weight_qkv[i * total_size : (i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], dim=0)) + + else: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head + total_size = q_size_tp + 2 * kv_size_tp + new_weight_qkv = torch.empty( + total_size * tp_size, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head + end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head + k_part = full_weight_k[start_idx:end_idx] + v_part = full_weight_v[start_idx:end_idx] + new_weight_qkv[i * total_size : (i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], dim=0)) + + tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) + if tensor is not None: + tensor.data.copy_(tensor_chunk[tp_rank]) + + # Embeddings + # ------------------- + print_rank_0("loading embeddings...") + gpt_model_module = _get_gpt_model(models[0]) + embed_tokens_weight = None + if pp_rank == 0: + embed_tokens_weight = gpt_model_module.model.embed_tokens.weight + _fetch_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") + + # Transformer layers + # ------------------- + layer_map = _megatron_calc_layer_map(config) + + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + num_layer_per_pp = config.num_hidden_layers // pp_size + vpp_size = mpu.get_virtual_pipeline_model_parallel_world_size() + + layer_list = [] + if vpp_size is not None: + for vpp_rank in range(vpp_size): + num_layer_vpp_chunk = num_layer_per_pp // vpp_size + num_layer_this_model = num_layer_vpp_chunk + offset = vpp_rank * (config.num_hidden_layers // mpu.get_virtual_pipeline_model_parallel_world_size()) + ( + mpu.get_pipeline_model_parallel_rank() * num_layer_vpp_chunk + ) + layer_list.extend(list(range(offset, offset + num_layer_this_model))) + else: + num_layer_this_model = num_layer_per_pp + offset = pp_rank * num_layer_per_pp + layer_list.extend(list(range(offset, offset + num_layer_this_model))) + + for layer in layer_list: + print_rank_0(f"loading layer #{layer}...") + layer_name = f"model.layers.{layer}" + dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] + + gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) + sync_layer = gpt_model_module.model.layers[dst_layer_idx] + + _fetch_tensor( + sync_layer.input_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.input_layernorm.weight", + ) + + _fetch_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_proj.weight", + f"{layer_name}.self_attn.k_proj.weight", + f"{layer_name}.self_attn.v_proj.weight", + ) + + _fetch_tp_shard_tensor( + sync_layer.self_attn.o_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.o_proj.weight", + chunk_dim=1, + ) + + _fetch_tensor( + sync_layer.post_attention_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.post_attention_layernorm.weight", + ) + + _fetch_tp_shard_tensor_gate_up( + sync_layer.mlp.gate_up_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.gate_proj.weight", + f"{layer_name}.mlp.up_proj.weight", + ) + + _fetch_tp_shard_tensor( + sync_layer.mlp.down_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.down_proj.weight", + chunk_dim=1, + ) + # Final Layernorm + # ------------------- + print_rank_0("loading final layernorm...") + gpt_model_module = _get_gpt_model(models[-1]) + _fetch_tensor( + getattr(gpt_model_module.model.norm, "weight", None), + "model.norm.weight", + ) + + print_rank_0("loading lm_head...") + if pp_rank + 1 == pp_size: + lm_head_weight = gpt_model_module.lm_head.weight + + if is_value_model: + if "lm_head.weight" in state_dict and state_dict["lm_head.weight"].shape[0] == 1: + _fetch_tensor(lm_head_weight, "lm_head.weight") + print_rank_0("load lm_head weight") + elif "reward_head.weight" in state_dict and state_dict["reward_head.weight"].shape[0] == 1: + _fetch_tensor(lm_head_weight, "reward_head.weight") + print_rank_0("load lm_head from value_head weight") + else: + _fetch_tensor(None, "lm_head.weight") + print_rank_0("fail to match lm_head in value_model") + else: + _fetch_tp_shard_tensor(lm_head_weight, "lm_head.weight") + + dist.barrier() + get_torch_device().empty_cache() + print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") diff --git a/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_loader_depracated.py b/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_loader_depracated.py new file mode 100644 index 0000000000000000000000000000000000000000..2f65bc6b1701bdb79cf1ed282de0212bd6396fdc --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_loader_depracated.py @@ -0,0 +1,458 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import torch +import torch.distributed as dist + +from verl.utils.device import get_device_id, get_torch_device + + +def _megatron_calc_layer_map(config): + """Calculate the mapping of global layer_idx to local layer_idx + Returns: + layer_map (Dict: int -> tuple(int, int, int)): + mapping from the global layer index to + a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) + """ + from megatron.core import mpu + + print(f"get megatron data parallel size: {mpu.get_data_parallel_world_size()}") + + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + + layer_map = dict() + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + for pp_rank_idx in range(pp_size): + for virtual_pp_rank_idx in range(virtual_pp_size): + layer_offset = ( + virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + pp_rank_idx * num_layers_per_model + ) + for layer_idx in range(num_layers_per_model): + layer_map[layer_offset + layer_idx] = ( + pp_rank_idx, + virtual_pp_rank_idx, + layer_idx, + ) + return layer_map + + +def load_state_dict_to_megatron_llama( + state_dict, wrapped_models, config, params_dtype, is_value_model=False, tie_word_embeddings=False +): + """Load merged state_dict to sharded Megatron module in training.""" + from megatron.core import DistributedDataParallel as LocalDDP + from megatron.core import mpu + from megatron.core.transformer.module import Float16Module + from torch.nn.parallel import DistributedDataParallel as torchDDP + + from verl.utils.logger import print_rank_0 + from verl.utils.megatron_utils import unwrap_model + + start_time = time.time() + + def _get_gpt_model(model): + return model + + def broadcast_params(module): + for param in module.parameters(): + torch.distributed.broadcast( + param.data, src=mpu.get_data_parallel_src_rank(), group=mpu.get_data_parallel_group() + ) + + dp_rank = mpu.get_data_parallel_rank() + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + mp_group = mpu.get_model_parallel_group() + + if torch.distributed.get_rank() == 0: + assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" + assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" + assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" + + if not isinstance(wrapped_models, list | tuple): + wrapped_models = list(wrapped_models) + + assert len(wrapped_models) == virtual_pp_size + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers, ( + f"num_layers_per_model: {num_layers_per_model} * pp_size: {pp_size} * virtual_pp_size " + f"{virtual_pp_size} != config.num_hidden_layers: {config.num_hidden_layers}" + ) + + models = [None] * len(wrapped_models) + + for i, wrapped_model in enumerate(wrapped_models): + models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) + gpt_model_module = _get_gpt_model(models[i]) + assert len(gpt_model_module.model.layers) == num_layers_per_model + + def _broadcast_tensor(tensor, name) -> torch.Tensor: + """broadcast tensor from rank0 across mp_group""" + nonlocal state_dict + nonlocal mp_group + if torch.distributed.get_rank() == 0: + if name in state_dict: + weight = state_dict[name] + tensor_shape = weight.shape + else: + tensor_shape = None + else: + weight = None + tensor_shape = None + + obj_list = [tensor_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + tensor_shape = obj_list[0] + + if tensor_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tensor:[{name}] not in state_dict, skip load") + return + + if tensor is None: + tensor = torch.empty( + tensor_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + if torch.distributed.get_rank() == 0: + tensor.data.copy_(weight) + dist.broadcast(tensor, src=0, group=mp_group) + + def _broadcast_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == 0: + if name in state_dict: + full_weight = state_dict[name] + + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == 0: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=0, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == 0: + if name in state_dict: + full_weight = state_dict[name] + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == 0: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=0, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == 0: + gate_weight = state_dict[gate_name] + up_weight = state_dict[up_name] + new_gate_up_weight = torch.empty( + config.intermediate_size * 2, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + intermediate_size_tp = config.intermediate_size // tp_size + gate_weight_tp = gate_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + up_weight_tp = up_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + new_gate_up_weight[intermediate_size_tp * 2 * i : intermediate_size_tp * 2 * (i + 1)].copy_( + torch.cat([gate_weight_tp, up_weight_tp], dim=0) + ) + + tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank() == 0:} tensor {gate_name, up_name} shape " + f"{tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == 0: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=0, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == 0: + assert q_name in state_dict and k_name in state_dict and v_name in state_dict + full_weight_q = state_dict[q_name] + full_weight_k = state_dict[k_name] + full_weight_v = state_dict[v_name] + + hidden_size_per_head = config.hidden_size // config.num_attention_heads + + if config.num_key_value_heads >= tp_size: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size + total_size = q_size_tp + 2 * kv_size_tp + new_weight_qkv = torch.empty( + total_size * tp_size, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + k_part = full_weight_k[i * kv_size_tp : (i + 1) * kv_size_tp] + v_part = full_weight_v[i * kv_size_tp : (i + 1) * kv_size_tp] + new_weight_qkv[i * total_size : (i + 1) * total_size].copy_( + torch.cat([q_part, k_part, v_part], dim=0) + ) + + else: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head + total_size = q_size_tp + 2 * kv_size_tp + new_weight_qkv = torch.empty( + total_size * tp_size, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head + end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head + k_part = full_weight_k[start_idx:end_idx] + v_part = full_weight_v[start_idx:end_idx] + new_weight_qkv[i * total_size : (i + 1) * total_size].copy_( + torch.cat([q_part, k_part, v_part], dim=0) + ) + + tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{q_name, k_name, v_name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {q_name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == 0: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=0, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + if dp_rank == 0: + # Embeddings + # ------------------- + print_rank_0("loading embeddings...") + gpt_model_module = _get_gpt_model(models[0]) + embed_tokens_weight = None + if pp_rank == 0: + embed_tokens_weight = gpt_model_module.model.embed_tokens.weight + _broadcast_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") + + # Transformer layers + # ------------------- + layer_map = _megatron_calc_layer_map(config) + + for layer in range(config.num_hidden_layers): + print_rank_0(f"loading layer #{layer}...") + layer_name = f"model.layers.{layer}" + dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] + + gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) + sync_layer = gpt_model_module.model.layers[dst_layer_idx] + + _broadcast_tensor( + sync_layer.input_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.input_layernorm.weight", + ) + + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_proj.weight", + f"{layer_name}.self_attn.k_proj.weight", + f"{layer_name}.self_attn.v_proj.weight", + ) + + _broadcast_tp_shard_tensor( + sync_layer.self_attn.o_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.o_proj.weight", + chunk_dim=1, + ) + + _broadcast_tensor( + sync_layer.post_attention_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.post_attention_layernorm.weight", + ) + + _broadcast_tp_shard_tensor_gate_up( + sync_layer.mlp.gate_up_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.gate_proj.weight", + f"{layer_name}.mlp.up_proj.weight", + ) + + _broadcast_tp_shard_tensor( + sync_layer.mlp.down_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.down_proj.weight", + chunk_dim=1, + ) + # Final Layernorm + # ------------------- + print_rank_0("loading final layernorm...") + gpt_model_module = _get_gpt_model(models[-1]) + _broadcast_tensor( + getattr(gpt_model_module.model.norm, "weight", None), + "model.norm.weight", + ) + + print_rank_0("loading lm_head...") + lm_head_weight = None + if pp_rank + 1 == pp_size: + lm_head_weight = gpt_model_module.lm_head.weight + + if is_value_model: + if "lm_head.weight" in state_dict and state_dict["lm_head.weight"].shape[0] == 1: + _broadcast_tensor(lm_head_weight, "lm_head.weight") + print_rank_0("load lm_head weight") + elif "reward_head.weight" in state_dict and state_dict["reward_head.weight"].shape[0] == 1: + _broadcast_tensor(lm_head_weight, "reward_head.weight") + print_rank_0("load lm_head from value_head weight") + else: + _broadcast_tensor(None, "lm_head.weight") + print_rank_0("fail to match lm_head in value_model") + else: + _broadcast_tp_shard_tensor(lm_head_weight, "lm_head.weight") + dist.barrier() + # Broadcast weights inside data parallel groups + for wrapped_model in wrapped_models: + broadcast_params(wrapped_model) + + get_torch_device().empty_cache() + print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") diff --git a/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_saver.py b/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_saver.py new file mode 100644 index 0000000000000000000000000000000000000000..595efcde376ea498ee65bc39310060a046b83d1b --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/checkpoint_utils/llama_saver.py @@ -0,0 +1,442 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import torch +import torch.distributed as dist +from megatron.core import mpu +from megatron.core.distributed import DistributedDataParallel as LocalDDP +from megatron.core.transformer.module import Float16Module +from torch.nn.parallel import DistributedDataParallel as torchDDP + +from verl.utils.device import get_device_id, get_torch_device +from verl.utils.logger import print_rank_0 +from verl.utils.megatron_utils import unwrap_model + + +def _megatron_calc_global_rank(tp_rank: int = 0, dp_rank: int = 0, pp_rank: int = 0): + """given TP,DP,PP rank to get the global rank.""" + + tp_size = mpu.get_tensor_model_parallel_world_size() + dp_size = mpu.get_data_parallel_world_size() + pp_size = mpu.get_pipeline_model_parallel_world_size() + assert tp_size * dp_size * pp_size == torch.distributed.get_world_size(), ( + f"{tp_size} x {dp_size} x {pp_size} != {torch.distributed.get_world_size()}" + ) + # We only support TP-DP-PP grouping, for correctness when resharding + return (pp_rank * dp_size + dp_rank) * tp_size + tp_rank + + +def _megatron_calc_layer_map(config): + """Calculate the mapping of global layer_idx to local layer_idx + Returns: + layer_map (Dict: int -> tuple(int, int, int)): + mapping from the global layer index to + a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) + """ + from megatron.core import mpu + + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + + layer_map = dict() + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + for pp_rank_idx in range(pp_size): + for virtual_pp_rank_idx in range(virtual_pp_size): + layer_offset = ( + virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + pp_rank_idx * num_layers_per_model + ) + for layer_idx in range(num_layers_per_model): + layer_map[layer_offset + layer_idx] = ( + pp_rank_idx, + virtual_pp_rank_idx, + layer_idx, + ) + return layer_map + + +def merge_megatron_ckpt_llama(wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False): + """Merge sharded parameters of a Megatron module into a merged checkpoint. + + Args: + wrapped_models (list of megatron.core.distributed.DistributedDataParallel): + The local DDP wrapped megatron modules. + config (str or None): + HF config for model + dtype: model params type + is_value_model: if model is value model + tie_word_embeddings: tie_word_embeddings, not used in llama, only to keep same interface with qwen2 + Returns: + state_dict (dict): + The merged state_dict in rank 0, and an empty dictionary in other ranks. + """ + start_time = time.time() + + def _get_gpt_model(model): + return model + + dp_rank = mpu.get_data_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + pp_rank = mpu.get_pipeline_model_parallel_rank() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + mp_group = mpu.get_model_parallel_group() + + if dist.get_rank() == 0: + assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" + assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" + assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" + + if not isinstance(wrapped_models, list | tuple): + wrapped_models = list(wrapped_models) + + assert len(wrapped_models) == virtual_pp_size + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + models = [None] * len(wrapped_models) + + for i, wrapped_model in enumerate(wrapped_models): + models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) + assert len(models[i].model.layers) == num_layers_per_model, ( + "len model layers {} not equal to num_layers_per_model {}".format( + len(models[i].model.layers), num_layers_per_model + ) + ) + + state_dict = dict() + + def _get_cpu_tensor(tensor: torch.Tensor): + if tensor is None: + return None + if tensor.device == torch.device("cpu"): + return tensor.detach().clone() + return tensor.detach().cpu() + + def _broadcast_tensor(tensor, name, src_pp_rank) -> torch.Tensor: + """broadcast tensor across mp_group""" + nonlocal state_dict + nonlocal mp_group + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) + + if torch.distributed.get_rank() == src_rank: + if tensor is None: + weight = None + tensor_shape = None + else: + weight = tensor + tensor_shape = weight.shape + else: + weight = None + tensor_shape = None + + obj_list = [tensor_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + tensor_shape = obj_list[0] + + if tensor_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tensor:[{name}] not exist, skip collect") + return + + if weight is None: + weight = torch.empty( + tensor_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + dist.broadcast(weight, src=src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + state_dict[name] = _get_cpu_tensor(weight) + + def _broadcast_tp_shard_tensor(tensor, name, src_pp_rank, concat_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=concat_dim) + if mutate_func is not None: + full_tensor = mutate_func(full_tensor) + state_dict[name] = full_tensor + + def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name, src_pp_rank) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=0) + intermediate_size_tp = config.intermediate_size // tp_size + gate_weight_list = [] + up_weight_list = [] + for i in range(tp_size): + gate_up_weight_tp = full_tensor[intermediate_size_tp * 2 * i : intermediate_size_tp * 2 * (i + 1)] + gate_weight_tp = gate_up_weight_tp[:intermediate_size_tp] + up_weight_tp = gate_up_weight_tp[intermediate_size_tp:] + gate_weight_list.append(gate_weight_tp) + up_weight_list.append(up_weight_tp) + + state_dict[gate_name] = torch.cat(gate_weight_list, dim=0) + state_dict[up_name] = torch.cat(up_weight_list, dim=0) + + def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, src_pp_rank): + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{q_name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=0) + q_weight_list = [] + k_weight_list = [] + v_weight_list = [] + hidden_size_per_head = config.hidden_size // config.num_attention_heads + + if config.num_key_value_heads >= tp_size: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size + total_size = q_size_tp + 2 * kv_size_tp + for i in range(tp_size): + qkv_part = full_tensor[i * total_size : (i + 1) * total_size] + q_part = qkv_part[:q_size_tp] + k_part = qkv_part[q_size_tp : q_size_tp + kv_size_tp] + v_part = qkv_part[q_size_tp + kv_size_tp : total_size] + q_weight_list.append(q_part) + k_weight_list.append(k_part) + v_weight_list.append(v_part) + else: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head + total_size = q_size_tp + 2 * kv_size_tp + for i in range(tp_size): + qkv_part = full_tensor[i * total_size : (i + 1) * total_size] + q_part = qkv_part[:q_size_tp] + k_part = qkv_part[q_size_tp : q_size_tp + kv_size_tp] + v_part = qkv_part[q_size_tp + kv_size_tp : total_size] + q_weight_list.append(q_part) + if i * config.num_key_value_heads % tp_size == 0: + k_weight_list.append(k_part) + v_weight_list.append(v_part) + + state_dict[q_name] = torch.cat(q_weight_list, dim=0) + state_dict[k_name] = torch.cat(k_weight_list, dim=0) + state_dict[v_name] = torch.cat(v_weight_list, dim=0) + + # empty cache before collecting weights + get_torch_device().empty_cache() + # Embeddings + # ------------------- + if dp_rank == 0: + # Embeddings + # ------------------- + print_rank_0("collecting embeddings...") + gpt_model_module = _get_gpt_model(models[0]) + _broadcast_tp_shard_tensor( + gpt_model_module.model.embed_tokens.weight if pp_rank == 0 else None, + "model.embed_tokens.weight", + src_pp_rank=0, + ) + + # Transformer layers + # ------------------- + layer_map = _megatron_calc_layer_map(config) + for layer in range(config.num_hidden_layers): + print_rank_0(f"collecting layer #{layer}...") + layer_name = f"model.layers.{layer}" + src_pp_rank, src_virtual_pp_rank, src_layer_idx = layer_map[layer] + + gpt_model_module = _get_gpt_model(models[src_virtual_pp_rank]) + sync_layer = gpt_model_module.model.layers[src_layer_idx] + + _broadcast_tensor( + sync_layer.input_layernorm.weight, + f"{layer_name}.input_layernorm.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.weight, + f"{layer_name}.self_attn.q_proj.weight", + f"{layer_name}.self_attn.k_proj.weight", + f"{layer_name}.self_attn.v_proj.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor( + sync_layer.self_attn.o_proj.weight, + f"{layer_name}.self_attn.o_proj.weight", + concat_dim=1, + src_pp_rank=src_pp_rank, + ) + + _broadcast_tensor( + sync_layer.post_attention_layernorm.weight, + f"{layer_name}.post_attention_layernorm.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor_gate_up( + sync_layer.mlp.gate_up_proj.weight, + f"{layer_name}.mlp.gate_proj.weight", + f"{layer_name}.mlp.up_proj.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor( + sync_layer.mlp.down_proj.weight, + f"{layer_name}.mlp.down_proj.weight", + concat_dim=1, + src_pp_rank=src_pp_rank, + ) + + # Final Layernorm + # ------------------- + print_rank_0("collecting final layernorm...") + gpt_model_module = _get_gpt_model(models[-1]) + _broadcast_tensor( + getattr(gpt_model_module.model.norm, "weight", None), + "model.norm.weight", + src_pp_rank=pp_size - 1, + ) + + print_rank_0("collecting lm_head...") + + if is_value_model: + if pp_rank == pp_size - 1: + print(f"gpt_model_module.lm_head.weight: {gpt_model_module.lm_head.weight.shape}") + _broadcast_tensor( + gpt_model_module.lm_head.weight if pp_rank == pp_size - 1 else None, + "lm_head.weight", + src_pp_rank=pp_size - 1, + ) + _broadcast_tensor( + gpt_model_module.reward_head.weight + if pp_rank == pp_size - 1 and getattr(gpt_model_module, "reward_weight", None) is not None + else None, + "reward_head.weight", + src_pp_rank=pp_size - 1, + ) + + else: + _broadcast_tp_shard_tensor( + getattr(gpt_model_module.lm_head, "weight", None) if pp_rank == pp_size - 1 else None, + "lm_head.weight", + src_pp_rank=pp_size - 1, + ) + + dist.barrier() + + get_torch_device().empty_cache() + if torch.distributed.get_rank() == 0: + if dtype not in [torch.float16, torch.bfloat16, torch.float32]: + print(f'Unknown/unsupported dtype to save: {dtype}"') + exit(1) + for k, v in state_dict.items(): + if dtype != v.dtype: + state_dict[k] = v.to(dtype) + + print_rank_0(f"merge megatron ckpt done, time elapsed {time.time() - start_time}s") + return state_dict diff --git a/e2c/verl/verl/models/llama/megatron/layers/__init__.py b/e2c/verl/verl/models/llama/megatron/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..352bc56086dcf1e7e2a6534f0e6e506796a1fb6d --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/layers/__init__.py @@ -0,0 +1,34 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .parallel_attention import ParallelLlamaAttention +from .parallel_decoder import ParallelLlamaDecoderLayer, ParallelLlamaDecoderLayerRmPad +from .parallel_linear import ( + LinearForLastLayer, + MergedColumnParallelLinear, + QKVParallelLinear, +) +from .parallel_mlp import ParallelLlamaMLP +from .parallel_rmsnorm import ParallelLlamaRMSNorm + +__all__ = [ + "LinearForLastLayer", + "MergedColumnParallelLinear", + "QKVParallelLinear", + "ParallelLlamaAttention", + "ParallelLlamaDecoderLayer", + "ParallelLlamaDecoderLayerRmPad", + "ParallelLlamaMLP", + "ParallelLlamaRMSNorm", +] diff --git a/e2c/verl/verl/models/llama/megatron/layers/parallel_attention.py b/e2c/verl/verl/models/llama/megatron/layers/parallel_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..e8aacbdb7dd63181cc538b4794babe1dc04bf89a --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/layers/parallel_attention.py @@ -0,0 +1,460 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Optional + +import torch +import torch.nn.functional as F +from einops import rearrange +from flash_attn.layers.rotary import apply_rotary_emb +from megatron.core import ModelParallelConfig, tensor_parallel +from megatron.core import parallel_state as mpu +from torch import nn +from transformers import LlamaConfig +from transformers.utils import is_flash_attn_2_available + +from verl.models.llama.megatron.layers.parallel_linear import QKVParallelLinear +from verl.utils.megatron import tensor_parallel as tp_utils + + +class LlamaRotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self._set_cos_sin_cache( + seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() + ) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if seq_len > self.max_seq_len_cached: + self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) + + return ( + self.cos_cached[:seq_len].to(dtype=x.dtype), + self.sin_cached[:seq_len].to(dtype=x.dtype), + ) + + +class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding): + """LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev""" + + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): + self.scaling_factor = scaling_factor + super().__init__(dim, max_position_embeddings, base, device) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + t = t / self.scaling_factor + + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + + +class LlamaDynamicNTKScalingRotaryEmbedding(LlamaRotaryEmbedding): + """LlamaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla""" + + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): + self.scaling_factor = scaling_factor + super().__init__(dim, max_position_embeddings, base, device) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + + if seq_len > self.max_position_embeddings: + base = self.base * ( + (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1) + ) ** (self.dim / (self.dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + + +class LlamaLlama3ScalingRotaryEmbedding(LlamaRotaryEmbedding): + def __init__(self, dim, config, max_position_embeddings=2048, base=10000, device=None): + super().__init__(dim, max_position_embeddings, base, device) + + self.factor = config.rope_scaling["factor"] # `8` in the original implementation + self.high_freq_factor = config.rope_scaling["high_freq_factor"] # `1` in the original implementation + self.low_freq_factor = config.rope_scaling["low_freq_factor"] # `4` in the original implementation + self.old_context_len = config.rope_scaling[ + "original_max_position_embeddings" + ] # `8192` in the original implementation + + low_freq_wavelen = self.old_context_len / self.low_freq_factor + high_freq_wavelen = self.old_context_len / self.high_freq_factor + + wavelen = 2 * math.pi / self.inv_freq + # wavelen < high_freq_wavelen: do nothing; wavelen > low_freq_wavelen: divide by factor + inv_freq_llama = torch.where(wavelen > low_freq_wavelen, self.inv_freq / self.factor, self.inv_freq) + # otherwise: interpolate between the two, using a smooth factor + smooth_factor = (self.old_context_len / wavelen - self.low_freq_factor) / ( + self.high_freq_factor - self.low_freq_factor + ) + smoothed_inv_freq = (1 - smooth_factor) * inv_freq_llama / self.factor + smooth_factor * inv_freq_llama + is_medium_freq = ~(wavelen < high_freq_wavelen) * ~(wavelen > low_freq_wavelen) + inv_freq = torch.where(is_medium_freq, smoothed_inv_freq, inv_freq_llama) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self._set_cos_sin_cache( + seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() + ) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids): + cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class ParallelLlamaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): + super().__init__() + self.config = config + self.megatron_config = megatron_config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + + # assign values after tp + tp_size = mpu.get_tensor_model_parallel_world_size() + assert self.num_heads % tp_size == 0, ( + f"num_head must be divisible by tp_size. Got num_head={self.num_heads}, tp_size={tp_size}" + ) + assert self.num_key_value_heads % tp_size == 0, ( + f"num_key_value_heads must be divisible by tp_size. Got num_key_value_heads=" + f"{self.num_key_value_heads}, tp_size={tp_size}" + ) + + self.num_heads_per_tp = self.num_heads // tp_size + self.num_key_value_heads_per_tp = self.num_key_value_heads // tp_size + self.hidden_size_per_tp = self.hidden_size // tp_size + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and " + f"`num_heads`: {self.num_heads})." + ) + + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + row_kwargs = tp_utils.get_default_kwargs_for_row_parallel_linear() + + if megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + assert row_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, megatron_config) + tp_utils.update_kwargs_with_config(row_kwargs, megatron_config) + + # [self.q_size, self.k_size, self.v_size] + self.qkv_proj = QKVParallelLinear( + input_size=self.hidden_size, + num_heads=self.num_heads, + num_key_value_heads=self.num_key_value_heads, + head_dim=self.head_dim, + bias=config.attention_bias, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + + self.q_size = self.num_heads_per_tp * self.head_dim + self.k_size = self.num_key_value_heads_per_tp * self.head_dim + self.v_size = self.num_key_value_heads_per_tp * self.head_dim + + self.o_proj = tensor_parallel.RowParallelLinear( + input_size=self.num_heads * self.head_dim, + output_size=self.hidden_size, + bias=config.attention_bias, + input_is_parallel=True, + skip_bias_add=False, + **row_kwargs, + ) + + self._init_rope() + + def _init_rope(self): + if self.config.rope_scaling is None: + self.rotary_emb = LlamaRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + else: + rope_type_key = "type" if "type" in self.config.rope_scaling else "rope_type" + scaling_type = self.config.rope_scaling[rope_type_key] + scaling_factor = self.config.rope_scaling["factor"] + if scaling_type == "linear": + self.rotary_emb = LlamaLinearScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + scaling_factor=scaling_factor, + base=self.rope_theta, + ) + elif scaling_type == "dynamic": + self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + scaling_factor=scaling_factor, + base=self.rope_theta, + ) + elif scaling_type == "llama3": + self.rotary_emb = LlamaLlama3ScalingRotaryEmbedding( + self.head_dim, + self.config, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + else: + raise ValueError(f"Unknown RoPE scaling type {scaling_type}") + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + qkv = self.qkv_proj(hidden_states)[0] + query_states, key_states, value_states = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + + query_states = query_states.view(bsz, q_len, self.num_heads_per_tp, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads_per_tp, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads_per_tp, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads_per_tp, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads_per_tp, q_len, kv_seq_len)}, " + f"but is {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads_per_tp, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads_per_tp, q_len, self.head_dim)}, " + f"but is {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size_per_tp) + attn_output = self.o_proj(attn_output)[0] + return attn_output + + +""" +Remove padding Attention +- Using Flash-attn 2 +- Compatible with sequence parallel +""" + + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + +def apply_rotary_pos_emb_rmpad(q, k, cos, sin, position_ids, indices, sequence_length): + batch_size = position_ids.shape[0] + + q = pad_input(q, indices, batch_size, sequence_length) # (batch_size, seqlen, num_head, head_dim) + k = pad_input(k, indices, batch_size, sequence_length) + cos = cos[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] + sin = sin[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + + q_embed = index_first_axis(rearrange(q_embed, "b s ... -> (b s) ..."), indices) + k_embed = index_first_axis(rearrange(k_embed, "b s ... -> (b s) ..."), indices) + + return q_embed, k_embed + + +# use flash-attn rotary embeddings with rmpad +# cos/sin shoudl be: (seq_length, rotary_dim / 2) +def apply_rotary_pos_emb_rmpad_flash(q, k, cos, sin, cu_seqlens, max_seqlen): + q_embed = apply_rotary_emb( + q, cos, sin, interleaved=False, inplace=False, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen + ) + k_embed = apply_rotary_emb( + k, cos, sin, interleaved=False, inplace=False, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen + ) + return q_embed, k_embed + + +class ParallelLlamaAttentionRmPad(ParallelLlamaAttention): + def forward( + self, + hidden_states: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + sequence_length: int = None, + indices: torch.Tensor = None, + cu_seqlens: torch.Tensor = None, + max_seqlen_in_batch: int = None, + ): + total_nnz, _, _ = hidden_states.size() # This is the total_nnz padded after sequence parallel + + if self.megatron_config.sequence_parallel: + total_nnz = total_nnz * mpu.get_tensor_model_parallel_world_size() + + qkv = self.qkv_proj(hidden_states)[0] + query_states, key_states, value_states = qkv.split( + [self.q_size, self.k_size, self.v_size], dim=-1 + ) # (total_nnz, 1, hidden_size) + + if self.megatron_config.sequence_parallel: + sequence_parallel_pad = total_nnz - cu_seqlens[-1] + total_nnz = cu_seqlens[-1] # total_nnz before sp padding + query_states = query_states[:total_nnz] + key_states = key_states[:total_nnz] + value_states = value_states[:total_nnz] + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dime x hidden_dim + # therefore we just need to keep the original shape + query_states = query_states.view(total_nnz, self.num_heads_per_tp, self.head_dim) + key_states = key_states.view(total_nnz, self.num_key_value_heads_per_tp, self.head_dim) + value_states = value_states.view(total_nnz, self.num_key_value_heads_per_tp, self.head_dim) + + cos, sin = self.rotary_emb(value_states, seq_len=sequence_length) + cos, sin = cos[:, : cos.shape[1] // 2], sin[:, : sin.shape[1] // 2] # flash attn only needs half + query_states, key_states = apply_rotary_pos_emb_rmpad_flash( + query_states, key_states, cos, sin, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen_in_batch + ) + # query_states, key_states = apply_rotary_pos_emb_rmpad(query_states, key_states, cos, sin, + # position_ids, indices, + + # TODO: llama does not have dropout in the config?? + # It is recommended to use dropout with FA according to the docs + # when training. + dropout_rate = 0.0 # if not self.training else self.attn_dropout + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (LlamaRMSNorm handles it correctly) + input_dtype = query_states.dtype + if input_dtype == torch.float32: + query_states = query_states.to(torch.float16) + key_states = key_states.to(torch.float16) + value_states = value_states.to(torch.float16) + + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=max_seqlen_in_batch, + max_seqlen_k=max_seqlen_in_batch, + dropout_p=dropout_rate, + softmax_scale=None, + causal=True, + ) + + attn_output_unpad = attn_output_unpad.to(input_dtype) + attn_output_unpad = attn_output_unpad.reshape(total_nnz, 1, self.hidden_size_per_tp).contiguous() + + # sequence parallel reduce_scatter is performed inside RowColumnParallel if enabled + # Here we need to repad + if self.megatron_config.sequence_parallel: + attn_output_unpad = F.pad(attn_output_unpad, pad=(0, 0, 0, 0, 0, sequence_parallel_pad)) + + attn_output_unpad = self.o_proj(attn_output_unpad)[0] + return attn_output_unpad diff --git a/e2c/verl/verl/models/llama/megatron/layers/parallel_decoder.py b/e2c/verl/verl/models/llama/megatron/layers/parallel_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..f46e9457c793ccc4a9dc72f6d471d58ef48e8bfe --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/layers/parallel_decoder.py @@ -0,0 +1,150 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +from megatron.core import ModelParallelConfig +from torch import nn +from transformers import LlamaConfig + +from verl.utils.megatron_utils import TransformerConfig, convert_config + +from .parallel_attention import ParallelLlamaAttention, ParallelLlamaAttentionRmPad +from .parallel_mlp import ParallelLlamaMLP +from .parallel_rmsnorm import ParallelLlamaRMSNorm + + +class ParallelLlamaDecoderLayer(nn.Module): + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig, layer_idx: int): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.self_attn = ParallelLlamaAttention(config=config, megatron_config=megatron_config) + + self.mlp = ParallelLlamaMLP(config, megatron_config=megatron_config) + self.input_layernorm = ParallelLlamaRMSNorm(config, megatron_config) + self.post_attention_layernorm = ParallelLlamaRMSNorm(config, megatron_config) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Note: sequence parallel is hidden inside ColumnParallelLinear + # reduce scatter is hidden inside RowParallelLinear + + # Self Attention + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + ) + + # TODO: add sequence parallel operator reduce_scatter here + + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + + # TODO: add sequence parallel operator all_gather here + + hidden_states = self.mlp(hidden_states) + + # TODO: add sequence parallel operator reduce_scatter here + + hidden_states = residual + hidden_states + + outputs = hidden_states + + return outputs + + +class ParallelLlamaDecoderLayerRmPad(nn.Module): + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig, layer_idx: int): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.self_attn = ParallelLlamaAttentionRmPad(config=config, megatron_config=megatron_config) + + self.mlp = ParallelLlamaMLP(config, megatron_config=megatron_config) + self.input_layernorm = ParallelLlamaRMSNorm(config, megatron_config) + self.post_attention_layernorm = ParallelLlamaRMSNorm(config, megatron_config) + + def forward( + self, + hidden_states: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + sequence_length: int = None, + indices: torch.Tensor = None, + cu_seqlens: int = None, + max_seqlen_in_batch: int = None, + ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]: + residual = hidden_states # (total_nnz // sp, 1, hidden_size) + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + # (total_nnz // sp, 1, hidden_size) -> all-gather (total_nnz, 1, hidden_size) + # -> col + row -> reduce-scatter -> (total_nnz // sp, 1, hidden_size) + hidden_states = self.self_attn( + hidden_states=hidden_states, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + hidden_states = residual + hidden_states + + # Fully Connected + # shape changes same as attn + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = hidden_states + + return outputs diff --git a/e2c/verl/verl/models/llama/megatron/layers/parallel_linear.py b/e2c/verl/verl/models/llama/megatron/layers/parallel_linear.py new file mode 100644 index 0000000000000000000000000000000000000000..043726c46c3705cf1bfa8ae10ab77d2b930e19d2 --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/layers/parallel_linear.py @@ -0,0 +1,106 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023 The vLLM team. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/linear.py + +import torch +from megatron.core import tensor_parallel + + +class QKVParallelLinear(tensor_parallel.ColumnParallelLinear): + def __init__( + self, + input_size, + num_heads, + num_key_value_heads, + head_dim, + *, + bias=True, + gather_output=True, + skip_bias_add=False, + **kwargs, + ): + # Keep input parameters, and already restrict the head numbers + self.input_size = input_size + self.q_output_size = num_heads * head_dim + self.kv_output_size = num_key_value_heads * head_dim + self.head_dim = head_dim + self.gather_output = gather_output + self.skip_bias_add = skip_bias_add + + input_size = self.input_size + output_size = (num_heads + 2 * num_key_value_heads) * self.head_dim + + super().__init__( + input_size=input_size, + output_size=output_size, + bias=bias, + gather_output=gather_output, + skip_bias_add=skip_bias_add, + **kwargs, + ) + + +class MergedColumnParallelLinear(tensor_parallel.ColumnParallelLinear): + def __init__( + self, + input_size, + gate_ouput_size, + up_output_size, + *, + bias=True, + gather_output=True, + skip_bias_add=False, + **kwargs, + ): + # Keep input parameters, and already restrict the head numbers + self.input_size = input_size + self.output_size = gate_ouput_size + up_output_size + self.gather_output = gather_output + self.skip_bias_add = skip_bias_add + + super().__init__( + input_size=self.input_size, + output_size=self.output_size, + bias=bias, + gather_output=gather_output, + skip_bias_add=skip_bias_add, + **kwargs, + ) + + +class LinearForLastLayer(torch.nn.Linear): + def __init__( + self, + input_size, + output_size, + *, + config, + bias=True, + ): + super().__init__(in_features=input_size, out_features=output_size, bias=bias) + self.sequence_parallel = config.sequence_parallel + if self.sequence_parallel: + self.weight.sequence_parallel = True + + def forward( + self, + input_, + weight=None, + runtime_gather_output=None, + ): + logits = super().forward(input_) + logits = logits.float() + if self.sequence_parallel: + logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) + return logits, None diff --git a/e2c/verl/verl/models/llama/megatron/layers/parallel_mlp.py b/e2c/verl/verl/models/llama/megatron/layers/parallel_mlp.py new file mode 100644 index 0000000000000000000000000000000000000000..583a317eb6aedadeb26d82cef54b815d2b9d22e6 --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/layers/parallel_mlp.py @@ -0,0 +1,74 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from megatron.core import ModelParallelConfig, tensor_parallel +from megatron.core import parallel_state as mpu +from torch import nn +from transformers.activations import ACT2FN + +from verl.models.llama.megatron.layers.parallel_linear import MergedColumnParallelLinear +from verl.utils.megatron import tensor_parallel as tp_utils + + +class ParallelLlamaMLP(nn.Module): + def __init__(self, config, megatron_config: ModelParallelConfig = None) -> None: + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + # The weight is only [hidden_size, intermediate_size // model_parallel_world_size] + + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + row_kwargs = tp_utils.get_default_kwargs_for_row_parallel_linear() + + if megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + assert row_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(row_kwargs, megatron_config) + tp_utils.update_kwargs_with_config(column_kwargs, megatron_config) + + tp_size = mpu.get_tensor_model_parallel_world_size() + + self.gate_up_proj = MergedColumnParallelLinear( + input_size=self.hidden_size, + gate_ouput_size=self.intermediate_size, + up_output_size=self.intermediate_size, + bias=False, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + self.gate_size = self.intermediate_size // tp_size + + self.down_proj = tensor_parallel.RowParallelLinear( + input_size=self.intermediate_size, + output_size=self.hidden_size, + bias=False, + input_is_parallel=True, + skip_bias_add=False, + **row_kwargs, + ) + + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + gate_up = self.gate_up_proj(x)[0] + gate, up = gate_up.split(self.gate_size, dim=-1) + return self.down_proj(self.act_fn(gate) * up)[0] diff --git a/e2c/verl/verl/models/llama/megatron/layers/parallel_rmsnorm.py b/e2c/verl/verl/models/llama/megatron/layers/parallel_rmsnorm.py new file mode 100644 index 0000000000000000000000000000000000000000..bc2e9ae36f05d6f5b1b51b719bf0225cbf95922d --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/layers/parallel_rmsnorm.py @@ -0,0 +1,48 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numbers + +import torch +from apex.normalization.fused_layer_norm import fused_rms_norm_affine +from megatron.core import ModelParallelConfig +from torch import nn +from transformers import LlamaConfig + +from verl.utils.megatron import sequence_parallel as sp_utils + + +class ParallelLlamaRMSNorm(nn.Module): + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): + """ + LlamaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + if isinstance(config.hidden_size, numbers.Integral): + normalized_shape = (config.hidden_size,) + self.normalized_shape = torch.Size(normalized_shape) + self.weight = nn.Parameter(torch.ones(self.normalized_shape)) + self.variance_epsilon = config.rms_norm_eps + + if megatron_config.sequence_parallel: + sp_utils.mark_parameter_as_sequence_parallel(self.weight) + + def forward(self, hidden_states): + return fused_rms_norm_affine( + input=hidden_states, + weight=self.weight, + normalized_shape=self.normalized_shape, + eps=self.variance_epsilon, + memory_efficient=True, + ) diff --git a/e2c/verl/verl/models/llama/megatron/modeling_llama_megatron.py b/e2c/verl/verl/models/llama/megatron/modeling_llama_megatron.py new file mode 100644 index 0000000000000000000000000000000000000000..ed5022e0c3b1a0dbbc91273548f8fd79f74331e4 --- /dev/null +++ b/e2c/verl/verl/models/llama/megatron/modeling_llama_megatron.py @@ -0,0 +1,688 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch LLaMA model with Megatron-style acceleration.""" + +from typing import Optional + +import torch +import torch.utils.checkpoint +from megatron.core import ModelParallelConfig, mpu, tensor_parallel +from torch import nn +from transformers.modeling_outputs import BaseModelOutputWithPast +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import CausalLMOutputWithPast + +from verl.utils.megatron import sequence_parallel as sp_utils +from verl.utils.megatron import tensor_parallel as tp_utils +from verl.utils.megatron_utils import TransformerConfig, convert_config + +from .layers import ParallelLlamaDecoderLayer, ParallelLlamaDecoderLayerRmPad, ParallelLlamaRMSNorm + +""" +TODO: +1. Add weight initialization. Here we need to be careful on TP weight init. +2. Add sequence parallel +3. Load checkpoint from meta LLama pretrained checkpoint +""" + + +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len) + + +# Copied from transformers.models.bart.modeling_bart._expand_mask +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +class ParallelLlamaModel(nn.Module): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] + + Args: + config: LlamaConfig + """ + + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() + if megatron_config is not None: + assert embedding_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) + self.embed_tokens = tensor_parallel.VocabParallelEmbedding( + num_embeddings=config.vocab_size, embedding_dim=config.hidden_size, **embedding_kwargs + ) + + self.layers = nn.ModuleList( + [ParallelLlamaDecoderLayer(config, megatron_config) for _ in range(config.num_hidden_layers)] + ) + self.norm = ParallelLlamaRMSNorm(config, megatron_config) + + # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + ) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | BaseModelOutputWithPast: + """ + + Args: + input_ids: input ids. shape (batch_size, seq_length) + attention_mask: attention_mask. shape (batch_size, seq_length) + position_ids: position ids. shape (batch_size, seq_length) + + Returns: + + """ + batch_size, seq_length = input_ids.shape + inputs_embeds = self.embed_tokens(input_ids) + # embed positions + + attention_mask = self._prepare_decoder_attention_mask(attention_mask, (batch_size, seq_length), inputs_embeds) + + hidden_states = inputs_embeds + + for idx, decoder_layer in enumerate(self.layers): + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + ) + + hidden_states = layer_outputs + + hidden_states = self.norm(hidden_states) + + return hidden_states + + +class ParallelLlamaForCausalLM(nn.Module): + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.model = ParallelLlamaModel(config, megatron_config=megatron_config) + self.vocab_size = config.vocab_size + + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + + self.lm_head = tensor_parallel.ColumnParallelLinear( + input_size=config.hidden_size, + output_size=config.vocab_size, + bias=False, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + ```""" + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + ) + + hidden_states = outputs + logits = self.lm_head(hidden_states)[0] + + logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) + + logits = logits.float() + return CausalLMOutputWithPast( + loss=None, + logits=logits, + past_key_values=None, + hidden_states=None, + attentions=None, + ) + + +from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + +class ParallelLlamaModelRmPad(nn.Module): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] + + Args: + config: LlamaConfig + """ + + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() + self.megatron_config = megatron_config + if megatron_config is not None: + assert embedding_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) + self.embed_tokens = tensor_parallel.VocabParallelEmbedding( + num_embeddings=config.vocab_size, embedding_dim=config.hidden_size, **embedding_kwargs + ) + + self.layers = nn.ModuleList( + [ParallelLlamaDecoderLayerRmPad(config, megatron_config) for _ in range(config.num_hidden_layers)] + ) + self.norm = ParallelLlamaRMSNorm(config, megatron_config) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + sequence_length: int = None, + indices: torch.Tensor = None, + cu_seqlens: int = None, + max_seqlen_in_batch: int = None, + ) -> tuple | BaseModelOutputWithPast: + """ + + Args: + input_ids: input ids. shape (1, totol_nnz) + position_ids: position ids. shape (batch_size, seq_length) + + Returns: + + """ + inputs_embeds = self.embed_tokens(input_ids) # (1, total_nnz) -> (1, total_nnz, hidden_size) + + # (1, total_nnz, hidden_size) -> (total_nnz, 1, hidden_size) -> (total_nnz // sp, 1, hidden_size) + inputs_embeds = inputs_embeds.transpose(0, 1) + if self.megatron_config.sequence_parallel: + inputs_embeds = tensor_parallel.scatter_to_sequence_parallel_region(inputs_embeds) + + hidden_states = inputs_embeds + for idx, decoder_layer in enumerate(self.layers): + layer_outputs = decoder_layer( + hidden_states, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + hidden_states = layer_outputs + + hidden_states = self.norm(hidden_states) + + return hidden_states + + +class ParallelLlamaForCausalLMRmPad(nn.Module): + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.megatron_config = megatron_config + self.model = ParallelLlamaModelRmPad(config, megatron_config=megatron_config) + self.vocab_size = config.vocab_size + self._init_head(config) + + def _init_head(self, config): + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if self.megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + self.lm_head = tensor_parallel.ColumnParallelLinear( + input_size=config.hidden_size, + output_size=config.vocab_size, + bias=False, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + + def _forward_head(self, hidden_states): + # all_gather from sequence parallel region is performed inside lm_head + logits = self.lm_head(hidden_states)[0] + logits = logits.float() # (total_nnz_padded, 1, vocab_size // tp) + logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) # (total_nnz_padded, 1, vocab_size) + return logits + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + ```""" + batch_size, sequence_length = input_ids.shape + + # remove padding here + input_ids, indices, cu_seqlens, max_seqlen_in_batch, *_ = unpad_input( + input_ids.unsqueeze(dim=-1), attention_mask + ) # (total_nnz, 1) + + # pad input_ids to multiple of tp for all tp ranks + # TODO: for better performance, the sp padding should be removed at each layer. Not sure the performance gap + if self.megatron_config.sequence_parallel: + input_ids = sp_utils.pad_to_sequence_parallel(input_ids) + + input_ids = input_ids.transpose(0, 1) # (1, total_nnz+pad) + + outputs = self.model( + input_ids=input_ids, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + hidden_states = outputs + + logits = self._forward_head(hidden_states) + + # remove padding from sequence parallel + if self.megatron_config.sequence_parallel: + totol_nnz = cu_seqlens[-1] + logits = logits[:totol_nnz] # (total_nnz_padded) + + logits = torch.squeeze(logits, dim=1) # remove the artificial batch dimension + # add removed padding back + logits = pad_input( + logits, indices, batch_size, seqlen=sequence_length + ) # (batch_size, sequence_length, vocab_size) + + return CausalLMOutputWithPast( + loss=None, + logits=logits, + past_key_values=None, + hidden_states=None, + attentions=None, + ) + + +class ParallelLlamaForValueRmPad(ParallelLlamaForCausalLMRmPad): + def _init_head(self, config): + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if self.megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + self.lm_head = nn.Linear(in_features=config.hidden_size, out_features=1, bias=False) + # lm_head is effectively the same as sequence parallel + sp_utils.mark_parameter_as_sequence_parallel(self.lm_head.weight) + + def _forward_head(self, hidden_states): + logits = self.lm_head(hidden_states) # (total_nnz_padded // tp, 1, 1) + logits = logits.float() + if self.megatron_config.sequence_parallel: + logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) + return logits + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + output = super().forward(input_ids, attention_mask, position_ids) + output.logits = torch.squeeze(output.logits, dim=-1) + return output + + +""" +Support pipeline parallelism +""" + + +class ParallelLlamaModelRmPadPP(nn.Module): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] + This model definition supports pipeline parallelism. To support pp and vpp, + - This model only contains layer in this pp stage and vpp chunk + - When calling get_model in Megatron, this rank will instantiate all the vpp chunks in this pp. + Args: + config: LlamaConfig + """ + + def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig, pre_process, post_process): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.pre_process = pre_process + self.post_process = post_process + self.megatron_config = megatron_config + embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() + if megatron_config is not None: + assert embedding_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) + if pre_process: + self.embed_tokens = tensor_parallel.VocabParallelEmbedding( + num_embeddings=config.vocab_size, embedding_dim=config.hidden_size, **embedding_kwargs + ) + else: + self.embed_tokens = None + + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = megatron_config.pipeline_model_parallel_size + self.num_layer_per_pp = config.num_hidden_layers // pp_size + vpp_size = megatron_config.virtual_pipeline_model_parallel_size + vpp_rank = mpu.get_virtual_pipeline_model_parallel_rank() + + if vpp_size is not None: + self.layers = nn.ModuleList() + self.num_layer_vpp_chunk = self.num_layer_per_pp // vpp_size + self.num_layer_this_model = self.num_layer_vpp_chunk + offset = vpp_rank * (config.num_hidden_layers // vpp_size) + (pp_rank * self.num_layer_vpp_chunk) + else: + self.num_layer_this_model = self.num_layer_per_pp + offset = pp_rank * self.num_layer_per_pp + + self.layers = nn.ModuleList() + for i in range(self.num_layer_this_model): + layer = ParallelLlamaDecoderLayerRmPad(config, megatron_config, layer_idx=offset + i) + self.layers.add_module(f"{i}", layer) + + if post_process: + self.norm = ParallelLlamaRMSNorm(config, megatron_config) + else: + self.norm = None + + def set_input_tensor(self, input_tensor): + """Set input tensor to be used instead of forward()'s input. + + When doing pipeline parallelism the input from the previous + stage comes from communication, not from the input, so the + model's forward_step_func won't have it. This function is thus + used by internal code to bypass the input provided by the + forward_step_func""" + self.input_tensor = input_tensor + + def forward( + self, + input_ids: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + sequence_length: int = None, + indices: torch.Tensor = None, + cu_seqlens: int = None, + max_seqlen_in_batch: int = None, + ) -> tuple | BaseModelOutputWithPast: + """ + + Args: + input_ids: input ids. shape (1, totol_nnz) + position_ids: position ids. shape (batch_size, seq_length) + + Returns: + + """ + if self.pre_process: + inputs_embeds = self.embed_tokens(input_ids) # (1, total_nnz) -> (1, total_nnz, hidden_size) + + # vocab parallel embedding will not do sequence parallel reduce-scatter in open source megatron + # so need to deal with it by handle here: + # (1, total_nnz, hidden_size) -> (total_nnz, 1, hidden_size) -> (total_nnz // sp, 1, hidden_size) + inputs_embeds = inputs_embeds.transpose(0, 1) + if self.megatron_config.sequence_parallel: + inputs_embeds = tensor_parallel.scatter_to_sequence_parallel_region(inputs_embeds) + + hidden_states = inputs_embeds + else: + # self.hidden_states should be passed by Megatron + hidden_states = self.input_tensor + + for idx, decoder_layer in enumerate(self.layers): + layer_outputs = decoder_layer( + hidden_states, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + hidden_states = layer_outputs + + if self.post_process: + hidden_states = self.norm(hidden_states) + + return hidden_states + + +class ParallelLlamaForCausalLMRmPadPP(nn.Module): + def __init__( + self, + config: LlamaConfig, + megatron_config: ModelParallelConfig, + pre_process, + post_process, + share_embeddings_and_output_weights=False, + ): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.megatron_config = megatron_config + self.model = ParallelLlamaModelRmPadPP( + config, megatron_config=megatron_config, pre_process=pre_process, post_process=post_process + ) + assert share_embeddings_and_output_weights is False, ( + "Llama Model not supports sharing embedding and output weights" + ) + self.share_embeddings_and_output_weights = share_embeddings_and_output_weights + self.vocab_size = config.vocab_size + self.pre_process = pre_process + self.post_process = post_process + if post_process: + self._init_head(config) + + def set_input_tensor(self, input_tensor): + """Set input tensor to be used instead of forward()'s input. + + When doing pipeline parallelism the input from the previous + stage comes from communication, not from the input, so the + model's forward_step_func won't have it. This function is thus + used by internal code to bypass the input provided by the + forward_step_func""" + assert len(input_tensor) == 1 + self.model.set_input_tensor(input_tensor[0]) + + def _init_head(self, config): + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if self.megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + self.lm_head = tensor_parallel.ColumnParallelLinear( + input_size=config.hidden_size, + output_size=config.vocab_size, + bias=False, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + + def _forward_head(self, hidden_states): + # all_gather from sequence parallel region is performed inside lm_head + # logits shape before forward_head hidden_states.shape: [4, 32, 4096] + logits = self.lm_head(hidden_states)[0] + # logits shape after forward_head logits.shape: [8, 32, 8] + logits = logits.float() # (total_nnz_padded, 1, vocab_size // tp) + return logits + + def forward( + self, + # original input + *, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + ```""" + + # Note that input_ids, attention_mask and position_ids should be passed to every pp layer. + # In the first pp, input_ids will be used, in other pp layers hidden_states will be used inside self.model + batch_size, sequence_length = input_ids.shape + # remove padding here + input_ids_rmpad, indices, cu_seqlens, max_seqlen_in_batch, *_ = unpad_input( + input_ids.unsqueeze(dim=-1), attention_mask + ) # (total_nnz, 1) + + # pad input_ids to multiple of tp for all tp ranks + # TODO: for better performance, the sp padding should be removed at each layer. Not sure the performance gap + if self.megatron_config.sequence_parallel: + input_ids_rmpad = sp_utils.pad_to_sequence_parallel(input_ids_rmpad) + + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz+pad) + + outputs = self.model( + input_ids=input_ids_rmpad, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + if self.post_process: + hidden_states = outputs + # print(f'hidden_states.shape = {hidden_states.shape}') # torch.Size([4, 32, 4096]) + logits = self._forward_head(hidden_states) + logits = torch.squeeze(logits, dim=1) # remove the artificial batch dimension # torch.Size([8, 32, 16]) + + # remove padding from sequence parallel + if self.megatron_config.sequence_parallel: + totol_nnz = cu_seqlens[-1] + logits = logits[:totol_nnz] # (total_nnz_padded) + # add removed padding back. If input is already rmpad, we let the caller pad_input + logits = pad_input( + logits, indices, batch_size, seqlen=sequence_length + ) # (batch_size, sequence_length, vocab_size) + + return CausalLMOutputWithPast( + loss=None, + logits=logits, + past_key_values=None, + hidden_states=None, + attentions=None, + ) + else: + return outputs + + +class ParallelLlamaForValueRmPadPP(ParallelLlamaForCausalLMRmPadPP): + def _init_head(self, config): + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if self.megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + self.lm_head = nn.Linear(in_features=config.hidden_size, out_features=1, bias=False) + # lm_head is effectively the same as sequence parallel + sp_utils.mark_parameter_as_sequence_parallel(self.lm_head.weight) + + def _forward_head(self, hidden_states): + logits = self.lm_head(hidden_states) # (total_nnz_padded // tp, 1, 1) + logits = logits.float() + if self.megatron_config.sequence_parallel: + logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) + return logits + + def forward( + self, + *, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + output = super().forward(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids) + if self.post_process: + output.logits = torch.squeeze(output.logits, dim=-1) + return output + else: + return output diff --git a/e2c/verl/verl/models/mcore/__init__.py b/e2c/verl/verl/models/mcore/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..29d0531775a8b81dd61d34e0e40b0f495fa26006 --- /dev/null +++ b/e2c/verl/verl/models/mcore/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .registry import ( + get_mcore_forward_fn, + get_mcore_forward_fused_fn, + get_mcore_weight_converter, + hf_to_mcore_config, + init_mcore_model, +) + +__all__ = [ + "hf_to_mcore_config", + "init_mcore_model", + "get_mcore_forward_fn", + "get_mcore_weight_converter", + "get_mcore_forward_fused_fn", +] diff --git a/e2c/verl/verl/models/mcore/config_converter.py b/e2c/verl/verl/models/mcore/config_converter.py new file mode 100644 index 0000000000000000000000000000000000000000..083e6d9b6835133b9461a20a76fd326838dee922 --- /dev/null +++ b/e2c/verl/verl/models/mcore/config_converter.py @@ -0,0 +1,396 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# convert huggingface config to mcore transformer config + + +import warnings +from typing import TypeVar + +import torch +import torch.nn.functional as F +from megatron.core import parallel_state as mpu +from megatron.core.transformer import MLATransformerConfig, TransformerConfig +from transformers import PretrainedConfig + +T = TypeVar("T", bound=TransformerConfig) + + +def _get_base_transformer_config( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> dict: + """ + Create a base TransformerConfig with common parameters across different model architectures. + TODO: (ycl) use dataclass or converter config? + + Args: + hf_config: HuggingFace model configuration + dtype: Data type for the model + override_transformer_config_kwargs: Additional parameters to override defaults + + Returns: + TransformerConfig with common parameters + """ + + # Common parallel state parameters + overlap_p2p_comm = ( + mpu.get_virtual_pipeline_model_parallel_world_size() is not None + and mpu.get_virtual_pipeline_model_parallel_world_size() > 1 + ) + batch_p2p_comm = False + + # Base configuration with common parameters + base_config = { + # Model architecture parameters + "num_layers": hf_config.num_hidden_layers, + "hidden_size": hf_config.hidden_size, + "num_attention_heads": hf_config.num_attention_heads, + "num_query_groups": hf_config.num_key_value_heads, + "ffn_hidden_size": hf_config.intermediate_size, + "attention_dropout": hf_config.attention_dropout, + "hidden_dropout": getattr(hf_config, "hidden_dropout", 0.0), + "kv_channels": getattr(hf_config, "head_dim", None), + "layernorm_epsilon": hf_config.rms_norm_eps, + "add_bias_linear": True, + # Activation and normalization + "activation_func": F.silu, + "normalization": "RMSNorm", + "gated_linear_unit": True, + # Data types + "pipeline_dtype": dtype, + "params_dtype": dtype, + "bf16": dtype is torch.bfloat16, + # Parallel configuration + "tensor_model_parallel_size": mpu.get_tensor_model_parallel_world_size(), + "pipeline_model_parallel_size": mpu.get_pipeline_model_parallel_world_size(), + "expert_model_parallel_size": mpu.get_expert_model_parallel_world_size(), + "expert_tensor_parallel_size": mpu.get_expert_tensor_parallel_world_size(), + "virtual_pipeline_model_parallel_size": mpu.get_virtual_pipeline_model_parallel_world_size(), + "context_parallel_size": mpu.get_context_parallel_world_size(), + "overlap_p2p_comm": overlap_p2p_comm, + "batch_p2p_comm": batch_p2p_comm, + "sequence_parallel": mpu.get_tensor_model_parallel_world_size() > 1, + # Common settings + "variable_seq_lengths": True, + "masked_softmax_fusion": True, + "moe_token_dispatcher_type": "alltoall", + } + + # Update with any provided overrides + # override_transformer_config_kwargs as kwargs shall never be none + base_config.update(override_transformer_config_kwargs) + + return base_config + + +def _get_mla_transformer_config( + hf_config: PretrainedConfig, mla_rope_config: dict, dtype: torch.dtype, **override_transformer_config_kwargs +) -> dict: + """ + Create a MLATransformerConfig with common parameters across different model architectures. + This is specifically for MLA models like DeepseekV3. + + Args: + hf_config: HuggingFace model configuration + mla_rope_config: MLA specific RoPE configuration + dtype: Data type for the model + override_transformer_config_kwargs: Additional parameters to override defaults + + Returns: + MLATransformerConfig with common parameters + """ + base_config = _get_base_transformer_config(hf_config=hf_config, dtype=dtype, **override_transformer_config_kwargs) + mla_config = { + # MLA specific parameters + "q_lora_rank": hf_config.q_lora_rank, + "kv_lora_rank": hf_config.kv_lora_rank, + "qk_head_dim": hf_config.qk_nope_head_dim, + "qk_pos_emb_head_dim": hf_config.qk_rope_head_dim, + "v_head_dim": hf_config.v_head_dim, + "rotary_base": hf_config.rope_theta, + "rotary_scaling_factor": mla_rope_config["factor"], + "rope_type": mla_rope_config["type"], + "max_position_embeddings": mla_rope_config["original_max_position_embeddings"], + "beta_fast": mla_rope_config["beta_fast"], + "beta_slow": mla_rope_config["beta_slow"], + "mscale": mla_rope_config["mscale"], + "mscale_all_dim": mla_rope_config["mscale_all_dim"], + } + + base_config.update(mla_config) + return base_config + + +def check_and_construct_configs(original_config: dict, cls: type[T]) -> T: + """ + Check and disable incompatible configurations for older Megatron version. + + Args: + original_config (dict): The original model configuration. + + Returns: + dict: The updated model configuration with incompatible settings disabled. + """ + removed_keys = [] + for key in original_config.keys(): + if not hasattr(cls, key): + removed_keys.append(key) + if removed_keys: + warnings.warn( + f"The following keys are not supported in the current Megatron version and will be removed: {removed_keys}", + stacklevel=2, + ) + for key in removed_keys: + original_config.pop(key) + + original_config = mapping_string_to_attn_backend(original_config) + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + print(f"Overridden {cls.__name__} init config: {original_config}") + return cls(**original_config) + + +def hf_to_mcore_config_dense( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> TransformerConfig: + # for LlamaForCausalLM or Qwen2ForCausalLM + qkv_bias = True if "Qwen2" in hf_config.architectures[0] else getattr(hf_config, "attention_bias", False) + qk_layernorm = True if "Qwen3" in hf_config.architectures[0] else False + + args: dict = _get_base_transformer_config( + hf_config=hf_config, + dtype=dtype, + use_cpu_initialization=False, + add_bias_linear=False, + add_qkv_bias=qkv_bias, + qk_layernorm=qk_layernorm, + ) + # override_transformer_config_kwargs as kwargs shall never be none + args.update(override_transformer_config_kwargs) + return check_and_construct_configs(args, TransformerConfig) + + +def hf_to_mcore_config_qwen2moe( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> TransformerConfig: + args: dict = _get_base_transformer_config( + hf_config=hf_config, + dtype=dtype, + use_cpu_initialization=False, + add_bias_linear=False, + layernorm_epsilon=hf_config.rms_norm_eps, + # MoE specific + moe_ffn_hidden_size=hf_config.moe_intermediate_size, + moe_router_bias_update_rate=0.001, + moe_router_topk=hf_config.num_experts_per_tok, + num_moe_experts=hf_config.num_experts, + moe_shared_expert_intermediate_size=hf_config.shared_expert_intermediate_size, + moe_aux_loss_coeff=hf_config.router_aux_loss_coef, + # moe_aux_loss_coeff=0.0, + moe_router_load_balancing_type="none", # turn off aux_loss as it hurts perf in RL + moe_shared_expert_overlap=True, + moe_grouped_gemm=True, + moe_router_score_function="softmax", + # Other optimizations + persist_layer_norm=True, + bias_activation_fusion=True, + bias_dropout_fusion=True, + # Qwen specific + moe_router_pre_softmax=True, + add_qkv_bias=True, + ) + # override_transformer_config_kwargs as kwargs shall never be none + args.update(override_transformer_config_kwargs) + return check_and_construct_configs(args, TransformerConfig) + + +def hf_to_mcore_config_mixtral( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> TransformerConfig: + args: dict = _get_base_transformer_config( + hf_config=hf_config, + dtype=dtype, + use_cpu_initialization=False, + add_bias_linear=False, + layernorm_epsilon=hf_config.rms_norm_eps, + # MoE specific + num_moe_experts=hf_config.num_local_experts, + moe_aux_loss_coeff=hf_config.router_aux_loss_coef, + moe_router_topk=hf_config.num_experts_per_tok, + moe_router_pre_softmax=True, + moe_router_load_balancing_type="none", # turn off aux_loss as it hurts perf in RL + moe_router_score_function="softmax", + moe_shared_expert_intermediate_size=None, # mixtral has no shared expert + moe_shared_expert_overlap=False, # mixtral has no shared expert + moe_ffn_hidden_size=hf_config.intermediate_size, + moe_router_bias_update_rate=0.001, + # moe_permute_fusion=True, # need TE 2.1+ + moe_grouped_gemm=True, + # Other optimizations + persist_layer_norm=True, + apply_rope_fusion=True, + bias_activation_fusion=True, + bias_dropout_fusion=True, + ) + # override_transformer_config_kwargs as kwargs shall never be none + args.update(override_transformer_config_kwargs) + return check_and_construct_configs(args, TransformerConfig) + + +def hf_to_mcore_config_qwen3moe( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> TransformerConfig: + args: dict = _get_base_transformer_config( + hf_config=hf_config, + dtype=dtype, + use_cpu_initialization=False, + add_bias_linear=False, + layernorm_epsilon=hf_config.rms_norm_eps, + # MoE specific + moe_ffn_hidden_size=hf_config.moe_intermediate_size, + moe_router_bias_update_rate=0.001, + moe_router_topk=hf_config.num_experts_per_tok, + num_moe_experts=hf_config.num_experts, + moe_aux_loss_coeff=hf_config.router_aux_loss_coef, + # moe_aux_loss_coeff=0.0, + moe_router_load_balancing_type="none", # turn off aux_loss as it hurts perf in RL + moe_grouped_gemm=True, + moe_router_score_function="softmax", + # Other optimizations + persist_layer_norm=True, + bias_activation_fusion=True, + bias_dropout_fusion=True, + # Qwen specific + moe_router_pre_softmax=False, + qk_layernorm=True, + ) + # override_transformer_config_kwargs as kwargs shall never be none + args.update(override_transformer_config_kwargs) + return check_and_construct_configs(args, TransformerConfig) + + +def hf_to_mcore_config_dpskv3( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> MLATransformerConfig: + # DeepseekV3ForCausalLM + from megatron.core.transformer.enums import AttnBackend + + from .patch_v012 import apply_patch + + apply_patch() + + mla_rope_config = { + "beta_fast": 32, + "beta_slow": 1, + "factor": 1, + "mscale": 1.0, + "mscale_all_dim": 1.0, + "original_max_position_embeddings": 4096, + "type": "rope", + } + if "rope_scaling" in hf_config and hf_config.rope_scaling is not None: + mla_rope_config.update(hf_config.rope_scaling) + moe_layer_freq = [1] * hf_config.num_hidden_layers + for i in range(min(hf_config.first_k_dense_replace, hf_config.num_hidden_layers)): + moe_layer_freq[i] = 0 + + # disable MTP and quantization for now + if "num_nextn_predict_layers" in hf_config: + assert hf_config.num_nextn_predict_layers == 0, ( + "MTP is not supported for now, please modify the config.json to set num_nextn_predict_layers to 0" + ) + assert "quantization_config" not in hf_config or not hf_config.quantization_config, ( + "quantization is not supported for now, please modify the config.json to remove quantization_config" + ) + + args: dict = _get_mla_transformer_config( + hf_config=hf_config, + mla_rope_config=mla_rope_config, + dtype=dtype, + # Additional parameters + use_cpu_initialization=False, + add_bias_linear=False, + attention_backend=AttnBackend.fused, + qk_layernorm=True, + # Standard MoE parameters + moe_ffn_hidden_size=hf_config.moe_intermediate_size, + moe_token_dispatcher_type="alltoall", + moe_router_bias_update_rate=0.001, + moe_router_enable_expert_bias=True, + moe_router_topk=hf_config.num_experts_per_tok, + num_moe_experts=hf_config.n_routed_experts, + moe_shared_expert_intermediate_size=hf_config.moe_intermediate_size * hf_config.n_shared_experts, + moe_aux_loss_coeff=getattr(hf_config, "aux_loss_alpha", 0.001), + moe_router_load_balancing_type="seq_aux_loss", + moe_shared_expert_overlap=True, + # moe_permute_fusion=True, # need TE 2.1+ + moe_grouped_gemm=True, + moe_router_score_function="sigmoid", + moe_router_pre_softmax=True, + moe_router_topk_scaling_factor=hf_config.routed_scaling_factor, + moe_layer_freq=moe_layer_freq, + # mcore 0.12 moe + moe_router_dtype="fp64", + disable_bf16_reduced_precision_matmul=True, + # Other optimizations + # deallocate_pipeline_outputs=True, + # gradient_accumulation_fusion=True, + persist_layer_norm=True, + bias_activation_fusion=True, + bias_dropout_fusion=True, + ) + # override_transformer_config_kwargs as kwargs shall never be none + args.update(override_transformer_config_kwargs) + transformer_config = check_and_construct_configs(args, MLATransformerConfig) + # MTP + if "num_nextn_predict_layers" in hf_config: + transformer_config.mtp_num_layers = hf_config.num_nextn_predict_layers + transformer_config.mtp_loss_scaling_factor = 0.1 + + return transformer_config + + +def hf_to_mcore_config_qwen2_5_vl( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> TransformerConfig: + # Qwen2_5_VLForConditionalGeneration + + args = _get_base_transformer_config( + hf_config=hf_config, + dtype=dtype, + add_bias_linear=False, + # qwen specific + add_qkv_bias=True, + mrope_section=hf_config.rope_scaling["mrope_section"], + ) + # override_transformer_config_kwargs as kwargs shall never be none + args.update(override_transformer_config_kwargs) + args = mapping_string_to_attn_backend(args) + return TransformerConfig(**args) + + +def hf_to_mcore_config_llama4( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> TransformerConfig: + # Llama4ForConditionalGeneration + raise NotImplementedError("Llama4ForConditionalGeneration is not supported yet") + + +def mapping_string_to_attn_backend(args: dict) -> dict: + if "attention_backend" in args and isinstance(args["attention_backend"], str): + from megatron.core.transformer.enums import AttnBackend + + args["attention_backend"] = AttnBackend[args["attention_backend"]] + return args diff --git a/e2c/verl/verl/models/mcore/loader.py b/e2c/verl/verl/models/mcore/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..577ffc5ecf4f138ab4183d9ee4bef445d6f8142c --- /dev/null +++ b/e2c/verl/verl/models/mcore/loader.py @@ -0,0 +1,495 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import torch +import torch.distributed as dist + +from verl.utils.device import get_device_id, get_torch_device + +from .saver import _megatron_calc_global_rank + + +def _megatron_calc_layer_map(config): + """Calculate the mapping of global layer_idx to local layer_idx + Returns: + layer_map (Dict: int -> tuple(int, int, int)): + mapping from the global layer index to + a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) + """ + from megatron.core import mpu + + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + + layer_map = dict() + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + for pp_rank_idx in range(pp_size): + for virtual_pp_rank_idx in range(virtual_pp_size): + layer_offset = ( + virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + pp_rank_idx * num_layers_per_model + ) + for layer_idx in range(num_layers_per_model): + layer_map[layer_offset + layer_idx] = ( + pp_rank_idx, + virtual_pp_rank_idx, + layer_idx, + ) + return layer_map + + +def load_state_dict_to_megatron_gptmodel(state_dict, wrapped_models, config, params_dtype, is_value_model=False): + """Load merged state_dict to sharded Megatron module in training.""" + from megatron.core import DistributedDataParallel as LocalDDP + from megatron.core import mpu + from megatron.core.transformer.module import Float16Module + from torch.nn.parallel import DistributedDataParallel as torchDDP + + from verl.utils.logger import print_rank_0 + from verl.utils.megatron_utils import unwrap_model + + start_time = time.time() + + def _get_gpt_model(model): + return model + + def broadcast_params(module): + for param in module.parameters(): + torch.distributed.broadcast( + param.data, src=mpu.get_data_parallel_src_rank(), group=mpu.get_data_parallel_group() + ) + + dp_rank = mpu.get_data_parallel_rank() + pp_rank = mpu.get_pipeline_model_parallel_rank() + cp_rank = mpu.get_context_parallel_rank() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=0, cp_rank=cp_rank) + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + mp_group = mpu.get_model_parallel_group() + + if torch.distributed.get_rank() == src_rank: + assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" + assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" + assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" + + if not isinstance(wrapped_models, list | tuple): + wrapped_models = list(wrapped_models) + + assert len(wrapped_models) == virtual_pp_size + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + models = [None] * len(wrapped_models) + + for i, wrapped_model in enumerate(wrapped_models): + models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) + gpt_model_module = _get_gpt_model(models[i]) + assert len(gpt_model_module.decoder.layers) == num_layers_per_model + + def _broadcast_tensor(tensor, name) -> torch.Tensor: + """broadcast tensor from rank0 across mp_group""" + nonlocal state_dict + nonlocal mp_group + if torch.distributed.get_rank() == src_rank: + if name in state_dict: + weight = state_dict[name] + tensor_shape = weight.shape + else: + tensor_shape = None + else: + weight = None + tensor_shape = None + + obj_list = [tensor_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + tensor_shape = obj_list[0] + + if tensor_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tensor:[{name}] not in state_dict, skip load") + return + + if tensor is None: + tensor = torch.empty( + tensor_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + if torch.distributed.get_rank() == src_rank: + tensor.data.copy_(weight) + dist.broadcast(tensor, src=src_rank, group=mp_group) + + def _broadcast_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == src_rank: + if name in state_dict: + full_weight = state_dict[name] + + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == src_rank: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=src_rank, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == src_rank: + if name in state_dict: + full_weight = state_dict[name] + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == src_rank: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=src_rank, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == src_rank: + gate_weight = state_dict[gate_name] + up_weight = state_dict[up_name] + new_gate_up_weight = torch.empty( + config.intermediate_size * 2, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + intermediate_size_tp = config.intermediate_size // tp_size + gate_weight_tp = gate_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + up_weight_tp = up_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + new_gate_up_weight[intermediate_size_tp * 2 * i : intermediate_size_tp * 2 * (i + 1)].copy_( + torch.cat([gate_weight_tp, up_weight_tp], dim=0) + ) + + tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank() == src_rank:} tensor {gate_name, up_name} shape " + f"{tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == src_rank: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=src_rank, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, bias=False) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == src_rank: + assert q_name in state_dict and k_name in state_dict and v_name in state_dict + full_weight_q = state_dict[q_name] + full_weight_k = state_dict[k_name] + full_weight_v = state_dict[v_name] + + hidden_size_per_head = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + + if config.num_key_value_heads >= tp_size: + q_size_tp = hidden_size_per_head * config.num_attention_heads // tp_size + kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size + total_size = q_size_tp + 2 * kv_size_tp + sizes = [total_size * tp_size] + if not bias: + sizes.append(config.hidden_size) + new_weight_qkv = torch.empty(*sizes, dtype=params_dtype, device=get_device_id()) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + k_part = full_weight_k[i * kv_size_tp : (i + 1) * kv_size_tp] + v_part = full_weight_v[i * kv_size_tp : (i + 1) * kv_size_tp] + num_query_groups_per_partition = models[0].config.num_query_groups // tp_size + new_weight_qkv_this_tp = new_weight_qkv[i * total_size : (i + 1) * total_size] + q_part_per_head = torch.chunk(q_part, num_query_groups_per_partition, dim=0) + k_part_per_head = torch.chunk(k_part, num_query_groups_per_partition, dim=0) + v_part_per_head = torch.chunk(v_part, num_query_groups_per_partition, dim=0) + total_size_per_head = total_size // num_query_groups_per_partition + for j in range(num_query_groups_per_partition): + new_weight_qkv_this_tp[j * total_size_per_head : (j + 1) * total_size_per_head].copy_( + torch.cat([q_part_per_head[j], k_part_per_head[j], v_part_per_head[j]], dim=0) + ) + + else: + q_size_tp = hidden_size_per_head * config.num_attention_heads // tp_size + kv_size_tp = hidden_size_per_head + total_size = q_size_tp + 2 * kv_size_tp + sizes = [total_size * tp_size] + if not bias: + sizes.append(config.hidden_size) + new_weight_qkv = torch.empty(*sizes, dtype=params_dtype, device=get_device_id()) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head + end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head + k_part = full_weight_k[start_idx:end_idx] + v_part = full_weight_v[start_idx:end_idx] + new_weight_qkv_this_tp = new_weight_qkv[i * total_size : (i + 1) * total_size] + q_part_per_head = torch.chunk(q_part, config.num_attention_heads, dim=0) + k_part_per_head = torch.chunk(k_part, config.num_attention_heads, dim=0) + v_part_per_head = torch.chunk(v_part, config.num_attention_heads, dim=0) + total_size_per_head = total_size // config.num_attention_heads + for j in range(config.num_attention_heads): + new_weight_qkv_this_tp[j * total_size_per_head : (j + 1) * total_size_per_head].copy_( + torch.cat([q_part_per_head[j], k_part_per_head[j], v_part_per_head[j]], dim=0) + ) + + tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{q_name, k_name, v_name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {q_name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == src_rank: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=src_rank, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + if dp_rank == 0: + # Embeddings + # ------------------- + print_rank_0("loading embeddings...") + gpt_model_module = _get_gpt_model(models[0]) + embed_tokens_weight = None + if pp_rank == 0: + embed_tokens_weight = gpt_model_module.embedding.word_embeddings.weight + _broadcast_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") + + # Transformer layers + # ------------------- + layer_map = _megatron_calc_layer_map(config) + + for layer in range(config.num_hidden_layers): + layer_name = f"model.layers.{layer}" + print_rank_0(f"loading layer #{layer}, with layer_name model.layers.{layer}...") + dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] + + gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) + sync_layer = gpt_model_module.decoder.layers[dst_layer_idx] + + _broadcast_tensor( + sync_layer.self_attention.linear_qkv.layer_norm_weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.input_layernorm.weight", + ) + + if f"{layer_name}.self_attn.q_norm.weight" in state_dict: + _broadcast_tensor( + sync_layer.self_attention.q_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_norm.weight", + ) + _broadcast_tensor( + sync_layer.self_attention.k_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.k_norm.weight", + ) + + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attention.linear_qkv.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_proj.weight", + f"{layer_name}.self_attn.k_proj.weight", + f"{layer_name}.self_attn.v_proj.weight", + ) + if f"{layer_name}.self_attn.q_proj.bias" in state_dict: + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attention.linear_qkv.bias if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_proj.bias", + f"{layer_name}.self_attn.k_proj.bias", + f"{layer_name}.self_attn.v_proj.bias", + bias=True, + ) + + _broadcast_tp_shard_tensor( + sync_layer.self_attention.linear_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.o_proj.weight", + chunk_dim=1, + ) + _broadcast_tensor( + sync_layer.mlp.linear_fc1.layer_norm_weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.post_attention_layernorm.weight", + ) + + _broadcast_tp_shard_tensor_gate_up( + sync_layer.mlp.linear_fc1.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.gate_proj.weight", + f"{layer_name}.mlp.up_proj.weight", + ) + + _broadcast_tp_shard_tensor( + sync_layer.mlp.linear_fc2.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.down_proj.weight", + chunk_dim=1, + ) + # Final Layernorm + # ------------------- + print_rank_0("loading final layernorm...") + gpt_model_module = _get_gpt_model(models[-1]) + _broadcast_tensor( + getattr(gpt_model_module.decoder.final_layernorm, "weight", None), + "model.norm.weight", + ) + + print_rank_0("loading lm_head...") + lm_head_weight = None + if pp_rank + 1 == pp_size: + lm_head_weight = gpt_model_module.output_layer.weight + + if is_value_model: + # if torch.distributed.get_rank() == src_rank: + if "lm_head.weight" in state_dict and state_dict["lm_head.weight"].shape[0] == 1: + _broadcast_tensor(lm_head_weight, "lm_head.weight") + elif "reward_head.weight" in state_dict and state_dict["reward_head.weight"].shape[0] == 1: + _broadcast_tensor(lm_head_weight, "reward_head.weight") + print_rank_0("load lm_head from value_head weight") + elif "score.weight" in state_dict and state_dict["score.weight"].shape[0] == 1: + _broadcast_tensor(lm_head_weight, "score.weight") + print_rank_0("load lm_head from score weight") + else: + _broadcast_tensor(None, "lm_head.weight") + print_rank_0("fail to match lm_head in value_model") + # else: + + # _broadcast_tensor(lm_head_weight, "lm_head.weight") + + else: + _broadcast_tp_shard_tensor(lm_head_weight, "lm_head.weight") + dist.barrier() + # Broadcast weights inside data parallel groups + for wrapped_model in wrapped_models: + broadcast_params(wrapped_model) + pass + get_torch_device().empty_cache() + print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") diff --git a/e2c/verl/verl/models/mcore/mbridge.py b/e2c/verl/verl/models/mcore/mbridge.py new file mode 100644 index 0000000000000000000000000000000000000000..35c32d6972c5d8ad3fba7eb67749f7d9b3ddd0e6 --- /dev/null +++ b/e2c/verl/verl/models/mcore/mbridge.py @@ -0,0 +1,23 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from mbridge import AutoBridge + from mbridge.utils.post_creation_callbacks import freeze_moe_router, make_value_model +except ImportError: + print("mbridge package not found. Please install mbridge with `pip install verl[mcore]` or `pip install mbridge`") + raise + +__all__ = ["AutoBridge", "make_value_model", "freeze_moe_router"] diff --git a/e2c/verl/verl/models/mcore/model_forward.py b/e2c/verl/verl/models/mcore/model_forward.py new file mode 100644 index 0000000000000000000000000000000000000000..e70e11f4ea1f25efc7506292d2b9fed908b135d7 --- /dev/null +++ b/e2c/verl/verl/models/mcore/model_forward.py @@ -0,0 +1,148 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from verl.utils.megatron_utils import unwrap_model + +from .util import postprocess_packed_seqs, preprocess_packed_seqs, recover_left_padding, remove_left_padding + + +def gptmodel_forward( + model, + input_ids, + attention_mask, + position_ids, + sequence_parallel, + value_model=False, + pack_seqs=True, + logits_processor=None, + logits_processor_args: dict = None, + **kwargs, +): + """Default forward pass for GPT models with optional sequence packing.""" + pre_process = unwrap_model(model).pre_process + post_process = unwrap_model(model).post_process + if pack_seqs: + batch_size, seq_len = attention_mask.shape[:2] + input_ids_rmpad, packed_seq_params = preprocess_packed_seqs(input_ids, attention_mask, pre_process=pre_process) + input_ids_rmpad = input_ids_rmpad.contiguous() + output_orig = model( + input_ids=input_ids_rmpad, + attention_mask=None, + position_ids=position_ids, + packed_seq_params=packed_seq_params, + ) + if post_process and logits_processor is not None: + args = { + k: preprocess_packed_seqs(v, attention_mask, pre_process=True)[0] + for k, v in logits_processor_args.items() + } + output_dict = logits_processor(output_orig, **args) + output = { + k: postprocess_packed_seqs( + v, packed_seq_params, attention_mask, batch_size, seq_len, post_process=post_process + ) + for k, v in output_dict.items() + } + else: + output = postprocess_packed_seqs( + output_orig, packed_seq_params, attention_mask, batch_size, seq_len, post_process=post_process + ) + else: + assert logits_processor is None, "logits_processor is not supported for non-packed sequence" + batch_size, sequence_length = attention_mask.shape + new_input_ids, new_attention_mask, new_position_ids = remove_left_padding( + input_ids, attention_mask, position_ids, sequence_parallel, pre_process=pre_process + ) + output = model(input_ids=new_input_ids, attention_mask=new_attention_mask, position_ids=new_position_ids) + output = recover_left_padding( + output, new_attention_mask, attention_mask, sequence_length, post_process=post_process + ) + if value_model and post_process: + output = output[..., 0] + return output + + +def gptmodel_forward_qwen2_5_vl( + model, + input_ids, + attention_mask, + position_ids, + sequence_parallel, + value_model=False, + pack_seqs=True, + multi_modal_inputs=None, + logits_processor=None, + logits_processor_args: dict = None, + **kwargs, +): + from megatron.core import parallel_state as mpu + + assert mpu.get_context_parallel_world_size() == 1, "qwen2_5_vl's context parallel is not accurate yet" + pre_process = unwrap_model(model).pre_process + post_process = unwrap_model(model).post_process + pixel_values = ( + multi_modal_inputs["pixel_values"].to(input_ids.device) if "pixel_values" in multi_modal_inputs else None + ) + image_grid_thw = ( + multi_modal_inputs["image_grid_thw"].to(input_ids.device) if "image_grid_thw" in multi_modal_inputs else None + ) + if pack_seqs: + batch_size, seq_len = attention_mask.shape[:2] + input_ids_rmpad, packed_seq_params = preprocess_packed_seqs(input_ids, attention_mask, pre_process=True) + input_ids_rmpad = input_ids_rmpad.contiguous() + output_orig = model( + input_ids=input_ids_rmpad, + attention_mask=None, + position_ids=position_ids, + packed_seq_params=packed_seq_params, + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + ) + + if post_process and logits_processor is not None: + args = { + k: preprocess_packed_seqs(v, attention_mask, pre_process=True)[0] + for k, v in logits_processor_args.items() + } + output_dict = logits_processor(output_orig, **args) + output = { + k: postprocess_packed_seqs( + v, packed_seq_params, attention_mask, batch_size, seq_len, post_process=post_process + ) + for k, v in output_dict.items() + } + else: + output = postprocess_packed_seqs( + output_orig, packed_seq_params, attention_mask, batch_size, seq_len, post_process=post_process + ) + else: + batch_size, sequence_length = attention_mask.shape + new_input_ids, new_attention_mask, new_position_ids = remove_left_padding( + input_ids, attention_mask, position_ids, sequence_parallel, pre_process=pre_process + ) + output = model( + input_ids=new_input_ids, + position_ids=new_position_ids, + attention_mask=new_attention_mask, + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + ) + output = recover_left_padding( + output, new_attention_mask, attention_mask, sequence_length, post_process=post_process + ) + if value_model and post_process: + output = output[..., 0] + return output diff --git a/e2c/verl/verl/models/mcore/model_forward_fused.py b/e2c/verl/verl/models/mcore/model_forward_fused.py new file mode 100644 index 0000000000000000000000000000000000000000..1f0b8daba084f43fb05e40283a2309b778675111 --- /dev/null +++ b/e2c/verl/verl/models/mcore/model_forward_fused.py @@ -0,0 +1,329 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict +from typing import Optional + +import torch +from megatron.core import parallel_state +from megatron.core.config_logger import has_config_logger_enabled, log_config_to_disk +from megatron.core.inference.contexts import BaseInferenceContext +from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region +from torch import Tensor + +from verl.models.mcore.util import preprocess_packed_seqs +from verl.utils.kernel.linear_cross_entropy import linear_cross_entropy +from verl.utils.megatron_utils import unwrap_model +from verl.utils.model import CausalLMOutputForPPO + +from .qwen2_5_vl.model import Qwen2_5VLModel +from .util import postprocess_packed_seqs_for_dict_output + + +def patch_fused_forward(model: torch.nn.Module): + model = unwrap_model(model) + if isinstance(model, GPTModel): + model = model + elif isinstance(model, Qwen2_5VLModel): + if not hasattr(model, "language_model"): + # the qwen2.5vl model might only have vision_model + return + model = model.language_model + else: + raise ValueError("Model is not a GPTModel or Qwen2_5VLModel") + model.forward_backup = model.forward + model.forward = _fused_GPTModel_forward.__get__(model, model.__class__) + return + + +def unpatch_fused_forward(model: torch.nn.Module): + model = unwrap_model(model) + if isinstance(model, GPTModel): + model = model + elif isinstance(model, Qwen2_5VLModel): + model = model.language_model + else: + raise ValueError("Model is not a GPTModel or Qwen2_5VLModel") + model.forward = model.forward_backup + return + + +def fused_forward_gptmodel( + model: GPTModel, + input_ids: Tensor, + position_ids: Tensor, + attention_mask: Tensor, + labels: Tensor, + labels_mask: Tensor, + temperature: float = 1.0, + **kwargs, +): + pre_process: bool = unwrap_model(model).pre_process + post_process: bool = unwrap_model(model).post_process + + batch_size, seq_len = attention_mask.shape[:2] + input_ids_rmpad, packed_seq_params = preprocess_packed_seqs(input_ids, attention_mask, pre_process=pre_process) + input_ids_rmpad = input_ids_rmpad.contiguous() + labels_rmpad, _ = preprocess_packed_seqs(labels, attention_mask, pre_process=True) + labels_mask_rmpad, _ = preprocess_packed_seqs(labels_mask, attention_mask, pre_process=True) + labels_rmpad = labels_rmpad.contiguous() + labels_mask_rmpad = labels_mask_rmpad.contiguous() + + output_orig: CausalLMOutputForPPO = model( + input_ids=input_ids_rmpad, + attention_mask=None, + position_ids=position_ids, + labels=labels_rmpad, + packed_seq_params=packed_seq_params, + temperature=temperature, + ) + + if post_process: + # output_orig is in type of CausalLMOutputForPPO + output = postprocess_packed_seqs_for_dict_output( + labels_mask_rmpad, + output_orig, + packed_seq_params, + attention_mask, + batch_size, + seq_len, + post_process=post_process, + ) + else: + output = output_orig + return output + + +def fused_forward_qwen2_5_vl( + model: Qwen2_5VLModel, + input_ids: Tensor, + position_ids: Tensor, + attention_mask: Tensor, + labels: Tensor, + labels_mask: Tensor, + multi_modal_inputs=None, + **kwargs, +): + # pre_process = unwrap_model(model).pre_process + post_process = unwrap_model(model).post_process + + pixel_values = ( + multi_modal_inputs["pixel_values"].to(input_ids.device) if "pixel_values" in multi_modal_inputs else None + ) + image_grid_thw = ( + multi_modal_inputs["image_grid_thw"].to(input_ids.device) if "image_grid_thw" in multi_modal_inputs else None + ) + + batch_size, seq_len = attention_mask.shape[:2] + input_ids_rmpad, packed_seq_params = preprocess_packed_seqs(input_ids, attention_mask, pre_process=True) + labels_rmpad, _ = preprocess_packed_seqs(labels, attention_mask, pre_process=True) + labels_mask_rmpad, _ = preprocess_packed_seqs(labels_mask, attention_mask, pre_process=True) + labels_rmpad = labels_rmpad.contiguous() + labels_mask_rmpad = labels_mask_rmpad.contiguous() + input_ids_rmpad = input_ids_rmpad.contiguous() + output_orig: CausalLMOutputForPPO = model( + input_ids=input_ids_rmpad, + attention_mask=None, + position_ids=position_ids, + packed_seq_params=packed_seq_params, + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + labels=labels_rmpad, + ) + if post_process: + # output_orig is in type of CausalLMOutputForPPO + output = postprocess_packed_seqs_for_dict_output( + labels_mask_rmpad, + output_orig, + packed_seq_params, + attention_mask, + batch_size, + seq_len, + post_process=post_process, + ) + else: + output = output_orig + return output + + +def _fused_GPTModel_forward( + self, + input_ids: Tensor, + position_ids: Tensor, + attention_mask: Tensor, + decoder_input: Tensor = None, + labels: Tensor = None, + inference_context: BaseInferenceContext = None, + packed_seq_params: PackedSeqParams = None, + extra_block_kwargs: dict = None, + runtime_gather_output: Optional[bool] = None, + *, + inference_params: Optional[BaseInferenceContext] = None, + loss_mask: Optional[Tensor] = None, + temperature: float = 1.0, +) -> CausalLMOutputForPPO: + """ + Forward pass for GPT models with fused kernel support. + + Patch https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/models/gpt/gpt_model.py + """ + + # If decoder_input is provided (not None), then input_ids and position_ids are ignored. + # Otherwise, apply embedding layer on input_ids and position_ids to get decoder_input. + + # Decoder embedding. + if decoder_input is not None: + pass + elif self.pre_process: + decoder_input = self.embedding(input_ids=input_ids, position_ids=position_ids) + else: + # intermediate stage of pipeline + # decoder will get hidden_states from encoder.input_tensor + decoder_input = None + + # Rotary positional embeddings (embedding is None for PP intermediate devices) + rotary_pos_emb = None + rotary_pos_cos = None + rotary_pos_sin = None + if self.position_embedding_type == "rope" and not self.config.multi_latent_attention: + if not self.training and self.config.flash_decode and inference_context: + assert inference_context.is_static_batching(), "GPTModel currently only supports static inference batching." + # Flash decoding uses precomputed cos and sin for RoPE + rotary_pos_cos, rotary_pos_sin = self.rotary_pos_emb_cache.setdefault( + inference_context.max_sequence_length, + self.rotary_pos_emb.get_cos_sin(inference_context.max_sequence_length), + ) + else: + rotary_seq_len = self.rotary_pos_emb.get_rotary_seq_len( + inference_context, self.decoder, decoder_input, self.config, packed_seq_params + ) + rotary_pos_emb = self.rotary_pos_emb( + rotary_seq_len, + packed_seq=packed_seq_params is not None and packed_seq_params.qkv_format == "thd", + ) + elif self.position_embedding_type == "mrope" and not self.config.multi_latent_attention: + if self.training or not self.config.flash_decode: + rotary_pos_emb = self.rotary_pos_emb(position_ids, self.mrope_section) + else: + # Flash decoding uses precomputed cos and sin for RoPE + raise NotImplementedError( + "Flash decoding uses precomputed cos and sin for RoPE, not implmented in MultimodalRotaryEmbedding yet." + ) + + if ( + (self.config.enable_cuda_graph or self.config.flash_decode) + and rotary_pos_cos is not None + and inference_context + and inference_context.is_static_batching() + and not self.training + ): + sequence_len_offset = torch.tensor( + [inference_context.sequence_len_offset] * inference_context.current_batch_size, + dtype=torch.int32, + device=rotary_pos_cos.device, # Co-locate this with the rotary tensors + ) + else: + sequence_len_offset = None + + # Wrap decoder_input to allow the decoder (TransformerBlock) to delete the + # reference held by this caller function, enabling early garbage collection for + # skip inference + + # Run decoder. + hidden_states = self.decoder( + hidden_states=decoder_input, + attention_mask=attention_mask, + inference_context=inference_context, + rotary_pos_emb=rotary_pos_emb, + rotary_pos_cos=rotary_pos_cos, + rotary_pos_sin=rotary_pos_sin, + packed_seq_params=packed_seq_params, + sequence_len_offset=sequence_len_offset, + **(extra_block_kwargs or {}), + ) + + # Process inference output. + if inference_context and not inference_context.is_static_batching(): + hidden_states = inference_context.last_token_logits(hidden_states.squeeze(1).unsqueeze(0)).unsqueeze(1) + + # logits and loss + output_weight = None + if self.share_embeddings_and_output_weights: + output_weight = self.shared_embedding_or_output_weight() + + if self.mtp_process: + hidden_states = self.mtp( + input_ids=input_ids, + position_ids=position_ids, + labels=labels, + loss_mask=loss_mask, + hidden_states=hidden_states, + attention_mask=attention_mask, + inference_params=inference_params, + rotary_pos_emb=rotary_pos_emb, + rotary_pos_cos=rotary_pos_cos, + rotary_pos_sin=rotary_pos_sin, + packed_seq_params=packed_seq_params, + sequence_len_offset=sequence_len_offset, + embedding=self.embedding, + output_layer=self.output_layer, + output_weight=output_weight, + runtime_gather_output=runtime_gather_output, + compute_language_model_loss=self.compute_language_model_loss, + **(extra_block_kwargs or {}), + ) + + if not self.post_process: + return hidden_states + + output = CausalLMOutputForPPO( + loss=None, + logits=None, + past_key_values=None, + hidden_states=hidden_states, + attentions=None, + ) + + if self.config.sequence_parallel: + hidden_states = gather_from_sequence_parallel_region(hidden_states) + logprobs, entropy = linear_cross_entropy( + hidden_states, + self.output_layer.weight, + labels, + temperature, + "none", + parallel_state.get_tensor_model_parallel_group(), + ) + + if has_config_logger_enabled(self.config): + payload = OrderedDict( + { + "input_ids": input_ids, + "position_ids": position_ids, + "attention_mask": attention_mask, + "decoder_input": decoder_input, + "logprobs": logprobs, + "entropy": entropy, + } + ) + log_config_to_disk(self.config, payload, prefix="input_and_logits") + + output.entropy = entropy + output.log_probs = logprobs + + return output diff --git a/e2c/verl/verl/models/mcore/model_initializer.py b/e2c/verl/verl/models/mcore/model_initializer.py new file mode 100644 index 0000000000000000000000000000000000000000..49a30bc9e2c982fa4e1182d6da745cdd34251dd5 --- /dev/null +++ b/e2c/verl/verl/models/mcore/model_initializer.py @@ -0,0 +1,276 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# use mcore transformer config to initialize the model +import inspect +from abc import ABC, abstractmethod + +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_decoder_block_spec, get_gpt_mtp_block_spec +from megatron.core.models.gpt.gpt_model import GPTModel + +from .config_converter import PretrainedConfig, TransformerConfig + + +class BaseModelInitializer(ABC): + """Base class for model initializers.""" + + def __init__(self, tfconfig: TransformerConfig, hf_config: PretrainedConfig): + self.tfconfig = tfconfig + self.hf_config = hf_config + self.has_vp_stage = inspect.signature(get_gpt_decoder_block_spec).parameters.get("vp_stage", None) is not None + + @abstractmethod + def get_transformer_layer_spec(self, vp_stage=None): + """Get the transformer layer specification. + https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/models/gpt/gpt_layer_specs.py""" + pass + + def get_rope_scaling_args(self) -> dict: + """Get rope scaling args.""" + rope_scaling_args = {} + if "rope_scaling" in self.hf_config: + if self.hf_config.rope_scaling is not None: + # assert self.hf_config.rope_scaling["type"] == "linear", "only linear scaling is supported for now" + rope_scaling_args["seq_len_interpolation_factor"] = self.hf_config.rope_scaling["factor"] + return rope_scaling_args + + def initialize( + self, + pre_process: bool = True, + post_process: bool = True, + share_embeddings_and_output_weights: bool = False, + value: bool = False, + **extra_kwargs, + ) -> GPTModel: + """Initialize a GPT model with the given configuration. + https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/models/gpt/gpt_model.py + + Args: + pre_process (bool): include embedding layer. + post_process (bool): including an output layer. + share_embeddings_and_output_weights (bool): input embeddings and output logit weights are shared. + value (bool): add an extra linear layer for classification or regression. + + Returns: + GPTModel: An initialized GPT model instance + """ + vp_stage = extra_kwargs.get("vp_stage", None) + transformer_layer_spec = self.get_transformer_layer_spec(vp_stage=vp_stage) + rope_scaling_args = self.get_rope_scaling_args() + mtp_block_spec = extra_kwargs.get("mtp_block_spec", None) + model = GPTModel( + config=self.tfconfig, + transformer_layer_spec=transformer_layer_spec, + vocab_size=self.hf_config.vocab_size, + max_sequence_length=self.hf_config.max_position_embeddings, + pre_process=pre_process, + post_process=post_process, + share_embeddings_and_output_weights=share_embeddings_and_output_weights, + position_embedding_type="rope", + rotary_base=self.hf_config.rope_theta, + **rope_scaling_args, + mtp_block_spec=mtp_block_spec, + **({} if not self.has_vp_stage else {"vp_stage": vp_stage}), + ) + + if post_process and value: + from verl.models.llama.megatron.layers.parallel_linear import LinearForLastLayer + + model.output_layer = LinearForLastLayer( + input_size=self.tfconfig.hidden_size, output_size=1, config=self.tfconfig + ) + + return model + + +class DenseModel(BaseModelInitializer): + """Initializer for dense models like Llama and Qwen2.""" + + def get_transformer_layer_spec(self, vp_stage=None): + assert self.tfconfig.normalization == "RMSNorm", "only RMSNorm is supported for now" + extra_kwargs = {} if not self.has_vp_stage else {"vp_stage": vp_stage} + return get_gpt_decoder_block_spec(self.tfconfig, use_transformer_engine=True, **extra_kwargs) + + +class Qwen2MoEModel(BaseModelInitializer): + """Initializer for Qwen2 MoE models.""" + + def get_transformer_layer_spec(self, vp_stage=None): + assert self.tfconfig.normalization == "RMSNorm", "only RMSNorm is supported for now" + extra_kwargs = {} if not self.has_vp_stage else {"vp_stage": vp_stage} + transformer_layer_spec = get_gpt_decoder_block_spec(self.tfconfig, use_transformer_engine=True, **extra_kwargs) + + # Patch layer spec for shared experts + for i in range(len(transformer_layer_spec.layer_specs)): + transformer_layer_spec.layer_specs[i].submodules.mlp.submodules.shared_experts.params["gate"] = True + + return transformer_layer_spec + + def initialize(self, **kwargs): + # Qwen default freeze_moe_router: true + model = super().initialize(**kwargs) + freeze_moe_router = kwargs.get("freeze_moe_router", True) + if freeze_moe_router: + for layer in model.decoder.layers: + layer.mlp.router.weight.requires_grad = False + return model + + +class MixtralModel(BaseModelInitializer): + """Initializer for Mixtral models.""" + + def get_transformer_layer_spec(self, vp_stage=None): + assert self.tfconfig.normalization == "RMSNorm", "only RMSNorm is supported for now" + extra_kwargs = {} if not self.has_vp_stage else {"vp_stage": vp_stage} + transformer_layer_spec = get_gpt_decoder_block_spec(self.tfconfig, use_transformer_engine=True, **extra_kwargs) + return transformer_layer_spec + + def initialize(self, **kwargs): + model = super().initialize(**kwargs) + freeze_moe_router = kwargs.get("freeze_moe_router", False) + if freeze_moe_router: + for layer in model.decoder.layers: + layer.mlp.router.weight.requires_grad = False + return model + + +class Qwen3MoEModel(BaseModelInitializer): + """Initializer for Qwen3 MoE models.""" + + def get_transformer_layer_spec(self, vp_stage=None): + assert self.tfconfig.normalization == "RMSNorm", "only RMSNorm is supported for now" + extra_kwargs = {} if not self.has_vp_stage else {"vp_stage": vp_stage} + transformer_layer_spec = get_gpt_decoder_block_spec(self.tfconfig, use_transformer_engine=True, **extra_kwargs) + return transformer_layer_spec + + def initialize(self, **kwargs): + # Qwen default freeze_moe_router: true + model = super().initialize(**kwargs) + freeze_moe_router = kwargs.get("freeze_moe_router", True) + if freeze_moe_router: + for layer in model.decoder.layers: + layer.mlp.router.weight.requires_grad = False + return model + + +class DeepseekV3Model(BaseModelInitializer): + """Initializer for DeepseekV3 models.""" + + def get_transformer_layer_spec(self, vp_stage=None): + extra_kwargs = {} if not self.has_vp_stage else {"vp_stage": vp_stage} + transformer_layer_spec = get_gpt_decoder_block_spec(self.tfconfig, use_transformer_engine=True, **extra_kwargs) + return transformer_layer_spec + + def get_rope_scaling_args(self) -> dict: + """Get rope scaling args.""" + rope_scaling_args = {} + return rope_scaling_args + + def initialize( + self, + **kwargs, + ): + vp_stage = kwargs.get("vp_stage", None) + freeze_moe_router = kwargs.get("freeze_moe_router", True) + if freeze_moe_router: + self.tfconfig.moe_router_load_balancing_type = "none" + # MTP + if self.tfconfig.mtp_num_layers is not None and self.tfconfig.mtp_num_layers > 0: + transformer_layer_spec = self.get_transformer_layer_spec(vp_stage=vp_stage) + mtp_block_spec = get_gpt_mtp_block_spec( + self.tfconfig, transformer_layer_spec, use_transformer_engine=True, vp_stage=vp_stage + ) + kwargs["mtp_block_spec"] = mtp_block_spec + + model = super().initialize(**kwargs) + if freeze_moe_router: + for layer in model.decoder.layers: + if hasattr(layer.mlp, "router"): + layer.mlp.router.weight.requires_grad = False + return model + + +class Qwen25VLModel(BaseModelInitializer): + """Initializer for Qwen2.5 VL models.""" + + def get_transformer_layer_spec(self, vp_stage=None): + extra_kwargs = {} if not self.has_vp_stage else {"vp_stage": vp_stage} + transformer_layer_spec = get_gpt_decoder_block_spec(self.tfconfig, use_transformer_engine=True, **extra_kwargs) + return transformer_layer_spec + + def initialize( + self, + pre_process=None, + post_process=None, + share_embeddings_and_output_weights=False, + value=False, + **extra_kwargs, + ): + tfconfig = self.tfconfig + hf_config = self.hf_config + # Qwen2_5_VLForConditionalGeneration + from copy import deepcopy + + transformer_layer_spec = self.get_transformer_layer_spec() + + from megatron.core.extensions.transformer_engine import TEColumnParallelLinear, TERowParallelLinear + from megatron.core.models.gpt.moe_module_specs import MLPSubmodules + from megatron.core.models.vision.vit_layer_specs import get_vit_layer_with_transformer_engine_spec + + from .qwen2_5_vl import Qwen2_5VLModel, get_vision_model_config, get_vision_projection_config + + vision_transformer_config = get_vision_model_config(deepcopy(tfconfig)) + vision_transformer_config.pipeline_model_parallel_size = 1 + vision_transformer_config.first_pipeline_num_layers = None + + vision_projection_config = get_vision_projection_config( + deepcopy(tfconfig), + vision_transformer_config.hidden_size, + spatial_merge_size=hf_config.vision_config.spatial_merge_size, + ) + vision_projection_layer_spec = MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ) + vision_transformer_layer_spec = get_vit_layer_with_transformer_engine_spec() + + qwen25_vl_model = Qwen2_5VLModel( + language_transformer_config=tfconfig, + language_transformer_layer_spec=transformer_layer_spec, + language_vocab_size=hf_config.vocab_size, + language_max_sequence_length=hf_config.max_position_embeddings, + vision_transformer_config=vision_transformer_config, + vision_transformer_layer_spec=vision_transformer_layer_spec, + vision_projection_config=vision_projection_config, + vision_projection_layer_spec=vision_projection_layer_spec, + vision_projection_type="mlp", + language_rotary_base=hf_config.rope_theta, + pre_process=pre_process, + post_process=post_process, + add_decoder=True, + add_encoder=True, + parallel_output=True, + language_share_embeddings_and_output_weights=share_embeddings_and_output_weights, + ) + + if post_process and value: + from verl.models.llama.megatron.layers.parallel_linear import LinearForLastLayer + + qwen25_vl_model.language_model.output_layer = LinearForLastLayer( + input_size=tfconfig.hidden_size, output_size=1, config=tfconfig + ) + + return qwen25_vl_model diff --git a/e2c/verl/verl/models/mcore/patch_v012.py b/e2c/verl/verl/models/mcore/patch_v012.py new file mode 100644 index 0000000000000000000000000000000000000000..d54a3eb346d272588c39f838a7723d0ff9f574fd --- /dev/null +++ b/e2c/verl/verl/models/mcore/patch_v012.py @@ -0,0 +1,215 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# there is some bug in mcore 0.12, so we need to patch it +# 1. `get_query_key_value_tensors` in `multi_latent_attention.py` works wrong when packed_seq_params is not None + + +def apply_patch(): + import torch + from megatron.core import parallel_state, tensor_parallel + from megatron.core.transformer.multi_latent_attention import ( + MLASelfAttention, + apply_rotary_pos_emb, + deprecate_inference_params, + gather_from_sequence_parallel_region, + gather_from_tensor_model_parallel_region, + scatter_to_sequence_parallel_region, + ) + + def patch_get_query_key_value_tensors( + self, + hidden_states, + key_value_states=None, + position_ids=None, + packed_seq_params=None, + inference_context=None, + *, + inference_params=None, + ): + """ + Derives `query`, `key` and `value` tensors from `hidden_states`. + """ + # s = sequence length, b = batch size, h = hidden size, n = num attention heads + # Attention heads [s, b, n*h] + assert hidden_states.ndim == 3, f"hidden_states should be 3D, [s, b, n*h], got {hidden_states.ndim}D" + + inference_context = deprecate_inference_params(inference_context, inference_params) + + # ========================================= + # Prepare RoPE and seqlen related params + # ========================================= + rotary_seq_len = self.rotary_pos_emb.get_rotary_seq_len( + inference_context, None, hidden_states, self.config, packed_seq_params + ) + + # rotary_pos_emb:[s, b, 1, 64] + mscale = 1.0 + if self.config.rope_type == "rope": + packed_seq = packed_seq_params is not None and packed_seq_params.qkv_format == "thd" + rotary_pos_emb = self.rotary_pos_emb(rotary_seq_len, packed_seq=packed_seq) + else: + rotary_pos_emb, mscale = self.rotary_pos_emb(rotary_seq_len) + + # ========================================= + # QKV down projection and layernorm + # ========================================= + if self.config.q_lora_rank is not None: + # if linear_q_down_proj is ColumnParallelLinear: + # q_compressed: [s, b, q_lora_rank / TP] + # elif linear_q_down_proj is Linear: + # q_compressed: [s / TP, b, q_lora_rank] + q_compressed, _ = self.linear_q_down_proj(hidden_states) + + # When output is sharded (ColumnParallelLinear), two things are needed to be + # identical to a normal Linear. + # 1. Manually gather output to restore output dim q_lora_rank; + # 2. Scatter sequence back to s / TP if sequence-parallel since it was + # gathered by ColumnParallelLinear. + if q_compressed.size(-1) != self.config.q_lora_rank: + q_compressed = gather_from_tensor_model_parallel_region(q_compressed) + if self.config.sequence_parallel: + q_compressed = scatter_to_sequence_parallel_region(q_compressed) + + q_compressed = self.q_layernorm(q_compressed) + else: + q_compressed = hidden_states + + # if linear_kv_down_proj is ColumnParallelLinear: + # kv_combined: [s, b, (kv_lora_rank + qk_pos_emb_head_dim) / TP] + # elif linear_kv_down_proj is Linear: + # kv_combined: [s / TP, b, (kv_lora_rank + qk_pos_emb_head_dim)] + kv_combined, _ = self.linear_kv_down_proj(hidden_states) + if kv_combined.size(-1) != self.config.kv_lora_rank + self.config.qk_pos_emb_head_dim: + # kv_combined: [s, b, (kv_lora_rank + qk_pos_emb_head_dim)] + kv_combined = gather_from_tensor_model_parallel_region(kv_combined) + # kv_compressed:[s, b, kv_lora_rank], k_pos_emb: [s, b, qk_pos_emb_head_dim] + kv_compressed, k_pos_emb = torch.split( + kv_combined, [self.config.kv_lora_rank, self.config.qk_pos_emb_head_dim], dim=-1 + ) + if self.config.sequence_parallel: + # kv_compressed:[s / TP, b, kv_lora_rank] + kv_compressed = scatter_to_sequence_parallel_region(kv_compressed) + else: + # kv_compressed:[s / TP, b, kv_lora_rank], k_pos_emb: [s / TP, b, qk_pos_emb_head_dim] + kv_compressed, k_pos_emb = torch.split( + kv_combined, [self.config.kv_lora_rank, self.config.qk_pos_emb_head_dim], dim=-1 + ) + if parallel_state.get_tensor_model_parallel_world_size() > 1: + # k_pos_emb: [s, b, qk_pos_emb_head_dim] + k_pos_emb = gather_from_sequence_parallel_region(k_pos_emb) + + kv_compressed = self.kv_layernorm(kv_compressed) + + # ========================================= + # QKV up projection and RoPE apply + # ========================================= + def qkv_up_proj_and_rope_apply(q_compressed, kv_compressed, k_pos_emb, rotary_pos_emb): + if self.config.q_lora_rank is not None: + q, _ = self.linear_q_up_proj(q_compressed) + else: + # hidden_states:[s, b, 2048], q: [s, b, n * 192] + q, _ = self.linear_q_proj(q_compressed) + + q_len, bsz, _ = q.size() + + # q: [s, b, n, 192] + q = q.view(q_len, bsz, self.num_attention_heads_per_partition, self.q_head_dim) + + # kv: [s, b, 2048] + kv, _ = self.linear_kv_up_proj(kv_compressed) + + # kv: [s, b, n, 256] + kv = kv.view( + q_len, + bsz, + self.num_attention_heads_per_partition, + self.config.qk_head_dim + self.config.v_head_dim, + ) + + if inference_context is not None: + # add offset to the sequence start for inference + sequence_start = inference_context.sequence_len_offset + sequence_end = sequence_start + q_len + rotary_pos_emb = rotary_pos_emb[sequence_start:sequence_end] + else: + # Shorten rotary_pos_emb to the sequence length when inference_params + # is not provided. This makes sure we can run forward directly with + # any sequence length. During training, the sequence length is always + # the full rotary_pos_emb length. + rotary_pos_emb = rotary_pos_emb[0:q_len] + + # [s, b, 64] -> [s, b, 1, 64] + k_pos_emb = torch.unsqueeze(k_pos_emb, 2) + + # q: [s, b, n, 128], q_pos_emb: [s, b, n, 64] + q_no_pe, q_pos_emb = torch.split(q, [self.config.qk_head_dim, self.config.qk_pos_emb_head_dim], dim=-1) + + # k_no_pe: [s, b, n, 128], value: [s, b, n, 128] + k_no_pe, value = torch.split(kv, [self.config.qk_head_dim, self.config.v_head_dim], dim=-1) + + if packed_seq_params is not None: + cu_seqlens_q = packed_seq_params.cu_seqlens_q + cu_seqlens_kv = packed_seq_params.cu_seqlens_kv + q_pos_emb = q_pos_emb.squeeze(1) + k_pos_emb = k_pos_emb.squeeze(1) + q_no_pe = q_no_pe.squeeze(1) + k_no_pe = k_no_pe.squeeze(1) + value = value.squeeze(1) + else: + cu_seqlens_q = cu_seqlens_kv = None + + # q_pos_emb: [s, b, n, 64], k_pos_emb:[s, b, 1, 64] + q_pos_emb = apply_rotary_pos_emb( + q_pos_emb, + rotary_pos_emb, + config=self.config, + cu_seqlens=cu_seqlens_q, + mscale=mscale, + ) + k_pos_emb = apply_rotary_pos_emb( + k_pos_emb, + rotary_pos_emb, + config=self.config, + cu_seqlens=cu_seqlens_kv, + mscale=mscale, + ) + + # query: [s, b, n, 192] + query = torch.cat([q_no_pe, q_pos_emb], dim=-1) + if packed_seq_params is not None: + k_pos_emb = k_pos_emb.expand(-1, self.num_attention_heads_per_partition, -1) + key = torch.cat([k_no_pe, k_pos_emb], dim=-1) + else: + # key: [s, b, n, 192] + k_pos_emb = k_pos_emb.expand(-1, -1, self.num_attention_heads_per_partition, -1) + key = torch.cat([k_no_pe, k_pos_emb], dim=-1) + + query = query.contiguous() + key = key.contiguous() + value = value.contiguous() + return query, key, value + + if self.recompute_up_proj: + self.qkv_up_checkpoint = tensor_parallel.CheckpointWithoutOutput() + query, key, value = self.qkv_up_checkpoint.checkpoint( + qkv_up_proj_and_rope_apply, q_compressed, kv_compressed, k_pos_emb, rotary_pos_emb + ) + else: + query, key, value = qkv_up_proj_and_rope_apply(q_compressed, kv_compressed, k_pos_emb, rotary_pos_emb) + + return query, key, value + + MLASelfAttention.get_query_key_value_tensors = patch_get_query_key_value_tensors diff --git a/e2c/verl/verl/models/mcore/qwen2_5_vl/__init__.py b/e2c/verl/verl/models/mcore/qwen2_5_vl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8842d0249e1fa5397734bb0929e65d20978f815f --- /dev/null +++ b/e2c/verl/verl/models/mcore/qwen2_5_vl/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024 Alibaba PAI Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .model import Qwen2_5VLModel +from .vision_config import get_vision_model_config, get_vision_projection_config + +__all__ = ["Qwen2_5VLModel", "get_vision_model_config", "get_vision_projection_config"] diff --git a/e2c/verl/verl/models/mcore/qwen2_5_vl/attention.py b/e2c/verl/verl/models/mcore/qwen2_5_vl/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..84e7ba8eda22172bc770bc57ac00dc395aa3f119 --- /dev/null +++ b/e2c/verl/verl/models/mcore/qwen2_5_vl/attention.py @@ -0,0 +1,224 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024 Alibaba PAI Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from megatron.core.transformer.attention import * + +from .rope_utils import apply_rotary_pos_emb_absolute + + +class Qwen2_5VLSelfAttention(SelfAttention): + """ + Overrides the SelfAttention class, the difference is that qwen2_5_vl uses apply_rotary_pos_emb_absolute + instead of apply_rotary_pos_emb + """ + + def forward( + self, + hidden_states: Tensor, + attention_mask: Tensor, + key_value_states: Optional[Tensor] = None, + inference_context: Optional[BaseInferenceContext] = None, + rotary_pos_emb: Optional[Union[Tensor, Tuple[Tensor, Tensor]]] = None, + rotary_pos_cos: Optional[Tensor] = None, + rotary_pos_sin: Optional[Tensor] = None, + attention_bias: Optional[Tensor] = None, + packed_seq_params: Optional[PackedSeqParams] = None, + sequence_len_offset: Optional[int] = None, + *, + inference_params: Optional[BaseInferenceContext] = None, + ) -> Tuple[Tensor, Tensor]: + """ + Perform a forward pass through the attention module. + + Args: + hidden_states (Tensor): Hidden states. + attention_mask (Tensor): Attention mask. + key_value_states (Optional[Tensor]): Key/value states (for cross attention). + inference_context (Optional[BaseInferenceContext]): Inference context that manages + KV cache. + rotary_pos_emb (Optional[Union[Tensor, Tuple[Tensor, Tensor]]]): Rotary + embedding tensor(s). + rotary_pos_cos (Optional[Tensor]): Rotary embedding cosine. + rotary_pos_sin (Optional[Tensor]): Rotary embedding sine. + attention_bias (Optional[Tensor]): Attention bias. + packed_seq_params (Optional[PackedSeqparams]): Parameters used for THD format. + sequence_len_offset (Optional[int]): Sequence length offset used for + inference CUDA graphs. + + Return: + (Tuple[Tensor, Tensor]) Attention output and bias. + + """ + + inference_context = deprecate_inference_params(inference_context, inference_params) + + if inference_context and inference_context.is_dynamic_batching(): + assert flash_decode_and_prefill_kernel is not None, ( + "Internal use only: install package `nvidia_chunked_flash_attn`." + ) + + # hidden_states: [sq, b, h] + if self.config.flash_decode and not self.training and inference_context is not None: + rotary_pos_emb = None + else: + assert rotary_pos_cos is None and rotary_pos_sin is None + + # For self attention we just duplicate the rotary_pos_emb if it isn't already + if rotary_pos_emb is not None and not isinstance(rotary_pos_emb, tuple): + rotary_pos_emb = (rotary_pos_emb,) * 2 + + # ===================== + # Query, Key, and Value + # ===================== + # Get the query, key and value tensors based on the type of attention - + # self or cross attn. + query, key, value = self.get_query_key_value_tensors(hidden_states, key_value_states) + + # =================================================== + # Adjust key, value, and rotary_pos_emb for inference + # =================================================== + + # This branch only runs in the decode phase of flash decoding and returns after the linear + # projection. This conditional is not used in the prefill phase or non-flash-decoding cases. + if ( + self.config.flash_decode + and inference_context is not None + and inference_context.is_decode_only() + and not self.training + and rotary_pos_cos is not None + ): + assert self.layer_number in inference_context.key_value_memory_dict + assert inference_context.sequence_len_offset is not None + inference_key_memory, inference_value_memory = inference_context.key_value_memory_dict[self.layer_number] + output = self.flash_decode( + sequence_len_offset=sequence_len_offset, + query_layer=query, + key_layer=key, + value_layer=value, + inference_key_memory=inference_key_memory, + inference_value_memory=inference_value_memory, + rotary_cos=rotary_pos_cos, + rotary_sin=rotary_pos_sin, + ) + out = output.transpose(0, 1).contiguous() + context_layer = out.view(out.size(0), out.size(1), -1) + output, bias = self.linear_proj(context_layer) + return output, bias + + # Use latest mcore 0.13 API and forward-compatible with previous versions. + outputs = self._adjust_key_value_for_inference( + inference_context, + query, + key, + value, + rotary_pos_emb, + rotary_pos_cos, + rotary_pos_sin, + sequence_len_offset, + ) + + query, key, value, rotary_pos_emb, attn_mask_type = outputs[:5] + + if packed_seq_params is not None: + query = query.squeeze(1) + key = key.squeeze(1) + value = value.squeeze(1) + + # ================================================ + # relative positional embedding (rotary embedding) + # ================================================ + if rotary_pos_emb is not None and not self.config.flash_decode: + q_pos_emb, k_pos_emb = rotary_pos_emb + + if packed_seq_params is not None: + if packed_seq_params.cu_seqlens_q_padded is not None: + cu_seqlens_q = packed_seq_params.cu_seqlens_q_padded + else: + cu_seqlens_q = packed_seq_params.cu_seqlens_q + if packed_seq_params.cu_seqlens_kv_padded is not None: + cu_seqlens_kv = packed_seq_params.cu_seqlens_kv_padded + else: + cu_seqlens_kv = packed_seq_params.cu_seqlens_kv + else: + cu_seqlens_q = cu_seqlens_kv = None + + if q_pos_emb is not None: + # TODO VIJAY: simplify + if inference_context is None or inference_context.is_static_batching(): + query = apply_rotary_pos_emb_absolute(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) + else: + query = inference_context.apply_rotary_emb_query(query, q_pos_emb, self.config, cu_seqlens_q) + if k_pos_emb is not None: + key = apply_rotary_pos_emb_absolute(key, k_pos_emb, config=self.config, cu_seqlens=cu_seqlens_kv) + + # TODO, can apply positional embedding to value_layer so it has + # absolute positional embedding. + # otherwise, only relative positional embedding takes effect + # value_layer = apply_rotary_pos_emb(value_layer, k_pos_emb) + + # ================================== + # core attention computation + # ================================== + + if self.checkpoint_core_attention and self.training: + core_attn_out = self._checkpointed_attention_forward( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type, + attention_bias=attention_bias, + packed_seq_params=packed_seq_params, + ) + else: + if inference_context is None or inference_context.is_static_batching(): + # Static batching attention kernel. + core_attn_out = self.core_attention( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type, + attention_bias=attention_bias, + packed_seq_params=packed_seq_params, + ) + + else: + # Dynamic batching attention kernel. + q, k, v = (query, key, value) + cu_query_lengths, max_seqlen_q = inference_context.cu_query_lengths() + cu_kv_lengths, max_seqlen_k = inference_context.cu_kv_lengths() + + core_attn_out = self.flash_decode_and_prefill( + q, k, v, max_seqlen_q, max_seqlen_k, cu_query_lengths, cu_kv_lengths + ) + core_attn_out = core_attn_out.squeeze(0).unsqueeze(1) + core_attn_out = rearrange(core_attn_out, "s b h d -> s b (h d)") + + if packed_seq_params is not None and packed_seq_params.qkv_format == "thd": + # reshape to same output shape as unpacked case + # (t, np, hn) -> (t, b=1, h=np*hn) + # t is the pack size = sum (sq_i) + # note that batch is a dummy dimension in the packed case + core_attn_out = core_attn_out.reshape(core_attn_out.size(0), 1, -1) + + # ================= + # Output. [sq, b, h] + # ================= + + output, bias = self.linear_proj(core_attn_out) + + return output, bias diff --git a/e2c/verl/verl/models/mcore/qwen2_5_vl/model.py b/e2c/verl/verl/models/mcore/qwen2_5_vl/model.py new file mode 100644 index 0000000000000000000000000000000000000000..74e4406c35d056b906cbcb7e04dbb4ed5f8bbbdb --- /dev/null +++ b/e2c/verl/verl/models/mcore/qwen2_5_vl/model.py @@ -0,0 +1,340 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024 Alibaba PAI Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging + +import torch +from megatron.core import InferenceParams, tensor_parallel +from megatron.core.models.gpt.gpt_model import GPTModel + +# from .transformer_config import Qwen2VLTransformerConfig +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer import MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig + +from .attention import Qwen2_5VLSelfAttention +from .vision_model import Qwen2_5VisionModel + + +# Note: This is under development and may be missing features. +class Qwen2_5VLModel(MegatronModule): + """Qwen2.5VL multi-modal model. + + Args: + language_transformer_config (TransformerConfig): Transformer config for the language model. + language_transformer_layer_spec (ModuleSpec): Specifies module to use for transformer layers of the + language model. + language_vocab_size (int): Language model vocabulary size. + language_max_sequence_length (int): Language model maximum sequence length. This is used for + positional embedding. + vision_transformer_config (TransformerConfig): Transformer config for the vision model. + vision_transformer_layer_spec (ModuleSpec): Specifies module to use for transformer layers of the + vision model. + vision_projection_config (TransformerConfig): Config for the projection from vision model outputs to + language model inputs. + vision_projection_layer_spec (ModuleSpec): Specifies the module to use for the vision + projection. + vision_projection_type (str): Type of the vision projection to use. Default is a 2-layer MLP. + parallel_output (bool): Do not gather the outputs, keep them split across tensor parallel ranks. This + is typically True for training and False for inference. + language_rotary_percent (float): Percent of rotary dimension to use for rotary position embeddings + in the language model. Defaults to 1.0. + pre_process (bool): Include the embedding layer in the gpt decoder (used with pipeline parallelism). + Defaults to True. + post_process (bool): Include an output layer and a layernorm in the gpt decoder (used with pipeline + parallelism). Defaults to True. + add_encoder (bool): Construct the encoder module (used with pipeline parallelism). Defaults to True. + When we use pipelining, the encoder + will live on only a subset of the pipeline stages (specifically, only the first stage). + add_decoder (bool): Construct the decoder module (used with pipeline parallelism). Defaults to True. + When we use pipelining, the decoder + will live on only a subset of the pipeline stages (specifically, every stage after the first one). + img_h (int): The height of each image that the ViT will see. + img_w (int): The width of each image that the ViT will see. + patch_dim (int): The size of each patch side. + img_embedding_idx (int): Index in the language_embeddings tensor where image_embeddings should be + inserted. Defaults to 0. + """ + + def __init__( + self, + language_transformer_config: TransformerConfig, + language_transformer_layer_spec: ModuleSpec, + language_vocab_size: int, + language_max_sequence_length: int, + vision_transformer_config: TransformerConfig, + vision_transformer_layer_spec: ModuleSpec, + vision_projection_config: TransformerConfig, + vision_projection_layer_spec: ModuleSpec, + vision_projection_type: str = "mlp", + parallel_output: bool = True, + language_rotary_percent: float = 1.0, + pre_process: bool = True, + post_process: bool = True, + add_encoder: bool = True, + add_decoder: bool = True, + language_rotary_base: int = 10000, + fp16_lm_cross_entropy: bool = False, + language_share_embeddings_and_output_weights: bool = False, + image_token_id: int = 151655, + video_token_id: int = 151656, + ) -> None: + super().__init__(config=language_transformer_config) + + # patch self_attention to use qwen2_5_vl attention + vision_transformer_layer_spec.submodules.self_attention.module = Qwen2_5VLSelfAttention + for layer_spec in language_transformer_layer_spec.layer_specs: + layer_spec.submodules.self_attention.module = Qwen2_5VLSelfAttention + + logging.getLogger(__name__).warning("Qwen2VL model is under development and may be missing features.") + + self.pre_process = pre_process + self.post_process = post_process + self.add_encoder = add_encoder + self.add_decoder = add_decoder + + self.encoder_hidden_state = None + self.vision_model = None + self.vision_projection = None + self.language_model = None + self.image_token_id = image_token_id + self.video_token_id = video_token_id + + self.square_merge_size = vision_projection_config.ffn_hidden_size // vision_transformer_config.hidden_size + + # This attribute is needed to check if an all-reduce is required + # on the word embeddings inside `finalize_model_grads._allreduce_word_embedding_grads`. + self.share_embeddings_and_output_weights = False + if self.pre_process: + self.vision_model = Qwen2_5VisionModel( + vision_transformer_config, + vision_transformer_layer_spec, + vision_projection_config, + vision_projection_layer_spec, + projection_type=vision_projection_type, + pre_process=True, + post_process=True, + ) + + self.language_model = GPTModel( + config=language_transformer_config, + transformer_layer_spec=language_transformer_layer_spec, + vocab_size=language_vocab_size, + max_sequence_length=language_max_sequence_length, + parallel_output=parallel_output, + position_embedding_type="mrope", + rotary_percent=language_rotary_percent, + pre_process=self.pre_process, + post_process=self.post_process, + rotary_base=language_rotary_base, + fp16_lm_cross_entropy=fp16_lm_cross_entropy, + share_embeddings_and_output_weights=language_share_embeddings_and_output_weights, + scatter_embedding_sequence_parallel=False, + ) + + self.share_embeddings_and_output_weights = self.language_model.share_embeddings_and_output_weights + + def shared_embedding_or_output_weight(self): + """This is a convenience method to surface the language model's word embeddings, which is + necessary for `finalize_model_grads._allreduce_word_embedding_grads`.""" + if self.add_decoder: + return self.language_model.shared_embedding_or_output_weight() + return None + + def set_input_tensor(self, input_tensor) -> None: + # This is usually handled in schedules.py but some inference code still + # gives us non-lists or None + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + assert len(input_tensor) == 1, "input_tensor should only be length 1 for Qwen2VL" + + if self.pre_process: + self.encoder_hidden_state = input_tensor[0] + else: + self.language_model.set_input_tensor(input_tensor[0]) + + def freeze(self, freeze_language_model: bool, freeze_vision_model: bool, freeze_vision_projection: bool): + """Freeze model modules. + + Make specific modules non-trainable by setting requires_grad to False for the module's parameters. + + Args: + freeze_language_model (bool): Freeze the language model module. + freeze_vision_model (bool): Freeze the vision model module. + freeze_vision_projection (bool): Freeze the vision projection module. + """ + modules = [] + if freeze_language_model and self.language_model is not None: + modules.append(self.language_model) + if freeze_vision_model and self.vision_model is not None: + modules.append(self.vision_model) + if freeze_vision_projection and self.vision_projection is not None: + modules.append(self.vision_projection) + + for module in modules: + for param in module.parameters(): + param.requires_grad = False + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + attention_mask: torch.Tensor = None, + labels: torch.Tensor = None, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + extra_block_kwargs: dict = None, + pixel_values: torch.Tensor = None, + pixel_values_videos: torch.Tensor = None, + image_grid_thw: torch.Tensor = None, + video_grid_thw: torch.Tensor = None, + ) -> torch.Tensor: + """Forward function of the Qwen2VL model. + + Args: + image_data (torch.Tensor): input image of shape [total_thw_size, n_features]. + input_ids (torch.Tensor): input text ids [batch, text_seq_len]. + position_ids (torch.Tensor): input text position ids [batch, text_seq_len]. + attention_mask (torch.Tensor): attention mask for the language model [batch, 1, combined_seq_len, + combined_seq_len]. + labels (torch.Tensor): Optional target text labels [batch, combined_seq_len]. + inference_params (InferenceParams): Inference-time parameters including KV cache. + + video_start_index: + 0 -- all video + len(video_seq) -- all image + others -- mixture + *_input_mask: should not be None in the first PP stage + Returns: + output (torch.Tensor): Loss of shape [b, s] if labels are provided, otherwise logits of shape + [b, s, vocab_size]. + """ + video_start_index = 0 + vision_grid_thw = None + vision_data = None + if image_grid_thw is not None: + image_mask = input_ids == self.image_token_id + vision_grid_thw = image_grid_thw + vision_data = pixel_values + video_start_index = image_mask.sum().item() + if video_grid_thw is not None: + video_mask = input_ids == self.video_token_id + vision_grid_thw = torch.cat([vision_grid_thw, video_grid_thw], dim=0) + vision_data = torch.cat([vision_data, pixel_values_videos], dim=0) + video_start_index = image_mask.sum().item() + video_mask.sum().item() + use_inference_kv_cache = ( + inference_params is not None and "image_tokens_count" in inference_params.key_value_memory_dict + ) + use_inference_kv_cache = ( + inference_params is not None and "image_tokens_count" in inference_params.key_value_memory_dict + ) + if use_inference_kv_cache: + raise NotImplementedError() + + if self.pre_process: + vision_embeds = None + if vision_grid_thw is not None and vision_grid_thw.shape[0] > 0: + vision_embeds = self.vision_model( + vision_data=vision_data, # If None, vision model should use intermediate outputs (EPP > 1) + grid_thw=vision_grid_thw, # should provided in each EPP stage + ) + + # If running inference, the language model KV cache will be updated for image token positions. + # Here we store the image tokens sequence length, which can be used as an offset to the KV cache later. + if inference_params is not None: + raise NotImplementedError() + # inference_params.key_value_memory_dict["image_tokens_count"] = ( + # vision_embeddings.shape[0] + # ) + + # If running inference, we can skip image token computation if they were computed already earlier + # for this sample. + if use_inference_kv_cache: + language_embeddings: torch.Tensor = self.language_model.embedding( + input_ids=input_ids, + position_ids=None, # NOTE: disable + ) # [text_seq_len, b, h_language] + # NOTE: why not cat here? is it the combined embeddings useless? + combined_embeddings = language_embeddings + elif vision_embeds is not None: + if video_start_index == 0: + image_embeds = None + video_embeds = vision_embeds + elif video_start_index == vision_embeds.shape[0]: + image_embeds = vision_embeds + video_embeds = None + elif 0 < video_start_index < vision_embeds.shape[0]: + image_embeds = vision_embeds[:video_start_index] + video_embeds = vision_embeds[video_start_index:] + else: + raise ValueError( + f"Expect video token start index in range [0, {vision_embeds.shape[0]}], but got " + f"{video_start_index}" + ) + + combined_embeddings = self.language_model.embedding( + input_ids=input_ids, + position_ids=None, # NOTE: disable + ) # [text_seq_len, b, h_language] + + if image_embeds is not None or video_embeds is not None: + combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + if image_embeds is not None: + image_mask = (input_ids == self.image_token_id).contiguous() + if image_mask.sum() > 0: + combined_embeddings = combined_embeddings.clone() + combined_embeddings[image_mask] = image_embeds.to( + dtype=combined_embeddings.dtype, device=combined_embeddings.device + ) + if video_embeds is not None: + video_mask = (input_ids == self.video_token_id).contiguous() + if video_mask.sum() > 0: + combined_embeddings = combined_embeddings.clone() + combined_embeddings[video_mask] = video_embeds.to( + dtype=combined_embeddings.dtype, device=combined_embeddings.device + ) + combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + + else: + combined_embeddings = self.language_model.embedding( + input_ids=input_ids, + position_ids=None, # NOTE: disable + ) # [text_seq_len, b, h_language] + if self.config.sequence_parallel: + combined_embeddings = tensor_parallel.scatter_to_sequence_parallel_region(combined_embeddings) + combined_embeddings = combined_embeddings.contiguous() + else: + combined_embeddings = None + from .rope_utils import get_rope_index + + position_ids, _ = get_rope_index( + input_ids, image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw, attention_mask=attention_mask + ) + + output = self.language_model( + input_ids=None, + position_ids=position_ids, # None in encoder + attention_mask=attention_mask, # None in encoder + decoder_input=combined_embeddings, # only not None in the first decoder PP stage + labels=labels, # only not None in the last decoder PP stage + # inference_params=inference_params, # currently always None + packed_seq_params=packed_seq_params, # currently always None + **(extra_block_kwargs or {}), + ) + + return output diff --git a/e2c/verl/verl/models/mcore/qwen2_5_vl/rope_utils.py b/e2c/verl/verl/models/mcore/qwen2_5_vl/rope_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fadc74daabe852f9e4561fe9981534815e5a148d --- /dev/null +++ b/e2c/verl/verl/models/mcore/qwen2_5_vl/rope_utils.py @@ -0,0 +1,266 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024 Alibaba PAI Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import logging +from typing import Optional + +import torch +from megatron.core.models.common.embeddings.rope_utils import * +from megatron.core.models.common.embeddings.rope_utils import _apply_rotary_pos_emb_bshd +from torch import Tensor + +logger = logging.getLogger(__name__) + + +# Slightly modified from Qwen2VLForConditionalGeneration.get_rope_index +def get_rope_index( + input_ids: Optional[torch.LongTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + second_per_grid_ts: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, +): + """ + Calculate the 3D rope index based on image and video's temporal, height and width in LLM. + + Explanation: + + Each embedding sequence contains vision embedding and text embedding or just contains text embedding. + + For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs. + + Examples: + + input_ids: [T T T T T], here T is for text. + temporal position_ids: [0, 1, 2, 3, 4] + height position_ids: [0, 1, 2, 3, 4] + width position_ids: [0, 1, 2, 3, 4] + + For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part + and 1D rotary position embedding for text part. + + Examples: + + Temporal (Time): 3 patches, representing different segments of the video in time. + Height: 2 patches, dividing each frame vertically. + Width: 2 patches, dividing each frame horizontally. + We also have some important parameters: + fps (Frames Per Second): The video's frame rate, set to 1. This means one frame is processed each + second. + tokens_per_second: This is a crucial parameter. It dictates how many "time-steps" or "temporal + tokens" are conceptually packed into a one-second interval of the video. + In this case, we have 25 tokens per second. So each second of the video will be + represented with 25 separate time points. It essentially defines the temporal + granularity. + temporal_patch_size: The number of frames that compose one temporal patch. Here, it's 2 frames. + interval: The step size for the temporal position IDs, calculated as tokens_per_second * + temporal_patch_size / fps. In this case, 25 * 2 / 1 = 50. This means that each temporal patch will be + have a difference of 50 in the temporal position IDs. + input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. + vision temporal position_ids: [0, 0, 0, 0, 50, 50, 50, 50, 100, 100, 100, 100] + vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] + vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + text temporal position_ids: [101, 102, 103, 104, 105] + text height position_ids: [101, 102, 103, 104, 105] + text width position_ids: [101, 102, 103, 104, 105] + Here we calculate the text start position_ids as the max vision position_ids plus 1. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): + The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) + mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) + """ + spatial_merge_size = 2 + tokens_per_second = 2 + image_token_id = 151655 + video_token_id = 151656 + vision_start_token_id = 151652 + mrope_position_deltas = [] + if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + attention_mask = attention_mask.to(total_input_ids.device) + for i, input_ids in enumerate(total_input_ids): + input_ids = input_ids[attention_mask[i] == 1] + image_nums, video_nums = 0, 0 + vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1) + vision_tokens = input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + second_per_grid_t = 0 + image_index += 1 + remain_images -= 1 + ed = ed_image + + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + if second_per_grid_ts is not None: + second_per_grid_t = second_per_grid_ts[video_index] + else: + second_per_grid_t = 1.0 + video_index += 1 + remain_videos -= 1 + ed = ed_video + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + range_tensor = torch.arange(llm_grid_t).view(-1, 1) + expanded_range = range_tensor.expand(-1, llm_grid_h * llm_grid_w) + + time_tensor = expanded_range * second_per_grid_t * tokens_per_second + + time_tensor_long = time_tensor.long() + t_index = time_tensor_long.flatten() + + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + text_len = len(input_tokens) - st + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) + mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) + mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) + max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas + + +def apply_rotary_pos_emb_thd_absolute( + t: Tensor, cu_seqlens: Tensor, freqs: Tensor, rotary_interleaved: bool = False +) -> Tensor: + """A baseline implementation of applying RoPE for `thd` format. + + Args: + t (Tensor): Input tensor T is of shape [t, h, d] + cu_seqlens(Tensor): Cumulative sum of sequence lengths in a batch for `t`, + with shape [b + 1] and dtype torch.int32. + freqs (Tensor): Rotary Positional embedding tensor freq is of shape [max_s, 1, 1, d] + + Returns: + Tensor: Shape [t, h, d]. The input tensor after applying RoPE. + """ + return _apply_rotary_pos_emb_bshd(t[:, None], freqs, rotary_interleaved=rotary_interleaved).squeeze(1) + + +def apply_rotary_pos_emb_absolute( + t: Tensor, + freqs: Tensor, + config: TransformerConfig, + cu_seqlens: Optional[Tensor] = None, +): + """ + Reroute to the appropriate apply_rotary_pos_emb function depending on + bshd (conventional) / thd (packed seq) format + + In Qwen2-VL, the shape of freqs is (seq_length, bs, 1, 2 * dim) instead of [max_seqlen, 1, 1, 2 * dim] + """ + + if config.apply_rope_fusion: + if cu_seqlens is None: + # NOTE: TE backends do not support mRoPE in bshd format when bs > 1 + if freqs.shape[1] > 1: + return _apply_rotary_pos_emb_bshd(t, freqs, rotary_interleaved=config.rotary_interleaved) + else: + return fused_apply_rotary_pos_emb(t, freqs) + else: + # NOTE: as expected, thd format can use bshd + return fused_apply_rotary_pos_emb(t[:, None], freqs).squeeze(1) + else: + if cu_seqlens is None: + return _apply_rotary_pos_emb_bshd(t, freqs, rotary_interleaved=config.rotary_interleaved) + else: + return apply_rotary_pos_emb_thd_absolute(t, cu_seqlens, freqs, rotary_interleaved=config.rotary_interleaved) diff --git a/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_config.py b/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_config.py new file mode 100644 index 0000000000000000000000000000000000000000..0631c90f61605f2ed0d659c8836f01c451e694a6 --- /dev/null +++ b/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_config.py @@ -0,0 +1,85 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024 Alibaba PAI Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from megatron.core import parallel_state +from megatron.core.transformer import TransformerConfig + + +def get_vision_model_config(config: TransformerConfig) -> TransformerConfig: + # Given a Transformer Config from decoder, build vision encoder config + # diff: out_hidden_size & intermediate_size + + # mlp: hidden_size -> intermediate_size -> embed_dim, silu + # NOTE: here we provide a workaround to solve the wrong layer amount when VPP of decoder is on + if config.num_layers in [28, 36]: + config.ffn_hidden_size = 3420 + else: + config.ffn_hidden_size = 3456 + + if parallel_state.get_virtual_pipeline_model_parallel_world_size() is not None: + config.num_layers = 32 * parallel_state.get_virtual_pipeline_model_parallel_world_size() # depth + else: + config.num_layers = 32 # depth + config.num_attention_heads = 16 # num_heads + config.add_bias_linear = True # all nn.Linear has bias (MLP, attn) + config.add_qkv_bias = True # qkv_proj in attn has bias + config.hidden_size = 1280 # hidden_size + config.hidden_dropout = 0.0 + config.attention_dropout = 0.0 + + # config.gated_linear_unit = False # no gated + # config.activation_func = quick_gelu # hidden_act + config.kv_channels = config.hidden_size // config.num_attention_heads + config.num_query_groups = config.num_attention_heads # no GQA + config.layernorm_zero_centered_gamma = False # False + config.apply_query_key_layer_scaling = False # factor=math.sqrt(head_dim) + config.bias_activation_fusion = False # no swiglu, set false + config.bias_dropout_fusion = False # no dropout, set false + config.attention_softmax_in_fp32 = True # use True + # config.normalization = 'LayerNorm' # use RMSNorm + config.seq_length = 1 + + config.tp_comm_overlap = False + config.sequence_parallel = False + config.temporal_patch_size = 2 + config.patch_size = 14 + config.in_channels = 3 + config.spatial_merge_size = 2 + + config.fullatt_block_indexes = [7, 15, 23, 31] + config._qwen2_5_vl_window_size = 112 + return config + + +def get_vision_projection_config( + config: TransformerConfig, embed_dim: int, spatial_merge_size: int +) -> TransformerConfig: + # merger: + # context_dim = hidden_size * merge_size**2 + # out_hidden_size = hidden_size + # context_dim -> context_dim -> out_hidden_size + # MLP: + # input_size -> ffn_hidden_size -> hidden_size + # spec: LN -> Linear(bias=True) -> GELU -> Linear(bias=True) + config.gated_linear_unit = False + config.bias_activation_fusion = False + config.add_bias_linear = True + config.ffn_hidden_size = embed_dim * (spatial_merge_size**2) + config.activation_func = torch.nn.functional.gelu + config.tp_comm_overlap = False + config.sequence_parallel = False + return config diff --git a/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_model.py b/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_model.py new file mode 100644 index 0000000000000000000000000000000000000000..06b4fd328064a1f50b32a7009aec8ecef573656e --- /dev/null +++ b/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_model.py @@ -0,0 +1,309 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024 Alibaba PAI Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +from megatron.core import InferenceParams +from megatron.core.models.common.vision_module.vision_module import VisionModule +from megatron.core.models.vision.multimodal_projector import MultimodalProjector +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.enums import ModelType +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig +from torch import nn +from torch.nn import functional as F + +from .vision_transformer_block import Qwen2_5VisionTransformerBlock as TransformerBlock + + +# copied from https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +class PatchEmbed(nn.Module): + def __init__( + self, + patch_size: int = 14, + temporal_patch_size: int = 2, + in_channels: int = 3, + embed_dim: int = 1152, + ) -> None: + super().__init__() + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.in_channels = in_channels + self.embed_dim = embed_dim + + kernel_size = [temporal_patch_size, patch_size, patch_size] + self.proj = nn.Conv3d(in_channels, embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=False) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + target_dtype = self.proj.weight.dtype + hidden_states = hidden_states.view( + -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size + ) + hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) + return hidden_states + + +# copied from https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +class VisionRotaryEmbedding(nn.Module): + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, seqlen: int) -> torch.Tensor: + seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.outer(seq, self.inv_freq) + return freqs.float() + + +class Qwen2_5VisionModel(VisionModule): + """Qwen2.5 ViT vision model. + + Args: + transformer_config (TransformerConfig): Transformer config. + transformer_layer_spec (ModuleSpec): Specifies module to use for transformer layers. + ln_pre_impl (ModuleSpec or type): Specifies the layer norm type to use for ln_pre. + add_class_token (bool, optional): Include a class token. Defaults to True. + class_token_len (int): Class token length. Defaults to 1 but 8 may be faster. + patch_dim (int): Image patch size. + img_h (int): Input image height. + img_w (int): Input image width. + """ + + def __init__( + self, + transformer_config: TransformerConfig, + transformer_layer_spec: ModuleSpec, + projection_config: TransformerConfig, + projection_layer_spec: ModuleSpec, + projection_type: str = "mlp", + pre_process: bool = True, + post_process: bool = False, + ) -> None: + super().__init__(config=transformer_config) + + self.spatial_merge_size = transformer_config.spatial_merge_size + + embed_dim = transformer_config.hidden_size + num_heads = transformer_config.num_attention_heads + temporal_patch_size = transformer_config.temporal_patch_size + patch_size = transformer_config.patch_size + in_channels = transformer_config.in_channels + + self.patch_size = transformer_config.patch_size + self.fullatt_block_indexes = transformer_config.fullatt_block_indexes + self.window_size = transformer_config._qwen2_5_vl_window_size + self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size + + self.max_sequence_length = transformer_config.seq_length + self.patch_embed = PatchEmbed( + patch_size=patch_size, + temporal_patch_size=temporal_patch_size, + in_channels=in_channels, + embed_dim=embed_dim, + ) + + head_dim = embed_dim // num_heads + self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2) + + self.model_type = ModelType.encoder_or_decoder + self.pre_process = pre_process + self.post_process = post_process + + # Transformer layers. + # TODO: Follow-up changes will make pre and post_process configurable. They are needed for supporting + # pipeline parallelism. + # NOTE: a final layer norm and/or linear layer present in some implementations are omitted here. + self.decoder = TransformerBlock( + config=transformer_config, + spec=transformer_layer_spec, + pre_process=self.pre_process, + post_process=self.post_process, + post_layer_norm=True, + ) + + self.merge_hidden_size = projection_config.ffn_hidden_size + self.square_merge_size = self.merge_hidden_size // embed_dim + + if self.post_process: + self.projection = MultimodalProjector( + projection_config, projection_layer_spec, projection_type, projection_config.ffn_hidden_size + ) + else: + self.projection = None + + self.input_tensor = None + + def set_input_tensor(self, input_tensor: torch.Tensor) -> None: + """Sets input tensor to the model. + + Args: + input_tensor (Tensor): Sets the input tensor for the model. + """ + if self.pre_process: # always True + self.input_tensor = input_tensor + else: + raise NotImplementedError() + + def rot_pos_emb(self, grid_thw): + pos_ids = [] + for t, h, w in grid_thw: + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + hpos_ids = hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + hpos_ids = hpos_ids.permute(0, 2, 1, 3) + hpos_ids = hpos_ids.flatten() + + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + wpos_ids = wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + wpos_ids = wpos_ids.permute(0, 2, 1, 3) + wpos_ids = wpos_ids.flatten() + pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) + pos_ids = torch.cat(pos_ids, dim=0).to(grid_thw.device) + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size).to(grid_thw.device) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb + + def get_window_index(self, grid_thw): + window_index: list = [] + cu_window_seqlens: list = [0] + window_index_id = 0 + vit_merger_window_size = self.window_size // self.spatial_merge_size // self.patch_size + + for grid_t, grid_h, grid_w in grid_thw: + llm_grid_h, llm_grid_w = ( + grid_h // self.spatial_merge_size, + grid_w // self.spatial_merge_size, + ) + index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w) + pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size + pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size + num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size + num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size + index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100) + index_padded = index_padded.reshape( + grid_t, + num_windows_h, + vit_merger_window_size, + num_windows_w, + vit_merger_window_size, + ) + index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape( + grid_t, + num_windows_h * num_windows_w, + vit_merger_window_size, + vit_merger_window_size, + ) + seqlens = (index_padded != -100).sum([2, 3]).reshape(-1) + index_padded = index_padded.reshape(-1) + index_new = index_padded[index_padded != -100] + window_index.append(index_new + window_index_id) + cu_seqlens_tmp = seqlens.cumsum(0) * self.spatial_merge_unit + cu_window_seqlens[-1] + cu_window_seqlens.extend(cu_seqlens_tmp.tolist()) + window_index_id += (grid_t * llm_grid_h * llm_grid_w).item() + window_index = torch.cat(window_index, dim=0) + + return window_index, cu_window_seqlens + + def forward( + self, + vision_data: Optional[torch.Tensor], + grid_thw: torch.Tensor, + inference_params: Optional[InferenceParams] = None, + extra_block_kwargs: dict = None, + ) -> torch.Tensor: + """Forward function of the Qwen2 Vision Model. This function passes the input tensors + through the embedding layer and then the transformer. + + Args: + x (torch.Tensor): input image/video data of shape [n_tokens, n_dims] + grid_thw (torch.Tensor): the size tensor indicates grid size of each image/frame + packed_seq_params (PackedSeqParams): parameters to build attention mask in the backend + + Returns: + x (torch.Tensor): output after final transformer block of shape [b, s, h]. + """ + assert grid_thw is not None + assert self.input_tensor is None + assert inference_params is None + + # Rotary positional embeddings (embedding is None for PP intermediate devices) + vision_data = self.patch_embed(vision_data) + window_index, cu_window_seqlens = self.get_window_index(grid_thw) + cu_window_seqlens = torch.tensor( + cu_window_seqlens, + device=vision_data.device, + dtype=torch.int32, + ) + cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens) + + seq_len, _ = vision_data.size() + vision_data = vision_data.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1) + vision_data = vision_data[window_index, :, :] + vision_data = vision_data.reshape(seq_len, 1, -1) + + rotary_pos_emb = self.rot_pos_emb(grid_thw) + rotary_pos_emb = rotary_pos_emb.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1) + rotary_pos_emb = rotary_pos_emb[window_index, :, :] + rotary_pos_emb = rotary_pos_emb.reshape(seq_len, 1, 1, -1).repeat(1, 1, 1, 2) + + hidden_states = self.decoder( + hidden_states=vision_data, + attention_mask=None, + inference_params=inference_params, + rotary_pos_emb=rotary_pos_emb, + packed_seq_params=self.build_packed_seq_params(None, cu_window_seqlens), + packed_seq_params_full=self.build_packed_seq_params(grid_thw), + fullatt_block_indexes=self.fullatt_block_indexes, + **(extra_block_kwargs or {}), + ) + + hidden_states = self.projection(hidden_states.view(-1, self.merge_hidden_size)) + reverse_indices = torch.argsort(window_index) + return hidden_states[reverse_indices, :] + + def build_packed_seq_params( + self, + grid_thw: Optional[torch.Tensor], + cu_seqlens: Optional[torch.Tensor] = None, + ) -> PackedSeqParams: + # NOTE: each frame is a sequence (rather than each grid) + if grid_thw is not None: + seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]) + cu_seqlens = seqlens.cumsum(dim=0) + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0).int() + else: + seqlens = cu_seqlens[1:] - cu_seqlens[:-1] + + max_seqlen_q = seqlens.max() + return PackedSeqParams( + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + qkv_format="thd", + max_seqlen_q=max_seqlen_q, + max_seqlen_kv=max_seqlen_q, + ) diff --git a/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_transformer_block.py b/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_transformer_block.py new file mode 100644 index 0000000000000000000000000000000000000000..8f765a0ff632f65771d1b1d19a4b0f052ee6ec37 --- /dev/null +++ b/e2c/verl/verl/models/mcore/qwen2_5_vl/vision_transformer_block.py @@ -0,0 +1,265 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024 Alibaba PAI Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from megatron.core.transformer.transformer_block import * + + +class Qwen2_5VisionTransformerBlock(TransformerBlock): + def _checkpointed_forward( + self, + hidden_states: Tensor, + attention_mask: Tensor, + context: Tensor, + context_mask: Tensor, + rotary_pos_emb: Tensor, + attention_bias: Tensor, + packed_seq_params: PackedSeqParams, + packed_seq_params_full: PackedSeqParams, + fullatt_block_indexes, + ): + """Forward method with activation checkpointing.""" + + def custom(start: int, end: int): + def custom_forward(hidden_states, attention_mask, context, context_mask, rotary_pos_emb): + for index in range(start, end): + if index in fullatt_block_indexes: + packed_seq_params_now = packed_seq_params_full + else: + packed_seq_params_now = packed_seq_params + layer = self._get_layer(index) + hidden_states, context = layer( + hidden_states=hidden_states, + attention_mask=attention_mask, + context=context, + context_mask=context_mask, + rotary_pos_emb=rotary_pos_emb, + attention_bias=attention_bias, + inference_context=None, + packed_seq_params=packed_seq_params_now, + ) + return hidden_states, context + + return custom_forward + + def checkpoint_handler(forward_func): + """Determines whether to use the `te_checkpoint` or `tensor_parallel.checkpoint`""" + if self.config.fp8: + return te_checkpoint( + forward_func, + self.config.distribute_saved_activations, + tensor_parallel.random.get_cuda_rng_tracker, + parallel_state.get_tensor_model_parallel_group(), + hidden_states, + attention_mask, + context, + context_mask, + rotary_pos_emb, + ) + else: + return tensor_parallel.checkpoint( + forward_func, + self.config.distribute_saved_activations, + hidden_states, + attention_mask, + context, + context_mask, + rotary_pos_emb, + ) + + if self.config.recompute_method == "uniform": + # Uniformly divide the total number of Transformer layers and checkpoint + # the input activation of each divided chunk. + # A method to further reduce memory usage reducing checkpoints. + layer_idx = 0 + while layer_idx < self.num_layers_per_pipeline_rank: + hidden_states, context = checkpoint_handler( + custom(layer_idx, layer_idx + self.config.recompute_num_layers) + ) + + layer_idx += self.config.recompute_num_layers + + elif self.config.recompute_method == "block": + # Checkpoint the input activation of only a set number of individual + # Transformer layers and skip the rest. + # A method fully use the device memory removing redundant re-computation. + recompute_skip_num_layers = 0 + for layer_idx in range(self.num_layers_per_pipeline_rank): + # Skip recomputation when input grad computation is not needed. + # Need to have at least one input tensor with gradient computation + # for re-enterant autograd engine. + if self.config.fp8 and not hidden_states.requires_grad: + recompute_skip_num_layers += 1 + if ( + layer_idx >= recompute_skip_num_layers + and layer_idx < self.config.recompute_num_layers + recompute_skip_num_layers + ): + hidden_states, context = checkpoint_handler(custom(layer_idx, layer_idx + 1)) + else: + hidden_states, context = custom(layer_idx, layer_idx + 1)( + hidden_states, attention_mask, context, context_mask, rotary_pos_emb + ) + else: + raise ValueError("Invalid activation recompute method.") + + return hidden_states + + def forward( + self, + hidden_states: Union[Tensor, WrappedTensor], + attention_mask: Optional[Tensor], + context: Optional[Tensor] = None, + context_mask: Optional[Tensor] = None, + rotary_pos_emb: Optional[Tensor] = None, + rotary_pos_cos: Optional[Tensor] = None, + rotary_pos_sin: Optional[Tensor] = None, + attention_bias: Optional[Tensor] = None, + inference_context: Optional[BaseInferenceContext] = None, + packed_seq_params: Optional[PackedSeqParams] = None, + sequence_len_offset: Optional[Tensor] = None, + packed_seq_params_full: PackedSeqParams = None, + fullatt_block_indexes=None, + *, + inference_params: Optional[BaseInferenceContext] = None, + ): + """ + Perform the forward pass through the transformer block. + + This method handles the core computation of the transformer, including + self-attention, optional cross-attention, and feed-forward operations. + + Args: + hidden_states (Union[Tensor, WrappedTensor]): Input tensor of shape [s, b, h] + where s is the sequence length, b is the batch size, and h is the hidden size. + Can be passed as a WrappedTensor during inference to avoid an obsolete + reference in the calling function. + attention_mask (Tensor): Boolean tensor of shape [1, 1, s, s] for masking + self-attention. + context (Tensor, optional): Context tensor for cross-attention. + context_mask (Tensor, optional): Mask for cross-attention context + rotary_pos_emb (Tensor, optional): Rotary positional embeddings. + attention_bias (Tensor): Bias tensor for Q * K.T of shape in shape broadcastable + to [b, num_head, sq, skv], e.g. [1, 1, sq, skv]. + Used as an alternative to apply attention mask for TE cuDNN attention. + inference_context (BaseInferenceContext, optional): Parameters for inference-time + optimizations. + packed_seq_params (PackedSeqParams, optional): Parameters for packed sequence + processing. + + Returns: + Union[Tensor, Tuple[Tensor, Tensor]]: The output hidden states tensor of shape + [s, b, h], and optionally the updated context tensor if cross-attention is used. + """ + + inference_context = deprecate_inference_params(inference_context, inference_params) + + # Delete the obsolete reference to the initial input tensor if necessary + if isinstance(hidden_states, WrappedTensor): + hidden_states = hidden_states.unwrap() + + if not self.pre_process: + # See set_input_tensor() + hidden_states = self.input_tensor + + # Update the inference parameters with the current batch size in case it is variable + if inference_context and not self.training: + inference_context.current_batch_size = hidden_states.size(1) + + # Viewless tensor. + # - We only need to create a viewless tensor in the case of micro batch + # size (mbs) == 1, since in this case, 'hidden_states.transpose()' + # above creates a view tensor, and '.contiguous()' is a pass-through. + # For mbs >= 2, '.contiguous()' creates a new tensor, eliminating + # the need to make it viewless. + # + # However, we don't explicitly check mbs == 1 here because + # make_viewless_tensor() has negligible overhead when its input + # is already viewless. + # + # - For the 'else' case above, calling make_viewless_tensor() here is + # likely redundant, since p2p_communication.py (likely originator) + # already creates viewless tensors. That said, make_viewless_tensor() + # is called here to be future-proof and corner-case-proof. + hidden_states = make_viewless_tensor(inp=hidden_states, requires_grad=True, keep_graph=True) + + if self.config.sequence_parallel: + rng_context = tensor_parallel.get_cuda_rng_tracker().fork() + else: + rng_context = nullcontext() + + # If fp8_recipe is delayed, wrap the entire pass with get_fp8_context(), + # otherwise do nothing extra at the outer level + # if we are using other fp8 recipes, then the context manager enter&exit are free + # we can wrap fp8_context within the for loop over layers, so that we can fine-grained + # control which layer will be fp8 or bf16 + use_outer_fp8_context = self.config.fp8 and self.config.fp8_recipe == Fp8Recipe.delayed + use_inner_fp8_context = self.config.fp8 and self.config.fp8_recipe != Fp8Recipe.delayed + outer_fp8_context = get_fp8_context(self.config) if use_outer_fp8_context else nullcontext() + + with rng_context, outer_fp8_context: + # Forward pass. + if self.config.recompute_granularity == "full" and self.training: + hidden_states = self._checkpointed_forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + context=context, + context_mask=context_mask, + rotary_pos_emb=rotary_pos_emb, + attention_bias=attention_bias, + packed_seq_params=packed_seq_params, + packed_seq_params_full=packed_seq_params_full, + fullatt_block_indexes=fullatt_block_indexes, + ) + else: + for l_no, layer in enumerate(self.layers): + inner_fp8_context = ( + get_fp8_context(self.config, layer.layer_number - 1) if use_inner_fp8_context else nullcontext() + ) + if l_no in fullatt_block_indexes: + packed_seq_params_now = packed_seq_params_full + else: + packed_seq_params_now = packed_seq_params + with self.offload_context, inner_fp8_context: + hidden_states, context = layer( + hidden_states=hidden_states, + attention_mask=attention_mask, + context=context, + context_mask=context_mask, + rotary_pos_emb=rotary_pos_emb, + rotary_pos_cos=rotary_pos_cos, + rotary_pos_sin=rotary_pos_sin, + attention_bias=attention_bias, + inference_context=inference_context, + packed_seq_params=packed_seq_params_now, + sequence_len_offset=sequence_len_offset, + ) + + if ( + torch.is_grad_enabled() + and self.config.cpu_offloading + and self.group_prefetch_offload_commit_async is not None + ): + hidden_states = self.group_prefetch_offload_commit_async(hidden_states) + + # Final layer norm. + if self.final_layernorm is not None: + hidden_states = self.final_layernorm(hidden_states) + # TENorm produces a "viewed" tensor. This will result in schedule.py's + # deallocate_output_tensor() throwing an error, so a viewless tensor is + # created to prevent this. + hidden_states = make_viewless_tensor(inp=hidden_states, requires_grad=True, keep_graph=True) + + return hidden_states diff --git a/e2c/verl/verl/models/mcore/readme.md b/e2c/verl/verl/models/mcore/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..606dcf1897a53f0a2b5194b8d80104e70d22596a --- /dev/null +++ b/e2c/verl/verl/models/mcore/readme.md @@ -0,0 +1,99 @@ +# verl Megatron-Core Models +The earlier versions of verl use `Megatron-LM` 0.4 and workaround huggingface model classes. To better use the latest features and speedup of modern Megatron, we are migrating to `Megatron-Core`(mcore), and use the recommended `GPTModel` class for all language models. With mcore `GPTModel`, we can use the latest features like `context parallel`, `expert parallel`, `dist_checkpointing`, etc. and we can update mcore with little effort in the future for new features. + +The migration has been successful with the help of the mcore team and the community. What we have done is: +1. update `Megatron` version to `0.11.0` +2. migrate `LlamaForCausalLM` and `Qwen2ForCausalLM` to mcore `GPTModel` +3. support sequence packing/thd format. +4. support `tensor parallel`, `pipeline parallel`, `sequence parallel`, `virtual pipeline parallel`, `context parallel`. +5. support the mcore `dist_checkpointing` feature and a basic offline weighs conversion script from huggingface to mcore `dist_checkpointing` format. + +We are working on the following features: +- support `Qwen2MoeForCausalLM` +- support `MixtralForCausalLM` +- support `DeepseekV3ForCausalLM` +- support `expert parallel` + +Features we invite the community to contribute: +- better scripts for offline weights conversion from huggingface to mcore `dist_checkpointing` format. + - conversion of large models with multiple GPUs + - conversion of large models with single GPU +- refactor the `megatron_checkpoint_manager.py` by `dist_checkpointing` format. +- support llama4 +- support qwen2.5-vl + +To track the progress of verl mcore integration, please refer to the [mcore integration issue](https://github.com/volcengine/verl/issues/1033). + +## How things work now +To engage the community in contributing, here are the key steps in our mcore integration process and features under development. + +The huggingface `transformers` is the de facto standard of model zoo while mcore is good at computation efficiency. The main challenge is conversion between the two. +main steps: +1. modelling the huggingface model with mcore `GPTModel` + - a. convert the huggingface config to mcore `TransformerConfig` + - b. init the mcore `GPTModel` with the converted config + - c. load the huggingface model weights to the `GPTModel` +2. online weight conversion from mcore to huggingface (due to the rollout engine `vLLM` is using huggingface format) + - a. bridge the gap between mcore and huggingface weights format and name mapping + - b. online resharding the mcore weights to rollout engine + - this part is very complicated with multiple parallel strategies composition between mcore and rollout engine +3. support the mcore features in verl + - a. support `tensor parallel`, `pipeline parallel`, `sequence parallel`, `virtual pipeline parallel`, `context parallel` + - b. support recompute and other mcore speed up features + +4. checkpointing + - a. support recovering the verl training. + - b. support exporting the mcore checkpoint to huggingface format, for downstream inference. + +### Modelling the huggingface model with mcore `GPTModel` +The first step is to convert huggingface config to mcore `TransformerConfig` and init the mcore `GPTModel` with the converted config. See code in `verl/models/mcore/config_converter.py` and `verl/verl/models/mcore/models/model_initializer.py`. The corresponding model forward code is in `verl/verl/models/mcore/models/model_forward.py`. + +There are two ways of loading the huggingface model weights to the `GPTModel` +1. Runtime loading + - every rank loads the entire huggingface model weights and then shard and convert to mcore weights. + - speed is slow and memory consumption is high. + - this way is deprecated and will not support new models. +2. Offline loading + - use offline script to convert the huggingface model weights to mcore weights and save with mcore `dist_checkpointing` format. + - online loading and sharding is automatically done by mcore `dist_checkpointing` format. The speed is fast and memory consumption is low. + - the offline script is in `verl/scripts/converter_hf_to_mcore.py`. + +### online weight conversion from mcore to huggingface +See function `convert_megatron_model_to_transformers_model` in `verl/utils/megatron_utils.py` for the details. + +It should be refatored for extensibility and better performance. + +### support the mcore features in verl +Most of the features of `GPTModel` is out-of-the-box supported in verl through changing the `TransformerConfig`, except those about parallel strategies, such as `expert parallel`. +Features about parallel strategies should be supported with changes about the online weights conversion(especially the resharding part) and verl work dispatching. + +### checkpointing +The existing checkpointing code is in `verl/utils/checkpoint/megatron_checkpoint_manager.py`. And the script to convert checkpoint to huggingface format is in `verl/scripts/model_merger`. + +The existing checkpoint format simply saves every rank's weights and optimizer states. It should be refactored by `dist_checkpointing` format. + + +## How to support new models +1. make sure the model is supported by vLLM +2. modelling the huggingface model with mcore `GPTModel` (The [Pai-Megatron-Path](https://github.com/alibaba/Pai-Megatron-Patch/tree/main) is a good reference) + - a. convert the huggingface config to mcore `TransformerConfig` + - b. init the mcore `GPTModel` with the converted config + - c. load the huggingface model weights to the `GPTModel` + - d. for VLM the interface might be different, it is ok to add a new model class with GPTModel as its module. +3. offline weights conversion from huggingface to mcore `dist_checkpointing` format +4. support online weights conversion from mcore to huggingface + - it is recommended to initialize a vLLM model with the converted mcore weights, and then test if the generating sequence is correct. + + +## How to scale up to larger models like deepseek-v3 or other 100B+ models +The greatest challenge for scaling up to larger models is the memory consumption. + +The necessary features under development for scaling up are +1. Training engine part + - expert parallel +2. Rollout engine part + - pipeline parallel + - expert parallel + - more efficient and general weight resharding and loading +3. Offline weights conversion + - support weights larger than single GPU memory diff --git a/e2c/verl/verl/models/mcore/registry.py b/e2c/verl/verl/models/mcore/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..2fafe5425810d5bbbeff5cd5e1b0822171a10545 --- /dev/null +++ b/e2c/verl/verl/models/mcore/registry.py @@ -0,0 +1,246 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Registry module for model architecture components. +""" + +from enum import Enum +from typing import Callable + +import torch +import torch.nn as nn + +from .config_converter import ( + PretrainedConfig, + TransformerConfig, + hf_to_mcore_config_dense, + hf_to_mcore_config_dpskv3, + hf_to_mcore_config_llama4, + hf_to_mcore_config_mixtral, + hf_to_mcore_config_qwen2_5_vl, + hf_to_mcore_config_qwen2moe, + hf_to_mcore_config_qwen3moe, +) +from .model_forward import ( + gptmodel_forward, + gptmodel_forward_qwen2_5_vl, +) +from .model_forward_fused import ( + fused_forward_gptmodel, + fused_forward_qwen2_5_vl, +) +from .model_initializer import ( + BaseModelInitializer, + DeepseekV3Model, + DenseModel, + MixtralModel, + Qwen2MoEModel, + Qwen3MoEModel, + Qwen25VLModel, +) +from .weight_converter import ( + McoreToHFWeightConverterDense, + McoreToHFWeightConverterDpskv3, + McoreToHFWeightConverterMixtral, + McoreToHFWeightConverterQwen2_5_VL, + McoreToHFWeightConverterQwen2Moe, + McoreToHFWeightConverterQwen3Moe, +) + + +class SupportedModel(Enum): + LLAMA = "LlamaForCausalLM" # tested + QWEN2 = "Qwen2ForCausalLM" # tested + QWEN2_MOE = "Qwen2MoeForCausalLM" # pending + DEEPSEEK_V3 = "DeepseekV3ForCausalLM" # not tested + MIXTRAL = "MixtralForCausalLM" # tested + QWEN2_5_VL = "Qwen2_5_VLForConditionalGeneration" # not supported + LLAMA4 = "Llama4ForConditionalGeneration" # not tested + QWEN3 = "Qwen3ForCausalLM" # tested + QWEN3_MOE = "Qwen3MoeForCausalLM" # tested + GLM4_MOE = "Glm4MoeForCausalLM" + + QWEN3_TOKEN_CLASSIFICATION = "Qwen3ForTokenClassification" + + +# Registry for model configuration converters +MODEL_CONFIG_CONVERTER_REGISTRY: dict[SupportedModel, Callable[[PretrainedConfig, torch.dtype], TransformerConfig]] = { + SupportedModel.LLAMA: hf_to_mcore_config_dense, + SupportedModel.QWEN2: hf_to_mcore_config_dense, + SupportedModel.QWEN2_MOE: hf_to_mcore_config_qwen2moe, + SupportedModel.DEEPSEEK_V3: hf_to_mcore_config_dpskv3, + SupportedModel.MIXTRAL: hf_to_mcore_config_mixtral, + SupportedModel.QWEN2_5_VL: hf_to_mcore_config_qwen2_5_vl, + SupportedModel.LLAMA4: hf_to_mcore_config_llama4, + SupportedModel.QWEN3: hf_to_mcore_config_dense, + SupportedModel.QWEN3_MOE: hf_to_mcore_config_qwen3moe, + SupportedModel.QWEN2_5_VL: hf_to_mcore_config_qwen2_5_vl, + SupportedModel.QWEN3_TOKEN_CLASSIFICATION: hf_to_mcore_config_dense, +} + +# Registry for model initializers +MODEL_INITIALIZER_REGISTRY: dict[SupportedModel, type[BaseModelInitializer]] = { + SupportedModel.LLAMA: DenseModel, + SupportedModel.QWEN2: DenseModel, + SupportedModel.QWEN2_MOE: Qwen2MoEModel, + SupportedModel.MIXTRAL: MixtralModel, + SupportedModel.DEEPSEEK_V3: DeepseekV3Model, + SupportedModel.QWEN2_5_VL: Qwen25VLModel, + SupportedModel.LLAMA4: DenseModel, + SupportedModel.QWEN3: DenseModel, + SupportedModel.QWEN3_MOE: Qwen3MoEModel, + SupportedModel.QWEN2_5_VL: Qwen25VLModel, + SupportedModel.QWEN3_TOKEN_CLASSIFICATION: DenseModel, +} + +# Registry for model forward functions +MODEL_FORWARD_REGISTRY: dict[SupportedModel, Callable] = { + SupportedModel.LLAMA: gptmodel_forward, + SupportedModel.QWEN2: gptmodel_forward, + SupportedModel.QWEN2_MOE: gptmodel_forward, + SupportedModel.MIXTRAL: gptmodel_forward, + SupportedModel.DEEPSEEK_V3: gptmodel_forward, + SupportedModel.QWEN2_5_VL: gptmodel_forward, + SupportedModel.LLAMA4: gptmodel_forward, + SupportedModel.QWEN3: gptmodel_forward, + SupportedModel.QWEN3_MOE: gptmodel_forward, + SupportedModel.QWEN2_5_VL: gptmodel_forward_qwen2_5_vl, + SupportedModel.DEEPSEEK_V3: gptmodel_forward, + SupportedModel.GLM4_MOE: gptmodel_forward, + SupportedModel.QWEN3_TOKEN_CLASSIFICATION: gptmodel_forward, +} + +# Registry for model forward functions +MODEL_FORWARD_FUSED_REGISTRY: dict[SupportedModel, Callable] = { + SupportedModel.LLAMA: fused_forward_gptmodel, + SupportedModel.QWEN2: fused_forward_gptmodel, + SupportedModel.QWEN2_MOE: fused_forward_gptmodel, + SupportedModel.MIXTRAL: fused_forward_gptmodel, + SupportedModel.DEEPSEEK_V3: fused_forward_gptmodel, + SupportedModel.QWEN2_5_VL: fused_forward_qwen2_5_vl, + SupportedModel.LLAMA4: fused_forward_gptmodel, + SupportedModel.QWEN3: fused_forward_gptmodel, + SupportedModel.QWEN3_MOE: fused_forward_gptmodel, + SupportedModel.QWEN2_5_VL: fused_forward_qwen2_5_vl, + SupportedModel.DEEPSEEK_V3: fused_forward_gptmodel, + SupportedModel.GLM4_MOE: fused_forward_gptmodel, +} + +# Registry for model weight converters +MODEL_WEIGHT_CONVERTER_REGISTRY: dict[SupportedModel, type] = { + SupportedModel.LLAMA: McoreToHFWeightConverterDense, + SupportedModel.QWEN2: McoreToHFWeightConverterDense, + SupportedModel.QWEN2_MOE: McoreToHFWeightConverterQwen2Moe, + SupportedModel.MIXTRAL: McoreToHFWeightConverterMixtral, + SupportedModel.DEEPSEEK_V3: McoreToHFWeightConverterDpskv3, + SupportedModel.QWEN3: McoreToHFWeightConverterDense, + SupportedModel.QWEN3_MOE: McoreToHFWeightConverterQwen3Moe, + SupportedModel.QWEN2_5_VL: McoreToHFWeightConverterQwen2_5_VL, + SupportedModel.QWEN3_TOKEN_CLASSIFICATION: McoreToHFWeightConverterDense, +} + + +def get_supported_model(model_type: str) -> SupportedModel: + try: + return SupportedModel(model_type) + except ValueError as err: + supported_models = [e.value for e in SupportedModel] + raise NotImplementedError( + f"Model Type: {model_type} not supported. Supported models: {supported_models}" + ) from err + + +def hf_to_mcore_config( + hf_config: PretrainedConfig, dtype: torch.dtype, **override_transformer_config_kwargs +) -> TransformerConfig: + """Convert huggingface PretrainedConfig to mcore TransformerConfig. + + Args: + hf_config: The huggingface PretrainedConfig. + dtype: The dtype of the model. + **override_transformer_config_kwargs: The kwargs to override the transformer config. + + Returns: + The mcore TransformerConfig. + """ + assert len(hf_config.architectures) == 1, "Only one architecture is supported for now" + model = get_supported_model(hf_config.architectures[0]) + return MODEL_CONFIG_CONVERTER_REGISTRY[model](hf_config, dtype, **override_transformer_config_kwargs) + + +def init_mcore_model( + tfconfig: TransformerConfig, + hf_config: PretrainedConfig, + pre_process: bool = True, + post_process: bool = None, + *, + share_embeddings_and_output_weights: bool = False, + value: bool = False, + **extra_kwargs, # may be used for vlm and moe +) -> nn.Module: + """ + Initialize a Mcore model. + + Args: + tfconfig: The transformer config. + hf_config: The HuggingFace config. + pre_process: Optional pre-processing function. + post_process: Optional post-processing function. + share_embeddings_and_output_weights: Whether to share embeddings and output weights. + value: Whether to use value. + **extra_kwargs: Additional keyword arguments. + + Returns: + The initialized model. + """ + assert len(hf_config.architectures) == 1, "Only one architecture is supported for now" + model = get_supported_model(hf_config.architectures[0]) + initializer_cls = MODEL_INITIALIZER_REGISTRY[model] + initializer = initializer_cls(tfconfig, hf_config) + return initializer.initialize( + pre_process=pre_process, + post_process=post_process, + share_embeddings_and_output_weights=share_embeddings_and_output_weights, + value=value, + **extra_kwargs, + ) + + +def get_mcore_forward_fn(hf_config: PretrainedConfig) -> Callable: + """ + Get the forward function for given model architecture. + """ + assert len(hf_config.architectures) == 1, "Only one architecture is supported for now" + model = get_supported_model(hf_config.architectures[0]) + return MODEL_FORWARD_REGISTRY[model] + + +def get_mcore_forward_fused_fn(hf_config: PretrainedConfig) -> Callable: + """ + Get the forward function for given model architecture. + """ + assert len(hf_config.architectures) == 1, "Only one architecture is supported for now" + model = get_supported_model(hf_config.architectures[0]) + return MODEL_FORWARD_FUSED_REGISTRY[model] + + +def get_mcore_weight_converter(hf_config: PretrainedConfig, dtype: torch.dtype) -> Callable: + """ + Get the weight converter for given model architecture. + """ + assert len(hf_config.architectures) == 1, "Only one architecture is supported for now" + model = get_supported_model(hf_config.architectures[0]) + tfconfig = hf_to_mcore_config(hf_config, dtype) + return MODEL_WEIGHT_CONVERTER_REGISTRY[model](hf_config, tfconfig) diff --git a/e2c/verl/verl/models/mcore/saver.py b/e2c/verl/verl/models/mcore/saver.py new file mode 100644 index 0000000000000000000000000000000000000000..2a954b2417cd5b8d09e88b9935e52eeb6ef5273a --- /dev/null +++ b/e2c/verl/verl/models/mcore/saver.py @@ -0,0 +1,497 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import torch +import torch.distributed as dist +from megatron.core import mpu +from megatron.core.distributed import DistributedDataParallel as LocalDDP +from megatron.core.transformer.module import Float16Module +from torch.nn.parallel import DistributedDataParallel as torchDDP + +from verl.utils.device import get_device_id, get_torch_device +from verl.utils.logger import print_rank_0 +from verl.utils.megatron_utils import unwrap_model + + +def _megatron_calc_global_rank( + tp_rank: int = 0, dp_rank: int = 0, pp_rank: int = 0, cp_rank: int = 0, ep_rank: int = 0 +): + """Calculate global rank with support for CP/EP parallelism""" + + # Get parallel sizes for each dimension + tp_size = mpu.get_tensor_model_parallel_world_size() + dp_size = mpu.get_data_parallel_world_size() + pp_size = mpu.get_pipeline_model_parallel_world_size() + cp_size = mpu.get_context_parallel_world_size() + # ep_size = mpu.get_expert_model_parallel_world_size() + + # Verify total GPU count matches (must be consistent with parallel_state.py) + total_size = tp_size * dp_size * pp_size * cp_size + assert total_size == torch.distributed.get_world_size(), ( + f"{tp_size}x{dp_size}x{pp_size}x{cp_size} != {torch.distributed.get_world_size()}" + ) + + # Core calculation logic (corresponds to RankGenerator order parameter) + # Assumes default order is "tp-cp-ep-dp-pp" + return ((pp_rank * dp_size + dp_rank) * cp_size + cp_rank) * tp_size + tp_rank + + +def _megatron_calc_layer_map(config): + """Calculate the mapping of global layer_idx to local layer_idx + Returns: + layer_map (Dict: int -> tuple(int, int, int)): + mapping from the global layer index to + a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) + """ + from megatron.core import mpu + + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + + layer_map = dict() + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + for pp_rank_idx in range(pp_size): + for virtual_pp_rank_idx in range(virtual_pp_size): + layer_offset = ( + virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + pp_rank_idx * num_layers_per_model + ) + for layer_idx in range(num_layers_per_model): + layer_map[layer_offset + layer_idx] = ( + pp_rank_idx, + virtual_pp_rank_idx, + layer_idx, + ) + return layer_map + + +def merge_megatron_ckpt_gptmodel(wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False): + """Merge sharded parameters of a Megatron module into a merged checkpoint. + + Args: + wrapped_models (list of megatron.core.distributed.DistributedDataParallel): + The local DDP wrapped megatron modules. + config (str or None): + HF config for model + dtype: model params type + is_value_model: if model is value model + tie_word_embeddings: tie_word_embeddings + Returns: + state_dict (dict): + The merged state_dict in rank 0, and an empty dictionary in other ranks. + """ + start_time = time.time() + + def _get_gpt_model(model): + return model + + dp_rank = mpu.get_data_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + pp_rank = mpu.get_pipeline_model_parallel_rank() + cp_rank = mpu.get_context_parallel_rank() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + mp_group = mpu.get_model_parallel_group() + + if dist.get_rank() == 0: + assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" + assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" + assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" + + if not isinstance(wrapped_models, list | tuple): + wrapped_models = list(wrapped_models) + + assert len(wrapped_models) == virtual_pp_size + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + models = [None] * len(wrapped_models) + + for i, wrapped_model in enumerate(wrapped_models): + models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) + assert len(models[i].decoder.layers) == num_layers_per_model, ( + "len model layers {} not equal to num_layers_per_model {}".format( + len(models[i].decoder.layers), num_layers_per_model + ) + ) + + state_dict = dict() + + def _get_cpu_tensor(tensor: torch.Tensor): + if tensor is None: + return None + if tensor.device == torch.device("cpu"): + return tensor.detach().clone() + return tensor.detach().cpu() + + def _broadcast_tensor(tensor, name, src_pp_rank) -> torch.Tensor: + """broadcast tensor across mp_group""" + nonlocal state_dict + nonlocal mp_group + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank, cp_rank=cp_rank) + + if torch.distributed.get_rank() == src_rank: + if tensor is None: + weight = None + tensor_shape = None + else: + weight = tensor + tensor_shape = weight.shape + else: + weight = None + tensor_shape = None + + obj_list = [tensor_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + tensor_shape = obj_list[0] + + if tensor_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tensor:[{name}] not exist, skip collect") + return + + if weight is None: + weight = torch.empty( + tensor_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + dist.broadcast(weight, src=src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + state_dict[name] = _get_cpu_tensor(weight) + + def _broadcast_tp_shard_tensor(tensor, name, src_pp_rank, concat_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + # tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank, cp_rank=cp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank, cp_rank=cp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=concat_dim) + if mutate_func is not None: + full_tensor = mutate_func(full_tensor) + state_dict[name] = full_tensor + + def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name, src_pp_rank) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + # tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank, cp_rank=cp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank, cp_rank=cp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=0) + intermediate_size_tp = config.intermediate_size // tp_size + gate_weight_list = [] + up_weight_list = [] + for i in range(tp_size): + gate_up_weight_tp = full_tensor[intermediate_size_tp * 2 * i : intermediate_size_tp * 2 * (i + 1)] + gate_weight_tp = gate_up_weight_tp[:intermediate_size_tp] + up_weight_tp = gate_up_weight_tp[intermediate_size_tp:] + gate_weight_list.append(gate_weight_tp) + up_weight_list.append(up_weight_tp) + + state_dict[gate_name] = torch.cat(gate_weight_list, dim=0) + state_dict[up_name] = torch.cat(up_weight_list, dim=0) + + def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, src_pp_rank): + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + # tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank, cp_rank=cp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{q_name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank, cp_rank=cp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=0) + q_weight_list = [] + k_weight_list = [] + v_weight_list = [] + hidden_size_per_head = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + + if config.num_key_value_heads >= tp_size: + q_size_tp = hidden_size_per_head * config.num_attention_heads // tp_size + kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size + total_size = q_size_tp + 2 * kv_size_tp + for i in range(tp_size): + num_query_groups_per_partition = wrapped_models[0].config.num_query_groups // tp_size + qkv_part = full_tensor[i * total_size : (i + 1) * total_size] + q_size_chunk = q_size_tp // num_query_groups_per_partition + kv_size_chunk = kv_size_tp // num_query_groups_per_partition + for qkv_part_chunk in qkv_part.chunk(num_query_groups_per_partition): + q_part = qkv_part_chunk[:q_size_chunk] + k_part = qkv_part_chunk[q_size_chunk : q_size_chunk + kv_size_chunk] + v_part = qkv_part_chunk[q_size_chunk + kv_size_chunk :] + q_weight_list.append(q_part) + k_weight_list.append(k_part) + v_weight_list.append(v_part) + else: + q_size_tp = hidden_size_per_head * config.num_attention_heads // tp_size + kv_size_tp = hidden_size_per_head + total_size = q_size_tp + 2 * kv_size_tp + for i in range(tp_size): + num_query_groups_per_partition = wrapped_models[0].config.num_query_groups // tp_size + qkv_part = full_tensor[i * total_size : (i + 1) * total_size] + q_size_chunk = q_size_tp // num_query_groups_per_partition + kv_size_chunk = kv_size_tp // num_query_groups_per_partition + for qkv_part_chunk in qkv_part.chunk(num_query_groups_per_partition): + q_part = qkv_part_chunk[:q_size_chunk] + k_part = qkv_part_chunk[q_size_chunk : q_size_chunk + kv_size_chunk] + v_part = qkv_part_chunk[q_size_chunk + kv_size_chunk :] + q_weight_list.append(q_part) + if i * config.num_key_value_heads % tp_size == 0: + k_weight_list.append(k_part) + v_weight_list.append(v_part) + + state_dict[q_name] = torch.cat(q_weight_list, dim=0) + state_dict[k_name] = torch.cat(k_weight_list, dim=0) + state_dict[v_name] = torch.cat(v_weight_list, dim=0) + + # empty cache before collecting weights + get_torch_device().empty_cache() + # Embeddings + # ------------------- + if dp_rank == 0 and cp_rank == 0: # models are identical across cp ranks + # Embeddings + # ------------------- + print_rank_0("collecting embeddings...") + gpt_model_module = _get_gpt_model(models[0]) + _broadcast_tp_shard_tensor( + gpt_model_module.embedding.word_embeddings.weight if pp_rank == 0 else None, + "model.embed_tokens.weight", + src_pp_rank=0, + ) + + # Transformer layers + # ------------------- + layer_map = _megatron_calc_layer_map(config) + for layer in range(config.num_hidden_layers): + print_rank_0(f"collecting layer #{layer}...") + layer_name = f"model.layers.{layer}" + src_pp_rank, src_virtual_pp_rank, src_layer_idx = layer_map[layer] + + gpt_model_module = _get_gpt_model(models[src_virtual_pp_rank]) + sync_layer = gpt_model_module.decoder.layers[src_layer_idx] + + _broadcast_tensor( + sync_layer.self_attention.linear_qkv.layer_norm_weight, + f"{layer_name}.input_layernorm.weight", + src_pp_rank=src_pp_rank, + ) + + if gpt_model_module.config.qk_layernorm: + _broadcast_tensor( + sync_layer.self_attention.q_layernorm.weight, + f"{layer_name}.self_attn.q_norm.weight", + src_pp_rank=src_pp_rank, + ) + _broadcast_tensor( + sync_layer.self_attention.k_layernorm.weight, + f"{layer_name}.self_attn.k_norm.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attention.linear_qkv.weight, + f"{layer_name}.self_attn.q_proj.weight", + f"{layer_name}.self_attn.k_proj.weight", + f"{layer_name}.self_attn.v_proj.weight", + src_pp_rank=src_pp_rank, + ) + + if gpt_model_module.config.add_qkv_bias: + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attention.linear_qkv.bias, + f"{layer_name}.self_attn.q_proj.bias", + f"{layer_name}.self_attn.k_proj.bias", + f"{layer_name}.self_attn.v_proj.bias", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor( + sync_layer.self_attention.linear_proj.weight, + f"{layer_name}.self_attn.o_proj.weight", + concat_dim=1, + src_pp_rank=src_pp_rank, + ) + + _broadcast_tensor( + sync_layer.mlp.linear_fc1.layer_norm_weight, + f"{layer_name}.post_attention_layernorm.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor_gate_up( + sync_layer.mlp.linear_fc1.weight, + f"{layer_name}.mlp.gate_proj.weight", + f"{layer_name}.mlp.up_proj.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor( + sync_layer.mlp.linear_fc2.weight, + f"{layer_name}.mlp.down_proj.weight", + concat_dim=1, + src_pp_rank=src_pp_rank, + ) + + # Final Layernorm + # ------------------- + print_rank_0("collecting final layernorm...") + gpt_model_module = _get_gpt_model(models[-1]) + _broadcast_tensor( + getattr(gpt_model_module.decoder.final_layernorm, "weight", None), + "model.norm.weight", + src_pp_rank=pp_size - 1, + ) + + if tie_word_embeddings: + print_rank_0("tie word embedding skip load lm_head...") + else: + print_rank_0("collecting lm_head...") + + if is_value_model: + lm_head_weight = None + if pp_rank == pp_size - 1: + lm_head_weight = getattr(gpt_model_module.output_layer, "weight", None) + _broadcast_tensor(lm_head_weight, "lm_head.weight", src_pp_rank=pp_size - 1) + + else: + _broadcast_tp_shard_tensor( + getattr(gpt_model_module.output_layer, "weight", None) if pp_rank == pp_size - 1 else None, + "lm_head.weight", + src_pp_rank=pp_size - 1, + ) + + dist.barrier() + get_torch_device().empty_cache() + if torch.distributed.get_rank() == 0: + for k, v in state_dict.items(): + if dtype != v.dtype: + state_dict[k] = v.to(dtype) + + print_rank_0(f"merge megatron ckpt done, time elapsed {time.time() - start_time}s") + return state_dict + + +def merge_megatron_ckpt_gptmodel_qwen_moe( + wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False +): + raise NotImplementedError("merge_megatron_ckpt_gptmodel_qwen_moe is not implemented") + + +def merge_megatron_ckpt_gptmodel_qwen2_5_vl( + wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False +): + raise NotImplementedError("merge_megatron_ckpt_gptmodel_qwen2_5_vl is not implemented") + + +def merge_megatron_ckpt_gptmodel_dpskv3(wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False): + raise NotImplementedError("merge_megatron_ckpt_gptmodel_dpskv3 is not implemented") + + +def merge_megatron_ckpt_gptmodel_mixtral( + wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False +): + raise NotImplementedError("merge_megatron_ckpt_gptmodel_mixtral is not implemented") diff --git a/e2c/verl/verl/models/mcore/util.py b/e2c/verl/verl/models/mcore/util.py new file mode 100644 index 0000000000000000000000000000000000000000..9904fc60d3d2ebdbfe637b05ae3a9a6d23f19cca --- /dev/null +++ b/e2c/verl/verl/models/mcore/util.py @@ -0,0 +1,260 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from megatron.core import parallel_state as mpu +from megatron.core.packed_seq_params import PackedSeqParams + +from verl.utils.model import CausalLMOutputForPPO + + +def preprocess_packed_seqs( + input_ids: torch.Tensor, attention_mask: torch.Tensor, pre_process: bool = True +) -> tuple[torch.Tensor, PackedSeqParams]: + """ + Preprocess packed sequences + CP splits sequence into CP*2 chunks, and each GPU gets 2 chunks (GPU0 gets first and last chunks, GPU1 + gets second and second last chunks, and so on), this is for load balancing with causal masking. + See https://github.com/NVIDIA/TransformerEngine/issues/1368 + """ + batch_size = input_ids.shape[0] + + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + tp_size = mpu.get_tensor_model_parallel_world_size() + cp_size = mpu.get_context_parallel_world_size() + cp_rank = mpu.get_context_parallel_rank() + align_size = tp_size * cp_size * 2 if cp_size > 1 else tp_size + + pad_size = (align_size - seqlens_in_batch % align_size) % align_size + seqlens_in_batch_padded = seqlens_in_batch + pad_size + + cu_seqlens = torch.zeros(batch_size + 1, dtype=torch.int32, device=input_ids.device) + cu_seqlens[1:] = torch.cumsum(seqlens_in_batch, dim=0) + cu_seqlens_padded = torch.zeros(batch_size + 1, dtype=torch.int32, device=input_ids.device) + cu_seqlens_padded[1:] = torch.cumsum(seqlens_in_batch_padded, dim=0) + + # ---------------------------------------------------------------------------- + # Move the index information needed in the subsequent loop to the CPU at once, + # to avoid frequent .item() calls in the loop that cause D2H synchronization + # ---------------------------------------------------------------------------- + seqlens_in_batch_cpu: list[int] = seqlens_in_batch.tolist() # original valid lengths + seqlens_in_batch_padded_cpu: list[int] = seqlens_in_batch_padded.tolist() # lengths after padding + cu_seqlens_padded_cpu: list[int] = cu_seqlens_padded.tolist() # start positions (after padding) + + # Pure Python int calculation to avoid further synchronization + max_seqlen_in_batch = max(seqlens_in_batch_padded_cpu) + + shape = list(input_ids.shape[1:]) + shape[0] = sum(seqlens_in_batch_padded_cpu) // cp_size + if pre_process: + input_ids_rmpad = torch.zeros(shape, dtype=input_ids.dtype, device=input_ids.device) + for i in range(batch_size): + # Use Python int, so no GPU→CPU sync in the loop + if cp_size <= 1: + seqlen = seqlens_in_batch_cpu[i] + start_idx = cu_seqlens_padded_cpu[i] + input_ids_rmpad[start_idx : start_idx + seqlen] = input_ids[i, attention_mask[i]] + continue + + seqlen_padded_i = seqlens_in_batch_padded_cpu[i] + seqlen = seqlen_padded_i // cp_size + half_seqlen = seqlen // 2 + start_idx = cu_seqlens_padded_cpu[i] // cp_size + # split to 2 chunks + d = input_ids[i, attention_mask[i]] + input_ids_rmpad[start_idx : start_idx + half_seqlen] = d[ + half_seqlen * cp_rank : half_seqlen * (cp_rank + 1) + ] + + remain_start = seqlen_padded_i - half_seqlen * (cp_rank + 1) + remain_end = seqlen_padded_i - half_seqlen * cp_rank + remain_end = min(remain_end, d.shape[0]) + remain_len = remain_end - remain_start + if remain_len > 0: + input_ids_rmpad[start_idx + half_seqlen : start_idx + half_seqlen + remain_len] = d[ + remain_start:remain_end + ] + + packed_seq_params = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens_padded, + max_seqlen_q=max_seqlen_in_batch, + cu_seqlens_kv=cu_seqlens_padded, + max_seqlen_kv=max_seqlen_in_batch, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + ) + if pre_process: + return input_ids_rmpad.unsqueeze(0), packed_seq_params + else: + return input_ids, packed_seq_params + + +def postprocess_packed_seqs( + output: torch.Tensor, + packed_seq_params: PackedSeqParams, + attention_mask: torch.Tensor, + batch_size: int, + seq_len: int, + post_process: bool = True, +) -> torch.Tensor: + """ + Postprocess packed sequences + """ + if not post_process: + return output + + # ------------------------------------------------------------------------- + # Move the lengths and offsets needed for subsequent Python-level indexing to the CPU in advance, + # to avoid a large number of .item() calls in the loop + # ------------------------------------------------------------------------- + cu_padded_cpu: list[int] = packed_seq_params.cu_seqlens_q_padded.tolist() + seq_lens_cpu: list[int] = attention_mask.sum(dim=1, dtype=torch.int32).cpu().tolist() + + shape = [batch_size, seq_len] + list(output.shape[2:]) # 1,packed, dim -> batch_size, seq_len, dim + output_new = torch.zeros(shape, dtype=output.dtype, device=output.device) + + cp_size = mpu.get_context_parallel_world_size() + # all gather output across context parallel group + if cp_size > 1: + # output shape: [1, packed_len, hidden_dim] + # need to gather across cp group and concatenate in sequence dimension + output_list = [torch.empty_like(output) for _ in range(cp_size)] + torch.distributed.all_gather(output_list, output.detach(), group=mpu.get_context_parallel_group()) + output_list[mpu.get_context_parallel_rank()] = output + else: + output_list = [output] + for i in range(batch_size): + if cp_size <= 1: + s = seq_lens_cpu[i] + start_idx = cu_padded_cpu[i] + output_new[i, attention_mask[i]] = output[0][start_idx : start_idx + s] + continue + s_len_padded_chunk = (cu_padded_cpu[i + 1] - cu_padded_cpu[i]) // cp_size + half_seqlen = s_len_padded_chunk // 2 + s_len = seq_lens_cpu[i] + s_len_padded = s_len_padded_chunk * cp_size + tmp = torch.empty(s_len_padded, *output.shape[2:], device=output.device) + for j in range(cp_size): + o = output_list[j][0] + # split to 2 chunks + packed_start_idx = cu_padded_cpu[i] // cp_size + o0, o1 = ( + o[packed_start_idx : packed_start_idx + half_seqlen], + o[packed_start_idx + half_seqlen : packed_start_idx + s_len_padded_chunk], + ) + tmp[j * half_seqlen : (j + 1) * half_seqlen] = o0 + tmp[s_len_padded - (j + 1) * half_seqlen : s_len_padded - j * half_seqlen] = o1 + output_new[i, attention_mask[i]] = tmp[:s_len] + + return output_new + + +def remove_left_padding( + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + position_ids: torch.Tensor, + sequence_parallel: bool = False, + pre_process: bool = True, +): + """ + Remove left padding from input_ids, attention_mask and position_ids + return new_input_ids, new_attention_mask, new_position_ids + """ + assert attention_mask.ndim == 2 + assert position_ids.ndim == 2 + cp_size = mpu.get_context_parallel_world_size() + assert cp_size == 1, "Context parallel size without seq_pack is not supported" + batch_size = input_ids.shape[0] + shape = list(input_ids.shape) # batch_size, seq_len,... + seq_lens = attention_mask.sum(dim=1) + seq_len = seq_lens.max().item() + if sequence_parallel: + sp_world_size = mpu.get_tensor_model_parallel_world_size() + pad_size = (sp_world_size - seq_len % sp_world_size) % sp_world_size + seq_len = seq_len + pad_size + shape[1] = seq_len + if pre_process: + new_input_ids = torch.zeros(dtype=input_ids.dtype, device=input_ids.device, size=shape) + new_attention_mask = torch.zeros( + dtype=attention_mask.dtype, device=attention_mask.device, size=(batch_size, seq_len) + ) + new_position_ids = torch.zeros(dtype=position_ids.dtype, device=position_ids.device, size=(batch_size, seq_len)) + for i in range(batch_size): + if pre_process: + new_input_ids[i, : seq_lens[i]] = input_ids[i, attention_mask[i]] + new_attention_mask[i, : seq_lens[i]] = attention_mask[i, attention_mask[i]] + new_position_ids[i, : seq_lens[i]] = position_ids[i, attention_mask[i]] + if pre_process: + return new_input_ids, new_attention_mask, new_position_ids + else: + return input_ids, new_attention_mask, new_position_ids + + +def recover_left_padding( + result, + attention_mask: torch.Tensor, + original_attention_mask: torch.Tensor, + origin_seqlen: int, + post_process: bool = True, +): + """ + Recover left padding from result + return result + """ + if not post_process: + return result + shape = list(result.shape) + batch_size = shape[0] + shape[1] = origin_seqlen + new_result = torch.zeros(dtype=result.dtype, device=result.device, size=shape) + for i in range(batch_size): + new_result[i, original_attention_mask[i]] = result[i, attention_mask[i]] + return new_result + + +def postprocess_packed_seqs_for_dict_output( + labels_mask: torch.Tensor, + output: CausalLMOutputForPPO, + packed_seq_params: PackedSeqParams, + attention_mask: torch.Tensor, + batch_size: int, + seq_len: int, + post_process: bool = True, +) -> dict[str, torch.Tensor]: + """_summary_ + For fused kernels, the output is a dictionary with keys like 'log_probs', 'entropy', etc. + This function post-processes each tensor in the output dictionary. + Args: + output (CausalLMOutputForPPO): _description_ + packed_seq_params (PackedSeqParams): _description_ + attention_mask (torch.Tensor): _description_ + batch_size (int): _description_ + seq_len (int): _description_ + post_process (bool, optional): _description_. Defaults to True. + Returns: + CausalLMOutputForPPO: _description_ + """ + ret = {} + output.entropy = output.entropy.view(1, -1) + output.log_probs = output.log_probs.view(1, -1) + output.log_probs = output.log_probs.masked_fill(~labels_mask, 0.0) + ret["entropy"] = postprocess_packed_seqs( + output.entropy, packed_seq_params, attention_mask, batch_size, seq_len, post_process=post_process + ) + ret["log_probs"] = postprocess_packed_seqs( + output.log_probs, packed_seq_params, attention_mask, batch_size, seq_len, post_process=post_process + ) + return ret diff --git a/e2c/verl/verl/models/mcore/weight_converter.py b/e2c/verl/verl/models/mcore/weight_converter.py new file mode 100644 index 0000000000000000000000000000000000000000..791513f32d1b7ab1e220d2c7f1abb5a2c8abeba3 --- /dev/null +++ b/e2c/verl/verl/models/mcore/weight_converter.py @@ -0,0 +1,479 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# online convert mcore weight to pure huggingface weight, no any fusion +# including format conversion and name mapping +# not including resharding +import torch +from megatron.core.transformer import TransformerConfig +from transformers import PretrainedConfig + + +class McoreToHFWeightConverterBase: + def __init__(self, hf_config: PretrainedConfig, mcore_config: TransformerConfig): + self.hf_config = hf_config + self.mcore_config = mcore_config + + def convert_param(self, name: str, params_one_group: list[torch.Tensor]) -> torch.Tensor: + raise NotImplementedError + + +class McoreToHFWeightConverterDense(McoreToHFWeightConverterBase): + def _convert_attention_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + # 'decoder.layers.0.self_attention.linear_proj.weight' + # 'decoder.layers.0.self_attention.linear_qkv.layer_norm_weight' + # 'decoder.layers.0.self_attention.linear_qkv.weight' + # 'decoder.layers.0.self_attention.linear_qkv.bias' + layer_number = name.split(".")[2] + convert_names = [] + if "self_attention.linear_qkv.bias" in name or "self_attention.linear_qkv.weight" in name: + param_type = name.split(".")[-1] + assert param_type == "bias" or param_type == "weight" + convert_names.append(f"model.layers.{layer_number}.self_attn.q_proj.{param_type}") + convert_names.append(f"model.layers.{layer_number}.self_attn.k_proj.{param_type}") + convert_names.append(f"model.layers.{layer_number}.self_attn.v_proj.{param_type}") + assert len(params) == 3 + elif "self_attention.linear_proj.weight" in name: + convert_names.append(f"model.layers.{layer_number}.self_attn.o_proj.weight") + assert len(params) == 1 + elif "self_attention.linear_qkv.layer_norm_weight" in name: + convert_names.append(f"model.layers.{layer_number}.input_layernorm.weight") + assert len(params) == 1 + elif "self_attention.q_layernorm.weight" in name: + convert_names.append(f"model.layers.{layer_number}.self_attn.q_norm.weight") + assert len(params) == 1 + elif "self_attention.k_layernorm.weight" in name: + convert_names.append(f"model.layers.{layer_number}.self_attn.k_norm.weight") + assert len(params) == 1 + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + return convert_names, params + + def _convert_mlp_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + # 'decoder.layers.0.mlp.linear_fc1.layer_norm_weight' + # 'decoder.layers.0.mlp.linear_fc1.weight' + # 'decoder.layers.0.mlp.linear_fc2.weight' + layer_number = name.split(".")[2] + convert_names = [] + if "mlp.linear_fc1.weight" in name: + # split gate_proj and up_proj + convert_names.append(f"model.layers.{layer_number}.mlp.gate_proj.weight") + convert_names.append(f"model.layers.{layer_number}.mlp.up_proj.weight") + assert len(params) == 2 + elif "mlp.linear_fc1.layer_norm_weight" in name: + convert_names.append(f"model.layers.{layer_number}.post_attention_layernorm.weight") + assert len(params) == 1 + elif "mlp.linear_fc2.weight" in name: + convert_names.append(f"model.layers.{layer_number}.mlp.down_proj.weight") + assert len(params) == 1 + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + return convert_names, params + + def convert_param(self, name: str, params_one_group: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + direct_name_mapping = { + "embedding.word_embeddings.weight": "model.embed_tokens.weight", + "decoder.final_layernorm.weight": "model.norm.weight", + "output_layer.weight": "lm_head.weight", + } + if name in direct_name_mapping: + return [direct_name_mapping[name]], [params_one_group[0]] + + if "self_attention" in name: + return self._convert_attention_param(name, params_one_group) + elif "mlp" in name: + return self._convert_mlp_param(name, params_one_group) + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + + +class McoreToHFWeightConverterQwen2Moe(McoreToHFWeightConverterDense): + def _convert_mlp_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + # 'decoder.layers.0.pre_mlp_layernorm.weight', + # 'decoder.layers.0.mlp.router.weight', + # 'decoder.layers.0.mlp.shared_experts.gate_weight', + # 'decoder.layers.0.mlp.shared_experts.linear_fc1.weight', + # 'decoder.layers.0.mlp.shared_experts.linear_fc2.weight' + # moe1 + # 'decoder.layers.0.mlp.experts.linear_fc1.weight0', + # 'decoder.layers.0.mlp.experts.linear_fc1.weight1', + # 'decoder.layers.0.mlp.experts.linear_fc1.weight2', + # 'decoder.layers.0.mlp.experts.linear_fc1.weight3', + # moe2 + # 'decoder.layers.0.mlp.experts.linear_fc2.weight0', + # 'decoder.layers.0.mlp.experts.linear_fc2.weight1', + layer_number = name.split(".")[2] + convert_names = [] + if "pre_mlp_layernorm" in name: + convert_names.append(f"model.layers.{layer_number}.post_attention_layernorm.weight") + assert len(params) == 1 + elif "mlp.router.weight" in name: + convert_names.append(f"model.layers.{layer_number}.mlp.gate.weight") + assert len(params) == 1 + elif "shared_experts.gate_weight" in name: + convert_names.append(f"model.layers.{layer_number}.mlp.shared_expert_gate.weight") + assert len(params) == 1 + elif "shared_experts.linear_fc1.weight" in name: # split gate_proj and up_proj + convert_names.append(f"model.layers.{layer_number}.mlp.shared_expert.gate_proj.weight") + convert_names.append(f"model.layers.{layer_number}.mlp.shared_expert.up_proj.weight") + assert len(params) == 2 + elif "shared_experts.linear_fc2.weight" in name: + convert_names.append(f"model.layers.{layer_number}.mlp.shared_expert.down_proj.weight") + assert len(params) == 1 + elif "mlp.experts.linear_fc1" in name: # split gate_proj and up_proj + expert_id = name.split("weight")[-1] + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.gate_proj.weight") + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.up_proj.weight") + assert len(params) == 2 + elif "mlp.experts.linear_fc2" in name: + expert_id = name.split("weight")[-1] + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.down_proj.weight") + assert len(params) == 1 + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + return convert_names, params + + +class McoreToHFWeightConverterQwen2_5_VL(McoreToHFWeightConverterDense): + def convert_param(self, name: str, params_one_group: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + direct_name_mapping = { + "language_model.embedding.word_embeddings.weight": "model.embed_tokens.weight", + "language_model.decoder.final_layernorm.weight": "model.norm.weight", + "language_model.output_layer.weight": "lm_head.weight", + "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight", + "vision_model.decoder.final_layernorm.weight": "visual.merger.ln_q.weight", + "vision_model.projection.encoder.linear_fc1.weight": "visual.merger.mlp.0.weight", + "vision_model.projection.encoder.linear_fc1.bias": "visual.merger.mlp.0.bias", + "vision_model.projection.encoder.linear_fc2.weight": "visual.merger.mlp.2.weight", + "vision_model.projection.encoder.linear_fc2.bias": "visual.merger.mlp.2.bias", + } + if name in direct_name_mapping: + return [direct_name_mapping[name]], [params_one_group[0]] + + if "self_attention" in name: + return self._convert_attention_param(name, params_one_group) + elif "mlp" in name: + return self._convert_mlp_param(name, params_one_group) + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + + def _convert_attention_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + model_type, _, _, layer_number = name.split(".")[:4] + + convert_names = [] + if model_type == "language_model": + name_map_after_layer = { + "self_attention.linear_qkv.bias": [ + "self_attn.q_proj.bias", + "self_attn.k_proj.bias", + "self_attn.v_proj.bias", + ], + "self_attention.linear_qkv.weight": [ + "self_attn.q_proj.weight", + "self_attn.k_proj.weight", + "self_attn.v_proj.weight", + ], + "self_attention.linear_proj.weight": "self_attn.o_proj.weight", + "self_attention.linear_qkv.layer_norm_weight": "input_layernorm.weight", + } + name_after_layer = ".".join(name.split(".")[-3:]) + mapped_name = name_map_after_layer.get(name_after_layer) + if isinstance(mapped_name, list): + assert len(params) == len(mapped_name) + for one in mapped_name: + convert_names.append(f"model.layers.{layer_number}.{one}") + else: + assert len(params) == 1 + convert_names.append(f"model.layers.{layer_number}.{mapped_name}") + elif model_type == "vision_model": + name_map_after_layer = { + "self_attention.linear_proj.weight": "attn.proj.weight", + "self_attention.linear_proj.bias": "attn.proj.bias", + "self_attention.linear_qkv.layer_norm_weight": "norm1.weight", + } + name_after_layer = ".".join(name.split(".")[-3:]) + mapped_name = name_map_after_layer.get(name_after_layer, None) + if mapped_name is None: + assert "linear_qkv" in name_after_layer + assert len(params) == 3 + new_param = torch.cat(params, dim=0) + params = [new_param] + if "bias" in name_after_layer: + convert_names.append(f"visual.blocks.{layer_number}.attn.qkv.bias") + else: + convert_names.append(f"visual.blocks.{layer_number}.attn.qkv.weight") + else: + assert len(params) == 1 + convert_names.append(f"visual.blocks.{layer_number}.{mapped_name}") + else: + raise NotImplementedError(f"Unsupported model type: {model_type}") + return convert_names, params + + def _convert_mlp_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + model_type, _, _, layer_number = name.split(".")[:4] + + convert_names = [] + if model_type == "language_model": + name_map_after_layer = { + "mlp.linear_fc1.weight": ["mlp.gate_proj.weight", "mlp.up_proj.weight"], + "mlp.linear_fc1.bias": ["mlp.gate_proj.bias", "mlp.up_proj.bias"], + "mlp.linear_fc2.weight": "mlp.down_proj.weight", + "mlp.linear_fc2.bias": "mlp.down_proj.bias", + "mlp.linear_fc1.layer_norm_weight": "post_attention_layernorm.weight", + } + name_after_layer = ".".join(name.split(".")[-3:]) + mapped_name = name_map_after_layer.get(name_after_layer) + if isinstance(mapped_name, list): + assert len(params) == len(mapped_name) + for one in mapped_name: + convert_names.append(f"model.layers.{layer_number}.{one}") + else: + assert len(params) == 1 + convert_names.append(f"model.layers.{layer_number}.{mapped_name}") + + elif model_type == "vision_model": + name_map_after_layer = { + "mlp.linear_fc1.weight": ["mlp.gate_proj.weight", "mlp.up_proj.weight"], + "mlp.linear_fc1.bias": ["mlp.gate_proj.bias", "mlp.up_proj.bias"], + "mlp.linear_fc2.weight": "mlp.down_proj.weight", + "mlp.linear_fc2.bias": "mlp.down_proj.bias", + "mlp.linear_fc1.layer_norm_weight": "norm2.weight", + } + name_after_layer = ".".join(name.split(".")[-3:]) + mapped_name = name_map_after_layer.get(name_after_layer) + if isinstance(mapped_name, list): + assert len(params) == len(mapped_name) + for one in mapped_name: + convert_names.append(f"visual.blocks.{layer_number}.{one}") + else: + assert len(params) == 1 + convert_names.append(f"visual.blocks.{layer_number}.{mapped_name}") + else: + raise NotImplementedError(f"Unsupported model type: {model_type}") + return convert_names, params + + +class McoreToHFWeightConverterDpskv3(McoreToHFWeightConverterBase): + def _convert_attention_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + # mcore + # 'decoder.layers.0.input_layernorm.weight' + # 'decoder.layers.0.self_attention.linear_proj.weight' + # 'decoder.layers.0.self_attention.linear_q_proj.weight' + # 'decoder.layers.0.self_attention.linear_kv_down_proj.weight' + # 'decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight' + # 'decoder.layers.0.self_attention.linear_kv_up_proj.weight' + # 'decoder.layers.0.self_attention.linear_q_down_proj.weight' + # 'decoder.layers.0.self_attention.linear_q_up_proj.weight' + # 'decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight' + # hf + # 'model.layers.0.input_layernorm.weight' + # 'model.layers.0.self_attn.o_proj.weight' + # 'model.layers.0.self_attn.q_proj.weight' + # 'model.layers.0.self_attn.kv_a_proj_with_mqa.weight' + # 'model.layers.0.self_attn.kv_a_layernorm.weight' + # 'model.layers.0.self_attn.kv_b_proj.weight' + # 'model.layers.0.self_attn.q_a_proj.weight' + # 'model.layers.0.self_attn.q_b_proj.weight' + # 'model.layers.0.self_attn.q_a_layernorm.weight' + name_map_after_layer = { + "input_layernorm.weight": "input_layernorm.weight", + "self_attention.linear_proj.weight": "self_attn.o_proj.weight", + "self_attention.linear_q_proj.weight": "self_attn.q_proj.weight", + "self_attention.linear_kv_down_proj.weight": "self_attn.kv_a_proj_with_mqa.weight", + "self_attention.linear_kv_up_proj.layer_norm_weight": "self_attn.kv_a_layernorm.weight", + "self_attention.linear_kv_up_proj.weight": "self_attn.kv_b_proj.weight", + "self_attention.linear_q_down_proj.weight": "self_attn.q_a_proj.weight", + "self_attention.linear_q_up_proj.weight": "self_attn.q_b_proj.weight", + "self_attention.linear_q_up_proj.layer_norm_weight": "self_attn.q_a_layernorm.weight", + } + assert len(params) == 1 + convert_names = [] + layer_number = name.split(".")[2] + name_after_layer = name.split(f".{layer_number}.")[1] + convert_names.append(f"model.layers.{layer_number}.{name_map_after_layer[name_after_layer]}") + return convert_names, params + + def _convert_mlp_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + # mcore dense + # 'decoder.layers.0.mlp.linear_fc1.layer_norm_weight' + # 'decoder.layers.0.mlp.linear_fc2.weight' + # 'decoder.layers.0.mlp.linear_fc1.weight' + # --- + # 'decoder.layers.1.mlp.shared_experts.linear_fc1.weight' + # --- + # 'decoder.layers.1.mlp.shared_experts.linear_fc2.weight' + # hf dense + # 'model.layers.0.post_attention_layernorm.weight' + # 'model.layers.0.mlp.down_proj.weight' + # 'model.layers.0.mlp.gate_proj.weight' + # 'model.layers.0.mlp.up_proj.weight' + # 'model.layers.1.mlp.shared_experts.gate_proj.weight' + # 'model.layers.1.mlp.shared_experts.up_proj.weight' + # 'model.layers.1.mlp.shared_experts.down_proj.weight' + + # mcore moe + # 'decoder.layers.1.pre_mlp_layernorm.weight' + # 'decoder.layers.1.mlp.router.weight' + # 'decoder.layers.1.mlp.router.expert_bias' + # 'decoder.layers.1.mlp.experts.linear_fc1.weight0' + # --- + # 'decoder.layers.1.mlp.experts.linear_fc2.weight0' + # hf moe + # 'model.layers.1.post_attention_layernorm.weight' + # 'model.layers.1.mlp.gate.weight' + # 'model.layers.1.mlp.gate.e_score_correction_bias' + # 'model.layers.1.mlp.experts.0.gate_proj.weight' + # 'model.layers.1.mlp.experts.0.up_proj.weight' + # 'model.layers.1.mlp.experts.0.down_proj.weight' + + name_map_after_layer = { + "mlp.linear_fc1.layer_norm_weight": "post_attention_layernorm.weight", + "mlp.linear_fc2.weight": "mlp.down_proj.weight", + "mlp.shared_experts.linear_fc2.weight": "mlp.shared_experts.down_proj.weight", + "mlp.linear_fc1.weight": ["mlp.gate_proj.weight", "mlp.up_proj.weight"], + "mlp.shared_experts.linear_fc1.weight": [ + "mlp.shared_experts.gate_proj.weight", + "mlp.shared_experts.up_proj.weight", + ], + "pre_mlp_layernorm.weight": "post_attention_layernorm.weight", + "mlp.router.weight": "mlp.gate.weight", + "mlp.router.expert_bias": "mlp.gate.e_score_correction_bias", + } + convert_names = [] + layer_number = name.split(".")[2] + name_after_layer = name.split(f".{layer_number}.")[1] + if name_after_layer in name_map_after_layer: + mapped_name = name_map_after_layer[name_after_layer] + if isinstance(mapped_name, list): + assert len(params) == len(mapped_name) + for one in mapped_name: + convert_names.append(f"model.layers.{layer_number}.{one}") + else: + assert len(params) == 1 + convert_names.append(f"model.layers.{layer_number}.{mapped_name}") + else: + if "mlp.experts.linear_fc1.weight" in name: + expert_id = name.split("weight")[-1] + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.gate_proj.weight") + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.up_proj.weight") + assert len(params) == 2 + elif "mlp.experts.linear_fc2.weight" in name: + expert_id = name.split("weight")[-1] + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.down_proj.weight") + assert len(params) == 1 + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + + return convert_names, params + + def _convert_mtp_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + assert self.mcore_config.mtp_num_layers == 1, "only support one mtp layer for now" + assert self.mcore_config.num_layers == 61, "only support 61 layers for now" + direct_name_mapping = { + "mtp.layers.0.enorm.weight": "model.layers.61.enorm.weight", + "mtp.layers.0.hnorm.weight": "model.layers.61.hnorm.weight", + "mtp.layers.0.eh_proj.weight": "model.layers.61.eh_proj.weight", + "mtp.layers.0.final_layernorm.weight": "model.layers.61.shared_head.norm.weight", + } + if name in direct_name_mapping: + return [direct_name_mapping[name]], [params[0]] + assert "mtp.layers.0.transformer_layer" in name, "only support transformer layer for now" + # use proxy name to convert + proxy_name = name.replace("mtp.layers.0.transformer_layer", "decoder.layers.61") + if "self_attention" in proxy_name or "input_layernorm.weight" in proxy_name: + convert_names, params = self._convert_attention_param(proxy_name, params) + elif "mlp" in proxy_name: + convert_names, params = self._convert_mlp_param(proxy_name, params) + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + return convert_names, params + + def convert_param(self, name: str, params_one_group: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + direct_name_mapping = { + "embedding.word_embeddings.weight": "model.embed_tokens.weight", + "decoder.final_layernorm.weight": "model.norm.weight", + "output_layer.weight": "lm_head.weight", + } + if name in direct_name_mapping: + return [direct_name_mapping[name]], [params_one_group[0]] + if "mtp" in name: + return self._convert_mtp_param(name, params_one_group) + elif "self_attention" in name or "input_layernorm.weight" in name: + return self._convert_attention_param(name, params_one_group) + elif "mlp" in name: + return self._convert_mlp_param(name, params_one_group) + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + + +class McoreToHFWeightConverterMixtral(McoreToHFWeightConverterDense): + def _convert_mlp_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + # decoder.layers.0.mlp.router.weight + # decoder.layers.0.mlp.experts.linear_fc1.weight0 - weight7 + # decoder.layers.0.mlp.experts.linear_fc2.weight0 - weight7 + + layer_number = name.split(".")[2] + convert_names = [] + if "pre_mlp_layernorm" in name: + convert_names.append(f"model.layers.{layer_number}.post_attention_layernorm.weight") + elif "mlp.router.weight" in name: + convert_names.append(f"model.layers.{layer_number}.block_sparse_moe.gate.weight") + elif "mlp.experts.linear_fc1.weight" in name: + expert_id = name.split("weight")[-1] + convert_names.append(f"model.layers.{layer_number}.block_sparse_moe.experts.{expert_id}.w1.weight") + convert_names.append(f"model.layers.{layer_number}.block_sparse_moe.experts.{expert_id}.w3.weight") + elif "mlp.experts.linear_fc2.weight" in name: + expert_id = name.split("weight")[-1] + convert_names.append(f"model.layers.{layer_number}.block_sparse_moe.experts.{expert_id}.w2.weight") + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + return convert_names, params + + +class McoreToHFWeightConverterQwen3Moe(McoreToHFWeightConverterDense): + def _convert_mlp_param(self, name: str, params: list[torch.Tensor]) -> tuple[list[str], list[torch.Tensor]]: + # qwen3 moe no share expert + + # 'decoder.layers.0.pre_mlp_layernorm.weight', + # 'decoder.layers.0.mlp.router.weight', + # moe1 + # 'decoder.layers.0.mlp.experts.linear_fc1.weight0', + # 'decoder.layers.0.mlp.experts.linear_fc1.weight1', + # 'decoder.layers.0.mlp.experts.linear_fc1.weight2', + # 'decoder.layers.0.mlp.experts.linear_fc1.weight3', + # moe2 + # 'decoder.layers.0.mlp.experts.linear_fc2.weight0', + # 'decoder.layers.0.mlp.experts.linear_fc2.weight1', + layer_number = name.split(".")[2] + convert_names = [] + if "pre_mlp_layernorm" in name: + convert_names.append(f"model.layers.{layer_number}.post_attention_layernorm.weight") + assert len(params) == 1 + elif "mlp.router.weight" in name: + convert_names.append(f"model.layers.{layer_number}.mlp.gate.weight") + assert len(params) == 1 + elif "mlp.experts.linear_fc1" in name: # split gate_proj and up_proj + expert_id = name.split("weight")[-1] + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.gate_proj.weight") + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.up_proj.weight") + assert len(params) == 2 + elif "mlp.experts.linear_fc2" in name: + expert_id = name.split("weight")[-1] + convert_names.append(f"model.layers.{layer_number}.mlp.experts.{expert_id}.down_proj.weight") + assert len(params) == 1 + else: + raise NotImplementedError(f"Unsupported parameter name: {name}") + return convert_names, params diff --git a/e2c/verl/verl/models/qwen2/__init__.py b/e2c/verl/verl/models/qwen2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/models/qwen2/megatron/__init__.py b/e2c/verl/verl/models/qwen2/megatron/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..57e33ee9e905a64eb92df812d2f0bc6126066042 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/__init__.py @@ -0,0 +1,34 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .modeling_qwen2_megatron import ( + ParallelQwen2ForCausalLM, + # rmpad with megatron + ParallelQwen2ForCausalLMRmPad, + # rmpad with megatron and pipeline parallelism + ParallelQwen2ForCausalLMRmPadPP, + ParallelQwen2ForValueRmPad, + ParallelQwen2ForValueRmPadPP, + # original model with megatron + ParallelQwen2Model, +) + +__all__ = [ + "ParallelQwen2ForCausalLM", + "ParallelQwen2ForCausalLMRmPad", + "ParallelQwen2ForCausalLMRmPadPP", + "ParallelQwen2ForValueRmPad", + "ParallelQwen2ForValueRmPadPP", + "ParallelQwen2Model", +] diff --git a/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/__init__.py b/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader.py b/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..3168635c7fe7b5b0e35a8e99b189057acbb8a5cb --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader.py @@ -0,0 +1,337 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import torch +import torch.distributed as dist + +from verl.utils.device import get_device_id, get_torch_device + + +def _megatron_calc_layer_map(config): + """Calculate the mapping of global layer_idx to local layer_idx + Returns: + layer_map (Dict: int -> tuple(int, int, int)): + mapping from the global layer index to + a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) + """ + from megatron.core import mpu + + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + + layer_map = dict() + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + for pp_rank_idx in range(pp_size): + for virtual_pp_rank_idx in range(virtual_pp_size): + layer_offset = ( + virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + pp_rank_idx * num_layers_per_model + ) + for layer_idx in range(num_layers_per_model): + layer_map[layer_offset + layer_idx] = ( + pp_rank_idx, + virtual_pp_rank_idx, + layer_idx, + ) + return layer_map + + +def load_state_dict_to_megatron_qwen2( + state_dict, wrapped_models, config, params_dtype, is_value_model=False, tie_word_embeddings=False +): + """Load merged state_dict to sharded Megatron module in training.""" + from megatron.core import DistributedDataParallel as LocalDDP + from megatron.core import mpu + from megatron.core.transformer.module import Float16Module + from torch.nn.parallel import DistributedDataParallel as torchDDP + + from verl.utils.logger import print_rank_0 + from verl.utils.megatron_utils import unwrap_model + + start_time = time.time() + + def _get_gpt_model(model): + return model + + def fetch_params(module): + for param in module.parameters(): + torch.distributed.fetch( + param.data, src=mpu.get_data_parallel_src_rank(), group=mpu.get_data_parallel_group() + ) + + dp_rank = mpu.get_data_parallel_rank() + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + mp_group = mpu.get_model_parallel_group() + + if torch.distributed.get_rank() == 0: + assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" + assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" + assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" + + if not isinstance(wrapped_models, list | tuple): + wrapped_models = list(wrapped_models) + + assert len(wrapped_models) == virtual_pp_size + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers, ( + f"num_layers_per_model: {num_layers_per_model} * pp_size: {pp_size} * virtual_pp_size: " + f"{virtual_pp_size} != config.num_hidden_layers: {config.num_hidden_layers}" + ) + + models = [None] * len(wrapped_models) + + for i, wrapped_model in enumerate(wrapped_models): + models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) + gpt_model_module = _get_gpt_model(models[i]) + assert len(gpt_model_module.model.layers) == num_layers_per_model + + def _fetch_tensor(tensor, name) -> torch.Tensor: + """fetch tensor""" + nonlocal state_dict + if tensor is not None: + tensor = tensor.data.copy_(state_dict[name], non_blocking=True) + + def _fetch_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """fetch tensor in tp shards""" + nonlocal state_dict + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + if name in state_dict: + full_weight = state_dict[name] + + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + if tensor is not None: + tensor = tensor.data.copy_(tensor_chunk[tp_rank], non_blocking=True) + else: + print(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + + def _fetch_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """fetch tensor in tp shards""" + nonlocal state_dict + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + if name in state_dict: + full_weight = state_dict[name] + + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + if tensor is not None: + tensor = tensor.data.copy_(tensor_chunk[tp_rank], non_blocking=True) + else: + print(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + + def _fetch_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: + """fetch gate_up tensor in tp shards""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + if gate_name in state_dict and up_name in state_dict: + gate_weight = state_dict[gate_name] + up_weight = state_dict[up_name] + new_gate_up_weight = torch.empty( + config.intermediate_size * 2, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + intermediate_size_tp = config.intermediate_size // tp_size + gate_weight_tp = gate_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + up_weight_tp = up_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + new_gate_up_weight[intermediate_size_tp * 2 * i : intermediate_size_tp * 2 * (i + 1)].copy_( + torch.cat([gate_weight_tp, up_weight_tp], dim=0) + ) + + tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) + if tensor is not None: + tensor = tensor.data.copy_(tensor_chunk[tp_rank], non_blocking=True) + else: + print(f"tp_shard tensor:[{gate_name}, {up_name}] not in state_dict, skip loading") + + def _fetch_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, bias=False) -> torch.Tensor: + """fetch tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + assert q_name in state_dict and k_name in state_dict and v_name in state_dict + full_weight_q = state_dict[q_name] + full_weight_k = state_dict[k_name] + full_weight_v = state_dict[v_name] + + hidden_size_per_head = config.hidden_size // config.num_attention_heads + + if config.num_key_value_heads >= tp_size: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size + total_size = q_size_tp + 2 * kv_size_tp + if not bias: + new_weight_qkv = torch.empty( + total_size * tp_size, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + else: + new_weight_qkv = torch.empty(total_size * tp_size, dtype=params_dtype, device=get_device_id()) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + k_part = full_weight_k[i * kv_size_tp : (i + 1) * kv_size_tp] + v_part = full_weight_v[i * kv_size_tp : (i + 1) * kv_size_tp] + new_weight_qkv[i * total_size : (i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], dim=0)) + + else: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head + total_size = q_size_tp + 2 * kv_size_tp + if not bias: + new_weight_qkv = torch.empty( + total_size * tp_size, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + else: + new_weight_qkv = torch.empty(total_size * tp_size, dtype=params_dtype, device=get_device_id()) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head + end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head + k_part = full_weight_k[start_idx:end_idx] + v_part = full_weight_v[start_idx:end_idx] + new_weight_qkv[i * total_size : (i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], dim=0)) + + tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) + if tensor is not None: + tensor = tensor.data.copy_(tensor_chunk[tp_rank], non_blocking=True) + + # Embeddings + # ------------------- + print_rank_0("loading embeddings...") + gpt_model_module = _get_gpt_model(models[0]) + if pp_rank == 0: + embed_tokens_weight = gpt_model_module.model.embed_tokens.weight + _fetch_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") + + # Transformer layers + # ------------------- + layer_map = _megatron_calc_layer_map(config) + + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + num_layer_per_pp = config.num_hidden_layers // pp_size + vpp_size = mpu.get_virtual_pipeline_model_parallel_world_size() + + layer_list = [] + if vpp_size is not None: + for vpp_rank in range(vpp_size): + num_layer_vpp_chunk = num_layer_per_pp // vpp_size + num_layer_this_model = num_layer_vpp_chunk + offset = vpp_rank * (config.num_hidden_layers // mpu.get_virtual_pipeline_model_parallel_world_size()) + ( + mpu.get_pipeline_model_parallel_rank() * num_layer_vpp_chunk + ) + layer_list.extend(list(range(offset, offset + num_layer_this_model))) + else: + num_layer_this_model = num_layer_per_pp + offset = pp_rank * num_layer_per_pp + layer_list.extend(list(range(offset, offset + num_layer_this_model))) + + for layer in layer_list: + print(f"{torch.distributed.get_rank()} loading layer #{layer}...") + layer_name = f"model.layers.{layer}" + dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] + + print( + f"{torch.distributed.get_rank()} offset: {offset}, num_layer_this_model: {num_layer_this_model}, " + f"layer_name: {layer_name}, layer_map[layer]: {layer_map[layer]}" + ) + + gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) + sync_layer = gpt_model_module.model.layers[dst_layer_idx] + + _fetch_tensor( + sync_layer.input_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.input_layernorm.weight", + ) + + _fetch_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_proj.weight", + f"{layer_name}.self_attn.k_proj.weight", + f"{layer_name}.self_attn.v_proj.weight", + ) + + _fetch_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.bias if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_proj.bias", + f"{layer_name}.self_attn.k_proj.bias", + f"{layer_name}.self_attn.v_proj.bias", + bias=True, + ) + + _fetch_tp_shard_tensor( + sync_layer.self_attn.o_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.o_proj.weight", + chunk_dim=1, + ) + + _fetch_tensor( + sync_layer.post_attention_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.post_attention_layernorm.weight", + ) + + _fetch_tp_shard_tensor_gate_up( + sync_layer.mlp.gate_up_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.gate_proj.weight", + f"{layer_name}.mlp.up_proj.weight", + ) + + _fetch_tp_shard_tensor( + sync_layer.mlp.down_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.down_proj.weight", + chunk_dim=1, + ) + # Final Layernorm + # ------------------- + print_rank_0("loading final layernorm...") + gpt_model_module = _get_gpt_model(models[-1]) + _fetch_tensor( + getattr(gpt_model_module.model.norm, "weight", None), + "model.norm.weight", + ) + + if tie_word_embeddings: + print_rank_0("tie_word_embeddings skip load lm_head") + else: + print_rank_0("loading lm_head...") + if pp_rank + 1 == pp_size: + lm_head_weight = gpt_model_module.lm_head.weight + + if is_value_model: + if "lm_head.weight" in state_dict and state_dict["lm_head.weight"].shape[0] == 1: + _fetch_tensor(lm_head_weight, "lm_head.weight") + print_rank_0("load lm_head from value_head weight") + elif "reward_head.weight" in state_dict and state_dict["reward_head.weight"].shape[0] == 1: + _fetch_tensor(lm_head_weight, "reward_head.weight") + print_rank_0("load lm_head from value_head weight") + else: + _fetch_tensor(None, "lm_head.weight") + print_rank_0("fail to match lm_head in value_model") + + else: + _fetch_tp_shard_tensor(lm_head_weight, "lm_head.weight") + + dist.barrier() + get_torch_device().empty_cache() + print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") diff --git a/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader_depracated.py b/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader_depracated.py new file mode 100644 index 0000000000000000000000000000000000000000..770e3653366321159ec079c42009052aeaf26510 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader_depracated.py @@ -0,0 +1,475 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import torch +import torch.distributed as dist + +from verl.utils.device import get_device_id, get_torch_device + + +def _megatron_calc_layer_map(config): + """Calculate the mapping of global layer_idx to local layer_idx + Returns: + layer_map (Dict: int -> tuple(int, int, int)): + mapping from the global layer index to + a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) + """ + from megatron.core import mpu + + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + + layer_map = dict() + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + for pp_rank_idx in range(pp_size): + for virtual_pp_rank_idx in range(virtual_pp_size): + layer_offset = ( + virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + pp_rank_idx * num_layers_per_model + ) + for layer_idx in range(num_layers_per_model): + layer_map[layer_offset + layer_idx] = ( + pp_rank_idx, + virtual_pp_rank_idx, + layer_idx, + ) + return layer_map + + +def load_state_dict_to_megatron_qwen2( + state_dict, wrapped_models, config, params_dtype, is_value_model=False, tie_word_embeddings=False +): + """Load merged state_dict to sharded Megatron module in training.""" + from megatron.core import DistributedDataParallel as LocalDDP + from megatron.core import mpu + from megatron.core.transformer.module import Float16Module + from torch.nn.parallel import DistributedDataParallel as torchDDP + + from verl.utils.logger import print_rank_0 + from verl.utils.megatron_utils import unwrap_model + + start_time = time.time() + + def _get_gpt_model(model): + return model + + def broadcast_params(module): + for param in module.parameters(): + torch.distributed.broadcast( + param.data, src=mpu.get_data_parallel_src_rank(), group=mpu.get_data_parallel_group() + ) + + dp_rank = mpu.get_data_parallel_rank() + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + mp_group = mpu.get_model_parallel_group() + + if torch.distributed.get_rank() == 0: + assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" + assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" + assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" + + if not isinstance(wrapped_models, list | tuple): + wrapped_models = list(wrapped_models) + + assert len(wrapped_models) == virtual_pp_size + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers, ( + f"num_layers_per_model: {num_layers_per_model} * pp_size: {pp_size} * virtual_pp_size: " + f"{virtual_pp_size} != config.num_hidden_layers: {config.num_hidden_layers}" + ) + + models = [None] * len(wrapped_models) + + for i, wrapped_model in enumerate(wrapped_models): + models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) + gpt_model_module = _get_gpt_model(models[i]) + assert len(gpt_model_module.model.layers) == num_layers_per_model + + def _broadcast_tensor(tensor, name) -> torch.Tensor: + """broadcast tensor from rank0 across mp_group""" + nonlocal state_dict + nonlocal mp_group + if torch.distributed.get_rank() == 0: + if name in state_dict: + weight = state_dict[name] + tensor_shape = weight.shape + else: + tensor_shape = None + else: + weight = None + tensor_shape = None + + obj_list = [tensor_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + tensor_shape = obj_list[0] + + if tensor_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tensor:[{name}] not in state_dict, skip load") + return + + if tensor is None: + tensor = torch.empty( + tensor_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + if torch.distributed.get_rank() == 0: + tensor.data.copy_(weight) + dist.broadcast(tensor, src=0, group=mp_group) + + def _broadcast_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == 0: + if name in state_dict: + full_weight = state_dict[name] + + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == 0: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=0, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == 0: + if name in state_dict: + full_weight = state_dict[name] + if mutate_func is not None: + full_weight = mutate_func(full_weight) + tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == 0: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=0, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == 0: + gate_weight = state_dict[gate_name] + up_weight = state_dict[up_name] + new_gate_up_weight = torch.empty( + config.intermediate_size * 2, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + for i in range(tp_size): + intermediate_size_tp = config.intermediate_size // tp_size + gate_weight_tp = gate_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + up_weight_tp = up_weight[i * intermediate_size_tp : (i + 1) * intermediate_size_tp] + new_gate_up_weight[intermediate_size_tp * 2 * i : intermediate_size_tp * 2 * (i + 1)].copy_( + torch.cat([gate_weight_tp, up_weight_tp], dim=0) + ) + + tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank() == 0:} tensor {gate_name, up_name} shape " + f"{tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == 0: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=0, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, bias=False) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + + if torch.distributed.get_rank() == 0: + assert q_name in state_dict and k_name in state_dict and v_name in state_dict + full_weight_q = state_dict[q_name] + full_weight_k = state_dict[k_name] + full_weight_v = state_dict[v_name] + + hidden_size_per_head = config.hidden_size // config.num_attention_heads + + if config.num_key_value_heads >= tp_size: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size + total_size = q_size_tp + 2 * kv_size_tp + if not bias: + new_weight_qkv = torch.empty( + total_size * tp_size, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + else: + new_weight_qkv = torch.empty(total_size * tp_size, dtype=params_dtype, device=get_device_id()) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + k_part = full_weight_k[i * kv_size_tp : (i + 1) * kv_size_tp] + v_part = full_weight_v[i * kv_size_tp : (i + 1) * kv_size_tp] + new_weight_qkv[i * total_size : (i + 1) * total_size].copy_( + torch.cat([q_part, k_part, v_part], dim=0) + ) + + else: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head + total_size = q_size_tp + 2 * kv_size_tp + if not bias: + new_weight_qkv = torch.empty( + total_size * tp_size, config.hidden_size, dtype=params_dtype, device=get_device_id() + ) + else: + new_weight_qkv = torch.empty(total_size * tp_size, dtype=params_dtype, device=get_device_id()) + for i in range(tp_size): + q_part = full_weight_q[i * q_size_tp : (i + 1) * q_size_tp] + start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head + end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head + k_part = full_weight_k[start_idx:end_idx] + v_part = full_weight_v[start_idx:end_idx] + new_weight_qkv[i * total_size : (i + 1) * total_size].copy_( + torch.cat([q_part, k_part, v_part], dim=0) + ) + + tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) + chunk_shape = tensor_chunk[0].shape + else: + chunk_shape = None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=0, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{q_name, k_name, v_name}] not in state_dict, skip loading") + return + + if tensor is None: + sync_tensor = torch.empty( + chunk_shape, + dtype=params_dtype, + device=get_device_id(), + requires_grad=False, + ) + else: + assert tensor.shape == chunk_shape, ( + f"rank #{torch.distributed.get_rank()} tensor {q_name} shape {tensor.shape} != {chunk_shape}" + ) + sync_tensor = torch.empty_like(tensor, device=get_device_id(), requires_grad=False) + + for i in range(tp_size): + if torch.distributed.get_rank() == 0: + sync_tensor.data.copy_(tensor_chunk[i]) + dist.broadcast(sync_tensor, src=0, group=mp_group) + if (i == tp_rank) and (tensor is not None): + tensor.data.copy_(sync_tensor) + + if dp_rank == 0: + # Embeddings + # ------------------- + print_rank_0("loading embeddings...") + gpt_model_module = _get_gpt_model(models[0]) + embed_tokens_weight = None + if pp_rank == 0: + embed_tokens_weight = gpt_model_module.model.embed_tokens.weight + _broadcast_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") + + # Transformer layers + # ------------------- + layer_map = _megatron_calc_layer_map(config) + + for layer in range(config.num_hidden_layers): + print_rank_0(f"loading layer #{layer}...") + layer_name = f"model.layers.{layer}" + dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] + + gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) + sync_layer = gpt_model_module.model.layers[dst_layer_idx] + + _broadcast_tensor( + sync_layer.input_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.input_layernorm.weight", + ) + + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_proj.weight", + f"{layer_name}.self_attn.k_proj.weight", + f"{layer_name}.self_attn.v_proj.weight", + ) + + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.bias if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.q_proj.bias", + f"{layer_name}.self_attn.k_proj.bias", + f"{layer_name}.self_attn.v_proj.bias", + bias=True, + ) + + _broadcast_tp_shard_tensor( + sync_layer.self_attn.o_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.self_attn.o_proj.weight", + chunk_dim=1, + ) + + _broadcast_tensor( + sync_layer.post_attention_layernorm.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.post_attention_layernorm.weight", + ) + + _broadcast_tp_shard_tensor_gate_up( + sync_layer.mlp.gate_up_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.gate_proj.weight", + f"{layer_name}.mlp.up_proj.weight", + ) + + _broadcast_tp_shard_tensor( + sync_layer.mlp.down_proj.weight if dst_pp_rank == pp_rank else None, + f"{layer_name}.mlp.down_proj.weight", + chunk_dim=1, + ) + # Final Layernorm + # ------------------- + print_rank_0("loading final layernorm...") + gpt_model_module = _get_gpt_model(models[-1]) + _broadcast_tensor( + getattr(gpt_model_module.model.norm, "weight", None), + "model.norm.weight", + ) + + if tie_word_embeddings: + print_rank_0("tie_word_embeddings skip load lm_head") + else: + print_rank_0("loading lm_head...") + lm_head_weight = None + if pp_rank + 1 == pp_size: + lm_head_weight = gpt_model_module.lm_head.weight + + if is_value_model: + if "lm_head.weight" in state_dict and state_dict["lm_head.weight"].shape[0] == 1: + _broadcast_tensor(lm_head_weight, "lm_head.weight") + print_rank_0("load lm_head from value_head weight") + elif "reward_head.weight" in state_dict and state_dict["reward_head.weight"].shape[0] == 1: + _broadcast_tensor(lm_head_weight, "reward_head.weight") + print_rank_0("load lm_head from value_head weight") + else: + _broadcast_tensor(None, "lm_head.weight") + print_rank_0("fail to match lm_head in value_model") + + else: + _broadcast_tp_shard_tensor(lm_head_weight, "lm_head.weight") + + dist.barrier() + # Broadcast weights inside data parallel groups + for wrapped_model in wrapped_models: + broadcast_params(wrapped_model) + + get_torch_device().empty_cache() + print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") diff --git a/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_saver.py b/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_saver.py new file mode 100644 index 0000000000000000000000000000000000000000..737f73b4c6163ee674d97466b4fb37b71df2534b --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/checkpoint_utils/qwen2_saver.py @@ -0,0 +1,448 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import torch +import torch.distributed as dist +from megatron.core import mpu +from megatron.core.distributed import DistributedDataParallel as LocalDDP +from megatron.core.transformer.module import Float16Module +from torch.nn.parallel import DistributedDataParallel as torchDDP + +from verl.utils.device import get_device_id, get_torch_device +from verl.utils.logger import print_rank_0 +from verl.utils.megatron_utils import unwrap_model + + +def _megatron_calc_global_rank(tp_rank: int = 0, dp_rank: int = 0, pp_rank: int = 0): + """given TP,DP,PP rank to get the global rank.""" + + tp_size = mpu.get_tensor_model_parallel_world_size() + dp_size = mpu.get_data_parallel_world_size() + pp_size = mpu.get_pipeline_model_parallel_world_size() + assert tp_size * dp_size * pp_size == torch.distributed.get_world_size(), ( + f"{tp_size} x {dp_size} x {pp_size} != {torch.distributed.get_world_size()}" + ) + # We only support TP-DP-PP grouping, for correctness when resharding + return (pp_rank * dp_size + dp_rank) * tp_size + tp_rank + + +def _megatron_calc_layer_map(config): + """Calculate the mapping of global layer_idx to local layer_idx + Returns: + layer_map (Dict: int -> tuple(int, int, int)): + mapping from the global layer index to + a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) + """ + from megatron.core import mpu + + pp_size = mpu.get_pipeline_model_parallel_world_size() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + + layer_map = dict() + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + for pp_rank_idx in range(pp_size): + for virtual_pp_rank_idx in range(virtual_pp_size): + layer_offset = ( + virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + pp_rank_idx * num_layers_per_model + ) + for layer_idx in range(num_layers_per_model): + layer_map[layer_offset + layer_idx] = ( + pp_rank_idx, + virtual_pp_rank_idx, + layer_idx, + ) + return layer_map + + +def merge_megatron_ckpt_qwen2(wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False): + """Merge sharded parameters of a Megatron module into a merged checkpoint. + + Args: + wrapped_models (list of megatron.core.distributed.DistributedDataParallel): + The local DDP wrapped megatron modules. + config (str or None): + HF config for model + dtype: model params type + is_value_model: if model is value model + tie_word_embeddings: tie_word_embeddings + Returns: + state_dict (dict): + The merged state_dict in rank 0, and an empty dictionary in other ranks. + """ + start_time = time.time() + + def _get_gpt_model(model): + return model + + dp_rank = mpu.get_data_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + pp_rank = mpu.get_pipeline_model_parallel_rank() + virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 + mp_group = mpu.get_model_parallel_group() + + if dist.get_rank() == 0: + assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" + assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" + assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" + + if not isinstance(wrapped_models, list | tuple): + wrapped_models = list(wrapped_models) + + assert len(wrapped_models) == virtual_pp_size + num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size + assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers + + models = [None] * len(wrapped_models) + + for i, wrapped_model in enumerate(wrapped_models): + models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) + assert len(models[i].model.layers) == num_layers_per_model, ( + "len model layers {} not equal to num_layers_per_model {}".format( + len(models[i].model.layers), num_layers_per_model + ) + ) + + state_dict = dict() + + def _get_cpu_tensor(tensor: torch.Tensor): + if tensor is None: + return None + if tensor.device == torch.device("cpu"): + return tensor.detach().clone() + return tensor.detach().cpu() + + def _broadcast_tensor(tensor, name, src_pp_rank) -> torch.Tensor: + """broadcast tensor across mp_group""" + nonlocal state_dict + nonlocal mp_group + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) + + if torch.distributed.get_rank() == src_rank: + if tensor is None: + weight = None + tensor_shape = None + else: + weight = tensor + tensor_shape = weight.shape + else: + weight = None + tensor_shape = None + + obj_list = [tensor_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + tensor_shape = obj_list[0] + + if tensor_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tensor:[{name}] not exist, skip collect") + return + + if weight is None: + weight = torch.empty( + tensor_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + dist.broadcast(weight, src=src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + state_dict[name] = _get_cpu_tensor(weight) + + def _broadcast_tp_shard_tensor(tensor, name, src_pp_rank, concat_dim=0, mutate_func=None) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=concat_dim) + if mutate_func is not None: + full_tensor = mutate_func(full_tensor) + state_dict[name] = full_tensor + + def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name, src_pp_rank) -> torch.Tensor: + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=0) + intermediate_size_tp = config.intermediate_size // tp_size + gate_weight_list = [] + up_weight_list = [] + for i in range(tp_size): + gate_up_weight_tp = full_tensor[intermediate_size_tp * 2 * i : intermediate_size_tp * 2 * (i + 1)] + gate_weight_tp = gate_up_weight_tp[:intermediate_size_tp] + up_weight_tp = gate_up_weight_tp[intermediate_size_tp:] + gate_weight_list.append(gate_weight_tp) + up_weight_list.append(up_weight_tp) + + state_dict[gate_name] = torch.cat(gate_weight_list, dim=0) + state_dict[up_name] = torch.cat(up_weight_list, dim=0) + + def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, src_pp_rank): + """broadcast tensor in tp shards across mp_group""" + nonlocal state_dict + nonlocal mp_group + tp_size = mpu.get_tensor_model_parallel_world_size() + src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) + + chunk_shape = tensor.shape if torch.distributed.get_rank() == src_rank else None + + obj_list = [chunk_shape] + dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) + chunk_shape = obj_list[0] + if chunk_shape is None: + # all or none ranks in the mp_group should reach here + print_rank_0(f"tp_shard tensor:[{q_name}] not exist, skip collecting") + return + + buffer_tensor = torch.empty( + chunk_shape, + dtype=dtype, + device=get_device_id(), + requires_grad=False, + ) + + chunk_tensors = [None] * tp_size + + for i in range(tp_size): + cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) + sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor + dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) + + if torch.distributed.get_rank() == 0: + chunk_tensors[i] = _get_cpu_tensor(sync_tensor) + + if torch.distributed.get_rank() == 0: + full_tensor = torch.concat(chunk_tensors, dim=0) + q_weight_list = [] + k_weight_list = [] + v_weight_list = [] + hidden_size_per_head = config.hidden_size // config.num_attention_heads + + if config.num_key_value_heads >= tp_size: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size + total_size = q_size_tp + 2 * kv_size_tp + for i in range(tp_size): + qkv_part = full_tensor[i * total_size : (i + 1) * total_size] + q_part = qkv_part[:q_size_tp] + k_part = qkv_part[q_size_tp : q_size_tp + kv_size_tp] + v_part = qkv_part[q_size_tp + kv_size_tp : total_size] + q_weight_list.append(q_part) + k_weight_list.append(k_part) + v_weight_list.append(v_part) + else: + q_size_tp = config.hidden_size // tp_size + kv_size_tp = hidden_size_per_head + total_size = q_size_tp + 2 * kv_size_tp + for i in range(tp_size): + qkv_part = full_tensor[i * total_size : (i + 1) * total_size] + q_part = qkv_part[:q_size_tp] + k_part = qkv_part[q_size_tp : q_size_tp + kv_size_tp] + v_part = qkv_part[q_size_tp + kv_size_tp : total_size] + q_weight_list.append(q_part) + if i * config.num_key_value_heads % tp_size == 0: + k_weight_list.append(k_part) + v_weight_list.append(v_part) + + state_dict[q_name] = torch.cat(q_weight_list, dim=0) + state_dict[k_name] = torch.cat(k_weight_list, dim=0) + state_dict[v_name] = torch.cat(v_weight_list, dim=0) + + # empty cache before collecting weights + get_torch_device().empty_cache() + # Embeddings + # ------------------- + if dp_rank == 0: + # Embeddings + # ------------------- + print_rank_0("collecting embeddings...") + gpt_model_module = _get_gpt_model(models[0]) + _broadcast_tp_shard_tensor( + gpt_model_module.model.embed_tokens.weight if pp_rank == 0 else None, + "model.embed_tokens.weight", + src_pp_rank=0, + ) + + # Transformer layers + # ------------------- + layer_map = _megatron_calc_layer_map(config) + for layer in range(config.num_hidden_layers): + print_rank_0(f"collecting layer #{layer}...") + layer_name = f"model.layers.{layer}" + src_pp_rank, src_virtual_pp_rank, src_layer_idx = layer_map[layer] + + gpt_model_module = _get_gpt_model(models[src_virtual_pp_rank]) + sync_layer = gpt_model_module.model.layers[src_layer_idx] + + _broadcast_tensor( + sync_layer.input_layernorm.weight, + f"{layer_name}.input_layernorm.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.weight, + f"{layer_name}.self_attn.q_proj.weight", + f"{layer_name}.self_attn.k_proj.weight", + f"{layer_name}.self_attn.v_proj.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor_qkv( + sync_layer.self_attn.qkv_proj.bias, + f"{layer_name}.self_attn.q_proj.bias", + f"{layer_name}.self_attn.k_proj.bias", + f"{layer_name}.self_attn.v_proj.bias", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor( + sync_layer.self_attn.o_proj.weight, + f"{layer_name}.self_attn.o_proj.weight", + concat_dim=1, + src_pp_rank=src_pp_rank, + ) + + _broadcast_tensor( + sync_layer.post_attention_layernorm.weight, + f"{layer_name}.post_attention_layernorm.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor_gate_up( + sync_layer.mlp.gate_up_proj.weight, + f"{layer_name}.mlp.gate_proj.weight", + f"{layer_name}.mlp.up_proj.weight", + src_pp_rank=src_pp_rank, + ) + + _broadcast_tp_shard_tensor( + sync_layer.mlp.down_proj.weight, + f"{layer_name}.mlp.down_proj.weight", + concat_dim=1, + src_pp_rank=src_pp_rank, + ) + + # Final Layernorm + # ------------------- + print_rank_0("collecting final layernorm...") + gpt_model_module = _get_gpt_model(models[-1]) + _broadcast_tensor( + getattr(gpt_model_module.model.norm, "weight", None), + "model.norm.weight", + src_pp_rank=pp_size - 1, + ) + + if tie_word_embeddings: + print_rank_0("tie word embedding skip load lm_head...") + else: + print_rank_0("collecting lm_head...") + + if is_value_model: + _broadcast_tensor( + gpt_model_module.lm_head.weight if pp_rank == pp_size - 1 else None, + "lm_head.weight", + src_pp_rank=pp_size - 1, + ) + _broadcast_tensor( + gpt_model_module.reward_head.weight + if pp_rank == pp_size - 1 and getattr(gpt_model_module, "reward_weight", None) is not None + else None, + "reward_head.weight", + src_pp_rank=pp_size - 1, + ) + + else: + _broadcast_tp_shard_tensor( + getattr(gpt_model_module.lm_head, "weight", None) if pp_rank == pp_size - 1 else None, + "lm_head.weight", + src_pp_rank=pp_size - 1, + ) + + dist.barrier() + + get_torch_device().empty_cache() + if torch.distributed.get_rank() == 0: + for k, v in state_dict.items(): + if dtype != v.dtype: + state_dict[k] = v.to(dtype) + + print_rank_0(f"merge megatron ckpt done, time elapsed {time.time() - start_time}s") + return state_dict diff --git a/e2c/verl/verl/models/qwen2/megatron/layers/__init__.py b/e2c/verl/verl/models/qwen2/megatron/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..263ea596fa758fdef2201e9e99e4a5c7d435e434 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/layers/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .parallel_attention import ParallelQwen2Attention +from .parallel_decoder import ParallelQwen2DecoderLayer, ParallelQwen2DecoderLayerRmPad +from .parallel_mlp import ParallelQwen2MLP +from .parallel_rmsnorm import ParallelQwen2RMSNorm + +__all__ = [ + "ParallelQwen2Attention", + "ParallelQwen2DecoderLayer", + "ParallelQwen2DecoderLayerRmPad", + "ParallelQwen2MLP", + "ParallelQwen2RMSNorm", +] diff --git a/e2c/verl/verl/models/qwen2/megatron/layers/parallel_attention.py b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..702c429c24343dc8d0fb634fa0ee0b48673b1cc6 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_attention.py @@ -0,0 +1,399 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Optional + +import torch.nn.functional as F +from einops import rearrange +from transformers.utils import is_flash_attn_2_available + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa +import torch +from flash_attn.layers.rotary import apply_rotary_emb +from megatron.core import ModelParallelConfig, tensor_parallel +from megatron.core import parallel_state as mpu +from torch import nn +from transformers import Qwen2Config + +from verl.models.qwen2.megatron.layers.parallel_linear import QKVParallelLinear +from verl.utils.megatron import tensor_parallel as tp_utils + + +class Qwen2RotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self._set_cos_sin_cache( + seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() + ) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if seq_len > self.max_seq_len_cached: + self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) + + return ( + self.cos_cached[:seq_len].to(dtype=x.dtype), + self.sin_cached[:seq_len].to(dtype=x.dtype), + ) + + +class Qwen2LinearScalingRotaryEmbedding(Qwen2RotaryEmbedding): + """Qwen2RotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev""" + + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): + self.scaling_factor = scaling_factor + super().__init__(dim, max_position_embeddings, base, device) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + t = t / self.scaling_factor + + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + + +class Qwen2DynamicNTKScalingRotaryEmbedding(Qwen2RotaryEmbedding): + """Qwen2RotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla""" + + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): + self.scaling_factor = scaling_factor + super().__init__(dim, max_position_embeddings, base, device) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + + if seq_len > self.max_position_embeddings: + base = self.base * ( + (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1) + ) ** (self.dim / (self.dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids): + cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class ParallelQwen2Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): + super().__init__() + self.config = config + self.megatron_config = megatron_config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + + # assign values after tp + tp_size = mpu.get_tensor_model_parallel_world_size() + assert self.num_heads % tp_size == 0, ( + f"num_head must be divisible by tp_size. Got num_head={self.num_heads}, tp_size={tp_size}" + ) + assert self.num_key_value_heads % tp_size == 0, ( + f"num_key_value_heads must be divisible by tp_size. Got num_key_value_heads=" + f"{self.num_key_value_heads}, tp_size={tp_size}" + ) + + self.num_heads_per_tp = self.num_heads // tp_size + self.num_key_value_heads_per_tp = self.num_key_value_heads // tp_size + self.hidden_size_per_tp = self.hidden_size // tp_size + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and " + f"`num_heads`: {self.num_heads})." + ) + + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + row_kwargs = tp_utils.get_default_kwargs_for_row_parallel_linear() + + if megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + assert row_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, megatron_config) + tp_utils.update_kwargs_with_config(row_kwargs, megatron_config) + + # [self.q_size, self.k_size, self.v_size] + self.qkv_proj = QKVParallelLinear( + input_size=self.hidden_size, + num_heads=self.num_heads, + num_key_value_heads=self.num_key_value_heads, + head_dim=self.head_dim, + # bias=config.attention_bias, + bias=True, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + + self.q_size = self.num_heads_per_tp * self.head_dim + self.k_size = self.num_key_value_heads_per_tp * self.head_dim + self.v_size = self.num_key_value_heads_per_tp * self.head_dim + + self.o_proj = tensor_parallel.RowParallelLinear( + input_size=self.num_heads * self.head_dim, + output_size=self.hidden_size, + # bias=config.attention_bias, + bias=False, + input_is_parallel=True, + skip_bias_add=False, + **row_kwargs, + ) + + self._init_rope() + + def _init_rope(self): + self.rotary_emb = Qwen2RotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + qkv = self.qkv_proj(hidden_states)[0] + query_states, key_states, value_states = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + + query_states = query_states.view(bsz, q_len, self.num_heads_per_tp, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads_per_tp, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads_per_tp, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads_per_tp, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads_per_tp, q_len, kv_seq_len)}, " + f"but is {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads_per_tp, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads_per_tp, q_len, self.head_dim)}, " + f"but is {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size_per_tp) + attn_output = self.o_proj(attn_output)[0] + return attn_output + + +""" +Remove padding Attention +- Using Flash-attn 2 +- Compatible with sequence parallel +""" + + +def apply_rotary_pos_emb_rmpad(q, k, cos, sin, position_ids, indices, sequence_length): + batch_size = position_ids.shape[0] + + q = pad_input(q, indices, batch_size, sequence_length) # (batch_size, seqlen, num_head, head_dim) + k = pad_input(k, indices, batch_size, sequence_length) + cos = cos[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] + sin = sin[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + + q_embed = index_first_axis(rearrange(q_embed, "b s ... -> (b s) ..."), indices) + k_embed = index_first_axis(rearrange(k_embed, "b s ... -> (b s) ..."), indices) + + return q_embed, k_embed + + +# use flash-attn rotary embeddings with rmpad +# cos/sin shoudl be: (seq_length, rotary_dim / 2) +def apply_rotary_pos_emb_rmpad_flash(q, k, cos, sin, cu_seqlens, max_seqlen): + q_embed = apply_rotary_emb( + q, cos, sin, interleaved=False, inplace=False, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen + ) + k_embed = apply_rotary_emb( + k, cos, sin, interleaved=False, inplace=False, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen + ) + return q_embed, k_embed + + +class ParallelQwen2AttentionRmPad(ParallelQwen2Attention): + def forward( + self, + hidden_states: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + sequence_length: int = None, + indices: torch.Tensor = None, + cu_seqlens: torch.Tensor = None, + max_seqlen_in_batch: int = None, + ): + total_nnz, _, _ = hidden_states.size() # This is the total_nnz padded after sequence parallel + + if self.megatron_config.sequence_parallel: + total_nnz = total_nnz * mpu.get_tensor_model_parallel_world_size() + + qkv = self.qkv_proj(hidden_states)[0] + query_states, key_states, value_states = qkv.split( + [self.q_size, self.k_size, self.v_size], dim=-1 + ) # (total_nnz, 1, hidden_size) + + if self.megatron_config.sequence_parallel: + sequence_parallel_pad = total_nnz - cu_seqlens[-1] + total_nnz = cu_seqlens[-1] # total_nnz before sp padding + query_states = query_states[:total_nnz] + key_states = key_states[:total_nnz] + value_states = value_states[:total_nnz] + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dime x hidden_dim + # therefore we just need to keep the original shape + query_states = query_states.view(total_nnz, self.num_heads_per_tp, self.head_dim) + key_states = key_states.view(total_nnz, self.num_key_value_heads_per_tp, self.head_dim) + value_states = value_states.view(total_nnz, self.num_key_value_heads_per_tp, self.head_dim) + + cos, sin = self.rotary_emb(value_states, seq_len=sequence_length) + cos, sin = cos[:, : cos.shape[1] // 2], sin[:, : sin.shape[1] // 2] # flash attn only needs half + query_states, key_states = apply_rotary_pos_emb_rmpad_flash( + query_states, key_states, cos, sin, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen_in_batch + ) + # query_states, key_states = apply_rotary_pos_emb_rmpad(query_states, key_states, cos, sin, + # position_ids, indices, + + # It is recommended to use dropout with FA according to the docs + # when training. + dropout_rate = 0.0 # if not self.training else self.attn_dropout + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (Qwen2RMSNorm handles it correctly) + input_dtype = query_states.dtype + if input_dtype == torch.float32: + query_states = query_states.to(torch.float16) + key_states = key_states.to(torch.float16) + value_states = value_states.to(torch.float16) + + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=max_seqlen_in_batch, + max_seqlen_k=max_seqlen_in_batch, + dropout_p=dropout_rate, + softmax_scale=None, + causal=True, + ) + + attn_output_unpad = attn_output_unpad.to(input_dtype) + attn_output_unpad = attn_output_unpad.reshape(total_nnz, 1, self.hidden_size_per_tp).contiguous() + + # sequence parallel reduce_scatter is performed inside RowColumnParallel if enabled + # Here we need to repad + if self.megatron_config.sequence_parallel: + attn_output_unpad = F.pad(attn_output_unpad, pad=(0, 0, 0, 0, 0, sequence_parallel_pad)) + + attn_output_unpad = self.o_proj(attn_output_unpad)[0] + return attn_output_unpad diff --git a/e2c/verl/verl/models/qwen2/megatron/layers/parallel_decoder.py b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..3c8a2a6ee946eb014658006a2da6d2d602c51063 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_decoder.py @@ -0,0 +1,150 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +from megatron.core import ModelParallelConfig +from torch import nn +from transformers import Qwen2Config + +from verl.utils.megatron_utils import TransformerConfig, convert_config + +from .parallel_attention import ParallelQwen2Attention, ParallelQwen2AttentionRmPad +from .parallel_mlp import ParallelQwen2MLP +from .parallel_rmsnorm import ParallelQwen2RMSNorm + + +class ParallelQwen2DecoderLayer(nn.Module): + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig, layer_idx: int): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.self_attn = ParallelQwen2Attention(config=config, megatron_config=megatron_config) + + self.mlp = ParallelQwen2MLP(config, megatron_config=megatron_config) + self.input_layernorm = ParallelQwen2RMSNorm(config, megatron_config) + self.post_attention_layernorm = ParallelQwen2RMSNorm(config, megatron_config) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Note: sequence parallel is hidden inside ColumnParallelLinear + # reduce scatter is hidden inside RowParallelLinear + + # Self Attention + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + ) + + # TODO: add sequence parallel operator reduce_scatter here + + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + + # TODO: add sequence parallel operator all_gather here + + hidden_states = self.mlp(hidden_states) + + # TODO: add sequence parallel operator reduce_scatter here + + hidden_states = residual + hidden_states + + outputs = hidden_states + + return outputs + + +class ParallelQwen2DecoderLayerRmPad(nn.Module): + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig, layer_idx: int): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.hidden_size = config.hidden_size + self.layer_idx = layer_idx + self.self_attn = ParallelQwen2AttentionRmPad(config=config, megatron_config=megatron_config) + + self.mlp = ParallelQwen2MLP(config, megatron_config=megatron_config) + self.input_layernorm = ParallelQwen2RMSNorm(config, megatron_config) + self.post_attention_layernorm = ParallelQwen2RMSNorm(config, megatron_config) + + def forward( + self, + hidden_states: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + sequence_length: int = None, + indices: torch.Tensor = None, + cu_seqlens: int = None, + max_seqlen_in_batch: int = None, + ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]: + residual = hidden_states # (total_nnz // sp, 1, hidden_size) + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + # (total_nnz // sp, 1, hidden_size) -> all-gather (total_nnz, 1, hidden_size) + # -> col + row -> reduce-scatter -> (total_nnz // sp, 1, hidden_size) + hidden_states = self.self_attn( + hidden_states=hidden_states, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + hidden_states = residual + hidden_states + + # Fully Connected + # shape changes same as attn + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = hidden_states + + return outputs diff --git a/e2c/verl/verl/models/qwen2/megatron/layers/parallel_linear.py b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_linear.py new file mode 100644 index 0000000000000000000000000000000000000000..e6d4a09f43013ed75feb03fdb427bc8ad86db093 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_linear.py @@ -0,0 +1,79 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023 The vLLM team. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/linear.py + + +from megatron.core import tensor_parallel + + +class QKVParallelLinear(tensor_parallel.ColumnParallelLinear): + def __init__( + self, + input_size, + num_heads, + num_key_value_heads, + head_dim, + *, + bias=True, + gather_output=True, + skip_bias_add=False, + **kwargs, + ): + # Keep input parameters, and already restrict the head numbers + self.input_size = input_size + self.q_output_size = num_heads * head_dim + self.kv_output_size = num_key_value_heads * head_dim + self.head_dim = head_dim + self.gather_output = gather_output + self.skip_bias_add = skip_bias_add + + input_size = self.input_size + output_size = (num_heads + 2 * num_key_value_heads) * self.head_dim + + super().__init__( + input_size=input_size, + output_size=output_size, + bias=bias, + gather_output=gather_output, + skip_bias_add=skip_bias_add, + **kwargs, + ) + + +class MergedColumnParallelLinear(tensor_parallel.ColumnParallelLinear): + def __init__( + self, + input_size, + gate_ouput_size, + up_output_size, + *, + bias=True, + gather_output=True, + skip_bias_add=False, + **kwargs, + ): + # Keep input parameters, and already restrict the head numbers + self.input_size = input_size + self.output_size = gate_ouput_size + up_output_size + self.gather_output = gather_output + self.skip_bias_add = skip_bias_add + + super().__init__( + input_size=self.input_size, + output_size=self.output_size, + bias=bias, + gather_output=gather_output, + skip_bias_add=skip_bias_add, + **kwargs, + ) diff --git a/e2c/verl/verl/models/qwen2/megatron/layers/parallel_mlp.py b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_mlp.py new file mode 100644 index 0000000000000000000000000000000000000000..672908a21ae8c8e69c0536eda7fadd0431cba5fe --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_mlp.py @@ -0,0 +1,74 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from megatron.core import ModelParallelConfig, tensor_parallel +from megatron.core import parallel_state as mpu +from torch import nn +from transformers.activations import ACT2FN + +from verl.models.qwen2.megatron.layers.parallel_linear import MergedColumnParallelLinear +from verl.utils.megatron import tensor_parallel as tp_utils + + +class ParallelQwen2MLP(nn.Module): + def __init__(self, config, megatron_config: ModelParallelConfig = None) -> None: + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + # The weight is only [hidden_size, intermediate_size // model_parallel_world_size] + + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + row_kwargs = tp_utils.get_default_kwargs_for_row_parallel_linear() + + if megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + assert row_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(row_kwargs, megatron_config) + tp_utils.update_kwargs_with_config(column_kwargs, megatron_config) + + tp_size = mpu.get_tensor_model_parallel_world_size() + + self.gate_up_proj = MergedColumnParallelLinear( + input_size=self.hidden_size, + gate_ouput_size=self.intermediate_size, + up_output_size=self.intermediate_size, + bias=False, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + self.gate_size = self.intermediate_size // tp_size + + self.down_proj = tensor_parallel.RowParallelLinear( + input_size=self.intermediate_size, + output_size=self.hidden_size, + bias=False, + input_is_parallel=True, + skip_bias_add=False, + **row_kwargs, + ) + + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + gate_up = self.gate_up_proj(x)[0] + gate, up = gate_up.split(self.gate_size, dim=-1) + return self.down_proj(self.act_fn(gate) * up)[0] diff --git a/e2c/verl/verl/models/qwen2/megatron/layers/parallel_rmsnorm.py b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_rmsnorm.py new file mode 100644 index 0000000000000000000000000000000000000000..2f4c90dd44e2b72f1116e3c097e52efca5567129 --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/layers/parallel_rmsnorm.py @@ -0,0 +1,48 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numbers + +import torch +from apex.normalization.fused_layer_norm import fused_rms_norm_affine +from megatron.core import ModelParallelConfig +from torch import nn +from transformers import Qwen2Config + +from verl.utils.megatron import sequence_parallel as sp_utils + + +class ParallelQwen2RMSNorm(nn.Module): + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): + """ + Qwen2RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + if isinstance(config.hidden_size, numbers.Integral): + normalized_shape = (config.hidden_size,) + self.normalized_shape = torch.Size(normalized_shape) + self.weight = nn.Parameter(torch.ones(self.normalized_shape)) + self.variance_epsilon = config.rms_norm_eps + + if megatron_config.sequence_parallel: + sp_utils.mark_parameter_as_sequence_parallel(self.weight) + + def forward(self, hidden_states): + return fused_rms_norm_affine( + input=hidden_states, + weight=self.weight, + normalized_shape=self.normalized_shape, + eps=self.variance_epsilon, + memory_efficient=True, + ) diff --git a/e2c/verl/verl/models/qwen2/megatron/modeling_qwen2_megatron.py b/e2c/verl/verl/models/qwen2/megatron/modeling_qwen2_megatron.py new file mode 100644 index 0000000000000000000000000000000000000000..92e81be8d76c5484cbe434d4268ffcf3f397bb8c --- /dev/null +++ b/e2c/verl/verl/models/qwen2/megatron/modeling_qwen2_megatron.py @@ -0,0 +1,737 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Qwen2 model.""" + +from typing import Optional + +import torch +import torch.utils.checkpoint +from megatron.core import ModelParallelConfig, mpu, parallel_state, tensor_parallel +from torch import nn +from transformers.modeling_outputs import BaseModelOutputWithPast +from transformers.models.qwen2.configuration_qwen2 import Qwen2Config +from transformers.models.qwen2.modeling_qwen2 import CausalLMOutputWithPast + +from verl.utils.device import get_device_name +from verl.utils.megatron import sequence_parallel as sp_utils +from verl.utils.megatron import tensor_parallel as tp_utils +from verl.utils.megatron_utils import TransformerConfig, convert_config + +from .layers import ParallelQwen2DecoderLayer, ParallelQwen2DecoderLayerRmPad, ParallelQwen2RMSNorm + +""" +TODO: +1. Add weight initialization. Here we need to be careful on TP weight init. +2. Add sequence parallel +3. Load checkpoint from Qwen2 pretrained checkpoint +""" + + +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len) + + +# Copied from transformers.models.bart.modeling_bart._expand_mask +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +class ParallelQwen2Model(nn.Module): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`] + + Args: + config: Qwen2Config + """ + + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() + if megatron_config is not None: + assert embedding_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(embedding_kwargs, megatron_config) + self.embed_tokens = tensor_parallel.VocabParallelEmbedding( + num_embeddings=config.vocab_size, embedding_dim=config.hidden_size, **embedding_kwargs + ) + + self.layers = nn.ModuleList( + [ParallelQwen2DecoderLayer(config, megatron_config) for _ in range(config.num_hidden_layers)] + ) + self.norm = ParallelQwen2RMSNorm(config, megatron_config) + + # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + ) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | BaseModelOutputWithPast: + """ + + Args: + input_ids: input ids. shape (batch_size, seq_length) + attention_mask: attention_mask. shape (batch_size, seq_length) + position_ids: position ids. shape (batch_size, seq_length) + + Returns: + + """ + batch_size, seq_length = input_ids.shape + inputs_embeds = self.embed_tokens(input_ids) + # embed positions + + attention_mask = self._prepare_decoder_attention_mask(attention_mask, (batch_size, seq_length), inputs_embeds) + + hidden_states = inputs_embeds + + for idx, decoder_layer in enumerate(self.layers): + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + ) + + hidden_states = layer_outputs + + hidden_states = self.norm(hidden_states) + + return hidden_states + + +class ParallelQwen2ForCausalLM(nn.Module): + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.model = ParallelQwen2Model(config, megatron_config=megatron_config) + self.vocab_size = config.vocab_size + + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + + self.lm_head = tensor_parallel.ColumnParallelLinear( + input_size=config.hidden_size, + output_size=config.vocab_size, + bias=False, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + ```""" + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + ) + + hidden_states = outputs + logits = self.lm_head(hidden_states)[0] + + logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) + + logits = logits.float() + return CausalLMOutputWithPast( + loss=None, + logits=logits, + past_key_values=None, + hidden_states=None, + attentions=None, + ) + + +from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + +class ParallelQwen2ModelRmPad(nn.Module): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`] + + Args: + config: Qwen2Config + """ + + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() + self.megatron_config = megatron_config + if megatron_config is not None: + assert embedding_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) + self.embed_tokens = tensor_parallel.VocabParallelEmbedding( + num_embeddings=config.vocab_size, embedding_dim=config.hidden_size, **embedding_kwargs + ) + + self.layers = nn.ModuleList( + [ParallelQwen2DecoderLayerRmPad(config, megatron_config) for _ in range(config.num_hidden_layers)] + ) + self.norm = ParallelQwen2RMSNorm(config, megatron_config) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + sequence_length: int = None, + indices: torch.Tensor = None, + cu_seqlens: int = None, + max_seqlen_in_batch: int = None, + ) -> tuple | BaseModelOutputWithPast: + """ + + Args: + input_ids: input ids. shape (1, totol_nnz) + position_ids: position ids. shape (batch_size, seq_length) + + Returns: + + """ + inputs_embeds = self.embed_tokens(input_ids) # (1, total_nnz) -> (1, total_nnz, hidden_size) + + # (1, total_nnz, hidden_size) -> (total_nnz, 1, hidden_size) -> (total_nnz // sp, 1, hidden_size) + inputs_embeds = inputs_embeds.transpose(0, 1) + if self.megatron_config.sequence_parallel: + inputs_embeds = tensor_parallel.scatter_to_sequence_parallel_region(inputs_embeds) + + hidden_states = inputs_embeds + for idx, decoder_layer in enumerate(self.layers): + layer_outputs = decoder_layer( + hidden_states, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + hidden_states = layer_outputs + + hidden_states = self.norm(hidden_states) + + return hidden_states + + +class ParallelQwen2ForCausalLMRmPad(nn.Module): + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.megatron_config = megatron_config + self.model = ParallelQwen2ModelRmPad(config, megatron_config=megatron_config) + self.vocab_size = config.vocab_size + self._init_head(config) + + def _init_head(self, config: Qwen2Config): + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if self.megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + self.lm_head = tensor_parallel.ColumnParallelLinear( + input_size=config.hidden_size, + output_size=config.vocab_size, + bias=False, + gather_output=False, + skip_bias_add=False, + **column_kwargs, + ) + + def _forward_head(self, hidden_states): + # all_gather from sequence parallel region is performed inside lm_head + logits = self.lm_head(hidden_states)[0] + logits = logits.float() # (total_nnz_padded, 1, vocab_size // tp) + logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) # (total_nnz_padded, 1, vocab_size) + return logits + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + ```""" + batch_size, sequence_length = input_ids.shape + + # remove padding here + input_ids, indices, cu_seqlens, max_seqlen_in_batch, *_ = unpad_input( + input_ids.unsqueeze(dim=-1), attention_mask + ) # (total_nnz, 1) + + # pad input_ids to multiple of tp for all tp ranks + # TODO: for better performance, the sp padding should be removed at each layer. Not sure the performance gap + if self.megatron_config.sequence_parallel: + input_ids = sp_utils.pad_to_sequence_parallel(input_ids) + + input_ids = input_ids.transpose(0, 1) # (1, total_nnz+pad) + + outputs = self.model( + input_ids=input_ids, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + hidden_states = outputs + + logits = self._forward_head(hidden_states) + + # remove padding from sequence parallel + if self.megatron_config.sequence_parallel: + totol_nnz = cu_seqlens[-1] + logits = logits[:totol_nnz] # (total_nnz_padded) + + logits = torch.squeeze(logits, dim=1) # remove the artificial batch dimension + # add removed padding back + logits = pad_input( + logits, indices, batch_size, seqlen=sequence_length + ) # (batch_size, sequence_length, vocab_size) + + return CausalLMOutputWithPast( + loss=None, + logits=logits, + past_key_values=None, + hidden_states=None, + attentions=None, + ) + + +class ParallelQwen2ForValueRmPad(ParallelQwen2ForCausalLMRmPad): + def _init_head(self, config): + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if self.megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + self.lm_head = nn.Linear(in_features=config.hidden_size, out_features=1, bias=False) + # lm_head is effectively the same as sequence parallel + sp_utils.mark_parameter_as_sequence_parallel(self.lm_head.weight) + + def _forward_head(self, hidden_states): + logits = self.lm_head(hidden_states) # (total_nnz_padded // tp, 1, 1) + logits = logits.float() + if self.megatron_config.sequence_parallel: + logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) + return logits + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + output = super().forward(input_ids, attention_mask, position_ids) + output.logits = torch.squeeze(output.logits, dim=-1) + return output + + +""" +Support pipeline parallelism +""" + + +class ParallelQwen2ModelRmPadPP(nn.Module): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`] + This model definition supports pipeline parallelism. To support pp and vpp, + - This model only contains layer in this pp stage and vpp chunk + - When calling get_model in Megatron, this rank will instantiate all the vpp chunks in this pp. + Args: + config: Qwen2Config + """ + + def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig, pre_process, post_process): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.pre_process = pre_process + self.post_process = post_process + self.megatron_config = megatron_config + embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() + if megatron_config is not None: + assert embedding_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) + if pre_process: + self.embed_tokens = tensor_parallel.VocabParallelEmbedding( + num_embeddings=config.vocab_size, embedding_dim=config.hidden_size, **embedding_kwargs + ) + else: + self.embed_tokens = None + + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = megatron_config.pipeline_model_parallel_size + self.num_layer_per_pp = config.num_hidden_layers // pp_size + vpp_size = megatron_config.virtual_pipeline_model_parallel_size + vpp_rank = mpu.get_virtual_pipeline_model_parallel_rank() + + if vpp_size is not None: + self.num_layer_vpp_chunk = self.num_layer_per_pp // vpp_size + self.num_layer_this_model = self.num_layer_vpp_chunk + offset = vpp_rank * (config.num_hidden_layers // vpp_size) + (pp_rank * self.num_layer_vpp_chunk) + else: + self.num_layer_this_model = self.num_layer_per_pp + offset = pp_rank * self.num_layer_per_pp + + self.layers = nn.ModuleList() + for i in range(self.num_layer_this_model): + layer = ParallelQwen2DecoderLayerRmPad(config, megatron_config, layer_idx=i + offset) + self.layers.add_module(f"{i}", layer) + + if post_process: + self.norm = ParallelQwen2RMSNorm(config, megatron_config) + else: + self.norm = None + + def set_input_tensor(self, input_tensor): + """Set input tensor to be used instead of forward()'s input. + + When doing pipeline parallelism the input from the previous + stage comes from communication, not from the input, so the + model's forward_step_func won't have it. This function is thus + used by internal code to bypass the input provided by the + forward_step_func""" + self.input_tensor = input_tensor + + def forward( + self, + input_ids: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + sequence_length: int = None, + indices: torch.Tensor = None, + cu_seqlens: int = None, + max_seqlen_in_batch: int = None, + ) -> tuple | BaseModelOutputWithPast: + """ + + Args: + input_ids: input ids. shape (1, totol_nnz) + position_ids: position ids. shape (batch_size, seq_length) + + Returns: + + """ + if self.pre_process: + inputs_embeds = self.embed_tokens(input_ids) # (1, total_nnz) -> (1, total_nnz, hidden_size) + + # vocab parallel embedding will not do sequence parallel reduce-scatter in open source megatron + # so need to deal with it by handle here: + # (1, total_nnz, hidden_size) -> (total_nnz, 1, hidden_size) -> (total_nnz // sp, 1, hidden_size) + inputs_embeds = inputs_embeds.transpose(0, 1) + if self.megatron_config.sequence_parallel: + inputs_embeds = tensor_parallel.scatter_to_sequence_parallel_region(inputs_embeds) + + hidden_states = inputs_embeds + else: + # self.hidden_states should be passed by Megatron + hidden_states = self.input_tensor + + for idx, decoder_layer in enumerate(self.layers): + layer_outputs = decoder_layer( + hidden_states, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + hidden_states = layer_outputs + + if self.post_process: + hidden_states = self.norm(hidden_states) + + return hidden_states + + +class ParallelQwen2ForCausalLMRmPadPP(nn.Module): + def __init__( + self, + config: Qwen2Config, + megatron_config: ModelParallelConfig, + pre_process, + post_process, + share_embeddings_and_output_weights, + ): + super().__init__() + self.config: TransformerConfig = convert_config(config, megatron_config) + self.megatron_config = megatron_config + self.model = ParallelQwen2ModelRmPadPP( + config, megatron_config=megatron_config, pre_process=pre_process, post_process=post_process + ) + self.share_embeddings_and_output_weights = share_embeddings_and_output_weights + self.vocab_size = config.vocab_size + self.pre_process = pre_process + self.post_process = post_process + if post_process: + self._init_head(config) + if pre_process or post_process: + self.setup_embeddings_and_output_layer() + + def set_input_tensor(self, input_tensor): + """Set input tensor to be used instead of forward()'s input. + + When doing pipeline parallelism the input from the previous + stage comes from communication, not from the input, so the + model's forward_step_func won't have it. This function is thus + used by internal code to bypass the input provided by the + forward_step_func""" + assert len(input_tensor) == 1 + self.model.set_input_tensor(input_tensor[0]) + + def _init_head(self, config): + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if self.megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + self.lm_head = tensor_parallel.ColumnParallelLinear( + input_size=config.hidden_size, + output_size=config.vocab_size, + bias=False, + gather_output=False, + skip_bias_add=False, + skip_weight_param_allocation=self.pre_process and self.share_embeddings_and_output_weights, + **column_kwargs, + ) + + def setup_embeddings_and_output_layer(self) -> None: + """Sets up embedding layer in first stage and output layer in last stage. + + This function initalizes word embeddings in the final stage when we are + using pipeline parallelism and sharing word embeddings, and sets up param + attributes on the embedding and output layers. + """ + # Set `is_embedding_or_output_parameter` attribute. + if self.pre_process: + self.model.embed_tokens.weight.is_embedding_or_output_parameter = True + if self.post_process and self.lm_head.weight is not None: + self.lm_head.weight.is_embedding_or_output_parameter = True + + if not self.share_embeddings_and_output_weights: + return + + if parallel_state.get_pipeline_model_parallel_world_size() == 1: + # Zero out wgrad if sharing embeddings between two layers on same + # pipeline stage to make sure grad accumulation into main_grad is + # correct and does not include garbage values (e.g., from torch.empty). + self.shared_embedding_or_output_weight().zero_out_wgrad = True + return + + if parallel_state.is_pipeline_first_stage() and self.pre_process and not self.post_process: + self.shared_embedding_or_output_weight().shared_embedding = True + + if self.post_process and not self.pre_process: + assert not parallel_state.is_pipeline_first_stage() + # set word_embeddings weights to 0 here, then copy first + # stage's weights using all_reduce below. + self.lm_head.weight.data.fill_(0) + self.lm_head.weight.shared = True + self.lm_head.weight.shared_embedding = True + + if torch.distributed.is_initialized() and parallel_state.is_rank_in_embedding_group(): + weight = self.shared_embedding_or_output_weight() + weight.data = weight.data.to(get_device_name()) + torch.distributed.all_reduce(weight.data, group=parallel_state.get_embedding_group()) + + def shared_embedding_or_output_weight(self) -> torch.Tensor: + if self.pre_process: + return self.model.embed_tokens.weight + elif self.post_process: + return self.lm_head.weight + return None + + def _forward_head(self, hidden_states): + # all_gather from sequence parallel region is performed inside lm_head + # print(f'logits shape before forward_head: {hidden_states.shape}, vocab_size = ' + # f'{self.config.vocab_size}') # [4, 32, 4096] + output_weight = None + if self.share_embeddings_and_output_weights: + output_weight = self.shared_embedding_or_output_weight() + logits = self.lm_head(hidden_states, weight=output_weight)[0] + # print(f'logits shape after forward_head: {logits.shape}') # [8, 32, 8] + logits = logits.float() # (total_nnz_padded, 1, vocab_size // tp) + return logits + + def forward( + self, + # original input + *, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + ```""" + + # Note that input_ids, attention_mask and position_ids should be passed to every pp layer. + # In the first pp, input_ids will be used, in other pp layers hidden_states will be used inside self.model + batch_size, sequence_length = input_ids.shape + # remove padding here + input_ids_rmpad, indices, cu_seqlens, max_seqlen_in_batch, *_ = unpad_input( + input_ids.unsqueeze(dim=-1), attention_mask + ) # (total_nnz, 1) + + # pad input_ids to multiple of tp for all tp ranks + # TODO: for better performance, the sp padding should be removed at each layer. Not sure the performance gap + if self.megatron_config.sequence_parallel: + input_ids_rmpad = sp_utils.pad_to_sequence_parallel(input_ids_rmpad) + + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz+pad) + + outputs = self.model( + input_ids=input_ids_rmpad, + position_ids=position_ids, + sequence_length=sequence_length, + indices=indices, + cu_seqlens=cu_seqlens, + max_seqlen_in_batch=max_seqlen_in_batch, + ) + + if self.post_process: + hidden_states = outputs + logits = self._forward_head(hidden_states) + logits = torch.squeeze(logits, dim=1) # remove the artificial batch dimension # torch.Size([8, 32, 16]) + + # remove padding from sequence parallel + if self.megatron_config.sequence_parallel: + totol_nnz = cu_seqlens[-1] + logits = logits[:totol_nnz] # (total_nnz_padded) + # add removed padding back. If input is already rmpad, we let the caller pad_input + logits = pad_input( + logits, indices, batch_size, seqlen=sequence_length + ) # (batch_size, sequence_length, vocab_size) + + return CausalLMOutputWithPast( + loss=None, + logits=logits, + past_key_values=None, + hidden_states=None, + attentions=None, + ) + else: + return outputs + + +class ParallelQwen2ForValueRmPadPP(ParallelQwen2ForCausalLMRmPadPP): + def _init_head(self, config): + column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() + if self.megatron_config is not None: + assert column_kwargs.get("config", False), "must have ModelParallelConfig" + tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) + self.lm_head = nn.Linear(in_features=config.hidden_size, out_features=1, bias=False) + # lm_head is effectively the same as sequence parallel + sp_utils.mark_parameter_as_sequence_parallel(self.lm_head.weight) + + def _forward_head(self, hidden_states): + logits = self.lm_head(hidden_states) # (total_nnz_padded // tp, 1, 1) + logits = logits.float() + if self.megatron_config.sequence_parallel: + logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) + return logits + + def forward( + self, + *, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> tuple | CausalLMOutputWithPast: + output = super().forward(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids) + if self.post_process: + output.logits = torch.squeeze(output.logits, dim=-1) + return output + else: + return output diff --git a/e2c/verl/verl/models/registry.py b/e2c/verl/verl/models/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..667df01417934846776f9f27b622806132e37314 --- /dev/null +++ b/e2c/verl/verl/models/registry.py @@ -0,0 +1,62 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from typing import Optional + +import torch.nn as nn + +# Supported models in Megatron-LM +# Architecture -> (module, class). +_MODELS = { + "LlamaForCausalLM": ( + "llama", + ("ParallelLlamaForCausalLMRmPadPP", "ParallelLlamaForValueRmPadPP", "ParallelLlamaForCausalLMRmPad"), + ), + "Qwen2ForCausalLM": ( + "qwen2", + ("ParallelQwen2ForCausalLMRmPadPP", "ParallelQwen2ForValueRmPadPP", "ParallelQwen2ForCausalLMRmPad"), + ), + "MistralForCausalLM": ( + "mistral", + ("ParallelMistralForCausalLMRmPadPP", "ParallelMistralForValueRmPadPP", "ParallelMistralForCausalLMRmPad"), + ), + "ApertusForCausalLM": ( + "apertus", + ("ParallelApertusForCausalLMRmPadPP", "ParallelApertusForValueRmPadPP", "ParallelApertusForCausalLMRmPad"), + ), +} + + +# return model class +class ModelRegistry: + @staticmethod + def load_model_cls(model_arch: str, value=False) -> Optional[type[nn.Module]]: + if model_arch not in _MODELS: + return None + + megatron = "megatron" + + module_name, model_cls_name = _MODELS[model_arch] + if not value: # actor/ref + model_cls_name = model_cls_name[0] + elif value: # critic/rm + model_cls_name = model_cls_name[1] + + module = importlib.import_module(f"verl.models.{module_name}.{megatron}.modeling_{module_name}_megatron") + return getattr(module, model_cls_name, None) + + @staticmethod + def get_supported_archs() -> list[str]: + return list(_MODELS.keys()) diff --git a/e2c/verl/verl/models/transformers/__init__.py b/e2c/verl/verl/models/transformers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/models/transformers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/models/transformers/apertus.py b/e2c/verl/verl/models/transformers/apertus.py new file mode 100644 index 0000000000000000000000000000000000000000..a42f50957b62e3ae3800b8aadf54793a2c97f2fc --- /dev/null +++ b/e2c/verl/verl/models/transformers/apertus.py @@ -0,0 +1,118 @@ +# Copyright 2025 The SwissAI Initiative +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from typing import Callable, Optional + +import torch + +if sys.version_info >= (3, 11): + pass +else: + pass + +from transformers.cache_utils import Cache +from transformers.models.apertus.modeling_apertus import apply_rotary_pos_emb +from transformers.utils import logging + +# Import compatibility wrapper for flash_attn_supports_top_left_mask +from verl.utils.ulysses import ( + gather_heads_scatter_seq, + gather_seq_scatter_heads, + get_ulysses_sequence_parallel_world_size, + validate_ulysses_config, +) + +logger = logging.get_logger(__name__) + + +def apertus_attn_forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_value: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, +) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + """ + Adapted from transformers 4.49.0 to support Ulysses sequence parallelism for transformers >= 4.48.0. + + Key differences from Llama attention: + - QK normalization applied after Q/K projections + + NOTE: This function has been tested only on transformers versions between 4.48.0 and 4.50.0. + """ + from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS + from transformers.models.apertus.modeling_apertus import eager_attention_forward + + bsz, q_len, _ = hidden_states.shape + + query_states = self.q_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + + query_states = self.q_norm(query_states) + key_states = self.k_norm(key_states) + + ########## AlltoAll for Ulysses ########## + ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + + if ulysses_sp_size > 1: + validate_ulysses_config(self.config.num_attention_heads, ulysses_sp_size) + + query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) + key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) + value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) + + full_q_len = query_states.size(2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False): + logger.warning_once( + "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. " + "Falling back to eager attention. This warning can be removed using the argument " + '`attn_implementation="eager"` when loading the model.' + ) + else: + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights diff --git a/e2c/verl/verl/models/transformers/dense_common.py b/e2c/verl/verl/models/transformers/dense_common.py new file mode 100644 index 0000000000000000000000000000000000000000..56fe293f5cbec4f9efa2a6a77a3374d09e358e56 --- /dev/null +++ b/e2c/verl/verl/models/transformers/dense_common.py @@ -0,0 +1,193 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Union + +import torch +from transformers.cache_utils import Cache +from transformers.modeling_outputs import CausalLMOutputWithPast + + +@dataclass +class CausalLMOutputForPPO(CausalLMOutputWithPast): + log_probs: Optional[torch.FloatTensor] = None + entropy: Optional[torch.FloatTensor] = None + + +def forward_base_model( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, +) -> CausalLMOutputWithPast: + r""" + Copy paste LLaMa's forward + https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/transformers/model/llama.py + + This function should be generic enough for all pure text models. + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + ) + + return outputs + + +def forward_with_torch_backend( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union["Cache", list[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: int | torch.Tensor = 0, + temperature: float = 1.0, + **loss_kwargs, +) -> tuple | CausalLMOutputForPPO: + from verl.utils.experimental.torch_functional import FusedLinearForPPO + + outputs = forward_base_model( + self, + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + cache_position=cache_position, + ) + + hidden_states = outputs[0] + + if not return_dict: + raise NotImplementedError("forward_with_torch_backend has to return_dict") + + # Loss calculations + if labels is not None: + rolled_labels = torch.roll(labels, shifts=-1, dims=-1) + elif input_ids is not None: + rolled_labels = torch.roll(input_ids, shifts=-1, dims=-1) + else: + raise RuntimeError("To use forward_with_torch_backend, either labels or input_ids must be provided.") + + fused_linear_for_ppo = FusedLinearForPPO() + log_probs, entropy = fused_linear_for_ppo.forward( + hidden_states=hidden_states, + vocab_weights=self.lm_head.weight, + input_ids=rolled_labels, + temperature=temperature, + ) + + return CausalLMOutputForPPO( + log_probs=log_probs, + entropy=entropy, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +def forward_with_triton_backend( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union["Cache", list[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: int | torch.Tensor = 0, + temperature: float = 1.0, + **loss_kwargs, +) -> tuple | CausalLMOutputForPPO: + from verl.utils.kernel.linear_cross_entropy import linear_cross_entropy + + outputs = forward_base_model( + self, + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + ) + + hidden_states = outputs[0] + + if not return_dict: + raise NotImplementedError("forward_with_triton_backend has to return_dict") + + # Loss calculations + if labels is not None: + rolled_labels = torch.roll(labels, shifts=-1, dims=-1) + elif input_ids is not None: + rolled_labels = torch.roll(input_ids, shifts=-1, dims=-1) + else: + raise RuntimeError("To use forward_with_triton_backend, either labels or input_ids must be provided.") + + log_probs, entropy = linear_cross_entropy( + hidden_states, + self.lm_head.weight, + rolled_labels, + temperature, + "none", + ) + + return CausalLMOutputForPPO( + log_probs=log_probs, + entropy=entropy, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/e2c/verl/verl/models/transformers/glm4v.py b/e2c/verl/verl/models/transformers/glm4v.py new file mode 100644 index 0000000000000000000000000000000000000000..92fd6824af23d66d9788cf3b90b5ae8184b1781b --- /dev/null +++ b/e2c/verl/verl/models/transformers/glm4v.py @@ -0,0 +1,533 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import itertools +import logging +import os +from dataclasses import dataclass +from typing import Optional + +import torch +import torch.distributed as dist +from transformers.modeling_flash_attention_utils import _flash_attention_forward, fa_peft_integration_check +from transformers.models.glm4v.modeling_glm4v import ( + Glm4vCausalLMOutputWithPast, + Glm4vForConditionalGeneration, + Glm4vTextAttention, +) +from transformers.utils import is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10 + +from verl.utils.device import is_npu_available +from verl.utils.ulysses import ( + gather_heads_scatter_seq, + gather_seq_scatter_heads, + get_ulysses_sequence_parallel_group, + get_ulysses_sequence_parallel_world_size, + validate_ulysses_config, +) + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + + _flash_supports_window_size = "window_size" in inspect.signature(flash_attn_func).parameters + _flash_supports_deterministic = "deterministic" in inspect.signature(flash_attn_func).parameters + _flash_use_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + +if is_npu_available: + from transformers.integrations.npu_flash_attention import npu_flash_attn_func as flash_attn_func + from transformers.integrations.npu_flash_attention import npu_flash_attn_varlen_func as flash_attn_varlen_func + from transformers.modeling_flash_attention_utils import flash_attn_supports_top_left_mask + + _flash_supports_window_size = "window_size" in inspect.signature(flash_attn_func).parameters + _flash_supports_deterministic = "deterministic" in inspect.signature(flash_attn_func).parameters + _flash_use_top_left_mask = flash_attn_supports_top_left_mask() + +_flash_deterministic_enabled = os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" + + +def get_rope_index( + processor, + input_ids: torch.Tensor, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + Gets the position ids for GLM4V in padding-free format. + The batch dim has been removed and the input_ids should be a 1D tensor representing a single example. + """ + spatial_merge_size = processor.image_processor.merge_size + image_token_id = processor.tokenizer.convert_tokens_to_ids("<|image|>") + video_start_token_id = processor.tokenizer.convert_tokens_to_ids("<|begin_of_video|>") + video_end_token_id = processor.tokenizer.convert_tokens_to_ids("<|end_of_video|>") + + if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + + position_ids = torch.ones(3, input_ids.size(0), dtype=input_ids.dtype, device=input_ids.device) # (3, seqlen) + image_index, video_index = 0, 0 + video_group_index = 0 + + input_ids_filtered = input_ids[attention_mask == 1] + input_tokens = input_ids_filtered.tolist() + + input_token_type = [] + video_check_flg = False + for token in input_tokens: + if token == video_start_token_id: + video_check_flg = True + elif token == video_end_token_id: + video_check_flg = False + + if token == image_token_id and not video_check_flg: + input_token_type.append("image") + elif token == image_token_id and video_check_flg: + input_token_type.append("video") + else: + input_token_type.append("text") + + input_type_group = [] + for key, group in itertools.groupby(enumerate(input_token_type), lambda x: x[1]): + group = list(group) + start_index = group[0][0] + end_index = group[-1][0] + 1 + input_type_group.append((key, start_index, end_index)) + + llm_pos_ids_list = [] + video_frame_num = 1 + + for modality_type, start_idx, end_idx in input_type_group: + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + + if modality_type == "image": + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx) + + image_index += 1 + video_frame_num = 1 + + elif modality_type == "video": + t, h, w = ( + video_frame_num, + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t, + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + + for t_idx in range(llm_grid_t): + t_index = torch.tensor(t_idx).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(1, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(1, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx) + + video_group_index += 1 + + if video_group_index >= video_grid_thw[video_index][0]: + video_index += 1 + video_group_index = 0 + + video_frame_num += 1 + + else: + text_len = end_idx - start_idx + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + video_frame_num = 1 + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., attention_mask == 1] = llm_positions.to(position_ids.device) + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = position_ids.unsqueeze(0).expand(3, -1).to(input_ids.device) + else: + position_ids = torch.arange(input_ids.shape[0], device=input_ids.device).view(1, -1).expand(3, -1) + + return position_ids + + +def prepare_fa2_from_position_ids( + query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, position_ids: torch.Tensor +): + assert position_ids.ndim == 2 # (batch_size, seq_length) + query = query.contiguous().view(-1, query.size(-2), query.size(-1)) + key = key.contiguous().view(-1, key.size(-2), key.size(-1)) + value = value.contiguous().view(-1, value.size(-2), value.size(-1)) + position_ids = position_ids.view(-1) + cu_seqlens = torch.cat( + ( + (position_ids == 0).nonzero().view(-1).to(torch.int32), + torch.tensor(position_ids.size(), device=position_ids.device, dtype=torch.int32), + ) + ) + max_length = cu_seqlens.diff().max() # use cu_seqlens to infer max_length for qwen2vl mrope + return (query, key, value, (cu_seqlens, cu_seqlens), (max_length, max_length)) + + +def _custom_flash_attention_forward( + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + attention_mask: Optional[torch.Tensor], + query_length: int, + is_causal: bool = True, + position_ids: Optional[torch.Tensor] = None, + use_top_left_mask: bool = False, + deterministic: Optional[bool] = None, + **kwargs, +): + """ + Patches flash attention forward to handle 3D position ids in mrope. (3, batch_size, seq_length) + """ + # Assuming 4D tensors, key_states.shape[1] is the key/value sequence length (source length). + flash_kwargs = {} + + if _flash_supports_deterministic: + flash_kwargs["deterministic"] = deterministic if deterministic is not None else _flash_deterministic_enabled + + if kwargs.get("softcap") is not None: + flash_kwargs["softcap"] = kwargs.pop("softcap") + + query_states, key_states, value_states = fa_peft_integration_check( + query_states, key_states, value_states, target_dtype=torch.bfloat16 + ) + + if position_ids is not None: + assert position_ids.ndim == 2 # (batch_size, seq_length / sp_size) + + sp_size = get_ulysses_sequence_parallel_world_size() + if sp_size > 1: + # qkv: (batch_size, seq_length / sp_size, num_head, head_size) + validate_ulysses_config(query_states.size(2), sp_size) + query_states = gather_seq_scatter_heads(query_states, seq_dim=1, head_dim=2) + key_states = gather_seq_scatter_heads(key_states, seq_dim=1, head_dim=2) + value_states = gather_seq_scatter_heads(value_states, seq_dim=1, head_dim=2) + position_ids_lst = [torch.empty_like(position_ids) for _ in range(sp_size)] + position_ids = dist.all_gather(position_ids_lst, position_ids, group=get_ulysses_sequence_parallel_group()) + position_ids = torch.cat(position_ids_lst, dim=-1) # (batch_size, seq_length) + + if position_ids is not None and query_length != 1 and not (torch.diff(position_ids, dim=-1) >= 0).all(): + batch_size = query_states.size(0) + q, k, v, (cu_seqlens_q, cu_seqlens_k), (max_seqlen_q, max_seqlen_k) = prepare_fa2_from_position_ids( + query_states, key_states, value_states, position_ids + ) + attn_output = flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + dropout_p=kwargs.pop("dropout", 0.0), + softmax_scale=kwargs.pop("softmax_scale", None), + causal=is_causal, + **flash_kwargs, + ) + attn_output = attn_output.view(batch_size, -1, attn_output.size(-2), attn_output.size(-1)) + else: + attn_output = _flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + query_length, + is_causal=is_causal, + use_top_left_mask=use_top_left_mask, + deterministic=deterministic, + **kwargs, + ) # do not pass position_ids to old flash_attention_forward + + if sp_size > 1: + # (batch_size, seq_length, num_head, head_size) + attn_output = gather_heads_scatter_seq(attn_output, head_dim=2, seq_dim=1) + + return attn_output + + +def glm4v_attn_forward( + self: "Glm4vTextAttention", + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + **kwargs, +) -> tuple[torch.Tensor, None, None]: + from transformers.models.glm4v.modeling_glm4v import apply_multimodal_rotary_pos_emb, repeat_kv + + bsz, q_len, _ = hidden_states.size() # q_len = seq_length / sp_size + query_states = self.q_proj(hidden_states) # (batch_size, seq_length / sp_size, num_heads * head_size) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + # Because the input can be padded, the absolute sequence length depends on the max position id. + cos, sin = position_embeddings + query_states, key_states = apply_multimodal_rotary_pos_emb( + query_states, key_states, cos, sin, self.rope_scaling["mrope_section"] + ) + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + dropout_rate = 0.0 if not self.training else self.attention_dropout + + # This is before the transpose + q_len = query_states.shape[2] + + # FA2 uses non-transposed inputs + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + attn_output = _custom_flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + query_length=q_len, + is_causal=getattr(self, "is_causal", True), + dropout=dropout_rate, + use_top_left_mask=_flash_use_top_left_mask, + position_ids=position_ids, # important: pass position ids + ) # (batch_size, seq_length / sp_size, num_head, head_size) + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, None + + +def _get_input_embeds( + model: "Glm4vForConditionalGeneration", + input_ids: torch.LongTensor, + attention_mask: Optional[torch.Tensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, +): + inputs_embeds = model.get_input_embeddings()(input_ids) + if pixel_values is not None: + pixel_values = pixel_values.type(model.visual.dtype) + image_embeds = model.visual(pixel_values, grid_thw=image_grid_thw) + n_image_tokens = (input_ids == model.config.image_token_id).sum().item() + n_image_features = image_embeds.shape[0] + if n_image_tokens != n_image_features: + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}" + ) + + mask = input_ids == model.config.image_token_id + mask_unsqueezed = mask.unsqueeze(-1) + mask_expanded = mask_unsqueezed.expand_as(inputs_embeds) + image_mask = mask_expanded.to(inputs_embeds.device) + + image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) + inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) + + if pixel_values_videos is not None: + pixel_values_videos = pixel_values_videos.type(model.visual.dtype) + video_embeds = model.visual(pixel_values_videos, grid_thw=video_grid_thw) + n_video_tokens = (input_ids == model.config.video_token_id).sum().item() + n_video_features = video_embeds.shape[0] + if n_video_tokens != n_video_features: + raise ValueError( + f"Video features and video tokens do not match: tokens: {n_video_tokens}, features {n_video_features}" + ) + + mask = input_ids == model.config.video_token_id + mask_unsqueezed = mask.unsqueeze(-1) + mask_expanded = mask_unsqueezed.expand_as(inputs_embeds) + video_mask = mask_expanded.to(inputs_embeds.device) + + video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) + inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) + + if model.training and pixel_values is None and pixel_values_videos is None: # handle mixed text-image data + pixel_values = torch.zeros((16, 1176), dtype=inputs_embeds.dtype, device=inputs_embeds.device) + image_grid_thw = torch.tensor([[1, 4, 4]], dtype=torch.long, device=inputs_embeds.device) + image_embeds = model.visual(pixel_values, grid_thw=image_grid_thw) + inputs_embeds += 0.0 * image_embeds.mean() + + if attention_mask is not None: + attention_mask = attention_mask.to(inputs_embeds.device) + + return inputs_embeds, attention_mask + + +def process_position_ids(position_ids: torch.Tensor) -> torch.Tensor: + if position_ids.ndim != 3 or position_ids.size(0) != 4: + # we concat the text position ids with the 3D vision position ids by default + # see https://github.com/huggingface/transformers/pull/39447 + raise ValueError("position_ids should be a 3D tensor of shape (4, batch_size, seq_length).") + + return position_ids + + +@dataclass +class Glm4vCausalLMOutputForPPO(Glm4vCausalLMOutputWithPast): + log_probs: Optional[torch.FloatTensor] = None + entropy: Optional[torch.FloatTensor] = None + + +def glm4v_base_forward( + self: "Glm4vForConditionalGeneration", + input_ids: torch.LongTensor, + attention_mask: Optional[torch.Tensor] = None, + labels: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs, +): + kwargs["inputs_embeds"], kwargs["attention_mask"] = _get_input_embeds( + self, input_ids, attention_mask, pixel_values, pixel_values_videos, image_grid_thw, video_grid_thw + ) # avoid lora module having multiple keyword arguments + return self.language_model( + input_ids=None, + **kwargs, + ) + + +def glm4v_forward( + self: "Glm4vForConditionalGeneration", + input_ids: torch.LongTensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs, +): + return self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=process_position_ids(position_ids), + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + **kwargs, + ) + + +def forward_with_normal_backend( + self: Glm4vForConditionalGeneration, + input_ids: torch.LongTensor = None, + labels: Optional[torch.LongTensor] = None, + temperature: float = 1.0, + **kwargs, +) -> "Glm4vCausalLMOutputWithPast": + outputs = glm4v_forward(self, input_ids, **kwargs) + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + + return Glm4vCausalLMOutputWithPast( + logits=logits, + hidden_states=outputs.hidden_states, + ) + + +def forward_with_torch_backend( + self: Glm4vForConditionalGeneration, + input_ids: torch.LongTensor = None, + labels: Optional[torch.LongTensor] = None, + temperature: float = 1.0, + **kwargs, +) -> tuple | Glm4vCausalLMOutputForPPO: + from verl.utils.experimental.torch_functional import FusedLinearForPPO + + outputs = glm4v_forward(self, input_ids, **kwargs) + hidden_states = outputs[0] + + # Loss calculations + if labels is not None: + rolled_labels = torch.roll(labels, shifts=-1, dims=-1) + elif input_ids is not None: + rolled_labels = torch.roll(input_ids, shifts=-1, dims=-1) + else: + raise RuntimeError("To use forward_with_torch_backend, either labels or input_ids must be provided.") + + fused_linear_for_ppo = FusedLinearForPPO() + log_probs, entropy = fused_linear_for_ppo.forward( + hidden_states=hidden_states, + vocab_weights=self.lm_head.weight, + input_ids=rolled_labels, + temperature=temperature, + ) + return Glm4vCausalLMOutputForPPO( + log_probs=log_probs, + entropy=entropy, + hidden_states=outputs.hidden_states, + ) + + +def forward_with_triton_backend( + self: Glm4vForConditionalGeneration, + input_ids: torch.LongTensor = None, + labels: Optional[torch.LongTensor] = None, + temperature: float = 1.0, + **kwargs, +) -> tuple | Glm4vCausalLMOutputForPPO: + from verl.utils.kernel.linear_cross_entropy import linear_cross_entropy + + outputs = glm4v_forward(self, input_ids, **kwargs) + hidden_states = outputs[0] + + # Loss calculations + if labels is not None: + rolled_labels = torch.roll(labels, shifts=-1, dims=-1) + elif input_ids is not None: + rolled_labels = torch.roll(input_ids, shifts=-1, dims=-1) + else: + raise RuntimeError("To use forward_with_triton_backend, either labels or input_ids must be provided.") + + log_probs, entropy = linear_cross_entropy( + hidden_states, + self.lm_head.weight, + rolled_labels, + temperature, + "none", + ) + return Glm4vCausalLMOutputForPPO( + log_probs=log_probs, + entropy=entropy, + hidden_states=outputs.hidden_states, + ) diff --git a/e2c/verl/verl/models/transformers/kimi_vl.py b/e2c/verl/verl/models/transformers/kimi_vl.py new file mode 100644 index 0000000000000000000000000000000000000000..cabb518f4a113fc52f421700d9f216b4ec3bd627 --- /dev/null +++ b/e2c/verl/verl/models/transformers/kimi_vl.py @@ -0,0 +1,192 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +import torch.nn.functional as F +from transformers.cache_utils import Cache +from transformers.modeling_flash_attention_utils import _flash_attention_forward + +from verl.models.transformers.monkey_patch import is_transformers_version_in_range + +# Import compatibility wrapper for flash_attn_supports_top_left_mask +from verl.utils.transformers_compat import flash_attn_supports_top_left_mask +from verl.utils.ulysses import ( + gather_heads_scatter_seq, + gather_seq_scatter_heads, + get_ulysses_sequence_parallel_world_size, + validate_ulysses_config, +) + + +# Copied from transformers.models.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb +def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos[position_ids].unsqueeze(unsqueeze_dim) + sin = sin[position_ids].unsqueeze(unsqueeze_dim) + + b, h, s, d = q.shape + q = q.view(b, h, s, d // 2, 2).transpose(4, 3).reshape(b, h, s, d) + + b, h, s, d = k.shape + k = k.view(b, h, s, d // 2, 2).transpose(4, 3).reshape(b, h, s, d) + + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +# Copied from transformers.models.llama.modeling_llama.repeat_kv +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def _ulysses_flash_attn_forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, +) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + if self.q_lora_rank is None: + q = self.q_proj(hidden_states) + else: + q = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(hidden_states))) + q = q.view(bsz, q_len, self.num_heads, self.q_head_dim).transpose(1, 2) + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dim x hidden_dim + # therefore we just need to keep the original shape + compressed_kv = self.kv_a_proj_with_mqa(hidden_states) + compressed_kv, k_pe = torch.split(compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + k_pe = k_pe.view(bsz, q_len, 1, self.qk_rope_head_dim).transpose(1, 2) + kv = ( + self.kv_b_proj(self.kv_a_layernorm(compressed_kv)) + .view(bsz, q_len, self.num_heads, self.qk_nope_head_dim + self.v_head_dim) + .transpose(1, 2) + ) + + k_nope, value_states = torch.split(kv, [self.qk_nope_head_dim, self.v_head_dim], dim=-1) + + # patch + ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + if ulysses_sp_size > 1: + validate_ulysses_config(self.num_heads, ulysses_sp_size) + + num_key_value_groups = self.config.num_attention_heads // self.config.num_key_value_heads + k_pe = repeat_kv(k_pe, ulysses_sp_size) # to keep heads=1 after a2a + k_nope = repeat_kv(k_nope, num_key_value_groups) + value_states = repeat_kv(value_states, num_key_value_groups) + q = gather_seq_scatter_heads(q, seq_dim=2, head_dim=1) + k_pe = gather_seq_scatter_heads(k_pe, seq_dim=2, head_dim=1) + k_nope = gather_seq_scatter_heads(k_nope, seq_dim=2, head_dim=1) + value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) + # (batch_size, num_head / sp_size, seq_length, head_size) + full_q_len = q.size(2) # full_q_len = seq_length + + else: + full_q_len = q_len + + q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + cos, sin = self.rotary_emb(value_states, seq_len=full_q_len) + q_pe, k_pe = apply_rotary_pos_emb(q_pe, k_pe, cos, sin, position_ids) + + query_states = k_pe.new_empty(bsz, self.num_heads // ulysses_sp_size, full_q_len, self.q_head_dim) + query_states[:, :, :, : self.qk_nope_head_dim] = q_nope + query_states[:, :, :, self.qk_nope_head_dim :] = q_pe + + key_states = k_pe.new_empty(bsz, self.num_heads // ulysses_sp_size, full_q_len, self.q_head_dim) + key_states[:, :, :, : self.qk_nope_head_dim] = k_nope + key_states[:, :, :, self.qk_nope_head_dim :] = k_pe + + if self.q_head_dim != self.v_head_dim: + value_states = F.pad(value_states, [0, self.q_head_dim - self.v_head_dim]) + + # TODO: These transpose are quite inefficient but Flash Attention requires the layout + # [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache + # to be able to avoid many of these transpose/reshape/view. + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + dropout_rate = self.attention_dropout if self.training else 0.0 + + attn_output = _flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + full_q_len, + dropout=dropout_rate, + sliding_window=None, + is_causal=self.is_causal, + use_top_left_mask=flash_attn_supports_top_left_mask(), + position_ids=position_ids, # important: pass position ids + softmax_scale=self.softmax_scale, + ) + + if ulysses_sp_size > 1: + attn_output = gather_heads_scatter_seq(attn_output, head_dim=2, seq_dim=1) + + if self.q_head_dim != self.v_head_dim: + attn_output = attn_output[:, :, :, : self.v_head_dim] + + attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.v_head_dim).contiguous() + attn_output = self.o_proj(attn_output) + + if is_transformers_version_in_range(min_version="4.53.0"): + return attn_output, None + else: + return attn_output, None, None diff --git a/e2c/verl/verl/models/transformers/llama.py b/e2c/verl/verl/models/transformers/llama.py new file mode 100644 index 0000000000000000000000000000000000000000..b3efb8646d55808bf647bb9d490ab69b80dc6fe1 --- /dev/null +++ b/e2c/verl/verl/models/transformers/llama.py @@ -0,0 +1,241 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from typing import Callable, Optional + +import torch + +if sys.version_info >= (3, 11): + pass +else: + pass + +from transformers.cache_utils import Cache +from transformers.modeling_flash_attention_utils import _flash_attention_forward +from transformers.models.llama.modeling_llama import apply_rotary_pos_emb +from transformers.utils import logging + +# Import compatibility wrapper for flash_attn_supports_top_left_mask +from verl.utils.transformers_compat import flash_attn_supports_top_left_mask +from verl.utils.ulysses import ( + gather_heads_scatter_seq, + gather_seq_scatter_heads, + get_ulysses_sequence_parallel_world_size, + validate_ulysses_config, +) + +logger = logging.get_logger(__name__) + + +def llama_flash_attn_forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + **kwargs, +) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + """ + Adapted from transformers 4.47.1 to support Ulysses sequence parallelism. + + NOTE: This function is used for transformers versions in the range [4.45.0, 4.47.1]. + """ + output_attentions = False + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dim x hidden_dim + # therefore we just need to keep the original shape + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + # trade off: repeat first and then all to all + # key_states = repeat_kv(key_states, self.num_key_value_groups) + # value_states = repeat_kv(value_states, self.num_key_value_groups) + + ########## AlltoAll for Ulysses ########## + ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + + if ulysses_sp_size > 1: + validate_ulysses_config(self.num_heads, ulysses_sp_size) + + # (bsz, n_head, seq_len/n, head_dim) -> (bsz, n_head/n, seq_len, head_dim) + query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) + key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) + value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) + + full_q_len = query_states.size(2) # full seq length + + if position_embeddings is None: + logger.warning_once( + "The attention layers in this model are transitioning from computing the RoPE embeddings internally " + "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " + "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " + "removed and `position_embeddings` will be mandatory." + ) + cos, sin = self.rotary_emb(value_states, position_ids) + else: + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # TODO: These transpose are quite inefficient but Flash Attention requires the layout + # [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache + # to be able to avoid many of these transpose/reshape/view. + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + dropout_rate = self.attention_dropout if self.training else 0.0 + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (LlamaRMSNorm handles it correctly) + + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to " + f"the fact you have upcasted embedding or layer norm layers in float32. We will cast back the " + f"input in {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + attn_output = _flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + full_q_len, + position_ids=position_ids, + dropout=dropout_rate, + sliding_window=getattr(self, "sliding_window", None), + use_top_left_mask=flash_attn_supports_top_left_mask(), + is_causal=self.is_causal, + **kwargs, + ) + + attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +def llama_attn_forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_value: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, +) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + """ + Adapted from transformers 4.49.0 to support Ulysses sequence parallelism for transformers >= 4.48.0. + + NOTE: This function has been tested only on transformers versions between 4.48.0 and 4.50.0. + """ + from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS + from transformers.models.llama.modeling_llama import eager_attention_forward + + bsz, q_len, _ = hidden_states.shape + + query_states = self.q_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + + ########## AlltoAll for Ulysses ########## + ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + + if ulysses_sp_size > 1: + validate_ulysses_config(self.config.num_attention_heads, ulysses_sp_size) + + query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) + key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) + value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) + + full_q_len = query_states.size(2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False): + logger.warning_once( + "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. " + "Falling back to eager attention. This warning can be removed using the argument " + '`attn_implementation="eager"` when loading the model.' + ) + else: + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights diff --git a/e2c/verl/verl/models/transformers/monkey_patch.py b/e2c/verl/verl/models/transformers/monkey_patch.py new file mode 100644 index 0000000000000000000000000000000000000000..7f2dc0b1f776799cd30805ffcad477d1540a1a74 --- /dev/null +++ b/e2c/verl/verl/models/transformers/monkey_patch.py @@ -0,0 +1,357 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Apply monkey-patch function to models +""" + +import sys +from types import SimpleNamespace +from typing import Optional + +import torch +from transformers.modeling_flash_attention_utils import _flash_attention_forward +from transformers.modeling_utils import PreTrainedModel + +from verl.utils.import_utils import is_trl_available +from verl.utils.transformers_compat import is_transformers_version_in_range +from verl.utils.ulysses import ( + gather_heads_scatter_seq, + gather_seq_scatter_heads, + get_ulysses_sequence_parallel_group, + get_ulysses_sequence_parallel_world_size, + slice_input_tensor, +) + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=2, repeats=n_rep). The hidden states go from (batch, + seqlen, num_key_value_heads, head_dim) to (batch, seqlen, num_attention_heads, head_dim) + """ + batch, slen, num_key_value_heads, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, :, None, :].expand(batch, slen, num_key_value_heads, n_rep, head_dim) + return hidden_states.reshape(batch, slen, num_key_value_heads * n_rep, head_dim) + + +def _ulysses_flash_attention_forward( + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + attention_mask: Optional[torch.Tensor], + query_length: int, + *args, + position_ids: Optional[torch.Tensor] = None, + **kwargs, +): + """Insert all-to-all before and after flash attention. + DeepSpeed-Ulysses: https://arxiv.org/pdf/2309.14509 + + For transformers>=4.55, the flash attention api has changed, + we need to pass the query_length after doing ulysses all2all. + See https://github.com/huggingface/transformers/issues/40399 + + Args: + query_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads, head_dim) + key_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads_k, head_dim) + value_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads_k, head_dim) + position_ids (torch.Tensor, optional): (batch_size, seqlen/sp_size) + + Returns: + torch.Tensor: (batch_size, seqlen/sp_size, nheads, head_dim) + + """ + ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + assert position_ids is not None, "position_ids is required for Ulysses sequence parallelism" + + # NOTE: repeat kv heads to be divided by sequence parallel. Instead of repeating nheads_q//nheads_k, + # we choose to repeat sp_size//nheads_k, since flash_attention supports MQA/GQA. + # For example: + # - nheads_k=4, sp=8, repeats=2 + # - nheads_k=8, sp=8, repeats=1 + # - nheads_k=16, sp=8, repeats=1 + repeats = max(ulysses_sp_size // key_states.size(2), 1) + key_states = repeat_kv(key_states, repeats) + value_states = repeat_kv(value_states, repeats) + + # (bsz, seq_len/n, n_head, head_dim) -> (bsz, seq_len, n_head/n, head_dim) + query_states = gather_seq_scatter_heads(query_states, seq_dim=1, head_dim=2) + key_states = gather_seq_scatter_heads(key_states, seq_dim=1, head_dim=2) + value_states = gather_seq_scatter_heads(value_states, seq_dim=1, head_dim=2) + + # TODO: all_gather position_ids because `prepare_fa2_from_position_ids` needs it, we can eliminate + # this all_gather by passing cu_seq_lens_q, cu_seq_lens_k, max_length_k, max_length_q explicitly. + # https://github.com/huggingface/transformers/pull/33932 + + # (bsz, seq_len/n) -> (bsz, seq_len) + position_ids_list = [torch.empty_like(position_ids) for _ in range(ulysses_sp_size)] + torch.distributed.all_gather(position_ids_list, position_ids, group=get_ulysses_sequence_parallel_group()) + position_ids = torch.concat(position_ids_list, dim=-1) + + # (bsz, seq_len, n_head/n, head_dim) + query_length = query_states.size(1) + attn_output = _flash_attention_forward( + query_states, key_states, value_states, attention_mask, query_length, *args, position_ids=position_ids, **kwargs + ) + + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + # (bsz, seq_len, n_head/n, head_dim) -> (bsz, seq_len/n, n_head, head_dim) + attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) + + return attn_output + + +def patch_vlm_for_ulysses_input_slicing(model_class: type): + """ + Applies a monkey patch to the forward method of a given model class + to enable Ulysses sequence parallelism input slicing. + """ + + def _create_ulysses_wrapped_decoder_forward(original_forward): + def ulysses_wrapped_decoder_forward(self, *args, **kwargs): + inputs_embeds = kwargs.get("inputs_embeds") + position_ids = kwargs.get("position_ids") + call_kwargs = kwargs.copy() + + current_ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + + slice_now = ( + inputs_embeds is not None + and current_ulysses_sp_size > 1 + and getattr(self, "_needs_initial_slice", True) + ) + if slice_now: + call_kwargs["inputs_embeds"] = slice_input_tensor(inputs_embeds, dim=1, padding=False) + call_kwargs["position_ids"] = slice_input_tensor(position_ids, dim=-1, padding=False) + self._needs_initial_slice = False + try: + return original_forward(self, *args, **call_kwargs) + finally: + if slice_now: + self._needs_initial_slice = True + + return ulysses_wrapped_decoder_forward + + original_forward = model_class.forward + wrapped_forward = _create_ulysses_wrapped_decoder_forward(original_forward) + model_class.forward = wrapped_forward + print(f"Monkey patch {model_class.__name__}.forward for Ulysses SP input slicing.") + + +def patch_forward_with_backends( + model: PreTrainedModel, + use_fused_kernels: bool = False, + fused_kernels_backend: str = None, +): + """ + Choose the forward function based on the model and backend. + Args: + model (PreTrainedModel): The model to apply the monkey patch. + use_fused_kernels (bool): Whether to use fused kernels. + fused_kernels_backend (str): The backend to use for fused kernels. + """ + if not use_fused_kernels or fused_kernels_backend not in ["triton", "torch"]: + print( + f"Skipping monkey patch for {model.__class__.__name__} as use_fused_kernels is " + f"{use_fused_kernels} or fused_kernels_backend is {fused_kernels_backend}" + ) + return + + forward_with_torch_backend_function = model.__class__.forward + forward_with_triton_backend_function = model.__class__.forward + if model.config.model_type in ["qwen2_5_vl", "qwen2_vl"]: + from verl.models.transformers.qwen2_vl import forward_with_torch_backend, forward_with_triton_backend + + forward_with_torch_backend_function = forward_with_torch_backend + forward_with_triton_backend_function = forward_with_triton_backend + elif model.config.model_type == "glm4v": + from verl.models.transformers.glm4v import forward_with_torch_backend, forward_with_triton_backend + + forward_with_torch_backend_function = forward_with_torch_backend + forward_with_triton_backend_function = forward_with_triton_backend + else: + from verl.models.transformers.dense_common import forward_with_torch_backend, forward_with_triton_backend + + forward_with_torch_backend_function = forward_with_torch_backend + forward_with_triton_backend_function = forward_with_triton_backend + + if fused_kernels_backend == "triton": + model.__class__.forward = forward_with_triton_backend_function + print(f"Using Triton backend for fused kernels in {model.__class__.__name__}") + elif fused_kernels_backend == "torch": + model.__class__.forward = forward_with_torch_backend_function + print(f"Using Torch backend for fused kernels in {model.__class__.__name__}") + else: + raise ValueError(f"Unsupported fused_kernels_backend: {fused_kernels_backend}. Choose 'triton' or 'torch'.") + + +def apply_monkey_patch( + model: PreTrainedModel, + ulysses_sp_size: int = 1, + use_remove_padding: bool = True, + use_fused_kernels: bool = False, + fused_kernels_backend: str = None, +): + """ + Apply monkey patch to the models for ulysses sequence parallel and fused kernel. + + In the end of this function forward function of the model is patched for fused kernel. + If the model is not supported with fused kernel, please return after patch. + """ + + """Replace _flash_attention_forward to _ulysses_flash_attention_forward""" + module = sys.modules[model.__module__] + + try: + num_attention_heads, num_key_value_heads = model.config.num_attention_heads, model.config.num_key_value_heads + except AttributeError: + num_attention_heads, num_key_value_heads = ( + model.config.text_config.num_attention_heads, + model.config.text_config.num_key_value_heads, + ) + + assert num_attention_heads % ulysses_sp_size == 0, ( + f"num_attention_heads {num_attention_heads} must be divisible by ulysses_sp_size {ulysses_sp_size}" + ) + assert num_key_value_heads % ulysses_sp_size == 0 or ulysses_sp_size % num_key_value_heads == 0, ( + f"num_key_value_heads {num_key_value_heads} must be divisible by ulysses_sp_size " + f"{ulysses_sp_size}or vise versa. Upon ulysses_sp_size % num_key_value_heads == 0," + f"kv heads are repeated to ensure correctness." + ) + + if is_trl_available(): + from trl import AutoModelForCausalLMWithValueHead # type: ignore + + def state_dict(self, *args, **kwargs): + return torch.nn.Module.state_dict(self, *args, **kwargs) + + AutoModelForCausalLMWithValueHead.state_dict = state_dict + print("Monkey patch state_dict in AutoModelForCausalLMWithValueHead. ") + + # TODO: VLM models only, unify monkey patch to LLM models. + if model.config.model_type in ["qwen2_5_vl", "qwen2_vl"]: + # Step 1: patch model to support image-text mixed data + if is_transformers_version_in_range(min_version="4.52.0"): + from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( + Qwen2_5_VLForConditionalGeneration, + Qwen2_5_VLModel, + Qwen2_5_VLTextModel, + ) + from transformers.models.qwen2_vl.modeling_qwen2_vl import ( + Qwen2VLForConditionalGeneration, + Qwen2VLModel, + Qwen2VLTextModel, + ) + else: + from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLForConditionalGeneration + from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLModel as Qwen2_5_VLTextModel + from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLForConditionalGeneration + from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLModel as Qwen2VLTextModel + + Qwen2_5_VLModel = SimpleNamespace(forward=None) + Qwen2VLModel = SimpleNamespace(forward=None) + + from verl.models.transformers.qwen2_vl import forward_with_normal_backend, qwen2_vl_base_forward + + Qwen2_5_VLModel.forward = qwen2_vl_base_forward + Qwen2VLModel.forward = qwen2_vl_base_forward + Qwen2_5_VLForConditionalGeneration.forward = forward_with_normal_backend + Qwen2VLForConditionalGeneration.forward = forward_with_normal_backend + print(f"Monkey patch {model.__class__.__name__} model forward") + + # Step 2: patch attention to support ulysses parallelism + if is_transformers_version_in_range(min_version="4.54.0"): + from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLAttention + from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLAttention + elif is_transformers_version_in_range(min_version="4.53.0"): + raise RuntimeError("Transformers 4.53.* is bugged. Use transformers 4.54.0 or later.") + else: + from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( + Qwen2_5_VLFlashAttention2 as Qwen2_5_VLAttention, + ) + from transformers.models.qwen2_vl.modeling_qwen2_vl import ( + Qwen2VLFlashAttention2 as Qwen2VLAttention, + ) + + if use_remove_padding or ulysses_sp_size > 1: + from verl.models.transformers.qwen2_vl import qwen2_vl_attn_forward + + Qwen2_5_VLAttention.forward = qwen2_vl_attn_forward + Qwen2VLAttention.forward = qwen2_vl_attn_forward + print(f"Monkey patch {model.__class__.__name__} attention layer") + + # Step 3: patch input for multimodal sequence parallelism + if ulysses_sp_size > 1: + patch_vlm_for_ulysses_input_slicing(Qwen2_5_VLTextModel) + patch_vlm_for_ulysses_input_slicing(Qwen2VLTextModel) + + if model.config.model_type == "glm4v": + # Step 1: patch model to support image-text mixed data + + from transformers.models.glm4v.modeling_glm4v import ( + Glm4vForConditionalGeneration, + Glm4vModel, + Glm4vTextAttention, + Glm4vTextModel, + ) + + from verl.models.transformers.glm4v import forward_with_normal_backend, glm4v_base_forward + + Glm4vModel.forward = glm4v_base_forward + Glm4vForConditionalGeneration.forward = forward_with_normal_backend + print(f"Monkey patch {model.__class__.__name__} model forward") + + # Step 2: patch attention to support ulysses parallelism + if use_remove_padding or ulysses_sp_size > 1: + from verl.models.transformers.glm4v import glm4v_attn_forward + + Glm4vTextAttention.forward = glm4v_attn_forward + print(f"Monkey patch {model.__class__.__name__} attention layer") + + # Step 3: patch input for multimodal sequence parallelism + if ulysses_sp_size > 1: + patch_vlm_for_ulysses_input_slicing(Glm4vTextModel) + + elif model.config.model_type == "kimi_vl": + if use_remove_padding or ulysses_sp_size > 1: + # TODO: Changes need to be made when transformers are adapted. + from verl.models.transformers.kimi_vl import _ulysses_flash_attn_forward + + module.DeepseekV3FlashAttention2.forward = _ulysses_flash_attn_forward + print("Monkey patch FlashAttention2.forward in KimiVL") + + if ulysses_sp_size > 1: + patch_vlm_for_ulysses_input_slicing(module.DeepseekV3ForCausalLM) + + if use_fused_kernels: + print("Not support fused kernels for KimiVL") + + return + + if use_remove_padding or ulysses_sp_size > 1: + if hasattr(module, "_flash_attention_forward"): # transformers <= 4.47.1 or legacy models + module._flash_attention_forward = _ulysses_flash_attention_forward + print(f"Monkey patch _flash_attention_forward in {model.__module__}") + else: + from transformers.integrations import flash_attention + + flash_attention._flash_attention_forward = _ulysses_flash_attention_forward + print(f"Monkey patch _flash_attention_forward in {flash_attention.__name__}") + + patch_forward_with_backends(model, use_fused_kernels=use_fused_kernels, fused_kernels_backend=fused_kernels_backend) diff --git a/e2c/verl/verl/models/transformers/npu_patch.py b/e2c/verl/verl/models/transformers/npu_patch.py new file mode 100644 index 0000000000000000000000000000000000000000..f9dcf021da6afb7642ce464217cbadae51fdd5cc --- /dev/null +++ b/e2c/verl/verl/models/transformers/npu_patch.py @@ -0,0 +1,207 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Copyright 2025 The Qwen Team and The HuggingFace Inc. team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from importlib.metadata import version as get_version +from typing import Optional + +import torch +import torch.nn.functional as F +import torch_npu +from torch_npu import npu_rotary_mul as apply_rotary_emb +from transformers.modeling_utils import PretrainedConfig, PreTrainedModel +from transformers.models.qwen2_5_vl import modeling_qwen2_5_vl +from transformers.models.qwen3 import modeling_qwen3 +from transformers.models.qwen3_moe import modeling_qwen3_moe +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +# This patch takes effect when using apply_rotary_pos_emb_flashatt on qwen2_5_vl and will be removed in +# subsequent versions +# https://github.com/huggingface/transformers/pull/38491 +def apply_rotary_pos_emb_flashatt_qwen2_5_vl_npu( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + cos = cos.chunk(2, dim=-1)[0].contiguous() + sin = sin.chunk(2, dim=-1)[0].contiguous() + cos = cos.repeat(1, 2) + sin = sin.repeat(1, 2) + q_embed = apply_rotary_emb( + q.float(), cos.unsqueeze(0).unsqueeze(2).float(), sin.unsqueeze(0).unsqueeze(2).float() + ).type_as(q) + k_embed = apply_rotary_emb( + k.float(), cos.unsqueeze(0).unsqueeze(2).float(), sin.unsqueeze(0).unsqueeze(2).float() + ).type_as(k) + return q_embed, k_embed + + +# This api can improve performance on ASCEND NPU +def rms_norm_forward(self, x): + return torch_npu.npu_rms_norm(x, self.weight, epsilon=self.variance_epsilon)[0] + + +def silu_forward(self, hidden_state): + """NPU optimized silu""" + gate_up = torch.cat((self.gate_proj(hidden_state), self.up_proj(hidden_state)), dim=-1) + return self.down_proj(torch_npu.npu_swiglu(gate_up, dim=-1)) + + +def apply_rotary_pos_emb_qwen3_npu(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = torch_npu.npu_rotary_mul(q, cos, sin) + k_embed = torch_npu.npu_rotary_mul(k, cos, sin) + return q_embed.to(q.dtype), k_embed.to(k.dtype) + + +class GmmFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, x, weight, group_list, split_size): + ctx.save_for_backward(x, weight) + ctx.group_list = group_list + ctx.split_size = split_size + + outputs = torch_npu.npu_grouped_matmul([x], [weight], group_list=group_list, group_type=0, split_item=2) + return outputs[0] + + @staticmethod + def backward(ctx, grad_outputs): + x, weight = ctx.saved_tensors + group_list = ctx.group_list + wt = weight.permute(0, 2, 1) + xt = x.permute(1, 0) + dx = torch_npu.npu_grouped_matmul([grad_outputs], [wt], group_list=group_list, group_type=0, split_item=2) + dw = torch.zeros_like(weight) + split_size = ctx.split_size + xt_list = torch.split(xt, split_size, dim=1) + grad_outputs_list = torch.split(grad_outputs, split_size, dim=0) + with torch.npu.amp.autocast(enabled=False): + dw = torch.stack([torch.matmul(xt_list[i], grad_outputs_list[i]) for i in range(len(xt_list))]) + + return dx[0], dw, None, None + + +def moe_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + """ """ + batch_size, sequence_length, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + # router_logits: (batch * sequence_length, n_experts) + router_logits = self.gate(hidden_states) + + routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float) + routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1) + if self.norm_topk_prob: # only diff with mixtral sparse moe block! + routing_weights /= routing_weights.sum(dim=-1, keepdim=True) + # we cast back to the input dtype + routing_weights = routing_weights.to(hidden_states.dtype) + + final_hidden_states = torch.zeros( + (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device + ) + + # One hot encode the selected experts to create an expert mask + # this will be used to easily index which expert is going to be sollicitated + expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0) + + # Loop over all available experts in the model and perform the computation on each expert + # Concat all weights + input_dtype = hidden_states.dtype + up_weight_list = [e.up_proj.weight.t().to(input_dtype) for e in self.experts] + gate_weight_list = [e.gate_proj.weight.t().to(input_dtype) for e in self.experts] + down_weight_list = [e.down_proj.weight.t().to(input_dtype) for e in self.experts] + w1 = torch.stack(up_weight_list) + w2 = torch.stack(gate_weight_list) + w3 = torch.stack(down_weight_list) + + # Copied from mindspeed moe_utils.py:permute + routing_map = selected_experts + flatten_indices = routing_map.view(-1) + sorted_indices = torch.sort(flatten_indices.float(), stable=True)[1] + permuted_tokens = hidden_states.index_select(0, sorted_indices // self.top_k) + + tokens_per_experts = torch.sum(expert_mask, dim=(1, 2)) + group_list = torch.cumsum(tokens_per_experts, dim=0) + + cpu_group_list = group_list.to("cpu", non_blocking=False) + cpu_group_list = [0] + cpu_group_list.tolist() + split_size = [cpu_group_list[i + 1] - cpu_group_list[i] for i in range(len(cpu_group_list) - 1)] + + up_res = GmmFunction.apply(permuted_tokens, w1, group_list, split_size) + gate_res = GmmFunction.apply(permuted_tokens, w2, group_list, split_size) + act_res = torch_npu.npu_swiglu(torch.cat([gate_res, up_res], dim=-1)) + down_res = GmmFunction.apply(act_res, w3, group_list, split_size) + + probs = routing_weights + num_unpermuted_tokens = probs.numel() + topk = self.top_k + permuted_tokens = down_res + + unpermuted_tokens = torch.zeros( + [num_unpermuted_tokens, permuted_tokens.shape[-1]], + dtype=permuted_tokens.dtype, + device=permuted_tokens.device, + ) + unpermuted_tokens.index_copy_(0, sorted_indices, permuted_tokens) + unpermuted_tokens = unpermuted_tokens.reshape(-1, topk, permuted_tokens.size(-1)) + unpermuted_tokens = unpermuted_tokens * probs.unsqueeze(-1) + unpermuted_tokens = unpermuted_tokens.sum(dim=1).to(hidden_states.dtype) + final_hidden_states = unpermuted_tokens + + return final_hidden_states, router_logits + + +@classmethod +def _check_and_enable_flash_attn_2( + cls, + config, + torch_dtype: Optional[torch.dtype] = None, + device_map: Optional[str | dict[str, int]] = None, + check_device_map: bool = True, + hard_check_only: bool = False, +) -> PretrainedConfig: + """ + Checks the availability of Flash Attention 2 and compatibility with the current model. + + If all checks pass and `hard_check_only` is False, the method will set the config attribute + `attn_implementation` to "flash_attention_2" so that the model can initialize + the correct attention module. + """ + if not cls._supports_flash_attn_2: + raise ValueError( + f"{cls.__name__} does not support Flash Attention 2.0 yet. Please request to add support where the" + f" model is hosted, on its model hub page: https://huggingface.co/{config._name_or_path}/discussions/new" + " or in the Transformers GitHub repo: https://github.com/huggingface/transformers/issues/new" + ) + + if not hard_check_only: + config._attn_implementation = "flash_attention_2" + logger.info("Detect using FlashAttention2 on Ascend NPU.") + return config + + +modeling_qwen2_5_vl.Qwen2RMSNorm.forward = rms_norm_forward +modeling_qwen2_5_vl.Qwen2_5_VLMLP.forward = silu_forward +modeling_qwen2_5_vl.apply_rotary_pos_emb_flashatt = apply_rotary_pos_emb_flashatt_qwen2_5_vl_npu +modeling_qwen3_moe.Qwen3MoeRMSNorm.forward = rms_norm_forward +modeling_qwen3_moe.Qwen3MoeSparseMoeBlock.forward = moe_block_forward +modeling_qwen3_moe.apply_rotary_pos_emb = apply_rotary_pos_emb_qwen3_npu +modeling_qwen3.Qwen3RMSNorm.forward = rms_norm_forward +modeling_qwen3.Qwen3MLP.forward = silu_forward + +if get_version("transformers") == "4.52.4": + PreTrainedModel._check_and_enable_flash_attn_2 = _check_and_enable_flash_attn_2 diff --git a/e2c/verl/verl/models/transformers/qwen2.py b/e2c/verl/verl/models/transformers/qwen2.py new file mode 100644 index 0000000000000000000000000000000000000000..3bac76e9a142530e86a32c3ad4228e6964afc19a --- /dev/null +++ b/e2c/verl/verl/models/transformers/qwen2.py @@ -0,0 +1,243 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Optional + +import torch +from transformers.cache_utils import Cache +from transformers.modeling_flash_attention_utils import _flash_attention_forward +from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv +from transformers.utils import logging + +# Import compatibility wrapper for flash_attn_supports_top_left_mask +from verl.utils.transformers_compat import flash_attn_supports_top_left_mask +from verl.utils.ulysses import ( + gather_heads_scatter_seq, + gather_seq_scatter_heads, + get_ulysses_sequence_parallel_world_size, + validate_ulysses_config, +) + +logger = logging.get_logger(__name__) + + +def qwen2_flash_attn_forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 +): + """ + Adapted from transformers 4.47.1 to support Ulysses sequence parallelism. + + NOTE: This function is only tested on transformers versions between 4.45.0 and 4.47.1. + """ + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + + ########## AlltoAll for Ulysses ########## + ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + + if ulysses_sp_size > 1: + validate_ulysses_config(self.num_heads, ulysses_sp_size) + + # (bsz, n_head, seq_len/n, head_dim) -> (bsz, n_head/n, seq_len, head_dim) + query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) + key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) + value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) + + full_q_len = query_states.size(2) # full seq length + + if position_embeddings is None: + logger.warning_once( + "The attention layers in this model are transitioning from computing the RoPE embeddings internally " + "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " + "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " + "removed and `position_embeddings` will be mandatory." + ) + cos, sin = self.rotary_emb(value_states, position_ids) + else: + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + dropout_rate = 0.0 if not self.training else self.attention_dropout + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to " + f"the fact you have upcasted embedding or layer norm layers in float32. We will cast back the " + f"input in {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # Reashape to the expected shape for Flash Attention + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + if ( + self.config.use_sliding_window + and getattr(self.config, "sliding_window", None) is not None + and self.layer_idx >= self.config.max_window_layers + ): + sliding_window = self.config.sliding_window + else: + sliding_window = None + + attn_output = _flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + full_q_len, + position_ids=position_ids, + dropout=dropout_rate, + sliding_window=sliding_window, + is_causal=self.is_causal, + use_top_left_mask=flash_attn_supports_top_left_mask(), + ) + + # use full_q_len to reshape + attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +def qwen2_attn_forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_value: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, +) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + """ + Adapted from transformers 4.49.0 to support Ulysses sequence parallelism for transformers >= 4.48.0. + + NOTE: This function has been tested only on transformers versions between 4.48.0 and 4.50.0. + """ + from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS + + bsz, q_len, _ = hidden_states.shape + hidden_shape = (bsz, q_len, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + ########## AlltoAll for Ulysses ########## + ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + + if ulysses_sp_size > 1: + validate_ulysses_config(self.config.num_attention_heads, ulysses_sp_size) + + # (bsz, n_head, seq_len/n, head_dim) -> (bsz, n_head/n, seq_len, head_dim) + query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) + key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) + value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) + + full_q_len = query_states.size(2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + sliding_window = None + if ( + self.config.use_sliding_window + and getattr(self.config, "sliding_window", None) is not None + and self.layer_idx >= self.config.max_window_layers + ): + sliding_window = self.config.sliding_window + + from transformers.models.qwen2.modeling_qwen2 import eager_attention_forward + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False): + logger.warning_once( + "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. " + "Falling back to eager attention. This warning can be removed using the argument " + '`attn_implementation="eager"` when loading the model.' + ) + else: + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + sliding_window=sliding_window, # main diff with Llama + **kwargs, + ) + + attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + # (bsz, seq_len, n_head/n, head_dim) -> (bsz, seq_len/n, n_head, head_dim) + attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights diff --git a/e2c/verl/verl/models/transformers/qwen2_vl.py b/e2c/verl/verl/models/transformers/qwen2_vl.py new file mode 100644 index 0000000000000000000000000000000000000000..5cfb7bae1be612ced607ee678b7e06eddd99e25f --- /dev/null +++ b/e2c/verl/verl/models/transformers/qwen2_vl.py @@ -0,0 +1,549 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import logging +import os +from dataclasses import dataclass +from typing import Optional + +import torch +import torch.distributed as dist +from transformers.modeling_flash_attention_utils import _flash_attention_forward, fa_peft_integration_check +from transformers.models.qwen2_vl.modeling_qwen2_vl import ( + Qwen2VLAttention, + Qwen2VLCausalLMOutputWithPast, + Qwen2VLForConditionalGeneration, +) +from transformers.utils import is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10 + +from verl.utils.device import is_npu_available +from verl.utils.transformers_compat import is_transformers_version_in_range +from verl.utils.ulysses import ( + gather_heads_scatter_seq, + gather_seq_scatter_heads, + get_ulysses_sequence_parallel_group, + get_ulysses_sequence_parallel_world_size, + validate_ulysses_config, +) + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + + _flash_supports_window_size = "window_size" in inspect.signature(flash_attn_func).parameters + _flash_supports_deterministic = "deterministic" in inspect.signature(flash_attn_func).parameters + _flash_use_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + +if is_npu_available: + from transformers.integrations.npu_flash_attention import npu_flash_attn_func as flash_attn_func + from transformers.integrations.npu_flash_attention import npu_flash_attn_varlen_func as flash_attn_varlen_func + from transformers.modeling_flash_attention_utils import flash_attn_supports_top_left_mask + + _flash_supports_window_size = "window_size" in inspect.signature(flash_attn_func).parameters + _flash_supports_deterministic = "deterministic" in inspect.signature(flash_attn_func).parameters + _flash_use_top_left_mask = flash_attn_supports_top_left_mask() + +_flash_deterministic_enabled = os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" + + +def get_rope_index( + processor, + input_ids: torch.Tensor, + image_grid_thw: Optional[torch.Tensor] = None, + video_grid_thw: Optional[torch.Tensor] = None, + second_per_grid_ts: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + Gets the position ids for Qwen2-VL, it should be generated before sharding the sequence. + The batch dim has been removed and the input_ids should be a 1D tensor representing a single example. + https://github.com/huggingface/transformers/blob/v4.52.4/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py#L1405 + """ + spatial_merge_size = processor.image_processor.merge_size + tokens_per_second = 2 + image_token_id = processor.tokenizer.convert_tokens_to_ids("<|image_pad|>") + video_token_id = processor.tokenizer.convert_tokens_to_ids("<|video_pad|>") + vision_start_token_id = processor.tokenizer.convert_tokens_to_ids("<|vision_start|>") + if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + + position_ids = torch.ones(3, input_ids.size(0), dtype=input_ids.dtype, device=input_ids.device) # (3, seqlen) + image_index, video_index = 0, 0 + input_ids = input_ids[attention_mask == 1] + image_nums, video_nums = 0, 0 + vision_start_indices = torch.argwhere(input_ids == vision_start_token_id) + vision_tokens = input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + second_per_grid_t = 0 + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + second_per_grid_t = second_per_grid_ts[video_index] if second_per_grid_ts is not None else 1.0 + + video_index += 1 + remain_videos -= 1 + ed = ed_video + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w) + t_index = (t_index * second_per_grid_t * tokens_per_second).long().flatten() + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + text_len = len(input_tokens) - st + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., attention_mask == 1] = llm_positions.to(position_ids.device) + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = position_ids.unsqueeze(0).expand(3, -1).to(input_ids.device) + else: + position_ids = torch.arange(input_ids.shape[1], device=input_ids.device).view(1, -1).expand(3, -1) + + return position_ids + + +def prepare_fa2_from_position_ids( + query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, position_ids: torch.Tensor +): + assert position_ids.ndim == 2 # (batch_size, seq_length) + query = query.contiguous().view(-1, query.size(-2), query.size(-1)) + key = key.contiguous().view(-1, key.size(-2), key.size(-1)) + value = value.contiguous().view(-1, value.size(-2), value.size(-1)) + position_ids = position_ids.view(-1) + cu_seqlens = torch.cat( + ( + (position_ids == 0).nonzero().view(-1).to(torch.int32), + torch.tensor(position_ids.size(), device=position_ids.device, dtype=torch.int32), + ) + ) + max_length = cu_seqlens.diff().max() # use cu_seqlens to infer max_length for qwen2vl mrope + return (query, key, value, (cu_seqlens, cu_seqlens), (max_length, max_length)) + + +def _custom_flash_attention_forward( + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + attention_mask: Optional[torch.Tensor], + query_length: int, + is_causal: bool = True, + position_ids: Optional[torch.Tensor] = None, + sliding_window: Optional[int] = None, + use_top_left_mask: bool = False, + deterministic: Optional[bool] = None, + **kwargs, +): + """ + Patches flash attention forward to handle 3D position ids in mrope. (3, batch_size, seq_length) + """ + # Assuming 4D tensors, key_states.shape[1] is the key/value sequence length (source length). + use_sliding_windows = ( + _flash_supports_window_size and sliding_window is not None and key_states.shape[1] > sliding_window + ) + flash_kwargs = {"window_size": (sliding_window, sliding_window)} if use_sliding_windows else {} + + if _flash_supports_deterministic: + flash_kwargs["deterministic"] = deterministic if deterministic is not None else _flash_deterministic_enabled + + if kwargs.get("softcap") is not None: + flash_kwargs["softcap"] = kwargs.pop("softcap") + + query_states, key_states, value_states = fa_peft_integration_check( + query_states, key_states, value_states, target_dtype=torch.bfloat16 + ) + + if position_ids is not None: + assert position_ids.ndim == 2 # (batch_size, seq_length / sp_size) + + sp_size = get_ulysses_sequence_parallel_world_size() + if sp_size > 1: + # qkv: (batch_size, seq_length / sp_size, num_head, head_size) + validate_ulysses_config(query_states.size(2), sp_size) + query_states = gather_seq_scatter_heads(query_states, seq_dim=1, head_dim=2) + key_states = gather_seq_scatter_heads(key_states, seq_dim=1, head_dim=2) + value_states = gather_seq_scatter_heads(value_states, seq_dim=1, head_dim=2) + position_ids_lst = [torch.empty_like(position_ids) for _ in range(sp_size)] + position_ids = dist.all_gather(position_ids_lst, position_ids, group=get_ulysses_sequence_parallel_group()) + position_ids = torch.cat(position_ids_lst, dim=-1) # (batch_size, seq_length) + + if position_ids is not None and query_length != 1 and not (torch.diff(position_ids, dim=-1) >= 0).all(): + batch_size = query_states.size(0) + q, k, v, (cu_seqlens_q, cu_seqlens_k), (max_seqlen_q, max_seqlen_k) = prepare_fa2_from_position_ids( + query_states, key_states, value_states, position_ids + ) + attn_output = flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + dropout_p=kwargs.pop("dropout", 0.0), + softmax_scale=kwargs.pop("softmax_scale", None), + causal=is_causal, + **flash_kwargs, + ) + attn_output = attn_output.view(batch_size, -1, attn_output.size(-2), attn_output.size(-1)) + else: + attn_output = _flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + query_length, + is_causal=is_causal, + sliding_window=sliding_window, + use_top_left_mask=use_top_left_mask, + deterministic=deterministic, + **kwargs, + ) # do not pass position_ids to old flash_attention_forward + + if sp_size > 1: + # (batch_size, seq_length, num_head, head_size) + attn_output = gather_heads_scatter_seq(attn_output, head_dim=2, seq_dim=1) + + return attn_output + + +def qwen2_vl_attn_forward( + self: "Qwen2VLAttention", + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + **kwargs, +) -> tuple[torch.Tensor, None, None]: + from transformers.models.qwen2_vl.modeling_qwen2_vl import apply_multimodal_rotary_pos_emb, repeat_kv + + bsz, q_len, _ = hidden_states.size() # q_len = seq_length / sp_size + query_states = self.q_proj(hidden_states) # (batch_size, seq_length / sp_size, num_heads * head_size) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + # Because the input can be padded, the absolute sequence length depends on the max position id. + cos, sin = position_embeddings + query_states, key_states = apply_multimodal_rotary_pos_emb( + query_states, key_states, cos, sin, self.rope_scaling["mrope_section"] + ) + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + dropout_rate = 0.0 if not self.training else self.attention_dropout + + sliding_window = None + if ( + self.config.use_sliding_window + and getattr(self.config, "sliding_window", None) is not None + and self.layer_idx >= self.config.max_window_layers + ): + sliding_window = self.config.sliding_window + + # This is before the transpose + q_len = query_states.shape[2] + + # FA2 uses non-transposed inputs + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + if position_ids.ndim == 3: + position_ids = position_ids[0] + + attn_output = _custom_flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + query_length=q_len, + is_causal=getattr(self, "is_causal", True), + dropout=dropout_rate, + sliding_window=sliding_window, + use_top_left_mask=_flash_use_top_left_mask, + position_ids=position_ids, # important: pass position ids + ) # (batch_size, seq_length / sp_size, num_head, head_size) + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + if is_transformers_version_in_range(min_version="4.54.0"): + return attn_output, None + else: + return attn_output, None, None + + +def _get_input_embeds( + model: "Qwen2VLForConditionalGeneration", + input_ids: torch.LongTensor, + attention_mask: Optional[torch.Tensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, +): + inputs_embeds = model.get_input_embeddings()(input_ids) + if pixel_values is not None: + pixel_values = pixel_values.type(model.visual.dtype) + image_embeds = model.visual(pixel_values, grid_thw=image_grid_thw) + n_image_tokens = (input_ids == model.config.image_token_id).sum().item() + n_image_features = image_embeds.shape[0] + if n_image_tokens != n_image_features: + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}" + ) + + mask = input_ids == model.config.image_token_id + mask_unsqueezed = mask.unsqueeze(-1) + mask_expanded = mask_unsqueezed.expand_as(inputs_embeds) + image_mask = mask_expanded.to(inputs_embeds.device) + + image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) + inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) + + if pixel_values_videos is not None: + pixel_values_videos = pixel_values_videos.type(model.visual.dtype) + video_embeds = model.visual(pixel_values_videos, grid_thw=video_grid_thw) + n_video_tokens = (input_ids == model.config.video_token_id).sum().item() + n_video_features = video_embeds.shape[0] + if n_video_tokens != n_video_features: + raise ValueError( + f"Video features and video tokens do not match: tokens: {n_video_tokens}, features {n_video_features}" + ) + + mask = input_ids == model.config.video_token_id + mask_unsqueezed = mask.unsqueeze(-1) + mask_expanded = mask_unsqueezed.expand_as(inputs_embeds) + video_mask = mask_expanded.to(inputs_embeds.device) + + video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) + inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) + + if model.training and pixel_values is None and pixel_values_videos is None: # handle mixed text-image data + pixel_values = torch.zeros((16, 1176), dtype=inputs_embeds.dtype, device=inputs_embeds.device) + image_grid_thw = torch.tensor([[1, 4, 4]], dtype=torch.long, device=inputs_embeds.device) + image_embeds = model.visual(pixel_values, grid_thw=image_grid_thw) + inputs_embeds += 0.0 * image_embeds.mean() + + if attention_mask is not None: + attention_mask = attention_mask.to(inputs_embeds.device) + + return inputs_embeds, attention_mask + + +def process_position_ids(position_ids: torch.Tensor) -> torch.Tensor: + if position_ids.ndim != 3 or position_ids.size(0) != 4: + # we concat the text position ids with the 3D vision position ids by default + # see https://github.com/huggingface/transformers/pull/39447 + raise ValueError("position_ids should be a 3D tensor of shape (4, batch_size, seq_length).") + + if is_transformers_version_in_range(max_version="4.53.3"): + # transformers < 4.54.0 only accepts vision position ids, so we discard the text position ids here + position_ids = position_ids[1:] + + return position_ids + + +@dataclass +class Qwen2VLCausalLMOutputForPPO(Qwen2VLCausalLMOutputWithPast): + log_probs: Optional[torch.FloatTensor] = None + entropy: Optional[torch.FloatTensor] = None + + +def qwen2_vl_base_forward( + self: "Qwen2VLForConditionalGeneration", + input_ids: torch.LongTensor, + attention_mask: Optional[torch.Tensor] = None, + labels: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs, +): + kwargs["inputs_embeds"], kwargs["attention_mask"] = _get_input_embeds( + self, input_ids, attention_mask, pixel_values, pixel_values_videos, image_grid_thw, video_grid_thw + ) # avoid lora module having multiple keyword arguments + return self.language_model( + input_ids=None, + **kwargs, + ) + + +def qwen2_vl_forward( + self: "Qwen2VLForConditionalGeneration", + input_ids: torch.LongTensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + **kwargs, +): + if is_transformers_version_in_range(min_version="4.52.0"): + return self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=process_position_ids(position_ids), + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + **kwargs, + ) + else: + inputs_embeds, attention_mask = _get_input_embeds( + self, input_ids, attention_mask, pixel_values, pixel_values_videos, image_grid_thw, video_grid_thw + ) + return self.model( + input_ids=None, + attention_mask=attention_mask, + position_ids=process_position_ids(position_ids), + inputs_embeds=inputs_embeds, + **kwargs, + ) + + +def forward_with_normal_backend( + self: Qwen2VLForConditionalGeneration, + input_ids: torch.LongTensor = None, + labels: Optional[torch.LongTensor] = None, + temperature: float = 1.0, + **kwargs, +) -> "Qwen2VLCausalLMOutputWithPast": + outputs = qwen2_vl_forward(self, input_ids, **kwargs) + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + + return Qwen2VLCausalLMOutputWithPast( + logits=logits, + hidden_states=outputs.hidden_states, + ) + + +def forward_with_torch_backend( + self: Qwen2VLForConditionalGeneration, + input_ids: torch.LongTensor = None, + labels: Optional[torch.LongTensor] = None, + temperature: float = 1.0, + **kwargs, +) -> tuple | Qwen2VLCausalLMOutputForPPO: + from verl.utils.experimental.torch_functional import FusedLinearForPPO + + outputs = qwen2_vl_forward(self, input_ids, **kwargs) + hidden_states = outputs[0] + + # Loss calculations + if labels is not None: + rolled_labels = torch.roll(labels, shifts=-1, dims=-1) + elif input_ids is not None: + rolled_labels = torch.roll(input_ids, shifts=-1, dims=-1) + else: + raise RuntimeError("To use forward_with_torch_backend, either labels or input_ids must be provided.") + + fused_linear_for_ppo = FusedLinearForPPO() + log_probs, entropy = fused_linear_for_ppo.forward( + hidden_states=hidden_states, + vocab_weights=self.lm_head.weight, + input_ids=rolled_labels, + temperature=temperature, + ) + return Qwen2VLCausalLMOutputForPPO( + log_probs=log_probs, + entropy=entropy, + hidden_states=outputs.hidden_states, + ) + + +def forward_with_triton_backend( + self: Qwen2VLForConditionalGeneration, + input_ids: torch.LongTensor = None, + labels: Optional[torch.LongTensor] = None, + temperature: float = 1.0, + **kwargs, +) -> tuple | Qwen2VLCausalLMOutputForPPO: + from verl.utils.kernel.linear_cross_entropy import linear_cross_entropy + + outputs = qwen2_vl_forward(self, input_ids, **kwargs) + hidden_states = outputs[0] + + # Loss calculations + if labels is not None: + rolled_labels = torch.roll(labels, shifts=-1, dims=-1) + elif input_ids is not None: + rolled_labels = torch.roll(input_ids, shifts=-1, dims=-1) + else: + raise RuntimeError("To use forward_with_triton_backend, either labels or input_ids must be provided.") + + log_probs, entropy = linear_cross_entropy( + hidden_states, + self.lm_head.weight, + rolled_labels, + temperature, + "none", + ) + return Qwen2VLCausalLMOutputForPPO( + log_probs=log_probs, + entropy=entropy, + hidden_states=outputs.hidden_states, + ) diff --git a/e2c/verl/verl/models/weight_loader_registry.py b/e2c/verl/verl/models/weight_loader_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..0904f14fad4ca6be2161e99a7dec7682cbe47983 --- /dev/null +++ b/e2c/verl/verl/models/weight_loader_registry.py @@ -0,0 +1,57 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def get_weight_loader(arch: str): + from verl.models.mcore.loader import load_state_dict_to_megatron_gptmodel + + _MODEL_WEIGHT_MEGATRON_LOADER_REGISTRY = { + "LlamaForCausalLM": load_state_dict_to_megatron_gptmodel, + "Qwen2ForCausalLM": load_state_dict_to_megatron_gptmodel, + } + + if arch in _MODEL_WEIGHT_MEGATRON_LOADER_REGISTRY: + return _MODEL_WEIGHT_MEGATRON_LOADER_REGISTRY[arch] + raise ValueError( + f"Model architectures {arch} loader are not supported for now. Supported architectures: " + f"{_MODEL_WEIGHT_MEGATRON_LOADER_REGISTRY.keys()}" + ) + + +def get_weight_saver(arch: str): + from verl.models.mcore.saver import ( + merge_megatron_ckpt_gptmodel, + merge_megatron_ckpt_gptmodel_dpskv3, + merge_megatron_ckpt_gptmodel_mixtral, + merge_megatron_ckpt_gptmodel_qwen2_5_vl, + merge_megatron_ckpt_gptmodel_qwen_moe, + ) + + _MODEL_WEIGHT_MEGATRON_SAVER_REGISTRY = { + "LlamaForCausalLM": merge_megatron_ckpt_gptmodel, + "Qwen2ForCausalLM": merge_megatron_ckpt_gptmodel, + "MixtralForCausalLM": merge_megatron_ckpt_gptmodel_mixtral, + "Qwen2MoeForCausalLM": merge_megatron_ckpt_gptmodel_qwen_moe, + "Qwen2_5_VLForConditionalGeneration": merge_megatron_ckpt_gptmodel_qwen2_5_vl, + "DeepseekV3ForCausalLM": merge_megatron_ckpt_gptmodel_dpskv3, + "Qwen3ForCausalLM": merge_megatron_ckpt_gptmodel, + "Qwen3ForTokenClassification": merge_megatron_ckpt_gptmodel, + "Qwen3MoeForCausalLM": merge_megatron_ckpt_gptmodel_qwen_moe, + } + if arch in _MODEL_WEIGHT_MEGATRON_SAVER_REGISTRY: + return _MODEL_WEIGHT_MEGATRON_SAVER_REGISTRY[arch] + raise ValueError( + f"Model architectures {arch} saver are not supported for now. Supported architectures: " + f"{_MODEL_WEIGHT_MEGATRON_SAVER_REGISTRY.keys()}" + ) diff --git a/e2c/verl/verl/protocol.py b/e2c/verl/verl/protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..b412dd454e1ab3fa463ad51231839dd7b8d21f12 --- /dev/null +++ b/e2c/verl/verl/protocol.py @@ -0,0 +1,1172 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Implement base data transfer protocol between any two functions, modules. +We can subclass Protocol to define more detailed batch info with specific keys +""" + +import contextlib +import copy +import logging +import math +import os +import pickle +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +import numpy as np +import ray +import tensordict +import torch +import torch.distributed +from packaging import version +from packaging.version import parse as parse_version +from tensordict import TensorDict +from torch.utils.data import DataLoader + +from verl.utils.device import get_device_id, get_torch_device +from verl.utils.py_functional import union_two_dict +from verl.utils.torch_functional import allgather_dict_tensors + +__all__ = ["DataProto", "union_tensor_dict"] + +with contextlib.suppress(Exception): + tensordict.set_lazy_legacy(False).set() + if parse_version(tensordict.__version__) < parse_version("0.10.0"): + tensordict.set_list_to_stack(True).set() + + +class _DataProtoConfigMeta(type): + _config = {} + + auto_padding_key = "_verl_auto_padding" + + @property + def auto_padding(cls): + enabled_by_env = os.getenv("VERL_AUTO_PADDING", "FALSE").upper() in ["TRUE", "1"] + return enabled_by_env or cls._config.get(cls.auto_padding_key, False) + + @auto_padding.setter + def auto_padding(cls, enabled: bool): + assert isinstance(enabled, bool), f"enabled must be a boolean, got {enabled} as {type(enabled)}" + cls._config[cls.auto_padding_key] = enabled + + +class DataProtoConfig(metaclass=_DataProtoConfigMeta): + pass + + +_padding_size_key = "_padding_size_key_x123d" + + +def pad_dataproto_to_divisor(data: "DataProto", size_divisor: int): + """Pad a DataProto to size divisible by size_divisor + + Args: + size_divisor (int): size divisor + + Returns: + data: (DataProto): the padded DataProto + pad_size (int) + """ + assert isinstance(data, DataProto), "data must be a DataProto" + if len(data) % size_divisor != 0: + pad_size = size_divisor - len(data) % size_divisor + padding_protos = [] + remaining_pad = pad_size + while remaining_pad > 0: + take_size = min(remaining_pad, len(data)) + padding_protos.append(data[:take_size]) + remaining_pad -= take_size + data_padded = DataProto.concat([data] + padding_protos) + else: + if len(data) == 0: + logging.warning("padding a DataProto with no item, no changed made") + pad_size = 0 + data_padded = data + return data_padded, pad_size + + +def unpad_dataproto(data: "DataProto", pad_size): + """Unpad the data proto with pad_size. i.e. `data[:-pad_size]`""" + if pad_size != 0: + data = data[:-pad_size] + return data + + +def union_tensor_dict(tensor_dict1: TensorDict, tensor_dict2: TensorDict) -> TensorDict: + """Union two tensordicts.""" + assert tensor_dict1.batch_size == tensor_dict2.batch_size, ( + f"Two tensor dict must have identical batch size. Got {tensor_dict1.batch_size} and {tensor_dict2.batch_size}" + ) + for key in tensor_dict2.keys(): + if key not in tensor_dict1.keys(): + tensor_dict1[key] = tensor_dict2[key] + else: + assert tensor_dict1[key].equal(tensor_dict2[key]), ( + f"{key} in tensor_dict1 and tensor_dict2 are not the same object" + ) + + return tensor_dict1 + + +def _array_equal(array1: np.ndarray, array2: np.ndarray, visited: set[int]) -> bool: + """ + Recursively compares two NumPy arrays for strict equality, with special + handling for object-dtype arrays, NaN values, and circular references. + This function assumes that the two arguments provided are NumPy arrays. + + Args: + array1: The first NumPy array. + array2: The second NumPy array. + + Returns: + True if the arrays' dtypes, shapes, and all elements are equal. + """ + # Check dtype and shape first, as this is the fastest failure path. + if array1.dtype != array2.dtype or array1.shape != array2.shape: + return False + + # For non-object dtypes, use NumPy's implementation with equal_nan=True. + if array1.dtype != "object": + return np.array_equal(array1, array2, equal_nan=True) + + # For object-dtype arrays, we must recursively compare each element. + # We delegate to _deep_equal to handle elements, as they could be any + # type, including other nested arrays or NaNs. + return all(_deep_equal(x, y, visited) for x, y in zip(array1.flat, array2.flat, strict=False)) + + +def _deep_equal(a: Any, b: Any, visited: set[int]) -> bool: + """ + Recursively performs a deep comparison between two Python objects. + - Handles NaN values correctly (NaN == NaN evaluates to True). + - Handling circular references. + - Dispatches to _array_equal if both objects are NumPy arrays. + - Otherwise, uses standard '==' comparison. + """ + if type(a) is not type(b): + return False + + # If we have seen this object ID before on this path, it's a cycle. + # Since we already know the types match, we can safely assume this part + # of the structure is equal. + obj_id = id(a) + if obj_id in visited: + return True + + visited.add(obj_id) + + # Perform the specific comparison based on type + result = False + if isinstance(a, float) and math.isnan(a) and math.isnan(b): + result = True + elif isinstance(a, np.ndarray): + # We know b is also an ndarray due to the initial type check + result = _array_equal(a, b, visited) + else: + # Standard equality for all other types + result = a == b + + # Clean up the visited set on the way out of the recursion + visited.remove(obj_id) + return result + + +def union_numpy_dict(tensor_dict1: dict[str, np.ndarray], tensor_dict2: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + for key, val in tensor_dict2.items(): + if key in tensor_dict1: + assert isinstance(tensor_dict2[key], np.ndarray) + assert isinstance(tensor_dict1[key], np.ndarray) + # to properly deal with nan and object type + assert _deep_equal(tensor_dict1[key], tensor_dict2[key], visited=set()), ( + f"`{key}` in tensor_dict1 and tensor_dict2 are not the same object." + ) + tensor_dict1[key] = val + + return tensor_dict1 + + +def list_of_dict_to_dict_of_list(list_of_dict: list[dict]): + if len(list_of_dict) == 0: + return {} + keys = list_of_dict[0].keys() + output = {key: [] for key in keys} + for data in list_of_dict: + for key, item in data.items(): + assert key in output + output[key].append(item) + return output + + +def fold_batch_dim(data: "DataProto", new_batch_size): + """ + Fold a batch dim from [bsz, xxx] into [new_bsz, bsz // new_bsz, xxx] + """ + batch_size = data.batch.batch_size[0] + + assert batch_size % new_batch_size == 0 + + tensor: TensorDict = data.batch + non_tensor = data.non_tensor_batch + + tensor = tensor.view(new_batch_size, -1) + tensor.auto_batch_size_(batch_dims=1) + + for key, val in non_tensor.items(): + non_tensor[key] = np.reshape(val, newshape=(new_batch_size, -1, *val.shape[1:])) + + return type(data)(batch=tensor, non_tensor_batch=non_tensor, meta_info=data.meta_info) + + +def unfold_batch_dim(data: "DataProto", batch_dims=2): + """ + Unfold the first n dims as new batch dim + """ + tensor: TensorDict = data.batch + non_tensor = data.non_tensor_batch + tensor.auto_batch_size_(batch_dims=batch_dims) + tensor = tensor.view(-1) + + batch_size = tensor.batch_size[0] + + non_tensor_new = {} + + for key, val in non_tensor.items(): + non_tensor_new[key] = np.reshape(val, newshape=(batch_size, *val.shape[batch_dims:])) + + return type(data)(batch=tensor, non_tensor_batch=non_tensor_new, meta_info=data.meta_info) + + +def serialize_single_tensor(obj: torch.Tensor) -> tuple[str, tuple[int, ...], int | memoryview]: + data = obj.flatten().contiguous().view(torch.uint8).numpy() + dtype = str(obj.dtype).removeprefix("torch.") + return dtype, obj.shape, data + + +def serialize_tensordict(batch: TensorDict) -> tuple[tuple[int, ...], Optional[str], dict[str, tuple[str, Any]]]: + encoded_items: dict[str, tuple[Any]] = {} + for k, v in batch.items(): + if not v.is_nested: + encoded_items[k] = serialize_single_tensor(v) + else: + layout = str(v.layout).removeprefix("torch.") + data = [serialize_single_tensor(tensor) for tensor in v.unbind()] + encoded_items[k] = (layout, data) + + batch_size = tuple(batch.batch_size) + device = str(batch.device) if batch.device is not None else None + return batch_size, device, encoded_items + + +def deserialize_single_tensor(arr: Any) -> torch.Tensor: + dtype, shape, data = arr + + torch_dtype = getattr(torch, dtype) + assert isinstance(torch_dtype, torch.dtype) + + buffer = bytearray(data) + # Create uint8 array + arr = torch.frombuffer(buffer, dtype=torch.uint8) + # Convert back to proper shape & type + return arr.view(torch_dtype).view(shape) + + +def deserialize_tensordict(arr: Any) -> TensorDict: + batch_size, device, encoded_items = arr + decoded_items: dict[str, Any] = {} + + for k, v in encoded_items.items(): + if len(v) == 3: + # decode single tensor + decoded_items[k] = deserialize_single_tensor(v) + elif len(v) == 2: + # decode nested tensor + layout, data = v + torch_layout = getattr(torch, layout) + decoded_items[k] = torch.nested.as_nested_tensor( + [deserialize_single_tensor(tensor) for tensor in data], layout=torch_layout + ) + else: + raise ValueError(f"Invalid tensor encoding format, expected length 2 or 3, got {len(v)}") + + return TensorDict(source=decoded_items, batch_size=batch_size, device=device) + + +def collate_fn(x: list["DataProtoItem"]): + batch = [] + non_tensor_batch = [] + for data in x: + batch.append(data.batch) + non_tensor_batch.append(data.non_tensor_batch) + batch = torch.stack(batch).contiguous() + non_tensor_batch = list_of_dict_to_dict_of_list(non_tensor_batch) + for key, val in non_tensor_batch.items(): + non_tensor_batch[key] = np.array(val, dtype=object) + return DataProto(batch=batch, non_tensor_batch=non_tensor_batch) + + +@dataclass +class DataProtoItem: + # TODO(zhangchi.usc1992) add consistency check + batch: TensorDict = None + non_tensor_batch: dict = field(default_factory=dict) + meta_info: dict = field(default_factory=dict) + + +@dataclass +class DataProto: + """ + A DataProto is a data structure that aims to provide a standard protocol for data exchange between functions. + It contains a batch (TensorDict) and a meta_info (Dict). The batch is a TensorDict https://pytorch.org/tensordict/. + TensorDict allows you to manipulate a dictionary of Tensors like a single Tensor. Ideally, the tensors with the + same batch size should be put inside batch. + """ + + batch: TensorDict = None + non_tensor_batch: dict = field(default_factory=dict) + meta_info: dict = field(default_factory=dict) + + def __post_init__(self): + # perform necessary checking + self.check_consistency() + + def __len__(self): + if self.batch is not None: + return self.batch.batch_size[0] + elif self.non_tensor_batch is not None and len(self.non_tensor_batch) > 0: + random_key = list(self.non_tensor_batch.keys())[0] + return self.non_tensor_batch[random_key].shape[0] + else: + return 0 + + def __getitem__(self, item): + """ + Enhanced indexing for DataProto objects. + + Args: + item: Can be one of: + - int: A single index + - slice: A slice object (start:stop:step) + - list: A list of indices + - numpy.ndarray: An array of indices + - torch.Tensor: A tensor of indices + + Returns: + DataProto: For all indexing types except single integers + DataProtoItem: Only for single integer indices + """ + # Case 1: Slice object - use the slice method + if isinstance(item, slice): + return self.slice(item.start, item.stop, item.step) + + # Case 2: List, numpy array, or torch tensor - use sel_idxs + elif isinstance(item, list | np.ndarray | torch.Tensor): + return self.select_idxs(item) + + # Case 3: Single integer - return DataProtoItem for backward compatibility + elif isinstance(item, int | np.integer): + tensor_data = self.batch[item] if self.batch is not None else None + non_tensor_data = {key: val[item] for key, val in self.non_tensor_batch.items()} + return DataProtoItem(batch=tensor_data, non_tensor_batch=non_tensor_data, meta_info=self.meta_info) + + # # Case 4: Unsupported type + else: + raise TypeError(f"Indexing with {type(item)} is not supported") + + def __getstate__(self): + if version.parse(tensordict.__version__) >= version.parse("0.5.0") and self.batch is not None: + batch = self.batch.contiguous().consolidate() + else: + batch = self.batch + + if os.getenv("VERL_DATAPROTO_SERIALIZATION_METHOD") == "numpy": + if batch is not None: + batch = serialize_tensordict(self.batch) + + return ( + batch, + self.non_tensor_batch, + self.meta_info, + ) + else: + import io + + buffer = io.BytesIO() + torch.save(batch, buffer) + buffer_bytes = buffer.getvalue() + return buffer_bytes, self.non_tensor_batch, self.meta_info + + def __setstate__(self, data): + batch_deserialized_bytes, non_tensor_batch, meta_info = data + + if os.getenv("VERL_DATAPROTO_SERIALIZATION_METHOD") == "numpy": + if batch_deserialized_bytes is not None: + self.batch = deserialize_tensordict(batch_deserialized_bytes) + else: + self.batch = None + else: + import io + + batch_deserialized = io.BytesIO(initial_bytes=batch_deserialized_bytes) + batch = torch.load( + batch_deserialized, + weights_only=False, + map_location="cpu" if not get_torch_device().is_available() else None, + ) + self.batch = batch + + self.non_tensor_batch = non_tensor_batch + self.meta_info = meta_info + + def save_to_disk(self, filepath): + with open(filepath, "wb") as f: + pickle.dump(self, f) + + @staticmethod + def load_from_disk(filepath) -> "DataProto": + with open(filepath, "rb") as f: + data = pickle.load(f) + return data + + def print_size(self, prefix=""): + size_of_tensordict = 0 + if self.batch is not None: + for _, tensor in self.batch.items(): + size_of_tensordict += tensor.element_size() * tensor.numel() + size_of_numpy_array = 0 + for _, numpy_array in self.non_tensor_batch.items(): + size_of_numpy_array += numpy_array.nbytes + + size_of_numpy_array /= 1024**3 + size_of_tensordict /= 1024**3 + + message = f"Size of tensordict: {size_of_tensordict} GB, size of non_tensor_batch: {size_of_numpy_array} GB" + + if prefix: + message = f"{prefix}, " + message + print(message) + + def check_consistency(self): + """Check the consistency of the DataProto. Mainly for batch and non_tensor_batch + We expose this function as a public one so that user can call themselves directly + """ + if self.batch is not None: + assert len(self.batch.batch_size) == 1, "only support num_batch_dims=1" + + if self.non_tensor_batch is not None: + for key, val in self.non_tensor_batch.items(): + assert isinstance(val, np.ndarray) + + if self.batch is not None and self.non_tensor_batch is not None and len(self.non_tensor_batch) != 0: + # TODO: we can actually lift this restriction if needed + assert len(self.batch.batch_size) == 1, "only support num_batch_dims=1 when non_tensor_batch is not empty." + + batch_size = self.batch.batch_size[0] + for key, val in self.non_tensor_batch.items(): + assert isinstance(val, np.ndarray), ( + f"data in the non_tensor_batch must be a numpy.array with dtype=object, but for " + f"{key=}, got {type(val)=}" + ) + assert val.shape[0] == batch_size, ( + f"key {key} length {len(val)} is not equal to batch size {batch_size}" + ) + + @classmethod + def from_single_dict(cls, data: dict[str, torch.Tensor | np.ndarray], meta_info=None, auto_padding=False): + """Create a DataProto from a dict of tensors and non_tensors""" + tensors = {} + non_tensors = {} + + for key, val in data.items(): + if isinstance(val, torch.Tensor): + tensors[key] = val + elif isinstance(val, np.ndarray): + non_tensors[key] = val + else: + raise ValueError(f"Unsupported type in data {type(val)}") + + return cls.from_dict(tensors=tensors, non_tensors=non_tensors, meta_info=meta_info, auto_padding=auto_padding) + + @classmethod + def from_dict( + cls, + tensors: Optional[dict[str, torch.Tensor]] = None, + non_tensors=None, + meta_info=None, + num_batch_dims=1, + auto_padding=False, + ): + """Create a DataProto from a dict of tensors. This assumes that + 1. All the tensor in tensors have the same dim0 + 2. Only dim0 is the batch dim + """ + + assert num_batch_dims > 0, "num_batch_dims must be greater than zero" + if non_tensors is not None: + assert num_batch_dims == 1, "only support num_batch_dims=1 when non_tensors is not None." + + if tensors is None: + tensors = {} + if meta_info is None: + meta_info = {} + if non_tensors is None: + non_tensors = {} + + assert isinstance(non_tensors, dict) + + # get and check batch size + batch_size = None + pivot_key = None + for key, tensor in tensors.items(): + if batch_size is None: + batch_size = tensor.shape[:num_batch_dims] + pivot_key = key + else: + current_batch = tensor.shape[:num_batch_dims] + assert batch_size == current_batch, ( + f"Not all the tensor in tensors have the same batch size with batch_dims={num_batch_dims}. " + f"Got {pivot_key} has {batch_size}, {key} has {current_batch}" + ) + + for key, val in non_tensors.items(): + if not isinstance(val, np.ndarray): + non_tensors[key] = np.array(val, dtype=object) + + tensor_dict = TensorDict(source=tensors, batch_size=batch_size) if tensors else None + if auto_padding: + meta_info[DataProtoConfig.auto_padding_key] = True + return cls(batch=tensor_dict, non_tensor_batch=non_tensors, meta_info=meta_info) + + def to(self, device) -> "DataProto": + """move the batch to device + + Args: + device (torch.device, str): torch device + + Returns: + DataProto: the current DataProto + + """ + if self.batch is not None: + self.batch = self.batch.to(device) + return self + + def select(self, batch_keys=None, non_tensor_batch_keys=None, meta_info_keys=None, deepcopy=False) -> "DataProto": + """Select a subset of the DataProto via batch_keys and meta_info_keys + + Args: + batch_keys (list, optional): a list of strings indicating the keys in batch to select + meta_info_keys (list, optional): a list of keys indicating the meta info to select + + Returns: + DataProto: the DataProto with the selected batch_keys and meta_info_keys + """ + # TODO (zhangchi.usc1992) whether to copy + if batch_keys is not None: + batch_keys = tuple(batch_keys) + sub_batch = self.batch.select(*batch_keys) + else: + sub_batch = self.batch + + if non_tensor_batch_keys is not None: + non_tensor_batch = {key: val for key, val in self.non_tensor_batch.items() if key in non_tensor_batch_keys} + else: + non_tensor_batch = self.non_tensor_batch + + if deepcopy: + non_tensor_batch = copy.deepcopy(non_tensor_batch) + + if meta_info_keys is not None: + sub_meta_info = {key: val for key, val in self.meta_info.items() if key in meta_info_keys} + else: + sub_meta_info = self.meta_info + + if deepcopy: + sub_meta_info = copy.deepcopy(sub_meta_info) + + return type(self)(batch=sub_batch, non_tensor_batch=non_tensor_batch, meta_info=sub_meta_info) + + def select_idxs(self, idxs): + """ + Select specific indices from the DataProto. + + Args: + idxs (torch.Tensor or numpy.ndarray or list): Indices to select + + Returns: + DataProto: A new DataProto containing only the selected indices + """ + if isinstance(idxs, list): + idxs = torch.tensor(idxs) + if idxs.dtype != torch.bool: + idxs = idxs.type(torch.int32) + + if isinstance(idxs, np.ndarray): + idxs_np = idxs + idxs_torch = torch.from_numpy(idxs) + else: # torch.Tensor + idxs_torch = idxs + idxs_np = idxs.detach().cpu().numpy() + + batch_size = int(idxs_np.sum()) if idxs_np.dtype == bool else idxs_np.shape[0] + + if self.batch is not None: + # Use TensorDict's built-in indexing capabilities + selected_batch = TensorDict( + source={key: tensor[idxs_torch] for key, tensor in self.batch.items()}, + batch_size=(batch_size,), + device=self.batch.device, + ) + else: + selected_batch = None + + selected_non_tensor = {} + for key, val in self.non_tensor_batch.items(): + selected_non_tensor[key] = val[idxs_np] + + return type(self)(batch=selected_batch, non_tensor_batch=selected_non_tensor, meta_info=self.meta_info) + + def slice(self, start=None, end=None, step=None): + """ + Slice the DataProto and return a new DataProto object. + This is an improved version of direct slicing which returns a DataProtoItem. + + Args: + start (int, optional): Start index. Defaults to None (start from beginning). + end (int, optional): End index (exclusive). Defaults to None (go to end). + step (int, optional): Step size. Defaults to None (step=1). + + Returns: + DataProto: A new DataProto containing the sliced data + + Examples: + # Using the slice method directly + sliced_data = data_proto.slice(10, 20) + + # Using enhanced indexing (returns DataProto) + sliced_data = data_proto[10:20] + sliced_data = data_proto[::2] # Every other element + + # Using list indexing (returns DataProto) + indices = [1, 5, 10] + selected_data = data_proto[indices] + + # Single index still returns DataProtoItem + single_item = data_proto[5] + """ + # Create a slice object + slice_obj = slice(start, end, step) + + # Handle the batch data + if self.batch is not None: + # Use TensorDict's built-in slicing capabilities + sliced_batch = self.batch[slice_obj] + else: + sliced_batch = None + + # Handle the non-tensor batch data + sliced_non_tensor = {} + for key, val in self.non_tensor_batch.items(): + sliced_non_tensor[key] = val[slice_obj] + + # Return a new DataProto object + return type(self)(batch=sliced_batch, non_tensor_batch=sliced_non_tensor, meta_info=self.meta_info) + + def pop(self, batch_keys=None, non_tensor_batch_keys=None, meta_info_keys=None) -> "DataProto": + """Pop a subset of the DataProto via `batch_keys` and `meta_info_keys` + + Args: + batch_keys (list, optional): a list of strings indicating the keys in batch to pop + meta_info_keys (list, optional): a list of keys indicating the meta info to pop + + Returns: + DataProto: the DataProto with the poped batch_keys and meta_info_keys + """ + if batch_keys is None: + batch_keys = [] + if meta_info_keys is None: + meta_info_keys = [] + if non_tensor_batch_keys is None: + non_tensor_batch_keys = [] + + tensors = {} + # tensor batch + for key in batch_keys: + assert key in self.batch.keys() + tensors[key] = self.batch.pop(key) + non_tensors = {} + # non tensor batch + for key in non_tensor_batch_keys: + assert key in self.non_tensor_batch.keys() + non_tensors[key] = self.non_tensor_batch.pop(key) + meta_info = {} + for key in meta_info_keys: + assert key in self.meta_info.keys() + meta_info[key] = self.meta_info.pop(key) + return DataProto.from_dict(tensors=tensors, non_tensors=non_tensors, meta_info=meta_info) + + def rename(self, old_keys=None, new_keys=None) -> "DataProto": + """ + Note that this function only rename the key in the batch + """ + + def validate_input(keys): + if keys is not None: + if isinstance(keys, str): + keys = [keys] + elif isinstance(keys, list): + pass + else: + raise TypeError(f"keys must be a list or a string, but got {type(keys)}") + return keys + + old_keys = validate_input(old_keys) + new_keys = validate_input(new_keys) + + if len(new_keys) != len(old_keys): + raise ValueError( + f"new_keys and old_keys must have the same length, but got {len(new_keys)} and {len(old_keys)}" + ) + + self.batch.rename_key_(tuple(old_keys), tuple(new_keys)) + + return self + + def union(self, other: "DataProto") -> "DataProto": + """Union with another DataProto. Union batch and meta_info separately. + Throw an error if + + - there are conflict keys in batch and they are not equal + - the batch size of two data batch is not the same + - there are conflict keys in meta_info and they are not the same. + + Args: + other (DataProto): another DataProto to union + + Returns: + DataProto: the DataProto after union + """ + self.batch = union_tensor_dict(self.batch, other.batch) + self.non_tensor_batch = union_numpy_dict(self.non_tensor_batch, other.non_tensor_batch) + self.meta_info = union_two_dict(self.meta_info, other.meta_info) + return self + + def make_iterator(self, mini_batch_size, epochs, seed=None, dataloader_kwargs=None): + r"""Make an iterator from the DataProto. This is built upon that TensorDict can be used as a normal Pytorch + dataset. See https://pytorch.org/tensordict/tutorials/data_fashion for more details. + + + Args: + mini_batch_size (int): mini-batch size when iterating the dataset. We require that + ``batch.batch_size[0] % mini_batch_size == 0``. + epochs (int): number of epochs when iterating the dataset. + dataloader_kwargs (Any): internally, it returns a DataLoader over the batch. The + dataloader_kwargs is the kwargs passed to the DataLoader. + + Returns: + Iterator: an iterator that yields a mini-batch data at a time. The total number of iteration + steps is ``self.batch.batch_size * epochs // mini_batch_size`` + """ + assert self.batch.batch_size[0] % mini_batch_size == 0, f"{self.batch.batch_size[0]} % {mini_batch_size} != 0" + # we can directly create a dataloader from TensorDict + if dataloader_kwargs is None: + dataloader_kwargs = {} + + if seed is not None: + generator = torch.Generator() + generator.manual_seed(seed) + else: + generator = None + + assert isinstance(dataloader_kwargs, dict) + train_dataloader = DataLoader( + dataset=self, batch_size=mini_batch_size, collate_fn=collate_fn, generator=generator, **dataloader_kwargs + ) + + def get_data(): + for _ in range(epochs): + for d in train_dataloader: + d.meta_info = self.meta_info + yield d + + return iter(get_data()) + + def is_padding_enabled(self): + """ + Check if padding is enabled for the DataProto. + Returns: + bool: True if padding is enabled, False otherwise. + """ + dataproto_specific_padding = self.meta_info.get(DataProtoConfig.auto_padding_key, False) + return dataproto_specific_padding or DataProtoConfig.auto_padding + + def padding(self, padding_size, padding_candidate=""): + """Pad the DataProto by concating with padding_candidate.repeat(padding_size) + + Args: + padding_size (int): the number of repeated padding_candidate + padding_candidate: the item to be repeated and appended to the DataProto, only supporting ["first", "last"] + """ + if padding_size == 0: + return + padding_candidate = self.select_idxs([0 if padding_candidate == "first" else len(self) - 1]) + padding_part = padding_candidate.repeat(padding_size) + padded_dp = DataProto.concat([self, padding_part]) + self.batch = padded_dp.batch + self.non_tensor_batch = padded_dp.non_tensor_batch + + def chunk(self, chunks: int) -> list["DataProto"]: + """Split the batch among dim=0 into chunks. The meta_info is passed to each DataProto after split. + + Args: + chunks (int): the number of chunks to split on dim=0 + + Returns: + List[DataProto]: a list of DataProto after splitting + """ + if not self.is_padding_enabled(): + assert len(self) % chunks == 0, ( + f"only support equal chunk. Got size of DataProto {len(self)} and chunk {chunks}." + ) + + bsz_in_batch = None + if self.batch is not None: + batch_lst = self.batch.chunk(chunks=chunks, dim=0) + bsz_in_batch = np.array([batch.batch_size[0] for batch in batch_lst]) + chunk_indices = np.cumsum(bsz_in_batch)[:-1] + else: + batch_lst = [None for _ in range(chunks)] + + non_tensor_batch_lst = [{} for _ in range(chunks)] + for key, val in self.non_tensor_batch.items(): + assert isinstance(val, np.ndarray) + if bsz_in_batch is not None: + non_tensor_lst = np.array_split(val, chunk_indices.tolist()) + else: + non_tensor_lst = np.array_split(val, chunks) + assert len(non_tensor_lst) == chunks + for i in range(chunks): + non_tensor_batch_lst[i][key] = non_tensor_lst[i] + + output = [] + for i in range(chunks): + output.append( + type(self)(batch=batch_lst[i], non_tensor_batch=non_tensor_batch_lst[i], meta_info=self.meta_info) + ) + + return output + + def split(self, split_size: int) -> list["DataProto"]: + """Split the batch among dim=0 into chunks. The meta_info is passed to each DataProto after split. + + Args: + split_size (int): the size of each split + + Returns: + List[DataProto]: a list of DataProto after splitting + """ + return [self[i : i + split_size] for i in range(0, len(self), split_size)] + + @staticmethod + def concat(data: list["DataProto"]) -> "DataProto": + """Concat a list of DataProto. The batch is concatenated among dim=0. + The meta_info is assumed to be identical and will use the first one. + + Args: + data (List[DataProto]): list of DataProto + + Returns: + DataProto: concatenated DataProto + """ + batch_lst = [] + for batch in data: + batch_lst.append(batch.batch) + new_batch = torch.cat(batch_lst, dim=0) if batch_lst[0] is not None else None + + non_tensor_batch = list_of_dict_to_dict_of_list(list_of_dict=[d.non_tensor_batch for d in data]) + for key, val in non_tensor_batch.items(): + non_tensor_batch[key] = np.concatenate(val, axis=0) + + cls = type(data[0]) if len(data) > 0 else DataProto + return cls(batch=new_batch, non_tensor_batch=non_tensor_batch, meta_info=data[0].meta_info) + + def reorder(self, indices): + """ + Note that this operation is in-place + """ + indices_np = indices.detach().numpy() + self.batch = self.batch[indices] + self.non_tensor_batch = {key: val[indices_np] for key, val in self.non_tensor_batch.items()} + + def repeat(self, repeat_times=2, interleave=True): + """ + Repeat the batch data a specified number of times. + + Args: + repeat_times (int): Number of times to repeat the data. + interleave (bool): Whether to interleave the repeated data. + + Returns: + DataProto: A new DataProto with repeated data. + """ + if self.batch is not None: + if interleave: + # Interleave the data + repeated_tensors = { + key: tensor.repeat_interleave(repeat_times, dim=0) for key, tensor in self.batch.items() + } + else: + # Stack the data + repeated_tensors = { + key: tensor.unsqueeze(0).expand(repeat_times, *tensor.shape).reshape(-1, *tensor.shape[1:]) + for key, tensor in self.batch.items() + } + + repeated_batch = TensorDict( + source=repeated_tensors, + batch_size=(self.batch.batch_size[0] * repeat_times,), + ) + else: + repeated_batch = None + + repeated_non_tensor_batch = {} + for key, val in self.non_tensor_batch.items(): + if interleave: + repeated_non_tensor_batch[key] = np.repeat(val, repeat_times, axis=0) + else: + repeated_non_tensor_batch[key] = np.tile(val, (repeat_times,) + (1,) * (val.ndim - 1)) + + return type(self)( + batch=repeated_batch, + non_tensor_batch=repeated_non_tensor_batch, + meta_info=self.meta_info, + ) + + def unfold_column_chunks(self, n_split: int, split_keys: Optional[list[str]] = None): + """Split along the second dim into `n_split`, unfold it to the first dim (batch dim) + Useful in passing grouped tensors that doesn't want to be shuffled in dataset. + keys not in split_keys are repeated to match the shape + Note that if the `split_keys` is not provided, it will repeat all the keys in the second dim. + """ + if self.batch is not None: + unfolded_batch = {} + for key in self.batch.keys(): + if key in split_keys if split_keys is not None else False: + shape = list(self.batch[key].shape) + shape[0] = self.batch[key].shape[0] * n_split + shape[1] = self.batch[key].shape[1] // n_split + unfolded_batch[key] = self.batch[key].reshape(*shape) + else: + unfolded_batch[key] = torch.repeat_interleave(self.batch[key], n_split, dim=0) + # locate the `unfolded_batch` as a TensorDict on the same device as the original batch + unfolded_batch = TensorDict( + source=unfolded_batch, batch_size=(self.batch.batch_size[0] * n_split,), device=self.batch.device + ) + else: + unfolded_batch = None + + repeated_non_tensor_batch = {} + for key, val in self.non_tensor_batch.items(): + if key in split_keys: + shape = list(val.shape) + shape[0] = val.shape[0] * n_split + shape[1] = val.shape[1] // n_split + repeated_non_tensor_batch[key] = val.reshape(*shape) + else: + repeated_non_tensor_batch[key] = np.repeat(val, n_split, axis=0) + + return type(self)( + batch=unfolded_batch, + non_tensor_batch=repeated_non_tensor_batch, + meta_info=self.meta_info, + ) + + def sample_level_repeat(self, repeat_times): + """ + Repeat each row of the batch data a specified number of times. + + Args: + repeat_times (torch.tensor, list, tuple, ndarray): Number of times to repeat the data. + + Returns: + DataProto: A new DataProto with repeated data. + """ + if isinstance(repeat_times, tuple): + repeat_times = list(repeat_times) + elif isinstance(repeat_times, torch.Tensor): + assert len(repeat_times.shape) == 1 + repeat_times = repeat_times.tolist() + elif isinstance(repeat_times, np.ndarray): + assert len(repeat_times.shape) == 1 + repeat_times = repeat_times.tolist() + else: + assert isinstance(repeat_times, list), ( + f"repeat_times type must be in [list, torch.Tensor, np.ndarray, tuple], got {type(repeat_times)}" + ) + repeat_times = torch.tensor(repeat_times) + + if self.batch is not None: + # Interleave the data + repeated_tensors = { + key: tensor.repeat_interleave(repeat_times, dim=0) for key, tensor in self.batch.items() + } + + repeated_batch = TensorDict( + source=repeated_tensors, + batch_size=(repeat_times.sum().item(),), + device=self.batch.device, + ) + else: + repeated_batch = None + + repeated_non_tensor_batch = {} + for key, val in self.non_tensor_batch.items(): + repeated_non_tensor_batch[key] = np.repeat(val, repeat_times, axis=0) + + return type(self)( + batch=repeated_batch, + non_tensor_batch=repeated_non_tensor_batch, + meta_info=self.meta_info, + ) + + def to_tensordict(self) -> TensorDict: + """Convert this DataProto to TensorDict. Note that this requires tensordict version at least 0.10 + + Returns: + + """ + assert parse_version(tensordict.__version__) >= parse_version("0.10"), ( + "Convert DataProto to TensorDict at least requires tensordict version 0.10" + ) + tensor_batch = self.batch.to_dict() + non_tensor_batch = self.non_tensor_batch + + from verl.utils import tensordict_utils as tu + + common_keys = set(tensor_batch.keys()) & set(non_tensor_batch.keys()) + assert len(common_keys) == 0, f"tensor_batch and non_tensor_batch have common keys {common_keys}" + + for key, val in non_tensor_batch.items(): + assert isinstance(val, np.ndarray) + tensor_batch[key] = val.tolist() + output = tu.get_tensordict(tensor_dict=tensor_batch, non_tensor_dict=self.meta_info) + return output + + def get_data_info(self) -> str: + """Return formatted information about stored data with nested type details. + + Returns: + str: Formatted string showing tensor details and recursive metadata types + """ + info = ["batch"] + + for key, tensor in self.batch.items(): + if hasattr(tensor, "shape") and hasattr(tensor, "dtype") and hasattr(tensor, "device"): + info.append(f" {key}: {tuple(tensor.shape)} ({tensor.dtype}) {tensor.device}") + elif hasattr(tensor, "shape") and hasattr(tensor, "dtype"): + info.append(f" {key}: {tuple(tensor.shape)} ({tensor.dtype})") + else: + info.append(f" {key}: {type(tensor).__name__}") + + info.append("non_tensor_batch") + for key, array in self.non_tensor_batch.items(): + info.append(f" {key}: ndarray{array.shape} ({array.dtype})") + + info.append("meta_info") + for k, v in self.meta_info.items(): + type_info = self._get_type_info(v) + info.append(f" {k}: {type_info}") + + return "\n".join(info) + + def _get_type_info(self, value): + """Recursively get type information for nested structures""" + if isinstance(value, list): + elem_types = {self._get_type_info(v) for v in value[:3]} + return f"list[{'|'.join(elem_types) if elem_types else '...'}]" + if isinstance(value, tuple): + elem_types = [self._get_type_info(v) for v in value] + return f"tuple({', '.join(elem_types)})" + if isinstance(value, dict): + if not value: + return "dict" + k, v = next(iter(value.items())) + return f"dict[{self._get_type_info(k)}: {self._get_type_info(v)}]" + if isinstance(value, np.ndarray): + return f"ndarray{value.shape} ({value.dtype})" + return type(value).__name__ + + +@dataclass +class DataProtoFuture: + """ + DataProtoFuture aims to eliminate actual data fetching on driver. By doing so, the driver doesn't have to wait + for data so that asynchronous execution becomes possible. + DataProtoFuture contains a list of futures from another WorkerGroup of size world_size. + - collect_fn is a Callable that reduces the list of futures to a DataProto + - dispatch_fn is a Callable that partitions the DataProto into a list of DataProto of size world_size + and then select + + Potential issue: we can optimize dispatch_fn(collect_fn) such that only needed data is fetched on destination + - DataProtoFuture only supports directly passing from the output of a method to another input. You can't perform any + operation on the DataProtoFuture in driver. + """ + + collect_fn: Callable + futures: list[ray.ObjectRef] + dispatch_fn: Callable = None + + @staticmethod + def concat(data: list[ray.ObjectRef]) -> "DataProtoFuture": + output = DataProtoFuture(collect_fn=DataProto.concat, futures=data) + return output + + def chunk(self, chunks: int) -> list["DataProtoFuture"]: + from functools import partial + + arg_future_lst = [] + for i in range(chunks): + # note that we can't directly pass i and chunks + def dispatch_fn(x, i, chunks): + return x.chunk(chunks=chunks)[i] + + arg_future = DataProtoFuture( + collect_fn=self.collect_fn, dispatch_fn=partial(dispatch_fn, i=i, chunks=chunks), futures=self.futures + ) + arg_future_lst.append(arg_future) + return arg_future_lst + + def get(self): + output = ray.get(self.futures) # dp_size. + for o in output: + assert isinstance(o, DataProto) + output = self.collect_fn(output) # select dp, concat + if self.dispatch_fn is not None: + output = self.dispatch_fn(output) # split in batch dim, select using dp + return output + + +def all_gather_data_proto(data: DataProto, process_group): + # Note that this is an inplace operator just like torch.distributed.all_gather + group_size = torch.distributed.get_world_size(group=process_group) + assert isinstance(data, DataProto) + prev_device = data.batch.device + data = data.to(get_device_id()) + data.batch = allgather_dict_tensors(data.batch.contiguous(), size=group_size, group=process_group, dim=0) + data = data.to(prev_device) + # all gather non_tensor_batch + all_non_tensor_batch = [None for _ in range(group_size)] + torch.distributed.all_gather_object(all_non_tensor_batch, data.non_tensor_batch, group=process_group) + data.non_tensor_batch = {k: np.concatenate([d[k] for d in all_non_tensor_batch]) for k in data.non_tensor_batch} diff --git a/e2c/verl/verl/py.typed b/e2c/verl/verl/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/e2c/verl/verl/single_controller/__init__.py b/e2c/verl/verl/single_controller/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ad6c42a80d188702247c23198e29a44611c81a0d --- /dev/null +++ b/e2c/verl/verl/single_controller/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from . import base +from .base import * + +version_folder = os.path.dirname(os.path.join(os.path.abspath(__file__))) + +# Note(haibin.lin): single_controller.__version__ is deprecated +with open(os.path.join(os.path.join(version_folder, os.pardir), "version/version")) as f: + __version__ = f.read().strip() + + +__all__ = base.__all__ diff --git a/e2c/verl/verl/single_controller/base/__init__.py b/e2c/verl/verl/single_controller/base/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b24bd9942b872b71f4c7b3a2dbfe6db5530cfe25 --- /dev/null +++ b/e2c/verl/verl/single_controller/base/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .worker import Worker +from .worker_group import ClassWithInitArgs, ResourcePool, WorkerGroup + +__all__ = ["Worker", "WorkerGroup", "ClassWithInitArgs", "ResourcePool"] diff --git a/e2c/verl/verl/single_controller/base/decorator.py b/e2c/verl/verl/single_controller/base/decorator.py new file mode 100644 index 0000000000000000000000000000000000000000..b246aaf7eecea2b77ccb3bfc01497a48dba503e2 --- /dev/null +++ b/e2c/verl/verl/single_controller/base/decorator.py @@ -0,0 +1,446 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from functools import partial, wraps +from types import FunctionType + +from verl.protocol import DataProtoFuture, _padding_size_key +from verl.utils.py_functional import DynamicEnum + +# here we add a magic number of avoid user-defined function already have this attribute +MAGIC_ATTR = "attrs_3141562937" + + +class Dispatch(DynamicEnum): + """Enum class defining different dispatch modes for distributed computation. + + Each mode represents a specific strategy for distributing data across + different ranks in a distributed system. The modes are used to control + how data is partitioned and processed across different worker groups. + """ + + _registry = {} + _next_value = 0 + + +def init_predefined_dispatch_mode(): + Dispatch.register("RANK_ZERO") + Dispatch.register("ONE_TO_ALL") + Dispatch.register("ALL_TO_ALL") + Dispatch.register("DP_COMPUTE") + Dispatch.register("DP_COMPUTE_PROTO") + Dispatch.register("DP_COMPUTE_PROTO_WITH_FUNC") + Dispatch.register("DP_COMPUTE_METRIC") + # This is a special dispatch mode for vllm ExternalRayDistributedExecutor + Dispatch.register("DIRECT_ROLLOUT_METHOD") + + +class Execute(DynamicEnum): + """Enum class defining different execution modes for distributed computation. + + These modes control how a function should be executed across different ranks + in a distributed system. + """ + + _registry = {} + _next_value = 0 + + +def init_predefined_execute_mode(): + Execute.register("ALL") + Execute.register("RANK_ZERO") + + +# Initialize the two Dynamic Enum Classes +init_predefined_dispatch_mode() +init_predefined_execute_mode() + + +def _split_args_kwargs_data_proto(chunks, *args, **kwargs): + from verl.protocol import DataProto, DataProtoFuture + + splitted_args = [] + for arg in args: + assert isinstance(arg, DataProto | DataProtoFuture) + splitted_args.append(arg.chunk(chunks=chunks)) + + splitted_kwargs = {} + for key, val in kwargs.items(): + assert isinstance(val, DataProto | DataProtoFuture) + splitted_kwargs[key] = val.chunk(chunks=chunks) + + return splitted_args, splitted_kwargs + + +def _split_args_kwargs_data_proto_with_auto_padding(chunks, *args, **kwargs): + from verl.protocol import DataProto, DataProtoFuture + + data_proto_len = None + padding_size = None + + def _padding_and_split_data(obj, chunks): + nonlocal data_proto_len, padding_size + assert isinstance(obj, DataProto | DataProtoFuture) + if isinstance(obj, DataProto) and obj.is_padding_enabled(): + # for padding, we only support DataProto with same length + if data_proto_len is None: + data_proto_len = len(obj) + padding_size = (chunks - (data_proto_len % chunks)) if (data_proto_len % chunks > 0) else 0 + else: + assert data_proto_len == len(obj), ( + f"expecting all arg share same length of {data_proto_len}, but got {len(obj)}" + ) + obj.padding(padding_size=padding_size) + return obj.chunk(chunks=chunks) + + splitted_args = [_padding_and_split_data(arg, chunks) for arg in args] + splitted_kwargs = {key: _padding_and_split_data(val, chunks) for key, val in kwargs.items()} + if padding_size is not None: + splitted_kwargs[_padding_size_key] = padding_size + + return splitted_args, splitted_kwargs + + +def dispatch_one_to_all(worker_group, *args, **kwargs): + args = tuple([arg] * worker_group.world_size for arg in args) + kwargs = {k: [v] * worker_group.world_size for k, v in kwargs.items()} + return args, kwargs + + +def dummy_direct_rollout_call(worker_group, *args, **kwargs): + raise NotImplementedError("Direct rollout call is forbidden.") + + +def dispatch_all_to_all(worker_group, *args, **kwargs): + return args, kwargs + + +def collect_all_to_all(worker_group, output): + return output + + +def _concat_data_proto_or_future(output: list): + import ray + + from verl.protocol import DataProto, DataProtoFuture + + # make sure all the elements in output has the same type + for o in output: + assert type(o) is type(output[0]) + + o = output[0] + + if isinstance(o, DataProto): + return DataProto.concat(output) + elif isinstance(o, ray.ObjectRef): + return DataProtoFuture.concat(output) + else: + raise NotImplementedError + + +def dispatch_dp_compute(worker_group, *args, **kwargs): + from verl.single_controller.base.worker_group import WorkerGroup + + assert isinstance(worker_group, WorkerGroup) + for arg in args: + assert isinstance(arg, tuple | list) and len(arg) == worker_group.world_size + for k, v in kwargs.items(): + assert isinstance(v, tuple | list) and len(v) == worker_group.world_size + return args, kwargs + + +def collect_dp_compute(worker_group, output): + from verl.single_controller.base.worker_group import WorkerGroup + + assert isinstance(worker_group, WorkerGroup) + assert len(output) == worker_group.world_size + return output + + +def dispatch_dp_compute_data_proto(worker_group, *args, **kwargs): + from verl.single_controller.base.worker_group import WorkerGroup + + assert isinstance(worker_group, WorkerGroup) + # Note: enable auto padding for dp compute DatapProto + splitted_args, splitted_kwargs = _split_args_kwargs_data_proto_with_auto_padding( + worker_group.world_size, + *args, + **kwargs, + ) + return splitted_args, splitted_kwargs + + +def dispatch_dp_compute_data_proto_with_func(worker_group, *args, **kwargs): + from verl.single_controller.base.worker_group import WorkerGroup + + assert isinstance(worker_group, WorkerGroup) + assert isinstance(args[0], FunctionType) # NOTE: The first one args is a function! + + splitted_args, splitted_kwargs = _split_args_kwargs_data_proto(worker_group.world_size, *args[1:], **kwargs) + splitted_args_with_func = [[args[0]] * worker_group.world_size] + splitted_args + return splitted_args_with_func, splitted_kwargs + + +def collect_dp_compute_data_proto(worker_group, output): + import ray + + from verl.protocol import DataProto + + for o in output: + assert isinstance(o, DataProto | ray.ObjectRef), f"expecting {o} to be DataProto, but got {type(o)}" + + output = collect_dp_compute(worker_group, output) + return _concat_data_proto_or_future(output) + + +def dispatch_nd_compute(dp_rank_mapping: list[int], dp_size, worker_group, *args, **kwargs): + import os + + from verl.single_controller.base.worker_group import WorkerGroup + from verl.utils.ray_utils import parallel_put + + assert isinstance(worker_group, WorkerGroup) + + max_workers = max(1, min(len(args[0]), os.cpu_count())) + + args = [parallel_put(arg, max_workers=max_workers) for arg in args] + kwargs = {k: parallel_put(v, max_workers=max_workers) for k, v in kwargs.items()} + + all_args = [] + for arg in args: + assert isinstance(arg, tuple | list) and len(arg) == dp_size + transformed_args = [] + for i in range(worker_group.world_size): + local_dp_rank = dp_rank_mapping[i] + transformed_args.append(arg[local_dp_rank]) + all_args.append(transformed_args) + all_args = tuple(all_args) + + all_kwargs = {} + for k, v in kwargs.items(): + assert isinstance(v, tuple | list) and len(v) == dp_size + transformed_v = [] + for i in range(worker_group.world_size): + local_dp_rank = dp_rank_mapping[i] + transformed_v.append(v[local_dp_rank]) + all_kwargs[k] = transformed_v + return all_args, all_kwargs + + +def collect_nd_compute(collect_mask: list[bool], worker_group, output): + from verl.single_controller.base.worker_group import WorkerGroup + + assert isinstance(worker_group, WorkerGroup) + assert len(output) == worker_group.world_size + + output_in_dp = [] + for global_rank in range(worker_group.world_size): + collect_dp_rank = collect_mask[global_rank] + if collect_dp_rank: + output_in_dp.append(output[global_rank]) + return output_in_dp + + +def dispatch_nd_compute_dataproto(dp_rank_mapping: list[int], dp_size, worker_group, *args, **kwargs): + splitted_args, splitted_kwargs = _split_args_kwargs_data_proto(dp_size, *args, **kwargs) + return dispatch_nd_compute(dp_rank_mapping, dp_size, worker_group, *splitted_args, **splitted_kwargs) + + +def collect_nd_compute_dataproto(collect_mask: list[bool], worker_group, output): + output = collect_nd_compute(collect_mask, worker_group, output) + import ray + + from verl.protocol import DataProto + + for o in output: + assert isinstance(o, DataProto | ray.ObjectRef), f"expecting {o} to be DataProto, but got {type(o)}" + return _concat_data_proto_or_future(output) + + +def dispatch_lazy_compute_data_proto(mesh_name, worker_group, *args, **kwargs): + from verl.single_controller.base.worker_group import WorkerGroup + + assert isinstance(worker_group, WorkerGroup) + + # query dispatch info of the worker group + if mesh_name not in worker_group._dispatch_info: + worker_group._dispatch_info[mesh_name] = worker_group._query_dispatch_info(mesh_name) + assert len(worker_group._dispatch_info[mesh_name]) == worker_group.world_size + + dp_rank_mapping = worker_group._dispatch_info[mesh_name] + # perform dispatch + dp_size = max(dp_rank_mapping) + 1 + return dispatch_nd_compute_dataproto(dp_rank_mapping, dp_size, worker_group, *args, **kwargs) + + +def collect_lazy_compute_data_proto(mesh_name, worker_group, *args, **kwargs): + from verl.single_controller.base.worker_group import WorkerGroup + + assert isinstance(worker_group, WorkerGroup) + + # the dispatch info is stored in the worker group + assert mesh_name in worker_group._dispatch_info + + if mesh_name not in worker_group._collect_info: + worker_group._collect_info[mesh_name] = worker_group._query_collect_info(mesh_name) + assert len(worker_group._collect_info[mesh_name]) == worker_group.world_size + + # a boolean of whether the dp_rank is used for collect + collect_mask = worker_group._collect_info[mesh_name] + # perform dispatch + return collect_nd_compute_dataproto(collect_mask, worker_group, *args, **kwargs) + + +def make_nd_compute_dataproto_dispatch_fn(mesh_name): + return { + "dispatch_fn": partial(dispatch_lazy_compute_data_proto, mesh_name), + "collect_fn": partial(collect_lazy_compute_data_proto, mesh_name), + } + + +# Global registry for dispatch mode. +DISPATCH_MODE_FN_REGISTRY = { + Dispatch.ONE_TO_ALL: { + "dispatch_fn": dispatch_one_to_all, + "collect_fn": collect_all_to_all, + }, + Dispatch.ALL_TO_ALL: { + "dispatch_fn": dispatch_all_to_all, + "collect_fn": collect_all_to_all, + }, + Dispatch.DP_COMPUTE: {"dispatch_fn": dispatch_dp_compute, "collect_fn": collect_dp_compute}, + Dispatch.DP_COMPUTE_PROTO: { + "dispatch_fn": dispatch_dp_compute_data_proto, + "collect_fn": collect_dp_compute_data_proto, + }, + Dispatch.DP_COMPUTE_PROTO_WITH_FUNC: { + "dispatch_fn": dispatch_dp_compute_data_proto_with_func, + "collect_fn": collect_dp_compute_data_proto, + }, + Dispatch.DP_COMPUTE_METRIC: {"dispatch_fn": dispatch_dp_compute_data_proto, "collect_fn": collect_dp_compute}, + Dispatch.DIRECT_ROLLOUT_METHOD: { + "dispatch_fn": dummy_direct_rollout_call, + "collect_fn": dummy_direct_rollout_call, + }, +} + + +def get_predefined_dispatch_fn(dispatch_mode): + return DISPATCH_MODE_FN_REGISTRY[dispatch_mode] + + +def register_dispatch_mode(dispatch_mode_name, dispatch_fn, collect_fn): + """ + Register a new dispatch mode. + """ + dispatch_mode = Dispatch.register(dispatch_mode_name) + _check_dispatch_mode(dispatch_mode) + assert dispatch_mode not in DISPATCH_MODE_FN_REGISTRY, f"dispatch_mode_name {dispatch_mode_name} already exists" + DISPATCH_MODE_FN_REGISTRY[dispatch_mode] = {"dispatch_fn": dispatch_fn, "collect_fn": collect_fn} + + +def update_dispatch_mode(dispatch_mode, dispatch_fn, collect_fn): + """ + Update the dispatch mode. + """ + _check_dispatch_mode(dispatch_mode) + assert dispatch_mode in DISPATCH_MODE_FN_REGISTRY, f"dispatch_mode {dispatch_mode} not found" + DISPATCH_MODE_FN_REGISTRY[dispatch_mode] = {"dispatch_fn": dispatch_fn, "collect_fn": collect_fn} + + +def get_predefined_execute_fn(execute_mode): + """ + Note that here we only asks execute_all and execute_rank_zero to be implemented + Leave the choice of how these two functions handle argument 'blocking' to users + """ + predefined_execute_mode_fn = { + Execute.ALL: {"execute_fn_name": "execute_all"}, + Execute.RANK_ZERO: {"execute_fn_name": "execute_rank_zero"}, + } + return predefined_execute_mode_fn[execute_mode] + + +def _check_dispatch_mode(dispatch_mode): + assert isinstance(dispatch_mode, Dispatch | dict), ( + f"dispatch_mode must be a Dispatch or a Dict. Got {dispatch_mode}" + ) + if isinstance(dispatch_mode, dict): + necessary_keys = ["dispatch_fn", "collect_fn"] + for key in necessary_keys: + assert key in dispatch_mode, f"key {key} should be in dispatch_mode if it is a dictionary" + + +def _check_execute_mode(execute_mode): + assert isinstance(execute_mode, Execute), f"execute_mode must be a Execute. Got {execute_mode}" + + +def _materialize_futures(*args, **kwargs): + new_args = [] + for arg in args: + if isinstance(arg, DataProtoFuture): + arg = arg.get() + # add more type to materialize + new_args.append(arg) + for k, v in kwargs.items(): + if isinstance(v, DataProtoFuture): + kwargs[k] = v.get() + + new_args = tuple(new_args) + return new_args, kwargs + + +def register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.ALL, blocking=True, materialize_futures=True): + """Register a function with distributed execution configuration. + + This decorator registers a function with specific dispatch and execution modes + for distributed computation. It handles both synchronous and asynchronous + functions, and optionally materializes futures before execution. + + Args: + dispatch_mode: + Dispatch mode for computation distribution. Default: Dispatch.ALL_TO_ALL. + execute_mode: + Execute mode for computation distribution. Default: Execute.ALL. + blocking: + Whether the execution should be blocking. Defaults to True. + materialize_futures: + Whether to materialize the data before dispatching. Defaults to True. + + Returns: + A decorator that wraps the original function with distributed execution + configuration. + """ + _check_dispatch_mode(dispatch_mode=dispatch_mode) + _check_execute_mode(execute_mode=execute_mode) + + def decorator(func): + @wraps(func) + def inner(*args, **kwargs): + if materialize_futures: + args, kwargs = _materialize_futures(*args, **kwargs) + return func(*args, **kwargs) + + @wraps(func) + async def async_inner(*args, **kwargs): + if materialize_futures: + args, kwargs = _materialize_futures(*args, **kwargs) + return await func(*args, **kwargs) + + wrapper = async_inner if inspect.iscoroutinefunction(func) else inner + attrs = {"dispatch_mode": dispatch_mode, "execute_mode": execute_mode, "blocking": blocking} + setattr(wrapper, MAGIC_ATTR, attrs) + return wrapper + + return decorator diff --git a/e2c/verl/verl/single_controller/base/worker.py b/e2c/verl/verl/single_controller/base/worker.py new file mode 100644 index 0000000000000000000000000000000000000000..59ab27a9c39989e347678a4d156c6b5be24feaec --- /dev/null +++ b/e2c/verl/verl/single_controller/base/worker.py @@ -0,0 +1,313 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +the class for Worker +""" + +import os +import socket +import warnings +from dataclasses import dataclass + +import ray + +from verl.utils.device import ( + get_torch_device, + get_visible_devices_keyword, + is_npu_available, +) + +from .decorator import Dispatch, Execute, register + + +@dataclass +class DistRankInfo: + tp_rank: int + dp_rank: int + pp_rank: int + cp_rank: int + + +@dataclass +class DistGlobalInfo: + tp_size: int + dp_size: int + pp_size: int + cp_size: int + + +class WorkerHelper: + @staticmethod + def _get_node_ip(): + if os.getenv("WG_BACKEND", None) == "ray": + return ray.util.get_node_ip_address() + else: + raise NotImplementedError("WG_BACKEND now just support ray mode.") + + @staticmethod + def _get_free_port(): + with socket.socket() as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] + + def get_availale_master_addr_port(self): + warnings.warn( + "This function is deprecated due to typo in name; Please use `get_available_master_addr_port` instead", + stacklevel=2, + ) + return self.get_available_master_addr_port() + + def get_available_master_addr_port(self): + return self._get_node_ip().strip("[]"), str(self._get_free_port()) + + +# we assume that in each WorkerGroup, there is a Master Worker +class Worker(WorkerHelper): + """A distributed worker that handles initialization and configuration for distributed training. + + This class manages worker initialization, configuration, and provides methods for executing + distributed operations. It handles communication settings, device configuration, and worker + metadata management. + """ + + fused_worker_attr_name = "fused_worker_dict" + + def _register_dispatch_collect_info(self, mesh_name: str, dp_rank: int, is_collect: bool): + """Register the dp_rank for a given mesh name. This function is meant to be called by the worker + + Args: + mesh_name (str): + Name of the mesh to register dp_rank for. + dp_rank (int): + dp_rank to register for the given mesh name. + is_collect (bool): + Whether the dp_rank is used for collect. + """ + if mesh_name in self.__dispatch_dp_rank or mesh_name in self.__collect_dp_rank: + raise ValueError(f"mesh_name {mesh_name} has been registered") + self.__dispatch_dp_rank[mesh_name] = dp_rank + self.__collect_dp_rank[mesh_name] = is_collect + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def _query_dispatch_info(self, mesh_name: str): + """Query the dispatch info for a given mesh name. + + Args: + mesh_name (str): + Name of the mesh to query dispatch info for. + + Returns: + int: + The dp_rank for the given mesh name. + """ + assert mesh_name in self.__dispatch_dp_rank, f"{mesh_name} is not registered in {self.__class__.__name__}" + # note that each rank store its own dp_rank + return self.__dispatch_dp_rank[mesh_name] + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def _query_collect_info(self, mesh_name: str): + """Query the collect info for a given mesh name. + + Args: + mesh_name (str): + Name of the mesh to query collect info for. + + Returns: + bool: + Whether the dp_rank is used for collect. + """ + assert mesh_name in self.__collect_dp_rank, f"{mesh_name} is not registered in {self.__class__.__name__}" + return self.__collect_dp_rank[mesh_name] + + @classmethod + def env_keys(cls): + """The keys of the environment variables that are used to configure the Worker.""" + return [ + "WORLD_SIZE", + "RANK", + "LOCAL_WORLD_SIZE", + "LOCAL_RANK", + "MASTER_ADDR", + "MASTER_PORT", + get_visible_devices_keyword().upper(), + ] + + def __init__(self, cuda_visible_devices=None) -> None: + """Initialize the worker with environment settings and device configuration. + + Args: + cuda_visible_devices (str, optional): + CUDA visible devices configuration. Defaults to None. + """ + # construct a meta from environment variable. Note that the import must be inside the class because + # it is executed remotely + import os + + self._setup_env_cuda_visible_devices() + + world_size = int(os.environ["WORLD_SIZE"]) + rank = int(os.environ["RANK"]) + self._rank = rank + self._world_size = world_size + + master_addr = os.environ["MASTER_ADDR"] + master_port = os.environ["MASTER_PORT"] + + local_world_size = int(os.getenv("LOCAL_WORLD_SIZE", "1")) + local_rank = int(os.getenv("LOCAL_RANK", "0")) + + store = { + "_world_size": world_size, + "_rank": rank, + "_local_world_size": local_world_size, + "_local_rank": local_rank, + "_master_addr": master_addr, + "_master_port": master_port, + } + if cuda_visible_devices is not None: + store[f"_{get_visible_devices_keyword()}".lower()] = cuda_visible_devices + + self._configure_with_store(store=store) + + self.fused_worker_dict = {} + self.__dispatch_dp_rank = {} + self.__collect_dp_rank = {} + + def get_fused_worker_by_name(self, worker_name: str): + """Get a fused worker by its name. + + Args: + worker_name (str): + Name of the worker to retrieve + """ + return self.fused_worker_dict.get(worker_name, None) + + def _setup_env_cuda_visible_devices(self): + from verl.utils.ray_utils import ray_noset_visible_devices + + is_ray_noset_visible_devices = ray_noset_visible_devices() + + # Prevent use of clashing `{CUDA/HIP/ROCR}_VISIBLE_DEVICES`` + rocr_val = os.environ.get("ROCR_VISIBLE_DEVICES", None) + hip_val = os.environ.get("HIP_VISIBLE_DEVICES", None) + cuda_val = os.environ.get("CUDA_VISIBLE_DEVICES", None) + if hip_val: + # Switch the use of HIP_VISIBLE_DEVICES to CUDA_VISIBLE_DEVICES for consistency. + # Make sure that the HIP_VISIBLE_DEVICES is set to the same value as CUDA_VISIBLE_DEVICES + # at this point. + val = os.environ.pop("HIP_VISIBLE_DEVICES") + hip_val = None + if cuda_val: + assert val == cuda_val, ( + f"Please use the same HIP_VISIBLE_DEVICES or CUDA_VISIBLE_DEVICES, inconsistant values " + f"found: {val} and {cuda_val}." + ) + else: + cuda_val = val + os.environ["CUDA_VISIBLE_DEVICES"] = val + # os.environ["HIP_VISIBLE_DEVICES"] = val + + if rocr_val: + # You must take care if both HIP/CUDA and ROCR env vars are set as they have + # different meanings. Both env vars accept either a list of ints or a + # list of UUIDs. The ROCR env var is processed first which then reduces + # the number of GPUs that HIP can select from. + # https://github.com/pytorch/pytorch/pull/144026 + # To avoid the complexity of this, we simply gives out error if both are set + # (Also to keep consistency with ray's practice with 2.45.0). + # Otherwise, we will set ROCR_VISIBLE_DEVICES to CUDA_VISIBLE_DEVICES + # and remove ROCR_VISIBLE_DEVICES. + if cuda_val: + raise ValueError("Please don't set ROCR_VISIBLE_DEVICES when HIP/CUDA_VISIBLE_DEVICES is set.") + + cuda_val = os.environ.pop("ROCR_VISIBLE_DEVICES") + os.environ["CUDA_VISIBLE_DEVICES"] = cuda_val + rocr_val = None + + if is_ray_noset_visible_devices: + # NOTE: Ray will automatically set the *_VISIBLE_DEVICES + # environment variable for each actor, unless + # RAY_EXPERIMENTAL_NOSET_*_VISIBLE_DEVICES is set, + # so we need to set local rank when the flag is set. + device_name = "NPU" if is_npu_available else "GPU" + local_rank = ray.get_runtime_context().get_accelerator_ids()[device_name][0] + os.environ["LOCAL_RANK"] = local_rank + get_torch_device().set_device(int(local_rank)) + + def _configure_with_store(self, store: dict): + """ + This function should only be called inside by WorkerGroup + """ + store_env_dict = {f"_{key.lower()}": store.get(f"_{key.lower()}", None) for key in type(self).env_keys()} + self.__dict__.update(store_env_dict) # this is hacky + # print(f"__dict__: {self.__dict__}") + for key in type(self).env_keys(): + val = self.__dict__.get(f"_{key.lower()}", None) + if val is not None: + # print(f"set {key} to {val}") + os.environ[key] = str(val) + os.environ["REDIS_STORE_SERVER_HOST"] = ( + str(self._master_addr).replace("[", "").replace("]", "") if self._master_addr else "" + ) + + def get_master_addr_port(self): + """Get the master address and port for distributed communication.""" + return self._master_addr, self._master_port + + def get_cuda_visible_devices(self): + """Get the CUDA visible devices configuration.""" + import os + + visible_devices = os.environ.get(get_visible_devices_keyword().upper(), "not set") + return visible_devices + + @property + def world_size(self): + """Get the total number of workers in the distributed setup.""" + return self._world_size + + @property + def rank(self): + """Get the rank of this worker in the distributed setup.""" + return self._rank + + @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO_WITH_FUNC) + def execute_with_func_generator(self, func, *args, **kwargs): + """Execute a function with function generator dispatch mode. + + Args: + func: + Function to execute + *args: + Positional arguments for the function + **kwargs: + Keyword arguments for the function + """ + ret_proto = func(self, *args, **kwargs) + return ret_proto + + @register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.RANK_ZERO) + def execute_func_rank_zero(self, func, *args, **kwargs): + """Execute a function in rank zero execution mode. + + Args: + func: + Function to execute + *args: + Positional arguments for the function + **kwargs: + Keyword arguments for the function + """ + result = func(*args, **kwargs) + return result diff --git a/e2c/verl/verl/single_controller/base/worker_group.py b/e2c/verl/verl/single_controller/base/worker_group.py new file mode 100644 index 0000000000000000000000000000000000000000..f5df3d6b31b32ce216dfcc6595a1e835171fb097 --- /dev/null +++ b/e2c/verl/verl/single_controller/base/worker_group.py @@ -0,0 +1,255 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +the class of WorkerGroup +""" + +import logging +import signal +import threading +import time +from typing import Any, Callable + +from .decorator import MAGIC_ATTR, Dispatch, get_predefined_dispatch_fn, get_predefined_execute_fn + + +class ResourcePool: + """ + Manages a pool of resources across multiple nodes, tracking process counts and GPU allocations. + The class provides methods to calculate world size, local world sizes, and local ranks + across all nodes in the pool. + """ + + def __init__(self, process_on_nodes=None, max_colocate_count: int = 10, n_gpus_per_node=8) -> None: + """Initialize the ResourcePool with node processes and GPU configuration. + + Args: + process_on_nodes (List[int], optional): List of process counts per node. Defaults to empty list. + max_colocate_count (int, optional): Maximum number of processes that can be colocated. Defaults to 10. + n_gpus_per_node (int, optional): Number of GPUs available per node. Defaults to 8. + """ + if process_on_nodes is None: + process_on_nodes = [] + self._store = process_on_nodes + self.max_colocate_count = max_colocate_count + self.n_gpus_per_node = n_gpus_per_node # this is left for future huawei GPU that contains 16 GPUs per node + + def add_node(self, process_count): + self._store.append(process_count) + + @property + def world_size(self): + """Total number of processes across all nodes in the pool.""" + return sum(self._store) + + def __call__(self) -> Any: + return self._store + + @property + def store(self): + return self._store + + def local_world_size_list(self) -> list[int]: + """Returns a flat list where each process has its local world size.""" + nested_local_world_size_list = [ + [local_world_size for _ in range(local_world_size)] for local_world_size in self._store + ] + return [item for row in nested_local_world_size_list for item in row] + + def local_rank_list(self) -> list[int]: + """Returns a flat list of local ranks for all processes across all nodes.""" + nested_local_rank_list = [[i for i in range(local_world_size)] for local_world_size in self._store] + return [item for row in nested_local_rank_list for item in row] + + +class ClassWithInitArgs: + """ + Wrapper class that stores constructor arguments for deferred instantiation. + This class is particularly useful for remote class instantiation where + the actual construction needs to happen at a different time or location. + """ + + def __init__(self, cls, *args, **kwargs) -> None: + """Initialize the ClassWithInitArgs instance. + + Args: + cls: The class to be instantiated later + *args: Positional arguments for the class constructor + **kwargs: Keyword arguments for the class constructor + """ + self.cls = cls + self.args = args + self.kwargs = kwargs + + self.fused_worker_used = False + + def __call__(self) -> Any: + """Instantiate the stored class with the stored arguments.""" + return self.cls(*self.args, **self.kwargs) + + +def check_workers_alive(workers: list, is_alive: Callable, gap_time: float = 1) -> None: + """Continuously monitors worker processes and raises SIGABRT if any worker dies. + + Args: + workers (List): + List of worker objects to monitor + is_alive (Callable): + Function to check if a worker is alive + gap_time (float): + Time interval between checks + """ + import time + + while True: + for worker in workers: + if not is_alive(worker): + logging.warning(f"worker {worker} is not alive sending signal to main thread") + signal.raise_signal(signal.SIGABRT) + time.sleep(gap_time) + + +class WorkerGroup: + """ + Base class for managing a group of workers in a distributed system. + The class provides methods for worker management, aliveness checking, and method binding. + """ + + fused_worker_execute_fn_name = "_fuw_execute" + + def __init__(self, resource_pool: ResourcePool, **kwargs) -> None: + self._is_init_with_detached_workers = resource_pool is None + + self.fused_worker_used = False + + if resource_pool is not None: + # handle the case when WorkGroup is attached to an existing one + self._procecss_dispatch_config = resource_pool() + else: + self._procecss_dispatch_config = None + + self._workers = [] + self._worker_names = [] + + self._dispatch_info = {} + self._collect_info = {} + + self._master_addr = None + self._master_port = None + + self._checker_thread: threading.Thread = None + + def _is_worker_alive(self, worker): + """Check if a worker is alive. Must be implemented by derived classes.""" + raise NotImplementedError("WorkerGroup._is_worker_alive called, should be implemented in derived class.") + + def _block_until_all_workers_alive(self) -> None: + """Blocks until all workers in the group are alive.""" + while True: + all_state = [self._is_worker_alive(worker) for worker in self._workers] + if False in all_state: + time.sleep(1) + else: + break + + def start_worker_aliveness_check(self, every_n_seconds=1) -> None: + """Starts a background thread to monitor worker aliveness. + + Args: + every_n_seconds (int): Interval between aliveness checks + """ + # before starting checking worker aliveness, make sure all workers are already alive + self._block_until_all_workers_alive() + + self._checker_thread = threading.Thread( + target=check_workers_alive, args=(self._workers, self._is_worker_alive, every_n_seconds) + ) + self._checker_thread.start() + + @property + def world_size(self): + """Number of workers in the group.""" + return len(self._workers) + + def _bind_worker_method(self, user_defined_cls, func_generator): + """Binds worker methods to the WorkerGroup based on registered attributes. + + Args: + user_defined_cls (type): The class containing methods to bind + func_generator (Callable): Function that generates the bound method + + Returns: + List[str]: List of method names that were successfully bound + """ + method_names = [] + for method_name in dir(user_defined_cls): + try: + method = getattr(user_defined_cls, method_name) + assert callable(method), f"{method_name} in {user_defined_cls} is not callable" + except Exception: + # if it is a property, it will fail because Class doesn't have instance property + continue + + if hasattr(method, MAGIC_ATTR): + # this method is decorated by register + attribute = getattr(method, MAGIC_ATTR) + assert isinstance(attribute, dict), f"attribute must be a dictionary. Got {type(attribute)}" + assert "dispatch_mode" in attribute, "attribute must contain dispatch_mode in its key" + + dispatch_mode = attribute["dispatch_mode"] + execute_mode = attribute["execute_mode"] + blocking = attribute["blocking"] + + # get dispatch fn + if isinstance(dispatch_mode, Dispatch): + # get default dispatch fn + fn = get_predefined_dispatch_fn(dispatch_mode=dispatch_mode) + dispatch_fn = fn["dispatch_fn"] + collect_fn = fn["collect_fn"] + else: + assert isinstance(dispatch_mode, dict) + assert "dispatch_fn" in dispatch_mode + assert "collect_fn" in dispatch_mode + dispatch_fn = dispatch_mode["dispatch_fn"] + collect_fn = dispatch_mode["collect_fn"] + + # get execute_fn_name + execute_mode = get_predefined_execute_fn(execute_mode=execute_mode) + wg_execute_fn_name = execute_mode["execute_fn_name"] + + # get execute_fn from string + try: + execute_fn = getattr(self, wg_execute_fn_name) + assert callable(execute_fn), "execute_fn must be callable" + except Exception: + print(f"execute_fn {wg_execute_fn_name} is invalid") + raise + + # bind a new method to the RayWorkerGroup + func = func_generator( + self, + method_name, + dispatch_fn=dispatch_fn, + collect_fn=collect_fn, + execute_fn=execute_fn, + blocking=blocking, + ) + + try: + setattr(self, method_name, func) + method_names.append(method_name) + except Exception as e: + raise ValueError(f"Fail to set method_name {method_name}") from e + + return method_names diff --git a/e2c/verl/verl/single_controller/ray/__init__.py b/e2c/verl/verl/single_controller/ray/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d2a5d6d3c71331c4d35002c93b0a7ce7d91eddc7 --- /dev/null +++ b/e2c/verl/verl/single_controller/ray/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import ( + RayClassWithInitArgs, + RayResourcePool, + RayWorkerGroup, + create_colocated_worker_cls, + create_colocated_worker_cls_fused, +) + +__all__ = [ + "RayClassWithInitArgs", + "RayResourcePool", + "RayWorkerGroup", + "create_colocated_worker_cls", + "create_colocated_worker_cls_fused", +] diff --git a/e2c/verl/verl/single_controller/ray/base.py b/e2c/verl/verl/single_controller/ray/base.py new file mode 100644 index 0000000000000000000000000000000000000000..3c6c1eab153ceda71be1c8859a4e7aaa0149434a --- /dev/null +++ b/e2c/verl/verl/single_controller/ray/base.py @@ -0,0 +1,888 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import logging +import socket +from copy import deepcopy +from typing import Any, Optional + +import ray +from ray.experimental.state.api import get_actor +from ray.util.placement_group import PlacementGroup, placement_group +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy, PlacementGroupSchedulingStrategy + +from verl.protocol import DataProto, _padding_size_key +from verl.single_controller.base import ClassWithInitArgs, ResourcePool, Worker, WorkerGroup +from verl.single_controller.base.decorator import MAGIC_ATTR, Dispatch +from verl.utils.py_functional import temp_env_var + +__all__ = ["Worker"] + + +def get_random_string(length: int) -> str: + import random + import string + + letters_digits = string.ascii_letters + string.digits + return "".join(random.choice(letters_digits) for _ in range(length)) + + +def func_generator(self, method_name, dispatch_fn, collect_fn, execute_fn, blocking): + class Functor: + def __call__(this, *args, **kwargs): + args, kwargs = dispatch_fn(self, *args, **kwargs) + padding_count = kwargs.pop(_padding_size_key, 0) + output = execute_fn(method_name, *args, **kwargs) + if blocking: + output = ray.get(output) + output = collect_fn(self, output) + if padding_count > 0: + if isinstance(output, DataProto): + indices = [i for i in range(len(output))][:-padding_count] + output = output.select_idxs(indices) + elif isinstance(output, list): + output = output[:-padding_count] + return output + + # use class type to pass the method_name to get a better observability + return type(method_name, (Functor,), {})() + + +def sort_placement_group_by_node_ip(pgs: list[PlacementGroup]) -> list[PlacementGroup]: + """ + Sort the placement groups by node ip, all bundles in a single placement group should be on the same node. + + FSDPCheckpointManager saves sharded model states and optimizer states in local storage, which requires RANK + to be consistent across nodes when resume from checkpoint. + + With this function, if there's only one resource pool and there's no node change, RANK should be consistent + across nodes in multiple ray jobs, even if the whole ray cluster is restarted. + """ + node_ip = {node["NodeID"]: node["NodeManagerAddress"] for node in ray.nodes()} + pg_ip = {} + for pg in pgs: + specs = ray._private.state.state.placement_group_table(pg.id) + # all bunles should be on the same node + node_id = specs["bundles_to_node_id"][0] + pg_ip[pg.id] = node_ip[node_id] + return sorted(pgs, key=lambda pg: pg_ip[pg.id]) + + +@ray.remote +def get_master_addr_port() -> tuple[str, str]: + addr = ray.util.get_node_ip_address().strip("[]") + with socket.socket() as sock: + sock.bind(("", 0)) + port = sock.getsockname()[1] + return addr, str(port) + + +class RayResourcePool(ResourcePool): + def __init__( + self, + process_on_nodes: Optional[list[int]] = None, + use_gpu: bool = True, + name_prefix: str = None, + max_colocate_count: int = 10, + detached=False, + accelerator_type: Optional[str] = None, + ) -> None: + super().__init__(process_on_nodes, max_colocate_count) + self.use_gpu = use_gpu + # print(f"in RayProcessDispatchConfiguration: name_prefix = {name_prefix}") + self.name_prefix = get_random_string(length=6) if name_prefix is None else name_prefix + self.pgs = None + self.detached = detached + self.accelerator_type = accelerator_type + + def get_placement_groups(self, strategy="STRICT_PACK", name=None, device_name="cuda"): + if self.pgs is not None: + return self.pgs + + pg_name_prefix = ( + name if name else f"{self.name_prefix}verl_group_{'_'.join([str(count) for count in self._store])}:" + ) + # print(f"pg_name_prefix = {pg_name_prefix}") + if device_name == "npu": + device_name = "NPU" + elif device_name == "cuda": + device_name = "GPU" + + bundle = {"CPU": self.max_colocate_count} + if self.use_gpu: + bundle[device_name] = 1 + if self.accelerator_type is not None: + bundle[self.accelerator_type] = 1e-4 + pg_scheme = [[bundle.copy() for _ in range(process_count)] for process_count in self._store] + + lifetime = "detached" if self.detached else None + + pgs = [ + placement_group(bundles=bundles, strategy=strategy, name=pg_name_prefix + str(idx), lifetime=lifetime) + for idx, bundles in enumerate(pg_scheme) + ] + + ray.get([pg.ready() for pg in pgs]) + + self.pgs = pgs + return pgs + + +def extract_pg_from_exist( + resource_pools: dict[str, RayResourcePool], src_role_names: list[str], resource_pool: RayResourcePool +) -> list: + src_pgs = [ + pg + for role_name, resource_pool in resource_pools.items() + for pg in resource_pool.get_placement_groups() + if role_name in src_role_names + ] + + sorted_src_pgs = sorted(src_pgs, key=lambda pg: pg.bundle_count, reverse=True) + sorted_process_on_nodes = sorted([(val, idx) for idx, val in enumerate(resource_pool.store)], reverse=True) + + unsorted_pgs: list[tuple[int, PlacementGroup]] = [] + searching_idx = 0 + for request_process, original_idx in sorted_process_on_nodes: + assert searching_idx < len(sorted_src_pgs), f"no enough nodes for request: searching {searching_idx} th node" + assert request_process <= sorted_src_pgs[searching_idx].bundle_count, ( + f"requesting {request_process} processes, bundle count cannot satisfy" + ) + unsorted_pgs.append((original_idx, sorted_src_pgs[searching_idx])) + searching_idx += 1 + + return [pg for _, pg in sorted(unsorted_pgs)] + + +def merge_resource_pool(rp1: RayResourcePool, rp2: RayResourcePool) -> RayResourcePool: + assert rp1.use_gpu == rp2.use_gpu, "Both RayResourcePool must either use_gpu or not" + assert rp1.max_colocate_count == rp2.max_colocate_count, "Both RayResourcePool must has the same max_colocate_count" + assert rp1.n_gpus_per_node == rp2.n_gpus_per_node, "Both RayResourcePool must has the same n_gpus_per_node" + assert rp1.detached == rp2.detached, "Detached ResourcePool cannot be merged with non-detached ResourcePool" + + new_store = rp1.store + rp2.store + + merged = type(rp1)(new_store, rp1.use_gpu, f"{rp1.name_prefix}_{rp2.name_prefix}") + merged.pgs = rp1.get_placement_groups() + rp2.get_placement_groups() + + return merged + + +class RayClassWithInitArgs(ClassWithInitArgs): + """A wrapper class for Ray actors with initialization arguments. + + This class extends ClassWithInitArgs to provide additional functionality for + configuring and creating Ray actors with specific resource requirements and + scheduling strategies. + """ + + def __init__(self, cls, *args, **kwargs) -> None: + # self._options = kwargs.pop('options', dict()) + super().__init__(cls, *args, **kwargs) + self._options = {} + self._additional_resource = {} + + def set_additional_resource(self, additional_resource): + """Set additional resource requirements for the actor. + + Args: + additional_resource: Dictionary specifying additional resource requirements + """ + self._additional_resource = additional_resource + + def update_options(self, options: dict): + """Update the Ray actor creation options. + + Args: + options: Dictionary of options to update + """ + self._options.update(options) + + def __call__( + self, + placement_group, + placement_group_bundle_idx, + use_gpu: bool = True, + num_gpus=1, + sharing_with=None, + device_name="cuda", + ) -> Any: + """Create and return a Ray actor with the configured options. + + Args: + placement_group: Ray placement group for scheduling + placement_group_bundle_idx: Index of the bundle in the placement group + use_gpu: Whether to use GPU resources + num_gpus: Number of GPUs to allocate + sharing_with: Actor to share resources with + device_name: Device for training + + Returns: + A Ray actor handle with the configured options + """ + if sharing_with is not None: + target_node_id = ray.get(sharing_with.get_node_id.remote()) + visible_devices = ray.get(sharing_with.get_cuda_visible_devices.remote()) + options = {"scheduling_strategy": NodeAffinitySchedulingStrategy(node_id=target_node_id, soft=False)} + return self.cls.options(**options).remote(*self.args, cuda_visible_devices=visible_devices, **self.kwargs) + + options = { + "scheduling_strategy": PlacementGroupSchedulingStrategy( + placement_group=placement_group, placement_group_bundle_index=placement_group_bundle_idx + ) + } + options.update(self._options) + + if use_gpu and device_name == "cuda": + options["num_gpus"] = num_gpus + if use_gpu and device_name == "npu": + options["resources"] = {"NPU": num_gpus} + + if len(self._additional_resource) > 1: + for k, v in self._additional_resource.items(): + options[k] = v + + # print("cls:", self.cls) + # print("args: ", self.args) + # print("kwargs: ", self.kwargs) + return self.cls.options(**options).remote(*self.args, **self.kwargs) + + +class RayWorkerGroup(WorkerGroup): + """A group of Ray workers that can be managed collectively. + + This class extends WorkerGroup to provide Ray-specific functionality for + creating and managing groups of Ray actors with specific resource requirements + and scheduling strategies. + """ + + def __init__( + self, + resource_pool: RayResourcePool = None, + ray_cls_with_init: RayClassWithInitArgs = None, + bin_pack: bool = True, + name_prefix: str = None, + detached=False, + worker_names=None, + worker_handles: list[ray.actor.ActorHandle] = None, + ray_wait_register_center_timeout: int = 300, + **kwargs, + ) -> None: + """Initialize a RayWorkerGroup. + + Args: + resource_pool: Resource pool for worker allocation + ray_cls_with_init: Class with initialization arguments for workers + bin_pack: Whether to use strict bin packing for resource allocation + name_prefix: Prefix for worker names + detached: Whether workers should be detached + worker_names: Names of existing workers to attach to + ray_wait_register_center_timeout: Timeout for waiting on register center + **kwargs: Additional keyword arguments + """ + super().__init__(resource_pool=resource_pool, **kwargs) + self.ray_cls_with_init = ray_cls_with_init + self.name_prefix = get_random_string(length=6) if name_prefix is None else name_prefix + self._ray_wait_register_center_timeout = ray_wait_register_center_timeout + # Whether the WorkerGroup is a Colocate WorkerGroup created by FusedWorker. + self.fused_worker_used = ray_cls_with_init.fused_worker_used + # if a WorkerGroup is spawned from Colocate WorkerGroup, this indicates which sub-class is binded to + # this WorkerGroup. + self.sub_cls_name = "" + self.device_name = kwargs.get("device_name", "cuda") + self.profile_steps = kwargs.get("profile_steps", None) + self.worker_nsight_options = kwargs.get("worker_nsight_options", None) + self.customized_worker_env = kwargs.get("worker_env", {}) + if self.worker_nsight_options is not None and self.worker_nsight_options["capture-range-end"] is None: + self.worker_nsight_options["capture-range-end"] = f"repeat-shutdown:{6 * len(self.profile_steps)}" + + if worker_names is not None and (not self.fused_worker_used): + assert self._is_init_with_detached_workers + self._worker_names = worker_names + + if self._is_init_with_detached_workers: + self._init_with_detached_workers(worker_names=worker_names, worker_handles=worker_handles) + else: + self._init_with_resource_pool( + resource_pool=resource_pool, + ray_cls_with_init=ray_cls_with_init, + bin_pack=bin_pack, + detached=detached, + worker_env=self.customized_worker_env, + ) + + if ray_cls_with_init is not None: + self._bind_worker_method(self.ray_cls_with_init.cls, func_generator) + + self.wg_dict = None + self.method_names = [] + + def _is_worker_alive(self, worker: ray.actor.ActorHandle): + """Check if a worker actor is still alive. + + Args: + worker: Ray actor handle to check + + Returns: + bool: True if the worker is alive, False otherwise + """ + worker_state_dict = get_actor(worker._actor_id.hex()) + return worker_state_dict.get("state", "undefined") == "ALIVE" if worker_state_dict is not None else False + + def _init_with_detached_workers(self, worker_names, worker_handles): + # ray.get_actor holds a weak reference to the actor, which causes actors garbage collected unexpectedly + # if we only hold spawn RayWorkerGroup. By passing actor handle explicitly, spawn RayWorkerGroup have + # strong reference to these actors. + # https://github.com/ray-project/ray/pull/45699 + workers = worker_handles if worker_handles else [ray.get_actor(name=name) for name in worker_names] + self._workers = workers + self._world_size = len(worker_names) + + def _get_master_addr_port(self, pg): + """Get master addr and port for this worker group""" + self._master_addr, self._master_port = ray.get( + get_master_addr_port.options( + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, placement_group_bundle_index=0 + ), + ).remote() + ) + + def _init_with_resource_pool(self, resource_pool, ray_cls_with_init, bin_pack, detached, worker_env=None): + """Initialize the worker group by creating new workers from a resource pool. + + Args: + resource_pool: Resource pool for worker allocation + ray_cls_with_init: Class with initialization arguments for workers + bin_pack: Whether to use strict bin packing for resource allocation + detached: Whether workers should be detached + """ + use_gpu = resource_pool.use_gpu + + strategy = "PACK" + if bin_pack: + strategy = "STRICT_PACK" + pgs = resource_pool.get_placement_groups(strategy=strategy, device_name=self.device_name) + world_size = resource_pool.world_size + self._world_size = world_size + # cia.add_kwarg("_world_size", world_size) + num_gpus = 1 / resource_pool.max_colocate_count + + rank = -1 + local_world_size = resource_pool.store[0] + for pg_idx, pg in enumerate(sort_placement_group_by_node_ip(pgs)): + assert local_world_size <= pg.bundle_count, f"when generating for {self.name_prefix}, for the " + if pg_idx == 0: + self._get_master_addr_port(pg) + + for local_rank in range(local_world_size): + rank += 1 + + # we pass in environment variable at option so that Worker can use environment variable to set + env_vars = { + "WORLD_SIZE": str(world_size), + "RANK": str(rank), + "WG_PREFIX": self.name_prefix, + "WG_BACKEND": "ray", + "RAY_LOCAL_WORLD_SIZE": str(local_world_size), + "MASTER_ADDR": self._master_addr, + "MASTER_PORT": self._master_port, + } + if worker_env is not None: + logging.debug(f"Appending ray class env, origin: {env_vars}, customized env: {worker_env}") + conflict_env_vars = set(env_vars.keys()) & set(worker_env.keys()) + if len(conflict_env_vars) > 0: + logging.error( + f"User customized env vars conflict with system env: {conflict_env_vars} " + f"Overriding may cause unexpected behavior." + ) + raise ValueError(f"Cannot override protected system env: {conflict_env_vars}") + env_vars.update(worker_env) + import re + + cia_name = type(ray_cls_with_init.cls).__name__ + match = re.search(r"ActorClass\(([^)]+)\)", cia_name) # ray.remote(Obj) -> "ActorClass(Obj)" + cia_name = match.group(1) if match else cia_name # "ActorClass(Obj)" -> "Obj" + name = f"{self.name_prefix}{cia_name}_{pg_idx}:{local_rank}" # e.g. Worker_2:5 + + if self.profile_steps and self.device_name == "cuda": + ray_cls_with_init.update_options( + { + "runtime_env": { + "env_vars": env_vars, + "nsight": self.worker_nsight_options, + }, + "name": name, + } + ) + else: + ray_cls_with_init.update_options({"runtime_env": {"env_vars": env_vars}, "name": name}) + + if detached: + ray_cls_with_init.update_options({"lifetime": "detached"}) + + # create a worker + worker = ray_cls_with_init( + placement_group=pg, + placement_group_bundle_idx=local_rank, + use_gpu=use_gpu, + num_gpus=num_gpus, + device_name=self.device_name, + ) + self._workers.append(worker) + self._worker_names.append(name) + + @property + def worker_names(self): + return self._worker_names + + @classmethod + def from_detached( + cls, + name_prefix=None, + worker_names=None, + worker_handles=None, + ray_cls_with_init=None, + **kwargs, + ): + """Create a worker group from existing detached workers. + + Args: + name_prefix: Prefix for worker names + worker_names: Names of existing workers to attach to + ray_cls_with_init: Class with initialization arguments for workers + + Returns: + A new RayWorkerGroup instance + """ + worker_group = cls( + resource_pool=None, + ray_cls_with_init=ray_cls_with_init, + name_prefix=name_prefix, + worker_names=worker_names, + worker_handles=worker_handles, + **kwargs, + ) + return worker_group + + def spawn(self, prefix_set): + """Spawn to a dictionary of worker groups, each with a subset of method with prefix. + + Args: + prefix_set: Set of prefixes to create worker groups for + + Returns: + Dictionary of worker groups keyed by prefix + """ + if self.fused_worker_used: + return self.spawn_fused(prefix_set) + + def _rebind_actor_methods(worker_group, actor_name): + prefix: str = actor_name + "_" + for method_name in dir(worker_group): + if method_name.startswith(prefix): + original_method_name = method_name.removeprefix(prefix) + method = getattr(worker_group, method_name) + setattr(worker_group, original_method_name, method) + + new_worker_group_dict = {} + for prefix in prefix_set: + new_worker_group = self.from_detached( + name_prefix=self.name_prefix, + worker_names=self._worker_names, + worker_handles=self._workers, + ray_cls_with_init=self.ray_cls_with_init, + profile_steps=self.profile_steps, + worker_nsight_options=self.worker_nsight_options, + ) + + _rebind_actor_methods(new_worker_group, prefix) + new_worker_group_dict[prefix] = new_worker_group + return new_worker_group_dict + + def spawn_fused(self, prefix_set): + """Create a dictionary of worker groups for fused workers. + + Args: + prefix_set: Set of prefixes to create worker groups for + + Returns: + Dictionary of worker groups keyed by prefix + """ + wg_dict = dict() + for key in prefix_set: + new_wg = deepcopy(self) + new_wg._bind_worker_method(self.ray_cls_with_init.cls.raw_cls_dict[key], func_generator) + new_wg.sub_cls_name = key + wg_dict[key] = new_wg + return wg_dict + + def fuse(self, prefix_set): + """Fuse multiple worker groups into the current worker group. + + Args: + prefix_set: Set of prefixes to fuse into the worker group + """ + if self.wg_dict is None: + self.wg_dict = self.spawn(prefix_set) + for role_name, role_wg in self.wg_dict.items(): + setattr(self, role_name, role_wg) + self.method_names = self._bind_worker_method(self.ray_cls_with_init.cls, func_generator) + + def _execute_remote_single_worker(self, worker, method_name: str, *args, **kwargs): + """Execute a method on a single worker remotely. + + Args: + worker: The worker actor handle + method_name: Name of the method to execute + *args: Positional arguments for the method + **kwargs: Keyword arguments for the method + + Returns: + Remote object reference to the method execution + """ + if self.fused_worker_used and method_name not in self.method_names: + remote_call = getattr(worker, self.fused_worker_execute_fn_name) + return remote_call.remote(f"{self.sub_cls_name}_fwmn_{method_name}", *args, **kwargs) + # fused worker not used + remote_call = getattr(worker, method_name) + return remote_call.remote(*args, **kwargs) + + def execute_rank_zero_sync(self, method_name: str, *args, **kwargs): + """Execute a method on rank zero worker synchronously. + + Args: + method_name: Name of the method to execute + *args: Positional arguments for the method + **kwargs: Keyword arguments for the method + + Returns: + Result of the method execution + """ + return ray.get(self.execute_rank_zero_async(method_name, *args, **kwargs)) + + def execute_rank_zero_async(self, method_name: str, *args, **kwargs): + """Execute a method on rank zero worker asynchronously. + + Args: + method_name: Name of the method to execute + *args: Positional arguments for the method + **kwargs: Keyword arguments for the method + + Returns: + Remote object reference to the method execution + """ + return self._execute_remote_single_worker(self._workers[0], method_name, *args, **kwargs) + + def execute_rank_zero(self, method_name: str, *args, **kwargs): + """Alias for execute_rank_zero_async. + + Args: + method_name: Name of the method to execute + *args: Positional arguments for the method + **kwargs: Keyword arguments for the method + + Returns: + Remote object reference to the method execution + """ + return self.execute_rank_zero_async(method_name, *args, **kwargs) + + def execute_all(self, method_name: str, *args, **kwargs): + """Alias for execute_all_async. + + Args: + method_name: Name of the method to execute + *args: Positional arguments for the method + **kwargs: Keyword arguments for the method + + Returns: + List of remote object references to the method executions + """ + return self.execute_all_async(method_name, *args, **kwargs) + + def execute_all_sync(self, method_name: str, *args, **kwargs): + """Execute a method on all workers synchronously. + + Args: + method_name: Name of the method to execute + *args: Positional arguments for the method + **kwargs: Keyword arguments for the method + + Returns: + List of results from all workers + """ + return ray.get(self.execute_all_async(method_name, *args, **kwargs)) + + def execute_all_async(self, method_name: str, *args, **kwargs): + """Execute a method on all workers asynchronously. + + Args: + method_name: Name of the method to execute + *args: Positional arguments for the method + **kwargs: Keyword arguments for the method + + Returns: + List of remote object references to the method executions + """ + # Here, we assume that if all arguments in args and kwargs are lists, + # and their lengths match len(self._workers), we'll distribute each + # element in these lists to the corresponding worker + # print(f"execute_all_async: method {method_name}({args}, {kwargs})") + length = len(self._workers) + if all(isinstance(arg, list) for arg in args) and all(isinstance(kwarg, list) for kwarg in kwargs.values()): + if all(len(arg) == length for arg in args) and all(len(kwarg) == length for kwarg in kwargs.values()): + # print(f"splitting args and kwargs into {length} shards") + result = [] + for i in range(length): + sliced_args = tuple(arg[i] for arg in args) + sliced_kwargs = {k: v[i] for k, v in kwargs.items()} + result.append( + self._execute_remote_single_worker(self._workers[i], method_name, *sliced_args, **sliced_kwargs) + ) + return result + + return [self._execute_remote_single_worker(worker, method_name, *args, **kwargs) for worker in self._workers] + + @property + def master_address(self): + return self._master_addr + + @property + def master_port(self): + return self._master_port + + @property + def workers(self): + return self._workers + + @property + def world_size(self): + return self._world_size + + +""" +Utilities that enables creating workers inside the same ray.Actor, +with code written in separate ray.Actors. +""" + + +# deprecated, switching to FusedWorker +def _bind_workers_method_to_parent(cls, key, user_defined_cls): + """ + Binds the methods of each worker to the WorkerDict. + Note that we only bind public methods that are decorated by register + """ + + for method_name in dir(user_defined_cls): + try: + method = getattr(user_defined_cls, method_name) + assert callable(method), f"{method_name} in {user_defined_cls} is not callable" + except Exception: + # if it is a property, it will fail because Class doesn't have instance property + continue + + if hasattr(method, MAGIC_ATTR): + + def generate_function(name, key=key): + def func(self, *args, **kwargs): + # dispatch to the actual worker + return getattr(self.worker_dict[key], name)(*args, **kwargs) + + async def async_func(self, *args, **kwargs): + # dispatch to the actual worker + return await getattr(self.worker_dict[key], name)(*args, **kwargs) + + wrapper = async_func if inspect.iscoroutinefunction(method) else func # noqa: B023 + + return wrapper + + func = generate_function(method_name) + # pass MAGIC_ATTR for outer worker group + attrs = getattr(method, MAGIC_ATTR) + setattr(func, MAGIC_ATTR, attrs) + try: + # bind direct rollout method to class without prefix + if attrs["dispatch_mode"] == Dispatch.DIRECT_ROLLOUT_METHOD and "rollout" in key: + assert not hasattr(cls, method_name), ( + f"conflict direct rollout method {method_name} with role {key}" + ) + setattr(cls, method_name, func) + print(f"bind role {key} method {method_name} to class {cls}") + else: + method_name_with_prefix = key + "_" + method_name + setattr(cls, method_name_with_prefix, func) + except Exception as e: + raise ValueError(f"Fail to set method_name {method_name}") from e + + +def _unwrap_ray_remote(cls): + if hasattr(cls, "__ray_actor_class__"): + cls = cls.__ray_actor_class__ + return cls + + +def _determine_fsdp_megatron_base_class(mros: list): + """ + - megatron: base class should be MegatronWorker + - fsdp: base class should be Worker + """ + for cls in mros[0]: + if cls.__name__ == "MegatronWorker": + return cls + if cls.__name__ == "Worker": + return cls + raise ValueError(f"Cannot determine base class for {mros}") + + +# deprecated, switching to FusedWorker +def create_colocated_worker_cls(class_dict: dict[str, RayClassWithInitArgs]): + """ + This function should return a class instance that delegates the calls to every + cls in cls_dict + """ + cls_dict = {} + init_args_dict = {} + worker_cls = _determine_fsdp_megatron_base_class( + [cls.cls.__ray_actor_class__.__mro__ for cls in class_dict.values()] + ) + assert issubclass(worker_cls, Worker), f"worker_cls {worker_cls} should be a subclass of Worker" + print(f"colocated worker base class {worker_cls}") + + for key, cls in class_dict.items(): + cls_dict[key] = cls.cls + init_args_dict[key] = {"args": cls.args, "kwargs": cls.kwargs} + + assert cls_dict.keys() == init_args_dict.keys() + + # TODO: create a class with customizable name + class WorkerDict(worker_cls): + def __init__(self): + super().__init__() + self.worker_dict = {} + for key, user_defined_cls in cls_dict.items(): + user_defined_cls = _unwrap_ray_remote(user_defined_cls) + # directly instantiate the class without remote + # in worker class, e.g. + # when DISABLE_WORKER_INIT == 1 it will return immediately + with temp_env_var("DISABLE_WORKER_INIT", "1"): + self.worker_dict[key] = user_defined_cls( + *init_args_dict[key].get("args", ()), **init_args_dict[key].get("kwargs", {}) + ) + + # now monkey-patch the methods from inner class to WorkerDict + for key, user_defined_cls in cls_dict.items(): + user_defined_cls = _unwrap_ray_remote(user_defined_cls) + _bind_workers_method_to_parent(WorkerDict, key, user_defined_cls) + + remote_cls = ray.remote(WorkerDict) + remote_cls = RayClassWithInitArgs(cls=remote_cls) + return remote_cls + + +FusedWorkerCLSName = "FusedWorker" + + +def create_colocated_worker_raw_cls(class_dict: dict[str, RayClassWithInitArgs]): + """ + This function returns a FusedWorker class. + + `FusedWorker.{class_name}` -> FusedClass + Use `class_name` as a param to directly access the underlying class. + + `FusedWorker._fuw_execute("{class_name}_fwmn_{method_name}", *args, **kwargs)` + First param must be "{class_name}_fwmn_{method_name}" in order to access `method_name` + of underlying class `{class_name}`. + + `FusedWorker.fused_worker_dict` -> {"class_name": FusedClass} + Stores all underlying classes. + + `FusedClass.fused_worker_dict` -> {"class_name": FusedClass} + The same as `FusedWorker.fused_worker_dict`, enables underlying class to access other + underlying classes. + """ + raw_cls_dict = {cls_name: _unwrap_ray_remote(cia.cls) for cls_name, cia in class_dict.items()} + init_args_dict = {cls_name: cia.args for cls_name, cia in class_dict.items()} + init_kwargs_dict = {cls_name: cia.kwargs for cls_name, cia in class_dict.items()} + cls_names = list(class_dict.keys()) + + # FusedWorker_Actor_Critic + class_name_renamed = "_".join([FusedWorkerCLSName] + cls_names) + + class FusedWorker(Worker): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cls_names = cls_names + self.raw_cls_dict = raw_cls_dict + self.init_args_dict = init_args_dict + self.init_kwargs_dict = init_kwargs_dict + + for cls_name, udc, ud_args, ud_kwargs in zip( + self.cls_names, + self.raw_cls_dict.values(), + self.init_args_dict.values(), + self.init_kwargs_dict.values(), + strict=True, + ): + with temp_env_var("DISABLE_WORKER_INIT", "1"): + udc._get_ray_actor_cls_name = lambda x, name_renamed=class_name_renamed: name_renamed + udc._get_ray_method_prefix = lambda x, name_prefixed=cls_name: f"{name_prefixed}_" + # cls_name = "actor", "critic", udc = ActorWorker, CriticWorker + self.fused_worker_dict[cls_name] = udc(*ud_args, **ud_kwargs) + setattr(self, cls_name, self.fused_worker_dict[cls_name]) + + # injecting fused_worker to each sub worker so they can be aware of existence of each other + for _, worker in self.fused_worker_dict.items(): + setattr(worker, Worker.fused_worker_attr_name, self.fused_worker_dict) + + def _fuw_execute(self, method_name: str, *args, **kwargs): + # for fused_worker, method_name is in a form of "{cls_name}_fwmn_{method_name}" + # where fwmn stands "fused worker method name" + names = method_name.split("_fwmn_") + cls_name = names[0] + method_name = names[1] + + assert cls_name in self.fused_worker_dict, ( + f"calling {cls_name}'s {method_name}, but {cls_name} not in fused_worker_dict" + ) + udc_method = getattr(self.fused_worker_dict[cls_name], method_name) + return udc_method(*args, **kwargs) + + renamed_fused_worker_cls = type(class_name_renamed, (FusedWorker,), {}) + renamed_fused_worker_cls.is_fused_worker = True + renamed_fused_worker_cls.raw_cls_dict = raw_cls_dict + + return renamed_fused_worker_cls + + +def create_colocated_worker_cls_fused(class_dict: dict[str, RayClassWithInitArgs]): + """ + This function returns a RayClassWithInitArgs instance of FusedWorker, which is an replacement + of `create_colocated_worker_cls`. WorkerGroup constructed using this class will be a colocated + WorkerGroup, which will be referenced as `ColocateWorkerGroup` below. + + `ColocateWorkerGroup.spawn(prefix_set)` + returns a dict of WorkerGroup {"class_name": WorkerGroup}, WorkerGroup in this dict will + have methods of underlying class `class_name` attached. + + `ColocateWorkerGroup.fuse(prefix_set)` + After executing this function, `ColocateWorkerGroup.{class_name}` will return WorkerGroup + with methods of underlying class `class_name` attached. + """ + raw_colocated_worker_cls = create_colocated_worker_raw_cls(class_dict) + + remote_cls = ray.remote(raw_colocated_worker_cls) + cia = RayClassWithInitArgs(cls=remote_cls) + cia.fused_worker_used = True + + return cia diff --git a/e2c/verl/verl/third_party/__init__.py b/e2c/verl/verl/third_party/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/third_party/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/third_party/sglang/__init__.py b/e2c/verl/verl/third_party/sglang/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..15593caaf36bebb7bdaa0ddf3d9364dd60111929 --- /dev/null +++ b/e2c/verl/verl/third_party/sglang/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/third_party/sglang/parallel_state.py b/e2c/verl/verl/third_party/sglang/parallel_state.py new file mode 100644 index 0000000000000000000000000000000000000000..cdec743d13f5926e6db45c04bc5ed035bed0eb90 --- /dev/null +++ b/e2c/verl/verl/third_party/sglang/parallel_state.py @@ -0,0 +1,328 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023 The SGlang team. +# Adapted from +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +"""Model and data parallel groups.""" + +import os +from typing import Optional + +import sglang.srt.distributed.parallel_state as ps +import torch +import torch.distributed +from sglang.srt.distributed.parallel_state import ( + get_pp_group, + get_world_group, + init_distributed_environment, + init_model_parallel_group, +) + +""" +This version is strongly tied with Megatron to implement HybridEngine and weight sharing between vllm and Megatron. +- We assume the Megatron tp+dp+pp world is already established before calling this function. + +""" + +# Device mesh for using DTensor +_DEVICE_MESH = None + +# Tensor model parallel group that the current rank belongs to. +_TP = None +# Pipeline model parallel group that the current rank belongs to. +_PP = None + + +# This method is for initializing the ParallelGroup when using HybridEngine +# NOTE(linjunrong): this function is for megatron +def initialize_parallel_state( + distributed_init_method: str = "env://", + backend: str = "nccl", + tensor_model_parallel_size: int = 1, + num_tp_per_train_tp: int = 1, + pipeline_model_parallel_size: int = 1, +): + # torch.distributed.all_reduce does not free the input tensor until + # the synchronization point. This causes the memory usage to grow + # as the number of all_reduce calls increases. This env var disables + # this behavior. + # Related issue: + # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 + os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" + + # NOTE(sgm): Modify for verl, Env vars will be set by TORCHRUN. + rank = int(os.getenv("RANK", "-1")) + local_rank = int(os.getenv("LOCAL_RANK", "0")) + + # Use the world_size set by TORCHRUN + world_size = int(os.getenv("WORLD_SIZE", "-1")) + assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" + init_distributed_environment(world_size, rank, distributed_init_method, local_rank, backend) + if torch.distributed.get_world_size() > 1: + # NOTE: build a separate inference group with infer tp & micro dp + initialize_model_parallel_for_sglang( + tensor_model_parallel_size=tensor_model_parallel_size, + num_tensor_model_parallel_groups_per_train_tp=num_tp_per_train_tp, + ) + else: + initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) + + +# NOTE(linjunrong): After init SGLang rollout using class EngineFragment, user should always remember to call +# this function to sync the _TP, _PP define at the beginning of this file. Otherwise, only the conterparts +# inside sglang.srt.distributed are init as ProcessGroup, the symbols defined in this file remain as None. +# It could be weird to maintain two _TP and _PP, I follow the same way to maintain an extra ones for +# verl itself as how it was done in verl.third_party.vllm.parallel_state. Note that the process is a little +# bit different +def ensure_model_parallel_initialized( + tensor_model_parallel_size: int, + pipeline_model_parallel_size: int = 1, + backend: Optional[str] = None, +) -> None: + """Helper to initialize model parallel groups if they are not initialized, + or ensure tensor-parallel and pipeline-parallel sizes are equal to expected + values if the model parallel groups are initialized. + """ + # get the backend of _DEVICE_WORLD_GROUP + backend = backend or torch.distributed.get_backend(get_world_group().device_group) + if not model_parallel_is_initialized(): + initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) + return + + assert get_tensor_model_parallel_world_size() == tensor_model_parallel_size, ( + f"tensor parallel group already initialized, but of unexpected size: " + f"{get_tensor_model_parallel_world_size()=} vs. {tensor_model_parallel_size=}" + ) + pp_world_size = get_pp_group().world_size + assert pp_world_size == pipeline_model_parallel_size, ( + f"pipeline parallel group already initialized, but of unexpected size: {pp_world_size=} vs. " + f"{pipeline_model_parallel_size=}" + ) + + +# TODO(sgm): deviate from the v0.5.4, not pp now +# NOTE(linjunrong): the SGLang version using _TP instead of ps._TP +def model_parallel_is_initialized(): + """Check if tensor and pipeline parallel groups are initialized.""" + return _TP is not None + # and _PIPELINE_MODEL_PARALLEL_GROUP is not None) + + +def initialize_model_parallel_for_sglang( + tensor_model_parallel_size: int, + num_tensor_model_parallel_groups_per_train_tp: int = 1, + pipeline_model_parallel_size: int = 1, +) -> None: + pass + + # Get world size and rank. Ensure some consistencies. + assert torch.distributed.is_initialized() + + assert isinstance(tensor_model_parallel_size, int) + + # assert num_tensor_model_parallel_groups_per_train_tp == 1 and not different_tp_group + # assert num_tensor_model_parallel_groups_per_train_tp > 1 and different_tp_group + + # Build the tensor model-parallel groups. + assert ps._TP is None, "tensor model parallel group is already initialized" + + global _TP + + world_size: int = torch.distributed.get_world_size() + + backend = torch.distributed.get_backend() + + num_tensor_model_parallel_groups = world_size // tensor_model_parallel_size + + if num_tensor_model_parallel_groups_per_train_tp == 1: + # if tensor_model_parallel_size == train_tensor_parallel_size: + # using the same tp group as Megatron/vllm + assert _TP is None, "tensor model parallel group is already initialized" + group_ranks = [] + for i in range(num_tensor_model_parallel_groups): + ranks = range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) + group_ranks.append(ranks) + _TP = init_model_parallel_group( + group_ranks=group_ranks, + local_rank=get_world_group().local_rank, + backend=backend, + use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer + use_message_queue_broadcaster=True, + ) + ps._TP = _TP + # _MICRO_DATA_PARALLEL_GROUP is move to hybrid engine + else: + # initialize a micro_dp group and a tp group + # assume training tp=4, infer tp=2, then, weight is partitioned as + # [1], [2], [3], [4] for training and [1,2], [1,2], [3,4], [3,4] for inference + + # Build the inference tp groups + # train_tp = train_tensor_parallel_size + train_tp = num_tensor_model_parallel_groups_per_train_tp * tensor_model_parallel_size + # num_tensor_model_parallel_groups_per_train_tp = train_tp // tensor_model_parallel_size + assert _TP is None, "tensor model parallel group is already initialized" + group_ranks = [] + for i in range(num_tensor_model_parallel_groups // num_tensor_model_parallel_groups_per_train_tp): + start = train_tp * i + end = train_tp * (i + 1) + for j in range(num_tensor_model_parallel_groups_per_train_tp): + ranks = list(range(start, end, num_tensor_model_parallel_groups_per_train_tp)) + for i in range(len(ranks)): + ranks[i] += j + group_ranks.append(ranks) + _TP = init_model_parallel_group( + group_ranks=group_ranks, + local_rank=get_world_group().local_rank, + backend=backend, + use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer + use_message_queue_broadcaster=True, + ) + ps._TP = _TP + + # Build the pipeline model-parallel groups. + # global _PIPELINE_MODEL_PARALLEL_GROUP + # global _PIPELINE_GLOBAL_RANKS + # assert ps._PIPELINE_MODEL_PARALLEL_GROUP is None, ("pipeline model parallel group is already initialized") + + # ps._PIPELINE_MODEL_PARALLEL_GROUP = mpu.get_pipeline_model_parallel_group() + # ps._PIPELINE_GLOBAL_RANKS = mpu.get_pipeline_model_parallel_ranks() + + # TODO: init using device mesh (not support hybrid engine now) + # Build the pipeline model-parallel groups. + num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size + global _PP + assert _PP is None, "pipeline model parallel group is already initialized" + group_ranks = [] + for i in range(num_pipeline_model_parallel_groups): + ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) + group_ranks.append(ranks) + # pipeline parallel does not need custom allreduce + _PP = init_model_parallel_group(group_ranks, get_world_group().local_rank, backend, use_custom_allreduce=False) + ps._PP = _PP # for verl + + +def initialize_model_parallel( + tensor_model_parallel_size: int = 1, + pipeline_model_parallel_size: int = 1, + backend: Optional[str] = None, +) -> None: + """ + NOTE: This method is a hack from the open-sourced version without + asertion of world_size = tp * pp + + Initialize model parallel groups. + + Arguments: + tensor_model_parallel_size: number of GPUs used for tensor model + parallelism. + pipeline_model_parallel_size: number of GPUs used for pipeline model + parallelism. + + Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we + use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize + the model pipeline. The present function will + create 4 tensor model-parallel groups and 2 pipeline model-parallel groups: + 4 tensor model-parallel groups: + [g0, g1], [g2, g3], [g4, g5], [g6, g7] + 2 pipeline model-parallel groups: + [g0, g2, g4, g6], [g1, g3, g5, g7] + Note that for efficiency, the caller should make sure adjacent ranks + are on the same DGX box. For example if we are using 2 DGX-1 boxes + with a total of 16 GPUs, rank 0 to 7 belong to the first box and + ranks 8 to 15 belong to the second box. + """ + # Get world size and rank. Ensure some consistencies. + assert torch.distributed.is_initialized() + world_size: int = torch.distributed.get_world_size() + backend = backend or torch.distributed.get_backend(ps.get_world_group().device_group) + + # NOTE(sgm) we don't assert world_size == tp * pp + # DP is not managed by vllm but by the VeRL WorkerGroup + # if (world_size != + # tensor_model_parallel_size * pipeline_model_parallel_size): + # raise RuntimeError( + # f"world_size ({world_size}) is not equal to " + # f"tensor_model_parallel_size ({tensor_model_parallel_size}) x " + # f"pipeline_model_parallel_size ({pipeline_model_parallel_size})") + + num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size + + global _TP + assert _TP is None, "tensor model parallel group is already initialized" + group_ranks = [] + for i in range(num_tensor_model_parallel_groups): + ranks = list(range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size)) + group_ranks.append(ranks) + + # message queue broadcaster is only used in tensor model parallel group + if ps._TP is not None: + _TP = ps._TP + else: + _TP = init_model_parallel_group( + group_ranks, + get_world_group().local_rank, + backend, + use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer + use_message_queue_broadcaster=True, + ) + ps._TP = _TP + + # TODO: init using device mesh (not support hybrid engine now) + # Build the pipeline model-parallel groups. + num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size + global _PP + assert _PP is None, "pipeline model parallel group is already initialized" + group_ranks = [] + for i in range(num_pipeline_model_parallel_groups): + ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) + group_ranks.append(ranks) + # pipeline parallel does not need custom allreduce + if ps._TP is not None: + _PP = ps._TP + else: + _PP = init_model_parallel_group(group_ranks, get_world_group().local_rank, backend, use_custom_allreduce=False) + ps._PP = _PP + + +""" +Device mesh utilities +""" + + +def get_device_mesh(): + assert _DEVICE_MESH is not None, "device mesh is not initialized" + return _DEVICE_MESH + + +""" +Tensor model parallel utilities +""" + + +# NOTE(linjunrong): In the vllm version parallel_state.py. verl created its own _TP and _PP as verl want to use +# the process group for some extra purpose. Under the hood, there is no difference between them and the original +# one in vllm.distributed.parallel_state. However, the implementation need to hack the init process of inference +# engine, as we do not maintain another SGLang here, I just use the original _TP and _PP directly. +def get_tensor_model_parallel_group(): + """Get the tensor model parallel group the caller rank belongs to.""" + + assert _TP is not None, "tensor model parallel group is not initialized" + return _TP.device_group + + +def get_tensor_model_parallel_world_size(): + """Return world size for the tensor model parallel group.""" + return torch.distributed.get_world_size(group=get_tensor_model_parallel_group()) + + +def get_tensor_model_parallel_rank(): + """Return my rank for the tensor model parallel group.""" + return torch.distributed.get_rank(group=get_tensor_model_parallel_group()) + + +def get_tensor_model_parallel_src_rank(): + """Calculate the global rank corresponding to the first local rank + in the tensor model parallel group.""" + global_rank = torch.distributed.get_rank() + local_world_size = get_tensor_model_parallel_world_size() + return (global_rank // local_world_size) * local_world_size diff --git a/e2c/verl/verl/third_party/torch/__init__.py b/e2c/verl/verl/third_party/torch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7664279b7411a806f615b52b2405fd2c40672517 --- /dev/null +++ b/e2c/verl/verl/third_party/torch/__init__.py @@ -0,0 +1,87 @@ +# official torch 2.6.0 set_model_state_dict API leads to OOM +# this is a copy of torch/distributed/checkpoint from torch 2.7.0 + +# From PyTorch: + +# Copyright (c) 2016- Facebook, Inc (Adam Paszke) +# Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +# Copyright (c) 2011-2013 NYU (Clement Farabet) +# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +# Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +# From Caffe2: + +# Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +# All contributions by Facebook: +# Copyright (c) 2016 Facebook Inc. + +# All contributions by Google: +# Copyright (c) 2015 Google Inc. +# All rights reserved. + +# All contributions by Yangqing Jia: +# Copyright (c) 2015 Yangqing Jia +# All rights reserved. + +# All contributions by Kakao Brain: +# Copyright 2019-2020 Kakao Brain + +# All contributions by Cruise LLC: +# Copyright (c) 2022 Cruise LLC. +# All rights reserved. + +# All contributions by Tri Dao: +# Copyright (c) 2024 Tri Dao. +# All rights reserved. + +# All contributions by Arm: +# Copyright (c) 2021, 2023-2024 Arm Limited and/or its affiliates + +# All contributions from Caffe: +# Copyright(c) 2013, 2014, 2015, the respective contributors +# All rights reserved. + +# All other contributions: +# Copyright(c) 2015, 2016 the respective contributors +# All rights reserved. + +# Caffe2 uses a copyright model similar to Caffe: each contributor holds +# copyright over their contributions to Caffe2. The project versioning records +# all such contribution and copyright details. If a contributor wants to further +# mark their specific copyright on a particular contribution, they should +# indicate their copyright solely in the commit message of the change when it is +# committed. + +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America +# and IDIAP Research Institute nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. diff --git a/e2c/verl/verl/third_party/torch/distributed/__init__.py b/e2c/verl/verl/third_party/torch/distributed/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7664279b7411a806f615b52b2405fd2c40672517 --- /dev/null +++ b/e2c/verl/verl/third_party/torch/distributed/__init__.py @@ -0,0 +1,87 @@ +# official torch 2.6.0 set_model_state_dict API leads to OOM +# this is a copy of torch/distributed/checkpoint from torch 2.7.0 + +# From PyTorch: + +# Copyright (c) 2016- Facebook, Inc (Adam Paszke) +# Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +# Copyright (c) 2011-2013 NYU (Clement Farabet) +# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +# Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +# From Caffe2: + +# Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +# All contributions by Facebook: +# Copyright (c) 2016 Facebook Inc. + +# All contributions by Google: +# Copyright (c) 2015 Google Inc. +# All rights reserved. + +# All contributions by Yangqing Jia: +# Copyright (c) 2015 Yangqing Jia +# All rights reserved. + +# All contributions by Kakao Brain: +# Copyright 2019-2020 Kakao Brain + +# All contributions by Cruise LLC: +# Copyright (c) 2022 Cruise LLC. +# All rights reserved. + +# All contributions by Tri Dao: +# Copyright (c) 2024 Tri Dao. +# All rights reserved. + +# All contributions by Arm: +# Copyright (c) 2021, 2023-2024 Arm Limited and/or its affiliates + +# All contributions from Caffe: +# Copyright(c) 2013, 2014, 2015, the respective contributors +# All rights reserved. + +# All other contributions: +# Copyright(c) 2015, 2016 the respective contributors +# All rights reserved. + +# Caffe2 uses a copyright model similar to Caffe: each contributor holds +# copyright over their contributions to Caffe2. The project versioning records +# all such contribution and copyright details. If a contributor wants to further +# mark their specific copyright on a particular contribution, they should +# indicate their copyright solely in the commit message of the change when it is +# committed. + +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America +# and IDIAP Research Institute nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. diff --git a/e2c/verl/verl/third_party/torch/distributed/_state_dict_utils.py b/e2c/verl/verl/third_party/torch/distributed/_state_dict_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d308449f7104e0c42afd48e38ed1696d2bf3072f --- /dev/null +++ b/e2c/verl/verl/third_party/torch/distributed/_state_dict_utils.py @@ -0,0 +1,840 @@ +# official torch 2.6.0 set_model_state_dict API leads to OOM +# this is a copy of torch/distributed/checkpoint from torch 2.7.0 + +# From PyTorch: + +# Copyright (c) 2016- Facebook, Inc (Adam Paszke) +# Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +# Copyright (c) 2011-2013 NYU (Clement Farabet) +# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +# Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +# From Caffe2: + +# Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +# All contributions by Facebook: +# Copyright (c) 2016 Facebook Inc. + +# All contributions by Google: +# Copyright (c) 2015 Google Inc. +# All rights reserved. + +# All contributions by Yangqing Jia: +# Copyright (c) 2015 Yangqing Jia +# All rights reserved. + +# All contributions by Kakao Brain: +# Copyright 2019-2020 Kakao Brain + +# All contributions by Cruise LLC: +# Copyright (c) 2022 Cruise LLC. +# All rights reserved. + +# All contributions by Tri Dao: +# Copyright (c) 2024 Tri Dao. +# All rights reserved. + +# All contributions by Arm: +# Copyright (c) 2021, 2023-2024 Arm Limited and/or its affiliates + +# All contributions from Caffe: +# Copyright(c) 2013, 2014, 2015, the respective contributors +# All rights reserved. + +# All other contributions: +# Copyright(c) 2015, 2016 the respective contributors +# All rights reserved. + +# Caffe2 uses a copyright model similar to Caffe: each contributor holds +# copyright over their contributions to Caffe2. The project versioning records +# all such contribution and copyright details. If a contributor wants to further +# mark their specific copyright on a particular contribution, they should +# indicate their copyright solely in the commit message of the change when it is +# committed. + +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America +# and IDIAP Research Institute nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + + +# ruff: noqa: B028, UP038, UP007, E721, E501 +# mypy: allow-untyped-defs +import copy +import io +import math +import weakref +from collections.abc import Mapping, MutableMapping +from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union, cast + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.distributed._functional_collectives import AsyncCollectiveTensor + +if dist.is_available() or TYPE_CHECKING: + from torch.distributed import distributed_c10d + from torch.distributed._shard.sharded_tensor import ShardedTensor + from torch.distributed.tensor import DTensor, Replicate, distribute_tensor + from torch.distributed.tensor._utils import compute_local_shape_and_global_offset + + +def _identity_func( + obj: torch.Tensor, + pg: Optional[dist.ProcessGroup], + device: Optional[torch.device], + companion_obj: Any, +) -> torch.Tensor: + return obj + + +def _all_gather_sharded_tensor( + sharded_tensor: "ShardedTensor", + pg: Optional[dist.ProcessGroup] = None, + device: Optional[torch.device] = None, +) -> torch.Tensor: + if pg is None: + pg = distributed_c10d._get_default_group() + world_size = dist.get_world_size(pg) + shards = sharded_tensor.local_shards() + dim_0_size = sharded_tensor.size()[0] # type: ignore[index] + tensor_numel = sharded_tensor.size().numel() # type: ignore[union-attr] + chunk_size = math.ceil(dim_0_size / world_size) * tensor_numel // dim_0_size + pg_device = distributed_c10d._get_pg_default_device(pg) if device is None else device + if shards: + local_tensor = shards[0].tensor.flatten() + if local_tensor.device.type != pg_device.type: + local_tensor = local_tensor.to(pg_device) + num_padding = chunk_size - local_tensor.numel() + if num_padding > 0: + local_tensor = F.pad(local_tensor, [0, num_padding]) + else: + local_tensor = torch.zeros(chunk_size, dtype=sharded_tensor.dtype, device=pg_device) + + tensor = torch.empty( + chunk_size * world_size, + dtype=local_tensor.dtype, + device=pg_device, + ) + dist.all_gather_into_tensor(tensor, local_tensor, group=pg) + + tensor = tensor.narrow(0, 0, tensor_numel).reshape(sharded_tensor.size()) + return tensor + + +class CompanionMismatch(Exception): + pass + + +def _iterate_state_dict( + iter_object: Any, + sharded_tensor_func: Callable, + dtensor_func: Callable, + tensor_func: Callable, + *, + pg: Optional[dist.ProcessGroup] = None, + device: Optional[torch.device] = None, + cpu_offload: bool = False, + companion_obj: Any = None, + ranks_only: tuple[int, ...] = (), + type_check: bool = True, + non_blocking: bool = True, +) -> dict[str, Any]: + """Iterate through the state dict, applying the given functions to each tensor type. + + Args: + iter_object (Any): the target state_dict. + sharded_tensor_func (Callable): the function to apply to ShardedTensor + dtensor_func (Callable): the function to apply to DTensor + tensor_func (Callable): the function to apply to Tensor + pg (Optional[dist.ProcessGroup]): process group passed to tensor functions + device (Optional[torch.device]): device passed to tensor functions + cpu_offload (bool): whether to offload the tensors to CPU memory. This option is ignored + if a companion_obj is supplied. + companion_obj (Any): A companion object to the state dict. If this object + is supplied, we attempt to copy the tensor to the companion object. + ranks_only (Tuple[int, ...]): if this tuple is empty, all ranks will + have the same state_dicts. Otherwise only ranks that in ``ranks_only`` + have the same state_dicts. Other ranks will get empty state_dicts. + type_check (bool): check if the instance data type is a supported type + that can be saved by DCP. The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + non_blocking (bool): whether to use non-blocking copy when copying to the companion object. + """ + # TODO: should we use pytree? + cpu_device = torch.device("cpu") + if isinstance(iter_object, ShardedTensor): + ret = sharded_tensor_func(iter_object, pg, device, companion_obj) + elif isinstance(iter_object, DTensor): + ret = dtensor_func(iter_object, pg, device, companion_obj) + elif isinstance(iter_object, torch.Tensor): + ret = tensor_func(iter_object, pg, device, companion_obj) + elif isinstance(iter_object, (int, float, str, bytes, io.BytesIO)) or iter_object is None: + ret = iter_object + elif isinstance(iter_object, dict): + if companion_obj is not None and ( + not isinstance(companion_obj, dict) or set(companion_obj.keys()) != set(iter_object.keys()) + ): + msg = "" if isinstance(companion_obj, dict) else f"{set(companion_obj.keys())=} {set(iter_object.keys())=}" + raise CompanionMismatch(msg) + + ret = { + key: _iterate_state_dict( + value, + sharded_tensor_func, + dtensor_func, + tensor_func, + pg=pg, + device=device, + cpu_offload=cpu_offload, + companion_obj=companion_obj[key] if companion_obj is not None else None, + ranks_only=ranks_only, + type_check=type_check, + non_blocking=non_blocking, + ) + for key, value in iter_object.items() + } + elif isinstance(iter_object, (list, tuple)): + if companion_obj is not None and ( + not isinstance(companion_obj, (list, tuple)) or len(companion_obj) != len(iter_object) + ): + raise CompanionMismatch + + ret = [ + _iterate_state_dict( + v, + sharded_tensor_func, + dtensor_func, + tensor_func, + pg=pg, + device=device, + cpu_offload=cpu_offload, + companion_obj=companion_obj[idx] if companion_obj is not None else None, + ranks_only=ranks_only, + type_check=type_check, + non_blocking=non_blocking, + ) + for idx, v in enumerate(iter_object) + ] + if isinstance(iter_object, tuple): + ret = tuple(ret) + elif not type_check: + ret = copy.deepcopy(iter_object) + else: + raise ValueError(f"Unexpected value type {type(iter_object)}") + + if not ranks_only or dist.get_rank(pg) in ranks_only: + if isinstance(ret, torch.Tensor): + if cpu_offload and companion_obj is None: + ret = ret.to(cpu_device) + + if companion_obj is not None: + if isinstance(companion_obj, DTensor): + assert isinstance(ret, DTensor) + companion_obj._local_tensor.copy_(ret._local_tensor, non_blocking=non_blocking) + else: + companion_obj.copy_(ret, non_blocking=non_blocking) + ret = companion_obj + else: + ret = {} if isinstance(ret, dict) else None + + return ret + + +def _gather_state_dict( + state_dict: dict[str, Any], + *, + pg: Optional[dist.ProcessGroup] = None, + device: Optional[torch.device] = None, + cpu_offload: bool = False, + ranks_only: tuple[int, ...] = (), + type_check: bool = True, +) -> dict[str, Any]: + """ + Given a state_dict, this API gathers all the ShardedTensors or DTensors in + the state_dict. + + + Args: + state_dict (Dict[str, Any]): the target sharded state_dict. + pg (Optional[dist.ProcessGroup]): the process group that is used to + gather ShardedTensor. Note that gathering a DTensor will use + the DeviceMesh. So this argument will be ignored when gathering a + DTensor. + device: (Optional[torch.device]): the device that is used to + perform allgather for ShardedTensor. Note that gathering a DTensor + will use the DeviceMesh. So this argument will be ignored when + gathering a DTensor. + cpu_offload (bool): whether to offload the tensors to CPU memory. The + default value is False. + ranks_only: (Tuple[int, ...]): if this tuple is empty, all ranks will + have the same state_dicts. Otherwise only ranks that in ``ranks_only`` + have the same state_dicts. Other ranks will get empty state_dicts. + type_check: (bool): check if the instance data type is a supported type + that can be saved by DCP. The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + + Returns: + The gathered state dictionary. + """ + + def sharded_tensor_func(value, pg, device, companion_obj): + # ShardedTensor does not seem to record the original device type. + # So if the tensor is moved to CPU, we won't know the original type. + # As a result, we have to rely on the user to tell us the correct one. + cpu_device = torch.device("cpu") + output_tensor = _all_gather_sharded_tensor(value, pg, device) + local_shard_device = value.local_shards()[0].tensor.device if value.local_shards() else cpu_device + if output_tensor.device != local_shard_device: + value = output_tensor.to(local_shard_device) + else: + value = output_tensor + return value + + def dtensor_func(value, pg, device, companion_obj): + if value.device != value.device_mesh.device_type: + value = value.to(value.device_mesh.device_type) + # FSDP all_gather: [Shard(0)] -> [Replicate()] + # HSDP all_gather: [Replicate(), Shard(0)] -> [Replicate(), Replicate()] + # 2D FSDP + TP all_gather: + # - [Shard(0), Shard(n)] -> [Replicate(), Replicate()] + # - [Shard(0), Replicate()] -> [Replicate(), Replicate()] + placements = [Replicate() for _ in value.placements] + value = value.redistribute( + device_mesh=value.device_mesh, + placements=placements, + ) + # Call `wait()` to force the tensor to be synchronous with respect + # to the main stream. + # See the discussion in https://github.com/pytorch/pytorch/pull/117799. + value = value.to_local() + if isinstance(value, AsyncCollectiveTensor): + value = value.wait() + return value + + return _iterate_state_dict( + state_dict, + sharded_tensor_func, + dtensor_func, + _identity_func, + pg=pg, + device=device, + cpu_offload=cpu_offload, + ranks_only=ranks_only, + type_check=type_check, + ) + + +def _offload_state_dict_to_cpu( + state_dict: dict[str, Any], + *, + ranks_only: tuple[int, ...] = (), + type_check: bool = True, +) -> dict[str, Any]: + """ + Given a state_dict, this API offload all the tensors to CPU memory. + + Args: + state_dict (Dict[str, Any]): the target state_dict. + pg (Optional[dist.ProcessGroup]): the process group that is used to + gather ShardedTensor. Note that gathering a DTensor will use + the DeviceMesh. So this argument will be ignored when gathering a + DTensor. + ranks_only: (Tuple[int, ...]): if this tuple is empty, all ranks will + have the same state_dicts. Otherwise only ranks that in ``ranks_only`` + have the same state_dicts. Other ranks will get empty state_dicts. + type_check: (bool): check if the instance data type is a supported type + that can be saved by DCP. The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + + Returns: + The gathered state dictionary. + """ + + ret = _iterate_state_dict( + state_dict, + _identity_func, + _identity_func, + _identity_func, + pg=None, + device=None, + cpu_offload=True, + ranks_only=ranks_only, + type_check=type_check, + ) + return ret + + +@torch.no_grad() +def _copy_state_dict( + state_dict: dict[str, Any], + copy_state_dict: dict[str, Any], + non_blocking: bool = False, + type_check: bool = True, +) -> dict[str, Any]: + """ + Copies all tensors in a given state dict into a different state_dict with the + same structure. Additionally, a copied state dict with the same value references + is returned. Editing the keys on this state dict will not affect the + passed in copy_state_dict (but the value references are the same). + + .. warning:: + It is expected by this function that state_dict and copy_state_dict share + the same structure and data types. + + .. warning:: + The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + + Args: + state_dict (Dict[str, Any]): the target state_dict. + copy_state_dict (Dict[str, Any]): + The state dict we are copying into. This state_dict must have exactly + the same structure as the source `state_dict`. + non_blocking: (bool): Whether copy ops should be performed asynchronously + type_check (bool): check if the instance data type is a supported type + that can be saved by DCP. The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + + Returns: + State Dict copy + """ + + return _iterate_state_dict( + state_dict, + _identity_func, + _identity_func, + _identity_func, + pg=None, + device=None, + cpu_offload=False, + ranks_only=(), + companion_obj=copy_state_dict, + type_check=type_check, + non_blocking=non_blocking, + ) + + +@torch.no_grad() +def _create_cpu_state_dict( + state_dict: dict[str, Any], pin_memory: bool = False, share_memory: bool = False +) -> dict[str, Any]: + """ + Given a state_dict, create another state_dict with the same structure and elements. + However, all tensors in the returned state_dict are new tensors on CPU. These + tensors can be placed on pin_memory or share_memory based on the provided arguments. + + .. warning:: + Setting both `pin_memory` and `share_memory` to True significantly increases the + latency of this method because of the nuances which require us to register memory + as pinned directly as opposed to relying on the pin_memory cache allocator. This + option should only be used for long lived tensors which are required to be shared. + This is not the case as long as at least one of `pin_memory` or `share_memory` is + set to False. + + """ + + def tensor_func( + obj: torch.Tensor, + pg: Optional[dist.ProcessGroup], + device: Optional[torch.device], + _: Any, + ) -> torch.Tensor: + if len(obj.size()) == 0: + return torch.tensor(0, dtype=obj.dtype) + + if share_memory: + t = torch.empty(*tuple(obj.size()), dtype=obj.dtype) + t = t.share_memory_() + if pin_memory: + + def unpin_memory(t): + succ = int(torch.cuda.cudart().cudaHostUnregister(t.data_ptr())) + assert succ == 0, f"Unpinning shared memory failed with error-code: {succ}" + + weakref.finalize(t, unpin_memory, t) + succ = int( + torch.cuda.cudart().cudaHostRegister( + t.data_ptr(), + t.numel() * t.element_size(), + 1, # lines up with 'cudaHostRegisterPortable' + ) + ) + assert succ == 0, f"Pinning shared memory failed with error-code: {succ}" + return t + elif pin_memory: + return torch.empty(*tuple(obj.size()), dtype=obj.dtype).pin_memory() + else: + return torch.empty(*tuple(obj.size()), dtype=obj.dtype) + + def dtensor_func( + obj: DTensor, + pg: Optional[dist.ProcessGroup], + device: Optional[torch.device], + _: Any, + ) -> DTensor: + if len(obj.size()) == 0: + return obj + + if obj.device != torch.device("cpu"): + ret = cast(DTensor, obj.to(device="cpu")) + else: + ret = copy.deepcopy(obj) + ret._local_tensor = tensor_func(ret._local_tensor, pg, device, None) + return ret + + ret = _iterate_state_dict( + state_dict, + _identity_func, + dtensor_func, + tensor_func, + pg=None, + device=None, + cpu_offload=False, + ranks_only=(), + type_check=False, + ) + return ret + + +def _check_state_dict_similarity( + state_dict: dict[str, Any], + compared_state_dict: dict[str, Any], +) -> bool: + """ + Given two state_dicts, check if the structures are the same. And + if a [key, tensor] pair exist in one state_dict there must be + the a corresponding pait, [key, other_tensor], in the other state_dict, + where tensor and other_tensor have the same size and dtype. + + Return the check result. + """ + + def tensor_func( + obj: torch.Tensor, + pg: Optional[dist.ProcessGroup], + device: Optional[torch.device], + companion_obj: Any, + ) -> torch.Tensor: + if companion_obj.dtype != obj.dtype or companion_obj.size() != obj.size(): + raise CompanionMismatch + return obj + + try: + _iterate_state_dict( + state_dict, + _identity_func, + _identity_func, + tensor_func, + pg=None, + device=None, + cpu_offload=False, + ranks_only=(), + companion_obj=compared_state_dict, + type_check=False, + ) + except CompanionMismatch: + return False + + return True + + +class _TensorInfo(NamedTuple): + size: torch.Size + dtype: torch.dtype + + +def _broadcast_tensors( + full_state_dict: dict[str, Any], + local_state_dict: dict[str, Any], + keys: list[str], + device: torch.device, + pg: Optional[dist.ProcessGroup] = None, +) -> None: + tensors = [] + for key in keys: + if dist.get_rank() == 0: + full_state = full_state_dict[key] + assert isinstance(full_state, torch.Tensor) + full_tensor = full_state.detach().to(device) + else: + tensor_info = full_state_dict[key] + full_tensor = torch.empty( + size=tensor_info.size, + device=device, + dtype=tensor_info.dtype, + ) + tensors.append(full_tensor) + local_state = local_state_dict.get(key, None) + if local_state is None: + continue + elif isinstance(local_state, DTensor): + local_state_dict[key] = (local_state, full_tensor) + else: + local_state_dict[key] = full_tensor + + if pg is None: + pg = dist.distributed_c10d._get_default_group() + + if len(tensors) > 1: + dist._broadcast_coalesced(pg, tensors, 500, 0) + else: + dist.broadcast(tensors[0], src=0, group=pg) + + _distribute_tensors(local_state_dict, keys, device, pg) + + +def _distribute_tensors( + local_state_dict: dict[str, Any], + keys: list[str], + device: torch.device, + pg: Optional[dist.ProcessGroup] = None, +) -> None: + if pg is None: + pg = dist.distributed_c10d._get_default_group() + for key in keys: + _local_state = local_state_dict.get(key, None) + if _local_state is None or torch.is_tensor(_local_state): + continue + + local_state = _local_state[0] + full_tensor = _local_state[1] + + shape, offset = compute_local_shape_and_global_offset( + full_tensor.shape, local_state.device_mesh, local_state.placements + ) + slices = [ + slice(cur_offset, cur_offset + cur_shape) for cur_shape, cur_offset in zip(shape, offset, strict=False) + ] + if local_state.is_meta: + # Use .clone() here rather than view to clone and return only the sliced portion, minimizing memory access and cost. + local_tensor = full_tensor[slices].detach().clone() + # TODO: currently, we cannot handle strided sharding if the dp dimension is not even. For example, + # one of the case that is not yet supported is when placements = (Shard(0), _StridedShard(0, sf=2)). + ret = DTensor.from_local( + local_tensor, + local_state.device_mesh, + local_state.placements, + shape=local_state.shape, + stride=local_state.stride(), + ) + else: + ret = local_state + # Copy full_tensor[slices] into local_state.to_local() to reduce memory footprint. + ret.to_local().copy_(full_tensor[slices]) + local_state_dict[key] = ret + + +def _broadcast_state_dict( + full_state_dict: dict[str, Any], + local_state_dict: dict[str, Any], + device: torch.device, + pg: Optional[dist.ProcessGroup] = None, + strict: bool = False, + cpu_offload: bool = False, +) -> None: + # Broadcast from rank0's `full_state_dict` to all ranks' `local_state_dict`. + # If strict is True, any keys in `local_state_dict` but not in `full_state_dict` + # will be removed from `local_state_dict`. + ret = {} + if dist.get_rank() == 0: + for key, value in full_state_dict.items(): + if not torch.is_tensor(value): + ret[key] = value + elif value.dim() == 0: + ret[key] = value.cpu() + else: + ret[key] = _TensorInfo(value.size(), value.dtype) + + broadcast_list = [ret] + dist.broadcast_object_list(broadcast_list, src=0, group=pg) + ret = broadcast_list[0] + # Gather values + keys = [] + local_state_dict_keys = set(local_state_dict.keys()) + global_keys = set() + for key, value in ret.items(): + global_keys.add(key) + if not isinstance(value, _TensorInfo): + if key in local_state_dict: + local_state_dict[key] = value + continue + + if dist.get_rank() == 0: + ret[key] = full_state_dict[key] + + keys.append(key) + # Broadcast every tensor to avoid OOM for now. + if len(keys) >= 1: + _broadcast_tensors(ret, local_state_dict, keys, device, pg) + if cpu_offload: + for key in keys: + local_state_dict[key] = local_state_dict[key].cpu() + keys.clear() + + if strict: + if missing_keys := (local_state_dict_keys - global_keys): + for key in missing_keys: + local_state_dict.pop(key) + + if keys: + _broadcast_tensors(ret, local_state_dict, keys, device, pg) + if cpu_offload: + for key in keys: + local_state_dict[key] = local_state_dict[key].cpu() + + +def _distribute_state_dict( + full_state_dict: dict[str, Any], + local_state_dict: dict[str, Any], + device: torch.device, + pg: Optional[dist.ProcessGroup] = None, +) -> None: + # Full_state_dict = True, broadcast_from_rank0 = False here. Each rank has + # full_state_dict. Skip the broadcast in ``_broadcast_state_dict`` and + # distribute tensors in each rank + for key, value in full_state_dict.items(): + if key not in full_state_dict: + continue + if not torch.is_tensor(value): + local_state_dict[key] = value + elif value.dim() == 0: + local_state_dict[key] = value.cpu() + else: + assert isinstance(value, torch.Tensor) + local_state = local_state_dict.get(key, None) + if local_state is None: + continue + elif isinstance(local_state, DTensor): + local_state_dict[key] = distribute_tensor( + value.detach().to(device), + local_state.device_mesh, + local_state.placements, + ) + else: + local_state_dict[key] = value.detach().to(device) + + +# These APIs are from torch.distributed.checkpoint. +# TODO: We should consolidate the code here as some not all modules can depend on +# DCP. +PATH_ITEM = Union[str, int] +OBJ_PATH = tuple[PATH_ITEM, ...] +FLATTEN_MAPPING = dict[str, OBJ_PATH] +STATE_DICT_TYPE = dict[str, Any] +CONTAINER_TYPE = MutableMapping[PATH_ITEM, Any] + + +def _traverse_state_dict( + state_dict: STATE_DICT_TYPE, + visitor: Callable[[OBJ_PATH, Any], None], +) -> None: + """ + Invoke ``visitor`` for each value recursively in ``state_dict``. + Mapping, list, and tuple will be flattened and other value types are treated + as the terminal values and will invoke ``visitor``. + """ + + def _traverse_obj(path: OBJ_PATH, value: Any) -> None: + if isinstance(value, Mapping): + for k, v in value.items(): + _traverse_obj(path + (str(k),), v) + elif isinstance(value, (list, tuple)): + for i, v in enumerate(value): + _traverse_obj(path + (i,), v) + else: + visitor(path, value) + + for key, value in state_dict.items(): + _traverse_obj((str(key),), value) + + +def _flatten_state_dict( + state_dict: STATE_DICT_TYPE, +) -> tuple[STATE_DICT_TYPE, FLATTEN_MAPPING]: + """ + Flatten ``state_dict`` made of nested dicts and lists into a top level dictionary. + + Use ``unflatten_state_dict`` to revert this process. + Returns: + A tuple with the flatten state_dict and a mapping from original to new state_dict. + N.B. The new keys are derived from the object paths, joined by dot. + For example: ``{ 'a': {'b':...}}`` results in the key `a.b`. + """ + flattened: STATE_DICT_TYPE = {} + mappings: FLATTEN_MAPPING = {} + + def flat_copy(path: OBJ_PATH, value: Any) -> None: + new_fqn = ".".join(map(str, path)) + if new_fqn in flattened: + raise ValueError(f"duplicated flatten key {new_fqn}") + flattened[new_fqn] = value + mappings[new_fqn] = path + + _traverse_state_dict(state_dict, flat_copy) + return flattened, mappings + + +def _set_element(root_dict: STATE_DICT_TYPE, path: OBJ_PATH, value: Any) -> None: + """Set ``value`` in ``root_dict`` along the ``path`` object path.""" + cur_container = cast(CONTAINER_TYPE, root_dict) + + def extend_list(lst: list[Any], idx: int) -> None: + while len(lst) <= idx: + lst.append(None) + + for i in range(1, len(path)): + prev_key = path[i - 1] + key = path[i] + def_val: CONTAINER_TYPE | list[Any] = {} if type(key) == str else [] + + if isinstance(cur_container, Mapping): + cur_container = cast(CONTAINER_TYPE, cur_container.setdefault(prev_key, def_val)) + else: + extend_list(cur_container, prev_key) + if cur_container[prev_key] is None: + cur_container[prev_key] = def_val + cur_container = cur_container[prev_key] + + key = path[-1] + if type(key) == int: + extend_list(cast(list[Any], cur_container), key) + + cur_container[key] = value + + +def _unflatten_state_dict(state_dict: STATE_DICT_TYPE, mapping: FLATTEN_MAPPING) -> STATE_DICT_TYPE: + """Restore the original nested state_dict according to ``mapping`` and the flattened ``state_dict``.""" + nested: STATE_DICT_TYPE = {} + for key, value in state_dict.items(): + _set_element(nested, mapping[key], value) + return nested diff --git a/e2c/verl/verl/third_party/torch/distributed/checkpoint/__init__.py b/e2c/verl/verl/third_party/torch/distributed/checkpoint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7664279b7411a806f615b52b2405fd2c40672517 --- /dev/null +++ b/e2c/verl/verl/third_party/torch/distributed/checkpoint/__init__.py @@ -0,0 +1,87 @@ +# official torch 2.6.0 set_model_state_dict API leads to OOM +# this is a copy of torch/distributed/checkpoint from torch 2.7.0 + +# From PyTorch: + +# Copyright (c) 2016- Facebook, Inc (Adam Paszke) +# Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +# Copyright (c) 2011-2013 NYU (Clement Farabet) +# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +# Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +# From Caffe2: + +# Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +# All contributions by Facebook: +# Copyright (c) 2016 Facebook Inc. + +# All contributions by Google: +# Copyright (c) 2015 Google Inc. +# All rights reserved. + +# All contributions by Yangqing Jia: +# Copyright (c) 2015 Yangqing Jia +# All rights reserved. + +# All contributions by Kakao Brain: +# Copyright 2019-2020 Kakao Brain + +# All contributions by Cruise LLC: +# Copyright (c) 2022 Cruise LLC. +# All rights reserved. + +# All contributions by Tri Dao: +# Copyright (c) 2024 Tri Dao. +# All rights reserved. + +# All contributions by Arm: +# Copyright (c) 2021, 2023-2024 Arm Limited and/or its affiliates + +# All contributions from Caffe: +# Copyright(c) 2013, 2014, 2015, the respective contributors +# All rights reserved. + +# All other contributions: +# Copyright(c) 2015, 2016 the respective contributors +# All rights reserved. + +# Caffe2 uses a copyright model similar to Caffe: each contributor holds +# copyright over their contributions to Caffe2. The project versioning records +# all such contribution and copyright details. If a contributor wants to further +# mark their specific copyright on a particular contribution, they should +# indicate their copyright solely in the commit message of the change when it is +# committed. + +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America +# and IDIAP Research Institute nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. diff --git a/e2c/verl/verl/third_party/torch/distributed/checkpoint/state_dict.py b/e2c/verl/verl/third_party/torch/distributed/checkpoint/state_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..e4555802aed8c4b5963892a688b1ff41ae97fb56 --- /dev/null +++ b/e2c/verl/verl/third_party/torch/distributed/checkpoint/state_dict.py @@ -0,0 +1,1493 @@ +# official torch 2.6.0 set_model_state_dict API leads to OOM +# this is a copy of torch/distributed/checkpoint from torch 2.7.0 + +# From PyTorch: + +# Copyright (c) 2016- Facebook, Inc (Adam Paszke) +# Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +# Copyright (c) 2011-2013 NYU (Clement Farabet) +# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +# Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +# From Caffe2: + +# Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +# All contributions by Facebook: +# Copyright (c) 2016 Facebook Inc. + +# All contributions by Google: +# Copyright (c) 2015 Google Inc. +# All rights reserved. + +# All contributions by Yangqing Jia: +# Copyright (c) 2015 Yangqing Jia +# All rights reserved. + +# All contributions by Kakao Brain: +# Copyright 2019-2020 Kakao Brain + +# All contributions by Cruise LLC: +# Copyright (c) 2022 Cruise LLC. +# All rights reserved. + +# All contributions by Tri Dao: +# Copyright (c) 2024 Tri Dao. +# All rights reserved. + +# All contributions by Arm: +# Copyright (c) 2021, 2023-2024 Arm Limited and/or its affiliates + +# All contributions from Caffe: +# Copyright(c) 2013, 2014, 2015, the respective contributors +# All rights reserved. + +# All other contributions: +# Copyright(c) 2015, 2016 the respective contributors +# All rights reserved. + +# Caffe2 uses a copyright model similar to Caffe: each contributor holds +# copyright over their contributions to Caffe2. The project versioning records +# all such contribution and copyright details. If a contributor wants to further +# mark their specific copyright on a particular contribution, they should +# indicate their copyright solely in the commit message of the change when it is +# committed. + +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America +# and IDIAP Research Institute nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# ruff: noqa: B028, UP038, UP007, E721 +# mypy: allow-untyped-defs +import contextlib +import functools +import gc +import warnings +from collections.abc import Generator, Iterable +from dataclasses import asdict, dataclass, field +from itertools import chain +from typing import Any, Callable, Optional, Union, cast, no_type_check + +import torch +import torch.distributed as dist +import torch.nn as nn +from torch.distributed._shard.sharded_tensor import ShardedTensor +from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( + _CHECKPOINT_PREFIX, +) +from torch.distributed.fsdp import ( + FullOptimStateDictConfig, + FullStateDictConfig, + OptimStateDictConfig, + ShardedOptimStateDictConfig, + ShardedStateDictConfig, + StateDictConfig, + StateDictType, +) +from torch.distributed.fsdp import ( + FullyShardedDataParallel as FSDP, +) +from torch.distributed.fsdp._common_utils import ( + FSDP_WRAPPED_MODULE, + _get_module_fsdp_state_if_fully_sharded_module, +) +from torch.distributed.tensor import DTensor +from torch.nn.modules.module import _IncompatibleKeys +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils._pytree import tree_map_only + +from verl.third_party.torch.distributed._state_dict_utils import ( + _broadcast_state_dict, + _distribute_state_dict, + _flatten_state_dict, + _gather_state_dict, + _offload_state_dict_to_cpu, + _unflatten_state_dict, +) + +__all__ = [ + "FQNS_T", + "PrimitiveType", + "ValueType", + "DictValueType", + "ListDictValueType", + "OptimizerStateType", + "StateDictOptions", + "get_model_state_dict", + "get_optimizer_state_dict", + "get_state_dict", + "set_model_state_dict", + "set_optimizer_state_dict", + "set_state_dict", +] + + +_FLAT_PARAM = "_flat_param" +_PG = "param_groups" +_PARAMS = "params" +_STATE = "state" + +FQNS_T = set[str] +PrimitiveType = Union[DTensor, ShardedTensor, torch.Tensor, int, float, str] +ValueType = Union[PrimitiveType, list[PrimitiveType], tuple[PrimitiveType], dict[str, "ValueType"]] +DictValueType = dict[str, ValueType] +ListDictValueType = list[DictValueType] +OptimizerStateType = dict[str, DictValueType | ListDictValueType] + + +_patched_state_dict: set[Callable] = set() + + +@contextlib.contextmanager +def _gc_context(): + is_enabled = gc.isenabled() + gc.disable() + try: + yield + finally: + if is_enabled: + gc.enable() + + +@dataclass +class StateDictOptions: + """ + This dataclass specifies how get_state_dict/set_state_dict will work. + + - ``full_state_dict``: if this is set to True, all the tensors in the + returned state_dict will be gathered. No ShardedTensor and DTensor + will be in the returned state_dict. + + - ``cpu_offload``: offload all the tensors to cpu. To prevent CPU OOM, if + ``full_state_dict`` is also true, then only the rank0 will get the + state_dict and all other ranks will get empty state_dict. + + - ``ignore_frozen_params``: if the value is True, the returned state_dict + won't contain any frozen parameters -- the ``requires_grad`` is False. + The default value is False. + + - ``keep_submodule_prefixes`` (deprecated): when ``submodules`` is not None, this option + indicates whether to keep the submodule prefixes from the state_dict keys. + or example, if the submodule is ``module.pretrain`` and the full FQN of + the parameter is ``pretrain.layer1.weight`` of the param. When this option + is True, the parameter's key in the returned state_dict will be + ``pretrain.layer1.weight``. If the options is False, the key will be + ``layer1.weight``. + Note that if ``keep_submodule_prefixes`` is False, there may be conflicted + FQNs, hence there should be only one submodule in ``submodules``. + + - ``strict``: the ``strict`` option when ``set_state_dict`` calls + model.load_state_dict(). + + - ``broadcast_from_rank0``: when the option is True, rank0 should receive a + full state_dict and will broadcast the tensors in the state_dict/ + optim_state_dict one by one to other ranks. Other ranks will receive + the tensors and shard according to the local shards in the model and + optimizer. ``full_state_dict`` must be set to True when using this option. + This option currently only supports DTensor, not the legacy ShardedTensor. + """ + + full_state_dict: bool = False + cpu_offload: bool = False + ignore_frozen_params: bool = False + keep_submodule_prefixes: bool = True + strict: bool = True + broadcast_from_rank0: bool = False + flatten_optimizer_state_dict: bool = False + dsd_fqn_modifiers: str = "_fqn_modifiers" + + +@dataclass +class _StateDictInfo(StateDictOptions): + fqn_param_mapping: dict[ + str | torch.Tensor, + FQNS_T | torch.Tensor, + ] = field(default_factory=dict) + shared_params_mapping: dict[ + str | torch.Tensor, + FQNS_T | torch.Tensor, + ] = field(default_factory=dict) + submodule_prefixes: set[str] = field(default_factory=set) + handle_model: bool = True + handle_optim: bool = True + fsdp_context: Callable = contextlib.nullcontext + fsdp_modules: list[nn.Module] = field(default_factory=list) + + +@functools.cache +def _get_fqns( + model: nn.Module, + name: str, + dsd_fqn_modifiers: str = "_fqn_modifiers", + skip_ddp_prefix: bool = True, + skip_compiler_prefix: bool = True, +) -> FQNS_T: + """ + This API is used to convert the name of a parameter to the FQNs. For FSDP + without `use_orig_params`, the name of FlatParameter can be mapped to + multiple original parameters. As a result, the return type of this function + is `set[str]`. + + Args: + module (nn.Module): the root model. + name (str): the name + skip_ddp_prefix (bool): whether to skip DDP's `module` prefix + + Returns: + The canonical FQNs based on the model traversal. + """ + + # Remove the checkpoint prefix, if it exists. + name = name.replace(_CHECKPOINT_PREFIX, "") + if "." not in name: + return {name} + + obj_names = name.split(".") + fqn_obj_names = [] + curr_obj = model + for i, curr_obj_name in enumerate(obj_names): + if isinstance(curr_obj, DDP): + assert curr_obj_name == "module" + curr_obj = curr_obj.module + if not skip_ddp_prefix: + fqn_obj_names.append(curr_obj_name) + elif isinstance(curr_obj, FSDP): + if i < len(obj_names) - 1 and obj_names[i + 1] == _FLAT_PARAM: + prefix = ".".join(fqn_obj_names) + flat_param = getattr(curr_obj, _FLAT_PARAM) + if prefix: + prefix = f"{prefix}." + return {f"{prefix}{fqn}" for fqn in flat_param._fqns} + curr_obj = getattr(curr_obj, FSDP_WRAPPED_MODULE) + if curr_obj_name != FSDP_WRAPPED_MODULE: + fqn_obj_names.append(curr_obj_name) + curr_obj = getattr(curr_obj, curr_obj_name) + elif isinstance(curr_obj, torch._dynamo.eval_frame.OptimizedModule): + assert curr_obj_name == "_orig_mod" + curr_obj = curr_obj._orig_mod + if not skip_compiler_prefix: + fqn_obj_names.append(curr_obj_name) + else: + # In some modeuls, _fqn_modifiers would not shown in the state_dict keys, + # skip them in the fqn to ensure load stat dict successfully for them. + if hasattr(curr_obj, dsd_fqn_modifiers): + if removed_fqn := getattr(curr_obj, dsd_fqn_modifiers)().get(curr_obj_name): + if hasattr(curr_obj, removed_fqn): + curr_obj = getattr(curr_obj, removed_fqn) + fqn_obj_names.append(curr_obj_name) + if curr_obj_name == nn.modules.module._EXTRA_STATE_KEY_SUFFIX: + if i != len(obj_names) - 1: + raise RuntimeError("Expect `_extra_state` to be the last obj name") + else: + curr_obj = getattr(curr_obj, curr_obj_name) + + return {".".join(fqn_obj_names).replace(_CHECKPOINT_PREFIX, "")} + + +class _EXTRA_STATE: + pass + + +def _iterate_valid_model_state(model, dsd_fqn_modifiers="_fqn_modifiers"): + visited_modules: set[nn.Module] = set() + + def recurse(module: nn.Module, curr_fqn: str) -> Generator: + visited_modules.add(module) + + curr_fqn = f"{curr_fqn}." if curr_fqn else "" + for name, submodule in module.named_children(): + if submodule in visited_modules: + continue + # if user have state_dict_hooks in their model, they can add the state_dict key changes + # at dsd_fqn_modifiers in input to align with the function of state_dict_hook + if hasattr(module, dsd_fqn_modifiers) and name in getattr(module, dsd_fqn_modifiers)().values(): + # skip _fqn_modifiers here thus remove the last `.` added + new_fqn = curr_fqn[:-1] + else: + new_fqn = f"{curr_fqn}{name}" + yield from recurse(submodule, new_fqn) + + for name, obj in chain(module.named_buffers(recurse=False), module.named_parameters(recurse=False)): + if name in module._non_persistent_buffers_set: + continue + new_fqn = f"{curr_fqn}{name}" + yield new_fqn, obj + + if getattr(module.__class__, "get_extra_state", nn.Module.get_extra_state) != nn.Module.get_extra_state: + new_fqn = f"{curr_fqn}{nn.modules.module._EXTRA_STATE_KEY_SUFFIX}" + yield new_fqn, _EXTRA_STATE() + + yield from recurse(model, "") + + +def _verify_options( + model: nn.Module, + optims: tuple[torch.optim.Optimizer, ...], + optim_only: bool, + *, + submodules: Optional[set[nn.Module]] = None, + options: Optional[StateDictOptions] = None, +) -> _StateDictInfo: + """ + Verify the model and options passed by the user and generates _StateDictInfo. + """ + if submodules: + warnings.warn( + "Getting submodules only model/optim state_dict is deprecated and " + "will be removed in 2.5. This feature can be achieved by manually " + "filtering out the state_dict returned from get_state_dict.", + FutureWarning, + ) + if optim_only and not optims: + raise RuntimeError("Optimizers are not passed in but optim_only is set to True.") + + options = options or StateDictOptions() + + fqn_param_mapping: dict[str | torch.Tensor, set[str] | torch.Tensor] = {} + shared_params_mapping: dict[str | torch.Tensor, set[str] | torch.Tensor] = {} + for name, param in _iterate_valid_model_state(model): + if isinstance(param, _EXTRA_STATE): + continue + + fqns = _get_fqns(model, name) + fqn = fqn_param_mapping.get(param, None) + if fqn is not None: + cast(set[str], fqn_param_mapping[param]).update(fqns) + shared_params_mapping[param] = fqn_param_mapping[param] + else: + # We need to do copy as _get_fqns is lru_cached + fqn_param_mapping[param] = fqns.copy() + for fqn in fqns: + if not isinstance(param, _EXTRA_STATE): + fqn_param_mapping[fqn] = param + + for param_, fqns_ in list(shared_params_mapping.items()): + for fqn in fqns_: + shared_params_mapping[fqn] = cast(torch.Tensor, param_) + + submodule_prefixes: set[str] = set() + if submodules: + submodules = set(submodules) + for name, module in model.named_modules(): + if module not in submodules: + continue + fqns = _get_fqns(model, name) + assert len(fqns) == 1, "Submodule FQN should only have 1 instance" + submodule_prefixes.update(f"{fqn}." for fqn in fqns) + + if options.broadcast_from_rank0 and not options.full_state_dict: + raise ValueError("full_state_dict must be True when broadcast_from_rank0 is True.") + fsdp_modules = FSDP.fsdp_modules(model) + state_dict_config: StateDictConfig + optim_state_dict_config: OptimStateDictConfig + fsdp_context: Callable + if fsdp_modules: + # FSDP API only work if at least one FSDP instance exists. + if options.full_state_dict: + state_dict_config = FullStateDictConfig(offload_to_cpu=options.cpu_offload, rank0_only=options.cpu_offload) + optim_state_dict_config = FullOptimStateDictConfig( + offload_to_cpu=options.cpu_offload, + rank0_only=(options.cpu_offload or options.broadcast_from_rank0), + ) + state_dict_type = StateDictType.FULL_STATE_DICT + else: + state_dict_config = ShardedStateDictConfig( + offload_to_cpu=options.cpu_offload, + ) + optim_state_dict_config = ShardedOptimStateDictConfig( + offload_to_cpu=options.cpu_offload, + ) + state_dict_type = StateDictType.SHARDED_STATE_DICT + + @contextlib.contextmanager + def fsdp_state_dict_type_without_warning( + module, + state_dict_type, + state_dict_config, + optim_state_dict_config, + ): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="FSDP.state_dict_type", category=FutureWarning) + with FSDP.state_dict_type( + module=module, + state_dict_type=state_dict_type, + state_dict_config=state_dict_config, + optim_state_dict_config=optim_state_dict_config, + ): + yield + + fsdp_context = functools.partial( + fsdp_state_dict_type_without_warning, + module=model, + state_dict_type=state_dict_type, + state_dict_config=state_dict_config, + optim_state_dict_config=optim_state_dict_config, + ) + else: + fsdp_context = contextlib.nullcontext + + return _StateDictInfo( + **asdict(options), + fqn_param_mapping=fqn_param_mapping, + shared_params_mapping=shared_params_mapping, + submodule_prefixes=submodule_prefixes, + fsdp_context=fsdp_context, + fsdp_modules=cast(list[nn.Module], fsdp_modules), + handle_model=not optim_only, + handle_optim=(len(optims) > 0), + ) + + +def _verify_state_dict( + model_state_dict: dict[str, ValueType], + optim_state_dict: OptimizerStateType, + info: _StateDictInfo, +) -> None: + for module in info.fsdp_modules: + fsdp_state = _get_module_fsdp_state_if_fully_sharded_module(module) + assert fsdp_state is not None, "Expected a fsdp_state with a fsdp module." + + # Verify if the model_state_dict and optim_state_dict are valid. This API + # should give the users an explicit error message to debug or report. + if ( + info.handle_model + and not model_state_dict + and not info.submodule_prefixes + and not info.ignore_frozen_params + and not (info.cpu_offload and info.full_state_dict) + and info.strict + and not info.broadcast_from_rank0 + ): + raise RuntimeError( + "The option indicates that model state_dict is required to save " + "or load, but model state_dict is empty." + f"rank = {dist.get_rank()=}." + ) + + if info.handle_optim: + if not optim_state_dict and not (info.cpu_offload and info.full_state_dict) and (not info.broadcast_from_rank0): + raise RuntimeError( + "The option indicates that model state_dict is required to save, " + f"or load but optim state_dict is empty. {optim_state_dict}" + ) + + for key in model_state_dict.keys(): + if _FLAT_PARAM in key: + raise RuntimeError(f"{key} contains {_FLAT_PARAM}. This can happen if the model is not the root module.") + + +def _state_dict_fn(obj: nn.Module | torch.optim.Optimizer, api: str) -> Callable: + call = getattr(obj, api) + if call in _patched_state_dict: + call = functools.partial(getattr(obj.__class__, api), self=obj) + return call + + +def _maybe_full_or_cpu_state_dict(state_dict: dict[str, Any], info: _StateDictInfo) -> dict[str, Any]: + if info.full_state_dict: + ranks_only = () if (not info.cpu_offload or not torch.distributed.is_initialized()) else (0,) + return _gather_state_dict(state_dict, cpu_offload=info.cpu_offload, ranks_only=ranks_only) + elif info.cpu_offload: + return _offload_state_dict_to_cpu(state_dict) + else: + return state_dict + + +@torch.no_grad() +def _get_model_state_dict(model: nn.Module, info: _StateDictInfo) -> dict[str, ValueType]: + if not info.handle_model: + return {} + + with info.fsdp_context(): + state_dict = _state_dict_fn(model, "state_dict")() + + for key in list(state_dict.keys()): + fqns = _get_fqns(model, key) + assert len(fqns) == 1, (key, fqns) + fqn = next(iter(fqns)) + if fqn != key: + # As we only support FSDP, DDP, and TP, the only cases are + # wrapper-based DDP and compiler. Verify if the assumption + # is correct. + def verify(key, fqn) -> bool: + if len(fqn) >= len(key): + return False + fqn_split = fqn.split(".") + key_split = key.split(".") + fqn_idx = 0 + for key_idx, key_name in enumerate(key_split): + if key_name == fqn_split[fqn_idx]: + fqn_idx += 1 + if fqn_idx == len(fqn_split): + return key_idx == len(key_split) - 1 + elif key_name in ("module", "_orig_mod"): + continue + else: + return False + return True + + if not verify(key, fqn): + raise RuntimeError(f"An unexpected key, {key}, exists. FQN is {fqn}") + state_dict[fqn] = state_dict.pop(key) + + if info.submodule_prefixes: + new_state_dict: dict[str, ValueType] = {} + # TODO: make this faster. + for fqn in state_dict.keys(): + for prefix in info.submodule_prefixes: + if not fqn.startswith(prefix): + continue + if info.keep_submodule_prefixes: + new_state_dict[fqn] = state_dict[fqn] + else: + new_fqn = fqn[len(prefix) :] + new_state_dict[new_fqn] = state_dict[fqn] + state_dict = new_state_dict + + if info.ignore_frozen_params: + for key, param in model.named_parameters(): + if param.requires_grad: + continue + fqns = _get_fqns(model, key) + for fqn in fqns: + state_dict.pop(fqn) + + for key, p in list(state_dict.items()): + if torch.is_tensor(p) and p.is_meta: + state_dict.pop(key) + + return _maybe_full_or_cpu_state_dict(state_dict, info) + + +@torch.no_grad() +def _load_model_state_dict( + model: nn.Module, + state_dict: dict[str, ValueType], + info: _StateDictInfo, +) -> _IncompatibleKeys: + if not info.handle_model or (not state_dict and not info.broadcast_from_rank0): + return _IncompatibleKeys({}, {}) + + local_state_dict = {} + for key, value in _iterate_valid_model_state(model, info.dsd_fqn_modifiers): + fqns = _get_fqns(model, key, info.dsd_fqn_modifiers) + fqns_with_prefix = _get_fqns( + model, + key, + info.dsd_fqn_modifiers, + skip_ddp_prefix=False, + skip_compiler_prefix=False, + ) + + for fqn, fqn_with_prefix in zip(fqns, fqns_with_prefix, strict=False): + if (not info.broadcast_from_rank0 or dist.get_rank() == 0) and fqn != fqn_with_prefix: + load_value = state_dict.pop(fqn, None) + if load_value is None: + if info.strict: + raise RuntimeError(f"Missing key: {fqn}.") + else: + state_dict[fqn_with_prefix] = load_value + local_state_dict[fqn_with_prefix] = value + + assign = False + if info.broadcast_from_rank0 or info.full_state_dict: + devices = set() + for key, value in local_state_dict.items(): + if torch.is_tensor(value) and value.dim() > 0: + devices.add(value.device) + # In lora state_dict, there could be multiple devices, with meta device inside. + # Take the other device in the broadcast/distribtue, and set assign to True + if torch.device("meta") in devices: + devices.remove(torch.device("meta")) + assign = True + if len(devices) == 0: + devices.add(dist.distributed_c10d._get_pg_default_device()) + elif len(devices) > 1: + raise ValueError("Multiple devices found") + + if info.broadcast_from_rank0: + _broadcast_state_dict( + state_dict, + local_state_dict, + device=devices.pop(), + strict=info.strict, + cpu_offload=info.cpu_offload, + ) + elif info.full_state_dict: + _distribute_state_dict(state_dict, local_state_dict, device=devices.pop()) + for fqn, local_state in local_state_dict.items(): + state_dict[fqn] = local_state + + with info.fsdp_context(): + return cast( + _IncompatibleKeys, + _state_dict_fn(model, "load_state_dict")(state_dict=state_dict, strict=info.strict, assign=assign), + ) + + +def _init_optim_state(optim: torch.optim.Optimizer) -> None: + """ + Initialize optim states by calling the step() with zero grads. + """ + if optim.state: + # The optimizer state is initialized. + return + + # There are some stateless optimizers like SGD. These optimizer will + # not return in the above condition. So if gradients exist, we should also + # return. If gradients do not exist, the following initialization should + # not disturb SGD because the gradients and lr are both zero. + for param_group in optim.param_groups: + for param in param_group[_PARAMS]: + if param.grad is not None: + return + + for param_group in optim.param_groups: + for param in param_group[_PARAMS]: + if param.requires_grad: + param.grad = torch.zeros_like(param) + + # Some optimizers will update parameters regardless of grads due to lr, so + # make lr to zero when calling `step()`. + lrs = [] + for param_group in optim.param_groups: + if "lr" in param_group: + lrs.append(param_group["lr"]) + param_group["lr"] = torch.tensor(0.0) if isinstance(param_group["lr"], torch.Tensor) else 0.0 + optim.step(closure=None) + # Whether to recover the "lr" should not matter too much as we will + # restore checkpointing later. + for param_group in optim.param_groups: + if "lr" in param_group: + param_group["lr"] = lrs.pop(0) + optim.zero_grad(set_to_none=True) + + +def _flatten_optim_state_dict(state_dict: OptimizerStateType) -> dict[str, ValueType]: + """ + This API flattens the optimizer state_dict to support optimizer resharding for + MPMD, e.g., pipeline parallelism. + + Without the API, the original optimizer state_dict looks like: + { + "state": { + "layer1.weight": { + "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor + }, + "layer2.weight": { + "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor + }, + }, + "param_group": [ + { + "lr": 0.0, + "betas": (0.9, 0.95), ..., + "params": ["layer1.weight", "layer2.weight"] + } + ] + } + + With this API, the optimizer state_dict looks like: + { + "state.layer1.weight.step": 10, + "state.layer2.weight.step": 10, + "state.layer1.weight.exp_avg": SomeTensor, + "state.layer2.weight.exp_avg": SomeTensor, + "state.layer1.weight.exp_avg_sq": SomeTensor, + "state.layer2.weight.exp_avg_sq": SomeTensor, + "param_group.layer1.weight.lr" : 0.1, + "param_group.layer2.weight.lr" : 0.1, + "param_group.layer1.weight.betas" : (0.9, 0.95), + "param_group.layer2.weight.betas" : (0.9, 0.95), + } + + Note that if any of the value is a container, like the betas in the example, + this API won't flattent it. + """ + + def _raise_if_type_not_supported(v): + if not isinstance(v, (torch.Tensor, int, float)): + raise NotImplementedError( + f"Flattening optimizer state_dict only supports tensor, int, float states now. Type is {type(v)}." + ) + + ret: dict[str, ValueType] = {} + for fqn, state in cast(DictValueType, state_dict[_STATE]).items(): + for k, v in cast(DictValueType, state).items(): + _raise_if_type_not_supported(v) + ret[f"{_STATE}.{fqn}.{k}"] = v + + for param_group in cast(ListDictValueType, state_dict[_PG]): + fqns = param_group.pop(_PARAMS) + for fqn in cast(list[str], fqns): + for k, v in param_group.items(): + ret[f"{_PG}.{fqn}.{k}"] = v + return ret + + +def _unflatten_optim_state_dict( + optim: torch.optim.Optimizer, + state_dict: dict[str, ValueType], + info: _StateDictInfo, +) -> OptimizerStateType: + """ + This API unflattens the state_dict generated by _flatten_optim_state_dict(). + See the docstring of _flatten_optim_state_dict() for more detail. + """ + state: DictValueType = {} + pg_state: ListDictValueType = [] + return_osd: OptimizerStateType = {_STATE: state, _PG: pg_state} + + for param_group in optim.param_groups: + pg_state.append({_PARAMS: []}) + for param in param_group[_PARAMS]: + for fqn in info.fqn_param_mapping[param]: + # If a parameter is shared, only one of the FQN will be used. + # So we need to verify which if this fqn is actually used in + # the state_dict. + if fqn in info.shared_params_mapping: + in_params = False + for k in param_group.keys(): + if k == _PARAMS: + continue + flatten_key = f"{_PG}.{fqn}.{k}" + if flatten_key in state_dict: + in_params = True + break + else: + in_params = True + + if not in_params: + continue + + params = pg_state[-1][_PARAMS] + assert isinstance(params, list) # typing + params.append(fqn) + if not param.requires_grad: + continue + state[fqn] = {} + for state_name in optim.state[param].keys(): + cast(DictValueType, state[fqn])[state_name] = state_dict[f"{_STATE}.{fqn}.{state_name}"] + + first_param_fqn = cast(list[str], pg_state[-1][_PARAMS])[0] + for k in param_group.keys(): + if k == _PARAMS: + continue + value = state_dict[f"{_PG}.{first_param_fqn}.{k}"] + if k not in pg_state[-1]: + pg_state[-1][k] = value + elif pg_state[-1][k] != value: + raise RuntimeError( + "All the parameters in the same parameter group should have " + f"the same saved param_group value. But {first_param_fqn}.{k} " + f"is {value} while other(s) is {pg_state[-1][k]}." + ) + + return return_osd + + +@torch.no_grad() +def _get_optim_state_dict( + model: nn.Module, + optimizers: tuple[torch.optim.Optimizer, ...], + info: _StateDictInfo, +) -> OptimizerStateType: + if not info.handle_optim: + return {} + + optim_state_dict: OptimizerStateType = {_STATE: {}, _PG: []} + for optim in optimizers: + _init_optim_state(optim) + osd = _state_dict_fn(optim, "state_dict")() + if info.fsdp_modules: + with info.fsdp_context(): + osd = FSDP.optim_state_dict(model, optim, osd) + + # We need to specially handle FlatParameter FSDP as + # FlatParameter FSDP converts the FQNs. + # There are no easy ways to do this conversion systematically. + # We can only use a string replacment without correctness check. + if not osd: + continue + for k in list(osd[_STATE].keys()): + if "_orig_mod" in k: + osd[_STATE][k.replace("_orig_mod.", "")] = osd[_STATE].pop(k) + for g in osd[_PG]: + params = [k.replace("_orig_mod.", "") for k in g[_PARAMS]] + g[_PARAMS] = params + else: + params = list(chain.from_iterable(g[_PARAMS] for g in optim.param_groups)) + param_pid_mapping = dict(zip(params, range(len(params)), strict=False)) + fqn_pid_mapping = {} + for key, param in model.named_parameters(): + fqns = _get_fqns(model, key) + assert len(fqns) == 1 + fqn = next(iter(fqns)) + if param not in param_pid_mapping: + continue + pid = param_pid_mapping[param] + fqn_pid_mapping[fqn] = pid + fqn_pid_mapping[pid] = fqn + + for key in list(osd[_STATE].keys()): + fqn = fqn_pid_mapping[key] + osd[_STATE][fqn] = osd[_STATE].pop(key) + + for group in osd[_PG]: + group[_PARAMS] = [fqn_pid_mapping[pid] for pid in group[_PARAMS]] + + if not osd: + continue + + cast(DictValueType, optim_state_dict[_STATE]).update(osd[_STATE]) + cast(ListDictValueType, optim_state_dict[_PG]).extend(osd[_PG]) + + if info.flatten_optimizer_state_dict: + optim_state_dict = cast(OptimizerStateType, _flatten_optim_state_dict(optim_state_dict)) + + return _maybe_full_or_cpu_state_dict(optim_state_dict, info) + + +def _split_optim_state_dict( + model: nn.Module, + optim: torch.optim.Optimizer, + optim_state_dict: OptimizerStateType, + info: _StateDictInfo, +) -> OptimizerStateType: + """ + Extract the corresponding optim state_dict from ``optim_state_dict`` for + ``optim`` and return the result optim state_dict. + + Args: + model (nn.Module): the root model. + optim (torch.optim.Optimizer): the optimizer. + optim_state_dict (Dict[str, ValueType]): the superset optim state_dict that + contains the optim state_dict of ``optim``. + info (_StateDictInfo): state dict information. + + Returns: + The optim state_dict of ``optim``. + """ + + state: DictValueType = {} + pg_state: ListDictValueType = [] + return_osd: OptimizerStateType = {_STATE: state, _PG: pg_state} + pg_mapping: dict[int, int] = {} + + if all(isinstance(k, int) for k in cast(DictValueType, optim_state_dict[_STATE]).keys()): + return optim_state_dict + + for param_group in optim.param_groups: + pg_state.append({_PARAMS: []}) + for param in param_group[_PARAMS]: + for fqn in info.fqn_param_mapping[param]: + if fqn in info.shared_params_mapping: + in_params = False + for loaded_param_group in cast(ListDictValueType, optim_state_dict[_PG]): + if fqn in cast(list[str], loaded_param_group[_PARAMS]): + in_params = True + break + else: + in_params = True + if not in_params: + continue + + params = pg_state[-1][_PARAMS] + assert isinstance(params, list) + params.append(fqn) + if param.requires_grad: + state[fqn] = cast(DictValueType, optim_state_dict[_STATE])[fqn] + for loaded_param_group in cast(ListDictValueType, optim_state_dict[_PG]): + if fqn in cast(list[str], loaded_param_group[_PARAMS]): + pg_mapping[id(loaded_param_group)] = len(return_osd[_PG]) - 1 + + if len(param_group[_PARAMS]) == 0: + # Param_group with empty params. + ret = [] + for loaded_param_group in cast(ListDictValueType, optim_state_dict[_PG]): + if len(cast(list[str], loaded_param_group[_PARAMS])) == 0: + ret.append(loaded_param_group) + if len(ret) != 1: + raise ValueError( + "There are param groups that have zero parameters. " + "In such a case, DSD only support exactly one param group " + "with zero parameters." + "But the loaded state_dict has zero or more than one param groups " + "that have zero parameters." + ) + if len(optim_state_dict[_PG]) != len(optim.param_groups): + raise ValueError( + "When there is a parameter group that has zero parameters, multiple optimizers are not supported." + ) + pg_mapping[id(loaded_param_group)] = len(return_osd[_PG]) - 1 + + for param_group in cast(ListDictValueType, optim_state_dict[_PG]): + pg_idx = pg_mapping.get(id(param_group), -1) + if pg_idx == -1: + continue + + for key, value in param_group.items(): + if key == _PARAMS: + continue + # TODO: check if value is the same if exists. + pg_state[pg_idx][key] = value + + return return_osd + + +@torch.no_grad() +def _load_optim_state_dict( + model: nn.Module, + optimizers: tuple[torch.optim.Optimizer, ...], + state_dict: OptimizerStateType, + info: _StateDictInfo, +) -> None: + if not info.handle_optim: + return + + for optim in optimizers: + _init_optim_state(optim) + if state_dict: + if _STATE in state_dict: + optim_state_dict = _split_optim_state_dict(model, optim, state_dict, info) + else: + optim_state_dict = _unflatten_optim_state_dict(optim, cast(dict[str, ValueType], state_dict), info) + else: + optim_state_dict = {} + if info.fsdp_modules: + # We need to specially handle FlatParameter FSDP as + # FlatParameter FSDP converts the FQNs. + for original_fqn, _ in model.named_parameters(): + fqns = _get_fqns(model, original_fqn) + fqns_with_compiler = _get_fqns(model, original_fqn, skip_compiler_prefix=False) + if fqns == fqns_with_compiler: + continue + + assert len(fqns) == 1 + fqn = fqns.pop() + fqn_with_compiler = fqns_with_compiler.pop() + for g in optim_state_dict[_PG]: + val = cast(dict[str, Any], g) + params = [key.replace(fqn, fqn_with_compiler) for key in val[_PARAMS]] + val[_PARAMS] = params + osd_state = cast(DictValueType, optim_state_dict[_STATE]) + for k in list(osd_state.keys()): + if fqn in k: + osd_state[k.replace(fqn, fqn_with_compiler)] = osd_state.pop(k) + + with info.fsdp_context(): + optim_state_dict = FSDP.optim_state_dict_to_load(model, optim, optim_state_dict) + elif info.full_state_dict: + info.full_state_dict = False + local_state_dict = _get_optim_state_dict(model, (optim,), info) + info.full_state_dict = True + device = None + + def _device(t): + if t.dim() > 0: + nonlocal device + if device is None: + device = t.device + elif device != t.device: + raise ValueError("Device mismatch") + return t + + _ = tree_map_only(torch.Tensor, _device, local_state_dict) + assert device is not None + flatten_osd, osd_mapping = _flatten_state_dict(optim_state_dict) + flatten_local_osd, local_osd_mapping = _flatten_state_dict(local_state_dict) + if info.broadcast_from_rank0: + _broadcast_state_dict(flatten_osd, flatten_local_osd, device=device) + else: + _distribute_state_dict(flatten_osd, flatten_local_osd, device=device) + # The modifications listed seek to address the problem where optim might possess + # dissimilar parameters in comparison to optim_state_dict. This is achieved by + # incorporating differential parameters within local, which may result in optim + # having additional parameters ultimately. + for optim_key in flatten_osd.keys(): + if optim_key not in flatten_local_osd: + assert optim_key in osd_mapping + flatten_local_osd[optim_key] = flatten_osd[optim_key] + local_osd_mapping[optim_key] = osd_mapping[optim_key] + optim_state_dict = _unflatten_state_dict(flatten_local_osd, local_osd_mapping) + for pg in optim_state_dict[_PG]: + if _PARAMS not in pg: + cast(dict[str, ValueType], pg)[_PARAMS] = [] + + # Note that we do not have to convert the FQN back to param id here if + # order in optim.param_groups[idx][_PARAMS] is the same as the one in + # optim_state_dict[_PG][idx][_PARAMS]. + _state_dict_fn(optim, "load_state_dict")(state_dict=optim_state_dict) + + +def get_model_state_dict( + model: nn.Module, + *, + submodules: Optional[set[nn.Module]] = None, + options: Optional[StateDictOptions] = None, +) -> dict[str, ValueType]: + """ + Return the model state_dict of ``model``. + + See ``get_state_dict`` for the detail usage. + + Args: + model (nn.Module): the nn.Module to the model. + submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters + that belong to the submodules. + options (StateDictOptions): the options to control how + model state_dict and optimizer state_dict should be returned. See + `StateDictOptions` for the details. + + Returns: + The state_dict for ``model``. + + :rtype: typing.Dict[str, ValueType] + """ + with _gc_context(): + info = _verify_options( + model, + (), + optim_only=False, + submodules=submodules, + options=options, + ) + model_state_dict = _get_model_state_dict(model, info) + _verify_state_dict(model_state_dict, {}, info) + return model_state_dict + + +def get_optimizer_state_dict( + model: nn.Module, + optimizers: torch.optim.Optimizer | Iterable[torch.optim.Optimizer], + *, + submodules: Optional[set[nn.Module]] = None, + options: Optional[StateDictOptions] = None, +) -> OptimizerStateType: + """ + Return the combined state_dict for optimizers. + + See ``get_state_dict`` for the detail usage. + + Args: + model (nn.Module): the nn.Module to the model. + optimizers (Union[None, Optimizer, Iterable[Optimizer]]): + The optimizers that are used to optimize ``model``. + submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters + that belong to the submodules. + options (StateDictOptions): the options to control how + model state_dict and optimizer state_dict should be returned. See + `StateDictOptions` for the details. + + Returns: + The state_dict for ``optimizers``. + + :rtype: OptimizerStateType + """ + with _gc_context(): + optimizers = (optimizers,) if isinstance(optimizers, torch.optim.Optimizer) else tuple(optimizers) + info = _verify_options( + model, + optimizers, + optim_only=True, + submodules=submodules, + options=options, + ) + optim_state_dict = _get_optim_state_dict(model, optimizers, info) + _verify_state_dict({}, optim_state_dict, info) + return optim_state_dict + + +def get_state_dict( + model: nn.Module, + optimizers: torch.optim.Optimizer | Iterable[torch.optim.Optimizer], + *, + submodules: Optional[set[nn.Module]] = None, + options: Optional[StateDictOptions] = None, +) -> tuple[dict[str, ValueType], OptimizerStateType]: + """ + Return the model state_dict and optimizers state_dict. + + ``get_state_dict`` can process any module that is parallelized by PyTorch + FSDP/fully_shard, DDP/replicate, tensor_parallel/parallelize_module, and any + combination of these parallelisms. The main functions of ``get_state_dict`` + are: 1.) returning a model and optimizer state_dict that can be resharded + with a different number of trainers and/or different parallelisms. + 2.) hiding the parallelism-specific state_dict APIs. Users don't have to call + these APIs. + 3.) sanity checking the result state_dict. + + The keys of the result state dictionary are the canonical FQNs (Fully + Qualified Names). A canonical FQN refers to the FQN based on a parameter's + position in an nn.Module hierarchy. More specifically, a canonical FQN to a + parameter is the FQN returned by ``module.named_parameters()`` or + ``module.named_buffers()`` when the module is not distributed by any + parallelisms. Since the optimizer internally uses parameter IDs to represent + a parameter, there will be a conversion from the parameter IDs to the + canonical FQNs when calling this API. + + ``get_state_dict`` can also process a module that is not parallelized. In + such a case, ``get_state_dict`` only performs one function -- converting the + optimizer parameter IDs to the canonical FQNs. + + Example: + >>> # xdoctest: +SKIP + >>> import torch + >>> from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + >>> from torch.nn.parallel import DistributedDataParallel as DDP + >>> from torch.distributed.checkpoint.state_dict import get_state_dict + + >>> fsdp_model = FSDP(copy.deepcopy(model)) + >>> fsdp_optim = torch.optim.Adam(model.parameters(), lr=1e-3) + >>> ddp_model = DDP(copy.deepcopy(model)) + >>> ddp_optim = torch.optim.Adam(model.parameters(), lr=1e-3) + + + >>> ddp_state_dict, ddp_optim_state_dict = get_state_dict(ddp_model, ddp_optim) + >>> fsdp_state_dict, fsdp_optim_state_dict = get_state_dict( + ... fsdp_model, fsdp_optim + ... ) + + >>> # if we simply call ddp_model.state_dict() and fsdp_model.state_dict(), + >>> # the asserts will fail. + >>> assert ddp_state_dict == fsdp_state_dict + >>> assert ddp_optim_state == fsdp_optim_state_dict + + + Args: + model (nn.Module): the nn.Module to the model. + optimizers (Union[None, Optimizer, Iterable[Optimizer]]): + The optimizers that are used to optimize ``model``. + submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters + that belong to the submodules. + options (StateDictOptions): the options to control how + model state_dict and optimizer state_dict should be returned. See + `StateDictOptions` for the details. + + Returns: + ``Tuple`` that contain model state_dict and optimizer state_dict. + + :rtype: typing.Tuple[typing.Dict[str, ValueType], OptimizerStateType] + """ + + with _gc_context(): + optimizers = (optimizers,) if isinstance(optimizers, torch.optim.Optimizer) else tuple(optimizers) + info = _verify_options( + model, + optimizers, + optim_only=False, + submodules=submodules, + options=options, + ) + model_state_dict = _get_model_state_dict(model, info) + optim_state_dict = _get_optim_state_dict(model, optimizers, info) + _verify_state_dict(model_state_dict, optim_state_dict, info) + return model_state_dict, optim_state_dict + + +def _unflatten_model_state_dict( + model: nn.Module, + state_dict: dict[nn.Module, dict[str, ValueType]] | dict[str, ValueType], +) -> dict[str, ValueType]: + if not state_dict: + return {} + + if isinstance(next(iter(state_dict.keys())), nn.Module): + warnings.warn( + "Passing model_state_dict as a ``Dict[nn.Module, Dict[str, Any]]``" + "is deprecated and will be removed in 2.5. If you need this " + "feature, please preprocessing the model_state_dict to achieve the " + "same functionality.", + FutureWarning, + ) + cast_state_dict = cast(dict[nn.Module, dict[str, ValueType]], state_dict) + new_state_dict: dict[str, ValueType] = {} + for submodule, sub_state_dict in cast_state_dict.items(): + for name, m in model.named_modules(): + if m != submodule: + continue + + fqns = _get_fqns(model, name) + assert len(fqns) == 1, "FQNs for a submodule should only have 1 element" + prefix = f"{next(iter(fqns))}." + new_state_dict.update({prefix + subfqn: value for subfqn, value in sub_state_dict.items()}) + return new_state_dict + else: + return cast(dict[str, ValueType], state_dict) + + +def set_model_state_dict( + model: nn.Module, + model_state_dict: dict[str, ValueType], + *, + options: Optional[StateDictOptions] = None, +) -> _IncompatibleKeys: + """Load the model state_dict. + + The counterpart of ``get_model_state_dict`` to set the state_dict to the + model. See ``set_state_dict`` for the detail usage. + + Args: + model (nn.Module): the nn.Module to the model. + model_state_dict: (Dict[str, ValueType]): + the model state_dict to load. If the key of the ``model_state_dict`` + is nn.Module, the key is a submodule of ``model`` and the value should + be the state_dict of the submodule. When loading the state_dict, + the prefix of the submodule will be append to the state_dict. + options (StateDictOptions): the options to control how + model state_dict and optimizer state_dict should be loaded. See + `StateDictOptions` for the details. + + Returns: + ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields: + * **missing_keys** is a list of str containing the missing keys + * **unexpected_keys** is a list of str containing the unexpected keys + + :type model_state_dict: typing.Dict[str, ValueType] + """ + model_state_dict: dict[str, ValueType] = _unflatten_model_state_dict(model, model_state_dict) + with _gc_context(): + info = _verify_options(model, (), optim_only=False, options=options) + + _verify_state_dict(model_state_dict, {}, info) + return _load_model_state_dict(model, model_state_dict, info) + + +def set_optimizer_state_dict( + model: nn.Module, + optimizers: torch.optim.Optimizer | Iterable[torch.optim.Optimizer], + optim_state_dict: OptimizerStateType, + *, + options: Optional[StateDictOptions] = None, +) -> None: + """Load the optimizers state_dict. + + The counterpart of ``get_optimizer_state_dict`` to set the state_dict to the + optimizers. See ``set_state_dict`` for the detail usage. + + WARN: ``set_optimizer_state_dict`` can only be called before ``backward()`` or after + ``step()`` is called on the optimizers. Otherwise, the optimizer states won't be + initialized correctly. + + Args: + model (nn.Module): the nn.Module to the model. + optimizers (Union[Optimizer, Iterable[Optimizer]]): + The optimizers that are used to optimize ``model``. + optim_state_dict: OptimizerStateType: + the optimizer state_dict to load. + options (StateDictOptions): the options to control how + model state_dict and optimizer state_dict should be loaded. See + `StateDictOptions` for the details. + + Returns: + None + + :type optim_state_dict: typing.OptimizerStateType + """ + with _gc_context(): + optimizers = (optimizers,) if isinstance(optimizers, torch.optim.Optimizer) else tuple(optimizers) + info = _verify_options(model, optimizers, optim_only=True, options=options) + + _verify_state_dict({}, optim_state_dict, info) + _load_optim_state_dict(model, optimizers, optim_state_dict, info) + + +def set_state_dict( + model: nn.Module, + optimizers: torch.optim.Optimizer | Iterable[torch.optim.Optimizer], + *, + model_state_dict: dict[str, ValueType], + optim_state_dict: OptimizerStateType, + options: Optional[StateDictOptions] = None, +) -> _IncompatibleKeys: + """Load the model state_dict and optimizers state_dict. + + The counterpart of ``get_state_dict`` to set the state_dict to the model and + optimizers. The given ``model_state_dict`` and ``optim_state_dict`` do not + have to be returned by ``get_state_dict`` but must meet the following + requirements: 1) all FQNs are canonical FQNs as defined in ``get_state_dict``, + 2) if a tensor is sharded, it must be either a ShardedTensor or DTensor, + 3) optimizer state_dict cannot contain the parameter IDs; the keys should be + the canonical FQNs. + + WARN: ``set_state_dict`` can only be called before ``backward()`` or after ``step()`` + is called on the optimizers. Otherwise, the optimizer states won't be initialized + correctly. + + Args: + model (nn.Module): the nn.Module to the model. + optimizers (Union[Optimizer, Iterable[Optimizer]]): + The optimizers that are used to optimize ``model``. + model_state_dict: (Union[Dict[nn.Module, Dict[str, ValueType]], Dict[str, ValueType]]): + the model state_dict to load. If the key of the ``model_state_dict`` + is nn.Module, the key is a submodule of ``model`` and the value should + be the state_dict of the submodule. When loading the state_dict, + the prefix of the submodule will be append to the state_dict. + optim_state_dict: OptimizerStateType: + the optimizer state_dict to load. + options (StateDictOptions): the options to control how + model state_dict and optimizer state_dict should be loaded. See + `StateDictOptions` for the details. + + Returns: + ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields: + * **missing_keys** is a list of str containing the missing keys of the model state_dict. + * **unexpected_keys** is a list of str containing the unexpected keys of the model state_dict. + + :type model_state_dict: typing.Dict[str, ValueType] + :type optim_state_dict: typing.OptimizerStateType + """ + + model_state_dict: dict[str, ValueType] = _unflatten_model_state_dict(model, model_state_dict) + with _gc_context(): + optimizers = (optimizers,) if isinstance(optimizers, torch.optim.Optimizer) else tuple(optimizers) + info = _verify_options(model, optimizers, optim_only=not model_state_dict, options=options) + + _verify_state_dict(model_state_dict, optim_state_dict, info) + _load_optim_state_dict(model, optimizers, optim_state_dict, info) + return _load_model_state_dict(model, model_state_dict, info) + + +# TODO: correct the state_dict function signature. +# TODO: this API is not yet fully tested. Make it private +@no_type_check +def _patch_model_state_dict( + model: nn.Module, + *, + options: Optional[StateDictOptions] = None, +) -> None: + """Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model``. + + Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model`` to + be a partial function to call ``get_state_dict`` and ``set_state_dict``. + + Example: + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + from torch.distributed.checkpoint.state_dict import patch_model_state_dict + + model = fsdp(model) + patch_model_state_dict(model) + + Args: + model (nn.Module): the nn.Module to the model. + options (StateDictOptions): the options to control how + model state_dict and optimizer state_dict should be loaded. See + `StateDictOptions` for the details. + Returns: + None + """ + + _state_dict_call = functools.partial( + get_model_state_dict, + model=model, + options=options, + ) + + def state_dict_call(): + return _state_dict_call() + + model.state_dict = state_dict_call + + _load_state_dict_call = functools.partial( + set_model_state_dict, + model=model, + options=options, + ) + + def load_state_dict_call(state_dict: dict[str, Any]): + _load_state_dict_call(model_state_dict=state_dict) + + model.load_state_dict = load_state_dict_call + + _patched_state_dict.add(state_dict_call) + _patched_state_dict.add(load_state_dict_call) + + +# TODO: correct the load_state_dict function signature. +# TODO: this API is not yet fully tested. Make it private +@no_type_check +def _patch_optimizer_state_dict( + model: nn.Module, + *, + optimizers: tuple[torch.optim.Optimizer, ...], + options: Optional[StateDictOptions] = None, +) -> None: + """Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers``. + + Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers`` to + be a partial function to call ``get_state_dict`` and ``set_state_dict``. + + Note that if there are multiple optimizers, all of the optimizers will be patched. + So users only need to call one of the state_dict() to get the full result. + + Example: + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + from torch.distributed.checkpoint.state_dict import patch_model_state_dict + + model = fsdp(model) + patch_model_state_dict(model) + + Args: + model (nn.Module): the nn.Module to the model. + options (StateDictOptions): the options to control how + model state_dict and optimizer state_dict should be loaded. See + `StateDictOptions` for the details. + Returns: + None + """ + + _state_dict_call = functools.partial( + get_optimizer_state_dict, + model=model, + optimizers=optimizers, + options=options, + ) + + def state_dict_call(): + return _state_dict_call() + + _load_state_dict_call = functools.partial( + set_optimizer_state_dict, + model=model, + optimizers=optimizers, + options=options, + ) + + def load_state_dict_call(state_dict: dict[str, Any]): + _load_state_dict_call(optim_state_dict=state_dict) + + _patched_state_dict.add(state_dict_call) + _patched_state_dict.add(load_state_dict_call) + optimizers = (optimizers,) if isinstance(optimizers, torch.optim.Optimizer) else tuple(optimizers) + for optim in optimizers: + optim.state_dict = state_dict_call + optim.load_state_dict = load_state_dict_call diff --git a/e2c/verl/verl/third_party/vllm/__init__.py b/e2c/verl/verl/third_party/vllm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2f6646f3b6939851190bc9ecf6b6e0b1cb8e63d5 --- /dev/null +++ b/e2c/verl/verl/third_party/vllm/__init__.py @@ -0,0 +1,64 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from importlib.metadata import PackageNotFoundError, version + +from packaging import version as vs + +from verl.utils.device import is_npu_available +from verl.utils.import_utils import is_sglang_available + + +def get_version(pkg): + try: + return version(pkg) + except PackageNotFoundError: + return None + + +package_name = "vllm" +package_version = get_version(package_name) +vllm_version = None +VLLM_SLEEP_LEVEL = 1 + +if package_version is None: + if not is_sglang_available(): + raise ValueError( + f"vllm version {package_version} not supported and SGLang also not Found. Currently supported " + f"vllm versions are 0.7.0+" + ) +elif is_npu_available: + # sleep_mode=2 is not supported on vllm-ascend for now, will remove this restriction when this ability is ready. + VLLM_SLEEP_LEVEL = 1 + from vllm import LLM + from vllm.distributed import parallel_state +elif vs.parse(package_version) >= vs.parse("0.7.0"): + vllm_version = package_version + if vs.parse(package_version) >= vs.parse("0.8.5"): + VLLM_SLEEP_LEVEL = 2 + from vllm import LLM + from vllm.distributed import parallel_state +else: + if vs.parse(package_version) in [vs.parse("0.5.4"), vs.parse("0.6.3")]: + raise ValueError( + f"vLLM version {package_version} support has been removed. vLLM 0.5.4 and 0.6.3 are no longer " + f"supported. Please use vLLM 0.7.0 or later." + ) + if not is_sglang_available(): + raise ValueError( + f"vllm version {package_version} not supported and SGLang also not Found. Currently supported " + f"vllm versions are 0.7.0+" + ) + +__all__ = ["LLM", "parallel_state"] diff --git a/e2c/verl/verl/tools/__init__.py b/e2c/verl/verl/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c4b932b1ae7eeeb4c53c98c684cf0ba9b670a86b --- /dev/null +++ b/e2c/verl/verl/tools/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/tools/base_tool.py b/e2c/verl/verl/tools/base_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..bec813a51870de77b1179808d98c289f46ddc609 --- /dev/null +++ b/e2c/verl/verl/tools/base_tool.py @@ -0,0 +1,93 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from typing import Any, Optional +from uuid import uuid4 + +from verl.utils.rollout_trace import rollout_trace_op + +from .schemas import OpenAIFunctionToolSchema, ToolResponse + + +class BaseTool: + """Base class for tools. + + A tool should support the following methods: + + - `get_openai_tool_schema`: return the tool schema in OpenAI format. + - `create`: create a tool instance for a trajectory. + - `execute`: execute the tool. + - `calc_reward`: calculate the reward respect to tool state. + - `release`: release the tool instance. + """ + + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + self.config = config + self.tool_schema = tool_schema or self.get_openai_tool_schema() + assert self.tool_schema is not None, "Tool schema is not set!" + self.name = self.tool_schema.function.name + print(json.dumps(self.tool_schema.model_dump(exclude_unset=True, exclude_none=True), indent=2)) + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + return self.tool_schema + + async def create(self, instance_id: Optional[str] = None, **kwargs) -> tuple[str, ToolResponse]: + """Create a tool instance. + + Args: + instance_id: The instance id of the tool. + + Returns: + The instance id of the tool. + tool_creation_response: The response of the tool when creating the instance. + """ + if instance_id is None: + return str(uuid4()), ToolResponse() + else: + return instance_id, ToolResponse() + + @rollout_trace_op + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + """Execute the tool. + + Args: + instance_id: The instance id of the tool. + parameters: The json string of the parameters of the tool. + + Returns: tool_response, tool_reward_score, tool_metrics + tool_response: The ToolResponse object containing text, image, and/or video content. + tool_reward_score: The step reward score of the tool. + tool_metrics: The metrics of the tool. + """ + return ToolResponse(text="Updated the tool state."), 0.0, {} + + async def calc_reward(self, instance_id: str, **kwargs) -> float: + """Calculate the reward of the tool. + + Args: + instance_id: The instance id of the tool. + + Returns: + The reward of the tool. + """ + return 0.0 + + async def release(self, instance_id: str, **kwargs) -> None: + """Release the tool instance. + + Args: + instance_id: The instance id of the tool. + """ + pass diff --git a/e2c/verl/verl/tools/geo3k_tool.py b/e2c/verl/verl/tools/geo3k_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..9697c757ee97668e3dfa3b9529ffa25016940b3c --- /dev/null +++ b/e2c/verl/verl/tools/geo3k_tool.py @@ -0,0 +1,101 @@ +# Copyright 2023-2025 SGLang Team +# Copyright Amazon.com, Inc. or its affiliates. +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from typing import Any, Optional +from uuid import uuid4 + +from verl.utils.reward_score import geo3k +from verl.utils.rollout_trace import rollout_trace_op + +from .base_tool import BaseTool +from .schemas import OpenAIFunctionToolSchema, ToolResponse + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class Geo3kTool(BaseTool): + """A demo tool for calculating the reward of geo3k. + - `get_openai_tool_schema`: return the tool schema in OpenAI format. + - `create`: create a tool instance for a trajectory. + - `execute`: execute the tool. + - `calc_reward`: calculate the reward respect to tool state. + - `release`: release the tool instance. + """ + + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + """ + _tool_schema = OpenAIFunctionToolSchema.model_validate({ + "type": "function", + "function": { + "name": "calc_geo3k_reward", + "description": "A tool for calculating the reward of geo3k", + "parameters": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The answer to the question, enclosed in \\boxed{}", + }, + }, + "required": ["answer"], + }, + } + }) + """ + super().__init__(config, tool_schema) + self._instance_dict = {} + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + return self.tool_schema + + async def create( + self, instance_id: Optional[str] = None, ground_truth: Optional[str] = None, **kwargs + ) -> tuple[str, ToolResponse]: + if instance_id is None: + instance_id = str(uuid4()) + self._instance_dict[instance_id] = { + "response": "", + "ground_truth": ground_truth, + "reward": 0.0, + } + return instance_id, ToolResponse() + + @rollout_trace_op + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + answer = parameters.get("answer", "") + if not isinstance(answer, str): + answer = str(answer) + self._instance_dict[instance_id]["response"] = answer + reward = await self.calc_reward(instance_id) + # penalty for non improved answer submission + tool_reward = 0.0 if reward > self._instance_dict[instance_id]["reward"] else -0.05 + # update the reward + self._instance_dict[instance_id]["reward"] = reward + return ToolResponse(text=f"Current parsed {answer=} {reward=}"), tool_reward, {} + + async def calc_reward(self, instance_id: str, **kwargs) -> float: + return geo3k.compute_score( + self._instance_dict[instance_id]["response"], + self._instance_dict[instance_id]["ground_truth"], + use_boxed=False, + format_score=0.0, + ) + + async def release(self, instance_id: str, **kwargs) -> None: + del self._instance_dict[instance_id] diff --git a/e2c/verl/verl/tools/gsm8k_tool.py b/e2c/verl/verl/tools/gsm8k_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..b9e6f0e66d48b9b2b95a72227b9b87828b280629 --- /dev/null +++ b/e2c/verl/verl/tools/gsm8k_tool.py @@ -0,0 +1,110 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from typing import Any, Optional +from uuid import uuid4 + +from verl.utils.reward_score import gsm8k +from verl.utils.rollout_trace import rollout_trace_op + +from .base_tool import BaseTool +from .schemas import OpenAIFunctionToolSchema, ToolResponse + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class Gsm8kTool(BaseTool): + """A demo tool for calculating the reward of gsm8k. + + - `get_openai_tool_schema`: return the tool schema in OpenAI format. + - `create`: create a tool instance for a trajectory. + - `execute`: execute the tool. + - `calc_reward`: calculate the reward respect to tool state. + - `release`: release the tool instance. + """ + + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + """ + _tool_schema = OpenAIFunctionToolSchema.model_validate({ + "type": "function", + "function": { + "name": "calc_gsm8k_reward", + "description": "A tool for calculating the reward of gsm8k", + "parameters": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The answer to the question", + }, + }, + "required": ["answer"], + }, + } + }) + """ + super().__init__(config, tool_schema) + self._instance_dict = {} + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + return self.tool_schema + + async def create( + self, instance_id: Optional[str] = None, ground_truth: Optional[str] = None, **kwargs + ) -> tuple[str, ToolResponse]: + if instance_id is None: + instance_id = str(uuid4()) + if ground_truth is None: + ground_truth = kwargs.get("create_kwargs", {}).get("ground_truth", None) + self._instance_dict[instance_id] = { + "response": "", + "ground_truth": ground_truth, + "reward": 0.0, + } + return instance_id, ToolResponse() + + @rollout_trace_op + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + answer = parameters.get("answer", "") + if not isinstance(answer, str): + answer = str(answer) + + if answer.startswith("#### "): + self._instance_dict[instance_id]["response"] = answer + else: + self._instance_dict[instance_id]["response"] = "#### " + answer + + reward = await self.calc_reward(instance_id) + # penalty for non improved answer submission + tool_reward = 0.0 if reward > self._instance_dict[instance_id]["reward"] else -0.05 + # update the reward + self._instance_dict[instance_id]["reward"] = reward + + return ToolResponse(text=f"Current parsed {answer=} {reward=}"), tool_reward, {} + + async def calc_reward(self, instance_id: str, **kwargs) -> float: + return gsm8k.compute_score( + self._instance_dict[instance_id]["response"], + self._instance_dict[instance_id]["ground_truth"], + method="flexible", + format_score=0.0, + score=1.0, + ) + + async def release(self, instance_id: str, **kwargs) -> None: + del self._instance_dict[instance_id] diff --git a/e2c/verl/verl/tools/image_zoom_in_tool.py b/e2c/verl/verl/tools/image_zoom_in_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..07529478b3b716d89158defe7aa996958c4621ec --- /dev/null +++ b/e2c/verl/verl/tools/image_zoom_in_tool.py @@ -0,0 +1,392 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import threading +from contextlib import ExitStack +from enum import Enum +from math import ceil, floor +from typing import Any, Callable, Optional, TypeVar +from uuid import uuid4 + +import ray +import ray.actor +from qwen_vl_utils import fetch_image + +from .base_tool import BaseTool +from .schemas import OpenAIFunctionToolSchema, ToolResponse + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +T = TypeVar("T") + + +# Adapted from verl/tools/sandbox_fusion_tools.py +class PoolMode(Enum): + """Execution pool mode enumeration.""" + + ThreadMode = 1 + ProcessMode = 2 + + +@ray.remote(concurrency_groups={"acquire": 1, "release": 10}) +class TokenBucketWorker: + """Ray actor for rate limiting using token bucket algorithm.""" + + def __init__(self, rate_limit: int): + self.rate_limit = rate_limit + self.current_count = 0 # For observability + self._semaphore = threading.Semaphore(rate_limit) + + @ray.method(concurrency_group="acquire") + def acquire(self): + """Acquire a token from the bucket.""" + self._semaphore.acquire() + self.current_count += 1 + + @ray.method(concurrency_group="release") + def release(self): + """Release a token back to the bucket.""" + self._semaphore.release() + self.current_count -= 1 + + def get_current_count(self): + """Get current number of acquired tokens.""" + return self.current_count + + +class VisualExecutionWorker: + """Worker for executing visual processing operations with optional rate limiting.""" + + def __init__(self, enable_global_rate_limit=True, rate_limit=10): + self.rate_limit_worker = self._init_rate_limit(rate_limit) if enable_global_rate_limit else None + + def _init_rate_limit(self, rate_limit): + """Initialize singleton rate limiter.""" + return TokenBucketWorker.options(name="rate-limiter", get_if_exists=True).remote(rate_limit) + + def ping(self): + """Health check method.""" + return True + + def execute(self, fn: Callable[..., T], *fn_args, **fn_kwargs) -> T: + """Execute function with optional rate limiting.""" + if self.rate_limit_worker: + with ExitStack() as stack: + stack.callback(self.rate_limit_worker.release.remote) + ray.get(self.rate_limit_worker.acquire.remote()) + try: + return fn(*fn_args, **fn_kwargs) + except Exception as e: + # TODO we should make this available to the tool caller + logger.warning(f"Error when executing visual processing: {e}") + else: + return fn(*fn_args, **fn_kwargs) + + +def init_visual_execution_pool( + num_workers: int, enable_global_rate_limit=True, rate_limit=10, mode: PoolMode = PoolMode.ThreadMode +): + """Initialize visual execution pool.""" + if mode == PoolMode.ThreadMode: + return ( + ray.remote(VisualExecutionWorker) + .options(max_concurrency=num_workers) + .remote(enable_global_rate_limit=enable_global_rate_limit, rate_limit=rate_limit) + ) + else: + raise NotImplementedError("Process mode is not implemented yet") + + +class ImageZoomInTool(BaseTool): + """A tool for zooming in on an image by cropping it based on a bounding box. + + This tool provides a zoom-in functionality by cropping a region from an image, + with rate limiting and concurrent execution support through Ray. + + Methods: + get_openai_tool_schema: Return the tool schema in OpenAI format + create: Create a tool instance for a trajectory + execute: Execute the zoom-in operation + calc_reward: Calculate the reward with respect to tool state + release: Release the tool instance + """ + + MIN_DIMENSION = 28 + + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + """ + _tool_schema = OpenAIFunctionToolSchema.model_validate({ + "type": "function", + "function": { + "name": "image_zoom_in_tool", + "description": ( + "Zoom in on a specific region of an image by cropping it based on a bounding box (bbox) and an " + "optional object label." + ), + "parameters": { + "type": "object", + "properties": { + "bbox_2d": { + "type": "array", + "items":{"type":"number"}, + "minItems":4, + "maxItems":4, + "description": ( + "The bounding box of the region to zoom in, as [x1, y1, x2, y2], where (x1, y1) is " + "the top-left corner and (x2, y2) is the bottom-right corner." + ), + }, + "label": { + "type": "string", + "description": "The name or label of the object in the specified bounding box (optional).", + }, + }, + "required": ["bbox_2d"], + }, + } + }) + """ + super().__init__(config, tool_schema) + self._instance_dict = {} + + # Worker and rate limiting configuration + self.num_workers = config.get("num_workers", 20) + self.rate_limit = config.get("rate_limit", 50) + self.timeout = config.get("timeout", 30) + + self.enable_global_rate_limit = config.get("enable_global_rate_limit", True) + self.execution_pool = init_visual_execution_pool( + num_workers=self.num_workers, + enable_global_rate_limit=self.enable_global_rate_limit, + rate_limit=self.rate_limit, + mode=PoolMode.ThreadMode, + ) + logger.info(f"Initialized ImageZoomInTool with config: {config}") + + def _validate_bbox(self, left: float, top: float, right: float, bottom: float) -> bool: + """Validate the bounding box dimensions and aspect ratio.""" + try: + if not (left < right and top < bottom): + logger.warning(f"Invalid bbox shape: left={left}, top={top}, right={right}, bottom={bottom}") + return False + + height = bottom - top + width = right - left + + # Prevent division by zero for zero-sized boxes + if min(height, width) == 0: + logger.warning(f"Bbox has zero width or height: left={left}, top={top}, right={right}, bottom={bottom}") + return False + + if max(height, width) / min(height, width) > 100: + logger.warning(f"Bbox aspect ratio > 100: left={left}, top={top}, right={right}, bottom={bottom}") + return False + + return True + except Exception as e: + logger.warning(f"Bbox validation error: {e}") + return False + + def _maybe_resize_bbox(self, bbox_2d: list[float], image_width: int, image_height: int) -> Optional[list[float]]: + """ + Clamp, validate, and potentially resize a bounding box. + + This function ensures the final bounding box is within image bounds and meets the minimum + dimension requirements. If the initial box is too small, it attempts to expand it + from its center. It performs a final check to guarantee the output dimensions are valid. + + Returns: + A valid bounding box as a list of coordinates, or None if validation fails. + """ + left, top, right, bottom = bbox_2d + + # 1. Clamp the initial bounding box to the image dimensions. + left = max(0.0, float(left)) + top = max(0.0, float(top)) + right = min(float(image_width), float(right)) + bottom = min(float(image_height), float(bottom)) + + # 2. If clamped bbox is invalid, return immediately. + if not self._validate_bbox(left, top, right, bottom): + return None + + current_bbox = [left, top, right, bottom] + height = bottom - top + width = right - left + + # 3. If the box is too small, attempt to resize it. + if height < self.MIN_DIMENSION or width < self.MIN_DIMENSION: + logger.info(f"Bbox {width}x{height} is smaller than {self.MIN_DIMENSION}, attempting resize.") + center_x = (left + right) / 2.0 + center_y = (top + bottom) / 2.0 + + min_dim = min(height, width) + if min_dim == 0: # Safeguard for zero-area boxes + return None + + # 1. Calculate the target dimensions to make the smallest side MIN_DIMENSION. + ratio = self.MIN_DIMENSION / min_dim + target_width = width * ratio + target_height = height * ratio + + # 2. If the target size is larger than the image, scale it down to fit. + # This preserves the aspect ratio while respecting image boundaries. + if target_width > image_width: + scale_down = image_width / target_width + target_width = image_width + target_height *= scale_down + + if target_height > image_height: + scale_down = image_height / target_height + target_height = image_height + target_width *= scale_down + + # 3. Determine the coordinates for the box centered on the original center. + new_half_width = target_width / 2.0 + new_half_height = target_height / 2.0 + new_left = center_x - new_half_width + new_top = center_y - new_half_height + + # 4. Shift the box if it extends beyond the image boundaries to keep its size. + if new_left < 0: + new_left = 0 + if new_top < 0: + new_top = 0 + if new_left + target_width > image_width: + new_left = image_width - target_width + if new_top + target_height > image_height: + new_top = image_height - target_height + + new_right = new_left + target_width + new_bottom = new_top + target_height + + # Use floor and ceil for final integer coordinates. + current_bbox = [floor(new_left), floor(new_top), ceil(new_right), ceil(new_bottom)] + + # 4. Final validation on the resulting bounding box (either original or resized). + final_left, final_top, final_right, final_bottom = current_bbox + if not self._validate_bbox(final_left, final_top, final_right, final_bottom): + logger.warning(f"Final bbox is invalid after processing: {current_bbox}") + return None + + final_height = floor(final_bottom) - floor(final_top) + final_width = floor(final_right) - floor(final_left) + + if final_height < self.MIN_DIMENSION or final_width < self.MIN_DIMENSION: + logger.warning( + f"Final bbox size ({final_width}x{final_height}) are still smaller than minimum ({self.MIN_DIMENSION})." + f"Original bbox: {bbox_2d}, original image size: {image_width}x{image_height}" + ) + return None + + return current_bbox + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + return self.tool_schema + + async def create(self, instance_id: Optional[str] = None, **kwargs) -> tuple[str, ToolResponse]: + """ + Creates a new instance for image zoom-in tool. + + This method initializes a new session for an image, which can then be used + for operations like zooming. It fetches the image from various sources + and stores it internally. + + Args: + instance_id: An optional unique identifier for the instance. If not + provided, a new UUID will be generated. + **kwargs: Should contain 'image' key with image data, or 'create_kwargs' + containing {'image': image_data}. Image can be one of the following: + - A PIL.Image.Image object. + - A string containing an HTTP or HTTPS URL. + - A string containing a local file path. + - A string containing a file URI (e.g., "file:///path/to/image.jpg"). + - A string containing a base64-encoded image in the format of "data:image/jpeg;base64,..." + + Returns: + Tuple of (instance_id, ToolResponse) + """ + if instance_id is None: + instance_id = str(uuid4()) + + # Handle create_kwargs parameter if passed + create_kwargs = kwargs.get("create_kwargs", {}) + if create_kwargs: + kwargs.update(create_kwargs) + + # Get image from kwargs + image = kwargs.get("image") + if image is None: + raise ValueError("Missing required 'image' parameter in kwargs") + + img = fetch_image({"image": image}) + self._instance_dict[instance_id] = { + "image": img, + "response": "", + "reward": 0.0, + } + return instance_id, ToolResponse() + + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + bbox_2d = parameters.get("bbox_2d") + label = parameters.get("label", "") + + if not bbox_2d or len(bbox_2d) != 4: + return ( + ToolResponse(text="Error: bbox_2d parameter is missing or not a list of 4 numbers."), + -0.05, + {"success": False}, + ) + + instance_data = self._instance_dict[instance_id] + image = instance_data["image"] + image_width, image_height = image.size + + try: + resized_bbox = self._maybe_resize_bbox(bbox_2d, image_width=image_width, image_height=image_height) + + if resized_bbox is None: + error_msg = ( + f"Error: The specified bounding box {bbox_2d} is invalid or results in a crop smaller than " + f"the minimum size of {self.MIN_DIMENSION}x{self.MIN_DIMENSION}." + ) + logger.warning(f"Tool execution failed: {error_msg}") + return ToolResponse(text=error_msg), -0.05, {"success": False} + + cropped_image = image.crop(resized_bbox) + logger.info(f"Cropped image size: {cropped_image.size}") + except Exception as e: + logger.error(f"Error processing image zoom-in: {e}") + return ToolResponse(text=f"Error processing image zoom-in: {e}"), -0.05, {"success": False} + + response_text = f"Zoomed in on the image to the region {bbox_2d}." + if label: + response_text = f"Zoomed in on the image to the region {bbox_2d} with label {label}." + + return ( + ToolResponse( + image=[cropped_image], + text=response_text, + ), + 0.0, + {"success": True}, + ) + + async def release(self, instance_id: str, **kwargs) -> None: + if instance_id in self._instance_dict: + del self._instance_dict[instance_id] diff --git a/e2c/verl/verl/tools/mcp_base_tool.py b/e2c/verl/verl/tools/mcp_base_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..fc845a569f1755a9086527e8576965f9d7900bb8 --- /dev/null +++ b/e2c/verl/verl/tools/mcp_base_tool.py @@ -0,0 +1,117 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +from typing import Any, Optional +from uuid import uuid4 + +from fastmcp.exceptions import ClientError + +from verl.tools.utils.mcp_clients.McpClientManager import ClientManager +from verl.utils.rollout_trace import rollout_trace_op + +from .base_tool import BaseTool +from .schemas import OpenAIFunctionToolSchema, ToolResponse + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class MCPBaseTool(BaseTool): + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + super().__init__(config, tool_schema) + self._instance_dict = {} + self.timeout = config.get("timeout", 30) + + # TODO(hechanghao): create a global client manager to manage the rate limit, client and pool + logger.info(f"Initialized MCPBaseTool with config: {config}") + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + """Return the OpenAI tool schema.""" + return self.tool_schema + + async def create(self, instance_id: Optional[str] = None, **kwargs) -> tuple[str, ToolResponse]: + """Create a tool instance. + + Args: + instance_id: The instance id of the tool. + + Returns: + The instance id of the tool. + tool_crtool_creation_response: The response of the tool when creating the instance. + """ + if instance_id is None: + instance_id = str(uuid4()) + self._instance_dict[instance_id] = { + "response": "", + "reward": [], + } + return instance_id, ToolResponse() + + async def _call_tool(self, instance_id, parameters) -> tuple[str, dict]: + err_msg = "" + try: + call_tool_result = await ClientManager.call_tool(self.name, parameters, self.timeout) + except ClientError as e: + err_msg = f"\n Tool call failed: {e}" + except ConnectionError as e: + err_msg = f"\n Connection failed: {e}" + except Exception as e: + err_msg = f"\n An unexpected error occurred: {e}" + + logger.debug(f"Tool result for instance {instance_id} with tool {self.name}: {call_tool_result.content}") + result, metadata = self._parse_tool_result(call_tool_result.content) + metadata["api_request_error"] = None if not err_msg else err_msg + return result, metadata + + @rollout_trace_op + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + if self.name == "" or self.name is None or parameters is None: + error_msg = "Error: 'parameters' is missing or empty." + logger.error(f"[MCPTool] {error_msg} Received tool name: {self.name}, parameters: {parameters}") + return ToolResponse(text=json.dumps({"result": error_msg})), 0.0, {} + + try: + result_text, metadata = await self._call_tool(instance_id, parameters) + + # Store results in instance dictionary + self._instance_dict[instance_id]["reward"].append(result_text.strip()) + + # Convert metadata to metrics + metrics = { + "query_count": metadata.get("query_count", 0), + "status": metadata.get("status", "unknown"), + "total_results": metadata.get("total_results", 0), + "api_request_error": metadata.get("api_request_error"), + } + + return ToolResponse(text=result_text), 0.0, metrics + + except Exception as e: + error_result = json.dumps({"result": f"Tool execution failed: {e}"}) + logger.error(f"[MCPBaseTool] Execution failed: {e}") + return ToolResponse(text=error_result), 0.0, {"error": str(e)} + + async def calc_reward(self, instance_id: str, **kwargs) -> str: + return self._instance_dict[instance_id]["reward"] + + async def release(self, instance_id: str, **kwargs) -> None: + if instance_id in self._instance_dict: + del self._instance_dict[instance_id] + + def _parse_tool_result(self, content: list) -> tuple[str, dict]: + tools_content = [part.text for part in filter(lambda x: x.type == "text", content)] + return " ".join(tools_content), {} diff --git a/e2c/verl/verl/tools/mcp_search_tool.py b/e2c/verl/verl/tools/mcp_search_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..ac823719bbb6ecdc0ca02b918b9a6ef6833407bf --- /dev/null +++ b/e2c/verl/verl/tools/mcp_search_tool.py @@ -0,0 +1,69 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import re + +from verl.tools.mcp_base_tool import MCPBaseTool + +from .schemas import OpenAIFunctionToolSchema + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class MCPSearchTool(MCPBaseTool): + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + super().__init__(config, tool_schema) + + def _parse_tool_result(self, content: list) -> tuple[str, dict]: + res = "" + res_cnt = 0 + query_list = [] + metadata = { + "api_request_error": "", + "status": "unknown", + "total_results": 0, + } + try: + for part in content: + if part.type != "text": + continue + text = part.text.replace("'", '"') + query_match = re.search(r'query"\s*:\s*"([^"]+)"', text) + query = query_match.group(1) if query_match else "" + query_list.append(query) + + title_matches = re.findall(r'"title"\s*:', text) + title_count = len(title_matches) + + results_match = re.search(r'"results"\s*:\s*(\[.*?\])', text, re.DOTALL) + results_content = results_match.group(1) if results_match else "" + + res += results_content + res_cnt += title_count + except json.JSONDecodeError: + err_msg = "json parse error." + logger.error(err_msg) + metadata["api_request_error"] = err_msg + metadata["status"] = "error" + + # update metadata + metadata["status"] = "success" + metadata["queries"] = query_list + metadata["query_count"] = len(query_list) + metadata["total_results"] = res_cnt + return res, metadata diff --git a/e2c/verl/verl/tools/sandbox_fusion_tools.py b/e2c/verl/verl/tools/sandbox_fusion_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..3cc467677e5298d678d7b4b79442599c41e1d763 --- /dev/null +++ b/e2c/verl/verl/tools/sandbox_fusion_tools.py @@ -0,0 +1,195 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import threading +from contextlib import ExitStack +from enum import Enum +from typing import Any, Callable, Optional, TypeVar +from uuid import uuid4 + +import ray + +from verl.tools.base_tool import BaseTool +from verl.utils.reward_score.sandbox_fusion.utils import _process_single_case +from verl.utils.rollout_trace import rollout_trace_op + +from .schemas import OpenAIFunctionToolSchema, ToolResponse + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +T = TypeVar("T") + + +class PoolMode(Enum): + ThreadMode = 1 + ProcessMode = 2 + + +@ray.remote(concurrency_groups={"acquire": 1, "release": 10}) +class TokenBucketWorker: + def __init__(self, rate_limit: int): + self.rate_limit = rate_limit + # this only used for observalability + self.current_count = 0 + self._semaphore = threading.Semaphore(rate_limit) + + @ray.method(concurrency_group="acquire") + def acquire(self): + self._semaphore.acquire() + self.current_count += 1 + + @ray.method(concurrency_group="release") + def release(self): + self._semaphore.release() + self.current_count -= 1 + + def get_current_count(self): + return self.current_count + + +class ExecutionWorker: + def __init__(self, enable_global_rate_limit=True, rate_limit=10): + self.rate_limit_worker = self._init_rate_limit(rate_limit) if enable_global_rate_limit else None + + def _init_rate_limit(self, rate_limit): + # TODO validation for rate_limit + # A Singleton Rate Limitor + return TokenBucketWorker.options(name="rate-limiter", get_if_exists=True).remote(rate_limit) + + def ping(self): + return True + + def execute(self, fn: Callable[..., T], *fn_args, **fn_kwargs) -> T: + with ExitStack() as stack: + stack.callback(self.rate_limit_worker.release.remote) + ray.get(self.rate_limit_worker.acquire.remote()) + try: + return fn(*fn_args, **fn_kwargs) + except Exception as e: + # TODO we should make this available to the tool caller + logger.warning(f"Error when executing code: {e}") + + +def init_execution_pool( + num_workers: int, enable_global_rate_limit=True, rate_limit=10, mode: PoolMode = PoolMode.ThreadMode +): + if mode == PoolMode.ThreadMode: + return ( + ray.remote(ExecutionWorker) + .options(max_concurrency=num_workers) + .remote(enable_global_rate_limit=enable_global_rate_limit, rate_limit=rate_limit) + ) + else: + raise NotImplementedError("Process mode is not implemented yet") + # return ray.util.multiprocessing.Pool(processes=num_workers) + + +class SandboxFusionTool(BaseTool): + """A tool for executing the code using sanbox fusion image. + + - `get_openai_tool_schema`: return the tool schema in OpenAI format. + - `create`: create a tool instance for a trajectory. + - `execute`: execute the tool. + - `calc_reward`: calculate the reward respect to tool state. + - `release`: release the tool instance. + """ + + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + """ + _tool_schema = OpenAIFunctionToolSchema.model_validate({ + "type": "function", + "function": { + "name": "code_interpreter", + "description": "A tool for execute code", + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "code needs to be execute and grad", + }, + }, + "required": ["code"], + }, + } + }) + """ + super().__init__(config, tool_schema) + self._instance_dict = {} + # TODO: better documentation for the config + self.num_workers = config.get("num_workers", 10) + self.rate_limit = config.get("rate_limit", 10) + self.default_timeout = config.get("default_timeout", 30) + self.default_language = config.get("default_language", "python") + self.enable_global_rate_limit = config.get("enable_global_rate_limit", True) + self.execution_pool = init_execution_pool( + num_workers=self.num_workers, + enable_global_rate_limit=self.enable_global_rate_limit, + rate_limit=self.rate_limit, + mode=PoolMode.ThreadMode, + ) + self.sandbox_fusion_url = config.get("sandbox_fusion_url", "") + self.memory_limit_mb = config.get("memory_limit_mb", 1024) + if self.sandbox_fusion_url == "": + raise ValueError("sandbox_fusion_url is not set") + log_msg = f"Init SandboxFusionTool with config: {config}" + logger.info(log_msg) + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + return self.tool_schema + + async def create( + self, instance_id: Optional[str] = None, ground_truth: Optional[str] = None, **kwargs + ) -> tuple[str, ToolResponse]: + if instance_id is None: + instance_id = str(uuid4()) + self._instance_dict[instance_id] = { + "response": "", + "ground_truth": ground_truth, + "reward": [], + } + return instance_id, ToolResponse() + + @rollout_trace_op + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + code = parameters.get("code", "") + timeout = parameters.get("timeout", self.default_timeout) + language = parameters.get("language", self.default_language) + if not isinstance(code, str): + code = str(code) + + result = await self.execution_pool.execute.remote(self.execute_code, instance_id, code, timeout, language) + # sandbox has no score or metrics, use Nones + return ToolResponse(text=result), None, None + + def execute_code(self, instance_id, code, timeout=30, language="python"): + result_status, metadata = _process_single_case( + 0, None, None, self.sandbox_fusion_url, code, timeout, self.memory_limit_mb, language + ) + # we should always expect this since we don't have correct answer + if metadata["run_status"] == "Finished": + actual_output = metadata["stdout"] + metadata["stderr"] + logger.debug(f"actual_output from sandbox fusion: {actual_output},{instance_id}") + return ToolResponse(text=actual_output) + else: + return ToolResponse(text="no stdout here") + + async def calc_reward(self, instance_id: str, **kwargs) -> str: + return self._instance_dict[instance_id]["reward"] + + async def release(self, instance_id: str, **kwargs) -> None: + del self._instance_dict[instance_id] diff --git a/e2c/verl/verl/tools/schemas.py b/e2c/verl/verl/tools/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..aa01ae724566d75c2bcd7b57979d0004e50fb3c5 --- /dev/null +++ b/e2c/verl/verl/tools/schemas.py @@ -0,0 +1,123 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from typing import Any, Literal + +from pydantic import BaseModel, Field, model_validator + + +class OpenAIFunctionPropertySchema(BaseModel): + """The schema of a parameter in OpenAI format.""" + + type: str + description: str | None = None + enum: list[str] | None = None + + +class OpenAIFunctionParametersSchema(BaseModel): + """The schema of parameters in OpenAI format.""" + + type: str + properties: dict[str, OpenAIFunctionPropertySchema] + required: list[str] + + +class OpenAIFunctionSchema(BaseModel): + """The schema of a function in OpenAI format.""" + + name: str + description: str + parameters: OpenAIFunctionParametersSchema = Field( + default_factory=lambda: OpenAIFunctionParametersSchema(type="object", properties={}, required=[]) + ) + strict: bool = False + + +class OpenAIFunctionToolSchema(BaseModel): + """The schema of a tool in OpenAI format.""" + + type: str + function: OpenAIFunctionSchema + + +class OpenAIFunctionParsedSchema(BaseModel): + """The parsed schema of a tool in OpenAI format.""" + + name: str + arguments: str # JSON string + + +class OpenAIFunctionCallSchema(BaseModel): + """The parsed schema of a tool in OpenAI format.""" + + name: str + arguments: dict[str, Any] + + @staticmethod + def from_openai_function_parsed_schema( + parsed_schema: OpenAIFunctionParsedSchema, + ) -> tuple["OpenAIFunctionCallSchema", bool]: + has_decode_error = False + try: + arguments = json.loads(parsed_schema.arguments) + except json.JSONDecodeError: + arguments = {} + has_decode_error = True + # If the arguments is not a dict, it means the arguments is not a valid JSON string + if not isinstance(arguments, dict): + arguments = {} + has_decode_error = True + + return OpenAIFunctionCallSchema(name=parsed_schema.name, arguments=arguments), has_decode_error + + +class OpenAIFunctionToolCall(BaseModel): + """The tool call in OpenAI format.""" + + id: str + type: Literal["function"] = "function" + function: OpenAIFunctionCallSchema + + +class ToolResponse(BaseModel): + """The response from a tool execution.""" + + text: str | None = None + image: list[Any] | None = None + video: list[Any] | None = None + + @model_validator(mode="before") + @classmethod + def initialize_request(cls, values): + if "image" in values and not isinstance(values["image"], list): + raise ValueError( + f"Image must be a list, but got {type(values['image'])}. Please check the tool.execute(). " + f"For single images, wrap in a list: [image]. " + f"Example: {{'image': [img1]}} or {{'image': [img1, img2, ...]}}." + ) + if "video" in values and not isinstance(values["video"], list): + raise ValueError( + f"Video must be a list, but got {type(values['video'])}. Please check the tool.execute(). " + f"For single videos, wrap in a list: [video]. " + f"Example: {{'video': [video1]}} or {{'video': [video1, video2, ...]}}." + ) + + return values + + def is_empty(self) -> bool: + return not self.text and not self.image and not self.video + + def is_text_only(self) -> bool: + return self.text and not self.image and not self.video diff --git a/e2c/verl/verl/tools/search_tool.py b/e2c/verl/verl/tools/search_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..b0f9f3ba87886952e5d06bc095e3a5ca8fb899b9 --- /dev/null +++ b/e2c/verl/verl/tools/search_tool.py @@ -0,0 +1,279 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import threading +from contextlib import ExitStack +from enum import Enum +from typing import Any, Callable, Optional, TypeVar +from uuid import uuid4 + +import ray +import ray.actor + +from verl.tools.utils.search_r1_like_utils import perform_single_search_batch +from verl.utils.rollout_trace import rollout_trace_op + +from .base_tool import BaseTool +from .schemas import OpenAIFunctionToolSchema, ToolResponse + +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +T = TypeVar("T") + + +# Adapted from verl/tools/sandbox_fusion_tools.py +class PoolMode(Enum): + """Execution pool mode enumeration.""" + + ThreadMode = 1 + ProcessMode = 2 + + +@ray.remote(concurrency_groups={"acquire": 1, "release": 10}) +class TokenBucketWorker: + """Ray actor for rate limiting using token bucket algorithm.""" + + def __init__(self, rate_limit: int): + self.rate_limit = rate_limit + self.current_count = 0 # For observability + self._semaphore = threading.Semaphore(rate_limit) + + @ray.method(concurrency_group="acquire") + def acquire(self): + """Acquire a token from the bucket.""" + self._semaphore.acquire() + self.current_count += 1 + + @ray.method(concurrency_group="release") + def release(self): + """Release a token back to the bucket.""" + self._semaphore.release() + self.current_count -= 1 + + def get_current_count(self): + """Get current number of acquired tokens.""" + return self.current_count + + +class SearchExecutionWorker: + """Worker for executing search operations with optional rate limiting.""" + + def __init__(self, enable_global_rate_limit=True, rate_limit=10): + self.rate_limit_worker = self._init_rate_limit(rate_limit) if enable_global_rate_limit else None + + def _init_rate_limit(self, rate_limit): + """Initialize singleton rate limiter.""" + return TokenBucketWorker.options(name="rate-limiter", get_if_exists=True).remote(rate_limit) + + def ping(self): + """Health check method.""" + return True + + def execute(self, fn: Callable[..., T], *fn_args, **fn_kwargs) -> T: + """Execute function with optional rate limiting.""" + if self.rate_limit_worker: + with ExitStack() as stack: + stack.callback(self.rate_limit_worker.release.remote) + ray.get(self.rate_limit_worker.acquire.remote()) + try: + return fn(*fn_args, **fn_kwargs) + except Exception as e: + # TODO we should make this available to the tool caller + logger.warning(f"Error when executing search: {e}") + else: + return fn(*fn_args, **fn_kwargs) + + +def init_search_execution_pool( + num_workers: int, enable_global_rate_limit=True, rate_limit=10, mode: PoolMode = PoolMode.ThreadMode +): + """Initialize search execution pool.""" + if mode == PoolMode.ThreadMode: + return ( + ray.remote(SearchExecutionWorker) + .options(max_concurrency=num_workers) + .remote(enable_global_rate_limit=enable_global_rate_limit, rate_limit=rate_limit) + ) + else: + raise NotImplementedError("Process mode is not implemented yet") + + +class SearchTool(BaseTool): + """Search tool for retrieving information using external retrieval services. + + This tool provides search functionality with rate limiting and concurrent execution + support through Ray. It integrates with external retrieval services to perform + semantic search operations. + + Methods: + get_openai_tool_schema: Return the tool schema in OpenAI format + create: Create a tool instance for a trajectory + execute: Execute the search tool + calc_reward: Calculate the reward with respect to tool state + release: Release the tool instance + """ + + def __init__(self, config: dict, tool_schema: OpenAIFunctionToolSchema): + """Initialize SearchTool with configuration and schema. + + Args: + config: Configuration dictionary containing tool settings + tool_schema: OpenAI function tool schema definition + + Example tool_schema: + { + "type": "function", + "function": { + "name": "search", + "description": "Searches for relevant information based on queries.", + "parameters": { + "type": "object", + "properties": { + "query_list": { + "type": "array", + "items": {"type": "string"}, + "description": "List of search queries" + } + }, + "required": ["query_list"] + } + } + } + """ + super().__init__(config, tool_schema) + self._instance_dict = {} + + # Worker and rate limiting configuration + self.num_workers = config.get("num_workers", 120) + self.rate_limit = config.get("rate_limit", 120) + self.timeout = config.get("timeout", 30) + + self.enable_global_rate_limit = config.get("enable_global_rate_limit", True) + self.execution_pool = init_search_execution_pool( + num_workers=self.num_workers, + enable_global_rate_limit=self.enable_global_rate_limit, + rate_limit=self.rate_limit, + mode=PoolMode.ThreadMode, + ) + + # Retrieval service configuration + self.retrieval_service_url = config.get("retrieval_service_url") + assert self.retrieval_service_url, "Configuration must include 'retrieval_service_url'" + self.topk = config.get("topk", 3) + if self.retrieval_service_url == "": + raise ValueError("retrieval_service_url is not set") + + logger.info(f"Initialized SearchTool with config: {config}") + + def get_openai_tool_schema(self) -> OpenAIFunctionToolSchema: + """Return the OpenAI tool schema.""" + return self.tool_schema + + async def create(self, instance_id: Optional[str] = None, **kwargs) -> tuple[str, ToolResponse]: + """Create a tool instance. + + Args: + instance_id: The instance id of the tool. + + Returns: + The instance id of the tool. + tool_creation_response: The response of the tool when creating the instance. + """ + if instance_id is None: + instance_id = str(uuid4()) + self._instance_dict[instance_id] = { + "response": "", + "reward": [], + } + return instance_id, ToolResponse() + + def execute_search(self, instance_id: str, query_list: list, retrieval_service_url: str, topk: int, timeout: int): + """Execute search operation using retrieval service. + + Args: + instance_id: Tool instance ID + query_list: List of search queries + retrieval_service_url: URL of the retrieval service + topk: Number of top results to return + timeout: Request timeout in seconds + + Returns: + Tuple of (result_text, metadata) + """ + result_text, metadata = perform_single_search_batch( + retrieval_service_url=retrieval_service_url, + query_list=query_list, + topk=topk, + concurrent_semaphore=None, # Ray handles concurrency control + timeout=timeout, + ) + logger.debug(f"Search result for instance {instance_id}: {result_text}") + return result_text, metadata + + @rollout_trace_op + async def execute(self, instance_id: str, parameters: dict[str, Any], **kwargs) -> tuple[ToolResponse, float, dict]: + """Execute the search tool. + + Args: + instance_id: The instance ID of the tool + parameters: Tool parameters containing query_list and optional timeout + + Returns: tool_response, tool_reward_score, tool_metrics + tool_response: The response str of the tool. + tool_reward_score: The step reward score of the tool. + tool_metrics: The metrics of the tool. + """ + timeout = self.timeout + query_list_from_params = parameters.get("query_list") + + if not query_list_from_params or not isinstance(query_list_from_params, list): + error_msg = "Error: 'query_list' is missing, empty, or not a list in parameters." + logger.error(f"[SearchTool] {error_msg} Received parameters: {parameters}") + return ToolResponse(text=json.dumps({"result": error_msg})), 0.0, {} + + # Execute search using Ray execution pool + try: + result_text, metadata = await self.execution_pool.execute.remote( + self.execute_search, instance_id, query_list_from_params, self.retrieval_service_url, self.topk, timeout + ) + + # Store results in instance dictionary + self._instance_dict[instance_id]["reward"].append(result_text.strip()) + + # Convert metadata to metrics + metrics = { + "query_count": metadata.get("query_count", 0), + "status": metadata.get("status", "unknown"), + "total_results": metadata.get("total_results", 0), + "api_request_error": metadata.get("api_request_error"), + } + + return ToolResponse(text=result_text), 0.0, metrics + + except Exception as e: + error_result = json.dumps({"result": f"Search execution failed: {e}"}) + logger.error(f"[SearchTool] Execution failed: {e}") + return ToolResponse(text=error_result), 0.0, {"error": str(e)} + + async def calc_reward(self, instance_id: str, **kwargs) -> str: + return self._instance_dict[instance_id]["reward"] + + async def release(self, instance_id: str, **kwargs) -> None: + if instance_id in self._instance_dict: + del self._instance_dict[instance_id] diff --git a/e2c/verl/verl/tools/utils/__init__.py b/e2c/verl/verl/tools/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c4b932b1ae7eeeb4c53c98c684cf0ba9b670a86b --- /dev/null +++ b/e2c/verl/verl/tools/utils/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/tools/utils/mcp_clients/McpClientManager.py b/e2c/verl/verl/tools/utils/mcp_clients/McpClientManager.py new file mode 100644 index 0000000000000000000000000000000000000000..ee5fe31191321f653230f6dc0cfb9e71a42e1722 --- /dev/null +++ b/e2c/verl/verl/tools/utils/mcp_clients/McpClientManager.py @@ -0,0 +1,97 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json +import logging +from typing import Any + +from fastmcp import Client +from fastmcp.client.transports import SSETransport + +from verl.tools.utils.mcp_clients.utils import TokenBucket, mcp2openai + +logger = logging.getLogger(__name__) + + +class MCPClientManager: + rootServerName = "mcpServers" + initialized = False + clients = [] + tool_client_mapping = {} + rate_limiter = None + + async def initialize(self, config_path, rate_limit: float = 10.0): + if self.initialized: + return + """Initialize the MCP Client Manager and start all clients""" + result = self._load_config(config_path) + servers = result[self.rootServerName] + exclude_sse_servers = {self.rootServerName: {}} + for server_name in servers.keys(): + server = servers[server_name] + if "auth_token" in server: + transport = SSETransport(url=server["url"], headers={"Authorization": f"Bearer {server['auth_token']}"}) + client = Client(transport) + self.clients.append(client) + else: + exclude_sse_servers[self.rootServerName][server_name] = server + + if exclude_sse_servers[self.rootServerName]: + self.clients.append(Client(exclude_sse_servers)) + + # Initialize rate limiter + self.rate_limiter = TokenBucket(rate_limit) + self.initialized = True + + async def call_tool(self, tool_name, parameters, timeout): + # Apply rate limiting + while not self.rate_limiter.acquire(): + await asyncio.sleep(0.1) + + client = self.get_client_with_tool_name(tool_name) + async with client: + return await client.call_tool_mcp(tool_name, parameters) + + async def fetch_tool_schemas(self, tool_selected_list: list[str]) -> list[dict]: + tool_schemas = [] + for client in self.clients: + async with client: + tools = await client.list_tools_mcp() + for tool in tools.tools: + if not tool_selected_list: + self.tool_client_mapping[tool.name] = client + tool_schemas.append(mcp2openai(tool)) + elif tool.name in tool_selected_list: + self.tool_client_mapping[tool.name] = client + tool_schemas.append(mcp2openai(tool)) + + return tool_schemas + + def get_client_with_tool_name(self, tool_name: str): + return self.tool_client_mapping[tool_name] + + def _load_config(self, file: str) -> dict[str, Any]: + try: + with open(file) as f: + return json.load(f) + except FileNotFoundError: + logger.warning(f'the "{file}" file was not found') + except Exception: + logger.error(f'there was an error reading the "{file}" file') + + return {} + + +ClientManager = MCPClientManager() diff --git a/e2c/verl/verl/tools/utils/mcp_clients/utils.py b/e2c/verl/verl/tools/utils/mcp_clients/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..22a5f63532713dcb895b0a940bf9bc9dfe42cfdf --- /dev/null +++ b/e2c/verl/verl/tools/utils/mcp_clients/utils.py @@ -0,0 +1,58 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import threading +import time + +from mcp import Tool + +logger = logging.getLogger(__file__) + + +class TokenBucket: + def __init__(self, rate_limit: float): + self.rate_limit = rate_limit # tokens per second + self.tokens = rate_limit + self.last_update = time.time() + self.lock = threading.Lock() + + def acquire(self) -> bool: + with self.lock: + now = time.time() + # Add new tokens based on time elapsed + new_tokens = (now - self.last_update) * self.rate_limit + self.tokens = min(self.rate_limit, self.tokens + new_tokens) + self.last_update = now + + if self.tokens >= 1: + self.tokens -= 1 + return True + return False + + +def mcp2openai(mcp_tool: Tool) -> dict: + """Convert a MCP Tool to an OpenAI ChatCompletionTool.""" + openai_format = { + "type": "function", + "function": { + "name": mcp_tool.name, + "description": mcp_tool.description, + "parameters": mcp_tool.inputSchema, + "strict": False, + }, + } + if not openai_format["function"]["parameters"].get("required", None): + openai_format["function"]["parameters"]["required"] = [] + return openai_format diff --git a/e2c/verl/verl/tools/utils/search_r1_like_utils.py b/e2c/verl/verl/tools/utils/search_r1_like_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..610698e3b602d44b1bc19919e397a2d4cfb08bc9 --- /dev/null +++ b/e2c/verl/verl/tools/utils/search_r1_like_utils.py @@ -0,0 +1,245 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import threading +import time +import traceback +import uuid +from typing import Any, Optional + +import requests + +DEFAULT_TIMEOUT = 30 # Default search request timeout +MAX_RETRIES = 10 +INITIAL_RETRY_DELAY = 1 +API_TIMEOUT = 10 + +logger = logging.getLogger(__name__) + + +def call_search_api( + retrieval_service_url: str, + query_list: list[str], + topk: int = 3, + return_scores: bool = True, + timeout: int = DEFAULT_TIMEOUT, +) -> tuple[Optional[dict[str, Any]], Optional[str]]: + """ + Calls the remote search API to perform retrieval with retry logic for various errors, + using increasing delay between retries. Logs internal calls with a unique ID. + + Args: + retrieval_service_url: The URL of the retrieval service API. + query_list: List of search queries. + topk: Number of top results to return. + return_scores: Whether to return scores. + timeout: Request timeout in seconds. + + Returns: + A tuple (response_json, error_message). + If successful, response_json is the API's returned JSON object, error_message is None. + If failed after retries, response_json is None, error_message contains the error information. + """ + request_id = str(uuid.uuid4()) + log_prefix = f"[Search Request ID: {request_id}] " + + payload = {"queries": query_list, "topk": topk, "return_scores": return_scores} + + headers = {"Content-Type": "application/json", "Accept": "application/json"} + + last_error = None + + for attempt in range(MAX_RETRIES): + try: + logger.info( + f"{log_prefix}Attempt {attempt + 1}/{MAX_RETRIES}: Calling search API at {retrieval_service_url}" + ) + response = requests.post( + retrieval_service_url, + headers=headers, + json=payload, + timeout=timeout, + ) + + # Check for Gateway Timeout (504) and other server errors for retrying + if response.status_code in [500, 502, 503, 504]: + last_error = ( + f"{log_prefix}API Request Error: Server Error ({response.status_code}) on attempt " + f"{attempt + 1}/{MAX_RETRIES}" + ) + logger.warning(last_error) + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + logger.info(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + + # Check for other HTTP errors (e.g., 4xx) + response.raise_for_status() + + # If successful (status code 2xx) + logger.info(f"{log_prefix}Search API call successful on attempt {attempt + 1}") + return response.json(), None + + except requests.exceptions.ConnectionError as e: + last_error = f"{log_prefix}Connection Error: {e}" + logger.warning(last_error) + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + logger.info(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + except requests.exceptions.Timeout as e: + last_error = f"{log_prefix}Timeout Error: {e}" + logger.warning(last_error) + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + logger.info(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + except requests.exceptions.RequestException as e: + last_error = f"{log_prefix}API Request Error: {e}" + break # Exit retry loop on other request errors + except json.JSONDecodeError as e: + raw_response_text = response.text if "response" in locals() else "N/A" + last_error = f"{log_prefix}API Response JSON Decode Error: {e}, Response: {raw_response_text[:200]}" + break # Exit retry loop on JSON decode errors + except Exception as e: + last_error = f"{log_prefix}Unexpected Error: {e}" + break # Exit retry loop on other unexpected errors + + # If loop finishes without returning success, return the last recorded error + logger.error(f"{log_prefix}Search API call failed. Last error: {last_error}") + return None, last_error.replace(log_prefix, "API Call Failed: ") if last_error else "API Call Failed after retries" + + +def _passages2string(retrieval_result): + """Convert retrieval results to formatted string.""" + format_reference = "" + for idx, doc_item in enumerate(retrieval_result): + content = doc_item["document"]["contents"] + title = content.split("\n")[0] + text = "\n".join(content.split("\n")[1:]) + format_reference += f"Doc {idx + 1} (Title: {title})\n{text}\n\n" + return format_reference.strip() + + +def perform_single_search_batch( + retrieval_service_url: str, + query_list: list[str], + topk: int = 3, + concurrent_semaphore: Optional[threading.Semaphore] = None, + timeout: int = DEFAULT_TIMEOUT, +) -> tuple[str, dict[str, Any]]: + """ + Performs a single batch search for multiple queries (original search tool behavior). + + Args: + retrieval_service_url: The URL of the retrieval service API. + query_list: List of search queries. + topk: Number of top results to return. + concurrent_semaphore: Optional semaphore for concurrency control. + timeout: Request timeout in seconds. + + Returns: + A tuple (result_text, metadata). + result_text: The search result JSON string. + metadata: Metadata dictionary for the batch search. + """ + logger.info(f"Starting batch search for {len(query_list)} queries.") + + api_response = None + error_msg = None + + try: + if concurrent_semaphore: + with concurrent_semaphore: + api_response, error_msg = call_search_api( + retrieval_service_url=retrieval_service_url, + query_list=query_list, + topk=topk, + return_scores=True, + timeout=timeout, + ) + else: + api_response, error_msg = call_search_api( + retrieval_service_url=retrieval_service_url, + query_list=query_list, + topk=topk, + return_scores=True, + timeout=timeout, + ) + except Exception as e: + error_msg = f"API Request Exception during batch search: {e}" + logger.error(f"Batch search: {error_msg}") + traceback.print_exc() + + metadata = { + "query_count": len(query_list), + "queries": query_list, + "api_request_error": error_msg, + "api_response": None, + "status": "unknown", + "total_results": 0, + "formatted_result": None, + } + + result_text = json.dumps({"result": "Search request failed or timed out after retries."}, ensure_ascii=False) + + if error_msg: + metadata["status"] = "api_error" + result_text = json.dumps({"result": f"Search error: {error_msg}"}, ensure_ascii=False) + logger.error(f"Batch search: API error occurred: {error_msg}") + elif api_response: + logger.debug(f"Batch search: API Response: {api_response}") + metadata["api_response"] = api_response + + try: + raw_results = api_response.get("result", []) + if raw_results: + pretty_results = [] + total_results = 0 + + for retrieval in raw_results: + formatted = _passages2string(retrieval) + pretty_results.append(formatted) + total_results += len(retrieval) if isinstance(retrieval, list) else 1 + + final_result = "\n---\n".join(pretty_results) + result_text = json.dumps({"result": final_result}, ensure_ascii=False) + metadata["status"] = "success" + metadata["total_results"] = total_results + metadata["formatted_result"] = final_result + logger.info(f"Batch search: Successful, got {total_results} total results") + else: + result_text = json.dumps({"result": "No search results found."}, ensure_ascii=False) + metadata["status"] = "no_results" + metadata["total_results"] = 0 + logger.info("Batch search: No results found") + except Exception as e: + error_msg = f"Error processing search results: {e}" + result_text = json.dumps({"result": error_msg}, ensure_ascii=False) + metadata["status"] = "processing_error" + logger.error(f"Batch search: {error_msg}") + else: + metadata["status"] = "unknown_api_state" + result_text = json.dumps( + {"result": "Unknown API state (no response and no error message)."}, ensure_ascii=False + ) + logger.error("Batch search: Unknown API state.") + + return result_text, metadata diff --git a/e2c/verl/verl/tools/utils/tool_registry.py b/e2c/verl/verl/tools/utils/tool_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..e3611cd4a5dca5acd5f28e2caee06c06284afc67 --- /dev/null +++ b/e2c/verl/verl/tools/utils/tool_registry.py @@ -0,0 +1,130 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import importlib +import logging +import os +import sys +import threading +from enum import Enum + +from omegaconf import OmegaConf + +from verl.tools.schemas import OpenAIFunctionToolSchema + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +class ToolType(Enum): + NATIVE = "native" + MCP = "mcp" + + +async def initialize_mcp_tool(tool_cls, tool_config) -> list: + from verl.tools.utils.mcp_clients.McpClientManager import ClientManager + + tool_list = [] + mcp_servers_config_path = tool_config.mcp.mcp_servers_config_path + tool_selected_list = tool_config.mcp.tool_selected_list if "tool_selected_list" in tool_config.mcp else None + await ClientManager.initialize(mcp_servers_config_path, tool_config.config.rate_limit) + # Wait for MCP client to be ready + max_retries = 10 + retry_interval = 2 # seconds + for i in range(max_retries): + tool_schemas = await ClientManager.fetch_tool_schemas(tool_selected_list) + if tool_schemas: + break + if i < max_retries - 1: + logger.debug(f"Waiting for MCP client to be ready, attempt {i + 1}/{max_retries}") + await asyncio.sleep(retry_interval) + else: + raise RuntimeError("Failed to initialize MCP tools after maximum retries") + # mcp registry + assert len(tool_schemas), "mcp tool is empty" + for tool_schema_dict in tool_schemas: + logger.debug(f"tool_schema_dict: {tool_schema_dict}") + tool_schema = OpenAIFunctionToolSchema.model_validate(tool_schema_dict) + tool = tool_cls( + config=OmegaConf.to_container(tool_config.config, resolve=True), + tool_schema=tool_schema, + ) + tool_list.append(tool) + return tool_list + + +def get_tool_class(cls_name): + module_name, class_name = cls_name.rsplit(".", 1) + if module_name not in sys.modules: + spec = importlib.util.find_spec(module_name) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + else: + module = sys.modules[module_name] + + tool_cls = getattr(module, class_name) + return tool_cls + + +def initialize_tools_from_config(tools_config_file): + tools_config = OmegaConf.load(tools_config_file) + tool_list = [] + + # Use a temporary event loop in a new thread because event + # loop may already exist in new async architecture while retaining + # backwards compatibility + tmp_event_loop = asyncio.new_event_loop() + thread = threading.Thread(target=tmp_event_loop.run_forever, name="mcp tool list fetcher", daemon=True) + + def run_coroutine(coroutine): + if not thread.is_alive(): + thread.start() + + future = asyncio.run_coroutine_threadsafe(coroutine, tmp_event_loop) + return future.result() + + async def stop_loop(): + tmp_event_loop.stop() + + try: + for tool_config in tools_config.tools: + cls_name = tool_config.class_name + tool_type = ToolType(tool_config.config.type) + tool_cls = get_tool_class(cls_name) + + match tool_type: + case ToolType.NATIVE: + if tool_config.get("tool_schema", None) is None: + tool_schema = None + else: + tool_schema_dict = OmegaConf.to_container(tool_config.tool_schema, resolve=True) + tool_schema = OpenAIFunctionToolSchema.model_validate(tool_schema_dict) + tool = tool_cls( + config=OmegaConf.to_container(tool_config.config, resolve=True), + tool_schema=tool_schema, + ) + tool_list.append(tool) + case ToolType.MCP: + mcp_tools = run_coroutine(initialize_mcp_tool(tool_cls, tool_config)) + tool_list.extend(mcp_tools) + case _: + raise NotImplementedError + finally: + if thread.is_alive(): + asyncio.run_coroutine_threadsafe(stop_loop(), tmp_event_loop) + thread.join() + + return tool_list diff --git a/e2c/verl/verl/trainer/__init__.py b/e2c/verl/verl/trainer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/trainer/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/trainer/config/__init__.py b/e2c/verl/verl/trainer/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a5105f7f340251cb44c8245bda8d8a1eeaaaeeff --- /dev/null +++ b/e2c/verl/verl/trainer/config/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .algorithm import * # noqa +from .config import * # noqa +from . import config, algorithm + +__all__ = config.__all__ + algorithm.__all__ diff --git a/e2c/verl/verl/trainer/config/_generated_ppo_megatron_trainer.yaml b/e2c/verl/verl/trainer/config/_generated_ppo_megatron_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ff14b9134fd49a57c44f6764127cf0527b73230 --- /dev/null +++ b/e2c/verl/verl/trainer/config/_generated_ppo_megatron_trainer.yaml @@ -0,0 +1,541 @@ +# This reference configration yaml is automatically generated via 'scripts/generate_trainer_config.sh' +# in which it invokes 'python3 scripts/print_cfg.py --cfg job --config-name=ppo_megatron_trainer.yaml' to flatten the 'verl/trainer/config/ppo_megatron_trainer.yaml' config fields into a single file. +# Do not modify this file directly. +# The file is usually only for reference and never used. + +actor_rollout_ref: + actor: + optim: + _target_: verl.workers.config.McoreOptimizerConfig + lr: 1.0e-06 + lr_warmup_steps_ratio: 0.0 + total_training_steps: -1 + weight_decay: 0.01 + lr_warmup_steps: -1 + betas: + - 0.9 + - 0.999 + clip_grad: 1.0 + optimizer: adam + lr_warmup_init: 0.0 + lr_decay_steps: null + lr_decay_style: constant + min_lr: 0.0 + weight_decay_incr_style: constant + lr_wsd_decay_style: exponential + lr_wsd_decay_steps: null + use_checkpoint_opt_param_scheduler: false + override_optimizer_config: {} + megatron: + _target_: verl.workers.config.McoreEngineConfig + param_offload: false + grad_offload: false + optimizer_offload: false + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null + context_parallel_size: 1 + sequence_parallel: true + use_distributed_optimizer: true + use_dist_checkpointing: false + dist_checkpointing_path: null + seed: 42 + override_ddp_config: {} + override_transformer_config: + recompute_granularity: null + recompute_modules: + - core_attn + recompute_method: null + recompute_num_layers: null + attention_backend: flash + override_mcore_model_config: {} + use_mbridge: false + forward_only: false + _target_: verl.workers.config.McoreActorConfig + strategy: megatron + ppo_mini_batch_size: 256 + ppo_micro_batch_size: null + ppo_micro_batch_size_per_gpu: null + use_dynamic_bsz: false + ppo_max_token_len_per_gpu: 16384 + clip_ratio: 0.2 + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + freeze_vision_tower: false + policy_loss: + _target_: verl.workers.config.PolicyLossConfig + loss_mode: vanilla + clip_cov_ratio: 0.0002 + clip_cov_lb: 1.0 + clip_cov_ub: 5.0 + kl_cov_ratio: 0.0002 + ppo_kl_coef: 0.1 + clip_ratio_c: 3.0 + loss_agg_mode: token-mean + entropy_coeff: 0 + tis_imp_ratio_cap: -1 + use_kl_loss: false + use_torch_compile: true + kl_loss_coef: 0.001 + kl_loss_type: low_var_kl + ppo_epochs: 1 + shuffle: false + checkpoint: + _target_: verl.trainer.config.CheckpointConfig + save_contents: + - model + - optimizer + - extra + load_contents: ${.save_contents} + async_save: false + use_fused_kernels: ${oc.select:actor_rollout_ref.model.use_fused_kernels,false} + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + data_loader_seed: null + load_weight: true + ref: + strategy: megatron + use_torch_compile: ${oc.select:actor_rollout_ref.actor.use_torch_compile,true} + log_prob_micro_batch_size: null + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + megatron: + _target_: verl.workers.config.MegatronEngineConfig + param_offload: false + grad_offload: false + optimizer_offload: false + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null + context_parallel_size: 1 + sequence_parallel: true + use_distributed_optimizer: true + use_dist_checkpointing: false + dist_checkpointing_path: null + seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42} + override_ddp_config: {} + override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}} + override_mcore_model_config: {} + use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False} + forward_only: false + load_weight: true + rollout: + _target_: verl.workers.config.RolloutConfig + name: ??? + mode: sync + temperature: 1.0 + top_k: -1 + top_p: 1 + prompt_length: ${oc.select:data.max_prompt_length,512} + response_length: ${oc.select:data.max_response_length,512} + dtype: bfloat16 + gpu_memory_utilization: 0.5 + ignore_eos: false + enforce_eager: false + cudagraph_capture_sizes: null + free_cache_engine: true + tensor_model_parallel_size: 2 + data_parallel_size: 1 + expert_parallel_size: 1 + max_num_batched_tokens: 8192 + max_model_len: null + max_num_seqs: 1024 + enable_chunked_prefill: true + enable_prefix_caching: true + load_format: dummy + log_prob_micro_batch_size: null + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + disable_log_stats: true + do_sample: true + 'n': 1 + over_sample_rate: 0 + multi_stage_wake_up: false + engine_kwargs: + vllm: {} + sglang: {} + val_kwargs: + _target_: verl.workers.config.SamplingConfig + top_k: -1 + top_p: 1.0 + temperature: 0 + 'n': 1 + do_sample: false + multi_turn: + _target_: verl.workers.config.MultiTurnConfig + enable: false + max_assistant_turns: null + tool_config_path: null + max_user_turns: null + max_parallel_calls: 1 + max_tool_response_length: 256 + tool_response_truncate_side: middle + interaction_config_path: null + use_inference_chat_template: false + tokenization_sanity_check_mode: strict + format: hermes + num_repeat_rollouts: null + calculate_log_probs: false + agent: + _target_: verl.workers.config.AgentLoopConfig + num_workers: 8 + agent_loop_config_path: null + custom_async_server: + _target_: verl.workers.config.CustomAsyncServerConfig + path: null + name: null + update_weights_bucket_megabytes: 512 + trace: + _target_: verl.workers.config.TraceConfig + backend: null + token2text: false + skip_rollout: false + skip_dump_dir: /tmp/rollout_dump + skip_tokenizer_init: true + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: ${oc.select:actor_rollout_ref.actor.profiler.enable,false} + all_ranks: ${oc.select:actor_rollout_ref.actor.profiler.all_ranks,false} + ranks: ${oc.select:actor_rollout_ref.actor.profiler.ranks,[]} + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} + layer_name_map: + qkv_layer_name: qkv + gate_proj_layer_name: gate_up + hybrid_engine: true + nccl_timeout: 600 + model: + path: ~/models/deepseek-llm-7b-chat + custom_chat_template: null + external_lib: null + override_config: + model_config: {} + moe_config: + freeze_moe_router: false + use_fused_kernels: false + trust_remote_code: false + use_remove_padding: false +data: + tokenizer: null + use_shm: false + train_files: ~/data/rlhf/gsm8k/train.parquet + val_files: ~/data/rlhf/gsm8k/test.parquet + prompt_key: prompt + reward_fn_key: data_source + max_prompt_length: 512 + max_response_length: 512 + train_batch_size: 1024 + val_batch_size: null + return_raw_input_ids: false + return_raw_chat: false + return_full_prompt: false + shuffle: true + dataloader_num_workers: 8 + validation_shuffle: false + filter_overlong_prompts: false + filter_overlong_prompts_workers: 1 + truncation: error + image_key: images + video_key: videos + trust_remote_code: false + custom_cls: + path: null + name: null + return_multi_modal_inputs: true + sampler: + class_path: null + class_name: null + datagen: + path: null + name: null + apply_chat_template_kwargs: {} +critic: + optim: + _target_: verl.workers.config.McoreOptimizerConfig + lr: 1.0e-05 + lr_warmup_steps_ratio: 0.0 + total_training_steps: -1 + weight_decay: 0.01 + lr_warmup_steps: -1 + betas: + - 0.9 + - 0.999 + clip_grad: 1.0 + optimizer: adam + lr_warmup_init: 0.0 + lr_decay_steps: null + lr_decay_style: constant + min_lr: 0.0 + weight_decay_incr_style: constant + lr_wsd_decay_style: exponential + lr_wsd_decay_steps: null + use_checkpoint_opt_param_scheduler: false + override_optimizer_config: {} + megatron: + _target_: verl.workers.config.McoreEngineConfig + param_offload: false + grad_offload: false + optimizer_offload: false + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null + context_parallel_size: 1 + sequence_parallel: true + use_distributed_optimizer: true + use_dist_checkpointing: false + dist_checkpointing_path: null + seed: 42 + override_ddp_config: {} + override_transformer_config: + recompute_granularity: null + recompute_modules: + - core_attn + recompute_method: null + recompute_num_layers: null + attention_backend: flash + override_mcore_model_config: {} + use_mbridge: false + forward_only: false + _target_: verl.workers.config.McoreCriticConfig + rollout_n: ${oc.select:actor_rollout_ref.rollout.n,1} + strategy: megatron + enable: null + model: + path: ~/models/deepseek-llm-7b-chat + tokenizer_path: ${oc.select:actor_rollout_ref.model.path,"~/models/deepseek-llm-7b-chat"} + override_config: + model_config: {} + moe_config: + freeze_moe_router: false + external_lib: ${oc.select:actor_rollout_ref.model.external_lib,null} + trust_remote_code: ${oc.select:actor_rollout_ref.model.trust_remote_code,false} + _target_: verl.trainer.config.BaseModelConfig + ppo_mini_batch_size: ${oc.select:actor_rollout_ref.actor.ppo_mini_batch_size,256} + ppo_micro_batch_size: null + ppo_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size,null} + use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + ppo_max_token_len_per_gpu: 32768 + forward_max_token_len_per_gpu: ${.ppo_max_token_len_per_gpu} + ppo_epochs: ${oc.select:actor_rollout_ref.actor.ppo_epochs,1} + shuffle: ${oc.select:actor_rollout_ref.actor.shuffle,false} + cliprange_value: 0.5 + loss_agg_mode: ${oc.select:actor_rollout_ref.actor.loss_agg_mode,token-mean} + checkpoint: + _target_: verl.trainer.config.CheckpointConfig + save_contents: + - model + - optimizer + - extra + load_contents: ${.save_contents} + async_save: false + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + nccl_timeout: 600 + load_weight: true + data_loader_seed: ${oc.select:actor_rollout_ref.actor.data_loader_seed,null} +reward_model: + enable: false + enable_resource_pool: false + n_gpus_per_node: 0 + nnodes: 0 + strategy: megatron + model: + input_tokenizer: ${actor_rollout_ref.model.path} + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + external_lib: ${actor_rollout_ref.model.external_lib} + trust_remote_code: false + micro_batch_size: null + micro_batch_size_per_gpu: null + max_length: null + use_dynamic_bsz: ${critic.use_dynamic_bsz} + forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + reward_manager: naive + launch_reward_fn_async: false + sandbox_fusion: + url: null + max_concurrent: 64 + memory_limit_mb: 1024 + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} + nccl_timeout: 600 + megatron: + _target_: verl.workers.config.MegatronEngineConfig + param_offload: false + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_size: 1 + virtual_pipeline_model_parallel_size: null + context_parallel_size: 1 + sequence_parallel: true + use_distributed_optimizer: false + use_dist_checkpointing: false + dist_checkpointing_path: null + seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42} + override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}} + use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False} + load_weight: true +custom_reward_function: + path: null + name: compute_score +algorithm: + _target_: verl.trainer.config.AlgoConfig + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + norm_adv_by_std_in_grpo: true + use_kl_in_reward: false + kl_penalty: kl + kl_ctrl: + _target_: verl.trainer.config.KLControlConfig + type: fixed + kl_coef: 0.001 + horizon: 10000 + target_kl: 0.1 + use_pf_ppo: false + pf_ppo: + reweight_method: pow + weight_pow: 2.0 +trainer: + balance_batch: true + total_epochs: 30 + total_training_steps: null + project_name: verl_examples + experiment_name: gsm8k + logger: + - console + - wandb + log_val_generations: 0 + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + esi_redundant_time: 0 + resume_mode: auto + resume_from_path: null + del_local_ckpt_after_load: false + val_before_train: true + test_freq: -1 + critic_warmup: 0 + default_hdfs_dir: null + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + max_actor_ckpt_to_keep: null + max_critic_ckpt_to_keep: null + ray_wait_register_center_timeout: 300 + device: cuda + rollout_data_dir: null +global_profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: null + steps: null + profile_continuous_steps: false + save_path: outputs/profile + global_tool_config: + nsys: + discrete: false + controller_nsight_options: + trace: cuda,nvtx,cublas,ucx + cuda-memory-usage: 'true' + cuda-graph-trace: graph + worker_nsight_options: + trace: cuda,nvtx,cublas,ucx + cuda-memory-usage: 'true' + cuda-graph-trace: graph + capture-range: cudaProfilerApi + capture-range-end: null + kill: none + torch_memory: + trace_alloc_max_entries: 100000 + stack_depth: 32 + context: all + stacks: all + kw_args: {} +ray_kwargs: + ray_init: + num_cpus: null + timeline_json_file: null diff --git a/e2c/verl/verl/trainer/config/_generated_ppo_trainer.yaml b/e2c/verl/verl/trainer/config/_generated_ppo_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61e5efb54501d7b747eeed43a3c4d103a1440b61 --- /dev/null +++ b/e2c/verl/verl/trainer/config/_generated_ppo_trainer.yaml @@ -0,0 +1,523 @@ +# This reference configration yaml is automatically generated via 'scripts/generate_trainer_config.sh' +# in which it invokes 'python3 scripts/print_cfg.py --cfg job ' to flatten the 'verl/trainer/config/ppo_trainer.yaml' config fields into a single file. +# Do not modify this file directly. +# The file is usually only for reference and never used. + +actor_rollout_ref: + actor: + optim: + _target_: verl.workers.config.FSDPOptimizerConfig + lr: 1.0e-06 + lr_warmup_steps_ratio: 0.0 + total_training_steps: -1 + weight_decay: 0.01 + lr_warmup_steps: -1 + betas: + - 0.9 + - 0.999 + clip_grad: 1.0 + min_lr_ratio: 0.0 + num_cycles: 0.5 + warmup_style: constant + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + wrap_policy: + min_num_params: 0 + param_offload: false + optimizer_offload: false + offload_policy: false + reshard_after_forward: true + fsdp_size: -1 + forward_prefetch: false + model_dtype: fp32 + use_orig_params: false + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + use_torch_compile: true + entropy_checkpointing: false + forward_only: false + strategy: fsdp + _target_: verl.workers.config.FSDPActorConfig + strategy: fsdp + ppo_mini_batch_size: 256 + ppo_micro_batch_size: null + ppo_micro_batch_size_per_gpu: null + use_dynamic_bsz: false + ppo_max_token_len_per_gpu: 16384 + clip_ratio: 0.2 + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + freeze_vision_tower: false + policy_loss: + _target_: verl.workers.config.PolicyLossConfig + loss_mode: vanilla + clip_cov_ratio: 0.0002 + clip_cov_lb: 1.0 + clip_cov_ub: 5.0 + kl_cov_ratio: 0.0002 + ppo_kl_coef: 0.1 + clip_ratio_c: 3.0 + loss_agg_mode: token-mean + entropy_coeff: 0 + tis_imp_ratio_cap: -1 + use_kl_loss: false + use_torch_compile: true + kl_loss_coef: 0.001 + kl_loss_type: low_var_kl + ppo_epochs: 1 + shuffle: false + checkpoint: + _target_: verl.trainer.config.CheckpointConfig + save_contents: + - model + - optimizer + - extra + load_contents: ${.save_contents} + async_save: false + use_fused_kernels: ${oc.select:actor_rollout_ref.model.use_fused_kernels,false} + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + grad_clip: 1.0 + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + entropy_checkpointing: false + use_remove_padding: ${oc.select:actor_rollout_ref.model.use_remove_padding,false} + ref: + strategy: ${actor_rollout_ref.actor.strategy} + use_torch_compile: ${oc.select:actor_rollout_ref.actor.use_torch_compile,true} + log_prob_micro_batch_size: null + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + wrap_policy: + min_num_params: 0 + param_offload: false + optimizer_offload: false + offload_policy: false + reshard_after_forward: true + fsdp_size: -1 + forward_prefetch: false + model_dtype: fp32 + use_orig_params: false + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + use_torch_compile: true + entropy_checkpointing: false + forward_only: false + strategy: fsdp + model: null + ulysses_sequence_parallel_size: ${oc.select:actor_rollout_ref.actor.ulysses_sequence_parallel_size,1} + entropy_from_logits_with_chunking: false + entropy_checkpointing: false + rollout: + _target_: verl.workers.config.RolloutConfig + name: ??? + mode: sync + temperature: 1.0 + top_k: -1 + top_p: 1 + prompt_length: ${oc.select:data.max_prompt_length,512} + response_length: ${oc.select:data.max_response_length,512} + dtype: bfloat16 + gpu_memory_utilization: 0.5 + ignore_eos: false + enforce_eager: false + cudagraph_capture_sizes: null + free_cache_engine: true + tensor_model_parallel_size: 2 + data_parallel_size: 1 + expert_parallel_size: 1 + max_num_batched_tokens: 8192 + max_model_len: null + max_num_seqs: 1024 + enable_chunked_prefill: true + enable_prefix_caching: true + load_format: dummy + log_prob_micro_batch_size: null + log_prob_micro_batch_size_per_gpu: null + log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + disable_log_stats: true + do_sample: true + 'n': 1 + over_sample_rate: 0 + multi_stage_wake_up: false + engine_kwargs: + vllm: {} + sglang: {} + val_kwargs: + _target_: verl.workers.config.SamplingConfig + top_k: -1 + top_p: 1.0 + temperature: 0 + 'n': 1 + do_sample: false + multi_turn: + _target_: verl.workers.config.MultiTurnConfig + enable: false + max_assistant_turns: null + tool_config_path: null + max_user_turns: null + max_parallel_calls: 1 + max_tool_response_length: 256 + tool_response_truncate_side: middle + interaction_config_path: null + use_inference_chat_template: false + tokenization_sanity_check_mode: strict + format: hermes + num_repeat_rollouts: null + calculate_log_probs: false + agent: + _target_: verl.workers.config.AgentLoopConfig + num_workers: 8 + agent_loop_config_path: null + custom_async_server: + _target_: verl.workers.config.CustomAsyncServerConfig + path: null + name: null + update_weights_bucket_megabytes: 512 + trace: + _target_: verl.workers.config.TraceConfig + backend: null + token2text: false + skip_rollout: false + skip_dump_dir: /tmp/rollout_dump + skip_tokenizer_init: true + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: ${oc.select:actor_rollout_ref.actor.profiler.enable,false} + all_ranks: ${oc.select:actor_rollout_ref.actor.profiler.all_ranks,false} + ranks: ${oc.select:actor_rollout_ref.actor.profiler.ranks,[]} + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} + layered_summon: false + model: + _target_: verl.workers.config.HFModelConfig + path: ~/models/deepseek-llm-7b-chat + hf_config_path: null + tokenizer_path: null + use_shm: false + trust_remote_code: false + custom_chat_template: null + external_lib: null + override_config: {} + enable_gradient_checkpointing: true + enable_activation_offload: false + use_remove_padding: false + lora_rank: 0 + lora_alpha: 16 + target_modules: all-linear + exclude_modules: null + use_liger: false + use_fused_kernels: false + fused_kernel_options: + impl_backend: torch + hybrid_engine: true + nccl_timeout: 600 +data: + tokenizer: null + use_shm: false + train_files: ~/data/rlhf/gsm8k/train.parquet + val_files: ~/data/rlhf/gsm8k/test.parquet + prompt_key: prompt + reward_fn_key: data_source + max_prompt_length: 512 + max_response_length: 512 + train_batch_size: 1024 + val_batch_size: null + return_raw_input_ids: false + return_raw_chat: false + return_full_prompt: false + shuffle: true + dataloader_num_workers: 8 + validation_shuffle: false + filter_overlong_prompts: false + filter_overlong_prompts_workers: 1 + truncation: error + image_key: images + video_key: videos + trust_remote_code: false + custom_cls: + path: null + name: null + return_multi_modal_inputs: true + sampler: + class_path: null + class_name: null + datagen: + path: null + name: null + apply_chat_template_kwargs: {} +critic: + optim: + _target_: verl.workers.config.FSDPOptimizerConfig + lr: 1.0e-05 + lr_warmup_steps_ratio: 0.0 + total_training_steps: -1 + weight_decay: 0.01 + lr_warmup_steps: -1 + betas: + - 0.9 + - 0.999 + clip_grad: 1.0 + min_lr_ratio: 0.0 + num_cycles: 0.5 + warmup_style: constant + model: + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + wrap_policy: + min_num_params: 0 + param_offload: false + optimizer_offload: false + offload_policy: false + reshard_after_forward: true + fsdp_size: -1 + forward_prefetch: false + model_dtype: fp32 + use_orig_params: false + ulysses_sequence_parallel_size: 1 + entropy_from_logits_with_chunking: false + use_torch_compile: true + entropy_checkpointing: false + forward_only: false + strategy: fsdp + path: ~/models/deepseek-llm-7b-chat + tokenizer_path: ${oc.select:actor_rollout_ref.model.path,"~/models/deepseek-llm-7b-chat"} + override_config: {} + external_lib: ${oc.select:actor_rollout_ref.model.external_lib,null} + trust_remote_code: ${oc.select:actor_rollout_ref.model.trust_remote_code,false} + _target_: verl.workers.config.FSDPCriticModelCfg + use_shm: false + enable_gradient_checkpointing: true + enable_activation_offload: false + use_remove_padding: false + lora_rank: 0 + lora_alpha: 16 + target_modules: all-linear + _target_: verl.workers.config.FSDPCriticConfig + rollout_n: ${oc.select:actor_rollout_ref.rollout.n,1} + strategy: fsdp + enable: null + ppo_mini_batch_size: ${oc.select:actor_rollout_ref.actor.ppo_mini_batch_size,256} + ppo_micro_batch_size: null + ppo_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size,null} + use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + ppo_max_token_len_per_gpu: 32768 + forward_max_token_len_per_gpu: ${.ppo_max_token_len_per_gpu} + ppo_epochs: ${oc.select:actor_rollout_ref.actor.ppo_epochs,1} + shuffle: ${oc.select:actor_rollout_ref.actor.shuffle,false} + cliprange_value: 0.5 + loss_agg_mode: ${oc.select:actor_rollout_ref.actor.loss_agg_mode,token-mean} + checkpoint: + _target_: verl.trainer.config.CheckpointConfig + save_contents: + - model + - optimizer + - extra + load_contents: ${.save_contents} + async_save: false + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + npu: + _target_: verl.utils.profiler.config.NPUToolConfig + contents: [] + level: level1 + analysis: true + discrete: false + torch: + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + step_start: 0 + step_end: null + torch_memory: + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + forward_micro_batch_size: ${oc.select:.ppo_micro_batch_size,null} + forward_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size_per_gpu,null} + ulysses_sequence_parallel_size: 1 + grad_clip: 1.0 +reward_model: + enable: false + enable_resource_pool: false + n_gpus_per_node: 0 + nnodes: 0 + strategy: fsdp + model: + input_tokenizer: ${actor_rollout_ref.model.path} + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + external_lib: ${actor_rollout_ref.model.external_lib} + trust_remote_code: false + use_shm: false + use_remove_padding: false + use_fused_kernels: ${actor_rollout_ref.model.use_fused_kernels} + fsdp_config: + _target_: verl.workers.config.FSDPEngineConfig + wrap_policy: + min_num_params: 0 + param_offload: false + reshard_after_forward: true + fsdp_size: -1 + forward_prefetch: false + micro_batch_size: null + micro_batch_size_per_gpu: null + max_length: null + use_dynamic_bsz: ${critic.use_dynamic_bsz} + forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + reward_manager: naive + launch_reward_fn_async: false + sandbox_fusion: + url: null + max_concurrent: 64 + memory_limit_mb: 1024 + profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: ${oc.select:global_profiler.tool,null} + enable: false + all_ranks: false + ranks: [] + save_path: ${oc.select:global_profiler.save_path,null} + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} + ulysses_sequence_parallel_size: 1 +custom_reward_function: + path: null + name: compute_score +algorithm: + _target_: verl.trainer.config.AlgoConfig + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + norm_adv_by_std_in_grpo: true + use_kl_in_reward: false + kl_penalty: kl + kl_ctrl: + _target_: verl.trainer.config.KLControlConfig + type: fixed + kl_coef: 0.001 + horizon: 10000 + target_kl: 0.1 + use_pf_ppo: false + pf_ppo: + reweight_method: pow + weight_pow: 2.0 +trainer: + balance_batch: true + total_epochs: 30 + total_training_steps: null + project_name: verl_examples + experiment_name: gsm8k + logger: + - console + - wandb + log_val_generations: 0 + rollout_data_dir: null + validation_data_dir: null + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + esi_redundant_time: 0 + resume_mode: auto + resume_from_path: null + val_before_train: true + val_only: false + test_freq: -1 + critic_warmup: 0 + default_hdfs_dir: null + del_local_ckpt_after_load: false + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + max_actor_ckpt_to_keep: null + max_critic_ckpt_to_keep: null + ray_wait_register_center_timeout: 300 + device: cuda + use_legacy_worker_impl: auto +global_profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: null + steps: null + profile_continuous_steps: false + save_path: outputs/profile + global_tool_config: + nsys: + _target_: verl.utils.profiler.config.NsightToolConfig + discrete: false + controller_nsight_options: + trace: cuda,nvtx,cublas,ucx + cuda-memory-usage: 'true' + cuda-graph-trace: graph + worker_nsight_options: + trace: cuda,nvtx,cublas,ucx + cuda-memory-usage: 'true' + cuda-graph-trace: graph + capture-range: cudaProfilerApi + capture-range-end: null + kill: none + torch_memory: + trace_alloc_max_entries: 100000 + stack_depth: 32 + context: all + stacks: all + kw_args: {} +ray_kwargs: + ray_init: + num_cpus: null + timeline_json_file: null diff --git a/e2c/verl/verl/trainer/config/actor/actor.yaml b/e2c/verl/verl/trainer/config/actor/actor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b5764e4a39bec650966babcd1c3699bbb038dd7 --- /dev/null +++ b/e2c/verl/verl/trainer/config/actor/actor.yaml @@ -0,0 +1,216 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# Target class for this configuration +_target_: verl.workers.config.ActorConfig + +# the abstract actor configs +# fsdp, fsdp2 or megatron. must be set. +strategy: ??? + +# Split each sample into sub-batches of this size for PPO +ppo_mini_batch_size: 256 + +# [Deprecated] Global micro batch size +ppo_micro_batch_size: null + +# Local per-GPU micro batch size +ppo_micro_batch_size_per_gpu: null + +# Whether to automatically adjust batch size at runtime +# oc.select: the default val for ref.log_prob_use_dynamic_bsz +use_dynamic_bsz: false + +# Max tokens per GPU in one PPO batch; affects gradient accumulation +# Typically it should be: n * ${data.max_prompt_length} + ${data.max_response_length} +# oc.select: the default val for ref.log_prob_max_token_len_per_gpu +ppo_max_token_len_per_gpu: 16384 + +# PPO clip ratio +clip_ratio: 0.2 + +# Lower bound for asymmetric clipping (used in dual-clip PPO) +clip_ratio_low: 0.2 + +# Upper bound for asymmetric clipping (used in dual-clip PPO) +clip_ratio_high: 0.2 + +# Whether to freeze vision model, if set true, it will be freeze vision model +freeze_vision_tower: false + +# policy loss config +policy_loss: + + # # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.PolicyLossConfig + + # Loss function mode: vanilla / clip-cov / kl-cov /gpg from https://arxiv.org/abs/2505.22617 + loss_mode: "vanilla" + + # Ratio of tokens to be clipped for clip-cov loss + clip_cov_ratio: 0.0002 + + # Lower bound for clip-cov loss + clip_cov_lb: 1.0 + + # Upper bound for clip-cov loss + clip_cov_ub: 5.0 + + # Ratio of tokens to be applied kl penalty for kl-cov loss + kl_cov_ratio: 0.0002 + + # KL divergence penalty coefficient + ppo_kl_coef: 0.1 + +# Constant C in Dual-clip PPO; clips when advantage < 0 and ratio > C +clip_ratio_c: 3.0 + +# Loss aggregation mode: "token-mean", "seq-mean-token-sum", or "seq-mean-token-mean" +loss_agg_mode: token-mean + +# Entropy regularization coefficient in PPO loss +entropy_coeff: 0 + +# Truncated Importance Sampling (TIS): https://fengyao.notion.site/off-policy-rl +# the truncation value C of truncated Importance Sampling (-1 for disable TIS) +tis_imp_ratio_cap: -1 + +# Whether to use KL loss instead of KL reward penalty. True for GRPO +use_kl_loss: false + +# Whether to use torch.compile() +# oc.select: the default val for ref.use_torch_compile +use_torch_compile: true + +# KL loss coefficient when use_kl_loss is enabled. For GRPO +kl_loss_coef: 0.001 + +# Type of KL divergence loss. Options: "kl"(k1), "abs", "mse"(k2), "low_var_kl"(k3), "full" +kl_loss_type: low_var_kl + +# Number of PPO epochs per batch +ppo_epochs: 1 + +# Shuffle training data across PPO epochs +shuffle: false + +# checkpoint configs +checkpoint: + + # Target dataclass for this configuration + _target_: verl.trainer.config.CheckpointConfig + + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + + # For more flexibility, you can specify the contents to load from the checkpoint. + # .xxx refers to the local variable xxx from the same level of hierarchy similar to python pkg + load_contents: ${.save_contents} + + # Whether to save checkpoints asynchronously. Only effective for Megatron as of now. + async_save: False + +# optimizer configs +optim: + + # Learning rate + lr: 1e-6 + + # Warmup steps ratio (used if lr_warmup_steps is 0 or negative) + lr_warmup_steps_ratio: 0.0 + + # Total training steps (must be overridden at runtime) + total_training_steps: -1 + + # Weight decay + weight_decay: 0.01 + + # Prioritized. None, 0 or Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps: -1 + + +# Whether to use custom fused kernels (e.g., FlashAttention, fused MLP) +use_fused_kernels: ${oc.select:actor_rollout_ref.model.use_fused_kernels,false} + +# profile the actor model in `update_policy` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # profiler tool, default same as profiler.tool in global config + # choices: nsys, npu, torch + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on Actor + enable: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config which only related to the role + tool_config: + + # nsys tool config + nsys: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NsightToolConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + + # npu config + npu: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NPUToolConfig + + # Contents to profile, can be empty + # options: npu, cpu, memory, shapes, module, stack + contents: [] + + # Collection level, optional values: level_none, level0, level1, level2. + level: "level1" + + # Whether to automatically parse the data. + analysis: True + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # torch profiler config + torch: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + + # start profile mini-batch in training + # NOTICE: different with global steps config which refers to iteration + # This field only related with mini-batch + step_start: 0 + + # stop profile mini-batch in training + step_end: null + + # torch memory profiler config + torch_memory: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + + # Maximum number of memory allocation entries to track + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + + # Stack trace depth for memory allocations + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} diff --git a/e2c/verl/verl/trainer/config/actor/dp_actor.yaml b/e2c/verl/verl/trainer/config/actor/dp_actor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab27304f7364f5cd8919a3253396fb6b736d64bf --- /dev/null +++ b/e2c/verl/verl/trainer/config/actor/dp_actor.yaml @@ -0,0 +1,42 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# defaults specify the default config from each component +defaults: + + # fsdp optimizer config + - ../optim@optim: fsdp + + # fsdp engine config + - ../engine@fsdp_config: fsdp + + # dp actor config, inheriting from trainer/config/actor/actor.yaml + - actor + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +# Target class for this configuration +_target_: verl.workers.config.FSDPActorConfig + +# TODO(haibin.lin): switch to fsdp2 +strategy: fsdp + +# Gradient clipping for actor updates, specific to the strategy. +grad_clip: 1.0 + +# Sequence parallelism size for Ulysses-style model parallelism +# oc.select: the default val for ref.ulysses_sequence_parallel_size +ulysses_sequence_parallel_size: 1 + +# calculate entropy with chunking to reduce memory peak +entropy_from_logits_with_chunking: False + +# recompute entropy +entropy_checkpointing: False + +# Whether to remove padding tokens in inputs during training +use_remove_padding: ${oc.select:actor_rollout_ref.model.use_remove_padding,false} diff --git a/e2c/verl/verl/trainer/config/actor/megatron_actor.yaml b/e2c/verl/verl/trainer/config/actor/megatron_actor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfb38467de220f3f7e522838105607d1d3e5f193 --- /dev/null +++ b/e2c/verl/verl/trainer/config/actor/megatron_actor.yaml @@ -0,0 +1,20 @@ +# megatron actor config, inheriting from trainer/config/actor/actor.yaml +defaults: + # megatron optimizer config + - ../optim@optim: megatron + + # megatron engine config + - ../engine@megatron: megatron + + - actor + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +_target_: verl.workers.config.McoreActorConfig + +strategy: megatron + +data_loader_seed: null + +load_weight: True diff --git a/e2c/verl/verl/trainer/config/algorithm.py b/e2c/verl/verl/trainer/config/algorithm.py new file mode 100644 index 0000000000000000000000000000000000000000..b66648b6ffb926a2586f1df1ad3b55a865f5f819 --- /dev/null +++ b/e2c/verl/verl/trainer/config/algorithm.py @@ -0,0 +1,87 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Any, Optional + +from verl.base_config import BaseConfig + +__all__ = ["AlgoConfig", "FilterGroupsConfig", "KLControlConfig"] + + +@dataclass +class KLControlConfig(BaseConfig): + """Configuration for KL control. + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + type (str): Type of KL control. Can be "fixed" or "adaptive". + kl_coef (float): Initial coefficient for KL penalty. + horizon (int): Horizon value for adaptive controller. + target_kl (float): Target KL divergence for adaptive controller. + """ + + type: str = "fixed" + kl_coef: float = 0.001 + horizon: int = 10000 + target_kl: float = 0.1 + + +@dataclass +class FilterGroupsConfig(BaseConfig): + """Configuration for filter groups (used in DAPO and Entropy). + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + enable (bool): Whether to enable filter groups. + metric (Optional[str]): Metric to use for filtering: "acc", "score", "seq_reward", "seq_final_reward", etc. + max_num_gen_batches (int): Non-positive values mean no upper limit. + """ + + enable: bool = False + metric: Optional[str] = None + max_num_gen_batches: int = 0 + + +@dataclass +class AlgoConfig(BaseConfig): + """Configuration for the algorithm. + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + gamma (float): Discount factor for future rewards. + lam (float): Trade-off between bias and variance in the GAE estimator. + adv_estimator (str): Advantage estimator type: "gae", "grpo", "reinforce_plus_plus", etc. + norm_adv_by_std_in_grpo (bool): Whether to normalize advantages by std (specific to GRPO). + use_kl_in_reward (bool): Whether to enable in-reward KL penalty. + kl_penalty (str): How to estimate KL divergence: "kl", "abs", "mse", "low_var_kl", or "full". + kl_ctrl (KLControlConfig): KL control configuration. + use_pf_ppo (bool): Whether to enable preference feedback PPO. + pf_ppo (dict[str, Any]): Preference feedback PPO settings. + filter_groups (Optional[FilterGroupsConfig]): Filter groups configuration, used in DAPO and Entropy + """ + + gamma: float = 1.0 + lam: float = 1.0 + adv_estimator: str = "gae" + norm_adv_by_std_in_grpo: bool = True + use_kl_in_reward: bool = False + kl_penalty: str = "kl" + kl_ctrl: KLControlConfig = field(default_factory=KLControlConfig) + use_pf_ppo: bool = False + pf_ppo: dict[str, Any] = field(default_factory=dict) + filter_groups: Optional[FilterGroupsConfig] = None diff --git a/e2c/verl/verl/trainer/config/config.py b/e2c/verl/verl/trainer/config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..3c326d0145b1f3122668057a36203591e782b4fd --- /dev/null +++ b/e2c/verl/verl/trainer/config/config.py @@ -0,0 +1,77 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Any, Optional + +from verl.base_config import BaseConfig + +__all__ = ["CheckpointConfig", "ProfileConfig", "BaseModelConfig"] + + +@dataclass +class CheckpointConfig(BaseConfig): + """Configuration for model checkpointing. + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + save_contents (list[str]): What to include in saved checkpoints. + Options: 'model', 'optimizer', 'extra', 'hf_model'. + load_contents (list[str]): Contents to load from checkpoint. Defaults to same as save_contents. + async_save (bool): Whether to save checkpoints asynchronously. Only implemented for Megatron as of now. + """ + + save_contents: list[str] = field(default_factory=lambda: ["model", "optimizer", "extra"]) + load_contents: list[str] = field(default_factory=lambda: ["model", "optimizer", "extra"]) + async_save: bool = False + + +@dataclass +class ProfileConfig(BaseConfig): + """Configuration for profiling. + + The inheritance from BaseConfig provides omegaconf.DictConfig-like interface for a dataclass config. + + Args: + profile_ranks (Optional[list[int]]): List of ranks to profile. None means all ranks. + step_start (int): Starting step for profiling. + step_end (int): Ending step for profiling. + save_path (Optional[str]): Path to save profiling results. + """ + + profile_ranks: Optional[list[int]] = None + step_start: int = -1 + step_end: int = -1 + save_path: Optional[str] = None + + +@dataclass +class BaseModelConfig(BaseConfig): + """Base configuration for a model. + Contains core settings for loading and initializing a pretrained model checkpoint. + + Args: + path (str): Path to pretrained model weights. + tokenizer_path (Optional[str]): Tokenizer path (defaults to actor's model path if not set). + override_config (dict): Hugging Face config override. + external_lib (Optional[str]): External model implementation (optional). + trust_remote_code (bool): Whether to trust remote code from Hugging Face models. + """ + + path: str = "~/models/deepseek-llm-7b-chat" + tokenizer_path: Optional[str] = None + override_config: dict[str, Any] = field(default_factory=dict) + external_lib: Optional[str] = None + trust_remote_code: bool = False diff --git a/e2c/verl/verl/trainer/config/critic/critic.yaml b/e2c/verl/verl/trainer/config/critic/critic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f201a34b40c36c5a45f3dda61084bc7dc78dd70a --- /dev/null +++ b/e2c/verl/verl/trainer/config/critic/critic.yaml @@ -0,0 +1,176 @@ +# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs +_target_: verl.workers.config.CriticConfig + +# Number of rollouts per update (mirrors actor rollout_n) +rollout_n: ${oc.select:actor_rollout_ref.rollout.n,1} + +# fsdp or fsdp2 strategy used for critic model training +strategy: ??? + +# whether to enable the critic worker. +# by default it is only enabled if advantage estimator is gae +# set it to True manually if you always want to enable critic worker +enable: null + +# optimizer configs +optim: + + # Learning rate + lr: 1e-5 + + # Warmup steps ratio; total steps will be injected at runtime + lr_warmup_steps_ratio: 0.0 + + # Total training steps (must be overridden at runtime) + total_training_steps: -1 + + # Weight decay + weight_decay: 0.01 + + # Prioritized. None, 0 or Negative values mean delegating to lr_warmup_steps_ratio. + lr_warmup_steps: -1 + + +# model config for the critic +model: + + # Path to pretrained model weights + path: ~/models/deepseek-llm-7b-chat + + # Tokenizer path (defaults to actor's model path) + tokenizer_path: ${oc.select:actor_rollout_ref.model.path,"~/models/deepseek-llm-7b-chat"} + + # Hugging Face config override + override_config: {} + + # External model implementation (optional) + external_lib: ${oc.select:actor_rollout_ref.model.external_lib,null} + + # Whether to trust remote code from Hugging Face models + trust_remote_code: ${oc.select:actor_rollout_ref.model.trust_remote_code,false} + +# PPO mini-batch size per update +ppo_mini_batch_size: ${oc.select:actor_rollout_ref.actor.ppo_mini_batch_size,256} + +# [Deprecated] Global micro batch size +ppo_micro_batch_size: null + +# Local per-GPU micro batch size +ppo_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size,null} + +# Whether to automatically adjust batch size at runtime +use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + +# Max tokens per GPU in one PPO batch (doubled for critic) +ppo_max_token_len_per_gpu: 32768 + +# Max token length per GPU in forward pass +forward_max_token_len_per_gpu: ${.ppo_max_token_len_per_gpu} + +# Number of PPO epochs per batch +ppo_epochs: ${oc.select:actor_rollout_ref.actor.ppo_epochs,1} + +# Shuffle training data across PPO epochs +shuffle: ${oc.select:actor_rollout_ref.actor.shuffle,false} + +# PPO value function clipping range +cliprange_value: 0.5 + +# Loss aggregation mode: "token-mean", "seq-mean-token-sum", or "seq-mean-token-mean" +loss_agg_mode: ${oc.select:actor_rollout_ref.actor.loss_agg_mode,token-mean} + +# checkpoint configs +checkpoint: + + # Target dataclass for this configuration + _target_: verl.trainer.config.CheckpointConfig + + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ['model', 'optimizer', 'extra'] + + # What to include when loading checkpoints + load_contents: ${.save_contents} + + # Whether to save checkpoints asynchronously. Only effective for Megatron as of now. + async_save: False + +# profile the critic model in `update_critic` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # profiler tool, default same as profiler.tool in global config + # choices: nsys, npu, torch, torch_memory + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on Critic + enable: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config which only related to the role + tool_config: + + # nsys tool config + nsys: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NsightToolConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + + # npu config + npu: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NPUToolConfig + + # Contents to profile, can be empty + # options: npu, cpu, memory, shapes, module, stack + contents: [] + + # Collection level, optional values: level_none, level0, level1, level2. + level: "level1" + + # Whether to automatically parse the data. + analysis: True + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # torch profiler config + torch: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + + # start profile mini-batch in training + # NOTICE: different with global steps config which refers to iteration + # This field only related with mini-batch + step_start: 0 + + # stop profile mini-batch in training + step_end: null + + # torch memory profiler config + torch_memory: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + + # Maximum number of memory allocation entries to track + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + + # Stack trace depth for memory allocations + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} + \ No newline at end of file diff --git a/e2c/verl/verl/trainer/config/critic/dp_critic.yaml b/e2c/verl/verl/trainer/config/critic/dp_critic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..240cc2de0a7f8f322be2aaca34811e9bf66e953e --- /dev/null +++ b/e2c/verl/verl/trainer/config/critic/dp_critic.yaml @@ -0,0 +1,65 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# defaults specify the default config from each component +defaults: + + # fsdp optimizer config + - ../optim@optim: fsdp + + # fsdp engine config + - ../engine@model.fsdp_config: fsdp + + # dp actor config, inheriting from trainer/config/critic/critic.yaml + - critic + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs +_target_: verl.workers.config.FSDPCriticConfig + +# distribution strategy. Options: fsdp (deprecating), fsdp2 +strategy: fsdp + +# model config for the critic +model: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.FSDPCriticModelCfg + + # Whether to use shared memory for loading the model + use_shm: False + + # Enable gradient checkpointing to save memory + enable_gradient_checkpointing: True + + # Offload activations to CPU to reduce GPU memory usage + enable_activation_offload: False + + # Use remove padding optimization (saves compute) + use_remove_padding: False + + # Set to positive value to enable LoRA (e.g., 32) + lora_rank: 0 + + # LoRA scaling factor + lora_alpha: 16 + + # LoRA target modules: "all-linear" or list of linear projection layers + target_modules: all-linear + +# Forward-only batch size during inference (global) +forward_micro_batch_size: ${oc.select:.ppo_micro_batch_size,null} + +# Forward-only batch size during inference (per GPU) +forward_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size_per_gpu,null} + +# Sequence parallelism size for Ulysses-style model parallelism +ulysses_sequence_parallel_size: 1 + +# Gradient clipping for critic updates +grad_clip: 1.0 diff --git a/e2c/verl/verl/trainer/config/critic/megatron_critic.yaml b/e2c/verl/verl/trainer/config/critic/megatron_critic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4a8509f2278a7ad845b7659196792683b6eaeb5 --- /dev/null +++ b/e2c/verl/verl/trainer/config/critic/megatron_critic.yaml @@ -0,0 +1,43 @@ +# defaults specify the default config from each component +defaults: + + # megatron optimizer config + - ../optim@optim: megatron + + # megatron engine config + - ../engine@megatron: megatron + + # dp actor config, inheriting from trainer/config/critic/critic.yaml + - critic + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs +_target_: verl.workers.config.McoreCriticConfig + +strategy: megatron + +# seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using megatron +nccl_timeout: 600 + +# model config for the critic +model: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.BaseModelConfig + + # override default empty mapping + override_config: + + model_config: {} + + moe_config: + + freeze_moe_router: False + +# Whether to load initial weights +load_weight: True + +# seed for data loader +data_loader_seed: ${oc.select:actor_rollout_ref.actor.data_loader_seed,null} diff --git a/e2c/verl/verl/trainer/config/data/legacy_data.yaml b/e2c/verl/verl/trainer/config/data/legacy_data.yaml new file mode 100644 index 0000000000000000000000000000000000000000..028405b4238e5e344fb1fa3f1f0a4b1a83403b69 --- /dev/null +++ b/e2c/verl/verl/trainer/config/data/legacy_data.yaml @@ -0,0 +1,112 @@ +# Tokenizer class or path. If null, it will be inferred from the model. +tokenizer: null + +# Whether to use shared memory for data loading. +use_shm: False + +# Training set parquet. Can be a list or a single file. +# The program will read all files into memory, so it can't be too large (< 100GB). +# The path can be either a local path or an HDFS path. +# For HDFS path, we provide utils to download it to DRAM and convert it to a local path. +train_files: ~/data/rlhf/gsm8k/train.parquet + +# Validation parquet. Can be a list or a single file. +val_files: ~/data/rlhf/gsm8k/test.parquet + +# The field in the dataset where the prompt is located. Default is 'prompt'. +prompt_key: prompt + +# The field used to select the reward function (if using different ones per example). +reward_fn_key: data_source + +# Maximum prompt length. All prompts will be left-padded to this length. +# An error will be reported if the length is too long. +# oc.select: default val for rollout.prompt_length +max_prompt_length: 512 + +# Maximum response length. Rollout in RL algorithms (e.g. PPO) generates up to this length. +# oc.select: default val for rollout.response_length +max_response_length: 512 + +# Batch size sampled for one training iteration of different RL algorithms. +train_batch_size: 1024 + +# Batch size used during validation. Can be null. +val_batch_size: null + +# Whether to return the original input_ids without adding chat template. +# This is used when the reward model's chat template differs from the policy. +# If using a model-based RM with different templates, this should be True. +return_raw_input_ids: False + +# Whether to return the original chat (prompt) without applying chat template. +return_raw_chat: False + +# Whether to return the full prompt with chat template. +return_full_prompt: False + +# Whether to shuffle the data in the dataloader. +shuffle: True + +# num dataloader workers +dataloader_num_workers: 8 + +# Whether to shuffle the validation set. +validation_shuffle: False + +# Whether to filter overlong prompts. +filter_overlong_prompts: False + +# Number of workers for filtering overlong prompts. +# For large-scale datasets, filtering can be time-consuming. +# Use multiprocessing to speed up. Default is 1. +filter_overlong_prompts_workers: 1 + +# Truncate the input_ids or prompt if they exceed max_prompt_length. +# Options: 'error', 'left', 'right', 'middle'. Default is 'error'. +truncation: error + +# The field in the multi-modal dataset where the image is located. Default is 'images'. +image_key: images + +# The field in the multi-modal dataset where the video is located. +video_key: videos + +# If the remote tokenizer has a Python file, this flag determines whether to allow using it. +trust_remote_code: False + +# Optional: specify a custom dataset class path and name if overriding default loading behavior. +custom_cls: + + # The path to the file containing your customized dataset class. If not specified, pre-implemented dataset will be used. + path: null + + # The name of the dataset class within the specified file. + name: null + +# Whether to return multi-modal inputs in the dataset. Set to False if rollout generates new multi-modal inputs. +return_multi_modal_inputs: True + +# settings related to data sampler +sampler: + + # the path to the module containing a curriculum class which implements the + # AbstractSampler interface + class_path: null + + # the name of the curriculum class like `MySampler` + class_name: null + +# Data generation configuration for augmenting the dataset. +datagen: + + # The path to the file containing your customized data generation class. + # E.g. 'pkg://verl.experimental.dynamic_dataset.dynamicgen_dataset' + path: null + + # The class name of the data generation class within the specified file. + # E.g. 'MockDataGenerator' + name: null + +# Additional kwargs when calling tokenizer.apply_chat_template +apply_chat_template_kwargs: {} diff --git a/e2c/verl/verl/trainer/config/engine/fsdp.yaml b/e2c/verl/verl/trainer/config/engine/fsdp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3f551adb669bff28a4254ef45d97b8d0a57d398 --- /dev/null +++ b/e2c/verl/verl/trainer/config/engine/fsdp.yaml @@ -0,0 +1,53 @@ +# Target class for this configuration +_target_: verl.workers.config.FSDPEngineConfig + +# policy for wrapping the model +wrap_policy: + + # Minimum number of parameters to trigger wrapping a layer with FSDP + min_num_params: 0 + +# Whether to offload model parameters to CPU (trades speed for memory) +# Note that this differs from the offload_policy in FSDP +param_offload: false + +# Whether to offload optimizer state to CPU +# Note that this differs from the offload_policy in FSDP +optimizer_offload: false + +# Only for FSDP2: offload param/grad/optimizer during train +offload_policy: false + +# Only for FSDP2: Reshard after forward pass to reduce memory footprint +reshard_after_forward: true + +# Number of GPUs in each FSDP shard group; -1 means auto +fsdp_size: -1 + +# Only for FSDP1: FSDP1 configuration, prefetch the next forward-pass all-gather +# before the current forward computation. +forward_prefetch: False + +# model dtype of fsdp +model_dtype: fp32 + +# Whether to use original parameters in fsdp. Only avaiable in fsdp1 +use_orig_params: false + +# ulysses sequence parallel size +ulysses_sequence_parallel_size: 1 + +# Whether to use entropy_from_logits_with_chunking in fsdp. +entropy_from_logits_with_chunking: false + +# Whether to use torch compile in fsdp. +use_torch_compile: true + +# Whether to use entropy checkpointing in fsdp. +entropy_checkpointing: false + +# Whether to use forward only in fsdp. +forward_only: false + +# fsdp or fsdp2 +strategy: fsdp diff --git a/e2c/verl/verl/trainer/config/engine/megatron.yaml b/e2c/verl/verl/trainer/config/engine/megatron.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9bb4ef01fb1e31eea6e645bbb2a7bcbc3824b59 --- /dev/null +++ b/e2c/verl/verl/trainer/config/engine/megatron.yaml @@ -0,0 +1,78 @@ +# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs +_target_: verl.workers.config.McoreEngineConfig + +# Whether to offload model parameters to CPU +param_offload: False + +# Whether to offload gradients to CPU +grad_offload: False + +# Whether to offload optimizer state to CPU +optimizer_offload: False + +# tensor model parallel size +tensor_model_parallel_size: 1 + +# expert model parallel size +expert_model_parallel_size: 1 + +# expert tensor parallel size +expert_tensor_parallel_size: 1 + +# pipeline model parallel size +pipeline_model_parallel_size: 1 + +# virtual pipeline model parallel size +virtual_pipeline_model_parallel_size: null + +# context parallel size +context_parallel_size: 1 + +# sequence parallel +sequence_parallel: True + +# Whether to use distributed optimizer +use_distributed_optimizer: True + +# Whether to use distributed checkpointing +use_dist_checkpointing: False + +# distributed checkpointing path +dist_checkpointing_path: null + +# oc.select: default val for ref.megatron.seed +seed: 42 + +# Allow to override Distributed Data Parallel (DDP) config +override_ddp_config: {} + +# additional transformer config like: num_layers_in_first(/last)_pipeline_stage +# oc.select: default val for ref.megatron.override_transformer_config +override_transformer_config: + # Recompute configuration, same as in megatron.training.arguments + # default use minimal performance-interference recompute methods + # Recompute granualarity, choices: ["full", "selective"] + recompute_granularity: null + + # Recompute modules, multiple choices: ["core_attn", "moe_act", "layernorm", "mla_up_proj", "mlp", "moe"] + # Please use correct module in matched model + recompute_modules: ["core_attn"] + + # 'uniform', 'block' + # 'uniform' divides the total number of transformer layers and checkpoints the input activation of each chunk + # 'block' checkpoints the specified number of layers per pipeline stage at the specified granularity + recompute_method: null + + # 'full' will checkpoint the entire transformer layer and 'selective' only checkpoints memory intensive part of attention + recompute_num_layers: null + + # Attention backend to use (flash,fused,unfused,local,auto). Defaults to auto in mcore, flash in verl + attention_backend: flash + +override_mcore_model_config: {} + +# oc.select: default val for ref.megatron.use_mbridge +use_mbridge: False + +# whether to use forward only +forward_only: False diff --git a/e2c/verl/verl/trainer/config/evaluation.yaml b/e2c/verl/verl/trainer/config/evaluation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a88d77f1e73b6c3cce1972f639fcafb412669fa --- /dev/null +++ b/e2c/verl/verl/trainer/config/evaluation.yaml @@ -0,0 +1,15 @@ +data: + path: /tmp/math_Qwen2-7B-Instruct.parquet + prompt_key: prompt + response_key: responses + data_source_key: data_source + reward_model_key: reward_model + +custom_reward_function: + path: null + name: compute_score + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. + timeline_json_file: null diff --git a/e2c/verl/verl/trainer/config/generation.yaml b/e2c/verl/verl/trainer/config/generation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a57858ff4bd65a88b370cffe7450b962f21b24fe --- /dev/null +++ b/e2c/verl/verl/trainer/config/generation.yaml @@ -0,0 +1,57 @@ +trainer: + nnodes: 1 + n_gpus_per_node: 8 + device: cuda + +data: + path: ~/data/rlhf/math/test.parquet + prompt_key: prompt + n_samples: 5 + output_path: /opt/tiger/math_Qwen2-7B-Instruct.parquet + batch_size: 128 + +model: + path: ~/models/Qwen2-7B-Instruct + external_lib: null +rollout: + _target_: verl.workers.config.RolloutConfig + name: vllm + mode: sync # sync: LLM, async: AsyncLLM + temperature: 1.0 + top_k: 50 # 0 for hf rollout, -1 for vllm rollout + top_p: 0.7 + prompt_length: 1536 + response_length: 512 + # for vllm rollout + dtype: bfloat16 # should align with FSDP + gpu_memory_utilization: 0.5 + ignore_eos: False + enforce_eager: True + free_cache_engine: True + load_format: auto + tensor_model_parallel_size: 1 + max_num_batched_tokens: 8192 + max_model_len: null + max_num_seqs: 1024 + log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu + log_prob_micro_batch_size_per_gpu: 8 + # for hf rollout + do_sample: True + disable_log_stats: True + enable_chunked_prefill: True + n: 1 + # support logging rollout prob for debugging purpose + calculate_log_probs: False +actor: + strategy: fsdp # This is for backward-compatibility + ulysses_sequence_parallel_size: 1 # sp size + entropy_from_logits_with_chunking: False # calculate entropy with chunking to reduce memory peak + entropy_checkpointing: False # recompute entropy + fsdp_config: + fsdp_size: -1 + forward_prefetch: False # FSDP1 forward_prefetch configuration + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. + timeline_json_file: null diff --git a/e2c/verl/verl/trainer/config/model/hf_model.yaml b/e2c/verl/verl/trainer/config/model/hf_model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac08c1bb7fd46b9bbd47a3f114690ed23a8daddb --- /dev/null +++ b/e2c/verl/verl/trainer/config/model/hf_model.yaml @@ -0,0 +1,64 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +_target_: verl.workers.config.HFModelConfig + +# path to the huggingface model +path: ~/models/deepseek-llm-7b-chat + +# config to the huggingface config. In case it is not the same as path +hf_config_path: null + +# path to the huggingface tokenizer. In case it is not the same as path +tokenizer_path: null + +# whether to use shared memory for model loading +use_shm: False + +# whether to trust remote code. +trust_remote_code: False + +# custom chat template for the model +custom_chat_template: null + +# whether to use external libs for the model +external_lib: null + +# override hf config +override_config: {} + +# whether to enable gradient checkpointing. Only valid when we use hf model definition +enable_gradient_checkpointing: True + +# whether to enable activation offload. Only valid when we use hf model definition +enable_activation_offload: False + +# whether to use remove padding. Only valid when we use hf model definition +use_remove_padding: False + +# Set to positive value to enable LoRA (e.g., 32) +lora_rank: 0 + +# LoRA scaling factor +lora_alpha: 16 + +# Target modules for LoRA adaptation +target_modules: all-linear + +# Exclude modules from LoRA adaptation +exclude_modules: null + +# whether to use liger. Only valid when we use hf model definition +use_liger: False + +# whether to use fused kernels. +use_fused_kernels: False + +# fused kernel options. +fused_kernel_options: + + # the implementation backend for fused kernels. + impl_backend: torch diff --git a/e2c/verl/verl/trainer/config/npu_profile/npu_profile.yaml b/e2c/verl/verl/trainer/config/npu_profile/npu_profile.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52bb52d3f40d7d6695708b7414c82c0136d8fba2 --- /dev/null +++ b/e2c/verl/verl/trainer/config/npu_profile/npu_profile.yaml @@ -0,0 +1,34 @@ +# Options for the npu profiler +options: + + # Storage path of collected data. + save_path: ./profiler_data + + # The roles that will be profiled. Only takes effect in discrete mode. + # optional values: all, rollout_generate, actor_compute_log_prob, actor_update and ref_compute_log_prob. + # "all" means all roles will be profiled. + roles: ["all"] + + # Collection level, optional values: level_none, level0, level1, level2. + level: level1 + + # Whether to enable memory analysis. + with_memory: False + + # Whether to record tensor shape. + record_shapes: False + + # Whether to record Device-side performance data. + with_npu: True + + # Whether to record Host-side performance data. + with_cpu: True + + # Whether to record Python call stack information. + with_module: False + + # Whether to record operator call stack information. + with_stack: False + + # Whether to automatically parse the data. + analysis: True \ No newline at end of file diff --git a/e2c/verl/verl/trainer/config/optim/fsdp.yaml b/e2c/verl/verl/trainer/config/optim/fsdp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2ca6b2e8da31fbd060f262a56e4c2e20fb17092 --- /dev/null +++ b/e2c/verl/verl/trainer/config/optim/fsdp.yaml @@ -0,0 +1,33 @@ +# Target class for this configuration +_target_: verl.workers.config.FSDPOptimizerConfig + +# Learning rate +lr: 1e-3 + +# LR warmup steps ratio +lr_warmup_steps_ratio: 0.0 + +# Total training steps +total_training_steps: -1 + +# Weight decay +weight_decay: 0.01 + +# LR warmup steps +lr_warmup_steps: -1 + +# Betas for Adam optimizer +betas: [0.9, 0.999] + +# Clip gradient +clip_grad: 1.0 + +# Minimum LR ratio for cosine schedule +min_lr_ratio: 0.0 + +# Number of cosine cycles in LR schedule +num_cycles: 0.5 + +# LR warmup style: "constant" or "cosine" +warmup_style: constant + diff --git a/e2c/verl/verl/trainer/config/optim/megatron.yaml b/e2c/verl/verl/trainer/config/optim/megatron.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3e49b7df8e59d33f51b50b943d9353af66d296c --- /dev/null +++ b/e2c/verl/verl/trainer/config/optim/megatron.yaml @@ -0,0 +1,49 @@ +_target_: verl.workers.config.McoreOptimizerConfig + +# Learning rate +lr: 1e-3 + +# LR warmup steps ratio +lr_warmup_steps_ratio: 0.0 + +# Total training steps +total_training_steps: -1 + +# Weight decay +weight_decay: 0.01 + +# LR warmup steps +lr_warmup_steps: -1 + +# Betas for Adam optimizer +betas: [0.9, 0.999] + +# Clip gradient +clip_grad: 1.0 + +# optimizer type +optimizer: adam + +# initial learning rate for warmup, default to 0.0 +lr_warmup_init: 0.0 + +lr_decay_steps: null + +# select from constant/linear/cosine/inverse_square_root +lr_decay_style: constant + +# minimum learning rate, default to 0.0 +min_lr: 0.0 + +# select from constant/linear/cosine +weight_decay_incr_style: constant + +# select from constant/exponential/cosine +lr_wsd_decay_style: exponential + +lr_wsd_decay_steps: null + +# use checkpoint optimizer parameter scheduler +use_checkpoint_opt_param_scheduler: False + +override_optimizer_config: {} diff --git a/e2c/verl/verl/trainer/config/ppo_megatron_trainer.yaml b/e2c/verl/verl/trainer/config/ppo_megatron_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..238522ebcecfb77c13c44c5bd5e1ed4e6e8a1b86 --- /dev/null +++ b/e2c/verl/verl/trainer/config/ppo_megatron_trainer.yaml @@ -0,0 +1,173 @@ +# specify the default per-component configs +defaults: + # @.: + # actor_rollout_ref.actor: trainer/config/actor/megatron_actor.yaml + - actor@actor_rollout_ref.actor: megatron_actor + # data: trainer/config/data/legacy_data.yaml + - data@data: legacy_data + # load the reference default config, then apply the fields in the current yaml + # Reference model config. + # Reference model will be enabled when actor.use_kl_loss or/and algorithm.use_kl_in_reward is/are True. + - ref@actor_rollout_ref.ref: megatron_ref + # Rollout model config. + - rollout@actor_rollout_ref.rollout: rollout + # Critic model config. + - critic@critic: megatron_critic + # Reward model config. + - reward_model@reward_model: megatron_reward_model + - _self_ + +actor_rollout_ref: + hybrid_engine: True + + nccl_timeout: 600 # seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using megatron + + model: + + path: ~/models/deepseek-llm-7b-chat + + custom_chat_template: null + + external_lib: null + + override_config: + model_config: {} + + moe_config: + freeze_moe_router: False + + use_fused_kernels: False # Whether to use custom fused kernels (PostProcessing, for memory efficiency) + + trust_remote_code: False + + # Whether to remove padding tokens in inputs during training + use_remove_padding: false + + rollout: + layer_name_map: + qkv_layer_name: qkv + gate_proj_layer_name: gate_up + +custom_reward_function: + path: null + name: compute_score + +algorithm: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.AlgoConfig + gamma: 1.0 + lam: 1.0 + adv_estimator: gae + norm_adv_by_std_in_grpo: True + use_kl_in_reward: False + kl_penalty: kl # how to estimate kl divergence + kl_ctrl: + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.KLControlConfig + type: fixed + kl_coef: 0.001 + horizon: 10000 + target_kl: 0.1 + use_pf_ppo: False + pf_ppo: + reweight_method: pow # ["pow", "max_min", "max_random"] + weight_pow: 2.0 + +trainer: + balance_batch: True + total_epochs: 30 + total_training_steps: null + project_name: verl_examples + experiment_name: gsm8k + logger: ["console", "wandb"] + log_val_generations: 0 + nnodes: 1 + n_gpus_per_node: 8 + save_freq: -1 + esi_redundant_time: 0 + + # auto: find the last ckpt to resume. If can't find, start from scratch + resume_mode: auto # or disable or resume_path if resume_from_path is set + resume_from_path: null + del_local_ckpt_after_load: False + val_before_train: True + test_freq: -1 + critic_warmup: 0 + default_hdfs_dir: null + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + max_actor_ckpt_to_keep: null + max_critic_ckpt_to_keep: null + # The timeout for ray worker group to wait for the register center to be ready + ray_wait_register_center_timeout: 300 + device: cuda + # Directory for logging rollout data; no dump if null + rollout_data_dir: null + +global_profiler: + _target_: verl.utils.profiler.ProfilerConfig + tool: null # choose between nsys, npu, torch, torch_memory + steps: null # profile steps + profile_continuous_steps: False + save_path: "outputs/profile" # profiler saving path + # Specific tool configs, can use +profiler.tool_config.[tool].xxx to config + global_tool_config: + # nsys config + nsys: + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # controller Nvidia Nsight Systems Options. Must set when profile_steps is not None. + ## reference https://docs.nvidia.com/nsight-systems/UserGuide/index.html + ## reference https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html + controller_nsight_options: + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # worker Nvidia Nsight Systems Options. Must set when profile_steps is not None. + worker_nsight_options: + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # Profiling only in a range of torch.cuda.profiler.start and stop. Do not change this config. + capture-range: "cudaProfilerApi" + + # Specify the desired behavior when a capture range ends. + # In verl we need the torch.cuda.profiler.start/stop pair to repeats n times. + # valid values are "repeat-shutdown:n" or null. + # For normal whole step profiling, n = len(profile_steps); + # but for discrete profiling, n = len(profile_steps) * Number(subtasks). + # Or you can just leave it null and the program will use n = len(profile_steps) * 6; + capture-range-end: null + + # Send signal to the target application's process group. We let the program to exit by itself. + kill: none + + # enable memory visualization for debugging memory usage + torch_memory: + # Maximum number of allocation entries to record + trace_alloc_max_entries: 100_000 + # The depth of the call stack to capture for each allocation + stack_depth: 32 + # 'alloc': records only allocation events || 'state': records memory state changes || 'all': records both. + context: "all" + # 'python': records Python stacks || 'cpp': records C++ stacks (available in some versions) || 'all': records both. + stacks: "all" + # devices, record_context etc. + kw_args: {} + +ray_kwargs: + ray_init: + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. + timeline_json_file: null diff --git a/e2c/verl/verl/trainer/config/ppo_trainer.yaml b/e2c/verl/verl/trainer/config/ppo_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8214833545f10ba92fcbc57afabaac4feaf32ac --- /dev/null +++ b/e2c/verl/verl/trainer/config/ppo_trainer.yaml @@ -0,0 +1,308 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# specify the default per-component configs +defaults: + + # @.: + # actor_rollout_ref.actor: trainer/config/actor/dp_actor.yaml + - actor@actor_rollout_ref.actor: dp_actor + + # data: trainer/config/data/legacy_data.yaml + - data@data: legacy_data + + # Reference model config. + # Reference model will be enabled when actor.use_kl_loss or/and algorithm.use_kl_in_reward is/are True. + - ref@actor_rollout_ref.ref: dp_ref + + # Rollout model config. + - rollout@actor_rollout_ref.rollout: rollout + + # Model config. + - model@actor_rollout_ref.model: hf_model + + # Critic model config. + - critic@critic: dp_critic + + # Reward model config. + - reward_model@reward_model: dp_reward_model + + # load the reference default config, then apply the fields in the current yaml + # self config override anything above + - _self_ + +# config for actor, rollout and reference model +actor_rollout_ref: + + # Whether it's a hybrid engine, currently only supports hybrid engine + hybrid_engine: true + + # Timeout for operations executed against the process group + nccl_timeout: 600 + + # Rollout model config. + rollout: + + # for huge model, layered summon can save memory (prevent OOM) but make it slower + layered_summon: False + +# custom reward function definition +custom_reward_function: + + # The path to the file containing your customized reward function. + # If not specified, pre-implemented reward functions will be used. + path: null + + # The name of the reward function within the specified file. Default is 'compute_score'. + name: compute_score + +# config for the algorithm +algorithm: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.AlgoConfig + + # Discount factor for future rewards + gamma: 1.0 + + # Trade-off between bias and variance in the GAE estimator + lam: 1.0 + + # Advantage estimator type: "gae", "grpo", "reinforce_plus_plus", etc. + adv_estimator: gae + + # Whether to normalize advantages by std (specific to GRPO) + norm_adv_by_std_in_grpo: True + + # Whether to enable in-reward KL penalty + use_kl_in_reward: False + + # How to estimate KL divergence: "kl", "abs", "mse", "low_var_kl", or "full" + kl_penalty: kl + + # KL control configuration + kl_ctrl: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.trainer.config.KLControlConfig + + # KL control type: "fixed" or "adaptive" + type: fixed + + # Initial coefficient for KL penalty + kl_coef: 0.001 + + # Horizon value for adaptive controller (if enabled) + horizon: 10000 + + # Target KL divergence (used for adaptive controller) + target_kl: 0.1 + + # Whether to enable preference feedback PPO + use_pf_ppo: False + + # Preference feedback PPO settings + pf_ppo: + + # Method for reweighting samples: "pow", "max_min", or "max_random" + reweight_method: pow + + # Power used for weight scaling in "pow" method + weight_pow: 2.0 + +# config for the trainer +trainer: + + # Whether to balance batch sizes across distributed workers + balance_batch: True + + # Number of epochs in training + total_epochs: 30 + + # Total training steps (can be set explicitly or derived from epochs) + total_training_steps: null + + # Project name for experiment tracking (e.g., wandb) + project_name: verl_examples + + # Experiment name for run identification in tracking tools + experiment_name: gsm8k + + # Logging backends to use: "console", "wandb", etc. + logger: ["console", "wandb"] + + # Number of generations to log during validation + log_val_generations: 0 + + # Directory for logging rollout data; no dump if null + rollout_data_dir: null + + # Directory for logging validation data; no dump if null + validation_data_dir: null + + # Number of nodes used in the training + nnodes: 1 + + # Number of GPUs per node + n_gpus_per_node: 8 + + # Save frequency (by iteration) for model checkpoints + save_freq: -1 + + # ESI refers to the elastic server instance used during training, similar to the training plan. For example, + # if you purchase 10 hours of computing power, the ESI will automatically shut down after 10 hours of training. + # To ensure a checkpoint is saved before ESI shuts down, the system will start saving a checkpoint in advance. + # The advance time is calculated as: Advance Time = Longest historical step duration + Checkpoint save duration + esi_redundant_time. + # Here, esi_redundant_time is a user-defined value that further extends the advance time for added safety. + esi_redundant_time: 0 + + # Resume mode: "auto", "disable", or "resume_path" + # "auto": resume from last checkpoint if available + # "disable": start from scratch + # "resume_path": resume from a user-defined path + resume_mode: auto + + # Path to resume training from (only used when resume_mode is "resume_path") + resume_from_path: null + + # Whether to run validation before training begins + val_before_train: True + + # Whether to run validation only + val_only: False + + # Validation frequency (in training iterations) + test_freq: -1 + + # Number of iterations to warm up the critic before updating policy + critic_warmup: 0 + + # Default path to distributed filesystem for saving checkpoints + default_hdfs_dir: null + + # Whether to delete local checkpoints after loading + del_local_ckpt_after_load: False + + # Default local directory for saving checkpoints + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + + # Maximum number of actor checkpoints to keep + max_actor_ckpt_to_keep: null + + # Maximum number of critic checkpoints to keep + max_critic_ckpt_to_keep: null + + # Timeout (in seconds) for Ray worker to wait for registration + ray_wait_register_center_timeout: 300 + + # Device to run training on (e.g., "cuda", "cpu") + device: cuda + + # whether to use legacy worker implementation + # mode: "auto", "enable", or "disable" + use_legacy_worker_impl: auto + +# profiler configs +global_profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # Profiling tool: choose between nsys, npu, torch, torch_memory + tool: null + + # profile steps + steps: null + + # Whether to combine continuous steps into one database. + ## If True, worker.profiler.discrete must be False, [1,2] in one, [5] in another. + ## If False, [1] in one, [2] in another, [5] in another. + profile_continuous_steps: False + + # Path to save profiling contents + save_path: "outputs/profile" + + # Specific tool configs, can use +profiler.tool_config.[tool].xxx to config + global_tool_config: + + # nsys config + nsys: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NsightToolConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # controller Nvidia Nsight Systems Options. Must set when profile_steps is not None. + ## reference https://docs.nvidia.com/nsight-systems/UserGuide/index.html + ## reference https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html + controller_nsight_options: + + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # worker Nvidia Nsight Systems Options. Must set when profile_steps is not None. + worker_nsight_options: + + # Select the API(s) to be traced. + trace: "cuda,nvtx,cublas,ucx" + + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". + cuda-memory-usage: "true" + + # CUDA graphs will be traced as a whole + cuda-graph-trace: "graph" + + # Profiling only in a range of torch.cuda.profiler.start and stop. Do not change this config. + capture-range: "cudaProfilerApi" + + # Specify the desired behavior when a capture range ends. + # In verl we need the torch.cuda.profiler.start/stop pair to repeats n times. + # valid values are "repeat-shutdown:n" or null. + # For normal whole step profiling, n = len(profile_steps); + # but for discrete profiling, n = len(profile_steps) * Number(subtasks). + # Or you can just leave it null and the program will use n = len(profile_steps) * 6; + capture-range-end: null + + # Send signal to the target application's process group. We let the program to exit by itself. + kill: none + + # enable memory visualization for debugging memory usage + torch_memory: + + # Maximum number of allocation entries to record + trace_alloc_max_entries: 100_000 + + # The depth of the call stack to capture for each allocation + stack_depth: 32 + + # 'alloc': records only allocation events || 'state': records memory state changes || 'all': records both. + context: "all" + + # 'python': records Python stacks || 'cpp': records C++ stacks (available in some versions) || 'all': records both. + stacks: "all" + + # devices, record_context etc. + kw_args: {} + +# configs related to ray +ray_kwargs: + + # configs related to ray initialization + ray_init: + + # Number of CPUs for Ray. Use a fixed number instead of null when using SLURM. + num_cpus: null + + # Path to save Ray timeline JSON for performance profiling + timeline_json_file: null diff --git a/e2c/verl/verl/trainer/config/ref/dp_ref.yaml b/e2c/verl/verl/trainer/config/ref/dp_ref.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d4ae0c882df83c0f166fd5497f9178ccd6bcd7 --- /dev/null +++ b/e2c/verl/verl/trainer/config/ref/dp_ref.yaml @@ -0,0 +1,26 @@ +# defaults specify the default config from each component +defaults: + + # dp ref config, inheriting from trainer/config/ref/ref.yaml + - ref + + # fsdp engine config + - ../engine@fsdp_config: fsdp + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +# ref model is assumed to be identical to actor model. Specify model.path for using a different ref model. +# Potential use case involves on policy distillation where we calculate KL divergence between student actor +# and teacher ref +model: null + +# sequence parallel size +# same as actor_rollout_ref.actor.ulysses_sequence_parallel_size if it exists, otherwise 1 +ulysses_sequence_parallel_size: ${oc.select:actor_rollout_ref.actor.ulysses_sequence_parallel_size,1} + +# calculate entropy with chunking to reduce memory peak +entropy_from_logits_with_chunking: False + +# recompute entropy +entropy_checkpointing: False diff --git a/e2c/verl/verl/trainer/config/ref/megatron_ref.yaml b/e2c/verl/verl/trainer/config/ref/megatron_ref.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e2769f2b7ae159cb26dd01b41ccf25148900fb2 --- /dev/null +++ b/e2c/verl/verl/trainer/config/ref/megatron_ref.yaml @@ -0,0 +1,19 @@ +# megatron ref config, inheriting from trainer/config/ref/ref.yaml +defaults: + - ref + + # megatron engine config + - ../engine@megatron: megatron + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +strategy: megatron + +megatron: + _target_: verl.workers.config.MegatronEngineConfig + seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42} + override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}} + use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False} + +load_weight: True \ No newline at end of file diff --git a/e2c/verl/verl/trainer/config/ref/ref.yaml b/e2c/verl/verl/trainer/config/ref/ref.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb4c811683d8fb43d8428bb765c72af2f2d9b171 --- /dev/null +++ b/e2c/verl/verl/trainer/config/ref/ref.yaml @@ -0,0 +1,99 @@ +# actor_rollout_ref.ref: FSDP config same as actor. For models larger than 7B, it’s recommended to turn on offload for ref by default +strategy: ${actor_rollout_ref.actor.strategy} + +# whether to enable torch.compile +# same as actor_rollout_ref.actor.use_torch_compile if it exists, otherwise 1 +use_torch_compile: ${oc.select:actor_rollout_ref.actor.use_torch_compile,true} + +# [Will be deprecated, use log_prob_micro_batch_size_per_gpu] +# The batch size for one forward pass in the computation of log_prob. Global batch size. +log_prob_micro_batch_size: null + +# The batch size for one forward pass in the computation of log_prob. Local batch size per GPU. +log_prob_micro_batch_size_per_gpu: null + +# enable dynamic batch size (sequence packing) for log_prob computation +# same as actor_rollout_ref.actor.use_dynamic_bsz if it exists, otherwise false +log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + +# the max token length per GPU +# same as actor_rollout_ref.actor.ppo_max_token_len_per_gpu if it exists, otherwise 16384 +log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + +# profile the ref model in `compute_log_prob` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # choices: nsys, npu, torch, torch_memory + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on Ref + enable: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config which only related to the role + tool_config: + + # nsys tool config + nsys: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NsightToolConfig + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete} + + # npu config + npu: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.NPUToolConfig + + # Contents to profile, can be empty + # options: npu, cpu, memory, shapes, module, stack + contents: [] + + # Collection level, optional values: level_none, level0, level1, level2. + level: "level1" + + # Whether to automatically parse the data. + analysis: True + + # True for each task has its own database, False for all tasks in one training step share one database. + discrete: False + + # torch profiler config + torch: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchProfilerToolConfig + + # start profile mini-batch in training + # NOTICE: different with global steps config which refers to iteration + # This field only related with mini-batch + step_start: 0 + + # stop profile mini-batch in training + step_end: null + + # torch memory profiler config + torch_memory: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.config.TorchMemoryToolConfig + + # Maximum number of memory allocation entries to track + trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000} + + # Stack trace depth for memory allocations + stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32} \ No newline at end of file diff --git a/e2c/verl/verl/trainer/config/reward_model/dp_reward_model.yaml b/e2c/verl/verl/trainer/config/reward_model/dp_reward_model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fff1f9f1f1d32100e77357781ee29a5728ef298c --- /dev/null +++ b/e2c/verl/verl/trainer/config/reward_model/dp_reward_model.yaml @@ -0,0 +1,55 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# defaults specify the default config from each component +defaults: + + # dp actor config, inheriting from trainer/config/reward_model/reward_model.yaml + - reward_model + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +strategy: fsdp + +model: + + # Whether to use shared memory for loading the model + use_shm: False + + # Use remove padding optimization (saves compute) + use_remove_padding: False + + # Whether to use fused reward kernels for speedup + use_fused_kernels: ${actor_rollout_ref.model.use_fused_kernels} + + # FSDP-specific config + fsdp_config: + + # Target configuration dataclass + _target_: verl.workers.config.FSDPEngineConfig + + # Policy for wrapping layers with FSDP + wrap_policy: + + # Minimum number of parameters to trigger wrapping + min_num_params: 0 + + # Whether to offload model parameters to CPU + param_offload: False + + # Only for FSDP2: Reshard after forward pass to reduce memory footprint + reshard_after_forward: True + + # Number of GPUs in each FSDP shard group; -1 means auto + fsdp_size: -1 + + # Only for FSDP1: FSDP1 configuration, prefetch the next forward-pass all-gather + # before the current forward computation. + forward_prefetch: False + +# Sequence parallelism size for Ulysses-style model parallelism +ulysses_sequence_parallel_size: 1 \ No newline at end of file diff --git a/e2c/verl/verl/trainer/config/reward_model/megatron_reward_model.yaml b/e2c/verl/verl/trainer/config/reward_model/megatron_reward_model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62a58d4b2dd31388560d8157980784c525ac659c --- /dev/null +++ b/e2c/verl/verl/trainer/config/reward_model/megatron_reward_model.yaml @@ -0,0 +1,65 @@ +# defaults specify the default config from each component +defaults: + + # dp actor config, inheriting from trainer/config/reward_model/reward_model.yaml + - reward_model + + # load the reference default config, then apply the fields in the current yaml + - _self_ + +strategy: megatron + +# seconds, default is 10 minutes for torch, you can set it to a larger value +# if you have long-running operations like 32B or 72B model using megatron +nccl_timeout: 600 + +# Megatron parallelism & checkpointing config +megatron: + + # Target configuration dataclass + _target_: verl.workers.config.MegatronEngineConfig + + # Whether to offload model parameters to CPU + param_offload: False + + # Number of GPUs in tensor model parallel group + tensor_model_parallel_size: 1 + + # Number of GPUs in expert model parallel group + expert_model_parallel_size: 1 + + # Expert tensor parallel size + expert_tensor_parallel_size: 1 + + # Number of pipeline model parallel stages + pipeline_model_parallel_size: 1 + + # change VPP interface for parallelism tests + virtual_pipeline_model_parallel_size: null + + # Context parallel size + context_parallel_size: 1 + + # Whether to use sequence parallelism + sequence_parallel: True + + # Whether to use distributed optimizer + use_distributed_optimizer: False + + # Whether to enable distributed checkpointing + use_dist_checkpointing: False + + # Path for distributed checkpoints + dist_checkpointing_path: null + + # RNG seed for megatron + seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42} + + # Any overrides to transformer config + override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}} + + # Whether to use mbridge for faster comms + use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False} + +# Whether to load weights (default True) +load_weight: True \ No newline at end of file diff --git a/e2c/verl/verl/trainer/config/reward_model/reward_model.yaml b/e2c/verl/verl/trainer/config/reward_model/reward_model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9ffc60fbc614377b81579454e7f88023db70d91 --- /dev/null +++ b/e2c/verl/verl/trainer/config/reward_model/reward_model.yaml @@ -0,0 +1,97 @@ +# configs for the reward model + +# Whether to enable reward model. If False, we compute the reward only with the user-defined reward functions. +# In GSM8K and Math examples, we disable reward model. +# For RLHF alignment example using full_hh_rlhf, we utilize reward model to assess the responses. +# If False, the following parameters are not effective +enable: False + +# Whether to deploy the model to a separate resource pool. +# If true, n_gpus_per_node & nnodes will be used to determine the resource node. +enable_resource_pool: False +n_gpus_per_node: 0 +nnodes: 0 + +# FSDP strategy: "fsdp" or "fsdp2" +strategy: ??? + +# model config for reward scoring +model: + + # Input tokenizer. If the reward model's chat template is inconsistent with the policy, + # we need to first decode to plaintext, then apply the rm's chat_template. + # Then score with RM. If chat_templates are consistent, it can be set to null. + # set this to null if the chat template is identical + input_tokenizer: ${actor_rollout_ref.model.path} + + # RM’s HDFS path or local path. Note that RM only supports AutoModelForSequenceClassification. + # Other model types need to define their own RewardModelWorker and pass it from the code. + path: ~/models/FsfairX-LLaMA3-RM-v0.1 + + # External model implementation (optional) + external_lib: ${actor_rollout_ref.model.external_lib} + + # Whether to enable loading a remote code model, default to False + trust_remote_code: False + +# [Deprecated] Global micro batch size +# will be deprecated, use micro_batch_size_per_gpu +micro_batch_size: null + +# Local per-GPU micro batch size +micro_batch_size_per_gpu: null + +# Maximum sequence length to process for scoring +max_length: null + +# Whether to dynamically adjust batch size at runtime +use_dynamic_bsz: ${critic.use_dynamic_bsz} + +# Maximum number of tokens per GPU in one forward pass +forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} + +# Reward Manager. This defines the mechanism of computing rule-based reward and handling different reward sources. +# Default is naive. If all verification functions are multiprocessing-safe, +# the reward manager can be set to prime for parallel verification. +reward_manager: naive + +# Whether to launch custom reward function asynchronously during log_prob +# custom reward function executed async on CPU, during log_prob +launch_reward_fn_async: False + +# Cloud/local sandbox fusion configuration for custom reward logic +sandbox_fusion: + + # Cloud /local function URL for sandbox execution + url: null + + # Max concurrent requests allowed to sandbox + max_concurrent: 64 + + # Max memory limit for each sandbox process in MB + memory_limit_mb: 1024 + +# profile the reward model in `compute_reward` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # profiler tool, default same as profiler.tool in global config + # choices: nsys, npu, torch + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on ref + enable: False + + # Whether to profile all ranks. + all_ranks: False + + # The ranks that will be profiled. [] or [0,1,...] + ranks: [] + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} \ No newline at end of file diff --git a/e2c/verl/verl/trainer/config/rollout/rollout.yaml b/e2c/verl/verl/trainer/config/rollout/rollout.yaml new file mode 100644 index 0000000000000000000000000000000000000000..366b553d840a51be8b4bda9cfefb5b1e18a16b66 --- /dev/null +++ b/e2c/verl/verl/trainer/config/rollout/rollout.yaml @@ -0,0 +1,291 @@ +# Target class for this configuration +_target_: verl.workers.config.RolloutConfig + +# actor_rollout_ref.rollout.name: hf/vllm/sglang. The default value will be removed in the future +name: ??? + +# sync: LLM, async: AsyncLLM +mode: sync + +# Sampling temperature for rollout. +temperature: 1.0 + +# Top-k sampling parameter. -1 for vLLM rollout, 0 for HF rollout. +top_k: -1 + +# Top-p sampling parameter. Default 1.0. +top_p: 1 + +# typically the same as data max prompt length +# same as data.max_prompt_length if it exists +prompt_length: ${oc.select:data.max_prompt_length,512} + +# typically the same as data max response length +# same as data.max_response_length if it exists +response_length: ${oc.select:data.max_response_length,512} + +# for vllm rollout +# Rollout model parameters type. Align with actor model's FSDP/Megatron type. +dtype: bfloat16 + +# Fraction of GPU memory used by vLLM/SGLang for KV cache. +gpu_memory_utilization: 0.5 + +# Whether to ignore EOS and continue generating after EOS is hit. +ignore_eos: False + +# Whether to disable CUDA graph. Default False to best performance. +enforce_eager: False + +# batch size of cudagraph to capture. Require enforce_eager: False to use this option +# Since cudagraph in inference engine can not be offloaded during update policy, +# you can use smaller batch size to save memory used in cuda graph, eg: [1 ,2, 4, 8, 16, 32] +# supported engines: vllm +cudagraph_capture_sizes: null + +# Whether to free engine KVCache after generation. +free_cache_engine: True + +# TP size for rollout. Not effective for hf +tensor_model_parallel_size: 2 + +# DP size for rollout +data_parallel_size: 1 + +# EP size for rollout +expert_parallel_size: 1 + +# max number of tokens in a batch +max_num_batched_tokens: 8192 + +# max length for rollout +max_model_len: null + +# max length of sequences +max_num_seqs: 1024 + +# may get higher throughput when set to True. When activated, Please increase max_num_batched_tokens or decrease max_model_len. +enable_chunked_prefill: True + +# Prefix caching kv-cache blocks is a popular optimization in LLM inference to avoid redundant prompt computations. +enable_prefix_caching: True + +# Which loader to use for rollout model weights: dummy, hf, megatron, etc. +# safetensors (for huge model, and set use_shm=True); dummy: randomly init model weight +load_format: dummy + +# [Will be deprecated, use log_prob_micro_batch_size_per_gpu] The batch size for one forward pass in the computation of log_prob. Global batch size. +log_prob_micro_batch_size: null + +# The batch size for one forward pass in the computation of log_prob. Local batch size per GPU. +log_prob_micro_batch_size_per_gpu: null + +# enable dynamic batch size (sequence packing) for log_prob computation +# same as actor_rollout_ref.actor.use_dynamic_bsz if it exists, otherwise false +log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false} + +# max token length for log_prob computation +# same as actor_rollout_ref.actor.ppo_max_token_len_per_gpu if it exists, otherwise 16384 +log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384} + +# disable logging statistics +disable_log_stats: True + +# for hf rollout +# Whether to sample during training rollout. False uses greedy sampling. +do_sample: True + +# number of responses (i.e. num sample times). > 1 for grpo +n: 1 + +# The over_sample_rate parameter controls the early termination threshold for training rollouts, +# where the system will abort remaining requests when (1 - over_sample_rate) * total_requests completions are reached. +over_sample_rate: 0 + +# Whether to wake up inference engine in multi-stage for SGLang +# to reduce peak memory during training-rollout transition. +# This is only effective for SGLang rollout. +multi_stage_wake_up: false + +# Extra inference engine arguments (vllm, sglang), please refer vllm/sglang official doc for detail +engine_kwargs: + + # vllm engine config + vllm: {} + + # sglang engine config + sglang: {} + +# Sampling parameters used during validation. +val_kwargs: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.SamplingConfig + + # sampling parameters for validation + # Top-k sampling parameter. -1 for vLLM rollout, 0 for HF rollout. + top_k: -1 + + # Top-p sampling parameter. Default 1.0. + top_p: 1.0 + + # Sampling temperature for rollout. + temperature: 0 + + # whether to repeat n times for validation + n: 1 + + # Whether to sample during training rollout. False uses greedy sampling. + do_sample: False + +# Multi-turn interaction config for tools or chat. +multi_turn: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.MultiTurnConfig + + # set to True for multi-turn tool interaction tasks; should set rollout.name to sglang as well + enable: False + + # null for no limit (default max_length // 3) + max_assistant_turns: null + + # null for no tool + tool_config_path: null + + # null for no limit (default max_length // 3) + max_user_turns: null + + # max parallel call for tools in single turn + max_parallel_calls: 1 + + # max length of tool response + max_tool_response_length: 256 + + # truncate side of tool response: left, middle, right + tool_response_truncate_side: middle + + # null for no interaction + interaction_config_path: null + + # - When set to True, the model's default chat template is used for multi-turn rollout, which typically matches production behavior. + # - When set to False, the token ids recorded for training are used instead; unlike the default chat template, these always include the model's full output, + # which may contain additional content such as reasoning content. This maintains the consistency between training and rollout, but it will lead to longer prompts. + use_inference_chat_template: False + + # Tokenization is performed turn by turn and the resulting token ids are concatenated to form the full conversation. + # To ensure this matches the result of tokenizing the entire conversation at once, a sanity check is run at the end of each multi-turn rollout to compare the two sets of token ids. + # Some models are known to produce different tokenization results when tokenizing turn by turn vs. all at once. aThis behavior has already been validated for them. + # To reduce excessive warnings, you can turn off the sanity check for these models if you are using their default chat template: + # Qwen/QwQ-32B, Qwen/Qwen3-xxB + # - disable: disable tokenization sanity check + # - strict: enable strict tokenization sanity check (default) + # - ignore_strippable: ignore strippable tokens when checking tokenization sanity + tokenization_sanity_check_mode: strict + + # Format of the multi-turn interaction. Options: hermes, llama3_json, ... + format: hermes + + # Number of repeat rollouts for each interaction + num_repeat_rollouts: null + +# support logging rollout prob for debugging purpose +# "Truncated importance sampling" requires rollout log probs, set to True when turning on Truncated importance sampling +calculate_log_probs: False + +# [Experimental] agent loop based rollout configs +agent: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.AgentLoopConfig + + # Number of agent loop workers + num_workers: 8 + + # custom agent loop config path, which should contain list of configs to intialize AgentLoop instances. + # https://hydra.cc/docs/advanced/instantiate_objects/overview/ + # + # - name: react_agent + # _target_: recipe.langgraph_agent.react_agent_loop.ReactAgentLoop + # tools: ["get_current_temperature"] + # - name: math_expression + # _target_: recipe.langgraph_agent.example.math_expression.MathExpressionReactAgentLoop + # min_terms: 2 + # max_terms: 6 + agent_loop_config_path: null + + # custom async server configs + custom_async_server: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.CustomAsyncServerConfig + + # Path to the custom async server implementation + path: null + + # Class name of the custom async server class (e.g. AsyncvLLMServer) + name: null + +# Specifies the tensor bucket size (in megabytes) for batch weight updates during rollout operations. +# This parameter controls the maximum payload size for a single weight update request. +# Reference: https://github.com/volcengine/verl/pull/2418 +# Currently only supported in SGLang rollout implementations +# Larger values may improve throughput but increase memory overhead +# Detailed performance comparison: +# https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/issues/169#issuecomment-3070686720 +# Default value (512MB) is optimized for typical GPU memory configurations +# For the best performance of `rebuild_cuda_tensor`, it is recommended to: +# 1. Enable `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES` +# 2. Manually set `CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7` +# when using Tensor Parallelism (TP) >= 8. +update_weights_bucket_megabytes: 512 + +# trace rollout data +trace: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.workers.config.TraceConfig + + # trace backend, support mlflow, weave + backend: null + + # whether translate token id to text in output + token2text: False + +# When enabled (True), the trainer will attempt to load previously generated rollout data from the specified directory instead of computing new rollouts. +# If no cached data is found or loading fails, new rollouts will be generated and automatically saved. +# This feature is useful for debugging or when you want to reuse computation results across multiple runs. +skip_rollout: False + +# Specifies the filesystem path where rollout data should be cached when skip_rollout is enabled. +# Note: Giving path under /tmp/ray/session* is not recommended as these are temporary Ray cluster directories. +skip_dump_dir: /tmp/rollout_dump + +# Whether to skip tokenizer initialization for rollout engine +# When enabled (True), the rollout assume token in token out for generation +skip_tokenizer_init: True + +# profile the rollout model in `generate_sequence` +profiler: + + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs + _target_: verl.utils.profiler.ProfilerConfig + + # profiler tool, default same as profiler.tool in global config + # choices: nsys, npu, torch + tool: ${oc.select:global_profiler.tool,null} + + # whether enable profile on ref + enable: ${oc.select:actor_rollout_ref.actor.profiler.enable,false} + + # Whether to profile all ranks. + all_ranks: ${oc.select:actor_rollout_ref.actor.profiler.all_ranks,false} + + # The ranks that will be profiled. [] or [0,1,...] + ranks: ${oc.select:actor_rollout_ref.actor.profiler.ranks,[]} + + # profile results saving path + save_path: ${oc.select:global_profiler.save_path,null} + + # specific tool config + tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null} diff --git a/e2c/verl/verl/trainer/config/sft_trainer.yaml b/e2c/verl/verl/trainer/config/sft_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb946be88abc550a8f3b7bd48d9dc6a42c55e38c --- /dev/null +++ b/e2c/verl/verl/trainer/config/sft_trainer.yaml @@ -0,0 +1,85 @@ +data: + train_batch_size: 256 + micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu + micro_batch_size_per_gpu: 4 # this is also val batch size + train_files: ~/data/gsm8k/train.parquet + val_files: ~/data/gsm8k/test.parquet + # Single-turn settings + prompt_key: question + response_key: answer + prompt_dict_keys: null + response_dict_keys: null + # Multi-turn settings + multiturn: + enable: false # Set to true to use multi-turn dataset + messages_key: messages # Key for messages list in multi-turn mode + tools_key: tools # Key for tools list in multi-turn mode + enable_thinking_key: enable_thinking # Whether to enable thinking in multi-turn mode + max_length: 1024 + truncation: error + balance_dp_token: False + chat_template: null + custom_cls: + path: null + name: null + use_shm: False + apply_chat_template_kwargs: {} +model: + partial_pretrain: ~/models/gemma-1.1-7b-it + use_shm: False + fsdp_config: + model_dtype: fp32 + wrap_policy: + min_num_params: 0 + cpu_offload: False + offload_params: False + external_lib: null + enable_gradient_checkpointing: True + trust_remote_code: False + lora_rank: 0 # Set to positive value to enable LoRA (e.g., 32) + lora_alpha: 16 # LoRA scaling factor + target_modules: all-linear # Target modules for LoRA adaptation + use_liger: False + strategy: fsdp2 +optim: + lr: 1e-5 + betas: [0.9, 0.95] + weight_decay: 0.01 + warmup_steps_ratio: 0.1 + clip_grad: 1.0 + lr_scheduler: cosine +ulysses_sequence_parallel_size: 1 +use_remove_padding: False +trainer: + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + default_hdfs_dir: null + project_name: gsm8k-sft + experiment_name: test + total_epochs: 4 + total_training_steps: null + logger: [ 'console', 'wandb' ] + seed: 1 + save_freq: -1 + test_freq: -1 + nnodes: 1 + n_gpus_per_node: 8 + max_ckpt_to_keep: null # Maximum number of checkpoints to keep, set to null to keep all + + # Resume mode: "auto", "disable", or "resume_path" + # "auto": resume from last checkpoint if available + # "disable": start from scratch + # "resume_path": resume from a user-defined path + resume_mode: auto + + # Path to resume training from (used when resume_mode is "resume_path" or "auto") + resume_from_path: null + + # Checkpoint configuration + checkpoint: + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ["model", "optimizer", "extra"] + + # For more flexibility, you can specify the contents to load from the checkpoint. + load_contents: ${trainer.checkpoint.save_contents} + device: cuda diff --git a/e2c/verl/verl/trainer/config/sft_trainer_engine.yaml b/e2c/verl/verl/trainer/config/sft_trainer_engine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cc4b32fe51451eff71525b7faa0b53746b2e692 --- /dev/null +++ b/e2c/verl/verl/trainer/config/sft_trainer_engine.yaml @@ -0,0 +1,71 @@ +# Format checks enforced on CI: +# 1. Comments must appear above each field. +# 2. There must be a blank line between each field. +# 3. Inline comments (after a field on the same line) are not allowed. +# 4. Indentation level is respected for nested fields. + +# @.: + +defaults: + - model@model: hf_model + - engine@engine: fsdp + - optim@optim: fsdp + - _self_ + +data: + train_batch_size: 256 # global batch size + micro_batch_size_per_gpu: 4 # this is also val batch size + max_token_len_per_gpu: 8192 + use_dynamic_bsz: True + train_files: ~/data/gsm8k/train.parquet + val_files: ~/data/gsm8k/test.parquet + # Multi-turn settings + messages_key: messages # Key for messages list in multi-turn mode + tools_key: tools # Key for tools list in multi-turn mode + enable_thinking_key: enable_thinking # Whether to enable thinking in multi-turn mode + pad_mode: left_right + # for right padding + max_length: 1024 + # for left right padding + max_prompt_length: 512 + max_response_length: 512 + truncation: error + balance_dp_token: False # to be implement + custom_cls: + path: null + name: null + use_shm: False + apply_chat_template_kwargs: {} + +# Checkpoint configuration +checkpoint: + _target_: verl.trainer.config.CheckpointConfig + # What to include in saved checkpoints + # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space + save_contents: ["model", "optimizer", "extra"] + + # For more flexibility, you can specify the contents to load from the checkpoint. + load_contents: ${checkpoint.save_contents} + +trainer: + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} + default_hdfs_dir: null + project_name: gsm8k-sft + experiment_name: test + total_epochs: 4 + total_training_steps: null + logger: [ 'console', 'wandb' ] + seed: 1 + save_freq: -1 + test_freq: -1 + max_ckpt_to_keep: null # Maximum number of checkpoints to keep, set to null to keep all + + # Resume mode: "auto", "disable", or "resume_path" + # "auto": resume from last checkpoint if available + # "disable": start from scratch + # "resume_path": resume from a user-defined path + resume_mode: auto + + # Path to resume training from (used when resume_mode is "resume_path" or "auto") + resume_from_path: null + device: cuda diff --git a/e2c/verl/verl/trainer/constants_ppo.py b/e2c/verl/verl/trainer/constants_ppo.py new file mode 100644 index 0000000000000000000000000000000000000000..30e325240a453ff81501f338c2dd6f5555af9c19 --- /dev/null +++ b/e2c/verl/verl/trainer/constants_ppo.py @@ -0,0 +1,52 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from ray._private.runtime_env.constants import RAY_JOB_CONFIG_JSON_ENV_VAR + +PPO_RAY_RUNTIME_ENV = { + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "WARN", + "VLLM_ALLOW_RUNTIME_LORA_UPDATING": "true", + "CUDA_DEVICE_MAX_CONNECTIONS": "1", + # To prevent hanging or crash during synchronization of weights between actor and rollout + # in disaggregated mode. See: + # https://docs.vllm.ai/en/latest/usage/troubleshooting.html?h=nccl_cumem_enable#known-issues + # https://github.com/vllm-project/vllm/blob/c6b0a7d3ba03ca414be1174e9bd86a97191b7090/vllm/worker/worker_base.py#L445 + "NCCL_CUMEM_ENABLE": "0", + }, +} + + +def get_ppo_ray_runtime_env(): + """ + A filter function to return the PPO Ray runtime environment. + To avoid repeat of some environment variables that are already set. + """ + working_dir = ( + json.loads(os.environ.get(RAY_JOB_CONFIG_JSON_ENV_VAR, "{}")).get("runtime_env", {}).get("working_dir", None) + ) + + runtime_env = { + "env_vars": PPO_RAY_RUNTIME_ENV["env_vars"].copy(), + **({"working_dir": None} if working_dir is None else {}), + } + for key in list(runtime_env["env_vars"].keys()): + if os.environ.get(key) is not None: + runtime_env["env_vars"].pop(key, None) + return runtime_env diff --git a/e2c/verl/verl/trainer/fsdp_sft_trainer.py b/e2c/verl/verl/trainer/fsdp_sft_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..a5a4b02410a7a2f81d17115c89214fcc87edd9fe --- /dev/null +++ b/e2c/verl/verl/trainer/fsdp_sft_trainer.py @@ -0,0 +1,844 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +A lightweight one-file FSDP SFT Trainer +TODO(zhangchi.usc1992) +- Add calculation of mfu +- Add validation +""" + +import os + +os.environ["NCCL_DEBUG"] = "WARN" +os.environ["TOKENIZERS_PARALLELISM"] = "true" + +import logging +import re +import time +from contextlib import nullcontext + +import hydra +import torch +import torch.distributed +from omegaconf import DictConfig, OmegaConf +from peft import LoraConfig, TaskType, get_peft_model +from tensordict import TensorDict +from torch import nn, optim +from torch.distributed.device_mesh import DeviceMesh, init_device_mesh +from torch.distributed.fsdp import CPUOffload, MixedPrecision, ShardingStrategy +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.utils.data import Dataset, DistributedSampler +from torchdata.stateful_dataloader import StatefulDataLoader +from tqdm import tqdm +from transformers import AutoConfig, AutoModelForCausalLM, PreTrainedModel + +import verl.utils.hdfs_io as hdfs_io +from verl.utils.attention_utils import index_first_axis, pad_input, rearrange, unpad_input +from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path, get_checkpoint_tracker_filename +from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager +from verl.utils.dataset import SFTDataset +from verl.utils.dataset.multiturn_sft_dataset import MultiTurnSFTDataset +from verl.utils.device import get_device_id, get_device_name, is_cuda_available, is_npu_available +from verl.utils.distributed import destroy_global_process_group, initialize_global_process_group +from verl.utils.fs import copy_to_local +from verl.utils.fsdp_utils import ( + CPUOffloadPolicy, + MixedPrecisionPolicy, + apply_fsdp2, + fsdp2_clip_grad_norm_, + fsdp2_load_full_state_dict, + get_fsdp_wrap_policy, + get_init_weight_context_manager, + init_fn, +) +from verl.utils.logger import log_with_rank +from verl.utils.profiler import log_gpu_memory_usage +from verl.utils.py_functional import convert_to_regular_types +from verl.utils.torch_dtypes import PrecisionType +from verl.utils.torch_functional import get_cosine_schedule_with_warmup, get_wsd_schedule_with_warmup +from verl.utils.tracking import Tracking +from verl.utils.ulysses import ( + gather_outputs_and_unpad, + get_ulysses_sequence_parallel_world_size, + ulysses_pad_and_slice_inputs, +) +from verl.workers.sharding_manager.fsdp_ulysses import FSDPUlyssesShardingManager + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_SFT_LOGGING_LEVEL", "WARN")) + + +def extract_step(path): + match = re.search(r"global_step_(\d+)", path) + if match: + return int(match.group(1)) + return None + + +class FSDPSFTTrainer: + def __init__( + self, + config, + device_mesh: DeviceMesh, + ulysses_device_mesh: DeviceMesh, + tokenizer, + train_dataset: Dataset, + val_dataset: Dataset, + ): + self.config = config + self.device_mesh = device_mesh + self.ulysses_device_mesh = ulysses_device_mesh + self.sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) + self.tokenizer = tokenizer + if self.config.data.chat_template is not None: + raise ValueError("Apply Chat template from config is not supported yet.") + + # normalize dp size + self._normalize_config_bsz() + + # Set sequence parallel size + self.config.ulysses_sequence_parallel_size = getattr(self.config, "ulysses_sequence_parallel_size", 1) + self.use_remove_padding = getattr(self.config, "use_remove_padding", False) + if self.device_mesh.get_rank() == 0: + print(f"Using sequence parallel size: {self.config.ulysses_sequence_parallel_size}") + print(f"Using remove padding: {self.use_remove_padding}") + + self._build_dataloader(train_dataset, val_dataset) + + # Initialize resume-related variables + self.resume_global_step = 0 + + # build model + self._build_model_optimizer() + + # Initialize checkpoint manager + self._init_checkpoint_manager() + + self.load_checkpoint() + + if self.device_mesh.get_rank() == 0: + print(self.config) + self.device_name = self.config.trainer.device + + def _normalize_config_bsz(self): + dp_size = self.device_mesh.size(0) if not self.ulysses_device_mesh else self.ulysses_device_mesh.size(0) + if self.device_mesh.get_rank() == 0: + print(f"Normalize batch size by dp {dp_size}") + + assert self.config.data.train_batch_size % dp_size == 0, ( + f"Global batch size {self.config.data.train_batch_size} is not divisible by dp size {dp_size}" + ) + + self.config.data.train_batch_size //= dp_size + + assert self.config.data.train_batch_size % self.config.data.micro_batch_size_per_gpu == 0 + + def _build_dataloader(self, train_dataset, val_dataset): + # build dataset + config = self.config + self.train_dataset, self.val_dataset = train_dataset, val_dataset + + # build dataloader + # Use data parallel rank and size instead of global rank and world size + + # If doing SP, we need to use the local rank and size + if self.config.ulysses_sequence_parallel_size > 1: + rank = self.ulysses_device_mesh.get_local_rank("dp") + world_size = self.ulysses_device_mesh.size(0) + if self.ulysses_device_mesh.get_rank() == 0: + print(f"Using SP rank {rank} and size {world_size} for data distribution") + print("Each SP rank gets different data, but the same data WITHIN the same rank") + else: + rank = self.device_mesh.get_rank() + world_size = self.device_mesh.size() + if self.device_mesh.get_rank() == 0: + print(f"Using FSDP rank {rank} and size {world_size} for data distribution") + + # Set pin_memory_device when pin_memory is enabled. + device_name = get_device_name() + + self.train_sampler = DistributedSampler( + self.train_dataset, shuffle=True, num_replicas=world_size, rank=rank, drop_last=True + ) + self.train_dataloader = StatefulDataLoader( + dataset=self.train_dataset, + batch_size=config.data.train_batch_size, + sampler=self.train_sampler, + num_workers=8, + pin_memory=True, + drop_last=True, + pin_memory_device=device_name, + ) + + self.val_sampler = DistributedSampler( + self.val_dataset, shuffle=False, num_replicas=world_size, rank=rank, drop_last=True + ) + self.val_dataloader = StatefulDataLoader( + dataset=self.val_dataset, + batch_size=config.data.micro_batch_size_per_gpu, + sampler=self.val_sampler, + num_workers=8, + pin_memory=True, + drop_last=True, + pin_memory_device=device_name, + ) + + def _build_model_optimizer(self): + # TODO (zhangchi.usc1992): + # 1. support pretrain from random weights + # 2. support init directly from sharded weights + local_model_path = copy_to_local(src=self.config.model.partial_pretrain, verbose=True) + + if self.config.model.get("external_lib", None) is not None: + # This is used to import external_lib into the huggingface systems + import importlib + + importlib.import_module(self.config.model.external_lib) + + log_gpu_memory_usage("Before model allocation", logger=logger) + + trust_remote_code = self.config.model.trust_remote_code + torch_dtype = self.config.model.fsdp_config.get("model_dtype", "fp32") + torch_dtype = PrecisionType.to_dtype(torch_dtype) + # load config first + config = AutoConfig.from_pretrained(local_model_path, trust_remote_code=trust_remote_code) + self.model_config = config + if hasattr(self.model_config, "max_position_embeddings"): + self.model_config.max_position_embeddings = max( + self.model_config.max_position_embeddings, self.config.data.max_length + ) + if self.config.ulysses_sequence_parallel_size > 1: + assert self.use_remove_padding, "Sequence parallel is only supported when remove_padding is enabled" + + # This may be very large + init_context = get_init_weight_context_manager( + use_meta_tensor=not config.tie_word_embeddings, mesh=self.device_mesh + ) + + with init_context(): + self.model: PreTrainedModel = AutoModelForCausalLM.from_pretrained( + local_model_path, + config=config, + torch_dtype=torch_dtype, + attn_implementation="flash_attention_2", + trust_remote_code=trust_remote_code, + ) + + if self.use_remove_padding or self.config.ulysses_sequence_parallel_size > 1: + from verl.models.transformers.monkey_patch import apply_monkey_patch + + apply_monkey_patch(model=self.model, ulysses_sp_size=self.config.ulysses_sequence_parallel_size) + + # Apply Liger kernel if use_liger is enabled + if self.config.model.get("use_liger", False): + from liger_kernel.transformers.monkey_patch import _apply_liger_kernel_to_instance + + _apply_liger_kernel_to_instance(model=self.model) + + if self.config.model.get("lora_rank", 0) > 0: + self.model.enable_input_require_grads() + # Convert config to regular Python types before creating PEFT model + lora_config = { + "task_type": TaskType.CAUSAL_LM, + "r": self.config.model.lora_rank, + "lora_alpha": self.config.model.lora_alpha, + "target_modules": convert_to_regular_types(self.config.model.target_modules), + "bias": "none", + } + self.model = get_peft_model(self.model, LoraConfig(**lora_config)) + self.model = self.model.to(torch_dtype) + + if self.config.model.enable_gradient_checkpointing: + self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + + log_gpu_memory_usage("After model allocation", logger=logger) + + mixed_precision = MixedPrecision( + param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32 + ) + + auto_wrap_policy = get_fsdp_wrap_policy( + self.model, + config=self.config.model.fsdp_config.wrap_policy, + is_lora=self.config.model.get("lora_rank", 0) > 0, + ) + if self.device_mesh.get_rank() == 0: + print(auto_wrap_policy) + + if not self.config.model.fsdp_config.cpu_offload: + cpu_offload = None + else: + cpu_offload = CPUOffload(offload_params=self.config.model.fsdp_config.offload_params) + + fsdp_strategy = self.config.model.strategy + if fsdp_strategy == "fsdp": + self.fsdp_model = FSDP( + self.model, + cpu_offload=cpu_offload, + param_init_fn=init_fn, + use_orig_params=False, + auto_wrap_policy=auto_wrap_policy, + device_id=get_device_id(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + sync_module_states=True, + device_mesh=self.device_mesh, + forward_prefetch=False, + ) + elif fsdp_strategy == "fsdp2": + assert CPUOffloadPolicy is not None, "PyTorch version >= 2.4 is required for using fully_shard API (FSDP2)" + mp_policy = MixedPrecisionPolicy( + param_dtype=torch.bfloat16, reduce_dtype=torch.float32, cast_forward_inputs=True + ) + + fsdp_kwargs = { + "mesh": self.device_mesh, + "mp_policy": mp_policy, + "offload_policy": cpu_offload, + "reshard_after_forward": True, + } + full_state = self.model.state_dict() + apply_fsdp2(self.model, fsdp_kwargs, self.config.model.fsdp_config) + fsdp2_load_full_state_dict(self.model, full_state, self.device_mesh, cpu_offload) + self.fsdp_model = self.model + else: + raise NotImplementedError(f"not implement {fsdp_strategy}") + + log_gpu_memory_usage("After FSDP wrapping", logger=logger) + + self.optimizer = optim.AdamW( + self.fsdp_model.parameters(), + lr=self.config.optim.lr, + betas=self.config.optim.betas, + weight_decay=self.config.optim.weight_decay, + ) + + log_gpu_memory_usage("After initialize optimizer", logger=logger) + + self.steps_per_epoch = len(self.train_dataloader) + self.total_steps = self.steps_per_epoch * self.config.trainer.total_epochs + + if self.device_mesh.get_rank() == 0: + print( + f"Number of steps/epoch {self.steps_per_epoch}, number of epochs " + f"{self.config.trainer.total_epochs}, total number of steps {self.total_steps}" + ) + + num_warmup_steps = int(self.total_steps * self.config.optim.warmup_steps_ratio) + + if not hasattr(self.config.optim, "lr_scheduler") or self.config.optim.lr_scheduler == "cosine": + self.lr_scheduler = get_cosine_schedule_with_warmup( + optimizer=self.optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=self.total_steps + ) + elif self.config.optim.lr_scheduler == "wsd": + self.lr_scheduler = get_wsd_schedule_with_warmup( + optimizer=self.optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=self.total_steps + ) + else: + raise ValueError(f"Unknown lr scheduler: {self.config.optim.lr_scheduler}") + + def _compute_loss_and_backward(self, batch, do_backward=True, n_micro_batches=1): + """Compute loss with optional sequence parallelism and remove padding features""" + use_sp = self.use_remove_padding and self.config.ulysses_sequence_parallel_size > 1 + + # Move inputs to GPU and prepare loss mask + input_ids = batch["input_ids"].to(self.device_name) + attention_mask = batch["attention_mask"].to(self.device_name) + position_ids = batch["position_ids"].to(self.device_name) + loss_mask = batch.pop("loss_mask")[:, 1:].reshape(-1).to(self.device_name) + loss_fct = nn.CrossEntropyLoss(reduction="none") + + # Context manager for sequence parallel if needed + context = self.sharding_manager if use_sp else nullcontext() + with context, torch.autocast(device_type=self.device_name, dtype=torch.bfloat16): + if not use_sp: + # Standard forward pass without sequence parallel + labels = input_ids[:, 1:].contiguous() + output = self.fsdp_model( + input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, use_cache=False + ) + logits = output.logits + + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels.contiguous() + # Flatten the tokens + shift_logits = shift_logits.view(-1, self.model.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + loss = loss * loss_mask.to(loss.device) + else: + # IMPORTANT: We have a big assumption here, so we can shard the SAME sequence across SP ranks + # i.e., each GPU has <1 sequence, and each SP group has 1 sequence + # 1. All SP ranks will receive the *SAME* batch + # 2. Different SP groups will receive *DIFFERENT* batches + # This is implemented by the DistributedSampler + + batch_size, seqlen = input_ids.shape + # Remove padding + input_ids_rmpad, indices, *_ = unpad_input( + input_ids.unsqueeze(-1), attention_mask + ) # input_ids_rmpad (total_nnz, ...) + input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) + + # Unpad position_ids to align rotary + position_ids_rmpad = index_first_axis( + rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), indices + ).transpose(0, 1) + + # Pad and slice inputs for sequence parallelism + input_ids_rmpad_sliced, position_ids_rmpad_padded, pad_size = ulysses_pad_and_slice_inputs( + input_ids_rmpad, position_ids_rmpad, sp_size=get_ulysses_sequence_parallel_world_size() + ) + # For computing loss + input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=1) # (1, total_nnz) + input_ids_rmpad_rolled, _, _ = ulysses_pad_and_slice_inputs( + input_ids_rmpad_rolled, None, get_ulysses_sequence_parallel_world_size() + ) + input_ids_rmpad_rolled = input_ids_rmpad_rolled.squeeze(0) # ((total_nnz / sp) + pad) + + # Forward pass + output = self.fsdp_model( + input_ids=input_ids_rmpad_sliced, + attention_mask=None, # Not needed with flash attention varlen + position_ids=position_ids_rmpad_padded, + use_cache=False, + ) + + # Compute loss locally then aggregate + logits_rmpad = output.logits.squeeze(0) + input_ids_rmpad_rolled = input_ids_rmpad_rolled.to(logits_rmpad.device) + loss = loss_fct(logits_rmpad, input_ids_rmpad_rolled) + # Gather and unpad for sequence parallelism + loss = gather_outputs_and_unpad(loss, gather_dim=0, unpad_dim=0, padding_size=pad_size) + + # This is the loss collected from all ulysses ranks + full_loss = pad_input( + hidden_states=loss.unsqueeze(-1), indices=indices, batch=batch_size, seqlen=seqlen + ) + full_loss = full_loss.squeeze(-1)[:, :-1] # Remove last token's loss + full_loss = full_loss.reshape(-1) + loss_mask = loss_mask.to(full_loss.device) + loss = full_loss * loss_mask + + valid_token_this_rank = torch.sum(loss_mask) + + if self.config.data.balance_dp_token: + torch.distributed.all_reduce(valid_token_this_rank) + dp_size = self.ulysses_device_mesh.size("dp") if use_sp else torch.distributed.get_world_size() + else: + dp_size = 1 + + loss = torch.sum(loss) / (valid_token_this_rank + 1e-8) * dp_size + + loss = loss / n_micro_batches # normalize loss + + if do_backward: + loss.backward() + return loss + + def training_step(self, batch: TensorDict): + start_time = time.time() + + self.fsdp_model.train() + + log_gpu_memory_usage("Before optimizer zero_grad", logger=logger) + + self.optimizer.zero_grad() + + log_gpu_memory_usage("After optimizer zero_grad", logger=logger) + + micro_batches = batch.split(self.config.data.micro_batch_size_per_gpu) + n_micro_batches = len(micro_batches) + step_loss = 0 + for micro_batch in micro_batches: + loss = self._compute_loss_and_backward(batch=micro_batch, n_micro_batches=n_micro_batches) + step_loss += loss.item() + + if self.config.model.strategy == "fsdp": + grad_norm = self.fsdp_model.clip_grad_norm_(max_norm=self.config.optim.clip_grad) + elif self.config.model.strategy == "fsdp2": + grad_norm = fsdp2_clip_grad_norm_(self.fsdp_model.parameters(), max_norm=self.config.optim.clip_grad) + else: + raise NotImplementedError(f"not implement {self.config.model.strategy}") + + log_gpu_memory_usage("Before optimizer step", logger=logger) + + # if grad_norm is not finite, skip the update + if not torch.isfinite(grad_norm): + print(f"WARN: grad_norm is not finite: {grad_norm}") + self.optimizer.zero_grad() + else: + self.optimizer.step() + + log_gpu_memory_usage("After optimizer step", logger=logger) + + self.lr_scheduler.step() + + # reduce loss across dp ranks + lr = self.lr_scheduler.get_last_lr()[0] + + log_gpu_memory_usage("After offload weights", logger=logger) + + step_loss = torch.tensor(step_loss).to(self.device_name) + + # compute time spent per step + end_time = time.time() + spend_time_per_step = end_time - start_time + + if is_cuda_available: + torch.distributed.all_reduce(step_loss, op=torch.distributed.ReduceOp.AVG) + elif is_npu_available: + torch.distributed.all_reduce(step_loss) + step_loss /= self.device_mesh.size(0) + return { + "train/loss": step_loss.detach().item(), + "train/lr(1e-3)": lr * 1e3, + "train/time(s)": spend_time_per_step, + } + + def validation_step(self, batch: TensorDict): + self.fsdp_model.eval() + with torch.no_grad(): + loss = self._compute_loss_and_backward(batch, do_backward=False) + if is_cuda_available: + torch.distributed.all_reduce(loss, op=torch.distributed.ReduceOp.AVG) + elif is_npu_available: + torch.distributed.all_reduce(loss) + loss /= self.device_mesh.size(0) + return loss + + def save_checkpoint(self, step): + """Save checkpoint using FSDPCheckpointManager with improved tracking""" + from verl.utils.fs import local_mkdir_safe + + # Determine checkpoint path + local_global_step_folder = os.path.join(self.config.trainer.default_local_dir, f"global_step_{step}") + + if self.device_mesh.get_rank() == 0: + print(f"Saving checkpoint to: {local_global_step_folder}") + + # Get max checkpoints to keep + max_ckpt_to_keep = getattr(self.config.trainer, "max_ckpt_to_keep", None) + + # Use checkpoint manager to save + self.checkpoint_manager.save_checkpoint( + local_path=local_global_step_folder, global_step=step, max_ckpt_to_keep=max_ckpt_to_keep + ) + + # Save dataloader state + if self.device_mesh.get_rank() == 0: + local_mkdir_safe(local_global_step_folder) + dataloader_local_path = os.path.join(local_global_step_folder, "data.pt") + + # Use StatefulDataLoader's built-in state dict functionality + dataloader_state_dict = self.train_dataloader.state_dict() + torch.save(dataloader_state_dict, dataloader_local_path) + print(f"Saved dataloader state to: {dataloader_local_path}") + + # Update latest checkpoint tracker (atomic write) + tracker_file = get_checkpoint_tracker_filename(self.config.trainer.default_local_dir) + temp_tracker_file = tracker_file + ".tmp" + with open(temp_tracker_file, "w") as f: + f.write(str(step)) + os.rename(temp_tracker_file, tracker_file) + print(f"Updated checkpoint tracker: {tracker_file}") + + # Copy to HDFS if configured + if self.device_mesh.get_rank() == 0 and getattr(self.config.trainer, "default_hdfs_dir", None): + hdfs_io.makedirs(self.config.trainer.default_hdfs_dir, exist_ok=True) + hdfs_io.copy(src=local_global_step_folder, dst=self.config.trainer.default_hdfs_dir, dirs_exist_ok=True) + + torch.distributed.barrier() + + def _init_checkpoint_manager(self): + """Initialize checkpoint manager with proper configuration""" + # Get checkpoint configuration from config, with defaults + checkpoint_config = getattr(self.config.trainer, "checkpoint", {}) + + # Set default values if not specified + save_contents = checkpoint_config.get("save_contents", ["model", "optimizer", "extra"]) + load_contents = checkpoint_config.get("load_contents", save_contents) + + # Create checkpoint config dict + checkpoint_config_dict = { + "load_contents": load_contents, + "save_contents": save_contents, + } + + # Convert to DictConfig for compatibility + checkpoint_config_dict = DictConfig(checkpoint_config_dict) + + # Initialize checkpoint manager + self.checkpoint_manager = FSDPCheckpointManager( + model=self.fsdp_model, + optimizer=self.optimizer, + lr_scheduler=self.lr_scheduler, + processing_class=self.tokenizer, + checkpoint_config=checkpoint_config_dict, + ) + + def load_checkpoint(self): + # Determine resume path based on configuration + checkpoint_path = self._determine_resume_path() + + if checkpoint_path is None: + return 0 + + # extract resume step from checkpoint path + resume_step = extract_step(checkpoint_path) + if resume_step is None: + log_with_rank( + f"Warning: Could not extract step number from {checkpoint_path}, starting from step 0", + logger=logger, + rank=self.device_mesh.get_rank(), + level=logging.WARNING, + log_only_rank_0=True, + ) + return 0 + self.resume_global_step = resume_step + + # Use checkpoint manager to load model state + self.checkpoint_manager.load_checkpoint(checkpoint_path) + log_with_rank( + f"Successfully loaded model checkpoint from {checkpoint_path} (step {resume_step})", + logger=logger, + rank=self.device_mesh.get_rank(), + log_only_rank_0=True, + ) + + # Always load dataloader state for StatefulDataLoader + self._load_dataloader_state(checkpoint_path) + + return resume_step + + def _load_dataloader_state(self, checkpoint_path: str): + """Load dataloader state from checkpoint""" + dataloader_path = os.path.join(checkpoint_path, "data.pt") + + if os.path.exists(dataloader_path): + # Use StatefulDataLoader's built-in state dict functionality + dataloader_state_dict = torch.load(dataloader_path, map_location="cpu", weights_only=False) + self.train_dataloader.load_state_dict(dataloader_state_dict) + + log_with_rank( + f"Successfully loaded dataloader state from {dataloader_path}", + logger=logger, + rank=self.device_mesh.get_rank(), + log_only_rank_0=True, + ) + + else: + log_with_rank( + f"Warning: No dataloader state found at {dataloader_path}, will start from scratch", + logger=logger, + rank=self.device_mesh.get_rank(), + level=logging.WARNING, + log_only_rank_0=True, + ) + + def _determine_resume_path(self): + """Determine the path to resume from based on resume_mode configuration""" + resume_mode = getattr(self.config.trainer, "resume_mode", "auto") + resume_from_path = getattr(self.config.trainer, "resume_from_path", None) + + if resume_mode == "disable": + return None + elif resume_mode == "auto": + if resume_from_path is not None: + assert os.path.exists(resume_from_path), ( + "resume_from_path must be null or an existing path when resume_mode is 'auto'" + ) + assert "global_step_" in resume_from_path, "resume_from_path must specify the global_steps" + return resume_from_path + # Try to find the latest checkpoint in the default directory + return self._find_latest_checkpoint() + elif resume_mode == "resume_path": + assert os.path.exists(resume_from_path), ( + "resume_from_path must be an existing path when resume_mode is 'resume_path'" + ) + assert "global_step_" in resume_from_path, "resume_from_path must specify the global_steps" + return resume_from_path + else: + raise ValueError(f"Invalid resume_mode: {resume_mode}. Must be 'auto', 'disable', or 'resume_path'") + + def _find_latest_checkpoint(self): + """Find the latest checkpoint in the default local directory""" + checkpoint_dir = self.config.trainer.default_local_dir + + if not os.path.exists(checkpoint_dir): + return None + + latest_checkpoint = find_latest_ckpt_path(checkpoint_dir) + + if latest_checkpoint and self.device_mesh.get_rank() == 0: + step_num = extract_step(latest_checkpoint) + print(f"Found latest checkpoint: {latest_checkpoint} (step {step_num})") + + return latest_checkpoint + + def fit(self): + rank = self.device_mesh.get_rank() + + # TODO: add a unified tracking + if rank == 0: + tracking = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + global_step = self.resume_global_step # Start from resumed step + last_valid_metric = None + # compute the total training steps. + # the total training steps in SFT is mainly for early exit + total_training_steps = len(self.train_dataloader) * self.config.trainer.total_epochs + + if self.config.trainer.total_training_steps is not None: + total_training_steps = self.config.trainer.total_training_steps + + self.total_training_steps = total_training_steps + log_with_rank( + f"Total training steps: {self.total_training_steps},", + logger=logger, + rank=self.device_mesh.get_rank(), + log_only_rank_0=True, + ) + + # With StatefulDataLoader, we don't need to manually calculate epochs and steps + # The dataloader will automatically resume from where it left off + if global_step > 0: + log_with_rank( + f"StatefulDataLoader will automatically resume from global step: {global_step}", + logger=logger, + rank=self.device_mesh.get_rank(), + log_only_rank_0=True, + ) + + # Calculate which epoch we're starting from for sampler.set_epoch() + start_epoch = global_step // self.steps_per_epoch + + train_time = 0 + for epoch in range(start_epoch, self.config.trainer.total_epochs): + self.train_sampler.set_epoch(epoch=epoch) + + for step_in_epoch, data in enumerate( + tqdm( + self.train_dataloader, + initial=global_step % self.steps_per_epoch if epoch == start_epoch else 0, + total=self.steps_per_epoch, + desc=f"Epoch {epoch + 1}/{self.config.trainer.total_epochs}", + disable=rank != 0, + ) + ): + global_step += 1 + data = TensorDict(data, batch_size=self.config.data.train_batch_size).to(self.device_name) + metric = self.training_step(data) + train_time += metric["train/time(s)"] + if rank == 0: + tracking.log(data=metric, step=global_step) + + is_last_step = global_step >= self.total_training_steps + is_valid_step = global_step % self.config.trainer.test_freq == 0 + is_save_step = global_step % self.config.trainer.save_freq == 0 + + # early exit or validation step + if is_last_step or (self.config.trainer.test_freq > 0 and is_valid_step): + # Perform validation + val_losses = [] + for val_data in self.val_dataloader: + val_data = TensorDict(val_data, batch_size=self.config.data.micro_batch_size_per_gpu).to( + self.device_name + ) + val_loss = self.validation_step(val_data) + val_losses.append(val_loss) + if rank == 0: + val_loss = torch.mean(torch.stack(val_losses)) + metric = {"val/loss": val_loss.detach().item()} + tracking.log(data=metric, step=global_step) + last_valid_metric = metric + torch.distributed.barrier() + + if is_last_step or (self.config.trainer.save_freq > 0 and is_save_step): + self.save_checkpoint(step=global_step) + + if is_last_step: + if rank == 0: + print(f"Total time for train steps: {train_time:.2f}s") + print(f"Final validation metrics: {last_valid_metric}") + return + + +def run_sft(config): + device_name = get_device_name() + local_rank, rank, world_size = initialize_global_process_group() + + device_mesh = init_device_mesh(device_type=device_name, mesh_shape=(world_size,), mesh_dim_names=("fsdp",)) + dp_size = world_size // config.ulysses_sequence_parallel_size + ulysses_device_mesh = init_device_mesh( + device_type=device_name, + mesh_shape=(dp_size, config.ulysses_sequence_parallel_size), + mesh_dim_names=("dp", "sp"), + ) + # build tokenizer and datasets first + from verl.utils import hf_tokenizer + + local_model_path = copy_to_local(src=config.model.partial_pretrain, verbose=True) + tokenizer = hf_tokenizer(local_model_path, trust_remote_code=config.model.trust_remote_code) + train_dataset = create_sft_dataset(config.data.train_files, config.data, tokenizer) + val_dataset = create_sft_dataset(config.data.val_files, config.data, tokenizer) + + trainer = FSDPSFTTrainer( + config=config, + device_mesh=device_mesh, + ulysses_device_mesh=ulysses_device_mesh, + tokenizer=tokenizer, + train_dataset=train_dataset, + val_dataset=val_dataset, + ) + + trainer.fit() + + destroy_global_process_group() + + +@hydra.main(config_path="config", config_name="sft_trainer", version_base=None) +def main(config): + run_sft(config) + + +def create_sft_dataset(data_paths, data_config, tokenizer): + """Create a dataset.""" + # build dataset + # First check if a custom dataset class is specified + if data_config.custom_cls.get("path", None): + from verl.utils.import_utils import load_extern_type + + dataset_cls = load_extern_type(data_config.custom_cls.path, data_config.custom_cls.name) + # Then check if multi-turn dataset should be used + elif data_config.get("multiturn", {}).get("enable", False): + dataset_cls = MultiTurnSFTDataset + # Default to single-turn dataset + else: + dataset_cls = SFTDataset + + # Create datasets based on the selected class + dataset = dataset_cls(parquet_files=data_paths, tokenizer=tokenizer, config=data_config) + return dataset + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/verl/trainer/main_eval.py b/e2c/verl/verl/trainer/main_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..579adbf803f151b0a004aa498f9001d7d2ed622e --- /dev/null +++ b/e2c/verl/verl/trainer/main_eval.py @@ -0,0 +1,81 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Offline evaluate the performance of a generated file using reward model and ground truth verifier. +The input is a parquet file that contains N generated sequences and (optional) the ground truth. + +""" + +from collections import defaultdict + +import hydra +import numpy as np +import pandas as pd +import ray +from omegaconf import OmegaConf +from tqdm import tqdm + +from verl.trainer.ppo.reward import get_custom_reward_fn +from verl.utils.fs import copy_to_local + + +@ray.remote +def process_item(reward_fn, data_source, response_lst, reward_data): + ground_truth = reward_data["ground_truth"] + score_lst = [reward_fn(data_source, r, ground_truth) for r in response_lst] + return data_source, np.mean(score_lst) + + +@hydra.main(config_path="config", config_name="evaluation", version_base=None) +def main(config): + local_path = copy_to_local(config.data.path, use_shm=config.data.get("use_shm", False)) + dataset = pd.read_parquet(local_path) + responses = dataset[config.data.response_key] + data_sources = dataset[config.data.data_source_key] + reward_model_data = dataset[config.data.reward_model_key] + + total = len(dataset) + + # Initialize Ray + if not ray.is_initialized(): + ray.init(**OmegaConf.to_container(config.ray_kwargs.get("ray_init", {}))) + + # evaluate test_score based on data source + data_source_reward = defaultdict(list) + compute_score = get_custom_reward_fn(config) + + # Create remote tasks + remote_tasks = [ + process_item.remote(compute_score, data_sources[i], responses[i], reward_model_data[i]) for i in range(total) + ] + + # Process results as they come in + with tqdm(total=total) as pbar: + while len(remote_tasks) > 0: + # Use ray.wait to get completed tasks + done_ids, remote_tasks = ray.wait(remote_tasks) + for result_id in done_ids: + data_source, score = ray.get(result_id) + data_source_reward[data_source].append(score) + pbar.update(1) + + metric_dict = {} + for data_source, rewards in data_source_reward.items(): + metric_dict[f"test_score/{data_source}"] = np.mean(rewards) + + print(metric_dict) + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/verl/trainer/main_generation.py b/e2c/verl/verl/trainer/main_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..791c17af7efb207f8909757bb0ae52d5a660c72c --- /dev/null +++ b/e2c/verl/verl/trainer/main_generation.py @@ -0,0 +1,153 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Generate responses given a dataset of prompts +""" + +import os + +import hydra +import numpy as np +import ray + +os.environ["NCCL_DEBUG"] = "WARN" +os.environ["TOKENIZERS_PARALLELISM"] = "true" +# os.environ['TORCH_COMPILE_DISABLE'] = '1' + +from pprint import pprint + +import pandas as pd +from omegaconf import OmegaConf + +from verl import DataProto +from verl.protocol import pad_dataproto_to_divisor, unpad_dataproto +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup +from verl.utils import hf_tokenizer +from verl.utils.fs import copy_to_local +from verl.utils.hdfs_io import makedirs +from verl.utils.model import compute_position_id_with_mask +from verl.workers.fsdp_workers import ActorRolloutRefWorker + + +@hydra.main(config_path="config", config_name="generation", version_base=None) +def main(config): + run_generation(config) + + +def run_generation(config) -> None: + if not ray.is_initialized(): + # this is for local ray cluster + default_runtime_env = {"env_vars": {"TOKENIZERS_PARALLELISM": "true", "NCCL_DEBUG": "WARN"}} + ray_init_kwargs = config.ray_kwargs.get("ray_init", {}) + runtime_env_kwargs = ray_init_kwargs.get("runtime_env", {}) + runtime_env = OmegaConf.merge(default_runtime_env, runtime_env_kwargs) + ray_init_kwargs = OmegaConf.create({**ray_init_kwargs, "runtime_env": runtime_env}) + print(f"ray init kwargs: {ray_init_kwargs}") + ray.init(**OmegaConf.to_container(ray_init_kwargs)) + + ray.get(main_task.remote(config)) + + +@ray.remote(num_cpus=1) +def main_task(config): + pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values + OmegaConf.resolve(config) + + local_path = copy_to_local(config.model.path) + trust_remote_code = config.data.get("trust_remote_code", False) + tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) + + if config.rollout.temperature == 0.0: + assert config.data.n_samples == 1, "When temperature=0, n_samples must be 1." + assert config.data.n_samples >= 1, "n_samples should always >= 1" + + # read dataset. Note that the dataset should directly contain chat template format (e.g., a list of dictionary) + dataset = pd.read_parquet(config.data.path) + chat_lst = dataset[config.data.prompt_key].tolist() + + chat_lst = [chat.tolist() for chat in chat_lst] + + tokenizer.padding_side = "left" + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + ray_cls_with_init = RayClassWithInitArgs(cls=ray.remote(ActorRolloutRefWorker), config=config, role="rollout") + resource_pool = RayResourcePool(process_on_nodes=[config.trainer.n_gpus_per_node] * config.trainer.nnodes) + wg = RayWorkerGroup( + resource_pool=resource_pool, + ray_cls_with_init=ray_cls_with_init, + device_name=config.trainer.device, + ) + wg.init_model() + + total_samples = len(dataset) + config_batch_size = config.data.batch_size + apply_chat_template_kwargs = config.data.get("apply_chat_template_kwargs", {}) + num_batch = -(-total_samples // config_batch_size) + output_lst = [[] for _ in range(config.data.n_samples)] + + for batch_idx in range(num_batch): + print(f"[{batch_idx + 1}/{num_batch}] Start to process.") + batch_chat_lst = chat_lst[batch_idx * config_batch_size : (batch_idx + 1) * config_batch_size] + inputs = tokenizer.apply_chat_template( + batch_chat_lst, + add_generation_prompt=True, + padding=True, + truncation=True, + max_length=config.rollout.prompt_length, + return_tensors="pt", + return_dict=True, + tokenize=True, + **apply_chat_template_kwargs, + ) + input_ids = inputs["input_ids"] + attention_mask = inputs["attention_mask"] + position_ids = compute_position_id_with_mask(attention_mask) + batch_dict = {"input_ids": input_ids, "attention_mask": attention_mask, "position_ids": position_ids} + + data = DataProto.from_dict(batch_dict) + data_padded, pad_size = pad_dataproto_to_divisor(data, wg.world_size) + + # START TO GENERATE FOR n_samples TIMES + print(f"[{batch_idx + 1}/{num_batch}] Start to generate.") + for n_sample in range(config.data.n_samples): + output_padded = wg.generate_sequences(data_padded) + output = unpad_dataproto(output_padded, pad_size=pad_size) + + output_texts = [] + for i in range(len(output)): + data_item = output[i] + prompt_length = data_item.batch["prompts"].shape[-1] + valid_response_length = data_item.batch["attention_mask"][prompt_length:].sum() + valid_response_ids = data_item.batch["responses"][:valid_response_length] + response_str = tokenizer.decode(valid_response_ids, skip_special_tokens=True) + output_texts.append(response_str) + + output_lst[n_sample].extend(output_texts) + + # convert output_lst from (n_samples, n_data) to (n_data, n_sampels) + output_lst = np.array(output_lst, dtype=object) + output_lst = np.transpose(output_lst, axes=(1, 0)).tolist() + + # add to the data frame + dataset["responses"] = output_lst + + # write to a new parquet + output_dir = os.path.dirname(config.data.output_path) + makedirs(output_dir, exist_ok=True) + dataset.to_parquet(config.data.output_path) + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/verl/trainer/main_ppo.py b/e2c/verl/verl/trainer/main_ppo.py new file mode 100644 index 0000000000000000000000000000000000000000..1320cdaa1455dc3f4cfe5a60c9c06d83b9596231 --- /dev/null +++ b/e2c/verl/verl/trainer/main_ppo.py @@ -0,0 +1,412 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Note that we don't combine the main with ray_trainer as ray_trainer is used by other mpain. +""" + +import os +import socket + +import hydra +import ray +from omegaconf import OmegaConf + +from verl.experimental.dataset.sampler import AbstractSampler +from verl.trainer.constants_ppo import get_ppo_ray_runtime_env +from verl.trainer.ppo.ray_trainer import RayPPOTrainer +from verl.trainer.ppo.reward import load_reward_manager +from verl.trainer.ppo.utils import need_critic, need_reference_policy +from verl.utils.config import validate_config +from verl.utils.device import is_cuda_available +from verl.utils.import_utils import load_extern_type + + +@hydra.main(config_path="config", config_name="ppo_trainer", version_base=None) +def main(config): + """Main entry point for PPO training with Hydra configuration management. + + Args: + config_dict: Hydra configuration dictionary containing training parameters. + """ + run_ppo(config) + + +# Define a function to run the PPO-like training process +def run_ppo(config) -> None: + """Initialize Ray cluster and run distributed PPO training process. + + Args: + config: Training configuration object containing all necessary parameters + for distributed PPO training including Ray initialization settings, + model paths, and training hyperparameters. + """ + # Check if Ray is not initialized + if not ray.is_initialized(): + # Initialize Ray with a local cluster configuration + # Set environment variables in the runtime environment to control tokenizer parallelism, + # NCCL debug level, VLLM logging level, and allow runtime LoRA updating + # `num_cpus` specifies the number of CPU cores Ray can use, obtained from the configuration + default_runtime_env = get_ppo_ray_runtime_env() + ray_init_kwargs = config.ray_kwargs.get("ray_init", {}) + runtime_env_kwargs = ray_init_kwargs.get("runtime_env", {}) + runtime_env = OmegaConf.merge(default_runtime_env, runtime_env_kwargs) + ray_init_kwargs = OmegaConf.create({**ray_init_kwargs, "runtime_env": runtime_env}) + print(f"ray init kwargs: {ray_init_kwargs}") + ray.init(**OmegaConf.to_container(ray_init_kwargs)) + + # Create a remote instance of the TaskRunner class, and + # Execute the `run` method of the TaskRunner instance remotely and wait for it to complete + if ( + is_cuda_available + and config.global_profiler.tool == "nsys" + and config.global_profiler.get("steps") is not None + and len(config.global_profiler.get("steps", [])) > 0 + ): + from verl.utils.import_utils import is_nvtx_available + + assert is_nvtx_available(), "nvtx is not available in CUDA platform. Please 'pip3 install nvtx'" + nsight_options = OmegaConf.to_container( + config.global_profiler.global_tool_config.nsys.controller_nsight_options + ) + runner = TaskRunner.options(runtime_env={"nsight": nsight_options}).remote() + else: + runner = TaskRunner.remote() + ray.get(runner.run.remote(config)) + + # [Optional] get the path of the timeline trace file from the configuration, default to None + # This file is used for performance analysis + timeline_json_file = config.ray_kwargs.get("timeline_json_file", None) + if timeline_json_file: + ray.timeline(filename=timeline_json_file) + + +@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head +class TaskRunner: + """Ray remote class for executing distributed PPO training tasks. + + This class encapsulates the main training logic and runs as a Ray remote actor + to enable distributed execution across multiple nodes and GPUs. + + Attributes: + role_worker_mapping: Dictionary mapping Role enums to Ray remote worker classes + mapping: Dictionary mapping Role enums to resource pool IDs for GPU allocation + """ + + def __init__(self): + self.role_worker_mapping = {} + self.mapping = {} + + def add_actor_rollout_worker(self, config): + """Add actor rollout worker based on the actor strategy.""" + from verl.single_controller.ray import RayWorkerGroup + + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + from verl.workers.fsdp_workers import ActorRolloutRefWorker, AsyncActorRolloutRefWorker + + actor_rollout_cls = ( + AsyncActorRolloutRefWorker + if config.actor_rollout_ref.rollout.mode == "async" + else ActorRolloutRefWorker + ) + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + from verl.workers.megatron_workers import ActorRolloutRefWorker, AsyncActorRolloutRefWorker + + actor_rollout_cls = ( + AsyncActorRolloutRefWorker + if config.actor_rollout_ref.rollout.mode == "async" + else ActorRolloutRefWorker + ) + ray_worker_group_cls = RayWorkerGroup + + else: + raise NotImplementedError + + from verl.trainer.ppo.ray_trainer import Role + + self.role_worker_mapping[Role.ActorRollout] = ray.remote(actor_rollout_cls) + + return actor_rollout_cls, ray_worker_group_cls + + def add_critic_worker(self, config): + """Add critic worker to role mapping.""" + if config.critic.strategy in {"fsdp", "fsdp2"}: + use_legacy_worker_impl = config.trainer.get("use_legacy_worker_impl", "auto") + if use_legacy_worker_impl in ["auto", "enable"]: + from verl.workers.fsdp_workers import CriticWorker + elif use_legacy_worker_impl == "disable": + from verl.workers.roles import CriticWorker + + print("Using new worker implementation") + else: + raise ValueError(f"Invalid use_legacy_worker_impl: {use_legacy_worker_impl}") + + elif config.critic.strategy == "megatron": + from verl.workers.megatron_workers import CriticWorker + + else: + raise NotImplementedError + + from verl.trainer.ppo.ray_trainer import Role + + self.role_worker_mapping[Role.Critic] = ray.remote(CriticWorker) + + def init_resource_pool_mgr(self, config): + """Initialize resource pool manager.""" + from verl.trainer.ppo.ray_trainer import Role + + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + # TODO Here you can use the new registration method to support dynamic registration of roles + if config.reward_model.enable_resource_pool: + if config.reward_model.n_gpus_per_node <= 0: + raise ValueError("config.reward_model.n_gpus_per_node must be greater than 0") + if config.reward_model.nnodes <= 0: + raise ValueError("config.reward_model.nnodes must be greater than 0") + + reward_pool = [config.reward_model.n_gpus_per_node] * config.reward_model.nnodes + resource_pool_spec["reward_pool"] = reward_pool + + self.mapping[Role.ActorRollout] = global_pool_id + self.mapping[Role.Critic] = global_pool_id + from verl.trainer.ppo.ray_trainer import ResourcePoolManager + + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=self.mapping) + return resource_pool_manager + + def add_reward_model_worker(self, config): + """Add reward model worker if enabled.""" + from verl.trainer.ppo.ray_trainer import Role + + if config.reward_model.enable: + use_legacy_worker_impl = config.trainer.get("use_legacy_worker_impl", "auto") + if use_legacy_worker_impl in ["auto", "enable"]: + if config.reward_model.strategy in {"fsdp", "fsdp2"}: + from verl.workers.fsdp_workers import RewardModelWorker + elif config.reward_model.strategy == "megatron": + from verl.workers.megatron_workers import RewardModelWorker + else: + raise NotImplementedError + elif use_legacy_worker_impl == "disable": + from verl.workers.roles import RewardModelWorker + + print("Using new worker implementation") + else: + raise ValueError(f"Invalid use_legacy_worker_impl: {use_legacy_worker_impl}") + + self.role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + if config.reward_model.enable_resource_pool: + self.mapping[Role.RewardModel] = "reward_pool" + else: + self.mapping[Role.RewardModel] = "global_pool" + + def add_ref_policy_worker(self, config, ref_policy_cls): + """Add reference policy worker if KL loss or KL reward is used.""" + from verl.trainer.ppo.ray_trainer import Role + + if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + self.role_worker_mapping[Role.RefPolicy] = ray.remote(ref_policy_cls) + self.mapping[Role.RefPolicy] = "global_pool" + + def run(self, config): + """Execute the main PPO training workflow. + + This method sets up the distributed training environment, initializes + workers, datasets, and reward functions, then starts the training process. + + Args: + config: Training configuration object containing all parameters needed + for setting up and running the PPO training process. + """ + # Print the initial configuration. `resolve=True` will evaluate symbolic values. + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_to_local + + print(f"TaskRunner hostname: {socket.gethostname()}, PID: {os.getpid()}") + pprint(OmegaConf.to_container(config, resolve=True)) + OmegaConf.resolve(config) + + actor_rollout_cls, ray_worker_group_cls = self.add_actor_rollout_worker(config) + self.add_critic_worker(config) + + # We should adopt a multi-source reward function here: + # - for rule-based rm, we directly call a reward score + # - for model-based rm, we call a model + # - for code related prompt, we send to a sandbox if there are test cases + # finally, we combine all the rewards together + # The reward type depends on the tag of the data + self.add_reward_model_worker(config) + + # Add a reference policy worker if KL loss or KL reward is used. + self.add_ref_policy_worker(config, actor_rollout_cls) + + # validate config + validate_config( + config=config, + use_reference_policy=need_reference_policy(self.role_worker_mapping), + use_critic=need_critic(config), + ) + + # Download the checkpoint from HDFS to the local machine. + # `use_shm` determines whether to use shared memory, which could lead to faster model loading if turned on + local_path = copy_to_local( + config.actor_rollout_ref.model.path, use_shm=config.actor_rollout_ref.model.get("use_shm", False) + ) + + # Instantiate the tokenizer and processor. + from verl.utils import hf_processor, hf_tokenizer + + trust_remote_code = config.data.get("trust_remote_code", False) + tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) + # Used for multimodal LLM, could be None + processor = hf_processor(local_path, trust_remote_code=trust_remote_code, use_fast=True) + + # Load the reward manager for training and validation. + reward_fn = load_reward_manager( + config, tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {}) + ) + val_reward_fn = load_reward_manager( + config, tokenizer, num_examine=1, **config.reward_model.get("reward_kwargs", {}) + ) + + resource_pool_manager = self.init_resource_pool_mgr(config) + + from verl.utils.dataset.rl_dataset import collate_fn + + # Create training and validation datasets. + train_dataset = create_rl_dataset(config.data.train_files, config.data, tokenizer, processor, is_train=True) + val_dataset = create_rl_dataset(config.data.val_files, config.data, tokenizer, processor, is_train=False) + train_sampler = create_rl_sampler(config.data, train_dataset) + + # Initialize the PPO trainer. + trainer = RayPPOTrainer( + config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=self.role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + train_dataset=train_dataset, + val_dataset=val_dataset, + collate_fn=collate_fn, + train_sampler=train_sampler, + ) + # Initialize the workers of the trainer. + trainer.init_workers() + + # Start the training process. + trainer.fit() + + +def create_rl_dataset(data_paths, data_config, tokenizer, processor, is_train=True): + """Create a dataset. + + Arguments: + data_paths: List of paths to data files. + data_config: The data config. + tokenizer (Tokenizer): The tokenizer. + processor (Processor): The processor. + + Returns: + dataset (Dataset): The dataset. + """ + from torch.utils.data import Dataset + + from verl.utils.dataset.rl_dataset import RLHFDataset + + # Check if a custom dataset class is specified in the data configuration + # and if the path to the custom class is provided + if "custom_cls" in data_config and data_config.custom_cls.get("path", None) is not None: + # Dynamically load the custom dataset class + dataset_cls = load_extern_type(data_config.custom_cls.path, data_config.custom_cls.name) + # Verify that the custom dataset class inherits from torch.utils.data.Dataset + if not issubclass(dataset_cls, Dataset): + raise TypeError( + f"The custom dataset class '{data_config.custom_cls.name}' from " + f"'{data_config.custom_cls.path}' must inherit from torch.utils.data.Dataset" + ) + elif "datagen" in data_config and data_config.datagen.get("path", None) is not None and is_train: + # If a data generation strategy is specified, use the DynamicGenDataset class + from verl.utils.dataset.dynamicgen_dataset import DynamicGenDataset + + dataset_cls = DynamicGenDataset + print("Using DynamicGenDataset for data generation.") + else: + # Use the default RLHFDataset class if no custom class is specified + dataset_cls = RLHFDataset + print(f"Using dataset class: {dataset_cls.__name__}") + + # Instantiate the dataset using the determined dataset class + dataset = dataset_cls( + data_files=data_paths, + tokenizer=tokenizer, + processor=processor, + config=data_config, + ) + + return dataset + + +def create_rl_sampler(data_config, dataset): + """Create a sampler for the dataset. + + Arguments: + data_config: The data config. + dataset (Dataset): The dataset. + + Returns: + sampler (Sampler): The sampler. + """ + import torch + from torch.utils.data import RandomSampler, SequentialSampler + + if data_config.sampler is not None and data_config.sampler.get("class_path", None) is not None: + curriculum_class = load_extern_type( + data_config.sampler.class_path, + data_config.sampler.class_name, + ) + sampler = curriculum_class( + data_source=dataset, + data_config=data_config, + ) + assert isinstance(sampler, AbstractSampler) + assert data_config.get("dataloader_num_workers", 8) == 0, ( + "If using curriculum, num_workers must be 0 to prevent data caching. " + "If the dataloader caches data before the batch is done the " + "curriculum sampler won't have the opportunity to reorder it. " + ) + + # Use a sampler to facilitate checkpoint resumption. + # If shuffling is enabled in the data configuration, create a random sampler. + elif data_config.shuffle: + train_dataloader_generator = torch.Generator() + train_dataloader_generator.manual_seed(data_config.get("seed", 1)) + sampler = RandomSampler(data_source=dataset, generator=train_dataloader_generator) + else: + # If shuffling is disabled, use a sequential sampler to iterate through the dataset in order. + sampler = SequentialSampler(data_source=dataset) + + return sampler + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/verl/trainer/ppo/__init__.py b/e2c/verl/verl/trainer/ppo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/trainer/ppo/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/trainer/ppo/core_algos.py b/e2c/verl/verl/trainer/ppo/core_algos.py new file mode 100644 index 0000000000000000000000000000000000000000..cf2a00f98bea9c0812643ddb9d8e6dcfc52e928a --- /dev/null +++ b/e2c/verl/verl/trainer/ppo/core_algos.py @@ -0,0 +1,1505 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Core functions to implement PPO algorithms. +The function implemented in this file should be used by trainer with different distributed strategies to +implement PPO-like algorithms. +""" + +__all__ = ["register_adv_est", "get_adv_estimator_fn", "AdvantageEstimator"] + +from collections import defaultdict +from enum import Enum +from typing import Any, Callable, Optional + +import numpy as np +import torch +from omegaconf import DictConfig + +import verl.utils.torch_functional as verl_F +from verl.trainer.config import AlgoConfig +from verl.utils import as_torch_index, group_mean_std +from verl.utils.import_utils import deprecated +from verl.workers.config import ActorConfig + +PolicyLossFn = Callable[ + [ + torch.Tensor, # old_log_prob + torch.Tensor, # log_prob + torch.Tensor, # advantages + torch.Tensor, # response_mask + str, # loss_agg_mode + Optional[DictConfig | AlgoConfig], # config + torch.Tensor | None, # rollout_log_probs + ], + tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], +] + +POLICY_LOSS_REGISTRY: dict[str, PolicyLossFn] = {} + + +def register_policy_loss(name: str) -> Callable[[PolicyLossFn], PolicyLossFn]: + """Register a policy loss function with the given name. + + Args: + name (str): The name to register the policy loss function under. + + Returns: + function: Decorator function that registers the policy loss function. + """ + + def decorator(func: PolicyLossFn) -> PolicyLossFn: + POLICY_LOSS_REGISTRY[name] = func + return func + + return decorator + + +def get_policy_loss_fn(name): + """Get the policy loss with a given name. + + Args: + name: `(str)` + The name of the policy loss. + + Returns: + `(callable)`: The policy loss function. + """ + loss_name = name + if loss_name not in POLICY_LOSS_REGISTRY: + raise ValueError( + f"Unsupported loss mode: {loss_name}. Supported modes are: {list(POLICY_LOSS_REGISTRY.keys())}" + ) + return POLICY_LOSS_REGISTRY[loss_name] + + +class AdvantageEstimator(str, Enum): + """Using an enumeration class to avoid spelling errors in adv_estimator. + + Note(haibin.lin): this enum class is immutable after creation. Extending this + enum for new estimators may not be necessary since users can always just call + `verl.trainer.ppo.core_algos.register` with string name for a custom advantage + estimator instead. + """ + + GAE = "gae" + GRPO = "grpo" + REINFORCE_PLUS_PLUS = "reinforce_plus_plus" + REINFORCE_PLUS_PLUS_BASELINE = "reinforce_plus_plus_baseline" + REMAX = "remax" + RLOO = "rloo" + OPO = "opo" + GRPO_PASSK = "grpo_passk" + GPG = "gpg" + RLOO_VECTORIZED = "rloo_vectorized" + GRPO_VECTORIZED = "grpo_vectorized" + + +ADV_ESTIMATOR_REGISTRY: dict[str, Any] = {} + + +def register_adv_est(name_or_enum: str | AdvantageEstimator) -> Any: + """Decorator to register a advantage estimator function with a given name. + + Args: + name_or_enum: `(str)` or `(AdvantageEstimator)` + The name or enum of the advantage estimator. + + """ + + def decorator(fn): + name = name_or_enum.value if isinstance(name_or_enum, Enum) else name_or_enum + if name in ADV_ESTIMATOR_REGISTRY and ADV_ESTIMATOR_REGISTRY[name] != fn: + raise ValueError( + f"Adv estimator {name} has already been registered: {ADV_ESTIMATOR_REGISTRY[name]} vs {fn}" + ) + ADV_ESTIMATOR_REGISTRY[name] = fn + return fn + + return decorator + + +def get_adv_estimator_fn(name_or_enum): + """Get the advantage estimator function with a given name. + + Args: + name_or_enum: `(str)` or `(AdvantageEstimator)` + The name or enum of the advantage estimator. + + Returns: + `(callable)`: The advantage estimator function. + """ + name = name_or_enum.value if isinstance(name_or_enum, Enum) else name_or_enum + if name not in ADV_ESTIMATOR_REGISTRY: + raise ValueError(f"Unknown advantage estimator simply: {name}") + return ADV_ESTIMATOR_REGISTRY[name] + + +class AdaptiveKLController: + """ + Adaptive KL controller described in the paper: + https://arxiv.org/pdf/1909.08593.pdf + """ + + def __init__(self, init_kl_coef, target_kl, horizon): + self.value = init_kl_coef + self.target = target_kl + self.horizon = horizon + + def update(self, current_kl, n_steps): + """Update the KL coefficient based on current KL divergence. + + Args: + current_kl (float): Current KL divergence value. + n_steps (int): Number of steps taken. + """ + target = self.target + proportional_error = np.clip(current_kl / target - 1, -0.2, 0.2) + mult = 1 + proportional_error * n_steps / self.horizon + self.value *= mult + + +class FixedKLController: + """Fixed KL controller.""" + + def __init__(self, kl_coef): + self.value = kl_coef + + def update(self, current_kl, n_steps): + """Update method for fixed KL controller (no-op). + + Args: + current_kl (float): Current KL divergence value (unused). + n_steps (int): Number of steps taken (unused). + """ + pass + + +def get_kl_controller(kl_ctrl): + """Factory function to create appropriate KL controller based on configuration. + + Args: + kl_ctrl: Configuration object containing KL controller settings. + + Returns: + KL controller instance (FixedKLController or AdaptiveKLController). + + Raises: + NotImplementedError: If controller type is not supported. + AssertionError: If adaptive controller horizon is not positive. + """ + if kl_ctrl.type == "fixed": + return FixedKLController(kl_coef=kl_ctrl.kl_coef) + elif kl_ctrl.type == "adaptive": + assert kl_ctrl.horizon > 0, f"horizon must be larger than 0. Got {kl_ctrl.horizon}" + return AdaptiveKLController(init_kl_coef=kl_ctrl.kl_coef, target_kl=kl_ctrl.target_kl, horizon=kl_ctrl.horizon) + else: + raise NotImplementedError + + +@register_adv_est(AdvantageEstimator.GAE) # or simply: @register_adv_est("gae") +def compute_gae_advantage_return( + token_level_rewards: torch.Tensor, + values: torch.Tensor, + response_mask: torch.Tensor, + gamma: torch.Tensor, + lam: torch.Tensor, +): + """Adapted from https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py + + Args: + token_level_rewards: `(torch.Tensor)` + shape is (bs, response_length) + values: `(torch.Tensor)` + shape is (bs, response_length) + response_mask: `(torch.Tensor)` + shape is (bs, response_length). [EOS] mask. The token after [EOS] have mask zero. + gamma is `(float)` + discounted factor used in RL + lam: `(float)` + lambda value when computing Generalized Advantage Estimation (https://arxiv.org/abs/1506.02438) + + Returns: + advantages: `(torch.Tensor)` + shape: (bs, response_length) + Returns: `(torch.Tensor)` + shape: (bs, response_length) + + """ + with torch.no_grad(): + nextvalues = 0 + lastgaelam = 0 + advantages_reversed = [] + gen_len = token_level_rewards.shape[-1] + + for t in reversed(range(gen_len)): + delta = token_level_rewards[:, t] + gamma * nextvalues - values[:, t] + lastgaelam_ = delta + gamma * lam * lastgaelam + + # skip values and TD-error on observation tokens + nextvalues = values[:, t] * response_mask[:, t] + (1 - response_mask[:, t]) * nextvalues + lastgaelam = lastgaelam_ * response_mask[:, t] + (1 - response_mask[:, t]) * lastgaelam + + advantages_reversed.append(lastgaelam) + advantages = torch.stack(advantages_reversed[::-1], dim=1) + + returns = advantages + values + advantages = verl_F.masked_whiten(advantages, response_mask) + return advantages, returns + + +# NOTE(sgm): this implementation only consider outcome supervision, where the reward is a scalar. +@register_adv_est(AdvantageEstimator.GRPO) # or simply: @register_adv_est("grpo") +def compute_grpo_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + epsilon: float = 1e-6, + norm_adv_by_std_in_grpo: bool = True, + config: Optional[AlgoConfig] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for GRPO, operating only on Outcome reward + (with only one scalar reward for each response). + + Args: + token_level_rewards: `(torch.Tensor)` + shape is (bs, response_length) + response_mask: `(torch.Tensor)` + shape is (bs, response_length) + index: `(np.ndarray)` + index array for grouping + epsilon: `(float)` + small value to avoid division by zero + norm_adv_by_std_in_grpo: `(bool)` + whether to scale the GRPO advantage + config: `(Optional[AlgoConfig])` + algorithm configuration object + + Note: + If norm_adv_by_std_in_grpo is True, the advantage is scaled by the std, as in the original GRPO. + If False, the advantage is not scaled, as in Dr.GRPO (https://arxiv.org/abs/2503.20783). + + Returns: + advantages: `(torch.Tensor)` + shape is (bs, response_length) + Returns: `(torch.Tensor)` + shape is (bs, response_length) + """ + scores = token_level_rewards.sum(dim=-1) + + id2score = defaultdict(list) + id2mean = {} + id2std = {} + + with torch.no_grad(): + bsz = scores.shape[0] + for i in range(bsz): + id2score[index[i]].append(scores[i]) + for idx in id2score: + if len(id2score[idx]) == 1: + id2mean[idx] = torch.tensor(0.0) + id2std[idx] = torch.tensor(1.0) + elif len(id2score[idx]) > 1: + scores_tensor = torch.stack(id2score[idx]) + id2mean[idx] = torch.mean(scores_tensor) + id2std[idx] = torch.std(scores_tensor) + else: + raise ValueError(f"no score in prompt index: {idx}") + for i in range(bsz): + if norm_adv_by_std_in_grpo: + scores[i] = (scores[i] - id2mean[index[i]]) / (id2std[index[i]] + epsilon) + else: + scores[i] = scores[i] - id2mean[index[i]] + scores = scores.unsqueeze(-1) * response_mask + + return scores, scores + + +@register_adv_est(AdvantageEstimator.GRPO_VECTORIZED) +def compute_grpo_vectorized_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + epsilon: float = 1e-6, + norm_adv_by_std_in_grpo: bool = True, + config: Optional[AlgoConfig] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Vectorized GRPO(outcome-only): + For each group g: + a_i = \\frac{r_i - \\mu_g}{\\sigma_g} (or without dividing by \\sigma_g), + then broadcast the scalar across the token dimension (multiplied by response_mask).。 + """ + with torch.no_grad(): + scores = token_level_rewards.sum(dim=-1) + g = as_torch_index(index, device=scores.device) + mean_g, std_g, _ = group_mean_std(scores, g, eps=epsilon) + if norm_adv_by_std_in_grpo: + scalars = (scores - mean_g[g]) / (std_g[g] + epsilon) + else: + scalars = scores - mean_g[g] + advantages = scalars.unsqueeze(-1) * response_mask + return advantages, advantages + + +@register_adv_est(AdvantageEstimator.GRPO_PASSK) # or simply: @register_adv_est("grpo_passk") +def compute_grpo_passk_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + epsilon: float = 1e-6, + norm_adv_by_std_in_grpo: bool = True, + config: Optional[AlgoConfig] = None, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for Pass@k using a GRPO-style outcome reward formulation. + Only the best response per group gets a non-zero advantage: r_max - r_second_max. + + Implemented as described in https://arxiv.org/abs/2503.19595. + + Args: + token_level_rewards: (bs, response_length) + response_mask: (bs, response_length) + index: (bs,) → group ID per sample + epsilon: float for numerical stability + config: (AlgoConfig) algorithm settings, which contains "norm_adv_by_std_in_grpo" + + Returns: + advantages: (bs, response_length) + returns: (bs, response_length) + """ + assert config is not None + # if True, normalize advantage by std within group + norm_adv_by_std_in_grpo = config.get("norm_adv_by_std_in_grpo", True) + scores = token_level_rewards.sum(dim=-1) # (bs,) + advantages = torch.zeros_like(scores) + + id2scores = defaultdict(list) + id2indices = defaultdict(list) + + with torch.no_grad(): + bsz = scores.shape[0] + for i in range(bsz): + idx = index[i] + id2scores[idx].append(scores[i]) + id2indices[idx].append(i) + + for idx in id2scores: + rewards = torch.stack(id2scores[idx]) # (k,) + if rewards.numel() < 2: + raise ValueError( + f"Pass@k requires at least 2 samples per group. Got {rewards.numel()} for group {idx}." + ) + topk, topk_idx = torch.topk(rewards, 2) + r_max, r_second_max = topk[0], topk[1] + i_max = id2indices[idx][topk_idx[0].item()] + advantage = r_max - r_second_max + if norm_adv_by_std_in_grpo: + std = torch.std(rewards) + advantage = advantage / (std + epsilon) + advantages[i_max] = advantage + + advantages = advantages.unsqueeze(-1) * response_mask + return advantages, advantages + + +@register_adv_est( + AdvantageEstimator.REINFORCE_PLUS_PLUS_BASELINE +) # or simply: @register_adv_est("reinforce_plus_plus_baseline") +def compute_reinforce_plus_plus_baseline_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: torch.Tensor, + epsilon: float = 1e-6, + config: Optional[AlgoConfig] = None, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for RF++-baseline (https://arxiv.org/abs/2501.03262), operating only on Outcome reward + (with only one scalar reward for each response). + + Args: + token_level_rewards: `(torch.Tensor)` + shape: (bs, response_length) + response_mask: `(torch.Tensor)` + shape: (bs, response_length) + config: (AlgoConfig) algorithm config + + Returns: + advantages: `(torch.Tensor)` + shape: (bs, response_length) + Returns: `(torch.Tensor)` + shape: (bs, response_length) + """ + response_length = token_level_rewards.shape[-1] + scores = token_level_rewards.sum(dim=-1) + + id2score = defaultdict(list) + id2mean = {} + + with torch.no_grad(): + bsz = scores.shape[0] + for i in range(bsz): + id2score[index[i]].append(scores[i]) + for idx in id2score: + if len(id2score[idx]) == 1: + id2mean[idx] = torch.tensor(0.0) + elif len(id2score[idx]) > 1: + id2mean[idx] = torch.mean(torch.stack(id2score[idx])) + else: + raise ValueError(f"no score in prompt index: {idx}") + for i in range(bsz): + scores[i] = scores[i] - id2mean[index[i]] + + scores = scores.unsqueeze(-1).tile([1, response_length]) * response_mask + scores = verl_F.masked_whiten(scores, response_mask) * response_mask + + return scores, scores + + +@register_adv_est(AdvantageEstimator.RLOO) # or simply: @register_adv_est("rloo") +def compute_rloo_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + epsilon: float = 1e-6, + config: Optional[AlgoConfig] = None, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for RLOO based on https://arxiv.org/abs/2402.14740 + + Args: + token_level_rewards: `(torch.Tensor)` + shape: (bs, response_length) + response_mask: `(torch.Tensor)` + shape: (bs, response_length) + config: (AlgoConfig) algorithm config + + Returns: + advantages: `(torch.Tensor)` + shape: (bs, response_length) + Returns: `(torch.Tensor)` + shape: (bs, response_length) + """ + scores = token_level_rewards.sum(dim=-1) + + id2score = defaultdict(list) + id2mean = {} + + with torch.no_grad(): + bsz = scores.shape[0] + for i in range(bsz): + id2score[index[i]].append(scores[i]) + for idx in id2score: + if len(id2score[idx]) == 1: + id2mean[idx] = torch.tensor(0.0) + elif len(id2score[idx]) > 1: + id2mean[idx] = torch.mean(torch.stack(id2score[idx])) + else: + raise ValueError(f"no score in prompt index: {idx}") + for i in range(bsz): + response_num = len(id2score[index[i]]) + if response_num > 1: + scores[i] = scores[i] * response_num / (response_num - 1) - id2mean[index[i]] * response_num / ( + response_num - 1 + ) + scores = scores.unsqueeze(-1) * response_mask + + return scores, scores + + +@register_adv_est(AdvantageEstimator.OPO) # or simply: @register_adv_est("opo") +def compute_opo_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + epsilon: float = 1e-6, + config: Optional[AlgoConfig] = None, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for OPO based on https://arxiv.org/pdf/2505.23585 + + Args: + token_level_rewards: `(torch.Tensor)` + shape: (bs, response_length) + response_mask: `(torch.Tensor)` + shape: (bs, response_length) + config: (AlgoConfig) algorithm config + + Returns: + advantages: `(torch.Tensor)` + shape: (bs, response_length) + Returns: `(torch.Tensor)` + shape: (bs, response_length) + """ + response_length = response_mask.sum(dim=-1) + scores = token_level_rewards.sum(dim=-1) + + id2score = defaultdict(list) + id2len = defaultdict(list) + id2bsl = {} + + with torch.no_grad(): + bsz = scores.shape[0] + for i in range(bsz): + id2score[index[i]].append(scores[i]) + id2len[index[i]].append(response_length[i]) + + for idx in id2score: + if len(id2score[idx]) == 1: + id2bsl[idx] = torch.tensor(0.0) + elif len(id2score[idx]) > 1: + score_tensor = torch.stack(id2score[idx]) + len_tensor = torch.stack(id2len[idx]) + id2bsl[idx] = (len_tensor * score_tensor).sum() / len_tensor.sum() + else: + raise ValueError(f"no score in prompt index: {idx}") + for i in range(bsz): + scores[i] = scores[i] - id2bsl[index[i]] + scores = scores.unsqueeze(-1) * response_mask + + return scores, scores + + +@register_adv_est(AdvantageEstimator.REINFORCE_PLUS_PLUS) # or simply: @register_adv_est("reinforce_plus_plus") +def compute_reinforce_plus_plus_outcome_advantage( + token_level_rewards: torch.Tensor, response_mask: torch.Tensor, config: Optional[AlgoConfig] = None, **kwargs +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for REINFORCE++. + This implementation is based on the paper: https://arxiv.org/abs/2501.03262 + + Args: + token_level_rewards: `(torch.Tensor)` + shape: (bs, response_length) + response_mask: `(torch.Tensor)` + shape: (bs, response_length) + config: (AlgoConfig) algorithm config + + Returns: + advantages: `(torch.Tensor)` + shape: (bs, response_length) + Returns: `(torch.Tensor)` + shape: (bs, response_length) + """ + assert config is not None + gamma = config.gamma + with torch.no_grad(): + returns = torch.zeros_like(token_level_rewards) + running_return = 0 + + for t in reversed(range(token_level_rewards.shape[1])): + running_return = token_level_rewards[:, t] + gamma * running_return + returns[:, t] = running_return + # Reset after EOS + running_return = running_return * response_mask[:, t] + + advantages = verl_F.masked_whiten(returns, response_mask) + advantages = advantages * response_mask + + return advantages, returns + + +@register_adv_est(AdvantageEstimator.REMAX) # or simply: @register_adv_est("remax") +def compute_remax_outcome_advantage( + token_level_rewards: torch.Tensor, + reward_baselines: torch.Tensor, + response_mask: torch.Tensor, + config: Optional[AlgoConfig] = None, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for ReMax, operating only on Outcome reward + This implementation is based on the paper: https://arxiv.org/abs/2310.10505 + (with only one scalar reward for each response). + + Args: + token_level_rewards: `(torch.Tensor)` + shape: (bs, response_length) + reward_baselines: `(torch.Tensor)` + shape: (bs,) + response_mask: `(torch.Tensor)` + shape: (bs, response_length) + config: (AlgoConfig) algorithm config + + Returns: + advantages: `(torch.Tensor)` + shape: (bs, response_length) + Returns: `(torch.Tensor)` + shape: (bs, response_length) + """ + + with torch.no_grad(): + returns = (token_level_rewards * response_mask).flip(dims=[-1]).cumsum(dim=-1).flip(dims=[-1]) + advantages = returns - reward_baselines.unsqueeze(-1) * response_mask + + return advantages, returns + + +@register_adv_est(AdvantageEstimator.GPG) # or simply: @register_adv_est("gpg") +def compute_gpg_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + epsilon: float = 1e-6, + f_norm: float = 1.0, + alpha: float = 1.0, + config=None, + **kwargs, +): + """ + Compute advantage for GPG, operating only on Outcome reward + (with only one scalar reward for each response). + Args: + token_level_rewards: `(torch.Tensor)` + shape: (bs, response_length) + response_mask: `(torch.Tensor)` + shape: (bs, response_length) + index: `(np.ndarray)` + shape: (bs,) + epsilon: (float) + f_norm: (float) + alpha: (float) + config: (dict) algorithm config + + Returns: + advantages: `(torch.Tensor)` + shape: (bs, response_length) + Returns: `(torch.Tensor)` + shape: (bs, response_length) + """ + scores = token_level_rewards.sum(dim=-1) + + id2score = defaultdict(list) + id2mean = {} + id2std = {} + + with torch.no_grad(): + bsz = scores.shape[0] + m = torch.count_nonzero(scores) + alpha = bsz / m.clamp(min=1) + + for i in range(bsz): + id2score[index[i]].append(scores[i]) + + for idx in id2score: + if len(id2score[idx]) == 1: + id2mean[idx] = torch.tensor(0.0) + id2std[idx] = torch.tensor(1.0) + elif len(id2score[idx]) > 1: + scores_tensor = torch.stack(id2score[idx]) + id2mean[idx] = torch.mean(scores_tensor) + id2std[idx] = torch.std(scores_tensor) + else: + raise ValueError(f"no score in prompt index: {idx}") + for i in range(bsz): + scores[i] = alpha * (scores[i] - id2mean[index[i]]) / (f_norm) + scores = scores.unsqueeze(-1) * response_mask + + return scores, scores + + +@register_adv_est(AdvantageEstimator.RLOO_VECTORIZED) # or simply: @register_adv_est("rloo_vectorized") +def compute_rloo_vectorized_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + epsilon: float = 1e-6, + config: Optional[AlgoConfig] = None, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for RLOO based on https://arxiv.org/abs/2402.14740 + + Args: + token_level_rewards: `(torch.Tensor)` + shape: (bs, response_length) + response_mask: `(torch.Tensor)` + shape: (bs, response_length) + config: (AlgoConfig) algorithm config + + Returns: + advantages: `(torch.Tensor)` + shape: (bs, response_length) + Returns: `(torch.Tensor)` + shape: (bs, response_length) + """ + scores = token_level_rewards.sum(dim=-1) + + with torch.no_grad(): + inv = torch.from_numpy(np.unique(index, return_inverse=True)[1]).to(scores.device) + + c = torch.bincount(inv)[inv].to(scores.dtype) + adv = ((c * scores - torch.bincount(inv, weights=scores)[inv]) / (c - 1).clamp_min(1)) * (c > 1) + + adv = adv.unsqueeze(-1) * response_mask + + return adv, adv + + +def compute_rewards(token_level_scores, old_log_prob, ref_log_prob, kl_ratio): + """Compute token-level rewards with KL penalty. + + Args: + token_level_scores (torch.Tensor): Token-level reward scores. + old_log_prob (torch.Tensor): Log probabilities from current policy. + ref_log_prob (torch.Tensor): Log probabilities from reference policy. + kl_ratio (float): KL penalty coefficient. + + Returns: + torch.Tensor: Token-level rewards with KL penalty applied. + """ + kl = old_log_prob - ref_log_prob + return token_level_scores - kl * kl_ratio + + +def agg_loss(loss_mat: torch.Tensor, loss_mask: torch.Tensor, loss_agg_mode: str): + """ + Aggregate the loss matrix into a scalar. + + Args: + loss_mat: `(torch.Tensor)`: + shape: (bs, response_length) + loss_mask: `(torch.Tensor)`: + shape: (bs, response_length) + loss_agg_mode: (str) choices: + method to aggregate the loss matrix into a scalar. + Returns: + loss: `a scalar torch.Tensor` + aggregated loss + """ + if loss_agg_mode == "token-mean": + loss = verl_F.masked_mean(loss_mat, loss_mask) + elif loss_agg_mode == "seq-mean-token-sum": + seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) # token-sum + loss = torch.mean(seq_losses) # seq-mean + elif loss_agg_mode == "seq-mean-token-mean": + seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) / torch.sum(loss_mask, dim=-1) # token-mean + loss = torch.mean(seq_losses) # seq-mean + elif loss_agg_mode == "seq-mean-token-sum-norm": + seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) + loss = torch.sum(seq_losses) / loss_mask.shape[-1] # The divisor + # (loss_mask.shape[-1]) should ideally be constant + # throughout training to well-replicate the DrGRPO paper. + # TODO: Perhaps add user-defined normalizer argument to + # agg_loss to ensure divisor stays constant throughout. + else: + raise ValueError(f"Invalid loss_agg_mode: {loss_agg_mode}") + + return loss + + +@deprecated("verl.trainer.ppo.core_algos.compute_policy_loss_vanilla") +def compute_policy_loss( + old_log_prob, + log_prob, + advantages, + response_mask, + cliprange=None, + cliprange_low=None, + cliprange_high=None, + clip_ratio_c=3.0, + loss_agg_mode: str = "token-mean", +): + """ + Compute the clipped policy objective and related metrics for PPO. + + Adapted from + https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py#L1122 + + Args: + old_log_prob (torch.Tensor): + Log-probabilities of actions under the old policy, shape (batch_size, response_length). + log_prob (torch.Tensor): + Log-probabilities of actions under the current policy, shape (batch_size, response_length). + advantages (torch.Tensor): + Advantage estimates for each action, shape (batch_size, response_length). + response_mask (torch.Tensor): + Mask indicating which tokens to include in the loss, shape (batch_size, response_length). + cliprange (float, optional): + Clipping parameter ε for standard PPO. See https://arxiv.org/abs/1707.06347. + Defaults to None (must be provided). + cliprange_low (float, optional): + Lower clip range for dual-clip PPO. Defaults to same as `cliprange`. + cliprange_high (float, optional): + Upper clip range for dual-clip PPO. Defaults to same as `cliprange`. + clip_ratio_c (float, optional): + Lower bound of the ratio for dual-clip PPO. See https://arxiv.org/pdf/1912.09729. + Defaults to 3.0. + loss_agg_mode (str, optional): + Aggregation mode for `agg_loss`. Defaults to "token-mean". + """ + assert clip_ratio_c > 1.0, ( + "The lower bound of the clip_ratio_c for dual-clip PPO should be greater than 1.0," + + f" but get the value: {clip_ratio_c}." + ) + + negative_approx_kl = log_prob - old_log_prob + # Clamp negative_approx_kl for stability + negative_approx_kl = torch.clamp(negative_approx_kl, min=-20.0, max=20.0) + ratio = torch.exp(negative_approx_kl) + ppo_kl = verl_F.masked_mean(-negative_approx_kl, response_mask) + + pg_losses1 = -advantages * ratio + if cliprange_low is None: + cliprange_low = cliprange + if cliprange_high is None: + cliprange_high = cliprange + pg_losses2 = -advantages * torch.clamp( + ratio, 1 - cliprange_low, 1 + cliprange_high + ) # - clip(ratio, 1-cliprange, 1+cliprange) * A + clip_pg_losses1 = torch.maximum( + pg_losses1, pg_losses2 + ) # max(-ratio * A, -clip(ratio, 1-cliprange, 1+cliprange) * A) + pg_clipfrac = verl_F.masked_mean(torch.gt(pg_losses2, pg_losses1).float(), response_mask) + + pg_losses3 = -advantages * clip_ratio_c + clip_pg_losses2 = torch.min(pg_losses3, clip_pg_losses1) + pg_clipfrac_lower = verl_F.masked_mean( + torch.gt(clip_pg_losses1, pg_losses3) * (advantages < 0).float(), response_mask + ) + + pg_losses = torch.where(advantages < 0, clip_pg_losses2, clip_pg_losses1) + pg_loss = agg_loss(loss_mat=pg_losses, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + + return pg_loss, pg_clipfrac, ppo_kl, pg_clipfrac_lower + + +@register_policy_loss("vanilla") +def compute_policy_loss_vanilla( + old_log_prob: torch.Tensor, + log_prob: torch.Tensor, + advantages: torch.Tensor, + response_mask: torch.Tensor, + loss_agg_mode: str = "token-mean", + config: Optional[DictConfig | AlgoConfig] = None, + rollout_log_probs: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Compute the clipped policy objective and related metrics for PPO. + + Adapted from + https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py#L1122 + + Args: + old_log_prob (torch.Tensor): + Log-probabilities of actions under the old policy, shape (batch_size, response_length). + log_prob (torch.Tensor): + Log-probabilities of actions under the current policy, shape (batch_size, response_length). + advantages (torch.Tensor): + Advantage estimates for each action, shape (batch_size, response_length). + response_mask (torch.Tensor): + Mask indicating which tokens to include in the loss, shape (batch_size, response_length). + loss_agg_mode (str, optional): + Aggregation mode for `agg_loss`. Defaults to "token-mean". + config: `(verl.trainer.config.ActorConfig)`: + config for the actor. + rollout_log_probs: `(torch.Tensor)`: + log probabilities of actions under the rollout policy, shape (batch_size, response_length). + """ + + assert config is not None + assert not isinstance(config, AlgoConfig) + clip_ratio = config.clip_ratio # Clipping parameter ε for standard PPO. See https://arxiv.org/abs/1707.06347. + clip_ratio_low = config.clip_ratio_low if config.clip_ratio_low is not None else clip_ratio + clip_ratio_high = config.clip_ratio_high if config.clip_ratio_high is not None else clip_ratio + clip_ratio_c = config.get( # Lower bound of the ratio for dual-clip PPO. See https://arxiv.org/pdf/1912.09729. + "clip_ratio_c", 3.0 + ) + + cliprange = clip_ratio + cliprange_low = clip_ratio_low + cliprange_high = clip_ratio_high + + assert clip_ratio_c > 1.0, ( + "The lower bound of the clip_ratio_c for dual-clip PPO should be greater than 1.0," + + f" but get the value: {clip_ratio_c}." + ) + + negative_approx_kl = log_prob - old_log_prob + # Clamp negative_approx_kl for stability + negative_approx_kl = torch.clamp(negative_approx_kl, min=-20.0, max=20.0) + ratio = torch.exp(negative_approx_kl) + ppo_kl = verl_F.masked_mean(-negative_approx_kl, response_mask) + + pg_losses1 = -advantages * ratio + if cliprange_low is None: + cliprange_low = cliprange + if cliprange_high is None: + cliprange_high = cliprange + pg_losses2 = -advantages * torch.clamp( + ratio, 1 - cliprange_low, 1 + cliprange_high + ) # - clip(ratio, 1-cliprange, 1+cliprange) * A + clip_pg_losses1 = torch.maximum( + pg_losses1, pg_losses2 + ) # max(-ratio * A, -clip(ratio, 1-cliprange, 1+cliprange) * A) + pg_clipfrac = verl_F.masked_mean(torch.gt(pg_losses2, pg_losses1).float(), response_mask) + + pg_losses3 = -advantages * clip_ratio_c + clip_pg_losses2 = torch.min(pg_losses3, clip_pg_losses1) + pg_clipfrac_lower = verl_F.masked_mean( + torch.gt(clip_pg_losses1, pg_losses3) * (advantages < 0).float(), response_mask + ) + + pg_losses = torch.where(advantages < 0, clip_pg_losses2, clip_pg_losses1) + + if config.tis_imp_ratio_cap > 0 and rollout_log_probs is not None: + # Apply truncated importance sampling -> https://fengyao.notion.site/off-policy-rl + tis_imp_ratio = torch.exp(old_log_prob - rollout_log_probs) + tis_imp_ratio = torch.clamp(tis_imp_ratio, max=config.tis_imp_ratio_cap) + pg_losses = pg_losses * tis_imp_ratio + + pg_loss = agg_loss(loss_mat=pg_losses, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + + return pg_loss, pg_clipfrac, ppo_kl, pg_clipfrac_lower + + +@register_policy_loss("gspo") +def compute_policy_loss_gspo( + old_log_prob: torch.Tensor, + log_prob: torch.Tensor, + advantages: torch.Tensor, + response_mask: torch.Tensor, + loss_agg_mode: str = "seq-mean-token-mean", + config: Optional[DictConfig | ActorConfig] = None, + rollout_log_probs: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Compute the clipped policy objective and related metrics for GSPO. + + See https://arxiv.org/pdf/2507.18071 for more details. + + Args: + old_log_prob (torch.Tensor): + Log-probabilities of actions under the old policy, shape (batch_size, response_length). + log_prob (torch.Tensor): + Log-probabilities of actions under the current policy, shape (batch_size, response_length). + advantages (torch.Tensor): + Advantage estimates for each action, shape (batch_size, response_length). + response_mask (torch.Tensor): + Mask indicating which tokens to include in the loss, shape (batch_size, response_length). + loss_agg_mode (str, optional): + Aggregation mode for `agg_loss`. For GSPO, it is recommended to use "seq-mean-token-mean". + """ + + assert config is not None + assert isinstance(config, ActorConfig) + clip_ratio_low = config.clip_ratio_low if config.clip_ratio_low is not None else config.clip_ratio + clip_ratio_high = config.clip_ratio_high if config.clip_ratio_high is not None else config.clip_ratio + + negative_approx_kl = log_prob - old_log_prob + + # compute sequence-level importance ratio: + # si(θ) = (π_θ(yi|x)/π_θold(yi|x))^(1/|yi|) = + # exp [(1/|y_i|) * Σ_t log(π_θ(y_i,t|x,y_i, tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Adapted from + https://github.com/AMAP-ML/GPG/blob/main/VisualThinker-R1-Zero/src/open-r1-multimodal/src/open_r1/trainer/grpo_trainer.py#L495 + Args: + log_prob: `(torch.Tensor)` + shape: (bs, response_length) + advantages: `(torch.Tensor)` + shape: (bs, response_length) + response_mask: `(torch.Tensor)` + shape: (bs, response_length) + return: + pg_loss: `a scalar torch.Tensor` + policy gradient loss computed via GPG + """ + pg_losses = -log_prob * advantages + + pg_loss = agg_loss(loss_mat=pg_losses, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + return pg_loss, torch.tensor(0.0), torch.tensor(0.0), torch.tensor(0.0) + + +@register_policy_loss("clip_cov") +def compute_policy_loss_clip_cov( + old_log_prob: torch.Tensor, + log_prob: torch.Tensor, + advantages: torch.Tensor, + response_mask: torch.Tensor, + loss_agg_mode: str = "token-mean", + config: Optional[DictConfig | AlgoConfig] = None, + rollout_log_probs: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Compute the clipped policy objective and related metrics for Clip-Cov. + + Adapted from + https://github.com/PRIME-RL/Entropy-Mechanism-of-RL/blob/main/verl/trainer/ppo/core_algos.py + + Args: + old_log_prob (torch.Tensor): + Log-probabilities of actions under the old policy, shape (batch_size, response_length). + log_prob (torch.Tensor): + Log-probabilities of actions under the current policy, shape (batch_size, response_length). + advantages (torch.Tensor): + Advantage estimates for each action, shape (batch_size, response_length). + response_mask (torch.Tensor): + Mask indicating which tokens to include in the loss, shape (batch_size, response_length). + cliprange (float, optional): + Clipping parameter ε for standard PPO. See https://arxiv.org/abs/1707.06347. + Defaults to None (must be provided). + cliprange_low (float, optional): + Lower clip range for dual-clip PPO. Defaults to same as `cliprange`. + cliprange_high (float, optional): + Upper clip range for dual-clip PPO. Defaults to same as `cliprange`. + loss_agg_mode (str, optional): + Aggregation mode for `agg_loss`. Defaults to "token-mean". + clip_cvo_ratio (float, optional): + Ratio for clipping the covariance. Defaults to 0.0002. + clip_cov_lb (float, optional): + Lower bound for clipping covariance. Defaults to 1.0. + clip_cov_ub (float, optional): + Upper bound for clipping covariance. Defaults to 5.0. + """ + assert config is not None + assert not isinstance(config, AlgoConfig), "passing AlgoConfig not supported yet" + assert config.policy_loss is not None + + clip_cov_ratio = config.policy_loss.clip_cov_ratio if config.policy_loss.clip_cov_ratio is not None else 0.0002 + cliprange = config.clip_ratio + cliprange_low = config.clip_ratio_low if config.clip_ratio_low is not None else cliprange + cliprange_high = config.clip_ratio_high if config.clip_ratio_high is not None else cliprange + clip_cov_ub = config.policy_loss.clip_cov_ub if config.policy_loss.clip_cov_ub is not None else 5.0 + clip_cov_lb = config.policy_loss.clip_cov_lb if config.policy_loss.clip_cov_lb is not None else 1.0 + + assert clip_cov_ratio > 0, "clip_ratio should be larger than 0." + + negative_approx_kl = log_prob - old_log_prob + ratio = torch.exp(negative_approx_kl) + ppo_kl = verl_F.masked_mean(-negative_approx_kl, response_mask) + + pg_losses1 = -advantages * ratio + + if cliprange_low is None: + cliprange_low = cliprange + if cliprange_high is None: + cliprange_high = cliprange + + corr = torch.ones_like(advantages) + pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high) + clip_by_origin = (pg_losses2 > pg_losses1) & (response_mask > 0) + + cov_all = (advantages - verl_F.masked_mean(advantages, response_mask)) * ( + log_prob - verl_F.masked_mean(log_prob.detach(), response_mask) + ) + cov_all[response_mask == 0] = -torch.inf + cov_all[clip_by_origin] = -torch.inf + + clip_num = max(int(clip_cov_ratio * response_mask.sum().item()), 1) + top_k_idx = (cov_all < clip_cov_ub) & (cov_all > clip_cov_lb) & (response_mask > 0) + top_k_idx = torch.nonzero(top_k_idx) + + if len(top_k_idx) > 0: + perm = torch.randperm(len(top_k_idx)) + top_k_idx = top_k_idx[perm[: min(clip_num, len(top_k_idx))]] + else: + top_k_idx = torch.empty((0, 2), device=cov_all.device, dtype=torch.long) + + corr[top_k_idx[:, 0], top_k_idx[:, 1]] = 0 + + pg_clipfrac = verl_F.masked_mean((corr == 0).float(), response_mask) + + pg_losses = torch.maximum(pg_losses1, pg_losses2) * corr + pg_loss = agg_loss(loss_mat=pg_losses, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + + return pg_loss, pg_clipfrac, ppo_kl, torch.tensor(0.0) + + +@register_policy_loss("kl_cov") +def compute_policy_loss_kl_cov( + old_log_prob: torch.Tensor, + log_prob: torch.Tensor, + advantages: torch.Tensor, + response_mask: torch.Tensor, + loss_agg_mode: str = "token-mean", + config: Optional[DictConfig | AlgoConfig] = None, + rollout_log_probs: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Compute the clipped policy objective and related metrics for Clip-Cov. + + Adapted from + https://github.com/PRIME-RL/Entropy-Mechanism-of-RL/blob/main/verl/trainer/ppo/core_algos.py + + Args: + old_log_prob (torch.Tensor): + Log-probabilities of actions under the old policy, shape (batch_size, response_length). + log_prob (torch.Tensor): + Log-probabilities of actions under the current policy, shape (batch_size, response_length). + advantages (torch.Tensor): + Advantage estimates for each action, shape (batch_size, response_length). + response_mask (torch.Tensor): + Mask indicating which tokens to include in the loss, shape (batch_size, response_length). + loss_agg_mode (str, optional): + Aggregation mode for `agg_loss`. Defaults to "token-mean". + kl_cov_ratio (float, optional): + Ratio for selecting the top-k covariance values. Defaults to 0.0002. + ppo_kl_coef (float, optional): + Coefficient for the KL penalty term in the loss. Defaults to 1. + """ + assert config is not None + assert not isinstance(config, AlgoConfig), "passing AlgoConfig not supported yet" + assert config.policy_loss is not None + + kl_cov_ratio = config.policy_loss.kl_cov_ratio if config.policy_loss.kl_cov_ratio is not None else 0.0002 + ppo_kl_coef = config.policy_loss.ppo_kl_coef if config.policy_loss.ppo_kl_coef is not None else 1.0 + + assert kl_cov_ratio > 0, "kl_cov_ratio should be larger than 0." + + negative_approx_kl = log_prob - old_log_prob + abs_kl = negative_approx_kl.abs() + ratio = torch.exp(negative_approx_kl) + ppo_kl_abs = verl_F.masked_mean(negative_approx_kl.abs(), response_mask) + pg_losses1 = -advantages * ratio + pg_losses_kl = -advantages * ratio + ppo_kl_coef * abs_kl + pg_losses = pg_losses1 + + all_valid = response_mask > 0 + all_valid_idx = torch.nonzero(all_valid.reshape(-1), as_tuple=True)[0] + all_valid_adv = advantages[all_valid].detach().reshape(-1).cpu() + all_valid_logp = log_prob[all_valid].detach().reshape(-1).cpu() + + k = min(kl_cov_ratio, len(all_valid_adv)) + + if k != 0: + cov_lst_all = (all_valid_adv - all_valid_adv.mean()) * (all_valid_logp - all_valid_logp.mean()) + k_percent_nums = max(1, int(len(cov_lst_all) * kl_cov_ratio)) + large_cov_idxs = torch.topk(cov_lst_all, k_percent_nums, largest=True).indices + + if len(large_cov_idxs) != 0: + large_cov_idxs = all_valid_idx[large_cov_idxs] + pg_losses[large_cov_idxs // advantages.shape[1], large_cov_idxs % advantages.shape[1]] = pg_losses_kl[ + large_cov_idxs // advantages.shape[1], large_cov_idxs % advantages.shape[1] + ] + + pg_loss = agg_loss(loss_mat=pg_losses, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + + return pg_loss, torch.tensor(0.0), ppo_kl_abs, torch.tensor(0.0) + + +@register_policy_loss("geo_mean") +def compute_policy_loss_geo_mean( + old_log_prob: torch.Tensor, + log_prob: torch.Tensor, + advantages: torch.Tensor, + response_mask: torch.Tensor, + loss_agg_mode: str = "token-mean", + config: Optional[DictConfig | AlgoConfig] = None, + rollout_log_probs: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Compute the clipped policy objective and related metrics for GMPO. + + Adapted from paper https://arxiv.org/abs/2507.20673 + https://github.com/callsys/GMPO/blob/main/train_zero_math_gmpo.py + + Args: + old_log_prob (torch.Tensor): + Log-probabilities of actions under the old policy, shape (batch_size, response_length). + log_prob (torch.Tensor): + Log-probabilities of actions under the current policy, shape (batch_size, response_length). + advantages (torch.Tensor): + Advantage estimates for each action, shape (batch_size, response_length). + response_mask (torch.Tensor): + Mask indicating which tokens to include in the loss, shape (batch_size, response_length). + loss_agg_mode (str, optional): + not used + """ + + assert config is not None + assert not isinstance(config, AlgoConfig) + clip_ratio = config.clip_ratio # Clipping parameter. See https://arxiv.org/abs/1707.06347. + clip_ratio_low = config.clip_ratio_low if config.clip_ratio_low is not None else clip_ratio + clip_ratio_high = config.clip_ratio_high if config.clip_ratio_high is not None else clip_ratio + + cliprange = clip_ratio + cliprange_low = clip_ratio_low + cliprange_high = clip_ratio_high + if cliprange_low is None: + cliprange_low = cliprange + if cliprange_high is None: + cliprange_high = cliprange + + negative_approx_kl = log_prob - old_log_prob + # Clamp negative_approx_kl for stability (uncomment it if you like) + # negative_approx_kl = torch.clamp(negative_approx_kl, min=-20.0, max=20.0) + ppo_kl = verl_F.masked_mean(-negative_approx_kl, response_mask) + + # Clipping at token-level & Clipping wider + sgn_advantage = torch.sign(advantages) + negative_approx_kl_clamp = torch.clamp(negative_approx_kl, -cliprange_low, cliprange_high) + negative_approx_kl_min = torch.min(sgn_advantage * negative_approx_kl, sgn_advantage * negative_approx_kl_clamp) + negative_approx_kl_min = sgn_advantage * negative_approx_kl_min + + # Geometric-Mean Policy Optimization + response_mask_sum = response_mask.sum(dim=-1) + ratio = torch.exp((negative_approx_kl_min * response_mask).sum(dim=-1) / (response_mask_sum + 1e-8)) + # we only support sequence level advantage for now, + # otherwise, below would be not consistent with the paper + advantage = (advantages * response_mask).sum(dim=-1) / (response_mask_sum + 1e-8) + pg_losses = -advantage * ratio + pg_loss = torch.mean(pg_losses) + + # higher: ratio is too large that need clamp to clip_high (when adv > 0) + clipped = torch.ne(negative_approx_kl, negative_approx_kl_clamp) + pg_clipfrac = verl_F.masked_mean((clipped * (advantages > 0)).float(), response_mask) + pg_clipfrac_lower = verl_F.masked_mean((clipped * (advantages < 0)).float(), response_mask) + + return pg_loss, pg_clipfrac, ppo_kl, pg_clipfrac_lower + + +def compute_entropy_loss(logits, response_mask, loss_agg_mode: str = "token-mean"): + """Compute categorical entropy loss (For backward compatibility) + + Args: + logits (torch.Tensor): shape is (bs, response_length, vocab_size) + response_mask (torch.Tensor): shape is (bs, response_length) + + Returns: + entropy: a scalar torch.Tensor + + """ + # compute entropy + token_entropy = verl_F.entropy_from_logits(logits) # (bs, response_len) + entropy_loss = agg_loss(loss_mat=token_entropy, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + return entropy_loss + + +def compute_value_loss( + vpreds: torch.Tensor, + returns: torch.Tensor, + values: torch.Tensor, + response_mask: torch.Tensor, + cliprange_value: float, + loss_agg_mode: str = "token-mean", +): + """ + Compute the clipped value-function loss for PPO. + + Copied from https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py#L1151 + + Args: + vpreds (torch.FloatTensor): + Predicted values from the value head, shape (batch_size, response_length). + values (torch.FloatTensor): + Old (baseline) values from the value head, shape (batch_size, response_length). + returns (torch.FloatTensor): + Ground-truth returns, shape (batch_size, response_length). + response_mask (torch.Tensor): + Mask indicating which tokens to include in the value loss calculation. + cliprange_value (float): + Clip range for value prediction updates. + loss_agg_mode (str, optional): + Aggregation mode for `agg_loss`. Defaults to "token-mean". + + Returns: + vf_loss (torch.FloatTensor): + A scalar tensor containing the aggregated value-function loss. + vf_clipfrac (float): + Fraction of elements where the clipped loss was used. + """ + vpredclipped = verl_F.clip_by_value(vpreds, values - cliprange_value, values + cliprange_value) + vf_losses1 = (vpreds - returns) ** 2 + vf_losses2 = (vpredclipped - returns) ** 2 + clipped_vf_losses = torch.max(vf_losses1, vf_losses2) + vf_loss = 0.5 * agg_loss(loss_mat=clipped_vf_losses, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + vf_clipfrac = verl_F.masked_mean(torch.gt(vf_losses2, vf_losses1).float(), response_mask) + return vf_loss, vf_clipfrac + + +def kl_penalty(logprob: torch.FloatTensor, ref_logprob: torch.FloatTensor, kl_penalty) -> torch.FloatTensor: + """Compute KL divergence given logprob and ref_logprob. Optionally using straight through to bind k2 on other + kl penalty compute method for unbiased KL gradient estimation. + See more description in http://joschu.net/blog/kl-approx.html + + Args: + logprob: + ref_logprob: + + Returns: + kl_estimate + """ + forward_score = kl_penalty_forward(logprob, ref_logprob, kl_penalty) + if not kl_penalty.endswith("+") or kl_penalty in ("mse", "k2"): + return forward_score + + """ + The expectation of k1 and k3 estimator is the expectaed value of KL, but the expected gradient of k1 and k3 + estimator is not the expectaed gradient of KL. On the other hand k2 estimator gives right gradient estimator, + so we use a straight through trick here if the kl_penalty method ends with '+', .e.g., k3+. + """ + backward_score = 0.5 * (logprob - ref_logprob).square() + + return backward_score - backward_score.detach() + forward_score.detach() + + +def kl_penalty_forward(logprob: torch.FloatTensor, ref_logprob: torch.FloatTensor, kl_penalty) -> torch.FloatTensor: + """Compute KL divergence given logprob and ref_logprob. + Copied from https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py#L1104 + See more description in http://joschu.net/blog/kl-approx.html + + Args: + logprob: + ref_logprob: + + Returns: + kl_estimate + """ + if kl_penalty in ("kl", "k1"): + return logprob - ref_logprob + + if kl_penalty == "abs": + return (logprob - ref_logprob).abs() + + if kl_penalty in ("mse", "k2"): + return 0.5 * (logprob - ref_logprob).square() + + # J. Schulman. Approximating kl divergence, 2020. + # # URL http://joschu.net/blog/kl-approx.html. + if kl_penalty in ("low_var_kl", "k3"): + kl = ref_logprob - logprob + # For numerical stability + kl = torch.clamp(kl, min=-20, max=20) + ratio = torch.exp(kl) + kld = (ratio - kl - 1).contiguous() + return torch.clamp(kld, min=-10, max=10) + + if kl_penalty == "full": + # so, here logprob and ref_logprob should contain the logits for every token in vocabulary + raise NotImplementedError + + raise NotImplementedError + + +def compute_pf_ppo_reweight_data( + data, + reweight_method: str = "pow", + weight_pow: float = 2.0, +): + """Reweight the data based on the token_level_scores. + + Args: + data: DataProto object, containing batch, non_tensor_batch and meta_info + reweight_method: str, choices: "pow", "max_min", "max_random" + weight_pow: float, the power of the weight + + Returns: + + """ + + @torch.no_grad() + def compute_weights(scores: torch.Tensor, reweight_method: str, weight_pow: float) -> torch.Tensor: + """Compute importance weights for resampling based on scores. + + Args: + scores (torch.Tensor): Tensor of scores to compute weights from. + reweight_method (str): Method for computing weights ('pow', 'max_min', 'max_random'). + weight_pow (float): Power exponent for 'pow' method. + + Returns: + torch.Tensor: Computed importance weights. + + Raises: + ValueError: If reweight_method is not supported. + """ + if reweight_method == "pow": + weights = torch.pow(torch.abs(scores), weight_pow) + elif reweight_method == "max_min": + max_score = torch.max(scores) + min_score = torch.min(scores) + weights = torch.where((scores == max_score) | (scores == min_score), 1.0, 0.0) + elif reweight_method == "max_random": + max_score = torch.max(scores) + weights = torch.where(scores == max_score, 0.4, 0.1) + else: + raise ValueError(f"Unsupported reweight_method: {reweight_method}") + return weights + + scores = data.batch["token_level_scores"].sum(dim=-1) + weights = compute_weights(scores, reweight_method, weight_pow) + weights = torch.clamp(weights + 1e-8, min=1e-8) + + batch_size = scores.shape[0] + sample_indices = torch.multinomial(weights, batch_size, replacement=True) + + resampled_batch = {key: tensor[sample_indices] for key, tensor in data.batch.items()} + + sample_indices_np = sample_indices.numpy() + resampled_non_tensor_batch = {} + for key, array in data.non_tensor_batch.items(): + if isinstance(array, np.ndarray): + resampled_non_tensor_batch[key] = array[sample_indices_np] + else: + resampled_non_tensor_batch[key] = [array[i] for i in sample_indices_np] + + resampled_meta_info = {} + for key, value in data.meta_info.items(): + if isinstance(value, list) and len(value) == batch_size: + resampled_meta_info[key] = [value[i] for i in sample_indices_np] + else: + resampled_meta_info[key] = value + + from copy import deepcopy + + resampled_data = deepcopy(data) + resampled_data.batch = type(data.batch)(resampled_batch) + resampled_data.batch.batch_size = data.batch.batch_size + resampled_data.non_tensor_batch = resampled_non_tensor_batch + resampled_data.meta_info = resampled_meta_info + + return resampled_data diff --git a/e2c/verl/verl/trainer/ppo/metric_utils.py b/e2c/verl/verl/trainer/ppo/metric_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..876d6907de417b314d993e31b2931395f1cf7e9c --- /dev/null +++ b/e2c/verl/verl/trainer/ppo/metric_utils.py @@ -0,0 +1,490 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Metrics related to the PPO trainer. +""" + +from collections import defaultdict +from functools import partial +from typing import Any, Callable + +import numpy as np +import torch + +from verl import DataProto +from verl.utils.import_utils import deprecated + + +@deprecated("verl.utils.metric.reduce_metrics") +def reduce_metrics(metrics: dict[str, list[Any]]) -> dict[str, Any]: + """ + Reduces a dictionary of metric lists by computing the mean of each list. + + Args: + metrics: A dictionary mapping metric names to lists of metric values. + + Returns: + A dictionary with the same keys but with each list replaced by its mean value. + + Example: + >>> metrics = {"loss": [1.0, 2.0, 3.0], "accuracy": [0.8, 0.9, 0.7]} + >>> reduce_metrics(metrics) + {"loss": 2.0, "accuracy": 0.8} + """ + from verl.utils.metric import reduce_metrics + + return reduce_metrics(metrics) + + +def _compute_response_info(batch: DataProto) -> dict[str, Any]: + """ + Computes information about prompts and responses from a batch. + + This is an internal helper function that extracts masks and lengths for prompts and responses. + + Args: + batch: A DataProto object containing batch data with responses and attention masks. + + Returns: + A dictionary containing: + - response_mask: Attention mask for the response tokens + - prompt_length: Tensor of prompt lengths for each item in the batch + - response_length: Tensor of response lengths for each item in the batch + """ + response_length = batch.batch["responses"].shape[-1] + + prompt_mask = batch.batch["attention_mask"][:, :-response_length] + response_mask = batch.batch["attention_mask"][:, -response_length:] + + prompt_length = prompt_mask.sum(-1).float() + response_length = response_mask.sum(-1).float() # (batch_size,) + + return dict( + response_mask=response_mask, + prompt_length=prompt_length, + response_length=response_length, + ) + + +def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, Any]: + """ + Computes various metrics from a batch of data for PPO training. + + This function calculates metrics related to scores, rewards, advantages, returns, values, + and sequence lengths from a batch of data. It provides statistical information (mean, max, min) + for each metric category. + + Args: + batch: A DataProto object containing batch data with token-level scores, rewards, advantages, etc. + use_critic: Whether to include critic-specific metrics. Defaults to True. + + Returns: + A dictionary of metrics including: + - critic/score/mean, max, min: Statistics about sequence scores + - critic/rewards/mean, max, min: Statistics about sequence rewards + - critic/advantages/mean, max, min: Statistics about advantages + - critic/returns/mean, max, min: Statistics about returns + - critic/values/mean, max, min: Statistics about critic values (if use_critic=True) + - critic/vf_explained_var: Explained variance of the value function (if use_critic=True) + - response_length/mean, max, min, clip_ratio: Statistics about response lengths + - prompt_length/mean, max, min, clip_ratio: Statistics about prompt lengths + - num_turns/mean, max, min: Statistics about the number of multi-turn conversations + """ + sequence_score = batch.batch["token_level_scores"].sum(-1) + sequence_reward = batch.batch["token_level_rewards"].sum(-1) + + advantages = batch.batch["advantages"] + returns = batch.batch["returns"] + + max_response_length = batch.batch["responses"].shape[-1] + + prompt_mask = batch.batch["attention_mask"][:, :-max_response_length].bool() + response_mask = batch.batch["response_mask"].bool() + + max_prompt_length = prompt_mask.size(-1) + + response_info = _compute_response_info(batch) + prompt_length = response_info["prompt_length"] + response_length = response_info["response_length"] + + aborted_mask = (response_length == 0).bool() + non_aborted_mask = ~aborted_mask + + non_aborted_sequence_score = sequence_score[non_aborted_mask] + non_aborted_sequence_reward = sequence_reward[non_aborted_mask] + + score_mean = torch.mean(non_aborted_sequence_score).detach().item() + score_max = torch.max(non_aborted_sequence_score).detach().item() + score_min = torch.min(non_aborted_sequence_score).detach().item() + + reward_mean = torch.mean(non_aborted_sequence_reward).detach().item() + reward_max = torch.max(non_aborted_sequence_reward).detach().item() + reward_min = torch.min(non_aborted_sequence_reward).detach().item() + + valid_adv = torch.masked_select(advantages, response_mask) + valid_returns = torch.masked_select(returns, response_mask) + + if use_critic: + values = batch.batch["values"] + valid_values = torch.masked_select(values, response_mask) + return_diff_var = torch.var(valid_returns - valid_values) + return_var = torch.var(valid_returns) + + # Aborted samples and non-aborted response length statistics + # response_length_non_aborted/*: statistics computed on non-aborted samples only + aborted_ratio = torch.mean(aborted_mask.float()).detach().item() + + non_aborted_response_length = response_length[non_aborted_mask] + if non_aborted_response_length.numel() > 0: + non_aborted_response_length_mean = torch.mean(non_aborted_response_length).detach().item() + non_aborted_response_length_max = torch.max(non_aborted_response_length).detach().item() + non_aborted_response_length_min = torch.min(non_aborted_response_length).detach().item() + non_aborted_response_length_clip_ratio = ( + torch.mean(torch.eq(non_aborted_response_length, max_response_length).float()).detach().item() + ) + else: + raise ValueError("All samples are aborted, this should not happen.") + + metrics = { + # score + "critic/score/mean": score_mean, + "critic/score/max": score_max, + "critic/score/min": score_min, + # reward + "critic/rewards/mean": reward_mean, + "critic/rewards/max": reward_max, + "critic/rewards/min": reward_min, + # adv + "critic/advantages/mean": torch.mean(valid_adv).detach().item(), + "critic/advantages/max": torch.max(valid_adv).detach().item(), + "critic/advantages/min": torch.min(valid_adv).detach().item(), + # returns + "critic/returns/mean": torch.mean(valid_returns).detach().item(), + "critic/returns/max": torch.max(valid_returns).detach().item(), + "critic/returns/min": torch.min(valid_returns).detach().item(), + **( + { + # values + "critic/values/mean": torch.mean(valid_values).detach().item(), + "critic/values/max": torch.max(valid_values).detach().item(), + "critic/values/min": torch.min(valid_values).detach().item(), + # vf explained var + "critic/vf_explained_var": (1.0 - return_diff_var / (return_var + 1e-5)).detach().item(), + } + if use_critic + else {} + ), + # response length + "response_length/mean": torch.mean(response_length).detach().item(), + "response_length/max": torch.max(response_length).detach().item(), + "response_length/min": torch.min(response_length).detach().item(), + "response_length/clip_ratio": torch.mean(torch.eq(response_length, max_response_length).float()) + .detach() + .item(), + # response length (non-aborted only) + # These statistics exclude aborted samples to avoid skew from zeros + "response_length_non_aborted/mean": non_aborted_response_length_mean, + "response_length_non_aborted/max": non_aborted_response_length_max, + "response_length_non_aborted/min": non_aborted_response_length_min, + "response_length_non_aborted/clip_ratio": non_aborted_response_length_clip_ratio, + # aborted ratio + # Fraction of samples whose response length is zero + "response/aborted_ratio": aborted_ratio, + # prompt length + "prompt_length/mean": torch.mean(prompt_length).detach().item(), + "prompt_length/max": torch.max(prompt_length).detach().item(), + "prompt_length/min": torch.min(prompt_length).detach().item(), + "prompt_length/clip_ratio": torch.mean(torch.eq(prompt_length, max_prompt_length).float()).detach().item(), + } + + # multi-turn conversation + if "__num_turns__" in batch.non_tensor_batch: + num_turns = batch.non_tensor_batch["__num_turns__"] + metrics["num_turns/min"] = num_turns.min() + metrics["num_turns/max"] = num_turns.max() + metrics["num_turns/mean"] = num_turns.mean() + + if "tool_call_counts" in batch.non_tensor_batch: + tool_call_counts = batch.non_tensor_batch["tool_call_counts"] + metrics["tool_call_counts/min"] = tool_call_counts.min() + metrics["tool_call_counts/max"] = tool_call_counts.max() + metrics["tool_call_counts/mean"] = tool_call_counts.mean() + + return metrics + + +def compute_timing_metrics(batch: DataProto, timing_raw: dict[str, float]) -> dict[str, Any]: + """ + Computes timing metrics for different processing stages in PPO training. + + This function calculates both raw timing metrics (in seconds) and per-token timing metrics + (in milliseconds) for various processing stages like generation, reference computation, + value computation, advantage computation, and model updates. + + Args: + batch: A DataProto object containing batch data with responses and attention masks. + timing_raw: A dictionary mapping stage names to their execution times in seconds. + + Returns: + A dictionary containing: + - timing_s/{name}: Raw timing in seconds for each stage + - timing_per_token_ms/{name}: Per-token timing in milliseconds for each stage + + Note: + Different stages use different token counts for normalization: + - "gen" uses only response tokens + - Other stages ("ref", "values", "adv", "update_critic", "update_actor") use all tokens + (prompt + response) + """ + response_info = _compute_response_info(batch) + num_prompt_tokens = torch.sum(response_info["prompt_length"]).item() + num_response_tokens = torch.sum(response_info["response_length"]).item() + num_overall_tokens = num_prompt_tokens + num_response_tokens + + num_tokens_of_section = { + "gen": num_response_tokens, + **{name: num_overall_tokens for name in ["ref", "values", "adv", "update_critic", "update_actor"]}, + } + + return { + **{f"timing_s/{name}": value for name, value in timing_raw.items()}, + **{ + f"timing_per_token_ms/{name}": timing_raw[name] * 1000 / num_tokens_of_section[name] + for name in set(num_tokens_of_section.keys()) & set(timing_raw.keys()) + }, + } + + +def compute_throughout_metrics(batch: DataProto, timing_raw: dict[str, float], n_gpus: int) -> dict[str, Any]: + """ + Computes throughput metrics for PPO training. + + This function calculates performance metrics related to token processing speed, + including the total number of tokens processed, time per step, and throughput + (tokens per second per GPU). + + Args: + batch: A DataProto object containing batch data with meta information about token counts. + timing_raw: A dictionary mapping stage names to their execution times in seconds. + Must contain a "step" key with the total step time. + n_gpus: Number of GPUs used for training. + + Returns: + A dictionary containing: + - perf/total_num_tokens: Total number of tokens processed in the batch + - perf/time_per_step: Time taken for the step in seconds + - perf/throughput: Tokens processed per second per GPU + + Note: + The throughput is calculated as total_tokens / (time * n_gpus) to normalize + across different GPU counts. + """ + total_num_tokens = sum(batch.meta_info["global_token_num"]) + time = timing_raw["step"] + # estimated_flops, promised_flops = flops_function.estimate_flops(num_tokens, time) + # f'Actual TFLOPs/s/GPU​': estimated_flops/(n_gpus), + # f'Theoretical TFLOPs/s/GPU​': promised_flops, + return { + "perf/total_num_tokens": total_num_tokens, + "perf/time_per_step": time, + "perf/throughput": total_num_tokens / (time * n_gpus), + } + + +def bootstrap_metric( + data: list[Any], + subset_size: int, + reduce_fns: list[Callable[[np.ndarray], float]], + n_bootstrap: int = 1000, + seed: int = 42, +) -> list[tuple[float, float]]: + """ + Performs bootstrap resampling to estimate statistics of metrics. + + This function uses bootstrap resampling to estimate the mean and standard deviation + of metrics computed by the provided reduction functions on random subsets of the data. + + Args: + data: List of data points to bootstrap from. + subset_size: Size of each bootstrap sample. + reduce_fns: List of functions that compute a metric from a subset of data. + n_bootstrap: Number of bootstrap iterations. Defaults to 1000. + seed: Random seed for reproducibility. Defaults to 42. + + Returns: + A list of tuples, where each tuple contains (mean, std) for a metric + corresponding to each reduction function in reduce_fns. + + Example: + >>> data = [1, 2, 3, 4, 5] + >>> reduce_fns = [np.mean, np.max] + >>> bootstrap_metric(data, 3, reduce_fns) + [(3.0, 0.5), (4.5, 0.3)] # Example values + """ + np.random.seed(seed) + + bootstrap_metric_lsts = [[] for _ in range(len(reduce_fns))] + for _ in range(n_bootstrap): + bootstrap_idxs = np.random.choice(len(data), size=subset_size, replace=True) + bootstrap_data = [data[i] for i in bootstrap_idxs] + for i, reduce_fn in enumerate(reduce_fns): + bootstrap_metric_lsts[i].append(reduce_fn(bootstrap_data)) + return [(np.mean(lst), np.std(lst)) for lst in bootstrap_metric_lsts] + + +def calc_maj_val(data: list[dict[str, Any]], vote_key: str, val_key: str) -> float: + """ + Calculate a value based on majority voting. + + This function identifies the most common value for a specified vote key + in the data, then returns the corresponding value for that majority vote. + + Args: + data: List of dictionaries, where each dictionary contains both vote_key and val_key. + vote_key: The key in each dictionary used for voting/counting. + val_key: The key in each dictionary whose value will be returned for the majority vote. + + Returns: + The value associated with the most common vote. + + Example: + >>> data = [ + ... {"pred": "A", "val": 0.9}, + ... {"pred": "B", "val": 0.8}, + ... {"pred": "A", "val": 0.7} + ... ] + >>> calc_maj_val(data, vote_key="pred", val_key="val") + 0.9 # Returns the first "val" for the majority vote "A" + """ + vote2vals = defaultdict(list) + for d in data: + vote2vals[d[vote_key]].append(d[val_key]) + + vote2cnt = {k: len(v) for k, v in vote2vals.items()} + maj_vote = max(vote2cnt, key=vote2cnt.get) + + maj_val = vote2vals[maj_vote][0] + + return maj_val + + +def process_validation_metrics( + data_sources: list[str], sample_uids: list[str], infos_dict: dict[str, list[Any]], seed: int = 42 +) -> dict[str, dict[str, dict[str, float]]]: + """ + Process validation metrics into a structured format with statistical analysis. + + This function organizes validation metrics by data source and prompt, then computes + various statistical measures including means, standard deviations, best/worst values, + and majority voting results. It also performs bootstrap sampling to estimate statistics + for different sample sizes. + + Args: + data_sources: List of data source identifiers for each sample. + sample_uids: List of sample uids corresponding to each sample. + infos_dict: Dictionary mapping variable names to lists of values for each sample. + seed: Random seed for bootstrap sampling. Defaults to 42. + + Returns: + A nested dictionary with the structure: + { + data_source: { + variable_name: { + metric_name: value + } + } + } + + Where metric_name includes: + - "mean@N": Mean value across N samples + - "std@N": Standard deviation across N samples + - "best@N/mean": Mean of the best values in bootstrap samples of size N + - "best@N/std": Standard deviation of the best values in bootstrap samples + - "worst@N/mean": Mean of the worst values in bootstrap samples + - "worst@N/std": Standard deviation of the worst values in bootstrap samples + - "maj@N/mean": Mean of majority voting results in bootstrap samples (if "pred" exists) + - "maj@N/std": Standard deviation of majority voting results (if "pred" exists) + + Example: + >>> data_sources = ["source1", "source1", "source2"] + >>> sample_uids = ["uid1", "uid1", "uid2"] + >>> infos_dict = {"score": [0.8, 0.9, 0.7], "pred": ["A", "A", "B"]} + >>> result = process_validation_metrics(data_sources, sample_uids, infos_dict) + >>> # result will contain statistics for each data source and variable + """ + # Group metrics by data source, prompt and variable + data_src2uid2var2vals = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + for sample_idx, data_source in enumerate(data_sources): + uid = sample_uids[sample_idx] + var2vals = data_src2uid2var2vals[data_source][uid] + for var_name, var_vals in infos_dict.items(): + var2vals[var_name].append(var_vals[sample_idx]) + + # Calculate metrics for each group + data_src2uid2var2metric = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) + for data_source, uid2var2vals in data_src2uid2var2vals.items(): + for uid, var2vals in uid2var2vals.items(): + for var_name, var_vals in var2vals.items(): + if isinstance(var_vals[0], str): + continue + + metric = {} + n_resps = len(var_vals) + metric[f"mean@{n_resps}"] = np.mean(var_vals) + + if n_resps > 1: + metric[f"std@{n_resps}"] = np.std(var_vals) + + ns = [] + n = 2 + while n < n_resps: + ns.append(n) + n *= 2 + ns.append(n_resps) + + for n in ns: + [(bon_mean, bon_std), (won_mean, won_std)] = bootstrap_metric( + data=var_vals, subset_size=n, reduce_fns=[np.max, np.min], seed=seed + ) + metric[f"best@{n}/mean"], metric[f"best@{n}/std"] = bon_mean, bon_std + metric[f"worst@{n}/mean"], metric[f"worst@{n}/std"] = won_mean, won_std + if var2vals.get("pred", None) is not None: + vote_data = [ + {"val": val, "pred": pred} for val, pred in zip(var_vals, var2vals["pred"], strict=True) + ] + [(maj_n_mean, maj_n_std)] = bootstrap_metric( + data=vote_data, + subset_size=n, + reduce_fns=[partial(calc_maj_val, vote_key="pred", val_key="val")], + seed=seed, + ) + metric[f"maj@{n}/mean"], metric[f"maj@{n}/std"] = maj_n_mean, maj_n_std + + data_src2uid2var2metric[data_source][uid][var_name] = metric + + # Aggregate metrics across uids + data_src2var2metric2uid_vals = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + for data_source, uid2var2metric in data_src2uid2var2metric.items(): + for uid, var2metric in uid2var2metric.items(): + for var_name, metric in var2metric.items(): + for metric_name, metric_val in metric.items(): + data_src2var2metric2uid_vals[data_source][var_name][metric_name].append(metric_val) + + data_src2var2metric2val = defaultdict(lambda: defaultdict(lambda: defaultdict(float))) + for data_source, var2metric2uid_vals in data_src2var2metric2uid_vals.items(): + for var_name, metric2uid_vals in var2metric2uid_vals.items(): + for metric_name, uid_vals in metric2uid_vals.items(): + data_src2var2metric2val[data_source][var_name][metric_name] = np.mean(uid_vals) + + return data_src2var2metric2val diff --git a/e2c/verl/verl/trainer/ppo/ray_trainer.py b/e2c/verl/verl/trainer/ppo/ray_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..2752f08feb245da5169d2ffdec3340954aaf6471 --- /dev/null +++ b/e2c/verl/verl/trainer/ppo/ray_trainer.py @@ -0,0 +1,1236 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +PPO Trainer with Ray-based single controller. +This trainer supports model-agonistic model initialization with huggingface +""" + +import json +import os +import uuid +from collections import defaultdict +from copy import deepcopy +from dataclasses import dataclass, field +from pprint import pprint +from typing import Optional + +import numpy as np +import ray +import torch +from omegaconf import OmegaConf, open_dict +from torch.utils.data import Dataset, Sampler +from torchdata.stateful_dataloader import StatefulDataLoader +from tqdm import tqdm + +from verl import DataProto +from verl.experimental.dataset.sampler import AbstractCurriculumSampler +from verl.protocol import pad_dataproto_to_divisor, unpad_dataproto +from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup +from verl.single_controller.ray.base import create_colocated_worker_cls +from verl.trainer.config import AlgoConfig +from verl.trainer.ppo import core_algos +from verl.trainer.ppo.core_algos import AdvantageEstimator, agg_loss +from verl.trainer.ppo.metric_utils import ( + compute_data_metrics, + compute_throughout_metrics, + compute_timing_metrics, + process_validation_metrics, +) +from verl.trainer.ppo.reward import compute_reward, compute_reward_async +from verl.trainer.ppo.utils import Role, WorkerType, need_critic, need_reference_policy, need_reward_model +from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path, should_save_ckpt_esi +from verl.utils.config import omega_conf_to_dataclass +from verl.utils.debug import marked_timer +from verl.utils.metric import reduce_metrics +from verl.utils.rollout_skip import RolloutSkip +from verl.utils.seqlen_balancing import get_seqlen_balanced_partitions, log_seqlen_unbalance +from verl.utils.torch_functional import masked_mean +from verl.utils.tracking import ValidationGenerationsLogger + + +@dataclass +class ResourcePoolManager: + """ + Define a resource pool specification. Resource pool will be initialized first. + """ + + resource_pool_spec: dict[str, list[int]] + mapping: dict[Role, str] + resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) + + def create_resource_pool(self): + """Create Ray resource pools for distributed training. + + Initializes resource pools based on the resource pool specification, + with each pool managing GPU resources across multiple nodes. + For FSDP backend, uses max_colocate_count=1 to merge WorkerGroups. + For Megatron backend, uses max_colocate_count>1 for different models. + """ + for resource_pool_name, process_on_nodes in self.resource_pool_spec.items(): + # max_colocate_count means the number of WorkerGroups (i.e. processes) in each RayResourcePool + # For FSDP backend, we recommend using max_colocate_count=1 that merge all WorkerGroups into one. + # For Megatron backend, we recommend using max_colocate_count>1 + # that can utilize different WorkerGroup for differnt models + resource_pool = RayResourcePool( + process_on_nodes=process_on_nodes, use_gpu=True, max_colocate_count=1, name_prefix=resource_pool_name + ) + self.resource_pool_dict[resource_pool_name] = resource_pool + + self._check_resource_available() + + def get_resource_pool(self, role: Role) -> RayResourcePool: + """Get the resource pool of the worker_cls""" + return self.resource_pool_dict[self.mapping[role]] + + def get_n_gpus(self) -> int: + """Get the number of gpus in this cluster.""" + return sum([n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes]) + + def _check_resource_available(self): + """Check if the resource pool can be satisfied in this ray cluster.""" + node_available_resources = ray._private.state.available_resources_per_node() + node_available_gpus = { + node: node_info.get("GPU", 0) if "GPU" in node_info else node_info.get("NPU", 0) + for node, node_info in node_available_resources.items() + } + + # check total required gpus can be satisfied + total_available_gpus = sum(node_available_gpus.values()) + total_required_gpus = sum( + [n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes] + ) + if total_available_gpus < total_required_gpus: + raise ValueError( + f"Total available GPUs {total_available_gpus} is less than total desired GPUs {total_required_gpus}" + ) + + +def apply_kl_penalty(data: DataProto, kl_ctrl: core_algos.AdaptiveKLController, kl_penalty="kl"): + """Apply KL penalty to the token-level rewards. + + This function computes the KL divergence between the reference policy and current policy, + then applies a penalty to the token-level rewards based on this divergence. + + Args: + data (DataProto): The data containing batched model outputs and inputs. + kl_ctrl (core_algos.AdaptiveKLController): Controller for adaptive KL penalty. + kl_penalty (str, optional): Type of KL penalty to apply. Defaults to "kl". + + Returns: + tuple: A tuple containing: + - The updated data with token-level rewards adjusted by KL penalty + - A dictionary of metrics related to the KL penalty + """ + response_mask = data.batch["response_mask"] + token_level_scores = data.batch["token_level_scores"] + batch_size = data.batch.batch_size[0] + + # compute kl between ref_policy and current policy + # When apply_kl_penalty, algorithm.use_kl_in_reward=True, so the reference model has been enabled. + kld = core_algos.kl_penalty( + data.batch["old_log_probs"], data.batch["ref_log_prob"], kl_penalty=kl_penalty + ) # (batch_size, response_length) + kld = kld * response_mask + beta = kl_ctrl.value + + token_level_rewards = token_level_scores - beta * kld + + current_kl = masked_mean(kld, mask=response_mask, axis=-1) # average over sequence + current_kl = torch.mean(current_kl, dim=0).item() + + # according to https://github.com/huggingface/trl/blob/951ca1841f29114b969b57b26c7d3e80a39f75a0/trl/trainer/ppo_trainer.py#L837 + kl_ctrl.update(current_kl=current_kl, n_steps=batch_size) + data.batch["token_level_rewards"] = token_level_rewards + + metrics = {"actor/reward_kl_penalty": current_kl, "actor/reward_kl_penalty_coeff": beta} + + return data, metrics + + +def compute_response_mask(data: DataProto): + """Compute the attention mask for the response part of the sequence. + + This function extracts the portion of the attention mask that corresponds to the model's response, + which is used for masking computations that should only apply to response tokens. + + Args: + data (DataProto): The data containing batched model outputs and inputs. + + Returns: + torch.Tensor: The attention mask for the response tokens. + """ + responses = data.batch["responses"] + response_length = responses.size(1) + attention_mask = data.batch["attention_mask"] + return attention_mask[:, -response_length:] + + +def compute_advantage( + data: DataProto, + adv_estimator: AdvantageEstimator, + gamma: float = 1.0, + lam: float = 1.0, + num_repeat: int = 1, + norm_adv_by_std_in_grpo: bool = True, + config: Optional[AlgoConfig] = None, +) -> DataProto: + """Compute advantage estimates for policy optimization. + + This function computes advantage estimates using various estimators like GAE, GRPO, REINFORCE++, etc. + The advantage estimates are used to guide policy optimization in RL algorithms. + + Args: + data (DataProto): The data containing batched model outputs and inputs. + adv_estimator (AdvantageEstimator): The advantage estimator to use (e.g., GAE, GRPO, REINFORCE++). + gamma (float, optional): Discount factor for future rewards. Defaults to 1.0. + lam (float, optional): Lambda parameter for GAE. Defaults to 1.0. + num_repeat (int, optional): Number of times to repeat the computation. Defaults to 1. + norm_adv_by_std_in_grpo (bool, optional): Whether to normalize advantages by standard deviation in + GRPO. Defaults to True. + config (dict, optional): Configuration dictionary for algorithm settings. Defaults to None. + + Returns: + DataProto: The updated data with computed advantages and returns. + """ + # Back-compatible with trainers that do not compute response mask in fit + if "response_mask" not in data.batch.keys(): + data.batch["response_mask"] = compute_response_mask(data) + # prepare response group + if adv_estimator == AdvantageEstimator.GAE: + # Compute advantages and returns using Generalized Advantage Estimation (GAE) + advantages, returns = core_algos.compute_gae_advantage_return( + token_level_rewards=data.batch["token_level_rewards"], + values=data.batch["values"], + response_mask=data.batch["response_mask"], + gamma=gamma, + lam=lam, + ) + data.batch["advantages"] = advantages + data.batch["returns"] = returns + if config.get("use_pf_ppo", False): + data = core_algos.compute_pf_ppo_reweight_data( + data, + config.pf_ppo.get("reweight_method"), + config.pf_ppo.get("weight_pow"), + ) + elif adv_estimator == AdvantageEstimator.GRPO: + # Initialize the mask for GRPO calculation + grpo_calculation_mask = data.batch["response_mask"] + + # Call compute_grpo_outcome_advantage with parameters matching its definition + advantages, returns = core_algos.compute_grpo_outcome_advantage( + token_level_rewards=data.batch["token_level_rewards"], + response_mask=grpo_calculation_mask, + index=data.non_tensor_batch["uid"], + norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo, + ) + data.batch["advantages"] = advantages + data.batch["returns"] = returns + else: + # handle all other adv estimator type other than GAE and GRPO + adv_estimator_fn = core_algos.get_adv_estimator_fn(adv_estimator) + adv_kwargs = { + "token_level_rewards": data.batch["token_level_rewards"], + "response_mask": data.batch["response_mask"], + "config": config, + } + if "uid" in data.non_tensor_batch: # optional + adv_kwargs["index"] = data.non_tensor_batch["uid"] + if "reward_baselines" in data.batch: # optional + adv_kwargs["reward_baselines"] = data.batch["reward_baselines"] + + # calculate advantage estimator + advantages, returns = adv_estimator_fn(**adv_kwargs) + data.batch["advantages"] = advantages + data.batch["returns"] = returns + return data + + +class RayPPOTrainer: + """Distributed PPO trainer using Ray for scalable reinforcement learning. + + This trainer orchestrates distributed PPO training across multiple nodes and GPUs, + managing actor rollouts, critic training, and reward computation with Ray backend. + Supports various model architectures including FSDP, Megatron, vLLM, and SGLang integration. + """ + + # TODO: support each role have individual ray_worker_group_cls, + # i.e., support different backend of different role + def __init__( + self, + config, + tokenizer, + role_worker_mapping: dict[Role, WorkerType], + resource_pool_manager: ResourcePoolManager, + ray_worker_group_cls: type[RayWorkerGroup] = RayWorkerGroup, + processor=None, + reward_fn=None, + val_reward_fn=None, + train_dataset: Optional[Dataset] = None, + val_dataset: Optional[Dataset] = None, + collate_fn=None, + train_sampler: Optional[Sampler] = None, + device_name=None, + ): + """ + Initialize distributed PPO trainer with Ray backend. + Note that this trainer runs on the driver process on a single CPU/GPU node. + + Args: + config: Configuration object containing training parameters. + tokenizer: Tokenizer used for encoding and decoding text. + role_worker_mapping (dict[Role, WorkerType]): Mapping from roles to worker classes. + resource_pool_manager (ResourcePoolManager): Manager for Ray resource pools. + ray_worker_group_cls (RayWorkerGroup, optional): Class for Ray worker groups. Defaults to RayWorkerGroup. + processor: Optional data processor, used for multimodal data + reward_fn: Function for computing rewards during training. + val_reward_fn: Function for computing rewards during validation. + train_dataset (Optional[Dataset], optional): Training dataset. Defaults to None. + val_dataset (Optional[Dataset], optional): Validation dataset. Defaults to None. + collate_fn: Function to collate data samples into batches. + train_sampler (Optional[Sampler], optional): Sampler for the training dataset. Defaults to None. + device_name (str, optional): Device name for training (e.g., "cuda", "cpu"). Defaults to None. + """ + + # Store the tokenizer for text processing + self.tokenizer = tokenizer + self.processor = processor + self.config = config + self.reward_fn = reward_fn + self.val_reward_fn = val_reward_fn + + self.hybrid_engine = config.actor_rollout_ref.hybrid_engine + assert self.hybrid_engine, "Currently, only support hybrid engine" + + if self.hybrid_engine: + assert Role.ActorRollout in role_worker_mapping, f"{role_worker_mapping.keys()=}" + + self.role_worker_mapping = role_worker_mapping + self.resource_pool_manager = resource_pool_manager + self.use_reference_policy = need_reference_policy(self.role_worker_mapping) + self.use_rm = need_reward_model(self.role_worker_mapping) + self.use_critic = need_critic(self.config) + self.ray_worker_group_cls = ray_worker_group_cls + self.device_name = device_name if device_name else self.config.trainer.device + self.validation_generations_logger = ValidationGenerationsLogger( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + ) + + # if ref_in_actor is True, the reference policy will be actor without lora applied + self.ref_in_actor = config.actor_rollout_ref.model.get("lora_rank", 0) > 0 + + # define in-reward KL control + # kl loss control currently not suppoorted + if self.config.algorithm.use_kl_in_reward: + self.kl_ctrl_in_reward = core_algos.get_kl_controller(self.config.algorithm.kl_ctrl) + + self._create_dataloader(train_dataset, val_dataset, collate_fn, train_sampler) + + def _create_dataloader(self, train_dataset, val_dataset, collate_fn, train_sampler: Optional[Sampler]): + """ + Creates the train and validation dataloaders. + """ + # TODO: we have to make sure the batch size is divisible by the dp size + from verl.trainer.main_ppo import create_rl_dataset, create_rl_sampler + + if train_dataset is None: + train_dataset = create_rl_dataset( + self.config.data.train_files, self.config.data, self.tokenizer, self.processor + ) + if val_dataset is None: + val_dataset = create_rl_dataset( + self.config.data.val_files, self.config.data, self.tokenizer, self.processor + ) + self.train_dataset, self.val_dataset = train_dataset, val_dataset + + if train_sampler is None: + train_sampler = create_rl_sampler(self.config.data, self.train_dataset) + if collate_fn is None: + from verl.utils.dataset.rl_dataset import collate_fn as default_collate_fn + + collate_fn = default_collate_fn + + num_workers = self.config.data["dataloader_num_workers"] + + self.train_dataloader = StatefulDataLoader( + dataset=self.train_dataset, + batch_size=self.config.data.get("gen_batch_size", self.config.data.train_batch_size), + num_workers=num_workers, + drop_last=True, + collate_fn=collate_fn, + sampler=train_sampler, + ) + + val_batch_size = self.config.data.val_batch_size # Prefer config value if set + if val_batch_size is None: + val_batch_size = len(self.val_dataset) + + self.val_dataloader = StatefulDataLoader( + dataset=self.val_dataset, + batch_size=val_batch_size, + num_workers=num_workers, + shuffle=self.config.data.get("validation_shuffle", True), + drop_last=False, + collate_fn=collate_fn, + ) + + assert len(self.train_dataloader) >= 1, "Train dataloader is empty!" + assert len(self.val_dataloader) >= 1, "Validation dataloader is empty!" + + print( + f"Size of train dataloader: {len(self.train_dataloader)}, Size of val dataloader: " + f"{len(self.val_dataloader)}" + ) + + total_training_steps = len(self.train_dataloader) * self.config.trainer.total_epochs + + if self.config.trainer.total_training_steps is not None: + total_training_steps = self.config.trainer.total_training_steps + + self.total_training_steps = total_training_steps + print(f"Total training steps: {self.total_training_steps}") + + try: + OmegaConf.set_struct(self.config, True) + with open_dict(self.config): + if OmegaConf.select(self.config, "actor_rollout_ref.actor.optim"): + self.config.actor_rollout_ref.actor.optim.total_training_steps = total_training_steps + if OmegaConf.select(self.config, "critic.optim"): + self.config.critic.optim.total_training_steps = total_training_steps + except Exception as e: + print(f"Warning: Could not set total_training_steps in config. Structure missing? Error: {e}") + + def _dump_generations(self, inputs, outputs, gts, scores, reward_extra_infos_dict, dump_path): + """Dump rollout/validation samples as JSONL.""" + os.makedirs(dump_path, exist_ok=True) + filename = os.path.join(dump_path, f"{self.global_steps}.jsonl") + + n = len(inputs) + base_data = { + "input": inputs, + "output": outputs, + "gts": gts, + "score": scores, + "step": [self.global_steps] * n, + } + + for k, v in reward_extra_infos_dict.items(): + if len(v) == n: + base_data[k] = v + + lines = [] + for i in range(n): + entry = {k: v[i] for k, v in base_data.items()} + lines.append(json.dumps(entry, ensure_ascii=False)) + + with open(filename, "w") as f: + f.write("\n".join(lines) + "\n") + + print(f"Dumped generations to {filename}") + + def _log_rollout_data( + self, batch: DataProto, reward_extra_infos_dict: dict, timing_raw: dict, rollout_data_dir: str + ): + """Log rollout data to disk. + Args: + batch (DataProto): The batch containing rollout data + reward_extra_infos_dict (dict): Additional reward information to log + timing_raw (dict): Timing information for profiling + rollout_data_dir (str): Directory path to save the rollout data + """ + with marked_timer("dump_rollout_generations", timing_raw, color="green"): + inputs = self.tokenizer.batch_decode(batch.batch["prompts"], skip_special_tokens=True) + outputs = self.tokenizer.batch_decode(batch.batch["responses"], skip_special_tokens=True) + scores = batch.batch["token_level_scores"].sum(-1).cpu().tolist() + sample_gts = [item.non_tensor_batch.get("reward_model", {}).get("ground_truth", None) for item in batch] + + reward_extra_infos_to_dump = reward_extra_infos_dict.copy() + if "request_id" in batch.non_tensor_batch: + reward_extra_infos_dict.setdefault( + "request_id", + batch.non_tensor_batch["request_id"].tolist(), + ) + + self._dump_generations( + inputs=inputs, + outputs=outputs, + gts=sample_gts, + scores=scores, + reward_extra_infos_dict=reward_extra_infos_to_dump, + dump_path=rollout_data_dir, + ) + + def _maybe_log_val_generations(self, inputs, outputs, scores): + """Log a table of validation samples to the configured logger (wandb or swanlab)""" + + generations_to_log = self.config.trainer.log_val_generations + + if generations_to_log == 0: + return + + import numpy as np + + # Create tuples of (input, output, score) and sort by input text + samples = list(zip(inputs, outputs, scores, strict=True)) + samples.sort(key=lambda x: x[0]) # Sort by input text + + # Use fixed random seed for deterministic shuffling + rng = np.random.RandomState(42) + rng.shuffle(samples) + + # Take first N samples after shuffling + samples = samples[:generations_to_log] + + # Log to each configured logger + self.validation_generations_logger.log(self.config.trainer.logger, samples, self.global_steps) + + def _get_gen_batch(self, batch: DataProto) -> DataProto: + reward_model_keys = set({"data_source", "reward_model", "extra_info", "uid"}) & batch.non_tensor_batch.keys() + + # pop those keys for generation + batch_keys_to_pop = ["input_ids", "attention_mask", "position_ids"] + non_tensor_batch_keys_to_pop = set(batch.non_tensor_batch.keys()) - reward_model_keys + gen_batch = batch.pop( + batch_keys=batch_keys_to_pop, + non_tensor_batch_keys=list(non_tensor_batch_keys_to_pop), + ) + + # For agent loop, we need reward model keys to compute score. + if self.async_rollout_mode: + gen_batch.non_tensor_batch.update(batch.non_tensor_batch) + + return gen_batch + + def _validate(self): + data_source_lst = [] + reward_extra_infos_dict: dict[str, list] = defaultdict(list) + + # Lists to collect samples for the table + sample_inputs = [] + sample_outputs = [] + sample_gts = [] + sample_scores = [] + sample_turns = [] + sample_uids = [] + + for test_data in self.val_dataloader: + test_batch = DataProto.from_single_dict(test_data) + + if "uid" not in test_batch.non_tensor_batch: + test_batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(test_batch.batch))], dtype=object + ) + + # repeat test batch + test_batch = test_batch.repeat( + repeat_times=self.config.actor_rollout_ref.rollout.val_kwargs.n, interleave=True + ) + + # we only do validation on rule-based rm + if self.config.reward_model.enable and test_batch[0].non_tensor_batch["reward_model"]["style"] == "model": + return {} + + # Store original inputs + input_ids = test_batch.batch["input_ids"] + # TODO: Can we keep special tokens except for padding tokens? + input_texts = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in input_ids] + sample_inputs.extend(input_texts) + sample_uids.extend(test_batch.non_tensor_batch["uid"]) + + ground_truths = [ + item.non_tensor_batch.get("reward_model", {}).get("ground_truth", None) for item in test_batch + ] + sample_gts.extend(ground_truths) + + test_gen_batch = self._get_gen_batch(test_batch) + test_gen_batch.meta_info = { + "eos_token_id": self.tokenizer.eos_token_id, + "pad_token_id": self.tokenizer.pad_token_id, + "recompute_log_prob": False, + "do_sample": self.config.actor_rollout_ref.rollout.val_kwargs.do_sample, + "validate": True, + "global_steps": self.global_steps, + } + print(f"test_gen_batch meta info: {test_gen_batch.meta_info}") + + # pad to be divisible by dp_size + size_divisor = ( + self.actor_rollout_wg.world_size + if not self.async_rollout_mode + else self.config.actor_rollout_ref.rollout.agent.num_workers + ) + test_gen_batch_padded, pad_size = pad_dataproto_to_divisor(test_gen_batch, size_divisor) + if not self.async_rollout_mode: + test_output_gen_batch_padded = self.actor_rollout_wg.generate_sequences(test_gen_batch_padded) + else: + test_output_gen_batch_padded = self.async_rollout_manager.generate_sequences(test_gen_batch_padded) + + # unpad + test_output_gen_batch = unpad_dataproto(test_output_gen_batch_padded, pad_size=pad_size) + + print("validation generation end") + + # Store generated outputs + output_ids = test_output_gen_batch.batch["responses"] + output_texts = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in output_ids] + sample_outputs.extend(output_texts) + + test_batch = test_batch.union(test_output_gen_batch) + test_batch.meta_info["validate"] = True + + # evaluate using reward_function + if self.val_reward_fn is None: + raise ValueError("val_reward_fn must be provided for validation.") + result = self.val_reward_fn(test_batch, return_dict=True) + reward_tensor = result["reward_tensor"] + scores = reward_tensor.sum(-1).cpu().tolist() + sample_scores.extend(scores) + + reward_extra_infos_dict["reward"].extend(scores) + print(f"len reward_extra_infos_dict['reward']: {len(reward_extra_infos_dict['reward'])}") + if "reward_extra_info" in result: + for key, lst in result["reward_extra_info"].items(): + reward_extra_infos_dict[key].extend(lst) + print(f"len reward_extra_infos_dict['{key}']: {len(reward_extra_infos_dict[key])}") + + # collect num_turns of each prompt + if "__num_turns__" in test_batch.non_tensor_batch: + sample_turns.append(test_batch.non_tensor_batch["__num_turns__"]) + + data_source_lst.append(test_batch.non_tensor_batch.get("data_source", ["unknown"] * reward_tensor.shape[0])) + + self._maybe_log_val_generations(inputs=sample_inputs, outputs=sample_outputs, scores=sample_scores) + + # dump generations + val_data_dir = self.config.trainer.get("validation_data_dir", None) + if val_data_dir: + self._dump_generations( + inputs=sample_inputs, + outputs=sample_outputs, + gts=sample_gts, + scores=sample_scores, + reward_extra_infos_dict=reward_extra_infos_dict, + dump_path=val_data_dir, + ) + + for key_info, lst in reward_extra_infos_dict.items(): + assert len(lst) == 0 or len(lst) == len(sample_scores), f"{key_info}: {len(lst)=}, {len(sample_scores)=}" + + data_sources = np.concatenate(data_source_lst, axis=0) + + data_src2var2metric2val = process_validation_metrics(data_sources, sample_uids, reward_extra_infos_dict) + metric_dict = {} + for data_source, var2metric2val in data_src2var2metric2val.items(): + core_var = "acc" if "acc" in var2metric2val else "reward" + for var_name, metric2val in var2metric2val.items(): + n_max = max([int(name.split("@")[-1].split("/")[0]) for name in metric2val.keys()]) + for metric_name, metric_val in metric2val.items(): + if ( + (var_name == core_var) + and any(metric_name.startswith(pfx) for pfx in ["mean", "maj", "best"]) + and (f"@{n_max}" in metric_name) + ): + metric_sec = "val-core" + else: + metric_sec = "val-aux" + pfx = f"{metric_sec}/{data_source}/{var_name}/{metric_name}" + metric_dict[pfx] = metric_val + + if len(sample_turns) > 0: + sample_turns = np.concatenate(sample_turns) + metric_dict["val-aux/num_turns/min"] = sample_turns.min() + metric_dict["val-aux/num_turns/max"] = sample_turns.max() + metric_dict["val-aux/num_turns/mean"] = sample_turns.mean() + + return metric_dict + + def init_workers(self): + """Initialize distributed training workers using Ray backend. + + Creates: + 1. Ray resource pools from configuration + 2. Worker groups for each role (actor, critic, etc.) + """ + self.resource_pool_manager.create_resource_pool() + + self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} + + # create actor and rollout + if self.hybrid_engine: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.ActorRollout) + actor_rollout_cls = RayClassWithInitArgs( + cls=self.role_worker_mapping[Role.ActorRollout], + config=self.config.actor_rollout_ref, + role="actor_rollout", + ) + self.resource_pool_to_cls[resource_pool]["actor_rollout"] = actor_rollout_cls + else: + raise NotImplementedError + + # create critic + if self.use_critic: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.Critic) + critic_cfg = omega_conf_to_dataclass(self.config.critic) + critic_cls = RayClassWithInitArgs(cls=self.role_worker_mapping[Role.Critic], config=critic_cfg) + self.resource_pool_to_cls[resource_pool]["critic"] = critic_cls + + # create reference policy if needed + if self.use_reference_policy: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) + ref_policy_cls = RayClassWithInitArgs( + self.role_worker_mapping[Role.RefPolicy], + config=self.config.actor_rollout_ref, + role="ref", + ) + self.resource_pool_to_cls[resource_pool]["ref"] = ref_policy_cls + + # create a reward model if reward_fn is None + if self.use_rm: + # we create a RM here + resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) + rm_cls = RayClassWithInitArgs(self.role_worker_mapping[Role.RewardModel], config=self.config.reward_model) + self.resource_pool_to_cls[resource_pool]["rm"] = rm_cls + + # initialize WorkerGroup + # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, + # you should not use `create_colocated_worker_cls`. + # Instead, directly pass different resource pool to different worker groups. + # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. + all_wg = {} + wg_kwargs = {} # Setting up kwargs for RayWorkerGroup + if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None: + wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout + if OmegaConf.select(self.config.global_profiler, "steps") is not None: + wg_kwargs["profile_steps"] = OmegaConf.select(self.config.global_profiler, "steps") + # Only require nsight worker options when tool is nsys + if OmegaConf.select(self.config.global_profiler, "tool") == "nsys": + assert ( + OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") + is not None + ), "worker_nsight_options must be set when using nsys with profile_steps" + wg_kwargs["worker_nsight_options"] = OmegaConf.to_container( + OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") + ) + wg_kwargs["device_name"] = self.device_name + + for resource_pool, class_dict in self.resource_pool_to_cls.items(): + worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + wg_dict = self.ray_worker_group_cls( + resource_pool=resource_pool, + ray_cls_with_init=worker_dict_cls, + **wg_kwargs, + ) + spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + all_wg.update(spawn_wg) + + if self.use_critic: + self.critic_wg = all_wg["critic"] + self.critic_wg.init_model() + + if self.use_reference_policy and not self.ref_in_actor: + self.ref_policy_wg = all_wg["ref"] + self.ref_policy_wg.init_model() + + self.rm_wg = None + if self.use_rm: + self.rm_wg = all_wg["rm"] + self.rm_wg.init_model() + + # we should create rollout at the end so that vllm can have a better estimation of kv cache memory + self.actor_rollout_wg = all_wg["actor_rollout"] + self.actor_rollout_wg.init_model() + + # create async rollout manager and request scheduler + self.async_rollout_mode = False + if self.config.actor_rollout_ref.rollout.mode == "async": + from verl.experimental.agent_loop import AgentLoopManager + + self.async_rollout_mode = True + self.async_rollout_manager = AgentLoopManager( + config=self.config, worker_group=self.actor_rollout_wg, rm_wg=self.rm_wg + ) + + def _save_checkpoint(self): + from verl.utils.fs import local_mkdir_safe + + # path: given_path + `/global_step_{global_steps}` + `/actor` + local_global_step_folder = os.path.join( + self.config.trainer.default_local_dir, f"global_step_{self.global_steps}" + ) + + print(f"local_global_step_folder: {local_global_step_folder}") + actor_local_path = os.path.join(local_global_step_folder, "actor") + + actor_remote_path = ( + None + if self.config.trainer.default_hdfs_dir is None + else os.path.join(self.config.trainer.default_hdfs_dir, f"global_step_{self.global_steps}", "actor") + ) + + remove_previous_ckpt_in_save = self.config.trainer.get("remove_previous_ckpt_in_save", False) + if remove_previous_ckpt_in_save: + print( + "Warning: remove_previous_ckpt_in_save is deprecated," + + " set max_actor_ckpt_to_keep=1 and max_critic_ckpt_to_keep=1 instead" + ) + max_actor_ckpt_to_keep = ( + self.config.trainer.get("max_actor_ckpt_to_keep", None) if not remove_previous_ckpt_in_save else 1 + ) + max_critic_ckpt_to_keep = ( + self.config.trainer.get("max_critic_ckpt_to_keep", None) if not remove_previous_ckpt_in_save else 1 + ) + + self.actor_rollout_wg.save_checkpoint( + actor_local_path, actor_remote_path, self.global_steps, max_ckpt_to_keep=max_actor_ckpt_to_keep + ) + + if self.use_critic: + critic_local_path = os.path.join(local_global_step_folder, "critic") + critic_remote_path = ( + None + if self.config.trainer.default_hdfs_dir is None + else os.path.join(self.config.trainer.default_hdfs_dir, f"global_step_{self.global_steps}", "critic") + ) + self.critic_wg.save_checkpoint( + critic_local_path, critic_remote_path, self.global_steps, max_ckpt_to_keep=max_critic_ckpt_to_keep + ) + + # save dataloader + local_mkdir_safe(local_global_step_folder) + dataloader_local_path = os.path.join(local_global_step_folder, "data.pt") + dataloader_state_dict = self.train_dataloader.state_dict() + torch.save(dataloader_state_dict, dataloader_local_path) + + # latest checkpointed iteration tracker (for atomic usage) + local_latest_checkpointed_iteration = os.path.join( + self.config.trainer.default_local_dir, "latest_checkpointed_iteration.txt" + ) + with open(local_latest_checkpointed_iteration, "w") as f: + f.write(str(self.global_steps)) + + def _load_checkpoint(self): + if self.config.trainer.resume_mode == "disable": + return 0 + + # load from hdfs + if self.config.trainer.default_hdfs_dir is not None: + raise NotImplementedError("load from hdfs is not implemented yet") + else: + checkpoint_folder = self.config.trainer.default_local_dir # TODO: check path + if not os.path.isabs(checkpoint_folder): + working_dir = os.getcwd() + checkpoint_folder = os.path.join(working_dir, checkpoint_folder) + global_step_folder = find_latest_ckpt_path(checkpoint_folder) # None if no latest + + # find global_step_folder + if self.config.trainer.resume_mode == "auto": + if global_step_folder is None: + print("Training from scratch") + return 0 + else: + if self.config.trainer.resume_mode == "resume_path": + assert isinstance(self.config.trainer.resume_from_path, str), "resume ckpt must be str type" + assert "global_step_" in self.config.trainer.resume_from_path, ( + "resume ckpt must specify the global_steps" + ) + global_step_folder = self.config.trainer.resume_from_path + if not os.path.isabs(global_step_folder): + working_dir = os.getcwd() + global_step_folder = os.path.join(working_dir, global_step_folder) + print(f"Load from checkpoint folder: {global_step_folder}") + # set global step + self.global_steps = int(global_step_folder.split("global_step_")[-1]) + + print(f"Setting global step to {self.global_steps}") + print(f"Resuming from {global_step_folder}") + + actor_path = os.path.join(global_step_folder, "actor") + critic_path = os.path.join(global_step_folder, "critic") + # load actor + self.actor_rollout_wg.load_checkpoint( + actor_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load + ) + # load critic + if self.use_critic: + self.critic_wg.load_checkpoint( + critic_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load + ) + + # load dataloader, + # TODO: from remote not implemented yet + dataloader_local_path = os.path.join(global_step_folder, "data.pt") + if os.path.exists(dataloader_local_path): + dataloader_state_dict = torch.load(dataloader_local_path, weights_only=False) + self.train_dataloader.load_state_dict(dataloader_state_dict) + else: + print(f"Warning: No dataloader state found at {dataloader_local_path}, will start from scratch") + + def _start_profiling(self, do_profile: bool) -> None: + """Start profiling for all worker groups if profiling is enabled.""" + if do_profile: + self.actor_rollout_wg.start_profile(role="e2e", profile_step=self.global_steps) + if self.use_reference_policy: + self.ref_policy_wg.start_profile(profile_step=self.global_steps) + if self.use_critic: + self.critic_wg.start_profile(profile_step=self.global_steps) + if self.use_rm: + self.rm_wg.start_profile(profile_step=self.global_steps) + + def _stop_profiling(self, do_profile: bool) -> None: + """Stop profiling for all worker groups if profiling is enabled.""" + if do_profile: + self.actor_rollout_wg.stop_profile() + if self.use_reference_policy: + self.ref_policy_wg.stop_profile() + if self.use_critic: + self.critic_wg.stop_profile() + if self.use_rm: + self.rm_wg.stop_profile() + + def _balance_batch(self, batch: DataProto, metrics, logging_prefix="global_seqlen"): + """Reorder the data on single controller such that each dp rank gets similar total tokens""" + attention_mask = batch.batch["attention_mask"] + batch_size = attention_mask.shape[0] + global_seqlen_lst = batch.batch["attention_mask"].view(batch_size, -1).sum(-1).tolist() # (train_batch_size,) + world_size = self.actor_rollout_wg.world_size + global_partition_lst = get_seqlen_balanced_partitions( + global_seqlen_lst, k_partitions=world_size, equal_size=True + ) + # reorder based on index. The data will be automatically equally partitioned by dispatch function + global_idx = torch.tensor([j for partition in global_partition_lst for j in partition]) + batch.reorder(global_idx) + global_balance_stats = log_seqlen_unbalance( + seqlen_list=global_seqlen_lst, partitions=global_partition_lst, prefix=logging_prefix + ) + metrics.update(global_balance_stats) + + def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the worker group through RPC + to construct the PPO dataflow. + The light-weight advantage computation is done on the driver process. + """ + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + self.global_steps = 0 + + # load checkpoint before doing anything + self._load_checkpoint() + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + val_metrics = self._validate() + assert val_metrics, f"{val_metrics=}" + pprint(f"Initial validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if self.config.trainer.get("val_only", False): + return + + if self.config.actor_rollout_ref.rollout.get("skip_rollout", False): + rollout_skip = RolloutSkip(self.config, self.actor_rollout_wg) + rollout_skip.wrap_generate_sequences() + + # add tqdm + progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress") + + # we start from step 1 + self.global_steps += 1 + last_val_metrics = None + self.max_steps_duration = 0 + + prev_step_profile = False + curr_step_profile = ( + self.global_steps in self.config.global_profiler.steps + if self.config.global_profiler.steps is not None + else False + ) + next_step_profile = False + + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + metrics = {} + timing_raw = {} + + with marked_timer("start_profile", timing_raw): + self._start_profiling( + not prev_step_profile and curr_step_profile + if self.config.global_profiler.profile_continuous_steps + else curr_step_profile + ) + batch: DataProto = DataProto.from_single_dict(batch_dict) + + # add uid to batch + batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(batch.batch))], dtype=object + ) + + gen_batch = self._get_gen_batch(batch) + + # pass global_steps to trace + gen_batch.meta_info["global_steps"] = self.global_steps + gen_batch = gen_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + + is_last_step = self.global_steps >= self.total_training_steps + with marked_timer("step", timing_raw): + # generate a batch + with marked_timer("gen", timing_raw, color="red"): + if not self.async_rollout_mode: + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + else: + gen_batch_output = self.async_rollout_manager.generate_sequences(gen_batch) + + timing_raw.update(gen_batch_output.meta_info["timing"]) + gen_batch_output.meta_info.pop("timing", None) + + if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: + if self.reward_fn is None: + raise ValueError("A reward_fn is required for REMAX advantage estimation.") + + with marked_timer("gen_max", timing_raw, color="purple"): + gen_baseline_batch = deepcopy(gen_batch) + gen_baseline_batch.meta_info["do_sample"] = False + if not self.async_rollout_mode: + gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) + else: + gen_baseline_output = self.async_rollout_manager.generate_sequences(gen_baseline_batch) + batch = batch.union(gen_baseline_output) + reward_baseline_tensor = self.reward_fn(batch) + reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) + + batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) + + batch.batch["reward_baselines"] = reward_baseline_tensor + + del gen_baseline_batch, gen_baseline_output + # repeat to align with repeated responses in rollout + batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) + batch = batch.union(gen_batch_output) + + if "response_mask" not in batch.batch.keys(): + batch.batch["response_mask"] = compute_response_mask(batch) + # Balance the number of valid tokens across DP ranks. + # NOTE: This usually changes the order of data in the `batch`, + # which won't affect the advantage calculation (since it's based on uid), + # but might affect the loss calculation (due to the change of mini-batching). + # TODO: Decouple the DP balancing and mini-batching. + if self.config.trainer.balance_batch: + self._balance_batch(batch, metrics=metrics) + + # compute global_valid tokens + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + with marked_timer("reward", timing_raw, color="yellow"): + # compute reward model score + if self.use_rm and "rm_scores" not in batch.batch.keys(): + reward_tensor = self.rm_wg.compute_rm_score(batch) + batch = batch.union(reward_tensor) + + if self.config.reward_model.launch_reward_fn_async: + future_reward = compute_reward_async.remote(data=batch, reward_fn=self.reward_fn) + else: + reward_tensor, reward_extra_infos_dict = compute_reward(batch, self.reward_fn) + + # recompute old_log_probs + with marked_timer("old_log_prob", timing_raw, color="blue"): + old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) + entropys = old_log_prob.batch["entropys"] + response_masks = batch.batch["response_mask"] + loss_agg_mode = self.config.actor_rollout_ref.actor.loss_agg_mode + entropy_agg = agg_loss(loss_mat=entropys, loss_mask=response_masks, loss_agg_mode=loss_agg_mode) + old_log_prob_metrics = {"actor/entropy": entropy_agg.detach().item()} + metrics.update(old_log_prob_metrics) + old_log_prob.batch.pop("entropys") + batch = batch.union(old_log_prob) + + if "rollout_log_probs" in batch.batch.keys(): + # TODO: we may want to add diff of probs too. + from verl.utils.debug.metrics import calculate_debug_metrics + + metrics.update(calculate_debug_metrics(batch)) + + if self.use_reference_policy: + # compute reference log_prob + with marked_timer("ref", timing_raw, color="olive"): + if not self.ref_in_actor: + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + else: + ref_log_prob = self.actor_rollout_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + + # compute values + if self.use_critic: + with marked_timer("values", timing_raw, color="cyan"): + values = self.critic_wg.compute_values(batch) + batch = batch.union(values) + + with marked_timer("adv", timing_raw, color="brown"): + # we combine with rule-based rm + reward_extra_infos_dict: dict[str, list] + if self.config.reward_model.launch_reward_fn_async: + reward_tensor, reward_extra_infos_dict = ray.get(future_reward) + batch.batch["token_level_scores"] = reward_tensor + + if reward_extra_infos_dict: + batch.non_tensor_batch.update({k: np.array(v) for k, v in reward_extra_infos_dict.items()}) + + # compute rewards. apply_kl_penalty if available + if self.config.algorithm.use_kl_in_reward: + batch, kl_metrics = apply_kl_penalty( + batch, kl_ctrl=self.kl_ctrl_in_reward, kl_penalty=self.config.algorithm.kl_penalty + ) + metrics.update(kl_metrics) + else: + batch.batch["token_level_rewards"] = batch.batch["token_level_scores"] + + # compute advantages, executed on the driver process + norm_adv_by_std_in_grpo = self.config.algorithm.get( + "norm_adv_by_std_in_grpo", True + ) # GRPO adv normalization factor + + batch = compute_advantage( + batch, + adv_estimator=self.config.algorithm.adv_estimator, + gamma=self.config.algorithm.gamma, + lam=self.config.algorithm.lam, + num_repeat=self.config.actor_rollout_ref.rollout.n, + norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo, + config=self.config.algorithm, + ) + + # update critic + if self.use_critic: + with marked_timer("update_critic", timing_raw, color="pink"): + critic_output = self.critic_wg.update_critic(batch) + critic_output_metrics = reduce_metrics(critic_output.meta_info["metrics"]) + metrics.update(critic_output_metrics) + + # implement critic warmup + if self.config.trainer.critic_warmup <= self.global_steps: + # update actor + with marked_timer("update_actor", timing_raw, color="red"): + batch.meta_info["multi_turn"] = self.config.actor_rollout_ref.rollout.multi_turn.enable + actor_output = self.actor_rollout_wg.update_actor(batch) + actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"]) + metrics.update(actor_output_metrics) + + # Log rollout generations if enabled + rollout_data_dir = self.config.trainer.get("rollout_data_dir", None) + if rollout_data_dir: + self._log_rollout_data(batch, reward_extra_infos_dict, timing_raw, rollout_data_dir) + + # validate + if ( + self.val_reward_fn is not None + and self.config.trainer.test_freq > 0 + and (is_last_step or self.global_steps % self.config.trainer.test_freq == 0) + ): + with marked_timer("testing", timing_raw, color="green"): + val_metrics: dict = self._validate() + if is_last_step: + last_val_metrics = val_metrics + metrics.update(val_metrics) + + # Check if the ESI (Elastic Server Instance)/training plan is close to expiration. + esi_close_to_expiration = should_save_ckpt_esi( + max_steps_duration=self.max_steps_duration, + redundant_time=self.config.trainer.esi_redundant_time, + ) + # Check if the conditions for saving a checkpoint are met. + # The conditions include a mandatory condition (1) and + # one of the following optional conditions (2/3/4): + # 1. The save frequency is set to a positive value. + # 2. It's the last training step. + # 3. The current step number is a multiple of the save frequency. + # 4. The ESI(Elastic Server Instance)/training plan is close to expiration. + if self.config.trainer.save_freq > 0 and ( + is_last_step or self.global_steps % self.config.trainer.save_freq == 0 or esi_close_to_expiration + ): + if esi_close_to_expiration: + print("Force saving checkpoint: ESI instance expiration approaching.") + with marked_timer("save_checkpoint", timing_raw, color="green"): + self._save_checkpoint() + + with marked_timer("stop_profile", timing_raw): + next_step_profile = ( + self.global_steps + 1 in self.config.global_profiler.steps + if self.config.global_profiler.steps is not None + else False + ) + self._stop_profiling( + curr_step_profile and not next_step_profile + if self.config.global_profiler.profile_continuous_steps + else curr_step_profile + ) + prev_step_profile = curr_step_profile + curr_step_profile = next_step_profile + + steps_duration = timing_raw["step"] + self.max_steps_duration = max(self.max_steps_duration, steps_duration) + + # training metrics + metrics.update( + { + "training/global_step": self.global_steps, + "training/epoch": epoch, + } + ) + # collect metrics + metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) + metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) + # TODO: implement actual tflpo and theoretical tflpo + n_gpus = self.resource_pool_manager.get_n_gpus() + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) + + # this is experimental and may be changed/removed in the future in favor of a general-purpose one + if isinstance(self.train_dataloader.sampler, AbstractCurriculumSampler): + self.train_dataloader.sampler.update(batch=batch) + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=self.global_steps) + + progress_bar.update(1) + self.global_steps += 1 + + if ( + hasattr(self.config.actor_rollout_ref.actor, "profiler") + and self.config.actor_rollout_ref.actor.profiler.tool == "torch_memory" + ): + self.actor_rollout_wg.dump_memory_snapshot( + tag=f"post_update_step{self.global_steps}", sub_dir=f"step{self.global_steps}" + ) + + if is_last_step: + pprint(f"Final validation metrics: {last_val_metrics}") + progress_bar.close() + return + + # this is experimental and may be changed/removed in the future + # in favor of a general-purpose data buffer pool + if hasattr(self.train_dataset, "on_batch_end"): + # The dataset may be changed after each training batch + self.train_dataset.on_batch_end(batch=batch) diff --git a/e2c/verl/verl/trainer/ppo/reward.py b/e2c/verl/verl/trainer/ppo/reward.py new file mode 100644 index 0000000000000000000000000000000000000000..7b1fdfc9c802bfdd56a09b15b206e795309b6a89 --- /dev/null +++ b/e2c/verl/verl/trainer/ppo/reward.py @@ -0,0 +1,191 @@ +# Copyright 2025 Individual Contributor: Thibaut Barroyer +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import multiprocessing +import os +import sys +import warnings +from functools import partial +from typing import Any, Optional + +import ray +import torch +from omegaconf import DictConfig + +from verl import DataProto +from verl.utils.reward_score import default_compute_score +from verl.workers.reward_manager import get_reward_manager_cls +from verl.workers.reward_manager.abstract import AbstractRewardManager, RawRewardFn + + +def _call_with_kwargs(raw_fn, extra_kwargs, *args, **kwargs): + """Calls `raw_fn` by merging `extra_kwargs` into call-time `kwargs`, with `extra_kwargs` taking precedence. + + This function is used to merge additional keyword arguments with the original function's arguments. + """ + merged_kwargs = {**kwargs, **extra_kwargs} + return raw_fn(*args, **merged_kwargs) + + +def get_custom_reward_fn(config: DictConfig) -> Optional[RawRewardFn]: + """Load and return a custom reward function from external file. + + Dynamically imports a reward function from a specified file path and wraps + it with additional keyword arguments from the configuration. + + Args: + config (dict): Configuration dictionary containing custom_reward_function + settings with 'path', 'name', and 'reward_kwargs' fields. + + Returns: + callable or None: Wrapped reward function with merged kwargs, or None + if no custom reward function is configured. + + Raises: + FileNotFoundError: If the specified reward function file doesn't exist. + RuntimeError: If there's an error loading the module from file. + AttributeError: If the specified function name isn't found in the module. + """ + + reward_fn_config = config.get("custom_reward_function") or {} + file_path = reward_fn_config.get("path") + if not file_path: + return None + + function_name = reward_fn_config.get("name") + assert function_name is not None + + module = sys.modules.get("custom_module", None) + if module is None: + if not os.path.exists(file_path): + raise FileNotFoundError(f"Reward function file '{file_path}' not found.") + + spec = importlib.util.spec_from_file_location("custom_module", file_path) + assert spec is not None + module = importlib.util.module_from_spec(spec) + try: + sys.modules["custom_module"] = module + assert spec.loader is not None + spec.loader.exec_module(module) + except Exception as e: + raise RuntimeError(f"Error loading module from '{file_path}': {e}") from e + + if not hasattr(module, function_name): + raise AttributeError(f"Reward function '{function_name}' not found in '{module.__file__}'.") + + print(f"using customized reward function '{function_name}' from '{module.__file__}'") + raw_fn = getattr(module, function_name) + + reward_kwargs = dict(reward_fn_config.get("reward_kwargs", {})) + + return partial(_call_with_kwargs, raw_fn, reward_kwargs) + + +def load_reward_manager( + config: DictConfig, tokenizer: Any, num_examine: int, **reward_kwargs: Any +) -> AbstractRewardManager: + """ + Load and initialize a reward manager based on the configuration. + + Args: + config: PPO trainer configuration object containing reward_model fields. + tokenizer: Tokenizer object used for processing text. + num_examine: Number of samples to examine. + **reward_kwargs: Additional keyword arguments for the reward manager. + + Returns: + An instance of the specified reward manager class. + """ + + # The list of pre-defined reward managers are defined in `verl/workers/reward_manager/`: + # naive: NaiveRewardManager + # prime: PrimeRewardManager + # batch: BatchRewardManager + # dapo: DAPORewardManager + # Note(haibin.lin): For custom reward managers, please make sure they are imported and + # registered via `verl.workers.reward_manager.register` + # By default reward_manager is set to naive (NaiveRewardManager) + reward_manager_name = config.reward_model.get("reward_manager", "naive") + reward_manager_cls = get_reward_manager_cls(reward_manager_name) + + # Try to get a custom reward function based on the configuration + compute_score = get_custom_reward_fn(config) + final_compute_score = compute_score + + if compute_score is None: + sandbox_config = config.reward_model.get("sandbox_fusion") + sandbox_url = sandbox_config.get("url") if sandbox_config else None + memory_limit_mb = sandbox_config.get("memory_limit_mb", 1024) + if sandbox_url: + sandbox_manager = multiprocessing.Manager() + # Create a semaphore to control concurrent access to the sandbox + _concurrent_semaphore = sandbox_manager.Semaphore(sandbox_config.get("max_concurrent", 64)) + final_compute_score = partial( + default_compute_score, + sandbox_fusion_url=sandbox_url, + concurrent_semaphore=_concurrent_semaphore, + memory_limit_mb=memory_limit_mb, + ) + else: + final_compute_score = default_compute_score + + # Instantiate and return the reward manager with the specified parameters + return reward_manager_cls( + tokenizer=tokenizer, + num_examine=num_examine, + compute_score=final_compute_score, + reward_fn_key=config.data.reward_fn_key, + **reward_kwargs, + ) + + +def compute_reward(data: DataProto, reward_fn: AbstractRewardManager) -> tuple[torch.Tensor, dict[str, Any]]: + """ + Compute reward for a batch of data. + Args: + data: DataProto object containing the input data. + reward_fn: Reward function to compute the reward. + Returns: + Tuple of reward tensor and extra info dictionary. + """ + try: + reward_result = reward_fn(data, return_dict=True) + reward_tensor = reward_result["reward_tensor"] + reward_extra_infos_dict = reward_result.get("reward_extra_info", {}) + except Exception as e: + print(f"Error in reward_fn: {e}") + reward_tensor = reward_fn(data) + reward_extra_infos_dict = {} + + return reward_tensor, reward_extra_infos_dict + + +@ray.remote(num_cpus=1) +def compute_reward_async(data: DataProto, config=None, tokenizer=None, reward_fn=None): + """ + Load the reward manager and compute the reward for a batch of data. + This is meant to be run in a separate Ray worker. + """ + if reward_fn is None: + assert config is not None and tokenizer is not None, ( + "config and tokenizer must not be None when reward_fn is None" + ) + + warnings.warn("using config and tokenizer with compute_reward_async is deprecated", stacklevel=2) + reward_fn = load_reward_manager( + config, tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {}) + ) + + return compute_reward(data, reward_fn) diff --git a/e2c/verl/verl/trainer/ppo/utils.py b/e2c/verl/verl/trainer/ppo/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..22d00a45052b04f05140f5c55710bbd517466512 --- /dev/null +++ b/e2c/verl/verl/trainer/ppo/utils.py @@ -0,0 +1,65 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from enum import Enum + +from omegaconf import DictConfig + +from verl.single_controller.base import Worker +from verl.trainer.ppo.core_algos import AdvantageEstimator + +WorkerType = type[Worker] + + +class Role(Enum): + """ + To create more roles dynamically, you can subclass Role and add new members + """ + + Actor = 0 + Rollout = 1 + ActorRollout = 2 + Critic = 3 + RefPolicy = 4 + RewardModel = 5 + ActorRolloutRef = 6 + + +def need_reference_policy( + role_worker_mapping: dict[Role, WorkerType], +) -> bool: + """Given a role worker mapping, do we need ref policy.""" + return Role.RefPolicy in role_worker_mapping + + +def need_reward_model( + role_worker_mapping: dict[Role, WorkerType], +) -> bool: + """Given a role worker mapping, do we need reward model.""" + return Role.RewardModel in role_worker_mapping + + +def need_critic(config: DictConfig) -> bool: + """Given a config, do we need critic.""" + if config.critic.enable is not None: + return bool(config.critic.enable) + elif config.algorithm.adv_estimator == AdvantageEstimator.GAE: + return True + else: + warnings.warn( + "Disabled critic as algorithm.adv_estimator != gae. If it is not intended, please set critic.enable=True", + stacklevel=2, + ) + return False diff --git a/e2c/verl/verl/trainer/runtime_env.yaml b/e2c/verl/verl/trainer/runtime_env.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63750cd72f720838343c5c118d9fddf5f5759a4c --- /dev/null +++ b/e2c/verl/verl/trainer/runtime_env.yaml @@ -0,0 +1,5 @@ +working_dir: ./ +excludes: ["/.git/"] +env_vars: + TORCH_NCCL_AVOID_RECORD_STREAMS: "1" + CUDA_DEVICE_MAX_CONNECTIONS: "1" diff --git a/e2c/verl/verl/trainer/sft_trainer.py b/e2c/verl/verl/trainer/sft_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..81d471bb1c89ed8f289f8179b2e4a7adf42c4c83 --- /dev/null +++ b/e2c/verl/verl/trainer/sft_trainer.py @@ -0,0 +1,384 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from functools import partial + +os.environ["NCCL_DEBUG"] = "WARN" +os.environ["TOKENIZERS_PARALLELISM"] = "true" + +import logging + +import hydra +import torch +import torch.distributed +from codetiming import Timer +from omegaconf import OmegaConf +from torch.utils.data import DistributedSampler +from torchdata.stateful_dataloader import StatefulDataLoader +from tqdm import tqdm + +from verl.utils import tensordict_utils as tu +from verl.utils.checkpoint import CheckpointHandler +from verl.utils.dataset.dataset_utils import SFTTensorCollator +from verl.utils.dataset.multiturn_sft_dataset import MultiTurnSFTDataset +from verl.utils.device import get_device_name, is_cuda_available, is_npu_available +from verl.utils.distributed import destroy_global_process_group +from verl.utils.flops_counter import FlopsCounter +from verl.utils.logger import log_with_rank +from verl.utils.tracking import Tracking + +if is_cuda_available: + pass +elif is_npu_available: + pass + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_SFT_LOGGING_LEVEL", "WARN")) + + +class SFTTrainer: + def __init__( + self, + config, + ): + self.config = config + + self.rank = torch.distributed.get_rank() + + self._build_config() + self._build_dataset() + + self._build_engine() + + self._build_dataloader() + + # Initialize resume-related variables + self.resume_global_step = 0 + + self._init_engine() + + self._build_ckpt_handler() + + self.ckpt_handler.load_checkpoint() + + self.device_name = self.config.trainer.device + + from verl.workers.roles.utils.losses import sft_loss + + self.loss_fn = partial(sft_loss, config=None) + + self.flops_counter = FlopsCounter(self.model_config.hf_config) + + if self.rank == 0: + print(self.config) + + def _build_ckpt_handler(self): + resume_mode = getattr(self.config.trainer, "resume_mode", "auto") + resume_from_path = getattr(self.config.trainer, "resume_from_path", None) + max_ckpt_to_keep = getattr(self.config.trainer, "max_ckpt_to_keep", None) + default_hdfs_dir = getattr(self.config.trainer, "default_hdfs_dir", None) + + self.ckpt_handler = CheckpointHandler( + engine=self.engine, + train_dataloader=self.train_dataloader, + default_local_dir=self.config.trainer.default_local_dir, + max_ckpt_to_keep=max_ckpt_to_keep, + default_hdfs_dir=default_hdfs_dir, + resume_mode=resume_mode, + resume_from_path=resume_from_path, + ) + + def _build_config(self): + from verl.utils.config import omega_conf_to_dataclass + + self.model_config = omega_conf_to_dataclass(self.config.model) + self.engine_config = omega_conf_to_dataclass(self.config.engine) + self.optimizer_config = omega_conf_to_dataclass(self.config.optim) + self.checkpoint_config = omega_conf_to_dataclass(self.config.checkpoint) + + def _build_engine(self): + from verl.workers.engine import BaseEngine, EngineRegistry + + self.engine: BaseEngine = EngineRegistry.new( + model_type="language_model", + backend=self.engine_config.strategy, + model_config=self.model_config, + engine_config=self.engine_config, + optimizer_config=self.optimizer_config, + checkpoint_config=self.checkpoint_config, + ) + + def _init_engine(self): + # patch optimizer config + if self.config.trainer.total_training_steps is not None: + self.total_training_steps = self.config.trainer.total_training_steps + else: + self.total_training_steps = len(self.train_dataloader) * self.config.trainer.total_epochs + self.optimizer_config.total_training_steps = self.total_training_steps + + self.steps_per_epoch = len(self.train_dataloader) + + # manage save and test frequency + self.save_freq = self.config.trainer.save_freq + if self.save_freq == "after_each_epoch": + self.save_freq = self.steps_per_epoch + + self.test_freq = self.config.trainer.test_freq + if self.test_freq == "after_each_epoch": + self.test_freq = self.steps_per_epoch + + self.engine.initialize() + + def _build_dataset(self): + config = self.config + tokenizer = self.model_config.tokenizer + train_dataset = create_sft_dataset(config.data.train_files, config.data, tokenizer) + val_dataset = create_sft_dataset(config.data.val_files, config.data, tokenizer) + + self.train_dataset, self.val_dataset = train_dataset, val_dataset + + def _build_dataloader(self): + # build dataset + config = self.config + # build dataloader + # Use data parallel rank and size instead of global rank and world size + + # Set pin_memory_device when pin_memory is enabled. + device_name = get_device_name() + + dp_rank = self.engine.get_data_parallel_rank() + dp_size = self.engine.get_data_parallel_size() + + self.train_sampler = DistributedSampler( + self.train_dataset, shuffle=True, num_replicas=dp_size, rank=dp_rank, drop_last=True + ) + + self.global_batch_size = config.data.train_batch_size + self.train_batch_size_per_dp = self.global_batch_size // dp_size + self.collate_fn = SFTTensorCollator(config.data.pad_mode) + + self.train_dataloader = StatefulDataLoader( + dataset=self.train_dataset, + batch_size=self.train_batch_size_per_dp, + sampler=self.train_sampler, + collate_fn=self.collate_fn, + num_workers=8, + pin_memory=True, + drop_last=True, + pin_memory_device=device_name, + ) + + self.val_sampler = DistributedSampler( + self.val_dataset, shuffle=False, num_replicas=dp_size, rank=dp_rank, drop_last=True + ) + self.val_dataloader = StatefulDataLoader( + dataset=self.val_dataset, + batch_size=self.train_batch_size_per_dp, + sampler=self.val_sampler, + collate_fn=self.collate_fn, + num_workers=8, + pin_memory=True, + drop_last=True, + pin_memory_device=device_name, + ) + + def fit(self): + is_logging = self.engine.is_mp_src_rank_with_outputs() and self.engine.get_data_parallel_rank() == 0 + + # TODO: add a unified tracking + if is_logging: + tracking = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + global_step = self.resume_global_step # Start from resumed step + last_valid_metric = None + + log_with_rank( + f"Total training steps: {self.total_training_steps},", + logger=logger, + rank=0, + log_only_rank_0=True, + ) + + # With StatefulDataLoader, we don't need to manually calculate epochs and steps + # The dataloader will automatically resume from where it left off + if global_step > 0: + log_with_rank( + f"StatefulDataLoader will automatically resume from global step: {global_step}", + logger=logger, + rank=0, + log_only_rank_0=True, + ) + + # Calculate which epoch we're starting from for sampler.set_epoch() + start_epoch = global_step // self.steps_per_epoch + + meta_info = { + "use_remove_padding": self.config.model.use_remove_padding, + "use_dynamic_bsz": self.config.data.use_dynamic_bsz, + "max_token_len_per_gpu": self.config.data.max_token_len_per_gpu, + "micro_batch_size_per_gpu": self.config.data.micro_batch_size_per_gpu, + "temperature": 1.0, + "global_batch_size": self.global_batch_size, + "pad_mode": self.config.data.pad_mode, + "pad_token_id": self.model_config.tokenizer.pad_token_id, + } + + train_time = 0 + for epoch in range(start_epoch, self.config.trainer.total_epochs): + self.train_sampler.set_epoch(epoch=epoch) + + for step_in_epoch, data in enumerate( + tqdm( + self.train_dataloader, + initial=global_step % self.steps_per_epoch if epoch == start_epoch else 0, + total=self.steps_per_epoch, + desc=f"Epoch {epoch + 1}/{self.config.trainer.total_epochs}", + disable=not is_logging, + ) + ): + global_step += 1 + + # construct tensordict + data = tu.get_tensordict(tensor_dict=data, non_tensor_dict=meta_info) + + with self.engine.train_mode(): + with Timer(name="update_policy", logger=None) as timer: + output = self.engine.train_batch(data=data, loss_function=self.loss_fn) + lr = self.engine.lr_scheduler_step() + + if self.engine.is_mp_src_rank_with_outputs(): + metrics = output["metrics"] + + loss = torch.mean(torch.tensor(metrics["loss"], device=self.device_name)) + + # mean over dp group + is_nested = data["input_ids"].is_nested + if is_nested: + batch_seqlens: torch.Tensor = data["input_ids"].offsets().diff() + else: + batch_seqlens: torch.Tensor = data["attention_mask"].sum(dim=-1) + batch_seqlens = batch_seqlens.to(self.device_name) # (global_bsz // dp) + + output_tensor = torch.randint( + 0, + 100, + (batch_seqlens.shape[0] * self.engine.get_data_parallel_size(),), + device=self.device_name, + ) # (global_bsz,) + + torch.distributed.all_gather_into_tensor( + output_tensor=output_tensor, + input_tensor=batch_seqlens, + group=self.engine.get_data_parallel_group(), + ) + torch.distributed.all_reduce( + loss, op=torch.distributed.ReduceOp.AVG, group=self.engine.get_data_parallel_group() + ) + + batch_seqlens = output_tensor.tolist() + loss = loss.item() + + # TODO: we can actual accumulate metrics for N steps and perform aggregate metrics + metrics["loss"] = loss + metrics["train/loss"] = metrics.pop("loss") + metrics["train/grad_norm"] = metrics.pop("grad_norm") + metrics["train/lr"] = lr + metrics["train/global_tokens"] = output_tensor.sum().item() + # mfu + delta_time = timer.last + estimated_flops, promised_flops = self.flops_counter.estimate_flops(batch_seqlens, delta_time) + metrics["train/mfu"] = estimated_flops / promised_flops / torch.distributed.get_world_size() + + if self.engine.get_data_parallel_rank() == 0: + tracking.log(data=metrics, step=global_step) + + is_last_step = global_step >= self.total_training_steps + is_valid_step = global_step % self.test_freq == 0 + is_save_step = global_step % self.save_freq == 0 + + # early exit or validation step + if is_last_step or (self.test_freq > 0 and is_valid_step): + # Perform validation + val_losses = [] + for val_data in self.val_dataloader: + with self.engine.eval_mode(): + # construct tensordict + val_data = tu.get_tensordict(tensor_dict=val_data, non_tensor_dict=meta_info) + output = self.engine.infer_batch(data=val_data, loss_function=self.loss_fn) + if self.engine.is_mp_src_rank_with_outputs(): + val_losses.extend(output["metrics"]["loss"]) + + if self.engine.is_mp_src_rank_with_outputs(): + val_loss = torch.mean(torch.tensor(val_losses, device=self.device_name)) + # average over data parallel group + torch.distributed.all_reduce( + val_loss, op=torch.distributed.ReduceOp.AVG, group=self.engine.get_data_parallel_group() + ) + + if is_logging: + metric = {"val/loss": val_loss.detach().item()} + tracking.log(data=metric, step=global_step) + last_valid_metric = metric + torch.distributed.barrier() + + if is_last_step or (self.save_freq > 0 and is_save_step): + self.ckpt_handler.save_checkpoint(step=global_step) + + if is_last_step: + if is_logging: + print(f"Total time for train steps: {train_time:.2f}s") + print(f"Final validation metrics: {last_valid_metric}") + return + + +def run_sft(config): + from verl.utils.distributed import initialize_global_process_group + + initialize_global_process_group() + trainer = SFTTrainer(config=config) + trainer.fit() + destroy_global_process_group() + + +@hydra.main(config_path="config", config_name="sft_trainer_engine", version_base=None) +def main(config): + run_sft(config) + + +def create_sft_dataset(data_paths, data_config, tokenizer): + """Create a dataset.""" + # build dataset + # First check if a custom dataset class is specified + if data_config.custom_cls.get("path", None): + from verl.utils.import_utils import load_extern_type + + dataset_cls = load_extern_type(data_config.custom_cls.path, data_config.custom_cls.name) + else: + # Default to multi-turn dataset + dataset_cls = MultiTurnSFTDataset + + # Create datasets based on the selected class + dataset = dataset_cls(parquet_files=data_paths, tokenizer=tokenizer, config=data_config) + return dataset + + +if __name__ == "__main__": + main() diff --git a/e2c/verl/verl/utils/__init__.py b/e2c/verl/verl/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc40ffb32e13ad3036c9d87655c949056ab786c1 --- /dev/null +++ b/e2c/verl/verl/utils/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import config, tokenizer +from .config import omega_conf_to_dataclass, validate_config +from .groupwise import as_torch_index, group_mean_std +from .tokenizer import hf_processor, hf_tokenizer + +__all__ = ( + tokenizer.__all__ + + config.__all__ + + ["hf_processor", "hf_tokenizer", "omega_conf_to_dataclass", "validate_config"] + + ["as_torch_index", "group_mean_std"] +) diff --git a/e2c/verl/verl/utils/activation_offload.py b/e2c/verl/verl/utils/activation_offload.py new file mode 100644 index 0000000000000000000000000000000000000000..73e2e83eb3b2d06cd7589ce6ee8084e82dd398b3 --- /dev/null +++ b/e2c/verl/verl/utils/activation_offload.py @@ -0,0 +1,558 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Functionality for CPU offloading of tensors saved for backward pass.""" + +from __future__ import annotations + +import functools +import logging +import os +from typing import Any, Optional + +import torch +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + +from verl.utils.device import get_torch_device +from verl.utils.fsdp_utils import FSDPModule as FSDP2 + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +def _get_unique_tensor_key(tensor): + key = (tensor.untyped_storage().data_ptr() + tensor.storage_offset(), tensor.dtype) + return key + + +class FSDPParameterFilter: + def __init__(self): + self.model_parameters_storage = set() + + def __call__(self, tensor): + return tensor.untyped_storage().data_ptr() not in self.model_parameters_storage + + def update_model_parameters(self, model): + new_storage = set() + for p in model.parameters(): + new_storage.add(p.data.untyped_storage().data_ptr()) + self.model_parameters_storage = new_storage + + +class CpuOffloadHookWithOffloadHandler: + """Context-manager that offloads/recovers tensors through an offload hander. + + The hook just offloads/recovers the tensor object to the handler through `tensor_push` + and `tensor_pop` interface. How the offload-handler manages the offloading, recovering + or prefetching timing is transparent to this hook. + """ + + def __init__( + self, + offload_handler: OffloadHandler, + handler_extra_kwargs: Optional[dict[str, Any]] = None, + ) -> None: + if handler_extra_kwargs is None: + handler_extra_kwargs = {} + self.offload_handler: OffloadHandler = offload_handler + self.handler_extra_kwargs: dict[str, Any] = handler_extra_kwargs + self.inside_context = False + + def __enter__(self): + self.inside_context = True + torch._C._autograd._push_saved_tensors_default_hooks(self.on_save_for_backward, self.on_get_saved_tensor) + + def __exit__(self, *args: Any): + self.inside_context = False + torch._C._autograd._pop_saved_tensors_default_hooks() + + def on_save_for_backward(self, tensor: torch.Tensor) -> Any: + retrieve_identifier = self.offload_handler.tensor_push(tensor, **self.handler_extra_kwargs) + return retrieve_identifier + + def on_get_saved_tensor(self, saved_state: Any) -> torch.Tensor: + tensor = self.offload_handler.tensor_pop(saved_state, **self.handler_extra_kwargs) + return tensor + + +class OffloadHandler: + """A base class for CPU offload-handler.""" + + def __init__(self) -> None: + pass + + def tensor_push(self, tensor: torch.Tensor, **kwargs) -> Any: + """Tensor push.""" + raise NotImplementedError( + "`tensor_push is not implented in OffloadHandler class. Inherit this class and implement your " + "custom tensor_push." + ) + + def tensor_pop(self, tensor_tag: Any, **kwargs): + """Tensor pop.""" + raise NotImplementedError( + "`tensor_pop is not implented in OffloadHandler class. Inherit this class and implement your " + "custom tensor_pop." + ) + + +class GroupCommitFunction(torch.autograd.Function): + """this is a dummy op with output identical to input. + However, it is necessary for marking a timepoint for offload handler to + accomplish all synchronizations. Implementing it as a function is necessary + because we need to actions in both forward and backward. + """ + + @staticmethod + def forward(ctx, tensor, cpu_offload_handler): + # pylint: disable=missing-function-docstring + cpu_offload_handler.on_group_commit_forward() + ctx.cpu_offload_handler = cpu_offload_handler + # return the identical tensor + return tensor + + @staticmethod + def backward(ctx, grad_output): + # pylint: disable=missing-function-docstring + cpu_offload_handler = ctx.cpu_offload_handler + cpu_offload_handler.on_group_commit_backward() + return grad_output, None + + +group_prefetch_offload_commit = GroupCommitFunction.apply + + +class SynchronizedGroupOffloadHandler(OffloadHandler): + """Offload Handler that offloads/reloads in a synchronized way. + The device-to-host and host-to-device copying happen in the same stream + as the computation kernels, thus the copying will block computation. + """ + + def __init__(self, num_offload_group, tensor_need_offloading_checker=(lambda _: True)) -> None: + super().__init__() + + self.num_offload_group = num_offload_group + self.tensor_need_offloading_checker = tensor_need_offloading_checker + + self.groupid_reset() + + def groupid_reset(self): + """Groupid reset.""" + # Data structures to label saved tensors and book-keep their cpu copies. + # Currently, on push, create a new cpu tensor and copies; on pop, copies + # the tensor back to gpu and deletes the cpu tensor. + # These will increment whenever `group_commit()` is invoked + self.current_group, self.tensor_count_current_group = (0, 0) + self.torch_tensor_count = 0 + self.tensor_tag_to_state = {} + + def on_group_commit_forward(self): + """On group commit forward.""" + # finishing up with updating current group and tensor count + self.current_group += 1 # increment + self.tensor_count_current_group = 0 # reset + + def on_group_commit_backward(self): + """On group commit backward.""" + self.current_group -= 1 + assert self.current_group >= 0 + + @staticmethod + def offload(src_tensor, pin_memory=True): + """Offload.""" + + cpu_backup = torch.empty( + src_tensor.size(), + dtype=src_tensor.dtype, + layout=src_tensor.layout, + device="cpu", + pin_memory=pin_memory, + ) + cpu_backup.copy_(src_tensor, non_blocking=True) + state = (src_tensor.device, cpu_backup) + return state + + @staticmethod + def reload(state, non_blocking=None): + """Reload.""" + dev, cpu_backup = state + if non_blocking is None: + non_blocking = cpu_backup.is_pinned() + return cpu_backup.to(dev, non_blocking=non_blocking) + + def tensor_push(self, tensor: torch.Tensor, **kwargs): + """Tensor push.""" + # obtain a unique tensor tag + tensor_tag = (self.current_group, self.tensor_count_current_group) + self.tensor_count_current_group += 1 + assert tensor_tag not in self.tensor_tag_to_state + if self.current_group < self.num_offload_group and self.tensor_need_offloading_checker(tensor): + state = SynchronizedGroupOffloadHandler.offload(tensor) + self.tensor_tag_to_state[tensor_tag] = state + else: + # will be offloaded together after group commit + self.tensor_tag_to_state[tensor_tag] = tensor + + return tensor_tag + + def tensor_pop(self, tensor_tag, **kwargs): + """Tensor pop.""" + assert tensor_tag in self.tensor_tag_to_state + state = self.tensor_tag_to_state.pop(tensor_tag) + if isinstance(state, tuple): + tensor = SynchronizedGroupOffloadHandler.reload(state) + else: + tensor = state + return tensor + + +class AsyncDoubleBufferGroupOffloadHandler(SynchronizedGroupOffloadHandler): + """Compared to synchronize, this uses more memory because of the buffer but + achieves better performance due to the overlapping. D2h and h2d copying are + completely hidden behind computation if computation time of a layer is longer + than host-device communication time. Bulk offloading with delay and bulk reloading + with prefetch are implemented.""" + + def __init__( + self, + num_offload_group, # must be <= actual number of groups (number of commits) + num_model_group, + tensor_need_offloading_checker=(lambda t: True), + ) -> None: + super().__init__( + num_offload_group=num_offload_group, + tensor_need_offloading_checker=tensor_need_offloading_checker, + ) + # Number of layers in the model + self.num_layers = num_model_group + # Data Structure to maintain reference to activation tensors + self.tensor_tag_to_buf = {} + # Tracking the number of layers offloaded + self.offloaded_group_count = 0 + # Core data structure that decides the window for offloading + self.layer_window_map = {} + self.group_offload_mapping = {} + + # Logic to make offloading load balance across computation + # for optimal CPU/GPU interconnect usage + constant = 0 + for i in range(self.num_offload_group): + self.layer_window_map[i] = ((self.num_layers // self.num_offload_group) * (i + 1)) - 1 + if i < (self.num_layers % self.num_offload_group): + self.layer_window_map[i] += i + 1 + constant = i + 1 + else: + self.layer_window_map[i] += constant + + # allocate streams and events for synchronization + self.d2h_stream = get_torch_device().Stream() + self.h2d_stream = get_torch_device().Stream() + + def tensor_push(self, tensor: torch.Tensor, **kwargs) -> Any: + torch_stray_tensor = isinstance( + tensor, + torch._subclasses.fake_tensor.FakeTensor | torch._subclasses.functional_tensor.FunctionalTensor, + ) + need_offload = not torch_stray_tensor + need_offload = need_offload and self.tensor_need_offloading_checker(tensor) + + if need_offload: + # obtain a unique tensor tag + tensor_tag = (self.current_group, self.tensor_count_current_group) + self.tensor_count_current_group += 1 + + assert tensor_tag not in self.tensor_tag_to_state + self.tensor_tag_to_state[tensor_tag] = tensor + + if self.current_group < self.num_offload_group: + self.tensor_tag_to_buf[tensor_tag] = tensor + else: + tensor_tag = tensor + return tensor_tag + + def tensor_pop(self, tensor_tag, **kwargs): + """Tensor pop.""" + if isinstance(tensor_tag, torch.Tensor): + return tensor_tag + assert tensor_tag in self.tensor_tag_to_state + tensor = self.tensor_tag_to_state.pop(tensor_tag) + self.tensor_tag_to_buf.pop(tensor_tag, None) + + # the tensor should have been copied back in on_group_commit_backward() + # which invokes bulk_reload_group. + assert not isinstance(tensor, tuple) + return tensor + + def bulk_offload_group(self, group_to_offload): + """Bulk offload group.""" + offload_mapping = {} + offload_size = 0 + with get_torch_device().stream(self.d2h_stream): + for tensor_tag, state in self.tensor_tag_to_state.items(): + group_id, _ = tensor_tag + if group_id == group_to_offload: + assert not isinstance(state, tuple) + key = _get_unique_tensor_key(state) + if key not in offload_mapping: + offload_mapping[key] = state + # if offload, return the reference to cpu copy + self.tensor_tag_to_state[tensor_tag] = (key, state.shape) + for key, tensor in offload_mapping.items(): + state = SynchronizedGroupOffloadHandler.offload(tensor) + offload_size += tensor.numel() * tensor.element_size() + offload_mapping[key] = state + + self.group_offload_mapping[group_to_offload] = offload_mapping + + def synchronize_on_group_commit_forward(self, current_group): + """Synchronize on group commit forward.""" + + # For the first group, kickstart the offload after we have + # the first compute completion + if current_group == 0: + self.d2h_stream.wait_stream(get_torch_device().current_stream()) + self.bulk_offload_group(current_group) + + # Window map data structure helps us synchronize based on number + # of layers offloaded + if self.layer_window_map[self.offloaded_group_count] == current_group: + # Stream synchronization both ways + self.d2h_stream.wait_stream(get_torch_device().current_stream()) + get_torch_device().current_stream().wait_stream(self.d2h_stream) + + # Time to free the activation memory after usage + for tensor_tag, _ in self.tensor_tag_to_buf.items(): + if tensor_tag[0] == self.offloaded_group_count: + self.tensor_tag_to_buf[tensor_tag] = None + + # Time to offload the next group + if self.offloaded_group_count < (self.num_offload_group - 1): + self.bulk_offload_group(self.offloaded_group_count + 1) + + # Increment the offload group count to keep track + self.offloaded_group_count += 1 + + def on_group_commit_forward(self): + """This function will cause host device synchronization""" + # handle synchronization events + self.synchronize_on_group_commit_forward(self.current_group) + + super().on_group_commit_forward() + + @torch.no_grad + def bulk_reload_group(self, group_to_reload): + """Bulk reload group.""" + assert group_to_reload < self.num_offload_group + + with get_torch_device().stream(self.h2d_stream): + # move back tensors + offload_mapping = self.group_offload_mapping.pop(group_to_reload) + assert offload_mapping is not None + for key, state in offload_mapping.items(): + offload_mapping[key] = SynchronizedGroupOffloadHandler.reload(state) + for tensor_label, state in self.tensor_tag_to_state.items(): + group_id, _ = tensor_label + if group_id == group_to_reload and not isinstance(state, torch.Tensor): + assert isinstance(state, tuple), f"{group_id} {state}" + key, shape = state + recovered_tensor = offload_mapping[key].view(shape) + self.tensor_tag_to_state[tensor_label] = recovered_tensor + + def on_group_commit_backward(self): + # first decrement the current group. + # after last commit in forward, the group will +1; in backward it -1. + # Finally it should be decremented to 0. + self.current_group -= 1 + assert self.current_group >= 0 + + # Layer window data structure helps us to reload at right times + if self.layer_window_map[self.offloaded_group_count - 1] == self.current_group: + # Stream synchronization both ways + self.h2d_stream.wait_stream(get_torch_device().current_stream()) + get_torch_device().current_stream().wait_stream(self.h2d_stream) + + # Time to reload the next group + self.bulk_reload_group(self.offloaded_group_count - 1) + + # Decrease the offloading group counter + self.offloaded_group_count -= 1 if self.offloaded_group_count > 1 else 0 + + # Last group computation needs to wait till all the reloads complete + if self.current_group == 0: + get_torch_device().current_stream().wait_stream(self.h2d_stream) + self.offloaded_group_count = 0 + + +def get_activation_offload_context( + num_layers: int = 1, model_layers: int = 1, tensor_need_offloading_checker=(lambda t: True) +): + cpu_offload_handler = AsyncDoubleBufferGroupOffloadHandler( + num_offload_group=num_layers, + num_model_group=model_layers, + tensor_need_offloading_checker=tensor_need_offloading_checker, + ) + + def group_prefetch_offload_commit_async(tensor): + return group_prefetch_offload_commit(tensor, cpu_offload_handler) + + return ( + CpuOffloadHookWithOffloadHandler(offload_handler=cpu_offload_handler), + group_prefetch_offload_commit_async, + ) + + +class ActivationHandler: + def __init__(self, offload_ctx, sync_func, tensor_filter, enable_ckpt): + self._offload_ctx = offload_ctx + self._sync_func = sync_func + self._enable_ckpt = enable_ckpt + self._tensor_filter = tensor_filter + if enable_ckpt: + self.checkpoint_fn = functools.partial( + torch.utils.checkpoint.checkpoint, + use_reentrant=True, + ) + + def pre_forward(self, module): + if module.training: + self._offload_ctx.__enter__() + self._tensor_filter.update_model_parameters(module) + + def post_forward(self, module): + if module.training: + self._offload_ctx.__exit__(None, None, None) + + def _pack_kwargs(self, *args, **kwargs): + kwarg_keys = [] + flat_args = list(args) + for k, v in kwargs.items(): + kwarg_keys.append(k) + flat_args.append(v) + + return tuple(flat_args), tuple(kwarg_keys) + + def _unpack_kwargs(self, flat_args, kwarg_keys): + assert len(kwarg_keys) <= len(flat_args), f"too many keys {len(kwarg_keys)} vs. {len(flat_args)}" + if len(kwarg_keys) == 0: + return flat_args, {} + args = flat_args[: -len(kwarg_keys)] + kwargs = dict(zip(kwarg_keys, flat_args[-len(kwarg_keys) :], strict=True)) + return args, kwargs + + def _ckpt_forward(self, forward_method, *args, **kwargs): + flat_args, kwarg_keys = self._pack_kwargs(*args, **kwargs) + + def my_function(*inputs): + # unpack back into args and kwargs + nonlocal forward_method, kwarg_keys + unpacked_args, unpacked_kwargs = self._unpack_kwargs(inputs, kwarg_keys) + # run original module + return forward_method(*unpacked_args, **unpacked_kwargs) + + return self.checkpoint_fn( + my_function, + *flat_args, + ) + + def forward(self, module, forward_method, *args, **kwargs): + if not module.training: + return forward_method(*args, **kwargs) + if not self._enable_ckpt: + ret = forward_method(*args, **kwargs) + else: + ret = self._ckpt_forward(forward_method, *args, **kwargs) + binded_tensor = ret + if isinstance(ret, tuple): + binded_tensor = ret[0] + binded_tensor = self._sync_func(binded_tensor) + final_ret = binded_tensor + if isinstance(ret, tuple): + final_ret = (final_ret,) + ret[1:] + return final_ret + + def wrap_module_forward_method(self, module): + orig_method = module.forward + handler = self + + @functools.wraps(orig_method) + def wrapped_method(model_self, *args, **kwargs): + nonlocal handler + handler.pre_forward(model_self) + out = handler.forward(model_self, orig_method, *args, **kwargs) + handler.post_forward(model_self) + return out + + module.forward = wrapped_method.__get__(module, type(module)) + + +def enable_activation_offloading(model, strategy, enable_ckpt=False): + """ + Enable activation offloading for the model. It groups activations by TransformerLayer and offloads activation + groups asynchronously. This means that the offloading of the i-th activation group and the computation of the i+1-th + activation group happen at the same time, and there are at most two activation groups in GPU memory. + + Args: + model: the model to enable activation offloading + strategy: the training strategy of the model, such as "fsdp" + enable_ckpt: whether activation checkpointing(also called gradient checkpointing) has been enabled for the model + + Note: + For best efficiency, activation offloading is usually combined with activation checkpointing. However, this + implementation of activation offloading is conflicted with the implementation of activation checkpointing in + some training strategies. This function resolves this conflict, and therefore requires the "strategy" and + "enable_ckpt" arguments. + + Returns: + + """ + + assert strategy == "fsdp" or strategy == "fsdp2", "activation offloading only supports fsdp strategy" + layers = [] + + def get_layers(module): + for name, child in module.named_children(): + if not isinstance(child, FSDP | FSDP2): + get_layers(child) + else: + wrapped_module = child + if isinstance(child, FSDP): + wrapped_module = child._fsdp_wrapped_module + # In some cases, torch.nn.Embedding is wrapped with FSDP alone. However, the activation + # size of torch.nn.Embedding is small, so it's not necessary to offload it. + if not isinstance(wrapped_module, torch.nn.Embedding): + layers.append(child) + + get_layers(model) + if len(layers) < 3: + logger.warning(f"Find only {len(layers)} fsdp layers, not neccessary to enable async activation offloading") + return + + tensor_filter = FSDPParameterFilter() + context, sync_func = get_activation_offload_context(len(layers) - 1, len(layers), tensor_filter) + if enable_ckpt: + # The implementation of activation checkpointing in transformers library is incompatible with + # activation offloading, + # so it will be disabled, but this implementation supports another version of activation checkpointing, so that + # these two features can be enabled at the same time. + for module in model.modules(): + if hasattr(module, "gradient_checkpointing_disable"): + module.gradient_checkpointing_disable() + + handler = ActivationHandler(context, sync_func, tensor_filter, enable_ckpt) + for layer in layers: + module = layer + if isinstance(layer, FSDP): + module = module._fsdp_wrapped_module + handler.wrap_module_forward_method(module) diff --git a/e2c/verl/verl/utils/attention_utils.py b/e2c/verl/verl/utils/attention_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b0c4f7247db49f437bf869585c6519835317c520 --- /dev/null +++ b/e2c/verl/verl/utils/attention_utils.py @@ -0,0 +1,113 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable + +_index_first_axis, _pad_input, _rearrange, _unpad_input = None, None, None, None + + +def _get_attention_functions() -> tuple[Callable, Callable, Callable, Callable]: + """Dynamically import attention functions based on available hardware.""" + + from verl.utils.device import is_cuda_available, is_npu_available + + global _index_first_axis, _pad_input, _rearrange, _unpad_input + + if is_cuda_available: + from flash_attn.bert_padding import index_first_axis, pad_input, rearrange, unpad_input + elif is_npu_available: + try: + from transformers.integrations.npu_flash_attention import ( + index_first_axis, + pad_input, + rearrange, + unpad_input, + ) + except ImportError: + # Since transformers v4.55.1, index_first_axis, pad_input, and unpad_input + # have been consolidated into `transformers.modeling_flash_attention_utils`. + from einops import rearrange + from transformers.modeling_flash_attention_utils import _index_first_axis as index_first_axis + from transformers.modeling_flash_attention_utils import _pad_input as pad_input + from transformers.modeling_flash_attention_utils import _unpad_input as unpad_input + + _index_first_axis, _pad_input, _rearrange, _unpad_input = index_first_axis, pad_input, rearrange, unpad_input + + return _index_first_axis, _pad_input, _rearrange, _unpad_input + + +def index_first_axis(*args, **kwargs): + """ + Unified entry point for `index_first_axis` across CUDA and NPU backends. + + Dynamically dispatches to the appropriate device-specific implementation: + - On CUDA: `flash_attn.bert_padding.index_first_axis` + - On NPU: `transformers.integrations.npu_flash_attention.index_first_axis` + (falls back to `transformers.modeling_flash_attention_utils._index_first_axis` + in newer versions of transformers). + + Users can call this function directly without worrying about the underlying device. + """ + func, *_ = _get_attention_functions() + return func(*args, **kwargs) + + +def pad_input(*args, **kwargs): + """ + Unified entry point for `pad_input` across CUDA and NPU backends. + + Dynamically dispatches to the appropriate device-specific implementation: + - On CUDA: `flash_attn.bert_padding.pad_input` + - On NPU: `transformers.integrations.npu_flash_attention.pad_input` + (falls back to `transformers.modeling_flash_attention_utils._pad_input` + in newer versions of transformers). + + Users can call this function directly without worrying about the underlying device. + """ + _, func, *_ = _get_attention_functions() + return func(*args, **kwargs) + + +def rearrange(*args, **kwargs): + """ + Unified entry point for `rearrange` across CUDA and NPU backends. + + Dynamically dispatches to the appropriate device-specific implementation: + - On CUDA: `flash_attn.bert_padding.rearrange` + - On NPU: `transformers.integrations.npu_flash_attention.rearrange` + (falls back to `einops.rearrange` if no dedicated NPU implementation exists). + + Users can call this function directly without worrying about the underlying device. + """ + *_, func, _ = _get_attention_functions() + return func(*args, **kwargs) + + +def unpad_input(*args, **kwargs): + """ + Unified entry point for `unpad_input` across CUDA and NPU backends. + + Dynamically dispatches to the appropriate device-specific implementation: + - On CUDA: `flash_attn.bert_padding.unpad_input` + - On NPU: `transformers.integrations.npu_flash_attention.unpad_input` + (falls back to `transformers.modeling_flash_attention_utils._unpad_input` + in newer versions of transformers). + + Users can call this function directly without worrying about the underlying device. + """ + *_, func = _get_attention_functions() + return func(*args, **kwargs) + + +__all__ = ["index_first_axis", "pad_input", "rearrange", "unpad_input"] diff --git a/e2c/verl/verl/utils/checkpoint/__init__.py b/e2c/verl/verl/utils/checkpoint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f10b6c49eeb98ab50f8c37b5f2e72b56dbef4406 --- /dev/null +++ b/e2c/verl/verl/utils/checkpoint/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .checkpoint_handler import CheckpointHandler + +__all__ = ["CheckpointHandler"] diff --git a/e2c/verl/verl/utils/checkpoint/checkpoint_handler.py b/e2c/verl/verl/utils/checkpoint/checkpoint_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..43a9cfbb69418d9de56fd0c1cde2e089c4bd32af --- /dev/null +++ b/e2c/verl/verl/utils/checkpoint/checkpoint_handler.py @@ -0,0 +1,204 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# TODO: add unit tests + +import logging +import os +import re + +import torch + +import verl.utils.hdfs_io as hdfs_io +from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path, get_checkpoint_tracker_filename +from verl.utils.logger import log_with_rank +from verl.workers.engine import BaseEngine + + +def extract_step(path): + match = re.search(r"global_step_(\d+)", path) + if match: + return int(match.group(1)) + return None + + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_SFT_LOGGING_LEVEL", "WARN")) + + +class CheckpointHandler: + """ + Checkpoint handler handles the path, global_step of a checkpoint folder. + Currently, it only works with a single model. + We can expand it to support multiple models. It is expected to be used with SPMD style (e.g., torchrun) + """ + + def __init__( + self, + engine: BaseEngine, + train_dataloader, + *, + default_local_dir, + max_ckpt_to_keep=None, + default_hdfs_dir=None, + resume_mode="auto", + resume_from_path=None, + ): + self.default_local_dir = default_local_dir + self.max_ckpt_to_keep = max_ckpt_to_keep + self.default_hdfs_dir = default_hdfs_dir + self.resume_mode = resume_mode + self.resume_from_path = resume_from_path + self.engine = engine + self.train_dataloader = train_dataloader + self.rank = torch.distributed.get_rank() + + def save_checkpoint(self, step): + """Save checkpoint using FSDPCheckpointManager with improved tracking""" + from verl.utils.fs import local_mkdir_safe + + # Determine checkpoint path + local_global_step_folder = os.path.join(self.default_local_dir, f"global_step_{step}") + if self.rank == 0: + print(f"Saving checkpoint to: {local_global_step_folder}") + + # Get max checkpoints to keep + max_ckpt_to_keep = self.max_ckpt_to_keep + + # Use checkpoint manager to save + self.engine.save_checkpoint( + local_path=local_global_step_folder, global_step=step, max_ckpt_to_keep=max_ckpt_to_keep + ) + + # Save dataloader state. Note that we only save the iterator in the train_dataloader. + # So it's identical in each dp rank. + if self.engine.is_mp_src_rank_with_outputs(): + dp_rank = self.engine.get_data_parallel_rank() + local_mkdir_safe(local_global_step_folder) + dataloader_local_path = os.path.join(local_global_step_folder, f"data_{dp_rank}.pt") + + # Use StatefulDataLoader's built-in state dict functionality + dataloader_state_dict = self.train_dataloader.state_dict() + torch.save(dataloader_state_dict, dataloader_local_path) + print(f"Saved dataloader state to: {dataloader_local_path}") + + if self.rank == 0: + # Update latest checkpoint tracker (atomic write) + tracker_file = get_checkpoint_tracker_filename(self.default_local_dir) + temp_tracker_file = tracker_file + ".tmp" + with open(temp_tracker_file, "w") as f: + f.write(str(step)) + os.rename(temp_tracker_file, tracker_file) + print(f"Updated checkpoint tracker: {tracker_file}") + + # Copy to HDFS if configured + if self.rank == 0 and self.default_hdfs_dir: + hdfs_io.makedirs(self.default_hdfs_dir, exist_ok=True) + hdfs_io.copy(src=local_global_step_folder, dst=self.default_hdfs_dir, dirs_exist_ok=True) + + torch.distributed.barrier() + + def load_checkpoint(self): + # Determine resume path based on configuration + checkpoint_path = self._determine_resume_path() + + if checkpoint_path is None: + return 0 + + # extract resume step from checkpoint path + resume_step = extract_step(checkpoint_path) + if resume_step is None: + log_with_rank( + f"Warning: Could not extract step number from {checkpoint_path}, starting from step 0", + logger=logger, + rank=self.rank, + level=logging.WARNING, + log_only_rank_0=True, + ) + return 0 + self.resume_global_step = resume_step + + # Use checkpoint manager to load model state + self.engine.load_checkpoint(checkpoint_path) + # Always load dataloader state for StatefulDataLoader + self._load_dataloader_state(checkpoint_path) + + return resume_step + + def _load_dataloader_state(self, checkpoint_path: str): + """Load dataloader state from checkpoint""" + dp_rank = self.engine.get_data_parallel_rank() + dataloader_path = os.path.join(checkpoint_path, f"data_{dp_rank}.pt") + + if os.path.exists(dataloader_path): + # Use StatefulDataLoader's built-in state dict functionality + dataloader_state_dict = torch.load(dataloader_path, map_location="cpu", weights_only=False) + self.train_dataloader.load_state_dict(dataloader_state_dict) + + log_with_rank( + f"Successfully loaded dataloader state from {dataloader_path}", + logger=logger, + rank=self.rank, + log_only_rank_0=True, + ) + + else: + log_with_rank( + f"Warning: No dataloader state found at {dataloader_path}, will start from scratch", + logger=logger, + rank=self.rank, + level=logging.WARNING, + log_only_rank_0=True, + ) + + def _determine_resume_path(self): + """Determine the path to resume from based on resume_mode configuration""" + resume_mode = self.resume_mode + resume_from_path = self.resume_from_path + + if resume_mode == "disable": + return None + elif resume_mode == "auto": + if resume_from_path is not None: + assert os.path.exists(resume_from_path), ( + "resume_from_path must be null or an existing path when resume_mode is 'auto'" + ) + assert "global_step_" in resume_from_path, "resume_from_path must specify the global_steps" + return resume_from_path + # Try to find the latest checkpoint in the default directory + return self._find_latest_checkpoint() + elif resume_mode == "resume_path": + assert os.path.exists(resume_from_path), ( + "resume_from_path must be an existing path when resume_mode is 'resume_path'" + ) + assert "global_step_" in resume_from_path, "resume_from_path must specify the global_steps" + return resume_from_path + else: + raise ValueError(f"Invalid resume_mode: {resume_mode}. Must be 'auto', 'disable', or 'resume_path'") + + def _find_latest_checkpoint(self): + """Find the latest checkpoint in the default local directory""" + checkpoint_dir = self.default_local_dir + + if not os.path.exists(checkpoint_dir): + return None + + latest_checkpoint = find_latest_ckpt_path(checkpoint_dir) + + if latest_checkpoint and self.rank == 0: + step_num = extract_step(latest_checkpoint) + print(f"Found latest checkpoint: {latest_checkpoint} (step {step_num})") + + return latest_checkpoint diff --git a/e2c/verl/verl/utils/checkpoint/checkpoint_manager.py b/e2c/verl/verl/utils/checkpoint/checkpoint_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..74fc83ecc3c2e51c3c08a971f02a0d3273456885 --- /dev/null +++ b/e2c/verl/verl/utils/checkpoint/checkpoint_manager.py @@ -0,0 +1,237 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random +import shutil + +import numpy as np +import torch +import torch.distributed +from omegaconf import DictConfig +from transformers import PreTrainedTokenizer, ProcessorMixin + +from verl.trainer.config import CheckpointConfig +from verl.utils.device import get_device_name, get_torch_device + + +class BaseCheckpointManager: + """ + A checkpoint manager that saves and loads the following states in a SPMD way: + - model + - optimizer + - lr_scheduler + - extra_states + + We save + - sharded model states and optimizer states + - full lr_scheduler states + - huggingface tokenizer and config for ckpt merge + """ + + def __init__( + self, + model, + optimizer: torch.optim.Optimizer, + lr_scheduler: torch.optim.lr_scheduler.LRScheduler = None, + processing_class: PreTrainedTokenizer | ProcessorMixin = None, + checkpoint_config: DictConfig | CheckpointConfig = None, + ): + self.checkpoint_config = checkpoint_config + checkpoint_load_contents = checkpoint_config.get("load_contents", None) if checkpoint_config else None + checkpoint_save_contents = checkpoint_config.get("save_contents", None) if checkpoint_config else None + if checkpoint_load_contents is None: + checkpoint_load_contents = ["model", "optimizer", "extra"] + if checkpoint_save_contents is None: + checkpoint_save_contents = ["model", "optimizer", "extra"] + self.previous_global_step = None + self.previous_saved_paths = [] + + self.model = model + self.optimizer = optimizer + self.lr_scheduler = lr_scheduler + self.processing_class = processing_class + self.checkpoint_load_contents = checkpoint_load_contents + self.checkpoint_save_contents = checkpoint_save_contents + + self.rank = torch.distributed.get_rank() + self.world_size = torch.distributed.get_world_size() + + @property + def should_save_model(self) -> bool: + """ + Returns True if 'model' is in checkpoint_save_contents, indicating the model state should be saved. + """ + return "model" in self.checkpoint_save_contents + + @property + def should_save_optimizer(self) -> bool: + """ + Returns True if 'optimizer' is in checkpoint_save_contents, indicating the optimizer state should be saved. + """ + return "optimizer" in self.checkpoint_save_contents + + @property + def should_save_extra(self) -> bool: + """ + Returns True if 'extra' is in checkpoint_save_contents, indicating the extra state should be saved. + """ + return "extra" in self.checkpoint_save_contents + + @property + def should_save_hf_model(self) -> bool: + """ + Returns True if 'hf_model' is in checkpoint_save_contents, indicating the model should be converted to hf + model and saved. + """ + return "hf_model" in self.checkpoint_save_contents + + @property + def should_load_model(self) -> bool: + """ + Returns True if 'model' is in checkpoint_load_contents, indicating the model state should be loaded. + """ + return "model" in self.checkpoint_load_contents + + @property + def should_load_optimizer(self) -> bool: + """ + Returns True if 'optimizer' is in checkpoint_load_contents, indicating the optimizer state should be loaded. + """ + return "optimizer" in self.checkpoint_load_contents + + @property + def should_load_extra(self) -> bool: + """ + Returns True if 'extra' is in checkpoint_load_contents, indicating the extra state should be loaded. + """ + return "extra" in self.checkpoint_load_contents + + def load_checkpoint(self, local_path: str, hdfs_path: str = None, del_local_after_load: bool = False): + raise NotImplementedError + + def save_checkpoint( + self, local_path: str, hdfs_path: str = None, global_step: int = 0, max_ckpt_to_keep: int = None + ): + raise NotImplementedError + + @staticmethod + def checkpath(local_path: str, hdfs_path: str): + assert local_path is not None or hdfs_path is not None, "local_path and hdfs_path cannot be both None" + return local_path is not None, local_path if local_path is not None else hdfs_path + + def remove_previous_save_local_path(self, path): + if isinstance(path, str): + path = [path] + for p in path: + abs_path = os.path.abspath(p) + print(f"Checkpoint manager remove previous save local path: {abs_path}") + if not os.path.exists(abs_path): + continue + shutil.rmtree(abs_path, ignore_errors=True) + + @staticmethod + def get_rng_state(): + rng_state = { + "cpu": torch.get_rng_state(), + "numpy": np.random.get_state(), + "random": random.getstate(), + } + + if get_device_name() != "cpu": + rng_state[get_device_name()] = get_torch_device().get_rng_state() + + return rng_state + + @staticmethod + def load_rng_state(rng_state): + torch.set_rng_state(rng_state["cpu"]) + np.random.set_state(rng_state["numpy"]) + random.setstate(rng_state["random"]) + + if get_device_name() != "cpu": + get_torch_device().set_rng_state(rng_state[get_device_name()]) + + +def find_latest_ckpt_path(path, directory_format="global_step_{}"): + """ + Return the most recent checkpoint directory based on a tracker file. + + Args: + path (str): Base directory containing the checkpoint tracker. + directory_format (str): Template for checkpoint subfolders with one + placeholder for the iteration number (default "global_step_{}"). + + Returns: + str or None: Full path to the latest checkpoint directory, or + None if the tracker or checkpoint folder is missing. + """ + if path is None: + return None + + tracker_file = get_checkpoint_tracker_filename(path) + if not os.path.exists(tracker_file): + print(f"Checkpoint tracker file does not exist: {tracker_file}") + return None + + with open(tracker_file, "rb") as f: + iteration = int(f.read().decode()) + ckpt_path = os.path.join(path, directory_format.format(iteration)) + if not os.path.exists(ckpt_path): + print("Checkpoint does not exist: %s", ckpt_path) + return None + + print("Found checkpoint: %s", ckpt_path) + return ckpt_path + + +def get_checkpoint_tracker_filename(root_path: str): + """ + Tracker file rescords the latest chckpoint during training to restart from. + """ + return os.path.join(root_path, "latest_checkpointed_iteration.txt") + + +def should_save_ckpt_esi(max_steps_duration: float, save_ckpt_duration: float = 60, redundant_time: float = 0) -> bool: + """ + Determine if checkpoint should be saved based on capacity esi expiration. + + Args: + max_steps_duration: Max estimated time (seconds) required to complete one training step + save_ckpt_duration: Estimated time (seconds) required to save checkpoint (default: 60) + redundant_time: Additional buffer time (seconds) for unexpected delays (default: 0) + """ + exp_ts_mlp = os.getenv("MLP_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP") # vemlp + exp_ts_aws = os.getenv("SAGEMAKER_CURRENT_CAPACITY_BLOCK_EXPIRATION_TIMESTAMP") # aws + if exp_ts_mlp: + try: + import time + + remaining = float(exp_ts_mlp) - time.time() + except ValueError: + return False + return ( + remaining > 0 + and max_steps_duration > 0 + and remaining <= save_ckpt_duration + max_steps_duration + redundant_time + ) + elif exp_ts_aws: + from datetime import datetime, timedelta + + expiration_time = datetime.fromtimestamp(int(exp_ts_aws)) + time_difference = expiration_time - datetime.now() + threshold_minutes = (save_ckpt_duration + max_steps_duration + redundant_time) / 60 + return time_difference < timedelta(minutes=threshold_minutes) + else: + return False diff --git a/e2c/verl/verl/utils/checkpoint/fsdp_checkpoint_manager.py b/e2c/verl/verl/utils/checkpoint/fsdp_checkpoint_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..4b3d7a8e7add0ee2330020aadba17a71c6e20b68 --- /dev/null +++ b/e2c/verl/verl/utils/checkpoint/fsdp_checkpoint_manager.py @@ -0,0 +1,367 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import warnings +from dataclasses import asdict, dataclass +from typing import Optional + +import torch +import torch.distributed +from accelerate import init_empty_weights +from omegaconf import DictConfig +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import ShardedOptimStateDictConfig, ShardedStateDictConfig, StateDictType +from transformers import GenerationConfig, PreTrainedTokenizer, ProcessorMixin +from transformers.dynamic_module_utils import custom_object_save + +from verl.utils.device import is_cuda_available +from verl.utils.fs import copy_to_local, is_non_local, local_mkdir_safe +from verl.utils.fsdp_utils import fsdp_version, get_fsdp_full_state_dict, get_fsdp_state_ctx +from verl.utils.logger import log_with_rank + +from .checkpoint_manager import BaseCheckpointManager + +# Setup logging +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "INFO")) + + +@dataclass +class FSDPConfig: + """Configuration for FSDP checkpointing. + + Args: + FSDP_version (int): Version of FSDP being used. + world_size (int): Number of processes in the distributed training setup. + """ + + FSDP_version: int + world_size: int + + +class FSDPCheckpointManager(BaseCheckpointManager): + """ + Manage FSDP checkpointing in SPMD training. + + - Saves/loads per-rank sharded model & optimizer states + - Persists full lr_scheduler and RNG state + - Stores HF tokenizer/processor and model/config for unified restore + + Args: + model (FSDP): Wrapped model instance. + optimizer (Optimizer): Training optimizer. + lr_scheduler (LRScheduler): Learning-rate scheduler. + processing_class (PreTrainedTokenizer or ProcessorMixin, optional): + Pre-/post-processing artifact handler. + checkpoint_contents DictConfig: Configuration for checkpoint contents. + - 'load': Components to load; must contain 'model'. Defaults to ['model', 'optimizer', 'extra']. + - 'save': Components to save; must contain 'model'. Defaults to ['model', 'optimizer', 'extra']. + """ + + def __init__( + self, + model: FSDP, + optimizer: Optional[torch.optim.Optimizer] = None, + lr_scheduler: Optional[torch.optim.lr_scheduler.LRScheduler] = None, + processing_class: PreTrainedTokenizer | ProcessorMixin = None, + checkpoint_config: DictConfig = None, + **kwargs, + ): + if processing_class is None and "tokenizer" in kwargs: + warnings.warn( + "`tokenizer` is deprecated. use `processing_class` instead.", DeprecationWarning, stacklevel=2 + ) + processing_class = kwargs.pop("tokenizer") + + super().__init__( + model, + optimizer, + lr_scheduler=lr_scheduler, + processing_class=processing_class, + checkpoint_config=checkpoint_config, + ) + + def load_checkpoint(self, local_path: str, hdfs_path: str = None, del_local_after_load=False): + """ + Load an FSDP checkpoint for this rank. + + Downloads and loads: + - model and optimizer shards + - extra state dict (scheduler + RNG) + + Args: + local_path: Directory with per-rank checkpoint files. + hdfs_path: Unused (for API compatibility). + del_local_after_load: Remove local files after loading. + """ + if local_path is None: + return + + # check if the checkpoint_load_contents is valid + if self.should_load_model: + assert self.model is not None, "model must be provided when checkpoint_contents.load includes ['model']" + if self.should_load_optimizer: + assert self.optimizer is not None, ( + "optimizer must be provided when checkpoint_contents.load includes ['optimizer']" + ) + + # every rank download its own checkpoint + state_dict_cfg = ( + ShardedStateDictConfig(offload_to_cpu=True if is_cuda_available else False) + if self.should_load_model + else None + ) + optim_cfg = ( + ShardedOptimStateDictConfig(offload_to_cpu=True if is_cuda_available else False) + if self.should_load_optimizer + else None + ) + with get_fsdp_state_ctx(self.model, StateDictType.SHARDED_STATE_DICT, state_dict_cfg, optim_cfg): + if self.should_load_model: + remote_model_path = os.path.join(local_path, f"model_world_size_{self.world_size}_rank_{self.rank}.pt") + local_model_path = copy_to_local(remote_model_path) + model_state_dict = torch.load(local_model_path, weights_only=False) + self.model.load_state_dict(model_state_dict) + log_with_rank(f"Loaded model from {remote_model_path}", rank=self.rank, logger=logger) + + if self.should_load_optimizer: + remote_optim_path = os.path.join(local_path, f"optim_world_size_{self.world_size}_rank_{self.rank}.pt") + local_optim_path = copy_to_local(remote_optim_path) + optimizer_state_dict = torch.load(local_optim_path, weights_only=False) + self.optimizer.load_state_dict(optimizer_state_dict) + log_with_rank(f"Loaded optimizer from {remote_optim_path}", rank=self.rank, logger=logger) + + if self.should_load_extra: + remote_extra_state_path = os.path.join( + local_path, f"extra_state_world_size_{self.world_size}_rank_{self.rank}.pt" + ) + local_extra_state_path = copy_to_local(remote_extra_state_path) + extra_state_dict = torch.load(local_extra_state_path, weights_only=False) + # recover random state + if "rng" in extra_state_dict: + # 'rng' may not exist for backward compatibility + self.load_rng_state(extra_state_dict["rng"]) + log_with_rank(f"Loaded rng from {remote_extra_state_path}", rank=self.rank, logger=logger) + + lr_scheduler_state_dict = extra_state_dict["lr_scheduler"] + if lr_scheduler_state_dict is not None and self.lr_scheduler is not None: + self.lr_scheduler.load_state_dict(lr_scheduler_state_dict) + log_with_rank(f"Loaded lr_scheduler from {remote_extra_state_path}", rank=self.rank, logger=logger) + + if self.rank == 0 and del_local_after_load: + try: + os.remove(local_model_path) if is_non_local(local_model_path) else None + os.remove(local_optim_path) if is_non_local(local_optim_path) else None + os.remove(local_extra_state_path) if is_non_local(local_extra_state_path) else None + except Exception as e: + log_with_rank( + f"remove local resume ckpt file after loading failed, exception {e} will be ignored", + rank=self.rank, + logger=logger, + ) + + # wait for everyone to load checkpoints + torch.distributed.barrier() + + def save_checkpoint(self, local_path: str, hdfs_path: str = None, global_step: int = 0, max_ckpt_to_keep=None): + """ + Save an FSDP checkpoint for this rank. + + Writes: + - model & optimizer shard files + - extra state dict (scheduler + RNG) + - HF tokenizer/processor and model/config on rank 0 + - optional full HF model under 'huggingface/' if requested + + Rotates old checkpoints, keeping at most `max_ckpt_to_keep`. + + Args: + local_path: Target directory for checkpoint files. + hdfs_path: Unused (for API compatibility). + global_step: Current training step (used for bookkeeping). + max_ckpt_to_keep: Number of recent checkpoints to retain. + """ + if local_path is None: + return + + # record the previous global step + self.previous_global_step = global_step + + # remove previous local_path, only rank 0 should do this + if ( + self.rank == 0 + and max_ckpt_to_keep + and isinstance(max_ckpt_to_keep, int) + and max_ckpt_to_keep > 0 + and len(self.previous_saved_paths) >= max_ckpt_to_keep + ): + keep_start = len(self.previous_saved_paths) - max_ckpt_to_keep + 1 + self.remove_previous_save_local_path(self.previous_saved_paths[:keep_start]) + self.previous_saved_paths = self.previous_saved_paths[keep_start:] + + local_path = local_mkdir_safe(local_path) + torch.distributed.barrier() + + # check if the checkpoint_save_contents is valid + if self.should_save_model: + assert self.model is not None, "model must be provided when checkpoint_contents.save includes ['model']" + if self.should_save_optimizer: + assert self.optimizer is not None, ( + "optimizer must be provided when checkpoint_contents.save includes ['optimizer']" + ) + + # every rank will save its own model and optim shard + state_dict_cfg = ShardedStateDictConfig(offload_to_cpu=True if is_cuda_available else False) + optim_cfg = ShardedOptimStateDictConfig(offload_to_cpu=True if is_cuda_available else False) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with get_fsdp_state_ctx(self.model, StateDictType.SHARDED_STATE_DICT, state_dict_cfg, optim_cfg): + model_path = os.path.join(local_path, f"model_world_size_{self.world_size}_rank_{self.rank}.pt") + optim_path = os.path.join(local_path, f"optim_world_size_{self.world_size}_rank_{self.rank}.pt") + extra_path = os.path.join(local_path, f"extra_state_world_size_{self.world_size}_rank_{self.rank}.pt") + + if self.should_save_model: + model_state_dict = self.model.state_dict() + torch.save(model_state_dict, model_path) + log_with_rank(f"Saved model to {os.path.abspath(model_path)}", rank=self.rank, logger=logger) + + if self.should_save_optimizer: + optimizer_state_dict = self.optimizer.state_dict() + torch.save(optimizer_state_dict, optim_path) + log_with_rank(f"Saved optim to {os.path.abspath(optim_path)}", rank=self.rank, logger=logger) + + if self.should_save_extra: + lr_scheduler_state_dict = self.lr_scheduler.state_dict() if self.lr_scheduler is not None else None + extra_state_dict = { + "lr_scheduler": lr_scheduler_state_dict, + "rng": self.get_rng_state(), + } + torch.save(extra_state_dict, extra_path) + log_with_rank(f"Saved extra_state to {os.path.abspath(extra_path)}", rank=self.rank, logger=logger) + + if self.rank == 0: + # Save HF tokenizer/processor and model config on rank 0 to huggingface/ directory, no matter whether + # huggingface model is requested to be saved or not. + + if fsdp_version(self.model) == 1: + unwrap_model = self.model._fsdp_wrapped_module + else: + unwrap_model = self.model + + hf_config_tokenizer_path = os.path.join(local_path, "huggingface") + local_mkdir_safe(hf_config_tokenizer_path) + model_config = unwrap_model.config + generation_config = None + if unwrap_model.can_generate() and hasattr(model_config, "name_or_path") and model_config.name_or_path: + try: + # Some model's name_or_path is empty if not initialized from pretrained, + # in this cases, we don't save generation config. + generation_config = GenerationConfig.from_pretrained(model_config.name_or_path) + generation_config.save_pretrained(hf_config_tokenizer_path) + except Exception: + # if the generation config isn't available, we don't save it + pass + + model_config.save_pretrained(hf_config_tokenizer_path) + if self.processing_class is not None: + self.processing_class.save_pretrained(hf_config_tokenizer_path) + log_with_rank( + f"Saved model config and tokenizer class to {os.path.abspath(hf_config_tokenizer_path)}", + rank=self.rank, + logger=logger, + log_only_rank_0=True, + ) + + # If we have a custom model, we copy the file defining it in the folder and set the attributes so it can be + # loaded from the Hub. + if hasattr(model_config, "auto_map"): + custom_object_save(unwrap_model, hf_config_tokenizer_path, config=model_config) + + # Also save runtime FSDP config + fsdp_config_path = os.path.join(local_path, "fsdp_config.json") + fsdp_config = FSDPConfig( + FSDP_version=fsdp_version(self.model), + world_size=self.world_size, + ) + with open(fsdp_config_path, "w") as f: + json.dump(asdict(fsdp_config), f, indent=4) + + # wait for everyone to dump to local + torch.distributed.barrier() + + if self.should_save_hf_model: + # Only rank 0 will save hf model and, + # offload to cpu to save LLMs which may be too large to fit in one GPU + state_dict = get_fsdp_full_state_dict(self.model, offload_to_cpu=True, rank0_only=True) + + if self.rank == 0: + hf_local_path = os.path.join(local_path, "huggingface") + os.makedirs(hf_local_path, exist_ok=True) + + if "ForTokenClassification" in model_config.architectures[0]: + from transformers import AutoModelForTokenClassification + + auto_model_cls = AutoModelForTokenClassification + elif "ForCausalLM" in model_config.architectures[0]: + from transformers import AutoModelForCausalLM + + auto_model_cls = AutoModelForCausalLM + elif "ForConditionalGeneration" in model_config.architectures[0]: + # Handle different transformers versions for Vision2Seq models + import transformers + from packaging import version + + if version.parse(transformers.__version__) >= version.parse("4.54.0"): + # transformers >= 4.54.0 uses AutoModelForImageTextToText + from transformers import AutoModelForImageTextToText + + auto_model_cls = AutoModelForImageTextToText + else: + # transformers < 4.54.0 uses AutoModelForVision2Seq + from transformers import AutoModelForVision2Seq + + auto_model_cls = AutoModelForVision2Seq + else: + raise NotImplementedError(f"Unknown architecture {model_config['architectures']}") + + with init_empty_weights(): + save_model = auto_model_cls.from_config(model_config, torch_dtype=torch.bfloat16) + save_model.to_empty(device="cpu") + + if save_model.can_generate(): + if generation_config is not None: + save_model.generation_config = generation_config + else: + print( + f"Warning: {self.__class__.__name__}.save_checkpoint: Generation config file not found " + f"in, using a generation config created from the model config when saving hf_model." + ) + + save_model.save_pretrained(hf_local_path, state_dict=state_dict) + log_with_rank( + f"Saved hf_model to {os.path.abspath(hf_local_path)}", + rank=self.rank, + logger=logger, + log_only_rank_0=True, + ) + del state_dict + del save_model + + # wait for rank0 to dump hf_model to local + torch.distributed.barrier() + + self.previous_saved_paths.append(local_path) diff --git a/e2c/verl/verl/utils/checkpoint/megatron_checkpoint_manager.py b/e2c/verl/verl/utils/checkpoint/megatron_checkpoint_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..b91ef6070b99c4423c51bf3f9569ed7fb8149ae4 --- /dev/null +++ b/e2c/verl/verl/utils/checkpoint/megatron_checkpoint_manager.py @@ -0,0 +1,557 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import random +from collections.abc import Callable +from dataclasses import asdict + +import numpy as np +import torch +import torch.distributed +from megatron.core import mpu, tensor_parallel +from megatron.core.dist_checkpointing.mapping import ShardedObject +from megatron.core.transformer.enums import AttnBackend +from transformers import GenerationConfig + +from verl.models.weight_loader_registry import get_weight_saver +from verl.utils.device import get_device_name, get_torch_device +from verl.utils.fs import is_non_local, local_mkdir_safe +from verl.utils.logger import log_with_rank +from verl.utils.megatron.dist_checkpointing import load_dist_checkpointing, save_dist_checkpointing +from verl.utils.megatron_utils import ( + get_dist_checkpoint_path, + get_hf_model_checkpoint_path, + get_transformer_config_checkpoint_path, +) + +from .checkpoint_manager import BaseCheckpointManager + +# Setup logging +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "INFO")) + + +class MegatronCheckpointManager(BaseCheckpointManager): + """ + Checkpoint manager for Megatron-LM distributed training. + + This class manages the saving and loading of model checkpoints in a Megatron-LM + distributed training environment. It handles various aspects of checkpointing + including model states, optimizer states, learning rate schedulers, and random + number generator states, ensuring compatibility with HuggingFace formats. + + Key features: + - Distributed checkpoint saving and loading using Megatron's dist_checkpointing + - Support for tensor parallel, pipeline parallel, and data parallel configurations + - Automatic handling of model state dictionaries across multiple pipeline stages + - Integration with HuggingFace model configurations and tokenizers + - Random number generator state management for reproducibility + - Support for both synchronous and asynchronous checkpoint operations + + The manager automatically handles: + - Directory structure creation based on global steps and process ranks + - Model configuration and tokenizer saving in HuggingFace format + - Optimizer and scheduler state persistence + - CUDA RNG state management for deterministic training + - Checkpoint cleanup and retention policies + + Args: + model: The Megatron model instance to checkpoint + optimizer: The optimizer instance (optional) + lr_scheduler: The learning rate scheduler instance (optional) + + Attributes: + model: Reference to the Megatron model being checkpointed + optimizer: Reference to the optimizer (if provided) + lr_scheduler: Reference to the learning rate scheduler (if provided) + rank: Current process rank in the distributed setup + + Example: + ```python + checkpoint_manager = MegatronCheckpointManager( + model=megatron_model, + optimizer=optimizer, + lr_scheduler=scheduler + ) + + checkpoint_manager.save_checkpoint( + local_path="checkpoints/step_1000", + global_step=1000 + ) + + checkpoint_manager.load_checkpoint( + local_path="checkpoints/step_1000" + ) + ``` + """ + + def __init__( + self, + config, + checkpoint_config, + model_config, + transformer_config, + role, + model: torch.nn.ModuleList, + arch: str, + hf_config, + param_dtype: torch.dtype, + share_embeddings_and_output_weights: bool, + processing_class, + optimizer, + optimizer_scheduler, + use_distributed_optimizer: bool, + use_checkpoint_opt_param_scheduler: bool = False, + use_dist_checkpointing: bool = True, + bridge=None, + **kwargs, + ): + super().__init__( + model, + optimizer=optimizer, + lr_scheduler=optimizer_scheduler, + processing_class=processing_class, + checkpoint_config=checkpoint_config, + ) + self.arch = arch + self.config = config + self.transformer_config = transformer_config + self.role = role + self.is_value_model = False + if self.role in ["reward", "critic"]: + self.is_value_model = True + self.model_config = model_config + self.hf_config = hf_config + self.param_dtype = param_dtype + self.share_embeddings_and_output_weights = share_embeddings_and_output_weights + self.model_path = self.config.model.path + self.use_distributed_optimizer = use_distributed_optimizer + self.use_checkpoint_opt_param_scheduler = use_checkpoint_opt_param_scheduler + self.bridge = bridge + self.rank = torch.distributed.get_rank() + self.use_dist_checkpointing = use_dist_checkpointing or not self.bridge or self.is_value_model + self.use_hf_checkpoint = not self.use_dist_checkpointing + + self.weight_saver = None + if self.bridge is None: + self.weight_saver = get_weight_saver(self.arch) + + def get_rng_state(self, use_dist_ckpt: bool = True, data_parallel_random_init: bool = False): + """collect rng state across data parallel ranks""" + rng_state = { + "random_rng_state": random.getstate(), + "np_rng_state": np.random.get_state(), + "torch_rng_state": torch.get_rng_state(), + "rng_tracker_states": tensor_parallel.get_cuda_rng_tracker().get_states(), + } + + if get_device_name() != "cpu": + rng_state[f"{get_device_name()}_rng_state"] = get_torch_device().get_rng_state() + + rng_state_list = None + if torch.distributed.is_initialized() and mpu.get_data_parallel_world_size() > 1 and data_parallel_random_init: + rng_state_list = [None for i in range(mpu.get_data_parallel_world_size())] + torch.distributed.all_gather_object(rng_state_list, rng_state, group=mpu.get_data_parallel_group()) + else: + rng_state_list = [rng_state] + + if use_dist_ckpt: + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + rng_state_list = ShardedObject( + "rng_state", + rng_state_list, + (pp_size, tp_size), + (pp_rank, tp_rank), + replica_id=mpu.get_data_parallel_rank(with_context_parallel=True), + ) + + return rng_state_list + + def get_checkpoint_name( + self, + checkpoints_path, + pipeline_parallel=None, + tensor_rank=None, + pipeline_rank=None, + cp_rank=None, + expert_parallel=None, + expert_rank=None, + return_base_dir=True, + basename="model.pt", + ): + """Determine the directory name for this rank's checkpoint.""" + # Use both the tensor and pipeline MP rank. + if pipeline_parallel is None: + pipeline_parallel = mpu.get_pipeline_model_parallel_world_size() > 1 + if tensor_rank is None: + tensor_rank = mpu.get_tensor_model_parallel_rank() + if pipeline_rank is None: + pipeline_rank = mpu.get_pipeline_model_parallel_rank() + if cp_rank is None: + cp_rank = mpu.get_context_parallel_rank() + if expert_parallel is None: + expert_parallel = mpu.get_expert_model_parallel_world_size() > 1 + if expert_rank is None: + expert_rank = mpu.get_expert_model_parallel_rank() + + # Use both the tensor and pipeline MP rank. If using the distributed + # optimizer, then the optimizer's path must additionally include the + # data parallel rank. + + # due to the fact that models are identical across cp ranks, cp rank is not used in the checkpoint path + if not pipeline_parallel: + common_path = os.path.join(checkpoints_path, f"mp_rank_{tensor_rank:02d}") + else: + common_path = os.path.join(checkpoints_path, f"mp_rank_{tensor_rank:02d}_{pipeline_rank:03d}") + + if expert_parallel: + common_path = common_path + f"_{expert_rank:03d}" + + os.makedirs(common_path, exist_ok=True) + + if return_base_dir: + return common_path + return os.path.join(common_path, basename) + + def generate_state_dict( + self, generate_model: bool = True, generate_optimizer: bool = True, generate_extra: bool = True + ): + # For save dist checkpointing + state_dict = {} + + # Should always generate model state dict + # All ranks Save Model to reduce memory pressure + # Get sharded state dict, notice that state_dict will collect among dp groups, causing memory pressure + for vpp_rank, model in enumerate(self.model): + if len(self.model) > 1: + mpu.set_virtual_pipeline_model_parallel_rank(vpp_rank) + key = f"model{vpp_rank}" if len(self.model) > 1 else "model" + else: + key = "model" + if hasattr(model, "module"): + model = model.module + state_dict[key] = model.sharded_state_dict() + + # Optimizer State Dict + if generate_optimizer: + torch.distributed.barrier() + optimizer_sharded_states = self.optimizer.sharded_state_dict(state_dict) + state_dict["optimizer"] = optimizer_sharded_states + + if self.lr_scheduler is not None: + lr_state_dict = self.lr_scheduler.state_dict() + state_dict["lr_scheduler"] = lr_state_dict + + if not generate_model: + state_dict.pop("model", None) + + # RNG States State Dict + if generate_extra: + torch.distributed.barrier() + rng_state = self.get_rng_state() + state_dict["rng_state"] = rng_state + + return state_dict + + def load_rng_states(self, rng_states, data_parallel_random_init=False, use_dist_ckpt=True): + # access rng_state for data parallel rank + if data_parallel_random_init: + rng_states = rng_states[mpu.get_data_parallel_rank()] + else: + rng_states = rng_states[0] + random.setstate(rng_states["random_rng_state"]) + np.random.set_state(rng_states["np_rng_state"]) + torch.set_rng_state(rng_states["torch_rng_state"]) + + if get_device_name() != "cpu": + get_torch_device().set_rng_state(rng_states[f"{get_device_name()}_rng_state"]) + + # Check for empty states array + if not rng_states["rng_tracker_states"]: + raise KeyError + tensor_parallel.get_cuda_rng_tracker().set_states(rng_states["rng_tracker_states"]) + + def load_checkpoint(self, local_path: str, hdfs_path: str = None, del_local_after_load=False): + if local_path is not None: + assert os.path.exists(local_path), f"Checkpoint path {local_path} does not exist." + + dist_checkpoint_path = get_dist_checkpoint_path(local_path) + + # Get State Dict for loading + sharded_state_dict = self.generate_state_dict( + self.should_load_model and self.use_dist_checkpointing, self.should_load_optimizer, self.should_load_extra + ) + log_with_rank(f"Generated state dict for loading: {sharded_state_dict.keys()}", rank=self.rank, logger=logger) + + # Load Dist Checkpointing + state_dict = load_dist_checkpointing( + sharded_state_dict=sharded_state_dict, + ckpt_dir=dist_checkpoint_path, + ) + + if self.should_load_model and self.use_dist_checkpointing: + assert "model" in state_dict or any( + f"model{vpp_rank}" in state_dict for vpp_rank in range(len(self.model)) + ), f"Model state dict not found in {state_dict.keys()}. Please check the checkpoint file {local_path}." + for vpp_rank, model in enumerate(self.model): + if len(self.model) == 1: + model_state_dict = state_dict["model"] + else: + assert f"model{vpp_rank}" in state_dict, f"model{vpp_rank} not found in state_dict" + model_state_dict = state_dict[f"model{vpp_rank}"] + mpu.set_virtual_pipeline_model_parallel_rank(vpp_rank) + self.model[vpp_rank].load_state_dict(model_state_dict) + log_with_rank(f"Loaded sharded model checkpoint from {local_path}", rank=self.rank, logger=logger) + elif self.should_load_model and self.use_hf_checkpoint: + hf_model_path = get_hf_model_checkpoint_path(local_path) + self.bridge.load_weights(self.model, hf_model_path) + log_with_rank(f"Loaded HF model checkpoint from {hf_model_path} with bridge", rank=self.rank, logger=logger) + + if self.should_load_optimizer: + assert "optimizer" in state_dict, ( + f"Optimizer state dict not found in {state_dict.keys()}. Please check the checkpoint file {local_path}." + ) + optimizer_state_dict = state_dict["optimizer"] + self.optimizer.load_state_dict(optimizer_state_dict) + log_with_rank(f"Loaded optimizer checkpoint from {local_path}", rank=self.rank, logger=logger) + if self.use_checkpoint_opt_param_scheduler: + assert "lr_scheduler" in state_dict, ( + f"LR scheduler state dict not found in {state_dict.keys()}. Please check the checkpoint file " + f"{local_path}." + ) + lr_scheduler_state_dict = state_dict["lr_scheduler"] + if self.lr_scheduler is not None: + self.lr_scheduler.load_state_dict(lr_scheduler_state_dict) + log_with_rank(f"Loaded LR scheduler checkpoint from {local_path}", rank=self.rank, logger=logger) + + if self.should_load_extra: + assert "rng_state" in state_dict, ( + f"RNG state dict not found in {state_dict.keys()}. Please check the checkpoint file {local_path}." + ) + rng_state = state_dict["rng_state"] + self.load_rng_states(rng_state) + log_with_rank(f"Loaded RNG states from {local_path}", rank=self.rank, logger=logger) + + if del_local_after_load: + try: + os.remove(local_path) if is_non_local(local_path) else None + except Exception as e: + log_with_rank( + f"remove local resume ckpt file after loading failed, exception {e} will be ignored", + rank=self.rank, + logger=logger, + ) + + def save_checkpoint(self, local_path: str, hdfs_path: str = None, global_step: int = 0, max_ckpt_to_keep=None): + # record the previous global step + self.previous_global_step = global_step + + # remove previous local_path + if ( + max_ckpt_to_keep + and isinstance(max_ckpt_to_keep, int) + and max_ckpt_to_keep > 0 + and len(self.previous_saved_paths) >= max_ckpt_to_keep + ): + keep_start = len(self.previous_saved_paths) - max_ckpt_to_keep + 1 + self.remove_previous_save_local_path(self.previous_saved_paths[:keep_start]) + self.previous_saved_paths = self.previous_saved_paths[keep_start:] + + local_path = local_mkdir_safe(local_path) + dist_checkpoint_path = get_dist_checkpoint_path(local_path) + + # Note that model weights, optimizer states, and extra states are generated + # together in a state dict, we save them in one time + if self.use_dist_checkpointing: + # Generate state dict for saving + state_dict = self.generate_state_dict( + self.should_save_model, self.should_save_optimizer, self.should_save_extra + ) + log_with_rank(f"Generated state dict for saving: {state_dict.keys()}", rank=self.rank, logger=logger) + for vpp_rank, model in enumerate(self.model): + if len(self.model) > 1: + model_i_keys = state_dict[f"model{vpp_rank}"].keys() + log_with_rank(f"Generated state dict for saving: {model_i_keys}", rank=self.rank, logger=logger) + else: + log_with_rank( + f"Generated state dict for saving: {state_dict['model'].keys()}", rank=self.rank, logger=logger + ) + # Start Async save if enabled + async_save_request = save_dist_checkpointing( + sharded_state_dict=state_dict, + ckpt_path=dist_checkpoint_path, + async_save=self.checkpoint_config.async_save, + ) + + # Synchronize all async save requests + if not self.checkpoint_config.async_save: + assert async_save_request is None, "Async save request should be None when not using async save." + torch.distributed.barrier() + else: + assert self.use_hf_checkpoint, "When not using distributed checkpointing, use_hf_checkpoint should be True." + # Generate optimizer and exra state dicts + state_dict = self.generate_state_dict( + generate_model=False, + generate_optimizer=self.should_save_optimizer, + generate_extra=self.should_save_extra, + ) + # Save optimizer and extra states to local path + # Start Async save if enabled + async_save_request = save_dist_checkpointing( + sharded_state_dict=state_dict, + ckpt_path=dist_checkpoint_path, + async_save=self.checkpoint_config.async_save, + ) + + # Synchronize all async save requests + if not self.checkpoint_config.async_save: + assert async_save_request is None, "Async save request should be None when not using async save." + torch.distributed.barrier() + + if self.should_save_model: + if self.use_hf_checkpoint: + # Use mbridge to save HF model checkpoint + log_with_rank(f"Saving HF model checkpoint to {local_path} with bridge", rank=self.rank, logger=logger) + hf_ckpt_path = get_hf_model_checkpoint_path(local_path) + self.bridge.save_weights(self.model, hf_ckpt_path) + log_with_rank(f"Saved bridge checkpoint to {hf_ckpt_path}", rank=self.rank, logger=logger) + + # Only rank 0 saves the hf config and tokenizer to huggingface path + # No matter whether we save hf model or not + if self.rank == 0: + # Save tokenizer + hf_config_tokenizer_path = get_hf_model_checkpoint_path(local_path) + if self.processing_class is not None: + self.processing_class.save_pretrained(hf_config_tokenizer_path) + # Save huggingface config + self.hf_config.save_pretrained(hf_config_tokenizer_path) + if hasattr(self.hf_config, "name_or_path") and self.hf_config.name_or_path: + try: + generation_config = GenerationConfig.from_pretrained(self.hf_config.name_or_path) + generation_config.save_pretrained(hf_config_tokenizer_path) + except Exception: + # if the generation config isn't available, we don't save it + pass + log_with_rank( + f"Saved Huggingface config and tokenizer to {hf_config_tokenizer_path}", + rank=self.rank, + logger=logger, + log_only_rank_0=True, + ) + + if self.should_save_extra: + if self.rank == 0: + # Save transformer config + print(self.transformer_config) + transformer_config_dict = asdict(self.transformer_config) + to_convert_types = {torch.dtype: str, AttnBackend: str} + ignore_types = [Callable] + pop_keys = [] + for key, value in transformer_config_dict.items(): + if type(value) in to_convert_types: + transformer_config_dict[key] = to_convert_types[type(value)](value) + if type(value) in ignore_types: + pop_keys.append(key) + if callable(value): + pop_keys.append(key) + for key in pop_keys: + transformer_config_dict.pop(key) + transformer_config_path = get_transformer_config_checkpoint_path(local_path) + with open(transformer_config_path, "w") as f: + json.dump(transformer_config_dict, f, indent=2) + + if self.should_save_hf_model and not self.use_hf_checkpoint: + # wait for everyone to dump to local + if self.bridge is not None: + hf_model_ckpt_path = get_hf_model_checkpoint_path(local_path) + self.bridge.save_weights(self.model, hf_model_ckpt_path) + else: + state_dict = self.weight_saver( + self.model, + self.hf_config, + dtype=self.param_dtype, + is_value_model=self.is_value_model, + tie_word_embeddings=self.share_embeddings_and_output_weights, + ) + + torch.distributed.barrier() + if self.rank == 0: + hf_model_ckpt_path = get_hf_model_checkpoint_path(local_path) + import warnings + + from accelerate import init_empty_weights + + with init_empty_weights(), warnings.catch_warnings(): + warnings.simplefilter("ignore") + if "mistral7b-rm" in self.config.model.path: + from transformers import MistralForSequenceClassification + + model = MistralForSequenceClassification.from_pretrained( + self.config.model.path + ) # use score head instead of lm_head + state_dict["score.weight"] = state_dict["score.weight"] + else: + from transformers import AutoModelForCausalLM + + model = AutoModelForCausalLM.from_pretrained(self.config.model.path, torch_dtype="auto") + model.save_pretrained(hf_model_ckpt_path, state_dict=state_dict) + log_with_rank( + f"Saved Huggingface config and tokenizer to {hf_model_ckpt_path}", + rank=self.rank, + logger=logger, + log_only_rank_0=True, + ) + + if hdfs_path is not None: + log_with_rank( + f"Uploading checkpoint to {hdfs_path}", rank=self.rank, logger=logger, log_only_rank_0=True + ) + from verl.utils import hdfs_io + + hdfs_io.makedirs(hdfs_path, exist_ok=True) + hdfs_io.copy(src=hf_model_ckpt_path, dst=hdfs_path, dirs_exist_ok=True) + log_with_rank( + f"HDFS checkpoint uploaded to {hdfs_path}", + rank=self.rank, + logger=logger, + log_only_rank_0=True, + ) + + def finalize_save_fn(): + # Rank 0 uploads checkpoint to HDFS if hdfs_path is provided + log_with_rank( + f"Dist checkpointing save completed for {dist_checkpoint_path}", rank=self.rank, logger=logger + ) + if self.rank == 0: + if hdfs_path is not None: + log_with_rank(f"Uploading checkpoint to {hdfs_path}", rank=self.rank, logger=logger) + from verl.utils import hdfs_io + + hdfs_io.makedirs(hdfs_path, exist_ok=True) + hdfs_io.copy(src=dist_checkpoint_path, dst=hdfs_path, dirs_exist_ok=True) + hdfs_io.copy(src=hf_config_tokenizer_path, dst=hdfs_path, dirs_exist_ok=True) + + if self.checkpoint_config.async_save: + assert async_save_request is not None, "Async save request should not be None when using async save." + async_save_request.add_finalize_fn(finalize_save_fn) + else: + finalize_save_fn() + + self.previous_saved_paths.append(local_path) diff --git a/e2c/verl/verl/utils/config.py b/e2c/verl/verl/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..fa3630c654cbad9a14f0443b9e6e9f054354ae7e --- /dev/null +++ b/e2c/verl/verl/utils/config.py @@ -0,0 +1,197 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import is_dataclass +from typing import Any, Optional + +from omegaconf import DictConfig, ListConfig, OmegaConf + +__all__ = ["omega_conf_to_dataclass", "validate_config"] + + +def omega_conf_to_dataclass(config: DictConfig | dict, dataclass_type: Optional[type[Any]] = None) -> Any: + """ + Convert an OmegaConf DictConfig to a dataclass. + + Args: + config: The OmegaConf DictConfig or dict to convert. + dataclass_type: The dataclass type to convert to. When dataclass_type is None, + the DictConfig must contain _target_ to be instantiated via hydra.instantiate API. + + Returns: + The dataclass instance. + """ + # Got an empty config + if not config: + return dataclass_type if dataclass_type is None else dataclass_type() + # Got an object + if not isinstance(config, DictConfig | ListConfig | dict | list): + return config + + if dataclass_type is None: + assert "_target_" in config, ( + "When dataclass_type is not provided, config must contain _target_. " + "See trainer/config/ppo_trainer.yaml algorithm section for an example. " + f"Got config: {config}" + ) + from hydra.utils import instantiate + + return instantiate(config, _convert_="partial") + + if not is_dataclass(dataclass_type): + raise ValueError(f"{dataclass_type} must be a dataclass") + cfg = OmegaConf.create(config) # in case it's a dict + # pop _target_ to avoid hydra instantiate error, as most dataclass do not have _target_ + # Updated (vermouth1992) We add _target_ to BaseConfig so that it is compatible. + # Otherwise, this code path can't support recursive instantiation. + # if "_target_" in cfg: + # cfg.pop("_target_") + cfg_from_dataclass = OmegaConf.structured(dataclass_type) + # let cfg override the existing vals in `cfg_from_dataclass` + cfg_merged = OmegaConf.merge(cfg_from_dataclass, cfg) + # now convert to `dataclass_type` + config_object = OmegaConf.to_object(cfg_merged) + return config_object + + +def update_dict_with_config(dictionary: dict, config: DictConfig): + for key in dictionary: + if hasattr(config, key): + dictionary[key] = getattr(config, key) + + +def validate_config( + config: DictConfig, + use_reference_policy: bool, + use_critic: bool, +) -> None: + """Validate an OmegaConf DictConfig. + + Args: + config (DictConfig): The OmegaConf DictConfig to validate. + use_reference_policy (bool): is ref policy needed + use_critic (bool): is critic needed + """ + # number of GPUs total + n_gpus = config.trainer.n_gpus_per_node * config.trainer.nnodes + + if not config.actor_rollout_ref.actor.use_dynamic_bsz: + if config.actor_rollout_ref.actor.strategy == "megatron": + model_parallel_size = ( + config.actor_rollout_ref.actor.megatron.tensor_model_parallel_size + * config.actor_rollout_ref.actor.megatron.pipeline_model_parallel_size + ) + assert ( + n_gpus % (model_parallel_size * config.actor_rollout_ref.actor.megatron.context_parallel_size) == 0 + ), ( + f"n_gpus ({n_gpus}) must be divisible by model_parallel_size ({model_parallel_size}) times " + f"context_parallel_size ({config.actor_rollout_ref.actor.megatron.context_parallel_size})" + ) + megatron_dp = n_gpus // ( + model_parallel_size * config.actor_rollout_ref.actor.megatron.context_parallel_size + ) + minimal_bsz = megatron_dp * config.actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu + else: + minimal_bsz = n_gpus + + # 1. Check total batch size for data correctness + real_train_batch_size = config.data.train_batch_size * config.actor_rollout_ref.rollout.n + assert real_train_batch_size % minimal_bsz == 0, ( + f"real_train_batch_size ({real_train_batch_size}) must be divisible by minimal possible batch size " + f"({minimal_bsz})" + ) + + # A helper function to check "micro_batch_size" vs "micro_batch_size_per_gpu" + # We throw an error if the user sets both. The new convention is "..._micro_batch_size_per_gpu". + def check_mutually_exclusive(mbs, mbs_per_gpu, name: str): + """Validate mutually exclusive micro batch size configuration options. + + Ensures that users don't set both deprecated micro_batch_size and + the new micro_batch_size_per_gpu parameters simultaneously. + + Args: + mbs: Deprecated micro batch size parameter value. + mbs_per_gpu: New micro batch size per GPU parameter value. + name (str): Configuration section name for error messages. + + Raises: + ValueError: If both parameters are set or neither is set. + """ + settings = { + "reward_model": "micro_batch_size", + "actor_rollout_ref.ref": "log_prob_micro_batch_size", + "actor_rollout_ref.rollout": "log_prob_micro_batch_size", + } + + if name in settings: + param = settings[name] + param_per_gpu = f"{param}_per_gpu" + + if mbs is None and mbs_per_gpu is None: + raise ValueError(f"[{name}] Please set at least one of '{name}.{param}' or '{name}.{param_per_gpu}'.") + + if mbs is not None and mbs_per_gpu is not None: + raise ValueError( + f"[{name}] You have set both '{name}.{param}' AND '{name}.{param_per_gpu}'. Please remove " + f"'{name}.{param}' because only '*_{param_per_gpu}' is supported (the former is deprecated)." + ) + + # Actor validation done in ActorConfig.__post_init__ and validate() + actor_config = omega_conf_to_dataclass(config.actor_rollout_ref.actor) + actor_config.validate(n_gpus, config.data.train_batch_size, config.actor_rollout_ref.model) + + if not config.actor_rollout_ref.actor.use_dynamic_bsz: + if use_reference_policy: + # reference: log_prob_micro_batch_size vs. log_prob_micro_batch_size_per_gpu + check_mutually_exclusive( + config.actor_rollout_ref.ref.log_prob_micro_batch_size, + config.actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu, + "actor_rollout_ref.ref", + ) + + # The rollout section also has log_prob_micro_batch_size vs. log_prob_micro_batch_size_per_gpu + check_mutually_exclusive( + config.actor_rollout_ref.rollout.log_prob_micro_batch_size, + config.actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu, + "actor_rollout_ref.rollout", + ) + + # Check for reward model micro-batch size conflicts + if config.reward_model.enable and not config.reward_model.use_dynamic_bsz: + check_mutually_exclusive( + config.reward_model.micro_batch_size, config.reward_model.micro_batch_size_per_gpu, "reward_model" + ) + + if config.algorithm.use_kl_in_reward and config.actor_rollout_ref.actor.use_kl_loss: + print("NOTICE: You have both enabled in-reward kl and kl loss.") + + # critic + if use_critic: + critic_config = omega_conf_to_dataclass(config.critic) + critic_config.validate(n_gpus, config.data.train_batch_size) + + if config.data.get("val_batch_size", None) is not None: + print( + "WARNING: val_batch_size is deprecated." + + " Validation datasets are sent to inference engines as a whole batch," + + " which will schedule the memory themselves." + ) + + # check eval config + if config.actor_rollout_ref.rollout.val_kwargs.do_sample: + assert config.actor_rollout_ref.rollout.temperature > 0, ( + "validation gen temperature should be greater than 0 when enabling do_sample" + ) + + print("[validate_config] All configuration checks passed successfully!") diff --git a/e2c/verl/verl/utils/dataset/README.md b/e2c/verl/verl/utils/dataset/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f886a70aabf443fb167453d667529b62f3311765 --- /dev/null +++ b/e2c/verl/verl/utils/dataset/README.md @@ -0,0 +1,16 @@ +# Dataset Format +## RLHF dataset +We combine all the data sources into a single parquet files. We directly organize the prompt into the chat format so that multi-turn chats can be easily incorporated. In the prompt, we may add instruction following texts to guide the model output the answers in a particular format so that we can extract the answers. + +Math problems +```json +{ + "data_source": "openai/gsm8k", + "prompt": [{"role": "user", "content": "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May? Let's think step by step and output the final answer after \"####\""}], + "ability": "math", + "reward_model": { + "style": "rule", + "ground_truth": ["72"] + }, +} +``` diff --git a/e2c/verl/verl/utils/dataset/__init__.py b/e2c/verl/verl/utils/dataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6032d68c86423f0e6c57afba684dff5e1b8362c0 --- /dev/null +++ b/e2c/verl/verl/utils/dataset/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .rl_dataset import RLHFDataset +from .rm_dataset import RMDataset +from .sft_dataset import SFTDataset + +__all__ = ["RLHFDataset", "RMDataset", "SFTDataset"] diff --git a/e2c/verl/verl/utils/dataset/dataset_utils.py b/e2c/verl/verl/utils/dataset/dataset_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7354a0c896d3910d2539b1453a372a0ad1b0dad7 --- /dev/null +++ b/e2c/verl/verl/utils/dataset/dataset_utils.py @@ -0,0 +1,70 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from enum import Enum + +import torch + + +class DatasetPadMode(str, Enum): + """Padding mode for dataset""" + + RIGHT = "right" + LEFT_RIGHT = "left_right" + NO_PADDING = "no_padding" + + +class SFTTensorCollator: + """ + A custom collate_fn that handles batching of sequences. + 1. for variable-length sequences, convert them into NestedTensors. + 2. for fixed-length sequences, use default_collate. + """ + + def __init__(self, pad_mode: DatasetPadMode = DatasetPadMode.LEFT_RIGHT): + self.pad_mode = pad_mode + + def __call__(self, batch: list[dict[str, any]]) -> dict[str, any]: + if self.pad_mode == DatasetPadMode.NO_PADDING: + return self.collate_variable_batch(batch) + elif self.pad_mode in [DatasetPadMode.RIGHT, DatasetPadMode.LEFT_RIGHT]: + from torch.utils.data import default_collate + + return default_collate(batch) + else: + raise NotImplementedError(f"pad_mode {self.pad_mode} not implemented") + + def collate_variable_batch(self, batch: list[dict[str, any]]) -> dict[str, any]: + """ + Collates a list of samples into a single batch. + + Args: + batch: A list of dictionary samples from the dataset. + + Returns: + A dictionary representing the batched data, with variable-length + sequences converted to NestedTensors. + """ + + final_batch = {} + + tensor_keys = [key for key in batch[0].keys() if isinstance(batch[0][key], torch.Tensor)] + + # Handle tensor values by creating a NestedTensor. + for key in tensor_keys: + tensors = [item[key] for item in batch] + final_batch[key] = torch.nested.as_nested_tensor(tensors, layout=torch.jagged) + + return final_batch diff --git a/e2c/verl/verl/utils/dataset/multiturn_sft_dataset.py b/e2c/verl/verl/utils/dataset/multiturn_sft_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..e669e292bd247ed21a2918ffa42758cf0526a0f6 --- /dev/null +++ b/e2c/verl/verl/utils/dataset/multiturn_sft_dataset.py @@ -0,0 +1,442 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2025 ModelBest Inc. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Multi-turn SFT dataset that supports training on conversation data with multiple turns +""" + +import logging +from typing import Any, Optional + +import numpy as np +import pandas as pd +import torch +from omegaconf import ListConfig +from torch.utils.data import Dataset +from transformers import PreTrainedTokenizer + +from verl.utils import hf_tokenizer +from verl.utils.dataset.dataset_utils import DatasetPadMode +from verl.utils.fs import copy_local_path_from_hdfs +from verl.utils.model import compute_position_id_with_mask +from verl.utils.torch_functional import pad_sequence_to_length, postprocess_data + + +def convert_nested_value_to_list_recursive(data_item): + if isinstance(data_item, dict): + return {k: convert_nested_value_to_list_recursive(v) for k, v in data_item.items()} + elif isinstance(data_item, list): + return [convert_nested_value_to_list_recursive(elem) for elem in data_item] + elif isinstance(data_item, np.ndarray): + # Convert to list, then recursively process the elements of the new list + return convert_nested_value_to_list_recursive(data_item.tolist()) + else: + # Base case: item is already a primitive type (int, str, float, bool, etc.) + return data_item + + +class MultiTurnSFTDataset(Dataset): + """ + Dataset for multi-turn conversations where each assistant response should be trained + """ + + def __init__(self, parquet_files: str | list[str], tokenizer, config=None): + # Set defaults and extract parameters from config if provided + config = config or {} + self.pad_mode = config.get("pad_mode", "right") + assert self.pad_mode in ["right", "left_right", "no_padding"], ( + f"Expect pad_mode to be 'right', 'left_right' or 'no_padding'. Got {self.pad_mode}" + ) + self.truncation = config.get("truncation", "error") + # for right padding + self.max_length = config.get("max_length", 1024) + # for left right paddding to be consistent with RL + self.max_prompt_length = config.get("max_prompt_length", 512) + self.max_response_length = config.get("max_response_length", 512) + # Get messages_key from the new multiturn config structure + multiturn_config = config.get("multiturn", {}) + self.messages_key = multiturn_config.get("messages_key", "messages") + self.tools_key = multiturn_config.get("tools_key", "tools") + self.enable_thinking_key = multiturn_config.get("enable_thinking_key", "enable_thinking") + self.apply_chat_template_kwargs = config.get("apply_chat_template_kwargs", {}) + assert self.truncation in ["error", "left", "right"] + + if not isinstance(parquet_files, list | ListConfig): + parquet_files = [parquet_files] + + self.parquet_files = parquet_files + if isinstance(tokenizer, str): + tokenizer = hf_tokenizer(tokenizer) + self.tokenizer: PreTrainedTokenizer = tokenizer + + self._download() + self._read_files_and_process() + + def _download(self): + for i, parquet_file in enumerate(self.parquet_files): + self.parquet_files[i] = copy_local_path_from_hdfs(parquet_file, verbose=True) + + def _read_files_and_process(self): + def series_to_item(ls): + import numpy + import pandas + + while isinstance(ls, pandas.core.series.Series | numpy.ndarray) and len(ls) == 1: + ls = ls[0] + return ls + + dataframes = [] + for parquet_file in self.parquet_files: + dataframe = pd.read_parquet(parquet_file) + dataframes.append(dataframe) + self.dataframe = pd.concat(dataframes) + + # Extract messages list from dataframe + self.messages = self.dataframe[self.messages_key].apply(series_to_item).tolist() + + # Extract tools list from dataframe + if self.tools_key in self.dataframe.columns: + self.tools = self.dataframe[self.tools_key].apply(convert_nested_value_to_list_recursive).tolist() + else: + self.tools = None + # Extract enable_thinking list from dataframe + if self.enable_thinking_key in self.dataframe.columns: + self.enable_thinking = self.dataframe[self.enable_thinking_key].tolist() + else: + self.enable_thinking = None + + def __len__(self): + return len(self.messages) + + def _process_message_tokens( + self, + messages: list[dict[str, Any]], + start_idx: int, + end_idx: int, + is_assistant: bool = False, + enable_thinking: Optional[bool] = None, + tools: Optional[list[dict[str, Any]]] = None, + ) -> tuple[list[int], list[int], list[int]]: + """ + Process tokens for a single message or a group of messages. + + Args: + messages: List of message dictionaries + start_idx: Start index in messages list + end_idx: End index in messages list + is_assistant: Whether this is an assistant message + enable_thinking: Whether to enable thinking mode + + Returns: + Tuple of (tokens, loss_mask, attention_mask) + """ + if start_idx > 0: + prev_applied_text = self.tokenizer.apply_chat_template( + messages[:start_idx], + tokenize=False, + add_generation_prompt=False, + enable_thinking=enable_thinking, + tools=tools, + **self.apply_chat_template_kwargs, + ) + if is_assistant: + prev_applied_text_w_generation_prompt = self.tokenizer.apply_chat_template( + messages[:start_idx], + tokenize=False, + add_generation_prompt=True, + enable_thinking=enable_thinking, + tools=tools, + **self.apply_chat_template_kwargs, + ) + + else: + prev_applied_text = "" + + cur_applied_text = self.tokenizer.apply_chat_template( + messages[:end_idx], + tokenize=False, + add_generation_prompt=False, + enable_thinking=enable_thinking, + tools=tools, + **self.apply_chat_template_kwargs, + ) + # Get tokens for the current message only + if is_assistant: + generation_prompt_text = prev_applied_text_w_generation_prompt[len(prev_applied_text) :] + generation_prompt_tokens = self.tokenizer.encode( + generation_prompt_text, + add_special_tokens=False, + ) + _message_tokens = self.tokenizer.encode( + cur_applied_text[len(prev_applied_text_w_generation_prompt) :], + add_special_tokens=False, + ) + message_tokens = generation_prompt_tokens + _message_tokens + loss_mask = [0] * (len(generation_prompt_tokens)) + [1] * ( + len(message_tokens) - len(generation_prompt_tokens) + ) + else: + message_tokens = self.tokenizer.encode( + cur_applied_text[len(prev_applied_text) :], + add_special_tokens=False, + ) + loss_mask = [0] * len(message_tokens) + + attention_mask = [1] * len(message_tokens) + + return message_tokens, loss_mask, attention_mask + + def _validate_and_convert_tokens( + self, + full_tokens: torch.Tensor, + concat_tokens: list[int], + concat_loss_mask: list[int], + concat_attention_mask: list[int], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Validate tokenization and convert to tensors. + + Args: + full_tokens: Full conversation tokens + concat_tokens: Concatenated tokens + concat_loss_mask: Concatenated loss mask + concat_attention_mask: Concatenated attention mask + + Returns: + Tuple of (input_ids, loss_mask, attention_mask) as tensors + """ + full_tokens_list = full_tokens.tolist() + + if len(concat_tokens) != len(full_tokens_list) or not all( + a == b for a, b in zip(concat_tokens, full_tokens_list, strict=True) + ): + logging.warning( + f"Token mismatch detected! Full tokenization length: {len(full_tokens_list)}, Concatenated tokens " + f"length: {len(concat_tokens)}. Using concatenated version." + # f"full tokens text: {self.tokenizer.decode(full_tokens_list)}" + # f"concat tokens text: {self.tokenizer.decode(concat_tokens)}" + ) + return ( + torch.tensor(concat_tokens, dtype=torch.long), + torch.tensor(concat_loss_mask, dtype=torch.long), + torch.tensor(concat_attention_mask, dtype=torch.long), + ) + + return ( + full_tokens, + torch.tensor(concat_loss_mask, dtype=torch.long), + torch.tensor(concat_attention_mask, dtype=torch.long), + ) + + def __getitem__(self, item): + tokenizer = self.tokenizer + messages = self.messages[item] + tools = self.tools[item] if self.tools is not None else None + enable_thinking = self.enable_thinking[item] if self.enable_thinking is not None else None + + # First, get the full conversation tokens + try: + full_tokens = tokenizer.apply_chat_template( + messages, + tools=tools, + tokenize=True, + return_tensors="pt", + add_generation_prompt=False, + enable_thinking=enable_thinking, + **self.apply_chat_template_kwargs, + ) + except Exception as e: + logging.error( + f"Error applying chat template: {e}\nMessages: {messages}\nTools: {tools}\nEnable thinking: " + f"{enable_thinking}" + ) + raise + + # Track concatenated tokens for validation + concat_tokens = [] + concat_loss_mask = [] + concat_attention_mask = [] + + i = 0 + while i < len(messages): + cur_messages = messages[i] + if cur_messages["role"] == "assistant": + # Process assistant message + tokens, loss_mask, attention_mask = self._process_message_tokens( + messages, i, i + 1, is_assistant=True, enable_thinking=enable_thinking, tools=tools + ) + i += 1 + elif cur_messages["role"] == "tool": + # Process consecutive tool messages + st = i + ed = i + 1 + while ed < len(messages) and messages[ed]["role"] == "tool": + ed += 1 + tokens, loss_mask, attention_mask = self._process_message_tokens( + messages, st, ed, enable_thinking=enable_thinking, tools=tools + ) + i = ed + elif cur_messages["role"] in ["user", "system"]: + # Process user or system message + if cur_messages["role"] == "system" and i != 0: + raise ValueError("System message should be the first message") + tokens, loss_mask, attention_mask = self._process_message_tokens( + messages, i, i + 1, enable_thinking=enable_thinking, tools=tools + ) + i += 1 + else: + raise ValueError(f"Unknown role: {cur_messages['role']}") + + # override loss mask with mask in the dataset to handle multi-turn conversation + override_loss_mask = cur_messages.get("loss_mask", None) + if override_loss_mask is not None: + if isinstance(override_loss_mask, np.ndarray): + override_loss_mask = override_loss_mask.item() + assert isinstance(override_loss_mask, int), f"loss_mask should be int, got {type(override_loss_mask)}" + assert override_loss_mask in [0, 1], f"loss_mask should be 0 or 1, got {override_loss_mask}" + loss_mask = [override_loss_mask] * len(tokens) + + concat_tokens.extend(tokens) + concat_loss_mask.extend(loss_mask) + concat_attention_mask.extend(attention_mask) + + # Validate and convert tokens + input_ids, loss_mask, attention_mask = self._validate_and_convert_tokens( + full_tokens[0], concat_tokens, concat_loss_mask, concat_attention_mask + ) + + # encode prompt + if messages[0]["role"] == "system": + assert messages[1]["role"] == "user" + assert messages[2]["role"] == "assistant" + prompt_message_length = 2 + elif messages[0]["role"] == "user": + assert messages[1]["role"] == "assistant" + prompt_message_length = 1 + else: + raise ValueError(f"Unknown role: {messages[0]['role']}") + + sequence_length = input_ids.shape[0] + # Handle sequence length + if self.pad_mode == DatasetPadMode.RIGHT: + if sequence_length < self.max_length: + # Pad sequences + pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else 0 + padded_input_ids = torch.full((self.max_length - sequence_length,), pad_token_id, dtype=input_ids.dtype) + padded_attention_mask = torch.zeros((self.max_length - sequence_length,), dtype=attention_mask.dtype) + padded_loss_mask = torch.zeros((self.max_length - sequence_length,), dtype=loss_mask.dtype) + + input_ids = torch.cat((input_ids, padded_input_ids)) + attention_mask = torch.cat((attention_mask, padded_attention_mask)) + loss_mask = torch.cat((loss_mask, padded_loss_mask)) + elif sequence_length > self.max_length: + if self.truncation == "left": + input_ids = input_ids[-self.max_length :] + attention_mask = attention_mask[-self.max_length :] + loss_mask = loss_mask[-self.max_length :] + elif self.truncation == "right": + input_ids = input_ids[: self.max_length] + attention_mask = attention_mask[: self.max_length] + loss_mask = loss_mask[: self.max_length] + elif self.truncation == "error": + raise ValueError(f"{sequence_length=} is larger than {self.max_length=}") + else: + raise ValueError(f"Unknown truncation method {self.truncation}") + + # Create position IDs + position_ids = torch.arange(len(input_ids), dtype=torch.long) + # Zero out position IDs for padding + position_ids = position_ids * attention_mask + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "loss_mask": loss_mask, + } + elif self.pad_mode == DatasetPadMode.LEFT_RIGHT: + assert self.truncation == "error", "Only support error truncation for left_right pad mode" + prompt_str = self.tokenizer.apply_chat_template( + messages[:prompt_message_length], + tools=tools, + tokenize=False, + add_generation_prompt=True, + enable_thinking=enable_thinking, + **self.apply_chat_template_kwargs, + ) + prompt_ids = self.tokenizer.encode(prompt_str, add_special_tokens=False) + prompt_length = len(prompt_ids) + prompt_ids = input_ids[:prompt_length].unsqueeze(0) + prompt_attention_mask = attention_mask[:prompt_length].unsqueeze(0) + prompt_loss_mask = loss_mask[:prompt_length].unsqueeze(0) + response_ids = input_ids[prompt_length:].unsqueeze(0) + response_attention_mask = attention_mask[prompt_length:].unsqueeze(0) + response_loss_mask = loss_mask[prompt_length:].unsqueeze(0) + + assert prompt_loss_mask.sum().item() == 0 + + prompt_ids, prompt_attention_mask = postprocess_data( + input_ids=prompt_ids, + attention_mask=prompt_attention_mask, + max_length=self.max_prompt_length, + pad_token_id=self.tokenizer.pad_token_id, + left_pad=True, + truncation=self.truncation, + ) + + response_ids, response_attention_mask = postprocess_data( + input_ids=response_ids, + attention_mask=response_attention_mask, + max_length=self.max_response_length, + pad_token_id=self.tokenizer.pad_token_id, + left_pad=False, + truncation=self.truncation, + ) + response_loss_mask = pad_sequence_to_length( + response_loss_mask, max_seq_len=self.max_response_length, pad_token_id=0, left_pad=False + ) + + prompt_ids = prompt_ids[0] + prompt_attention_mask = prompt_attention_mask[0] + response_ids = response_ids[0] + response_attention_mask = response_attention_mask[0] + response_loss_mask = response_loss_mask[0] + + assert response_attention_mask[0].item() == 1 + assert response_loss_mask[0].item() == 1 + + input_ids = torch.cat((prompt_ids, response_ids), dim=0) + attention_mask = torch.cat((prompt_attention_mask, response_attention_mask), dim=0) + position_ids = compute_position_id_with_mask(attention_mask) + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "responses": response_ids, + "response_mask": response_loss_mask, + } + elif self.pad_mode == DatasetPadMode.NO_PADDING: + # truncate input_ids if it is longer than max_length + if len(input_ids) > self.max_length: + input_ids = input_ids[: self.max_length] + loss_mask = loss_mask[: self.max_length] + # create position IDs + position_ids = torch.arange(len(input_ids), dtype=torch.long) + # return nested tensor with out padding + return { + "input_ids": input_ids, + "position_ids": position_ids, + "loss_mask": loss_mask, + } diff --git a/e2c/verl/verl/utils/dataset/rl_dataset.py b/e2c/verl/verl/utils/dataset/rl_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..15e197577454a0a14cb1858363829101dc6ccd6c --- /dev/null +++ b/e2c/verl/verl/utils/dataset/rl_dataset.py @@ -0,0 +1,379 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import logging +import os +import re +from collections import defaultdict +from typing import Optional + +import datasets +import numpy as np +import torch +from omegaconf import DictConfig, ListConfig +from torch.utils.data import Dataset +from transformers import PreTrainedTokenizer, ProcessorMixin + +import verl.utils.torch_functional as verl_F +from verl.utils.model import compute_position_id_with_mask + +logger = logging.getLogger(__name__) + + +def collate_fn(data_list: list[dict]) -> dict: + """ + Collate a batch of sample dicts into batched tensors and arrays. + + Args: + data_list: List of dicts mapping feature names to torch.Tensor or other values. + + Returns: + Dict where tensor entries are stacked into a torch.Tensor of shape + (batch_size, \*dims) and non-tensor entries are converted to + np.ndarray of dtype object with shape (batch_size,). + """ + tensors = defaultdict(list) + non_tensors = defaultdict(list) + + for data in data_list: + for key, val in data.items(): + if isinstance(val, torch.Tensor): + tensors[key].append(val) + else: + non_tensors[key].append(val) + + for key, val in tensors.items(): + tensors[key] = torch.stack(val, dim=0) + + for key, val in non_tensors.items(): + non_tensors[key] = np.fromiter(val, dtype=object, count=len(val)) + + return {**tensors, **non_tensors} + + +class RLHFDataset(Dataset): + """ + Load and preprocess RLHF data from Parquet files. + + - Caches files locally. + - Reads into a HuggingFace Dataset and tokenizes prompts. + - Optionally handles images/videos via a ProcessorMixin. + - Filters prompts over a max length. + - Supports resuming from checkpoints. + + Args: + data_files (str or list): Path(s) to Parquet file(s). + tokenizer (PreTrainedTokenizer): For the tokenization of text to token IDs. + config (DictConfig): Options like cache_dir, prompt_key, max_prompt_length, truncation, etc. + processor (ProcessorMixin, optional): Multimodal preprocessor for images/videos. + """ + + def __init__( + self, + data_files: str | list[str], + tokenizer: PreTrainedTokenizer, + config: DictConfig, + processor: Optional[ProcessorMixin] = None, + ): + if not isinstance(data_files, list | ListConfig): + data_files = [data_files] + + self.data_files = copy.deepcopy(data_files) + self.original_data_files = copy.deepcopy(data_files) # use for resume + self.tokenizer = tokenizer + self.processor = processor + self.config = config + + self.cache_dir = os.path.expanduser(config.get("cache_dir", "~/.cache/verl/rlhf")) + self.prompt_key = config.get("prompt_key", "prompt") + self.image_key = config.get("image_key", "images") + self.video_key = config.get("video_key", "videos") + self.max_prompt_length = config.get("max_prompt_length", 1024) + self.return_raw_chat = config.get("return_raw_chat", False) + self.return_full_prompt = config.get("return_full_prompt", False) + self.truncation = config.get("truncation", "error") + self.filter_overlong_prompts = config.get("filter_overlong_prompts", True) + self.apply_chat_template_kwargs = config.get("apply_chat_template_kwargs", {}) + + self.num_workers = config.get("filter_overlong_prompts_workers", max(1, os.cpu_count() // 4)) + self.num_workers = min(self.num_workers, os.cpu_count()) + self.use_shm = config.get("use_shm", False) + self.chat_template_func = config.get("chat_template_func", None) + self.need_tools_kwargs = config.get("need_tools_kwargs", False) + self.filter_prompts = config.get("filter_prompts", True) + self.serialize_dataset = False + self.return_multi_modal_inputs = config.get("return_multi_modal_inputs", True) + + self._download() + self._read_files_and_tokenize() + + def _download(self, use_origin_parquet=False): + from verl.utils.fs import copy_to_local + + data_files = self.data_files if not use_origin_parquet else self.original_data_files + for i, parquet_file in enumerate(data_files): + self.data_files[i] = copy_to_local(src=parquet_file, cache_dir=self.cache_dir, use_shm=self.use_shm) + + def _read_files_and_tokenize(self): + dataframes = [] + for parquet_file in self.data_files: + # read parquet files and cache + dataframe = datasets.load_dataset("parquet", data_files=parquet_file)["train"] + dataframes.append(dataframe) + self.dataframe: datasets.Dataset = datasets.concatenate_datasets(dataframes) + + print(f"dataset len: {len(self.dataframe)}") + + self.dataframe = self.maybe_filter_out_long_prompts(self.dataframe) + + def maybe_filter_out_long_prompts(self, dataframe: datasets.Dataset = None): + # filter out too long prompts + if self.filter_overlong_prompts: + tokenizer = self.tokenizer + processor = self.processor + prompt_key = self.prompt_key + image_key = self.image_key + video_key = self.video_key + + if processor is not None: + from verl.utils.dataset.vision_utils import process_image, process_video + + def doc2len(doc) -> int: + messages = self._build_messages(doc) + raw_prompt = self.processor.apply_chat_template( + messages, add_generation_prompt=True, tokenize=False, **self.apply_chat_template_kwargs + ) + images = ( + [process_image(image) for image in doc[image_key]] + if image_key in doc and doc[image_key] + else None + ) + videos = ( + [process_video(video) for video in doc[video_key]] + if video_key in doc and doc[video_key] + else None + ) + + return len(processor(text=[raw_prompt], images=images, videos=videos)["input_ids"][0]) + + else: + + def doc2len(doc) -> int: + return len( + tokenizer.apply_chat_template( + doc[prompt_key], add_generation_prompt=True, **self.apply_chat_template_kwargs + ) + ) + + dataframe = dataframe.filter( + lambda doc: doc2len(doc) <= self.max_prompt_length, + num_proc=self.num_workers, + desc=f"Filtering prompts longer than {self.max_prompt_length} tokens", + ) + + print(f"filter dataset len: {len(dataframe)}") + return dataframe + + def resume_dataset_state(self): + self.serialize_dataset = not hasattr(self, "original_data_files") + # resume dataframe if not it's serialized in data.pt + if not self.serialize_dataset: + self._download(use_origin_parquet=True) # download and resume from original parquet files + self._read_files_and_tokenize() + else: + print(r"old dataloader ckpt file is used, please train from scratch for better ckpt performance") + + def __len__(self): + return len(self.dataframe) + + def _build_messages(self, example: dict): + messages: list = example.pop(self.prompt_key) + + if self.image_key in example or self.video_key in example: + for message in messages: + content = message["content"] + content_list = [] + segments = re.split("(|
\n<|assistant|>\n", " [/INST]") + .replace(" \n<|assistant|>\n", " [/INST]") + + "" + ) + if print_decode and torch.distributed.get_rank() == 0: + # only print first decode result + print( + f"device {get_device_id()}: sft decode result:\n{decode_result}\n \ + \ndevice {get_device_id()}: sft decode result with \ + rm chat template:\n{decode_with_rm_chat}\n\n" + ) + print_decode = False + # 3. encode by rm_tokenizer + rm_input_ids = self.rm_tokenizer(decode_with_rm_chat, return_tensors="pt")["input_ids"][0].to( + input_ids.device + ) + # 4. generate attention_mask and position_ids + rm_attention_mask = torch.ones_like(rm_input_ids, device=input_ids.device) + cur_seqlen = rm_input_ids.shape[-1] + # NOTE(gh): the later reward compute will process the shape (bs, seqlen_pad_128) + if cur_seqlen > ori_seqlen: + print(f"warninig: rm encode seqlen {cur_seqlen} > sft encode seqlen {ori_seqlen}") + rm_input_ids = rm_input_ids[:ori_seqlen] + rm_attention_mask = rm_attention_mask[:ori_seqlen] + else: + # right padding + rm_input_ids = pad_sequence_to_length(rm_input_ids, ori_seqlen, self.rm_tokenizer.pad_token_id) + rm_attention_mask = pad_sequence_to_length(rm_attention_mask, ori_seqlen, 0) + rm_position_ids = torch.arange(0, ori_seqlen, device=input_ids.device) + input_ids_for_rm.append(torch.unsqueeze(rm_input_ids, dim=0)) + attention_mask_for_rm.append(torch.unsqueeze(rm_attention_mask, dim=0)) + position_ids_for_rm.append(torch.unsqueeze(rm_position_ids, dim=0)) + input_ids_for_rm = torch.cat(input_ids_for_rm, dim=0) + attention_mask_for_rm = torch.cat(attention_mask_for_rm, dim=0) + position_ids_for_rm = torch.cat(position_ids_for_rm, dim=0) + + # (bs, seqlen) will not change, but input_ids, attention_mask and position_ids will change + # NOTE(gh): need to replace into origin values after compute reward! + data.batch["input_ids"] = input_ids_for_rm + data.batch["attention_mask"] = attention_mask_for_rm + data.batch["position_ids"] = position_ids_for_rm + + return data, ori_values + + @torch.no_grad() + def compute_reward(self, data: DataProto) -> DataProto: + if self.config.megatron.param_offload: + self.load_params_to_cuda() + + if self.use_different_tokenizer: + data, ori_values = self.re_encode_by_rm_tokenizer(data) + + input_ids = data.batch["input_ids"] # (bs, seq_len') + attention_mask = data.batch["attention_mask"] + position_ids = data.batch["position_ids"] + use_dynamic_bsz = data.meta_info.get("use_dynamic_bsz", False) + micro_batch_size = data.meta_info.get("micro_batch_size", None) + max_token_len = data.meta_info.get("max_token_len", None) + assert micro_batch_size is not None, "micro batch size is needed for forward compute" + if use_dynamic_bsz: + assert max_token_len is not None, "use_dynamic_bsz is True, but max_token_len is None!" + max_token_len = max_token_len * self.config.megatron.context_parallel_size + + responses = data.batch["responses"] + batch_size = responses.size(0) + response_length = responses.size(1) + + with torch.no_grad(): + output = self.forward_batch( + data, use_dynamic_bsz=use_dynamic_bsz, micro_batch_size=micro_batch_size, max_token_len=max_token_len + ) + if mpu.is_pipeline_last_stage(ignore_virtual=True): + logits = torch.cat(output["output"], dim=0) + if use_dynamic_bsz: + indices = output["indices"] + indices = list(itertools.chain.from_iterable(indices)) + assert len(indices) == logits.size(0), f"{len(indices)} vs. {logits.size()}" + revert_indices = torch.tensor(get_reverse_idx(indices), dtype=torch.long) + logits = logits[revert_indices] + else: + logits = torch.empty( + (input_ids.shape[0], input_ids.shape[1]), + device=input_ids.device, + ) + logits = logits.to(torch.float32) + + # broadcast across pp ranks + torch.distributed.broadcast( + tensor=logits, + src=mpu.get_pipeline_model_parallel_last_rank(), + group=mpu.get_pipeline_model_parallel_group(), + async_op=False, + ) + + # (bs, seqlen', hidden_size) -> (bs, seqlen', 1) -> (bs, seqlen') + token_level_rewards = logits + # find the last token reward + ends = attention_mask.cumsum(dim=-1).argmax(dim=-1).view(-1, 1) # (bs, 1) + rewards = torch.gather(token_level_rewards, dim=1, index=ends) # (bs, 1) + + if self.use_different_tokenizer: + data.batch.update(ori_values) + input_ids = ori_values["input_ids"] + attention_mask = ori_values["attention_mask"] + position_ids = ori_values["position_ids"] + + token_level_rewards = rewards.expand(attention_mask.shape[0], attention_mask.shape[1]) # (bs, ori_seqlen) + + # assign last valid token reward to ori position + if position_ids.dim() == 3: # qwen2vl mrope [bs, 3, seq_len] + position_ids = position_ids[:, 0, :] + eos_mask_idx = torch.argmax(position_ids * attention_mask, dim=-1) # (bs,) + eos_mask = torch.zeros_like(attention_mask) + eos_mask[torch.arange(batch_size), eos_mask_idx] = 1.0 + + token_level_rewards = token_level_rewards * eos_mask + token_level_rewards = token_level_rewards[:, -response_length:] + + if self.config.megatron.param_offload: + self.offload_params_to_cpu() + else: + # add empty cache after each compute + get_torch_device().empty_cache() + + batch = TensorDict({"rm_scores": token_level_rewards}, batch_size=input_ids.shape[0]) + + return DataProto(batch=batch) + + def forward_batch(self, data: DataProto, use_dynamic_bsz=False, micro_batch_size=None, max_token_len=None): + """ + We assume: + - The model takes input: (input_ids, attention_mask, position_ids). No rmpad for the input + - The communication shape is (total_nnz_pad_to_sp // tp_size, 1, hidden_size) if sequence parallel is enabled + """ + # broadcast from last pp rank to all other pp ranks + # TODO: actually, we just need to control the sampling order. + mini_batch = data + mini_batch.batch = mini_batch.batch.contiguous() + broadcast_dict_tensor( + mini_batch.batch, + src=mpu.get_pipeline_model_parallel_last_rank(), + group=mpu.get_pipeline_model_parallel_group(), + ) + + mini_batch.batch["attention_mask"] = mini_batch.batch["attention_mask"].to(bool) + + self.has_multi_modal_inputs = "multi_modal_inputs" in mini_batch.non_tensor_batch.keys() + if self.has_multi_modal_inputs: + mini_batch.batch["multi_modal_inputs"] = mini_batch.non_tensor_batch["multi_modal_inputs"] + mini_batch.batch["multi_modal_inputs_idx"] = torch.Tensor( + list(range(len(mini_batch.non_tensor_batch["multi_modal_inputs"]))) + ).to(torch.int64) + + indices = None + if use_dynamic_bsz: + assert max_token_len is not None, "max_token_len must be set when use_dynamic_bsz is True" + vpp_size = mpu.get_virtual_pipeline_model_parallel_world_size() + if vpp_size is not None and vpp_size > 1: + microbatch_group_size_per_vp_stage = self.tf_config.microbatch_group_size_per_vp_stage + micro_batches, indices = rearrange_micro_batches( + batch=mini_batch.batch, + num_batches_divided_by=microbatch_group_size_per_vp_stage, + max_token_len=max_token_len, + ) + assert len(micro_batches) % self.tf_config.microbatch_group_size_per_vp_stage == 0, ( + f"micro_batches {micro_batches} must be divisible by microbatch_group_size_per_vp_stage " + f"{microbatch_group_size_per_vp_stage} for megatron backend" + ) + else: + micro_batches, indices = rearrange_micro_batches(batch=mini_batch.batch, max_token_len=max_token_len) + total_seqlen = max_token_len + else: + assert micro_batch_size is not None, ( + "micro_batch_size is needed to be passed in when not using dynamic batch size" + ) + micro_batches = mini_batch.batch.split(micro_batch_size) + seq_len = micro_batches[0]["input_ids"].shape[1] + total_seqlen = micro_batch_size * seq_len + n_micro_batch = len(micro_batches) + + # compute input shapes for pp stages + forward_backward_func = get_forward_backward_func() + + def loss_func(output): + return torch.tensor(1.0, device=output.device), output + + def forward_step(batch_iter, model): + batch = next(batch_iter) + input_ids = batch["input_ids"] + attention_mask = batch["attention_mask"] + position_ids = batch["position_ids"] + from verl.models.mcore import get_mcore_forward_fn + + forward_fn = get_mcore_forward_fn(self.hf_config) + + multi_modal_inputs = {} + if "multi_modal_inputs" in batch: + from verl.utils.model import extract_multi_modal_inputs + + indices = batch.get("multi_modal_inputs_idx", None) + multi_modal_inputs = extract_multi_modal_inputs(batch["multi_modal_inputs"], indices) + output = forward_fn( + model, + input_ids, + attention_mask, + position_ids, + sequence_parallel=self.tf_config.sequence_parallel, + value_model=True, + multi_modal_inputs=multi_modal_inputs, + ) + + return output, loss_func + + # batch should be a list of batches inside micro-batches + batch_generator = make_batch_generator(micro_batches, vpp_size=len(self.reward_model_module)) + + # TODO: we may use the new schedule instead + # for flash-attn: (seq_len, batch_size, hidden_size) = (mbs*seq_len, 1, hidden_size) + if mpu.get_pipeline_model_parallel_world_size() > 1: + losses_reduced = forward_backward_func( + forward_step_func=forward_step, + data_iterator=batch_generator, + model=self.reward_model_module, + num_microbatches=n_micro_batch, + seq_length=total_seqlen, # no use when input_shapes was set + micro_batch_size=1, # no use when input_shapes was set + forward_only=True, + ) + else: + losses_reduced = forward_backward_func( + forward_step_func=forward_step, + data_iterator=batch_generator, + model=self.reward_model_module, + num_microbatches=n_micro_batch, + seq_length=total_seqlen, # in use for pp = 1 + micro_batch_size=1, # in use for pp = 1 + forward_only=True, + ) + + if self.has_multi_modal_inputs: + data.batch.pop("multi_modal_inputs") + data.batch.pop("multi_modal_inputs_idx") + data.non_tensor_batch.pop("multi_modal_inputs") + # loss_reduces contains the stats returned from loss_func + losses_reduced = {"output": losses_reduced} + if use_dynamic_bsz: + losses_reduced["indices"] = indices + return losses_reduced + + def offload_params_to_cpu(self): + if self.device in ["cuda", "npu"]: + for reward_model_module in self.reward_model_module: + for name, param in reward_model_module.named_parameters(): + param.data = param.data.to("cpu", non_blocking=True) + self.device = "cpu" + get_torch_device().empty_cache() + + def load_params_to_cuda(self): + if self.device == "cpu": + for reward_model_module in self.reward_model_module: + for name, param in reward_model_module.named_parameters(): + param.data = param.data.to(get_device_id(), non_blocking=True) + self.device = get_device_name() diff --git a/e2c/verl/verl/workers/roles/__init__.py b/e2c/verl/verl/workers/roles/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7c50a44650517821fa0ccaf192f54b95dcb8e8b2 --- /dev/null +++ b/e2c/verl/verl/workers/roles/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .actor import ActorWorker +from .critic import CriticWorker + +try: + from .reward_model import RewardModelWorker +except ImportError: + RewardModelWorker = None + +__all__ = ["CriticWorker", "ActorWorker"] + +if RewardModelWorker is not None: + __all__.append("RewardModelWorker") diff --git a/e2c/verl/verl/workers/roles/actor.py b/e2c/verl/verl/workers/roles/actor.py new file mode 100644 index 0000000000000000000000000000000000000000..b9b70de0edd04af1bded39fcaa2af975fe6219bc --- /dev/null +++ b/e2c/verl/verl/workers/roles/actor.py @@ -0,0 +1,184 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from functools import partial + +import psutil +from codetiming import Timer + +from verl import DataProto +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.utils.device import ( + get_device_id, + get_device_name, + get_torch_device, +) +from verl.utils.distributed import initialize_global_process_group_ray +from verl.utils.flops_counter import FlopsCounter +from verl.utils.profiler import DistProfiler, DistProfilerExtension +from verl.utils.py_functional import append_to_dict +from verl.workers.config import ActorConfig +from verl.workers.roles.utils.losses import ppo_loss + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +device_name = get_device_name() + + +class ActorWorker(Worker, DistProfilerExtension): + """ + This worker can be instantiated as a standalone actor or a standalone rollout or a standalone reference policy + or a hybrid engine based on the config.rollout + """ + + def __init__(self, config: ActorConfig): + self.config = config + Worker.__init__(self) + self.profiler_config = self.config.profiler + tool_config = self.profiler_config.tool_config + DistProfilerExtension.__init__( + self, DistProfiler(rank=self.rank, config=self.profiler_config, tool_config=tool_config) + ) + + initialize_global_process_group_ray(timeout_second=None) + + self.loss_fn = partial(ppo_loss, config=self.config) + + def _build_engine(self): + self.model_config = self.config.model_config + self.engine_config = self.config.engine + self.optimizer_config = self.config.optim + self.checkpoint_config = self.config.checkpoint + + from verl.workers.engine import BaseEngine, EngineRegistry + + self.engine: BaseEngine = EngineRegistry.new( + model_type="language_model", + backend=self.config.strategy, + model_config=self.model_config, + engine_config=self.engine_config, + optimizer_config=self.optimizer_config, + checkpoint_config=self.checkpoint_config, + ) + + # build dispatch info + self._register_dispatch_collect_info( + mesh_name="actor", + dp_rank=self.engine.get_data_parallel_rank(), + is_collect=self.engine.is_mp_src_rank_with_outputs(), + ) + + # aggregate with bon sampling + self.ppo_mini_batch_size = self.config.ppo_mini_batch_size * self.config.rollout_n + assert self.ppo_mini_batch_size % self.engine.get_data_parallel_size() == 0, ( + f"{self.ppo_mini_batch_size=} is not divisible by {self.engine.get_data_parallel_size()=}" + ) + self.ppo_mini_batch_size_per_dp = self.ppo_mini_batch_size // self.engine.get_data_parallel_size() + + # setup flops counter + self.flops_counter = FlopsCounter(self.model_config.hf_config) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + self._build_engine() + self.engine.initialize() + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_loss_fn(self, loss_fn): + self.loss_fn = loss_fn + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") + def compute_log_prob(self, data: DataProto): + data.meta_info["use_dynamic_bsz"] = self.config.use_dynamic_bsz + data.meta_info["use_fused_kernels"] = self.config.use_fused_kernels + data.meta_info["calculate_entropy"] = True + if self.config.use_dynamic_bsz: + data.meta_info["max_token_len_per_gpu"] = self.config.ppo_infer_max_token_len_per_gpu + else: + data.meta_info["micro_batch_size_per_gpu"] = self.config.ppo_infer_micro_batch_size_per_gpu + + with self.engine.eval_mode(): + # TODO: make worker API to accept TensorDict as well + data = data.to_tensordict() + output = self.engine.infer_batch(data) + + if self.engine.is_mp_src_rank_with_outputs(): + output = output["model_output"] + # in megatron, only last pp contains valid data and returned to the single controller + output = DataProto.from_dict( + tensors={"old_log_probs": output["log_probs"].float(), "entropy": output["entropy"].float()}, + ) + output = output.to("cpu") + + return output + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + @DistProfiler.annotate(color="red", role="actor_update") + def update_actor(self, data: DataProto): + data.meta_info["use_dynamic_bsz"] = self.config.use_dynamic_bsz + data.meta_info["use_fused_kernels"] = self.config.use_fused_kernels + data.meta_info["calculate_entropy"] = self.config.entropy_coeff != 0.0 + if self.config.use_dynamic_bsz: + data.meta_info["max_token_len_per_gpu"] = self.config.ppo_max_token_len_per_gpu + else: + data.meta_info["micro_batch_size_per_gpu"] = self.config.ppo_micro_batch_size_per_gpu + + metrics = {} + # Support all hardwares + data = data.to(get_device_id()) + # perform forward computation + with self.engine.train_mode(): + dataloader = data.make_iterator( + mini_batch_size=self.ppo_mini_batch_size_per_dp, + epochs=self.config.ppo_epochs, + seed=self.config.data_loader_seed + self.engine.get_data_parallel_rank(), + dataloader_kwargs={"shuffle": self.config.shuffle}, + ) + with Timer(name="update_policy", logger=None) as timer: + for batch_idx, mini_batch in enumerate(dataloader): + mini_batch.meta_info["global_batch_size"] = self.config.ppo_mini_batch_size + # TODO: make worker API to accept TensorDict as well + mini_batch = mini_batch.to_tensordict() + output = self.engine.train_batch(mini_batch, self.loss_fn) + mini_batch_metrics = output.get("metrics", {}) + append_to_dict(metrics, mini_batch_metrics, prefix="actor/") + + delta_time = timer.last + + global_num_tokens = data.meta_info["global_token_num"] + estimated_flops, promised_flops = self.flops_counter.estimate_flops(global_num_tokens, delta_time) + metrics["perf/mfu/actor"] = estimated_flops * self.config.ppo_epochs / promised_flops / self.world_size + metrics["perf/max_memory_allocated_gb"] = get_torch_device().max_memory_allocated() / (1024**3) + metrics["perf/max_memory_reserved_gb"] = get_torch_device().max_memory_reserved() / (1024**3) + metrics["perf/cpu_memory_used_gb"] = psutil.virtual_memory().used / (1024**3) + + lr = self.engine.lr_scheduler_step() + metrics["actor/lr"] = lr + + output = DataProto(batch=None, meta_info={"metrics": metrics}) + + return output + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): + return self.engine.save_checkpoint(local_path, hdfs_path, global_step, max_ckpt_to_keep) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=False): + return self.engine.load_checkpoint(local_path, hdfs_path, del_local_after_load) diff --git a/e2c/verl/verl/workers/roles/critic.py b/e2c/verl/verl/workers/roles/critic.py new file mode 100644 index 0000000000000000000000000000000000000000..24b644149fd3dc865e84c6697df7813c5633a7c7 --- /dev/null +++ b/e2c/verl/verl/workers/roles/critic.py @@ -0,0 +1,206 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import warnings +from functools import partial + +import psutil +from codetiming import Timer + +from verl import DataProto +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.utils.device import ( + get_device_id, + get_device_name, + get_torch_device, +) +from verl.utils.distributed import initialize_global_process_group_ray +from verl.utils.flops_counter import FlopsCounter +from verl.utils.profiler import DistProfiler, DistProfilerExtension +from verl.utils.py_functional import append_to_dict +from verl.workers.config import CriticConfig +from verl.workers.roles.utils.losses import value_loss + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +device_name = get_device_name() + + +class CriticWorker(Worker, DistProfilerExtension): + """ + This worker can be instantiated as a standalone actor or a standalone rollout or a standalone reference policy + or a hybrid engine based on the config.rollout + """ + + def __init__(self, config: CriticConfig): + self.config = config + Worker.__init__(self) + self.profiler_config = self.config.profiler + tool_config = self.profiler_config.tool_config + DistProfilerExtension.__init__( + self, DistProfiler(rank=self.rank, config=self.profiler_config, tool_config=tool_config) + ) + + initialize_global_process_group_ray(timeout_second=None) + + self.loss_fn = partial(value_loss, config=self.config) + + def _build_engine(self): + from copy import copy, deepcopy + + self.model_config = copy(self.config.model_config) + self.model_config.hf_config = deepcopy(self.config.model_config.hf_config) + self.engine_config = self.config.engine + self.optimizer_config = self.config.optim + self.checkpoint_config = self.config.checkpoint + + from verl.workers.engine import BaseEngine, EngineRegistry + + # replace AutoModelForSequenceClassification to AutoModelForTokenClassification + hf_config = self.model_config.hf_config + + arch = hf_config.architectures[0] + # This logic assumes the critic is a token classification model. + # If the provided model is a CausalLM, we adapt it. + if "ForCausalLM" in arch: + model_name = arch.split("ForCausalLM")[0] + new_arch = f"{model_name}ForTokenClassification" + warnings.warn(f"Implicitly changing critic architecture from '{arch}' to '{new_arch}'", stacklevel=2) + hf_config.architectures[0] = new_arch + elif "ForTokenClassification" not in arch and "ForSequenceClassification" not in arch: + raise ValueError( + f"Unsupported critic architecture: {arch}. " + f"Critic worker expects an architecture suitable for value function estimation, " + f"such as '...ForTokenClassification' or '...ForSequenceClassification'." + ) + + # make sure output dropout is 0 + hf_config.classifier_dropout = 0 + + self.engine: BaseEngine = EngineRegistry.new( + model_type="value_model", + backend=self.config.strategy, + model_config=self.model_config, + engine_config=self.engine_config, + optimizer_config=self.optimizer_config, + checkpoint_config=self.checkpoint_config, + ) + + # build dispatch info + self._register_dispatch_collect_info( + mesh_name="critic", + dp_rank=self.engine.get_data_parallel_rank(), + is_collect=self.engine.is_mp_src_rank_with_outputs(), + ) + + # aggregate with bon sampling + self.ppo_mini_batch_size = self.config.ppo_mini_batch_size * self.config.rollout_n + assert self.ppo_mini_batch_size % self.engine.get_data_parallel_size() == 0, ( + f"{self.ppo_mini_batch_size=} is not divisible by {self.engine.get_data_parallel_size()=}" + ) + self.ppo_mini_batch_size_per_dp = self.ppo_mini_batch_size // self.engine.get_data_parallel_size() + + # setup flops counter + self.flops_counter = FlopsCounter(self.model_config.hf_config) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + self._build_engine() + self.engine.initialize() + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_loss_fn(self, loss_fn): + self.loss_fn = loss_fn + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="critic")) + @DistProfiler.annotate(color="blue", role="critic_compute_values") + def compute_values(self, data: DataProto): + data.meta_info["use_dynamic_bsz"] = self.config.use_dynamic_bsz + if self.config.use_dynamic_bsz: + data.meta_info["max_token_len_per_gpu"] = self.config.ppo_infer_max_token_len_per_gpu + else: + data.meta_info["micro_batch_size_per_gpu"] = self.config.ppo_infer_micro_batch_size_per_gpu + + with self.engine.eval_mode(): + # TODO: make worker API to accept TensorDict as well + data = data.to_tensordict() + output = self.engine.infer_batch(data) + + if self.engine.is_mp_src_rank_with_outputs(): + # in megatron, only last pp contains valid data and returned to the single controller + output = output["model_output"] + output = DataProto.from_dict( + tensors={"values": output["values"].float()}, + ) + output = output.to("cpu") + + return output + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="critic")) + @DistProfiler.annotate(color="red", role="critic_update") + def update_critic(self, data: DataProto): + data.meta_info["use_dynamic_bsz"] = self.config.use_dynamic_bsz + if self.config.use_dynamic_bsz: + data.meta_info["max_token_len_per_gpu"] = self.config.ppo_max_token_len_per_gpu + else: + data.meta_info["micro_batch_size_per_gpu"] = self.config.ppo_micro_batch_size_per_gpu + + metrics = {} + # Support all hardwares + data = data.to(get_device_id()) + # perform forward computation + with self.engine.train_mode(): + dataloader = data.make_iterator( + mini_batch_size=self.ppo_mini_batch_size_per_dp, + epochs=self.config.ppo_epochs, + seed=self.config.data_loader_seed + self.engine.get_data_parallel_rank(), + dataloader_kwargs={"shuffle": self.config.shuffle}, + ) + with Timer(name="update_policy", logger=None) as timer: + for batch_idx, mini_batch in enumerate(dataloader): + mini_batch.meta_info["global_batch_size"] = self.config.ppo_mini_batch_size + # TODO: make worker API to accept TensorDict as well + mini_batch = mini_batch.to_tensordict() + output = self.engine.train_batch(mini_batch, self.loss_fn) + mini_batch_metrics = output.get("metrics", {}) + append_to_dict(metrics, mini_batch_metrics, prefix="critic/") + + delta_time = timer.last + + global_num_tokens = data.meta_info["global_token_num"] + estimated_flops, promised_flops = self.flops_counter.estimate_flops(global_num_tokens, delta_time) + metrics["perf/mfu/critic"] = estimated_flops * self.config.ppo_epochs / promised_flops / self.world_size + metrics["perf/max_memory_allocated_gb"] = get_torch_device().max_memory_allocated() / (1024**3) + metrics["perf/max_memory_reserved_gb"] = get_torch_device().max_memory_reserved() / (1024**3) + metrics["perf/cpu_memory_used_gb"] = psutil.virtual_memory().used / (1024**3) + + lr = self.engine.lr_scheduler_step() + metrics["critic/lr"] = lr + + output = DataProto(batch=None, meta_info={"metrics": metrics}) + + return output + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): + return self.engine.save_checkpoint(local_path, hdfs_path, global_step, max_ckpt_to_keep) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=False): + return self.engine.load_checkpoint(local_path, hdfs_path, del_local_after_load) diff --git a/e2c/verl/verl/workers/roles/hybrid_engine.py b/e2c/verl/verl/workers/roles/hybrid_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..1cd1e8433dffa0b3ba420be3e346f4f5cd062014 --- /dev/null +++ b/e2c/verl/verl/workers/roles/hybrid_engine.py @@ -0,0 +1,13 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/workers/roles/reward_model.py b/e2c/verl/verl/workers/roles/reward_model.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7ac382ecd52c3f1ac1c5fa8fe630c8151c269c --- /dev/null +++ b/e2c/verl/verl/workers/roles/reward_model.py @@ -0,0 +1,203 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The main entry point to run the PPO algorithm +""" + +import logging +import os + +import torch + +from verl import DataProto +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.utils.device import ( + get_device_name, + get_torch_device, +) +from verl.utils.distributed import initialize_global_process_group_ray +from verl.utils.profiler import DistProfiler, DistProfilerExtension, log_gpu_memory_usage +from verl.workers.config import HFModelConfig, RewardModelConfig, RewardModelDataProcessorConfig +from verl.workers.roles.reward_model_engine import get_reward_model_class + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +device_name = get_device_name() + + +class RewardModelWorker(Worker, DistProfilerExtension): + def __init__(self, config: RewardModelConfig) -> None: + self.config = config + self.model_config = config.model_config + self.input_model_config = config.input_model_config + self.model_type = config.model_type + assert self.model_type in ["discriminative", "generative"], f"model_type: {self.model_type} is not supported" + Worker.__init__(self) + self.profiler_config = self.config.profiler + tool_config = self.profiler_config.tool_config + DistProfilerExtension.__init__( + self, DistProfiler(rank=self.rank, config=self.profiler_config, tool_config=tool_config) + ) + + initialize_global_process_group_ray(timeout_second=None) + + def _build_reward_model(self): + from torch.distributed.device_mesh import init_device_mesh + + # 1. parse reward model and huggingface model config + reward_model_config: RewardModelConfig = self.config + model_config: HFModelConfig = self.config.model_config + data_processor_config: RewardModelDataProcessorConfig = self.config.data_processor_config + self.tokenizer = self.model_config.get_processor() + if self.input_model_config is None: + self._do_switch_chat_template = False + self.src_tokenizer = self.tokenizer + else: + self._do_switch_chat_template = True + self.src_tokenizer = self.input_model_config.get_processor() + self.preprocess_fn, self.postprocess_fn = data_processor_config.get_process_fn() + if self.model_type == "generative": + assert self.preprocess_fn is not None and self.postprocess_fn is not None, ( + "generative reward model must have preprocess_fn and postprocess_fn" + ) + + # 2. build reward model device mesh + infer_tp = self.config.tensor_model_parallel_size + dp = self.world_size // infer_tp + assert self.world_size % infer_tp == 0, ( + f"reward model world_size: {self.world_size} is not divisible by infer_tp: {infer_tp}" + ) + reward_model_device_mesh = init_device_mesh( + device_name, mesh_shape=(dp, infer_tp), mesh_dim_names=["dp", "infer_tp"] + ) + is_collect = reward_model_device_mesh["infer_tp"].get_local_rank() == 0 + self._register_dispatch_collect_info( + "reward_model", dp_rank=reward_model_device_mesh["dp"].get_local_rank(), is_collect=is_collect + ) + + # 3. init trainer and reward model random states + self.torch_random_states = get_torch_device().get_rng_state() + gen_dp_rank = reward_model_device_mesh["dp"].get_local_rank() + get_torch_device().manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + + # 4. build reward model + log_gpu_memory_usage("Before building sglang reward model", logger=logger) + self.reward_model = get_reward_model_class(reward_model_config.name)( + config=reward_model_config, model_config=model_config, device_mesh=reward_model_device_mesh + ) + log_gpu_memory_usage("After building sglang reward model", logger=logger) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + self._build_reward_model() + + def _expand_to_token_level(self, data: DataProto, scores: torch.Tensor): + batch_size = data.batch.batch_size[0] + # expand as token_level_reward + attention_mask = data.batch["attention_mask"] + position_ids = data.batch["position_ids"] + response_length = data.batch["responses"].shape[-1] + if position_ids.dim() == 3: # qwen2vl mrope [bs, 3, seq_len] + position_ids = position_ids[:, 0, :] + eos_mask_idx = torch.argmax(position_ids * attention_mask, dim=-1) # (bsz,) + token_level_scores = torch.zeros_like(attention_mask, dtype=scores.dtype) # (bsz, seqlen) + token_level_scores[torch.arange(batch_size), eos_mask_idx] = scores + + # select the response part + token_level_scores = token_level_scores[:, -response_length:] + + return token_level_scores + + def _preprocess_reward_inputs(self, data: DataProto): + src_tokenizer = self.src_tokenizer + tokenizer = self.tokenizer + rm_inputs = [] + for i in range(len(data)): + data_item = data[i] + + # get rollout question + if "extra_infos" in data_item.non_tensor_batch and "question" in data_item.non_tensor_batch["extra_infos"]: + rollout_question = data_item.non_tensor_batch["extra_infos"]["question"] + else: + # use prompt_str as a substitute for question + prompt_ids = data_item.batch["prompts"] + prompt_length = prompt_ids.shape[-1] + valid_prompt_length = data_item.batch["attention_mask"][:prompt_length].sum() + valid_prompt_ids = prompt_ids[-valid_prompt_length:] + rollout_question = src_tokenizer.decode(valid_prompt_ids, skip_special_tokens=True) + + # get rollout response + response_ids = data_item.batch["responses"] + response_length = response_ids.shape[-1] + valid_response_length = data_item.batch["attention_mask"][-response_length:].sum() + valid_response_ids = response_ids[:valid_response_length] + rollout_response = src_tokenizer.decode(valid_response_ids, skip_special_tokens=True) + + # get ground truth + ground_truth = data_item.non_tensor_batch.get("reward_model", {}).get("ground_truth", None) + + if self.model_type == "discriminative": + if self._do_switch_chat_template: + chats = [ + {"role": "user", "content": rollout_question}, + {"role": "assistant", "content": rollout_response}, + ] + rm_input = tokenizer.apply_chat_template(chats, tokenize=True) + else: + non_pad_indices = torch.nonzero(data_item.batch["attention_mask"], as_tuple=True)[0] + start_idx, end_idx = non_pad_indices[0], non_pad_indices[-1] + rm_input = data_item.batch["input_ids"][start_idx : end_idx + 1].tolist() + else: + assert self.preprocess_fn is not None, "generative reward model must have preprocess_fn" + + input_str = self.preprocess_fn( + rollout_question=rollout_question, + rollout_response=rollout_response, + ground_truth=ground_truth, + ) + chats = [{"role": "user", "content": input_str}] + rm_input = tokenizer.apply_chat_template(chats, add_generation_prompt=True, tokenize=True) + + rm_inputs.append(rm_input) + + return rm_inputs + + def _postprocess_reward_outputs(self, data: DataProto, output: list[float] | list[list[int]]): + if self.model_type == "discriminative": + scores = torch.tensor(output) + else: + assert self.postprocess_fn is not None, "generative reward model must have postprocess_fn" + output_text = [self.tokenizer.decode(o) for o in output] + # postprocess genrm responses to scores + scores = [self.postprocess_fn(o) for o in output_text] + scores = torch.tensor(scores) + + token_level_scores = self._expand_to_token_level(data, scores) + return token_level_scores + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="reward_model")) + @DistProfiler.annotate(color="brown") + def compute_rm_score(self, data: DataProto): + data = data.to("cpu") + rm_data = self._preprocess_reward_inputs(data) + + output = self.reward_model.compute_reward(rm_data) + token_level_scores = self._postprocess_reward_outputs(data, output) + # Note that this is only the scores, may not be the final rewards used to train RL + output = DataProto.from_dict(tensors={"rm_scores": token_level_scores}) + return output diff --git a/e2c/verl/verl/workers/roles/reward_model_engine/__init__.py b/e2c/verl/verl/workers/roles/reward_model_engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..745952d15ff572898fae34bd37b0ca37a5bace37 --- /dev/null +++ b/e2c/verl/verl/workers/roles/reward_model_engine/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import get_reward_model_class + +__all__ = ["get_reward_model_class"] diff --git a/e2c/verl/verl/workers/roles/reward_model_engine/base.py b/e2c/verl/verl/workers/roles/reward_model_engine/base.py new file mode 100644 index 0000000000000000000000000000000000000000..8adb5e33a4800d6fad753f75a4db5a519aa40ac5 --- /dev/null +++ b/e2c/verl/verl/workers/roles/reward_model_engine/base.py @@ -0,0 +1,94 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The base class for reward model +""" + +import importlib +from abc import ABC, abstractmethod + +from torch.distributed.device_mesh import DeviceMesh + +from verl import DataProto +from verl.workers.config import HFModelConfig, RewardModelConfig + +__all__ = ["BaseRewardModel"] + + +class BaseRewardModel(ABC): + """base class for reward model""" + + def __init__( + self, + config: RewardModelConfig, + model_config: HFModelConfig, + device_mesh: DeviceMesh, + ): + self.config = config + self.model_config = model_config + self.device_mesh = device_mesh + + @abstractmethod + async def resume(self, tags: list[str]): + """Resume reward model weights or kv cache in GPU memory. + + Args: + tags: weights or kv_cache. + """ + pass + + @abstractmethod + async def release(self): + """Release weights and kv cache in GPU memory.""" + pass + + @abstractmethod + def compute_reward(self, data: DataProto) -> DataProto: + """Computing reward given input_ids. The transformers should output a tensor with shape + [batch_size, sequence_length], and the value at [EOS] mask should be gathered. + + Args: + data: must contain keys "input_ids", "attention_mask" and "position_ids". + - input_ids: [batch_size, sequence_length] + - attention_mask: [batch_size, sequence_length] + - position_ids: [batch_size, sequence_length] + + Returns: a data pass protocol containing "reward". Only the [EOS] position contains the reward. + Other position should have zero reward. Note that this may change in the future if we use + dense reward. So, we leave the interface for general case. + - reward: [batch_size, sequence_length]. + + """ + pass + + +_REWARD_MODEL_REGISTRY = { + "sglang": "verl.workers.roles.reward_model_engine.sglang_reward_model.SGLangRewardModel", +} + + +def get_reward_model_class(reward_model_name: str) -> type[BaseRewardModel]: + """Get the reward model class by name. + + Args: + reward_model_name: The name of the reward model. + + Returns: + The reward model class. + """ + assert reward_model_name in _REWARD_MODEL_REGISTRY, f"Reward Model {reward_model_name} with mode not found" + fqdn = _REWARD_MODEL_REGISTRY[reward_model_name] + module_name, class_name = fqdn.rsplit(".", 1) + reward_model_module = importlib.import_module(module_name) + return getattr(reward_model_module, class_name) diff --git a/e2c/verl/verl/workers/roles/reward_model_engine/sglang_reward_model.py b/e2c/verl/verl/workers/roles/reward_model_engine/sglang_reward_model.py new file mode 100644 index 0000000000000000000000000000000000000000..44d7192e96c765e3fe34849267deffd60c9bd931 --- /dev/null +++ b/e2c/verl/verl/workers/roles/reward_model_engine/sglang_reward_model.py @@ -0,0 +1,283 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import asyncio +import logging +import multiprocessing as mp +import os + +import sglang.srt.entrypoints.engine +import torch +import torch.distributed as dist +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import ( + assert_pkg_version, + get_ip, + get_open_port, + is_cuda, + set_prometheus_multiproc_dir, + set_ulimit, +) +from torch.distributed.device_mesh import DeviceMesh, init_device_mesh + +from verl.utils.net_utils import is_ipv6 +from verl.workers.config import HFModelConfig, RewardModelConfig +from verl.workers.roles.reward_model_engine.base import BaseRewardModel +from verl.workers.rollout.sglang_rollout.http_server_engine import AsyncHttpServerAdapter +from verl.workers.rollout.sglang_rollout.utils import broadcast_pyobj + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +# patch to avoid issue https://github.com/sgl-project/sglang/issues/6723 +def _set_envs_and_config(server_args: ServerArgs): + # Set global environments + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["NCCL_NVLS_ENABLE"] = str(int(server_args.enable_nccl_nvls)) + os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" + os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "4" + os.environ["CUDA_MODULE_LOADING"] = "AUTO" + + # Set prometheus env vars + if server_args.enable_metrics: + set_prometheus_multiproc_dir() + + # Set ulimit + set_ulimit() + + # Check flashinfer version + if server_args.attention_backend == "flashinfer": + assert_pkg_version( + "flashinfer_python", + "0.2.5", + "Please uninstall the old version and reinstall the latest version by following the instructions at https://docs.flashinfer.ai/installation.html.", + ) + if is_cuda(): + assert_pkg_version( + "sgl-kernel", + "0.1.1", + "Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`", + ) + + # Set mp start method + mp.set_start_method("spawn", force=True) + + +sglang.srt.entrypoints.engine._set_envs_and_config = _set_envs_and_config + + +def _post_process_outputs(output, is_embedding): + if is_embedding: + scores = [o["embedding"][-1] for o in output] + return scores + else: + output_ids = [o["output_ids"] for o in output] + return output_ids + + +class SGLangRewardModel(BaseRewardModel): + def __init__( + self, + config: RewardModelConfig, + model_config: HFModelConfig, + device_mesh: DeviceMesh, + ): + super().__init__(config, model_config, device_mesh) + + actor_module = model_config.local_path + trust_remote_code = model_config.trust_remote_code + port = None + + reward_type = self.config.model_type + if reward_type == "discriminative": + self.is_embedding = True + elif reward_type == "generative": + self.is_embedding = False + self._init_sampling_params() + else: + raise ValueError(f"reward type {reward_type} not supported") + + os.environ.setdefault("SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK", "true") + + self._init_distributed_env(device_mesh_cpu=None) + self._init_inference_engine(trust_remote_code, actor_module, port) + + def _init_distributed_env(self, device_mesh_cpu): + self._device_mesh_cpu = device_mesh_cpu + os.environ.setdefault("SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK", "true") + self.tensor_parallel_size = self.config.get("tensor_model_parallel_size", 1) + assert self.tensor_parallel_size <= dist.get_world_size(), ( + "tensor parallel size should be less than or equal to the world size" + ) + + tp_size = self.tensor_parallel_size + world_size = int(os.getenv("WORLD_SIZE", "-1")) + + # init device mesh + if self._device_mesh_cpu is None: + device_mesh_kwargs = dict( + mesh_shape=(world_size // tp_size, tp_size, 1), + mesh_dim_names=["dp", "tp", "pp"], + ) + + self._device_mesh_cpu = init_device_mesh("cpu", **device_mesh_kwargs) + + self._rank = self._device_mesh_cpu.get_rank() + self._tp_rank = self._device_mesh_cpu["tp"].get_local_rank() + self._tp_size = self._device_mesh_cpu["tp"].size() + if self._rank == 0: + logger.info(f"_init_distributed_env: :tp_world: {self._tp_size}, global_world: {world_size}") + # get tp_rank of this process in this tp group + visible_devices = [None] * self._device_mesh_cpu.size(1) + + torch.distributed.all_gather_object( + visible_devices, os.environ["CUDA_VISIBLE_DEVICES"], self._device_mesh_cpu.get_group("tp") + ) + self.visible_devices_set = set(",".join(visible_devices).split(",")) + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(sorted(list(self.visible_devices_set))) + + def _init_inference_engine(self, trust_remote_code, actor_module, port): + # initialize the inference engine + nnodes = -(-self._tp_size // len(self.visible_devices_set)) + if nnodes > 1: + ip = get_ip() + port = get_open_port() if port is None else port + [ip, port] = broadcast_pyobj( + [ip, port], + rank=self._rank, + dist_group=self._device_mesh_cpu.get_group("tp"), + src=self._device_mesh_cpu["tp"].mesh[0].item(), + force_cpu_device=False, + ) + dist_init_addr = f"[{ip}]:{port}" if is_ipv6(ip) else f"{ip}:{port}" + else: + dist_init_addr = None + + tp_size_per_node = self._tp_size // nnodes + node_rank = self._tp_rank // tp_size_per_node + first_rank_in_node = self._tp_rank % tp_size_per_node == 0 + engine_kwargs = self.config.get("engine_kwargs", {}).get("sglang", {}) or {} + engine_kwargs = {key: val for key, val in engine_kwargs.items() if val is not None} + + # attention backend will be changed to fa3 if not specified + attention_backend = engine_kwargs.pop("attention_backend", None) + max_running_requests = self.config.get("max_num_seqs", None) + + if first_rank_in_node: + rank = dist.get_rank() + os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" + args = { + "model_path": actor_module, + "dtype": self.config.dtype, + "mem_fraction_static": self.config.gpu_memory_utilization, + "enable_memory_saver": True, + "base_gpu_id": 0, + "gpu_id_step": 1, + "tp_size": self._tp_size, + "node_rank": node_rank, + "dist_init_addr": dist_init_addr, + "nnodes": nnodes, + "trust_remote_code": trust_remote_code, + "max_running_requests": max_running_requests, + # NOTE(linjunrong): add rank to prevent SGLang generate same port inside PortArgs.init_new + # when random.seed is being set during training + "port": 30000 + rank, + # NOTE(Chenyang): if you want to debug the SGLang engine output + # please set the following parameters + # Otherwise, it will make the engine run too slow + "log_level": "info", + # "log_level": "error", + # log_requests=True, + # log_requests_level=2, + # NOTE(Chenyang): turn on max_running_requests to set the max concurrent running requests + # max_running_requests=1, + "mm_attention_backend": "fa3", + "attention_backend": attention_backend if attention_backend is not None else "fa3", + # In async mode, we want token in token out. + "skip_tokenizer_init": True, + # For embedding models + "is_embedding": self.is_embedding, + # server specific args + "timeout": self.config.server_config["timeout"], + "max_attempts": self.config.server_config["max_attempts"], + "retry_delay": self.config.server_config["retry_delay"], + "max_connections": self.config.server_config["max_connections"], + "max_start_wait_time": self.config.server_config["max_start_wait_time"], + "first_rank_in_node": first_rank_in_node, + } + self._engine = AsyncHttpServerAdapter(**args) + else: + self._engine = None + + self.sharding_manager = None + self.is_sleep = True + + def _init_sampling_params(self): + kwargs = dict( + n=1, + max_new_tokens=self.config.response_length, + presence_penalty=0.0, + frequency_penalty=0.0, + repetition_penalty=self.config.get("repetition_penalty", 1.0), + ) + # supporting adding any sampling params from the config file + for k in self.config.keys(): + if hasattr(SamplingParams(), str(k)) or "stop" in str(k): + kwargs[k] = self.config.get(k) + kwargs["n"] = 1 # already repeat in ray_trainer + self.sampling_params = kwargs + + def compute_reward(self, rm_input_ids): + if self._tp_rank == 0: + loop = asyncio.new_event_loop() + if self.is_embedding: + output = loop.run_until_complete(self._engine.async_reward_score(prompt=None, input_ids=rm_input_ids)) + else: + output = loop.run_until_complete( + self._engine.async_generate( + prompt=None, sampling_params=self.sampling_params, input_ids=rm_input_ids + ) + ) + else: + output = None + + dist.barrier() + [output] = broadcast_pyobj( + data=[output], + rank=self._rank, + dist_group=self._device_mesh_cpu["tp"].get_group(), + src=self._device_mesh_cpu["tp"].mesh[0].item(), + force_cpu_device=False, + ) + + output = _post_process_outputs(output, self.is_embedding) + return output + + async def resume(self, tags: list[str]): + """Resume reward model weights or kv cache in GPU memory. + + Args: + tag: weights or kv_cache. + """ + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.config.free_cache_engine: + await self._engine.resume_memory_occupation(tags=tags) + + async def release(self): + """Release weights and kv cache in GPU memory.""" + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.config.free_cache_engine: + await self._engine.release_memory_occupation(tags=["kv_cache", "weights"]) diff --git a/e2c/verl/verl/workers/roles/utils/__init__.py b/e2c/verl/verl/workers/roles/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1cd1e8433dffa0b3ba420be3e346f4f5cd062014 --- /dev/null +++ b/e2c/verl/verl/workers/roles/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/workers/roles/utils/losses.py b/e2c/verl/verl/workers/roles/utils/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..91bf3e33aa9a8eced965f9e0c5568724e28ff172 --- /dev/null +++ b/e2c/verl/verl/workers/roles/utils/losses.py @@ -0,0 +1,133 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch +from tensordict import TensorDict + +from verl.trainer.ppo.core_algos import agg_loss, compute_value_loss, get_policy_loss_fn, kl_penalty +from verl.utils import tensordict_utils as tu +from verl.utils.dataset.dataset_utils import DatasetPadMode +from verl.utils.torch_functional import masked_mean +from verl.workers.config import ActorConfig, CriticConfig + + +def sft_loss(config: ActorConfig, model_output, data: TensorDict, dp_group=None): + pad_mode = tu.get_non_tensor_data(data=data, key="pad_mode", default=DatasetPadMode.LEFT_RIGHT) + + log_prob = model_output["log_probs"] + + if pad_mode == DatasetPadMode.NO_PADDING: + # log_prob and loss mask are nested tensors of shape [bsz, j1] + # for each sample, loss mask shape is [1, prompt_length + response_length] + loss_mask = data["loss_mask"] + + log_prob_flatten = log_prob.values() + cu_seqlens = log_prob.offsets() + loss_mask_flatten = loss_mask.values() + + # left-shift the loss mask by one token to align with log_prob + loss_mask_flatten = torch.roll(loss_mask_flatten, shifts=-1, dims=0) + loss_mask_flatten[cu_seqlens[1:] - 1] = 0 + loss = -masked_mean(log_prob_flatten, loss_mask_flatten) + else: + response_mask = data["response_mask"].to(bool) + loss = -masked_mean(log_prob, response_mask) + + return loss, {"loss": loss.detach().item()} + + +def ppo_loss(config: ActorConfig, model_output, data: TensorDict, dp_group=None): + log_prob = model_output["log_probs"] + entropy = model_output.get("entropy", None) + + metrics = {} + + response_mask = data["response_mask"].to(bool) + # compute policy loss + old_log_prob = data["old_log_probs"] + advantages = data["advantages"] + + loss_agg_mode = config.loss_agg_mode + + loss_mode = config.policy_loss.get("loss_mode", "vanilla") + + policy_loss_fn = get_policy_loss_fn(loss_mode) + pg_loss, pg_clipfrac, ppo_kl, pg_clipfrac_lower = policy_loss_fn( + old_log_prob=old_log_prob, + log_prob=log_prob, + advantages=advantages, + response_mask=response_mask, + loss_agg_mode=loss_agg_mode, + config=config, + ) + + metrics.update( + { + "pg_loss": pg_loss.detach().item(), + "pg_clipfrac": pg_clipfrac.detach().item(), + "ppo_kl": ppo_kl.detach().item(), + "pg_clipfrac_lower": pg_clipfrac_lower.detach().item(), + } + ) + policy_loss = pg_loss + + # add entropy loss + if entropy is not None: + entropy_loss = agg_loss(loss_mat=entropy, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) + entropy_coeff = config.entropy_coeff + policy_loss -= entropy_coeff * entropy_loss + + # add kl loss + if config.use_kl_loss: + ref_log_prob = data["ref_log_prob"] + # compute kl loss + kld = kl_penalty(logprob=log_prob, ref_logprob=ref_log_prob, kl_penalty=config.kl_loss_type) + kl_loss = agg_loss(loss_mat=kld, loss_mask=response_mask, loss_agg_mode=config.loss_agg_mode) + + policy_loss += kl_loss * config.kl_loss_coef + metrics["kl_loss"] = kl_loss.detach().item() + metrics["kl_coef"] = config.kl_loss_coef + + return policy_loss, metrics + + +def value_loss(config: CriticConfig, model_output, data: TensorDict, dp_group=None): + vpreds = model_output["values"] + values = data["values"] + + values = data["values"] + returns = data["returns"] + response_mask = data["response_mask"].to(bool) + + vf_loss, vf_clipfrac = compute_value_loss( + vpreds=vpreds, + values=values, + returns=returns, + response_mask=response_mask, + cliprange_value=config.cliprange_value, + loss_agg_mode=config.loss_agg_mode, + ) + + metrics = {} + + metrics.update( + { + "critic/vf_loss": vf_loss.detach().item(), + "critic/vf_clipfrac": vf_clipfrac.detach().item(), + "critic/vpred_mean": masked_mean(vpreds, response_mask).detach().item(), + } + ) + + return vf_loss, metrics diff --git a/e2c/verl/verl/workers/rollout/__init__.py b/e2c/verl/verl/workers/rollout/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5d9263c9c56b202f94f23251d599f9dacda0f412 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import BaseRollout, get_rollout_class +from .hf_rollout import HFRollout +from .naive import NaiveRollout + +__all__ = ["BaseRollout", "NaiveRollout", "HFRollout", "get_rollout_class"] diff --git a/e2c/verl/verl/workers/rollout/base.py b/e2c/verl/verl/workers/rollout/base.py new file mode 100644 index 0000000000000000000000000000000000000000..dcc8a977fa1f78ddf445fabda49de9293279b6ee --- /dev/null +++ b/e2c/verl/verl/workers/rollout/base.py @@ -0,0 +1,102 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from abc import ABC, abstractmethod +from typing import Generator + +import torch +from torch.distributed.device_mesh import DeviceMesh + +from verl import DataProto +from verl.workers.config import HFModelConfig, RolloutConfig + +__all__ = ["BaseRollout"] + + +class BaseRollout(ABC): + """Base class for rollout.""" + + def __init__( + self, + config: RolloutConfig, + model_config: HFModelConfig, + device_mesh: DeviceMesh, + ): + self.config = config + self.model_config = model_config + self.device_mesh = device_mesh + + @abstractmethod + async def resume(self, tags: list[str]): + """Resume rollout weights or kv cache in GPU memory. + + Args: + tags: weights or kv_cache. + """ + pass + + @abstractmethod + async def update_weights( + self, + weights: Generator[tuple[str, torch.Tensor], None, None], + **kwargs, + ): + """Update the weights of the rollout model. + + Args: + weights: A generator that yields the name of the weight tensor and the tensor itself. + """ + pass + + @abstractmethod + async def release(self): + """Release weights and kv cache in GPU memory.""" + pass + + def generate_sequences(self, prompts: DataProto) -> DataProto: + """Batch generate sequences in sync mode. + + Args: + prompts: The input prompts. + + Returns: + The output sequences. + """ + raise NotImplementedError + + +_ROLLOUT_REGISTRY = { + ("vllm", "sync"): "verl.workers.rollout.vllm_rollout.vLLMRollout", + ("vllm", "async"): "verl.workers.rollout.vllm_rollout.vLLMAsyncRollout", + ("sglang", "sync"): "verl.workers.rollout.sglang_rollout.sglang_rollout.SGLangRollout", + ("sglang", "async"): "verl.workers.rollout.sglang_rollout.sglang_rollout.ServerAdapter", +} + + +def get_rollout_class(rollout_name: str, mode: str) -> type[BaseRollout]: + """Get the rollout class by name. + + Args: + rollout_name: The name of the rollout. + mode: The mode of the rollout, sync: spmd mode, async: server mode. + + Returns: + The rollout class. + """ + assert (rollout_name, mode) in _ROLLOUT_REGISTRY, f"Rollout {rollout_name} with mode {mode} not found" + fqdn = _ROLLOUT_REGISTRY[(rollout_name, mode)] + module_name, class_name = fqdn.rsplit(".", 1) + rollout_module = importlib.import_module(module_name) + return getattr(rollout_module, class_name) diff --git a/e2c/verl/verl/workers/rollout/hf_rollout.py b/e2c/verl/verl/workers/rollout/hf_rollout.py new file mode 100644 index 0000000000000000000000000000000000000000..e596507cdb9cab9362be448b34c9702ea9dc7061 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/hf_rollout.py @@ -0,0 +1,177 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Rollout with huggingface models. +TODO: refactor this class. Currently, it will hang when using FSDP HybridShard. We should actually create a single +GPU model. Then, get full state_dict and bind the state_dict to the single GPU model. Then, use the single GPU model +to perform generation. +""" + +import contextlib + +import torch +import torch.distributed +from tensordict import TensorDict +from torch import nn +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from transformers import GenerationConfig + +from verl import DataProto +from verl.utils.device import get_device_name, get_torch_device +from verl.utils.torch_functional import get_response_mask + +from .base import BaseRollout + +__all__ = ["HFRollout"] + + +class HFRollout(BaseRollout): + def __init__(self, module: nn.Module, config): + super().__init__() + self.config = config + self.module = module + + def generate_sequences(self, prompts: DataProto) -> DataProto: + batch_size = prompts.batch.batch_size[0] + num_chunks = max(batch_size // self.config.get("micro_batch_size", batch_size), 1) + batch_prompts = prompts.chunk(chunks=num_chunks) + output = [self._generate_minibatch(p) for p in batch_prompts] + output = DataProto.concat(output) + return output + + @torch.no_grad() + def _generate_minibatch(self, prompts: DataProto) -> DataProto: + # make sampling args can be overridden by inputs + do_sample = prompts.meta_info.get("do_sample", self.config.do_sample) + is_validate = prompts.meta_info.get("validate", False) + + temperature = prompts.meta_info.get("temperature", self.config.temperature) + response_length = prompts.meta_info.get("response_length", self.config.response_length) + top_p = prompts.meta_info.get("top_p", self.config.get("top_p", 1.0)) + top_k = max(0, prompts.meta_info.get("top_k", self.config.get("top_k", 0))) # to be compatible with vllm + + if not do_sample: + # do_sample==False -> greedy decoding + kwargs = { + "do_sample": False, + "num_beams": 1, + } + elif is_validate: + # do validate and do sample -> use val_kwargs + kwargs = { + "do_sample": True, + "num_beams": 1, + "top_k": max(0, self.config.val_kwargs.top_k), # to be compatible with vllm + "top_p": self.config.val_kwargs.top_p, + "temperature": self.config.val_kwargs.temperature, + "num_return_sequences": 1, # if validate, already repeat in ray_trainer + } + else: + # do_sample -> use rollout config + kwargs = { + "do_sample": True, + "num_beams": 1, + "top_p": top_p, + "top_k": top_k, + "temperature": temperature, + # already repeat in ray_trainer + # https://github.com/volcengine/verl/blob/2fdfbdcba6f2e076f64bc47922d8fe6cf7dc7da5/verl/trainer/ppo/ray_trainer.py#L1117 + "num_return_sequences": 1, + } + + # make config according to generate mode + generation_config = GenerationConfig(**kwargs) + + idx = prompts.batch["input_ids"] # (bs, prompt_length) + prompt_length = idx.size(1) + attention_mask = prompts.batch["attention_mask"] # left-padded attention_mask + position_ids = prompts.batch["position_ids"] + + # used to construct attention_mask + eos_token_id = prompts.meta_info["eos_token_id"] + pad_token_id = prompts.meta_info["pad_token_id"] + + self.module.eval() + param_ctx = contextlib.nullcontext() + + if isinstance(self.module, FSDP): + # recurse need to set to False according to https://github.com/pytorch/pytorch/issues/100069 + param_ctx = FSDP.summon_full_params(self.module, writeback=False, recurse=False) + with param_ctx, torch.autocast(device_type=get_device_name(), dtype=torch.bfloat16): + output = self.module.generate( + input_ids=idx, + attention_mask=attention_mask, + position_ids=position_ids, + do_sample=do_sample, + max_new_tokens=response_length, + eos_token_id=eos_token_id, + pad_token_id=pad_token_id, + generation_config=generation_config, + output_scores=False, # this is potentially very large + return_dict_in_generate=True, + use_cache=True, + ) + + # TODO: filter out the seq with no answers like ds-chat + seq = output.sequences + generated_batch_size = seq.size(0) # bs * num_return_sequences + + # huggingface generate will stop generating when all the batch reaches [EOS]. + # We have to pad to response_length + sequence_length = prompt_length + self.config.response_length + delta_length = sequence_length - seq.shape[1] + + if delta_length > 0: + delta_tokens = torch.ones(size=(generated_batch_size, delta_length), device=seq.device, dtype=seq.dtype) + delta_tokens = pad_token_id * delta_tokens + seq = torch.cat((seq, delta_tokens), dim=1) + assert seq.shape[1] == sequence_length + + # make necessary reputations if num_return_sequences > 1 + num_return_sequences = kwargs.get("num_return_sequences", 1) + if num_return_sequences > 1: + position_ids = position_ids.repeat_interleave(num_return_sequences, dim=0) + attention_mask = attention_mask.repeat_interleave(num_return_sequences, dim=0) + + prompt = seq[:, :prompt_length] # (generated_batch_size, prompt_length) + response = seq[:, prompt_length:] # (generated_batch_size, response_length) + + response_length = response.size(1) + delta_position_id = torch.arange(1, response_length + 1, device=position_ids.device) + delta_position_id = delta_position_id.unsqueeze(0).repeat(generated_batch_size, 1) + + response_position_ids = position_ids[:, -1:] + delta_position_id + position_ids = torch.cat([position_ids, response_position_ids], dim=-1) + + response_attention_mask = get_response_mask( + response_id=response, eos_token=eos_token_id, dtype=attention_mask.dtype + ) + attention_mask = torch.cat((attention_mask, response_attention_mask), dim=-1) + + batch = TensorDict( + { + "prompts": prompt, + "responses": response, + "input_ids": seq, + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=generated_batch_size, + ) + + # empty cache before compute old_log_prob + get_torch_device().empty_cache() + + self.module.train() + return DataProto(batch=batch) diff --git a/e2c/verl/verl/workers/rollout/naive/__init__.py b/e2c/verl/verl/workers/rollout/naive/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb6c23bf4327ef199ea9b454f00be88cbaa27967 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/naive/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .naive_rollout import NaiveRollout + +__all__ = ["NaiveRollout"] diff --git a/e2c/verl/verl/workers/rollout/naive/naive_rollout.py b/e2c/verl/verl/workers/rollout/naive/naive_rollout.py new file mode 100644 index 0000000000000000000000000000000000000000..fe56dc4c929b05aa2279b7e1b46e6d9a74e1b175 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/naive/naive_rollout.py @@ -0,0 +1,120 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +In single GPU rollout, the sequences are generated directly by sampling from the model. +The output will contain +1. output_ids +2. attention_masks (left padding) +3. eos_masks +4. log_probs +""" + +import torch +import torch.nn.functional as F +from tensordict import TensorDict +from torch import nn + +from verl import DataProto +from verl.utils.torch_functional import logprobs_from_logits + +from ..base import BaseRollout + +__all__ = ["NaiveRollout"] + + +class NaiveRollout(BaseRollout): + def __init__(self, module: nn.Module, config): + """A naive rollout. It requires the module to be compatible with huggingface APIs. That is: + The module should define __call__ to receive input_ids, attention_mask and position_ids. + It outputs a structure that contains logits field. + + Args: + module: module here follows huggingface APIs + config: DictConfig + """ + super().__init__() + self.config = config + self.module = module + + @torch.no_grad() + def generate_sequences(self, prompts: DataProto) -> DataProto: + """Generate sequences""" + idx = prompts.batch["input_ids"] # (bs, prompt_length) + attention_mask = prompts.batch["attention_mask"] # left-padded attention_mask + position_ids = prompts.batch["position_ids"] + + # used to construct attention_mask + eos_token_id = prompts.meta_info["eos_token_id"] + + batch_size = idx.size(0) + prompt_length = idx.size(1) + + self.module.eval() + + prev_attention_mask = torch.ones(size=(batch_size, 1), dtype=attention_mask.dtype, device=attention_mask.device) + + logits_lst = [] + for _ in range(self.config.response_length): + # if the sequence context is growing too long we must crop it at block_size + # idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:] + idx_cond = idx + # forward the model to get the logits for the index in the sequence + # we use huggingface APIs here + output = self.module(input_ids=idx_cond, attention_mask=attention_mask, position_ids=position_ids) + logits = output.logits + # pluck the logits at the final step and scale by desired temperature + logits = logits[:, -1, :] / self.config.temperature # (bs, vocab_size) + # optionally crop the logits to only the top k options + if self.config.top_k is not None: + v, _ = torch.topk(logits, min(self.config.top_k, logits.size(-1))) + logits[logits < v[:, [-1]]] = -float("Inf") + # apply softmax to convert logits to (normalized) probabilities + probs = F.softmax(logits, dim=-1) + # sample from the distribution + if self.config.do_sample: + idx_next = torch.multinomial(probs, num_samples=1) + else: + idx_next = torch.argmax(probs, dim=-1, keepdim=True) + + attention_mask = torch.cat((attention_mask, prev_attention_mask), dim=-1) + + for token_id in eos_token_id: + prev_attention_mask = torch.logical_and(idx_next != token_id, prev_attention_mask.bool()) + prev_attention_mask.to(attention_mask.dtype) + + position_ids = torch.cat((position_ids, position_ids[:, -1:] + 1), dim=-1) + + # append sampled index to the running sequence and continue + idx = torch.cat((idx, idx_next), dim=1) + logits_lst.append(logits) + + logits = torch.stack(logits_lst, dim=1) # (bs, response_length, vocab_size) + prompts = idx[:, :prompt_length] # (bs, prompt_length) + response = idx[:, prompt_length:] # (bs, response_length) + log_probs = logprobs_from_logits(logits=logits, labels=response) + batch = TensorDict( + { + "input_ids": prompts, + "responses": response, + "sequences": idx, + "old_log_probs": log_probs, + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=batch_size, + ) + + self.module.train() + + return DataProto(batch=batch) diff --git a/e2c/verl/verl/workers/rollout/replica.py b/e2c/verl/verl/workers/rollout/replica.py new file mode 100644 index 0000000000000000000000000000000000000000..c30ce46156532ccfdea05699a68cc930b2ed7d05 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/replica.py @@ -0,0 +1,202 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import logging +import os +from abc import ABC, abstractmethod +from enum import Enum +from typing import Optional + +from pydantic import BaseModel +from ray.actor import ActorHandle + +from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup +from verl.trainer.ppo.ray_trainer import RayResourcePool, ResourcePoolManager +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RewardModelConfig, RolloutConfig + +logger = logging.getLogger(__file__) + + +class TokenOutput(BaseModel): + token_ids: list[int] + """response token ids""" + log_probs: Optional[list[float]] = None + """logprobs of response token ids""" + + +class RolloutMode(Enum): + # Rollout engine and training engine(fsdp/megatron) fused in same process + # Rollout and trainer share GPUs, switch context with weight synchronization. + # Usage scenarios: on-policy training. + HYBRID = "hybrid" + + # Rollout engine colocated with hybrid engine in same ray placement group but in separate process. + # Rollout and hybrid processes share GPUs, switch context without weight synchronization. + # Usage scenarios: GRM (LLM as a judge). + COLOCATED = "colocated" + + # Standalone rollout server with separate GPU resource, disaggregated architecture. + # Usage scenarios: off-policy training. + STANDALONE = "standalone" + + +class RolloutReplica(ABC): + """Rollout replica is an individual server instance, which may be deployed on single or multiple nodes. + It is equivalent to launch server in each node with command line: + + SGLang: + ``` + python -m sglang.launch_server --node-rank 0 --nnode 2 ... + python -m sglang.launch_server --node-rank 1 --nnode 2 ... + ``` + + vLLM: + ``` + vllm serve --data-parallel-size 16 --data-parallel-size-local 8 --data-parallel-start-rank 0 ... + vllm serve --data-parallel-size 16 --data-parallel-size-local 8 --data-parallel-start-rank 8 ... + ``` + + Args: + replica_rank: int, rank of this rollout replica. + config: RolloutConfig | RewardModelConfig, full config. + gpus_per_node: int, number of gpus per node. + """ + + def __init__( + self, + replica_rank: int, + config: RolloutConfig | RewardModelConfig, + model_config: HFModelConfig, + gpus_per_node: int = 8, + ) -> None: + self.replica_rank = replica_rank + self.config = omega_conf_to_dataclass(config) + self.model_config: HFModelConfig = omega_conf_to_dataclass(model_config, dataclass_type=HFModelConfig) + + self.world_size = self.config.tensor_model_parallel_size * self.config.data_parallel_size + self.gpus_per_node = min(gpus_per_node, self.world_size) + assert self.world_size % self.gpus_per_node == 0, ( + f"world_size {self.world_size} must be divisible by gpus_per_node {self.gpus_per_node}" + ) + self.nnodes = self.world_size // self.gpus_per_node + + self.rollout_mode: RolloutMode = None + self.workers: list[ActorHandle] = [] + self.resource_pool: RayResourcePool = None + + self.servers: list[ActorHandle] = [] + self._server_address: str = None + self._server_handle: ActorHandle = None + + async def init_hybrid(self, worker_group: RayWorkerGroup): + """Init hybrid rollout server, rollout engine and training engine(fsdp/megatron) fused in same process. + + Args: + worker_group: RayWorkerGroup, fused workers where training engine(fsdp/megatron) have been initialized. + """ + self.rollout_mode = RolloutMode.HYBRID + self.workers = worker_group.workers[ + self.world_size * self.replica_rank : self.world_size * (self.replica_rank + 1) + ] + await self.launch_servers() + + async def init_colocated(self, resource_pool: RayResourcePool): + """Init colocated rollout server, rollout engine and hybrid engine colocated in same ray placement group + but in separate processes. + + Args: + resource_pool: RayResourcePool, ray placement group where hybrid engine processes have been launched. + """ + raise NotImplementedError + + async def init_standalone(self): + """Init standalone rollout server, create new resource pool for this rollout.""" + # create resource pool for this rollout + self.rollout_mode = RolloutMode.STANDALONE + resource_pool_spec = { + f"rollout_pool_{self.replica_rank}": [self.gpus_per_node] * self.nnodes, + } + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=None) + resource_pool_manager.create_resource_pool() + self.resource_pool = resource_pool_manager.resource_pool_dict[f"rollout_pool_{self.replica_rank}"] + + # create worker group for this rollout + worker_group = RayWorkerGroup( + resource_pool=self.resource_pool, + ray_cls_with_init=self.get_ray_class_with_init_args(), + bin_pack=False, + name_prefix=f"rollout_standalone_{self.replica_rank}", + ) + self.workers = worker_group.workers + await self.launch_servers() + + @abstractmethod + def get_ray_class_with_init_args(self) -> RayClassWithInitArgs: + """Get rollout worker actor class for colocated and standalone mode.""" + raise NotImplementedError + + @abstractmethod + async def launch_servers(self): + """Launch http server in each node.""" + raise NotImplementedError + + @property + def server_address(self) -> str: + """Get rollout server address for OpenAI chat completion.""" + return self._server_address + + @property + def server_handle(self) -> ActorHandle: + """Get rollout server handle for Token-in-token-out generation.""" + return self._server_handle + + async def wake_up(self): + """Wake up each rollout server.""" + await asyncio.gather(*[server.wake_up.remote() for server in self.servers]) + + async def sleep(self): + """Sleep each rollout server.""" + await asyncio.gather(*[server.sleep.remote() for server in self.servers]) + + +def get_rollout_replica_class(rollout: str) -> type[RolloutReplica]: + if rollout == "vllm": + from verl.workers.rollout.vllm_rollout.vllm_async_server import vLLMReplica + + return vLLMReplica + elif rollout == "sglang": + # NOTE: verl driver is cpu only, avoid sglang fp8 quantization import error. + os.environ["SGLANG_USE_CPU_ENGINE"] = "1" + + # TODO: remove this once we bump to sglang>=0.5.1 + try: + import vllm # noqa: F401 + except ImportError: + import sys + from unittest.mock import Mock + + mock_vllm = Mock() + mock_vllm._custom_ops = Mock() + mock_vllm._custom_ops.scaled_fp8_quant = Mock() + + sys.modules["vllm"] = mock_vllm + sys.modules["vllm._custom_ops"] = mock_vllm._custom_ops + + from verl.workers.rollout.sglang_rollout.async_sglang_server import SGLangReplica + + del os.environ["SGLANG_USE_CPU_ENGINE"] + return SGLangReplica + else: + raise ValueError(f"Unknown rollout mode: {rollout}") diff --git a/e2c/verl/verl/workers/rollout/schemas.py b/e2c/verl/verl/workers/rollout/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..a572985e6f49be9898938daa0d5b18d4e9d84f8f --- /dev/null +++ b/e2c/verl/verl/workers/rollout/schemas.py @@ -0,0 +1,669 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import difflib +import logging +import os +from enum import Enum +from typing import Any, Optional + +import torch +from pydantic import BaseModel, ConfigDict, model_validator +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, ProcessorMixin + +from verl.tools.schemas import OpenAIFunctionToolCall, OpenAIFunctionToolSchema, ToolResponse +from verl.utils.model import compute_position_id_with_mask + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +BASE_CHAT_HISTORY = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "I am a user."}, +] + + +class FinishReasonTypeEnum(str, Enum): + """The enum for finish reason type.""" + + LENGTH = "length" + STOP = "stop" + TOOL_CALL = "tool_calls" + + @classmethod + def from_str(cls, value: str) -> "FinishReasonTypeEnum": + if value == "stop": + return cls.STOP + elif value == "length": + return cls.LENGTH + elif value == "tool_calls": + return cls.TOOL_CALL + else: + raise ValueError(f"Unsupported finish reason type: {value}") + + +class Message(BaseModel): + role: str + content: str | dict[str, Any] | list[dict[str, Any]] | ToolResponse + tool_calls: Optional[list[OpenAIFunctionToolCall]] = None + + +class AsyncRolloutRequestStateEnum(str, Enum): + """The enum for async rollout request state.""" + + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + TOOL_CALLING = "tool_calling" + INTERACTING = "interacting" + + +class TokenizationSanityCheckModeEnum(str, Enum): + """The enum for tokenization sanity check mode.""" + + DISABLE = "disable" + STRICT = "strict" + IGNORE_STRIPPABLE = "ignore_strippable" + + +class AsyncRolloutRequest(BaseModel): + """The data model for async rollout.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + batch_data_id: int = 0 + rollout_offset: int = 0 + request_id: str + state: AsyncRolloutRequestStateEnum + messages: list[Message] + multi_modal_keys: Optional[list[str]] = None + multi_modal_data: Optional[dict[str, Any]] = None + multi_modal_inputs: Optional[dict[str, torch.Tensor]] = None + tool_schemas: Optional[list[OpenAIFunctionToolSchema]] = None + tools_kwargs: dict[str, Any] = {} + interaction_kwargs: dict[str, Any] = {} + input_ids: Optional[torch.Tensor] = None + prompt_ids: Optional[torch.Tensor] = None + response_ids: Optional[torch.Tensor] = None + attention_mask: Optional[torch.Tensor] = None + prompt_attention_mask: Optional[torch.Tensor] = None + response_attention_mask: Optional[torch.Tensor] = None + position_ids: Optional[torch.Tensor] = None + prompt_position_ids: Optional[torch.Tensor] = None + response_position_ids: Optional[torch.Tensor] = None + loss_mask: Optional[torch.Tensor] = None + prompt_loss_mask: Optional[torch.Tensor] = None + response_loss_mask: Optional[torch.Tensor] = None + reward_scores: dict[str, float] + max_prompt_len: int + max_response_len: int = 8192 + max_model_len: int = 32768 + metrics: dict[str, list[Any]] = {} + output_token_ids: torch.Tensor | None = None + rollout_log_probs: torch.Tensor | None = None + + use_inference_chat_template: bool + tokenization_sanity_check_mode: TokenizationSanityCheckModeEnum + generation_prompt_ids: Optional[torch.Tensor] = None + base_conv_wo_gen_prompt_end_pos: int + base_conv_with_gen_prompt_end_pos: int + + @model_validator(mode="before") + @classmethod + def initialize_request(cls, values): + if not (messages := values.get("messages")): + raise ValueError("messages is required for AsyncRolloutRequest initialization") + if not (max_prompt_len := values.get("max_prompt_len")): + raise ValueError("max_prompt_len is required for AsyncRolloutRequest initialization") + if not (processing_class := values.pop("processing_class", None)): + raise ValueError("processing_class is required for AsyncRolloutRequest initialization") + + values["messages"] = [Message.model_validate(msg) for msg in messages] + + # If there is no multi_modal_keys, we assume the multi-modal data is image and video. + if not values.get("multi_modal_keys"): + values["multi_modal_keys"] = ["image", "video"] + if not values.get("multi_modal_data"): + values["multi_modal_data"] = {key: [] for key in values["multi_modal_keys"]} + else: + # check if all multi_modal_keys are in multi_modal_data + for key in values["multi_modal_keys"]: + if key not in values["multi_modal_data"]: + values["multi_modal_data"][key] = [] + if not values.get("multi_modal_inputs"): + values["multi_modal_inputs"] = {} + + tools = ( + [tool.model_dump() for tool in tool_schemas] if (tool_schemas := values.get("tool_schemas", [])) else None + ) + + multi_modal_data = values["multi_modal_data"] + tokens_without_prompt = cls._handle_apply_chat_template( + processing_class, + messages, + multi_modal_data=multi_modal_data, + tools=tools, + add_generation_prompt=False, + tokenize=True, + ) + if ( + values.get("input_ids") is None + or values.get("attention_mask") is None + or values.get("position_ids") is None + ): + tokenization_dict_with_prompt = cls._handle_apply_chat_template( + processing_class, + messages, + multi_modal_data=multi_modal_data, + tools=tools, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + ) + + values["input_ids"], values["attention_mask"] = ( + tokenization_dict_with_prompt["input_ids"], + tokenization_dict_with_prompt["attention_mask"], + ) + if values["input_ids"].shape[-1] > max_prompt_len: + # Only log the warning to avoid truncating in the middle of generation prompt. Consider raising an + # error for this case in the future. + logger.warning( + f"Prompt {values['batch_data_id']} has length {values['input_ids'].shape[-1]} " + f"which is greater than max_prompt_len {max_prompt_len} after applied chat template with tools." + ) + + # Process multi_modal_inputs + multi_modal_inputs = tokenization_dict_with_prompt.copy() + multi_modal_inputs.pop("input_ids", None) + multi_modal_inputs.pop("attention_mask", None) + values["multi_modal_inputs"] = multi_modal_inputs + + values["position_ids"] = values["prompt_position_ids"] = cls._get_position_ids( + processing_class, values["input_ids"], values["attention_mask"], multi_modal_inputs + ) + + values["prompt_ids"], values["prompt_attention_mask"] = values["input_ids"], values["attention_mask"] + values["loss_mask"] = values["prompt_loss_mask"] = torch.zeros_like(values["input_ids"], dtype=torch.bool) + values["generation_prompt_ids"] = values["input_ids"][..., tokens_without_prompt.shape[-1] :] + values["base_conv_wo_gen_prompt_end_pos"] = cls._handle_apply_chat_template( + processing_class, + BASE_CHAT_HISTORY, + multi_modal_data=multi_modal_data, + tools=tools, + add_generation_prompt=False, + tokenize=True, + ).shape[-1] + + values["base_conv_with_gen_prompt_end_pos"] = cls._handle_apply_chat_template( + processing_class, + BASE_CHAT_HISTORY, + multi_modal_data=multi_modal_data, + tools=tools, + add_generation_prompt=True, + tokenize=True, + ).shape[-1] + + return values + + @staticmethod + def _handle_apply_chat_template( + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, + messages: list[Message], + multi_modal_data: dict[str, Any], + tools: Optional[list[OpenAIFunctionToolSchema]] = None, + add_generation_prompt: bool = False, + tokenize: bool = False, + return_dict: bool = False, + ): + raw_prompt = processing_class.apply_chat_template( + messages, tools=tools, add_generation_prompt=add_generation_prompt, tokenize=False + ) + if not tokenize: + return raw_prompt + + if isinstance(processing_class, PreTrainedTokenizer) or isinstance(processing_class, PreTrainedTokenizerFast): + if any(len(values) > 0 for values in multi_modal_data.values()): + logger.warning( + "There is multi_modal_data but you are not using a processor. Multi-modal data will be ignored." + ) + model_inputs = processing_class(text=[raw_prompt], return_tensors="pt") + elif isinstance(processing_class, ProcessorMixin): + # When we update multi_model_keys, we also need to update this logic + images = images if len(images := multi_modal_data.get("image", [])) > 0 else None + videos = videos if len(videos := multi_modal_data.get("video", [])) > 0 else None + model_inputs = processing_class(text=[raw_prompt], images=images, videos=videos, return_tensors="pt") + else: + raise ValueError(f"Unsupported processing class type: {type(processing_class)}") + + model_inputs = dict(model_inputs) + if return_dict: + return model_inputs + else: + return model_inputs["input_ids"] + + @staticmethod + def _get_position_ids( + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + multi_modal_inputs: Optional[dict[str, torch.Tensor]] = None, + ) -> torch.Tensor: + # special case for qwen2vl + is_qwen2vl = ( + hasattr(processing_class, "image_processor") + and "Qwen2VLImageProcessor" in processing_class.image_processor.__class__.__name__ + ) + if is_qwen2vl: + from verl.models.transformers.qwen2_vl import get_rope_index + + image_grid_thw = video_grid_thw = second_per_grid_ts = None + if multi_modal_inputs: + image_grid_thw = multi_modal_inputs.get("image_grid_thw") + video_grid_thw = multi_modal_inputs.get("video_grid_thw") + second_per_grid_ts = multi_modal_inputs.get("second_per_grid_ts") + + assert input_ids.dim() == 2 and input_ids.shape[0] == 1, ( + f"input_ids should be 2D with batch size 1, but got shape {input_ids.shape}" + ) + assert attention_mask.dim() == 2 and attention_mask.shape[0] == 1, ( + f"attention_mask should be 2D with batch size 1, but got shape {attention_mask.shape}" + ) + new_position_ids = get_rope_index( + processing_class, + input_ids=input_ids.squeeze(0), + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + second_per_grid_ts=second_per_grid_ts, + attention_mask=attention_mask.squeeze(0), + ) + return new_position_ids # (3, seq_len) + else: + return compute_position_id_with_mask(attention_mask) # (1, seq_len) + + def _update_input_ids( + self, + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, + new_input_ids: torch.Tensor, + attention_mask: bool, + loss_mask: bool, + new_multi_modal_inputs: Optional[dict[str, torch.Tensor]] = None, + ) -> None: + """ + Update the input_ids, attention_mask, position_ids, and loss_mask of the request in additive manner. + """ + self.input_ids = torch.cat([self.input_ids, new_input_ids], dim=-1) + attention_mask = torch.ones_like(new_input_ids) * int(attention_mask) + self.attention_mask = torch.cat([self.attention_mask, attention_mask], dim=-1) + loss_mask = torch.ones_like(new_input_ids) * int(loss_mask) + self.loss_mask = torch.cat([self.loss_mask, loss_mask], dim=-1) + + if new_multi_modal_inputs: + self._update_multi_modal_inputs(new_multi_modal_inputs) + + new_position_ids = self._get_position_ids( + processing_class, new_input_ids, attention_mask, new_multi_modal_inputs + ) + + last_pos = self.position_ids[..., -1:] + new_position_ids = new_position_ids + (last_pos + 1) + + self.position_ids = torch.cat([self.position_ids, new_position_ids], dim=-1) + + assert ( + self.input_ids.shape[-1] + == self.attention_mask.shape[-1] + == self.position_ids.shape[-1] + == self.loss_mask.shape[-1] + ), f"""Request {self.request_id} has different length of {self.input_ids.shape[-1]=}, + {self.attention_mask.shape[-1]=}, {self.position_ids.shape[-1]=}, {self.loss_mask.shape[-1]=}""" + + def _update_multi_modal_inputs(self, new_multi_modal_inputs: dict[str, torch.Tensor]) -> None: + """ + Update the multi_modal_inputs of the request in additive manner. + """ + for key in new_multi_modal_inputs: + input_tensor = new_multi_modal_inputs[key] + self.multi_modal_inputs[key] = ( + torch.cat([self.multi_modal_inputs[key], input_tensor], dim=0) + if key in self.multi_modal_inputs + else input_tensor + ) + + def get_generation_prompt_ids( + self, processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin + ) -> list[int]: + """ + Get the generation prompt ids for rollout engine. + + Because rollout engine(SGLang) requires the ids to be a list, we need to convert the tensor to a list. + """ + generation_prompt_ids = ( + None + if self.input_ids[..., -self.generation_prompt_ids.shape[-1] :].eq(self.generation_prompt_ids).all() + else self.generation_prompt_ids + ) + if generation_prompt_ids is not None: + self._update_input_ids(processing_class, generation_prompt_ids, attention_mask=True, loss_mask=False) + + if self.use_inference_chat_template: + messages = [msg.model_dump() for msg in self.messages] + tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None + generation_prompt_ids = self._handle_apply_chat_template( + processing_class, + messages, + multi_modal_data=self.multi_modal_data, + tools=tools, + add_generation_prompt=True, + tokenize=True, + ) + return generation_prompt_ids.squeeze(0).tolist() + else: + return self.input_ids.squeeze(0).tolist() + + def add_user_message( + self, + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, + content: str, + ) -> None: + self.messages.append(Message(role="user", content=content)) + messages = [*BASE_CHAT_HISTORY, self.messages[-1]] + tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None + + # We don't need to pass multi_modal_data here because we don't have any multi-modal data from Engine + # Inference, it is pure text. + content_ids = self._handle_apply_chat_template( + processing_class, messages, multi_modal_data={}, tools=tools, add_generation_prompt=False, tokenize=True + )[..., self.base_conv_wo_gen_prompt_end_pos :] + self._update_input_ids(processing_class, content_ids, attention_mask=True, loss_mask=False) + + def add_assistant_message( + self, + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, + content: str, + content_ids: Optional[torch.Tensor] = None, + tool_calls: Optional[list[OpenAIFunctionToolCall]] = None, + ) -> None: + self.messages.append(Message(role="assistant", content=content, tool_calls=tool_calls)) + if content_ids is None: + messages = [*BASE_CHAT_HISTORY, self.messages[-1]] + tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None + + # We don't need to pass multi_modal_data here because we don't have any multi-modal data from Engine + # Inference, it is pure text. + content_ids = self._handle_apply_chat_template( + processing_class, messages, multi_modal_data={}, tools=tools, add_generation_prompt=False, tokenize=True + )[..., self.base_conv_with_gen_prompt_end_pos :] + self._update_input_ids(processing_class, content_ids, attention_mask=True, loss_mask=True) + + def add_tool_response_messages( + self, + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, + contents: list[ToolResponse], + ) -> None: + if not contents or all(content.is_empty() for content in contents): + return + # We also handle the case when tool returns image + # We require the processing of the image and video to be done at tool.execute() level + delta_multi_modal_data = {key: [] for key in self.multi_modal_keys} + for content in contents: + if content.is_text_only(): + self.messages.append(Message(role="tool", content=content.text)) + else: + content_list = [] + # When we update multi_model_keys, we also need to update this logic + if content.image: + content_list.extend([{"type": "image"} for _ in content.image]) + delta_multi_modal_data["image"].extend(content.image) + if content.video: + content_list.extend([{"type": "video"} for _ in content.video]) + delta_multi_modal_data["video"].extend(content.video) + if content.text: + content_list.append({"type": "text", "text": content.text}) + self.messages.append(Message(role="tool", content=content_list)) + + messages = [*BASE_CHAT_HISTORY, *self.messages[-len(contents) :]] + tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None + + for key in self.multi_modal_keys: + if len(delta_multi_modal_data[key]) > 0: + self.multi_modal_data[key].extend(delta_multi_modal_data[key]) + + # We just passed the new multi-modal data to the chat template to update the input_ids. + content_info = self._handle_apply_chat_template( + processing_class, + messages, + multi_modal_data=delta_multi_modal_data, + tools=tools, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + ) + content_ids = content_info["input_ids"][..., self.base_conv_wo_gen_prompt_end_pos :] + + # process multi_modal_inputs + multi_modal_inputs = content_info.copy() + multi_modal_inputs.pop("input_ids", None) + multi_modal_inputs.pop("attention_mask", None) + + # chat templates include generation prompt tokens (e.g., "assistant\n") + # So when tool response is added, we need to explicitly remove these tokens. + self._remove_generation_prompt_ids_if_present() + + self._update_input_ids( + processing_class, + content_ids, + attention_mask=True, + loss_mask=False, + new_multi_modal_inputs=multi_modal_inputs, + ) + + def update_metrics(self, metrics: Any, tool_id: str) -> None: + """ + metrics: should be a dict of tools_name -> Any + """ + if self.metrics.get(tool_id) is None: + self.metrics[tool_id] = [] + self.metrics[tool_id].append(metrics) + + def _get_prompt_diffs( + self, + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, + full_prompt_ids: torch.Tensor, + current_prompt_ids: torch.Tensor, + diff_surrounding_chars: int = 10, + ) -> list[dict[str, Any]]: + """Get differences between full prompt and current prompt with surrounding context. + + This function helps debug tokenization mismatches by showing the differences between + full prompt and current prompt with surrounding context. Instead of just showing + the exact diff, it includes additional tokens before and after to help locate + the issue in the chat template. + + For example, if the actual diff is a newline change from "\n\n" to "\n", with + diff_surrounding_chars the output might look like: + + full_prompt_chunk: "<|im_start|>assistant\n\nI think..." + current_prompt_chunk: "<|im_start|>assistant\nI think..." + + This context makes it much easier to identify where in the chat template the + mismatch occurs. + + Args: + processing_class: The processing class to use for decoding the token IDs + full_prompt_ids: Token IDs from applying chat template to all messages at once + current_prompt_ids: Token IDs from incremental chat template application + diff_surrounding_chars: Number of surrounding characters to include for context (default: 10) + + Returns: + List of dicts containing the differing chunks with context and their indices + """ + full_prompt_ids = full_prompt_ids.squeeze(0) + current_prompt_ids = current_prompt_ids.squeeze(0) + full_prompt = processing_class.decode(full_prompt_ids, skip_special_tokens=False) + current_prompt = processing_class.decode(current_prompt_ids, skip_special_tokens=False) + s = difflib.SequenceMatcher(None, full_prompt, current_prompt, autojunk=False) + diffs = [] + for tag, i1, i2, j1, j2 in s.get_opcodes(): + if tag == "equal": + continue + + # Get the surrounding context for better readability + start_i = max(0, i1 - diff_surrounding_chars) + end_i = min(len(full_prompt), i2 + diff_surrounding_chars) + start_j = max(0, j1 - diff_surrounding_chars) + end_j = min(len(current_prompt), j2 + diff_surrounding_chars) + + diffs.append( + { + "full_prompt_chunk": full_prompt[start_i:end_i], + "current_prompt_chunk": current_prompt[start_j:end_j], + "indices": (start_i, end_i, start_j, end_j), + } + ) + return diffs + + def _remove_generation_prompt_ids_if_present(self) -> None: + """ + Remove generation prompt IDs from input tensors if they are present at the end. + """ + if self.input_ids[..., -self.generation_prompt_ids.shape[-1] :].eq(self.generation_prompt_ids).all(): + self.input_ids = self.input_ids[..., : -self.generation_prompt_ids.shape[-1]] + self.attention_mask = self.attention_mask[..., : -self.generation_prompt_ids.shape[-1]] + self.position_ids = self.position_ids[..., : -self.generation_prompt_ids.shape[-1]] + self.loss_mask = self.loss_mask[..., : -self.generation_prompt_ids.shape[-1]] + + def finalize( + self, + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, + reward_scores: dict[str, list[float]], + finish_reason_type: FinishReasonTypeEnum = FinishReasonTypeEnum.STOP, + ) -> None: + self.state = AsyncRolloutRequestStateEnum.COMPLETED + self.reward_scores = reward_scores + + # In case we failed to generate the assistant message and the generation prompt ids were already added to + # input_ids, remove them from the end of input_ids + self._remove_generation_prompt_ids_if_present() + + self.response_ids = self.input_ids[..., self.prompt_ids.shape[-1] :] + + if self.tokenization_sanity_check_mode != TokenizationSanityCheckModeEnum.DISABLE: + # When there is a diff, we log the diffs with diff_surrounding_chars context + diff_surrounding_chars = 10 + + messages = [msg.model_dump() for msg in self.messages] + tools = [tool.model_dump() for tool in self.tool_schemas] if self.tool_schemas else None + full_prompt_info = self._handle_apply_chat_template( + processing_class, + messages, + multi_modal_data=self.multi_modal_data, + tools=tools, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + ) + full_prompt_ids = full_prompt_info["input_ids"] + + # We must use dict(full_prompt_info) to convert BatchFeature values to a new dict + # because np.array() only keeps the keys for BatchFeature. + full_prompt_multi_modal_inputs = full_prompt_info.copy() + full_prompt_multi_modal_inputs.pop("input_ids", None) + full_prompt_multi_modal_inputs.pop("attention_mask", None) + + for multi_modal_inputs_key in self.multi_modal_inputs: + if multi_modal_inputs_key in full_prompt_multi_modal_inputs: + if ( + not self.multi_modal_inputs[multi_modal_inputs_key] + .eq(full_prompt_multi_modal_inputs[multi_modal_inputs_key]) + .all() + ): + logger.warning( + f"Multi-modal data {multi_modal_inputs_key} is not consistent. " + f"This may lead to unexpected behavior during training. " + f"Please review your multi_modal_inputs logic." + ) + else: + logger.warning( + f"Multi-modal inputs key {multi_modal_inputs_key} is not found in the multi_modal_inputs. " + f"This may lead to unexpected behavior during training." + f"Please review your multi_modal_inputs logic." + ) + + if diffs := self._get_prompt_diffs( + processing_class, full_prompt_ids, self.input_ids, diff_surrounding_chars=diff_surrounding_chars + ): + log_warning = False + if self.tokenization_sanity_check_mode == TokenizationSanityCheckModeEnum.STRICT: + log_warning = True + elif self.tokenization_sanity_check_mode == TokenizationSanityCheckModeEnum.IGNORE_STRIPPABLE: + non_strippable_diffs_exist = any( + d["full_prompt_chunk"].strip() or d["current_prompt_chunk"].strip() for d in diffs + ) + if non_strippable_diffs_exist: + log_warning = True + + if log_warning: + mode_str = f" ({self.tokenization_sanity_check_mode.value})" + logger.warning( + f"Inconsistent training and inference tokenization detected{mode_str}. This may lead to " + f"unexpected behavior during training. Please review your chat template to determine if this " + f"is intentional. For more information, refer to the multiturn README.md." + ) + logger.warning( + f"Showing {diff_surrounding_chars} characters before and after the diffs for context and " + f"better readability." + ) + diff_details_list = [] + for d in diffs: + i1, i2, j1, j2 = d["indices"] + diff_details_list.append( + f"idx {i1}:{i2} -> {j1}:{j2} | full_prompt_chunk: {repr(d['full_prompt_chunk'])} | " + f"current_prompt_chunk: {repr(d['current_prompt_chunk'])}" + ) + diff_details = "\n".join(diff_details_list) + logger.warning(f"Found differences:\n{diff_details}") + + if finish_reason_type == FinishReasonTypeEnum.STOP: + pass + elif finish_reason_type == FinishReasonTypeEnum.LENGTH: + pass + else: + raise ValueError(f"Unsupported finalize finish reason type: {finish_reason_type}") + self.truncate_output_ids(processing_class) + + assert ( + self.input_ids.shape[-1] + == self.attention_mask.shape[-1] + == self.position_ids.shape[-1] + == self.loss_mask.shape[-1] + ), f"""Request {self.request_id} has different length of {self.input_ids.shape[-1]=}, + {self.attention_mask.shape[-1]=}, {self.position_ids.shape[-1]=}, {self.loss_mask.shape[-1]=}""" + + def truncate_output_ids( + self, processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin + ) -> None: + self.input_ids = self.input_ids[..., : self.max_model_len] + self.attention_mask = self.attention_mask[..., : self.max_model_len] + self.position_ids = self.position_ids[..., : self.max_model_len] + self.loss_mask = self.loss_mask[..., : self.max_model_len] + self.response_ids = self.input_ids[..., self.prompt_ids.shape[-1] :][..., : self.max_response_len] + self.response_attention_mask = self.attention_mask[..., self.prompt_attention_mask.shape[-1] :][ + ..., : self.max_response_len + ] + self.response_position_ids = self.position_ids[..., self.prompt_position_ids.shape[-1] :][ + ..., : self.max_response_len + ] + self.response_loss_mask = self.loss_mask[..., self.prompt_loss_mask.shape[-1] :][..., : self.max_response_len] diff --git a/e2c/verl/verl/workers/rollout/sglang_rollout/__init__.py b/e2c/verl/verl/workers/rollout/sglang_rollout/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..337ec74bce1922b8af69a9879fcbf0f2b382c555 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/sglang_rollout/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and diff --git a/e2c/verl/verl/workers/rollout/sglang_rollout/async_sglang_server.py b/e2c/verl/verl/workers/rollout/sglang_rollout/async_sglang_server.py new file mode 100644 index 0000000000000000000000000000000000000000..44c19b34992c76c4ca9abfc9bcd1879698ce0da8 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/sglang_rollout/async_sglang_server.py @@ -0,0 +1,302 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import logging +import os +from typing import Any, Optional + +import ray +import sglang.srt.entrypoints.engine +import torch +from ray.actor import ActorHandle +from sglang.srt.entrypoints.http_server import ( + ServerArgs, + _GlobalState, + _launch_subprocesses, + app, + set_global_state, +) +from sglang.srt.managers.io_struct import ( + GenerateReqInput, + ReleaseMemoryOccupationReqInput, + ResumeMemoryOccupationReqInput, +) + +from verl.single_controller.ray import RayClassWithInitArgs +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RewardModelConfig, RolloutConfig +from verl.workers.rollout.replica import RolloutMode, RolloutReplica, TokenOutput +from verl.workers.rollout.sglang_rollout.sglang_rollout import ServerAdapter, _set_envs_and_config +from verl.workers.rollout.utils import get_free_port, is_valid_ipv6_address, run_unvicorn + +logger = logging.getLogger(__file__) +logger.setLevel(logging.INFO) + + +@ray.remote(num_cpus=1) +class SGLangHttpServer: + """SGLang http server in single node, this is equivalent to launch server with command line: + ``` + python -m sglang.launch_server --node-rank 0 --nnode 1 ... + ``` + + Args: + config (DictConfig): full config. + rollout_mode (RolloutMode): rollout mode. + replica_rank (int): replica rank, a replica may contain multiple nodes. + node_rank (int): node rank. + nnodes (int): number of nodes. + cuda_visible_devices (str): cuda visible devices. + """ + + def __init__( + self, + config: RolloutConfig | RewardModelConfig, + model_config: HFModelConfig, + rollout_mode: RolloutMode, + workers: list[ActorHandle], + replica_rank: int, + node_rank: int, + nnodes: int, + cuda_visible_devices: str, + ): + print(f"SGLang http server: {rollout_mode=}, {replica_rank=}, {node_rank=}, {nnodes=}, {cuda_visible_devices=}") + os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices + assert torch.cuda.is_available(), "SGLang http server should run on GPU node" + + self.config: RolloutConfig | RewardModelConfig = omega_conf_to_dataclass(config) + self.model_config: HFModelConfig = omega_conf_to_dataclass(model_config, dataclass_type=HFModelConfig) + self.config.max_model_len = self.config.prompt_length + self.config.response_length + self.rollout_mode = rollout_mode + self.workers = workers + + self.replica_rank = replica_rank + self.node_rank = node_rank + self.nnodes = nnodes + + if self.rollout_mode != RolloutMode.HYBRID and self.config.load_format == "dummy": + logger.warning(f"rollout mode is {self.rollout_mode}, load_format is dummy, set to auto") + self.config.load_format = "auto" + + # used for http server + self._server_address = ray.util.get_node_ip_address().strip("[]") + self._server_port = None + + # used for NCCL process group + if self.node_rank == 0: + self._master_address = self._server_address + self._master_port, self._master_sock = get_free_port(self._server_address) + logger.info( + f"SGLangHttpServer, replica_rank: {self.replica_rank}, " + f"master address: {self._master_address}, port: {self._master_port}" + ) + else: + self._master_address = None + self._master_port = None + + def get_master_address(self): + """Get master address and port for init NCCL process group.""" + return self._master_address, self._master_port + + def get_server_address(self): + """Get http server address and port.""" + assert self._server_port is not None, "http server is not launched, port is None" + return self._server_address, self._server_port + + async def launch_server(self, master_address: str = None, master_port: int = None): + if self.node_rank != 0: + assert master_address and master_port, "non-master node should provide master address and port" + self._master_address = master_address + self._master_port = master_port + + engine_kwargs = self.config.get("engine_kwargs", {}).get("sglang", {}) or {} + attention_backend = engine_kwargs.pop("attention_backend", None) + dist_init_addr = ( + f"[{self._master_address}]:{self._master_port}" + if is_valid_ipv6_address(self._master_address) + else f"{self._master_address}:{self._master_port}" + ) + + args = { + "model_path": self.model_config.local_path, + "dtype": self.config.dtype, + "mem_fraction_static": self.config.gpu_memory_utilization, + "disable_cuda_graph": self.config.enforce_eager, + "enable_memory_saver": True, + "base_gpu_id": 0, + "gpu_id_step": 1, + "tp_size": self.config.tensor_model_parallel_size, + "dp_size": self.config.data_parallel_size, + "ep_size": self.config.expert_parallel_size, + "node_rank": self.node_rank, + "load_format": self.config.load_format, + "dist_init_addr": dist_init_addr, + "nnodes": self.nnodes, + "trust_remote_code": self.model_config.trust_remote_code, + "max_running_requests": self.config.get("max_num_seqs", None), + "log_level": "error", + "mm_attention_backend": "fa3", + "attention_backend": attention_backend if attention_backend is not None else "fa3", + "skip_tokenizer_init": self.config.skip_tokenizer_init, + } + + # NOTE: We can't directly call SGLang's launch_server since it's not an async function. + # https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/http_server.py + sglang.srt.entrypoints.engine._set_envs_and_config = _set_envs_and_config + os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" + server_args = ServerArgs(**args) + self.tokenizer_manager, self.template_manager, self.scheduler_info = _launch_subprocesses( + server_args=server_args + ) + + # In multi-node cases, non-zero rank nodes should not launch http server. + if self.node_rank > 0: + return + + set_global_state( + _GlobalState( + tokenizer_manager=self.tokenizer_manager, + template_manager=self.template_manager, + scheduler_info=self.scheduler_info, + ) + ) + app.is_single_tokenizer_mode = True + self._server_port, self._server_task = await run_unvicorn(app, server_args, self._server_address) + + async def wake_up(self): + if self.rollout_mode == RolloutMode.HYBRID: + # Call all workers to switch between trainer mode and rollout mode. + await asyncio.gather(*[worker.wake_up.remote() for worker in self.workers]) + elif self.rollout_mode == RolloutMode.COLOCATED: + # Directly call engine to wake up without sync weights. + # FIXME(@wuxibin): sglang seems resume with random weights. + obj = ResumeMemoryOccupationReqInput(tags=["kv_cache", "weights"]) + await self.tokenizer_manager.resume_memory_occupation(obj, None) + await self.tokenizer_manager.flush_cache() + elif self.rollout_mode == RolloutMode.STANDALONE: + logger.info("skip wake_up in standalone mode") + + async def sleep(self): + if self.rollout_mode == RolloutMode.HYBRID: + await asyncio.gather(*[worker.sleep.remote() for worker in self.workers]) + elif self.rollout_mode == RolloutMode.COLOCATED: + obj = ReleaseMemoryOccupationReqInput(tags=["kv_cache", "weights"]) + await self.tokenizer_manager.release_memory_occupation(obj, None) + elif self.rollout_mode == RolloutMode.STANDALONE: + logger.info("skip sleep in standalone mode") + + async def generate( + self, + prompt_ids: torch.Tensor, + sampling_params: dict[str, Any], + request_id: str, + image_data: Optional[list[Any]] = None, + ) -> TokenOutput: + """Generate sequence with token-in-token-out.""" + # TODO(@wuxibin): switch to `/generate` http endpoint once multi-modal support ready. + max_new_tokens = min(self.config.response_length, self.config.max_model_len - len(prompt_ids) - 1) + sampling_params["max_new_tokens"] = max_new_tokens + return_logprob = sampling_params.pop("logprobs", False) + + request = GenerateReqInput( + rid=request_id, + input_ids=prompt_ids, + sampling_params=sampling_params, + return_logprob=return_logprob, + image_data=image_data, + ) + output = await self.tokenizer_manager.generate_request(request, None).__anext__() + if return_logprob: + output_token_logprobs = output["meta_info"]["output_token_logprobs"] + log_probs, token_ids = zip( + *[(log_prob, token_ids) for log_prob, token_ids, _ in output_token_logprobs], strict=True + ) + else: + token_ids = output["output_ids"] + log_probs = None + return TokenOutput(token_ids=token_ids, log_probs=log_probs) + + +_rollout_worker_actor_cls = ray.remote(ServerAdapter) + + +class SGLangReplica(RolloutReplica): + def get_ray_class_with_init_args(self) -> RayClassWithInitArgs: + """Get rollout worker actor class for colocated and standalone mode.""" + worker_dict_cls = RayClassWithInitArgs( + cls=_rollout_worker_actor_cls, + config=self.config, + model_config=self.model_config, + device_mesh=None, + ) + return worker_dict_cls + + async def launch_servers(self): + """Launch http server in each node.""" + assert len(self.workers) == self.world_size, ( + f"worker number {len(self.workers)} not equal to world size {self.world_size}" + ) + + # get (node_id, CUDA_VISIBLE_DEVICES) of all workers + worker_infos = await asyncio.gather( + *[ + worker.__ray_call__.remote( + lambda self: (ray.get_runtime_context().get_node_id(), os.environ["CUDA_VISIBLE_DEVICES"]) + ) + for worker in self.workers + ] + ) + worker_cuda_visible_devices = [worker_info[1] for worker_info in worker_infos] + worker_node_ids = [worker_info[0] for worker_info in worker_infos] + + # create server actor in each node with node affinity and cuda visible devices + for node_rank in range(self.nnodes): + workers = self.workers[node_rank * self.gpus_per_node : (node_rank + 1) * self.gpus_per_node] + node_cuda_visible_devices = ",".join( + worker_cuda_visible_devices[node_rank * self.gpus_per_node : (node_rank + 1) * self.gpus_per_node] + ) + node_id = worker_node_ids[node_rank * self.gpus_per_node] + server = SGLangHttpServer.options( + scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=node_id, + soft=False, + ), + runtime_env={"env_vars": {"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1"}}, + name=f"sglang_server_{self.replica_rank}_{node_rank}", + ).remote( + config=self.config, + model_config=self.model_config, + rollout_mode=self.rollout_mode, + workers=workers, + replica_rank=self.replica_rank, + node_rank=node_rank, + nnodes=self.nnodes, + cuda_visible_devices=node_cuda_visible_devices, + ) + self.servers.append(server) + + # launch http server in each node + master_address, master_port = await self.servers[0].get_master_address.remote() + await asyncio.gather( + *[ + server.launch_server.remote(master_address=master_address, master_port=master_port) + for server in self.servers + ] + ) + + # get http server address from first server + server_address, server_port = await self.servers[0].get_server_address.remote() + self._server_handle = self.servers[0] + self._server_address = f"{server_address}:{server_port}" diff --git a/e2c/verl/verl/workers/rollout/sglang_rollout/http_server_engine.py b/e2c/verl/verl/workers/rollout/sglang_rollout/http_server_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..6822a9e52dac74cbb8320bc2d54e7eab421df57b --- /dev/null +++ b/e2c/verl/verl/workers/rollout/sglang_rollout/http_server_engine.py @@ -0,0 +1,954 @@ +# Copyright 2025 z.ai +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file is adapted from multiple sources: +# 1. THUDM/slime project +# Original source: https://github.com/THUDM/slime/blob/main/slime/backends/sglang_utils/http_server_engine.py +# Copyright 2025 z.ai +# Licensed under the Apache License, Version 2.0 +# 2. SGLang project +# Original source: https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/http_server_engine.py +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 +# +# Modifications made by z.ai and ModelBest Inc. include but are not limited to: +# - Enhanced error handling and retry logic +# - Added async support with connection pooling +# - Extended functionality for distributed weight updates +# - Improved logging and monitoring capabilities +# - Additional configuration options and optimizations + +"""HTTP Server Engine Adapter for SGLang. + +This module provides HTTP-based adapters for SGLang engines, allowing communication +with SGLang servers through HTTP requests instead of direct engine calls. + +Classes: + HttpServerAdapter: Synchronous HTTP adapter for SGLang engines + AsyncHttpServerAdapter: Asynchronous HTTP adapter for SGLang engines + +Functions: + launch_server_process: Launch and initialize an SGLang HTTP server process +""" + +import asyncio +import logging +import multiprocessing +import os +import time +from contextlib import asynccontextmanager +from typing import Any, Callable, Optional + +import aiohttp +import requests +from sglang.srt.entrypoints.EngineBase import EngineBase +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.managers.io_struct import ( + UpdateWeightsFromTensorReqInput, +) +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import kill_process_tree + +# Configure logger +logger = logging.getLogger(__name__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +# Default configuration constants +DEFAULT_TIMEOUT = 60.0 +DEFAULT_MAX_ATTEMPTS = 3 +DEFAULT_RETRY_DELAY = 2.0 +DEFAULT_MAX_CONNECTIONS = 2000 +DEFAULT_MAX_WAIT_TIME = 300.0 + + +def _read_response(response: requests.Response): + if response.status_code == 204 or not response.content: + return {} + try: + return response.json() + except ValueError: + return { + "content_type": response.headers.get("Content-Type", ""), + "text": response.text, + } + + +async def _read_async_response(resp: aiohttp.ClientResponse) -> dict[str, Any]: + if resp.status == 204 or (resp.content_length == 0): + return {} + + try: + return await resp.json(content_type=None) + except Exception: + try: + text = await resp.text() + except Exception: + return {} + return { + "content_type": (resp.headers.get("Content-Type") or ""), + "text": text, + } + + +def launch_server_process( + server_args: ServerArgs, + timeout: float = DEFAULT_TIMEOUT, + max_wait_time=DEFAULT_MAX_WAIT_TIME, + first_rank_in_node=False, +) -> multiprocessing.Process: + """Launch an SGLang HTTP server process and wait for it to be ready. + + This function starts a new process running an SGLang HTTP server, then waits + for the server to become ready by polling its health endpoints. It ensures + the server is fully operational before returning. + + Args: + server_args (ServerArgs): Server configuration arguments including host, port, and other settings + timeout (float, optional): Timeout for individual HTTP requests during health checks. + Defaults to DEFAULT_TIMEOUT. + + Returns: + multiprocessing.Process: The launched multiprocessing.Process instance + + Raises: + RuntimeError: If the server process terminates unexpectedly during startup or cache flush + TimeoutError: If server fails to become ready within reasonable time (300 seconds) + requests.RequestException: If health check requests fail repeatedly + + Note: + This function will return immediately for non-master nodes (node_rank != 0), + but the process will still be started and returned. + This is for consistency; except for the process obtained by node_rank = 0, + other processes have no actual effect. + """ + p = multiprocessing.Process(target=launch_server, args=(server_args,)) + if server_args.node_rank != 0 or not first_rank_in_node: + logger.info(f"Server process started with PID {p.pid} for node rank {server_args.node_rank}", flush=True) + return p + + p.start() + + base_url = server_args.url() + headers = { + "Content-Type": "application/json; charset=utf-8", + "Authorization": f"Bearer {server_args.api_key}", + } + + # Health check with overall timeout + start_time = time.time() + + with requests.Session() as session: + while time.time() - start_time < max_wait_time: + if not p.is_alive(): + raise RuntimeError("Server process terminated unexpectedly during startup") + + try: + if server_args.is_embedding: + response = session.get(f"{base_url}/health", headers=headers, timeout=timeout) + else: + response = session.get(f"{base_url}/health_generate", headers=headers, timeout=timeout) + if response.status_code == 200: + break + except requests.RequestException as e: + logger.debug(f"Health check failed: {e}") + + time.sleep(2) + else: + p.terminate() + logger.error(f"Server in {base_url} failed to become healthy within timeout period") + raise TimeoutError("Server failed to become healthy within timeout period") + + # Ensure cache is ready + while time.time() - start_time < max_wait_time: + if not p.is_alive(): + raise RuntimeError("Server process terminated unexpectedly during cache flush") + + try: + response = session.get(f"{base_url}/flush_cache", headers=headers, timeout=timeout) + if response.status_code == 200: + break + except requests.RequestException as e: + logger.debug(f"Cache flush check failed: {e}") + + time.sleep(2) + else: + p.terminate() + raise TimeoutError("Server cache flush failed within timeout period") + + return p + + +class HttpServerAdapter(EngineBase): + """HTTP-based adapter for SGLang engines. + + This adapter allows interaction with SGLang engines through HTTP requests + instead of direct engine calls. It launches an HTTP server process and + provides methods to communicate with it via REST API calls. + + You can use this class to launch a server from a HttpServerAdapter instance. + We recommend using this class only when you need to use http server. + Otherwise, you can use Engine directly. + + Attributes: + router_ip (Optional[str]): IP address of the router for worker registration + router_port (Optional[int]): Port of the router for worker registration + server_args (ServerArgs): Server configuration arguments + node_rank (int): Rank of this node in distributed setup + process (multiprocessing.Process): The launched server process + timeout (float): HTTP request timeout in seconds + max_attempts (int): Maximum number of attempts for requests + retry_delay (float): Base delay between retries in seconds + """ + + def __init__( + self, + router_ip: Optional[str] = None, + router_port: Optional[int] = None, + timeout: float = DEFAULT_TIMEOUT, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + retry_delay: float = DEFAULT_RETRY_DELAY, + first_rank_in_node: bool = False, + max_start_wait_time: float = DEFAULT_MAX_WAIT_TIME, + launch_server: bool = True, + **kwargs: Any, + ) -> None: + """Initialize the HTTP server engine adapter. + + Args: + router_ip (Optional[str], optional): IP address of router for worker registration. + Defaults to None. + router_port (Optional[int], optional): Port of router for worker registration. + Defaults to None. + timeout (float, optional): HTTP request timeout in seconds. + Defaults to DEFAULT_TIMEOUT. + max_attempts (int, optional): Maximum number of retry attempts for failed requests. + Defaults to DEFAULT_MAX_ATTEMPTS. + retry_delay (float, optional): Base delay between retries in seconds. + Defaults to DEFAULT_RETRY_DELAY. + launch_server (bool, optional): Whether to launch the server process. + Defaults to True. + **kwargs (Any): Additional arguments passed to ServerArgs + + Note: + TODO: @ChangyiYang Enable SGLang router for this http server engine + If both router_ip and router_port are provided and this is the master node + (node_rank == 0), the adapter will automatically register with the router. + """ + self.router_ip: Optional[str] = router_ip + self.router_port: Optional[int] = router_port + self.timeout: float = timeout + self.max_attempts: int = max_attempts + self.retry_delay: float = retry_delay + self.server_args: ServerArgs = ServerArgs(**kwargs) + self.node_rank: int = self.server_args.node_rank + self.max_start_wait_time: float = max_start_wait_time + + logger.info( + f"Launch HttpServerAdapter at: {self.server_args.host}:{self.server_args.port} with {first_rank_in_node}" + ) + if launch_server: + self.process: multiprocessing.Process = launch_server_process( + self.server_args, self.timeout, self.max_start_wait_time, first_rank_in_node + ) + + if self.node_rank == 0 and self.router_ip and self.router_port: + self._register_with_router() + + def _register_with_router(self) -> None: + """Register worker with router with error handling. + + This method attempts to register the current worker with a router service. + If registration fails, it logs an error but does not raise an exception, + allowing the server to continue operating without router integration. + + Raises: + Does not raise exceptions - all errors are logged and handled gracefully. + """ + try: + url = f"http://{self.router_ip}:{self.router_port}/add_worker" + params = {"url": f"http://{self.server_args.host}:{self.server_args.port}"} + response = requests.post(url, params=params, timeout=self.timeout) + response.raise_for_status() + logger.info("Successfully registered with router") + except Exception as e: + logger.error(f"Failed to register with router: {e}") + # Don't raise here - server can still work without router + + def _make_request( + self, + endpoint: str, + payload: Optional[dict[str, Any]] = None, + method: str = "POST", + timeout: float = DEFAULT_TIMEOUT, + only_master: bool = True, + ) -> dict[str, Any]: + """Make a HTTP request with retry logic and consistent error handling. + + Args: + endpoint (str): The API endpoint to call (without leading slash) + payload (Optional[Dict[str, Any]], optional): The JSON payload to send. + Defaults to empty dict if None. + method (str, optional): HTTP method to use. Defaults to "POST". + + Returns: + Dict[str, Any]: The JSON response from the server + + Raises: + requests.HTTPError: If the HTTP request fails with a client/server error + RuntimeError: If all retry attempts are exhausted + + Note: + - For non-master nodes (node_rank != 0), returns empty dict immediately + - Uses exponential backoff for retries + - Logs warnings for timeout and connection errors, errors for HTTP errors + """ + if only_master and self.node_rank != 0: + return {} + + url = f"http://{self.server_args.host}:{self.server_args.port}/{endpoint}" + + for attempt in range(self.max_attempts): + try: + if method.upper() == "GET": + response = requests.get(url, timeout=self.timeout) + else: + response = requests.post(url, json=payload or {}, timeout=self.timeout) + + response.raise_for_status() + return _read_response(response) + + except requests.exceptions.Timeout: + logger.warning(f"Request to {endpoint} timed out (attempt {attempt + 1})") + except requests.exceptions.ConnectionError: + logger.warning(f"Connection error for {endpoint} (attempt {attempt + 1})") + except requests.exceptions.HTTPError as e: + logger.error(f"HTTP error for {endpoint}: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error for {endpoint}: {e}") + if attempt == self.max_attempts - 1: + raise + + if attempt < self.max_attempts - 1: + time.sleep(self.retry_delay * (2**attempt)) + + raise RuntimeError(f"Failed to complete request to {endpoint} after {self.max_attempts} attempts") + + def update_weights_from_tensor(self, req: UpdateWeightsFromTensorReqInput) -> dict[str, Any]: + """Update model weights from tensor data. + + The HTTP server will only post meta data, and the real weights will be + copied directly from GPUs. + + Args: + serialized_named_tensors (List[str]): List of serialized tensor data + load_format (Optional[str], optional): Format specification for loading weights. + Defaults to None. + flush_cache (bool, optional): Whether to flush cache after updating weights. + Defaults to False. + + Returns: + Dict[str, Any]: Server response containing update status + + Note: + The model should be on GPUs rather than CPU for this functionality to work properly. + If you encounter issues, ensure your model is loaded on GPU devices rather than CPU. + """ + import base64 + + named_tensors = req.serialized_named_tensors + load_format = req.load_format + flush_cache = req.flush_cache + + if named_tensors: + serialized_named_tensors = [ + base64.b64encode(named_tensor).decode("utf-8") for named_tensor in named_tensors + ] + else: + serialized_named_tensors = [] + + return self._make_request( + "update_weights_from_tensor", + { + "serialized_named_tensors": serialized_named_tensors, + "load_format": load_format, + "flush_cache": flush_cache, + }, + ) + + def shutdown(self) -> None: + """Shutdown the HTTP server and clean up resources. + + This method performs the following cleanup operations: + 1. Unregisters the worker from the router (if configured) + 2. Terminates the server process tree + + All operations are performed with error handling to ensure graceful shutdown + even if individual steps fail. + + Note: + This method should be called when the adapter is no longer needed + to ensure proper cleanup of resources and processes. + """ + # Unregister from router + if self.router_ip and self.router_port: + try: + url = f"http://{self.router_ip}:{self.router_port}/remove_worker" + params = {"url": f"http://{self.server_args.host}:{self.server_args.port}"} + requests.post(url, params=params, timeout=5.0) # Short timeout for shutdown + logger.info("Successfully unregistered from router") + except Exception as e: + logger.warning(f"Failed to unregister from router: {e}") + + # Kill server process + if hasattr(self, "process") and self.process is not None: + try: + kill_process_tree(self.process.pid) + logger.info("Server process terminated") + except Exception as e: + logger.error(f"Failed to terminate server process: {e}") + + def generate( + self, + prompt: Optional[str] = None, + sampling_params: Optional[dict[str, Any]] = None, + input_ids: Optional[list[int]] = None, + image_data: Optional[Any] = None, + return_logprob: bool = False, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + token_ids_logprob: Optional[list[int]] = None, + lora_path: Optional[str] = None, + custom_logit_processor: Optional[Callable] = None, + ) -> dict[str, Any]: + """Generate text using the SGLang server. + + Args: + prompt (Optional[str], optional): Text prompt for generation. Defaults to None. + sampling_params (Optional[Dict[str, Any]], optional): Parameters controlling + text generation sampling. Defaults to None. + input_ids (Optional[List[int]], optional): Alternative to prompt, direct token IDs input. + Defaults to None. + image_data (Optional[Any], optional): Image data for multimodal generation. + Defaults to None. + return_logprob (bool, optional): Whether to return log probabilities. + Defaults to False. + logprob_start_len (Optional[int], optional): Starting length for log probability calculation. + Defaults to None. + top_logprobs_num (Optional[int], optional): Number of top log probabilities to return. + Defaults to None. + token_ids_logprob (Optional[List[int]], optional): Specific token IDs for + log probability calculation. Defaults to None. + lora_path (Optional[str], optional): Path to LoRA adapter weights. Defaults to None. + custom_logit_processor (Optional[Callable], optional): Custom logit processing function. + Defaults to None. + + Returns: + Dict[str, Any]: Generated text and associated metadata from the server + + Note: + Either prompt or input_ids should be provided, but not both. + The response format depends on the server configuration and parameters. + """ + payload = { + "text": prompt, + "sampling_params": sampling_params, + "input_ids": input_ids, + "image_data": image_data, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + "token_ids_logprob": token_ids_logprob, + "lora_path": lora_path, + "custom_logit_processor": custom_logit_processor, + } + # Filter out None values + payload = {k: v for k, v in payload.items() if v is not None} + + return self._make_request("generate", payload, only_master=False) + + def reward_score( + self, + prompt: Optional[str] = None, + input_ids: Optional[list[int]] = None, + image_data: Optional[Any] = None, + lora_path: Optional[str] = None, + ) -> dict[str, Any]: + assert self.server_args.is_embedding, "Score is only supported for embedding models" + payload = { + "text": prompt, + "input_ids": input_ids, + "image_data": image_data, + "lora_path": lora_path, + } + # Filter out None values + payload = {k: v for k, v in payload.items() if v is not None} + + return self._make_request("classify", payload, only_master=False) + + def flush_cache(self) -> dict[str, Any]: + """Flush the cache of the server. + + This method repeatedly attempts to flush the server cache until successful. + The flush operation will not return status 200 when there are pending requests. + + Returns: + Dict[str, Any]: Server response indicating cache flush status. + For non-master nodes, returns empty dict. + + Note: + Uses retry logic with limited attempts (max_attempts * 2) to avoid infinite loops. + Each retry includes a delay to allow pending requests to complete. + """ + if self.node_rank != 0: + return {} + + # Use retry logic with limited attempts to avoid infinite loops + for attempt in range(self.max_attempts * 2): # Allow more retries for cache flush + try: + response = requests.get( + f"http://{self.server_args.host}:{self.server_args.port}/flush_cache", timeout=self.timeout + ) + if response.status_code == 200: + return _read_response(response) + except Exception as e: + logger.warning(f"Error flushing cache (attempt {attempt + 1}): {e}") + + time.sleep(self.retry_delay) + + logger.error("Failed to flush cache after maximum attempts") + return {} + + def release_memory_occupation(self, tags: Optional[list[str]] = None) -> dict[str, Any]: + """Release GPU memory occupation temporarily. + + Args: + tags (Optional[List[str]], optional): List of tags to specify which memory to release. + If None, releases all memory. Defaults to None. ["weights", "kv_cache"] + + Returns: + Dict[str, Any]: Server response indicating memory release status + """ + return self._make_request("release_memory_occupation", {"tags": tags}) + + def resume_memory_occupation(self, tags: Optional[list[str]] = None) -> dict[str, Any]: + """Resume GPU memory occupation. + + Args: + tags (Optional[List[str]], optional): List of tags to specify which memory to resume. + If None, resumes all memory. Defaults to None. ["weights", "kv_cache"] + + Returns: + Dict[str, Any]: Server response indicating memory resume status + """ + return self._make_request("resume_memory_occupation", {"tags": tags}) + + def abort_request(self, rid: str = "", abort_all: bool = False) -> dict[str, Any]: + """Abort a request. + + Args: + rid (str): The ID of the request to abort + abort_all (bool, optional): Whether to abort all requests. Defaults to False. + + Returns: + Dict[str, Any]: Server response indicating abort status + """ + return self._make_request("abort_request", {"rid": rid, "abort_all": abort_all}) + + +class AsyncHttpServerAdapter(HttpServerAdapter): + """Asynchronous HTTP-based adapter for SGLang engines. + + This class inherits from HttpServerAdapter and adds async capabilities + for non-blocking HTTP requests to the SGLang server. It provides the same + functionality as the synchronous version but with async/await support. + + The async adapter is useful when you need to make multiple concurrent requests + or integrate with async frameworks. It uses aiohttp for efficient async HTTP + communication and maintains connection pooling for better performance. + + Attributes: + max_connections (int): Maximum number of connections in the connection pool + """ + + def __init__( + self, + router_ip: Optional[str] = None, + router_port: Optional[int] = None, + timeout: float = DEFAULT_TIMEOUT, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + retry_delay: float = DEFAULT_RETRY_DELAY, + max_connections: int = DEFAULT_MAX_CONNECTIONS, + first_rank_in_node: bool = False, + launch_server: bool = True, + **kwargs: Any, + ) -> None: + """Initialize the async HTTP server engine adapter. + + Args: + router_ip (Optional[str], optional): IP address of router for worker registration. + Defaults to None. + router_port (Optional[int], optional): Port of router for worker registration. + Defaults to None. + timeout (float, optional): HTTP request timeout in seconds. + Defaults to DEFAULT_TIMEOUT. + max_attempts (int, optional): Maximum number of retry attempts for failed requests. + Defaults to DEFAULT_MAX_ATTEMPTS. + retry_delay (float, optional): Base delay between retries in seconds. + Defaults to DEFAULT_RETRY_DELAY. + max_connections (int, optional): Maximum number of connections in the connection pool. + Defaults to DEFAULT_MAX_CONNECTIONS. + launch_server (bool, optional): Whether to launch the server process. + Defaults to True. + **kwargs (Any): Additional arguments passed to ServerArgs + """ + super().__init__( + router_ip, + router_port, + timeout, + max_attempts, + retry_delay, + first_rank_in_node, + launch_server=launch_server, + **kwargs, + ) + self.max_connections: int = max_connections + + @asynccontextmanager + async def _get_session(self) -> aiohttp.ClientSession: + """Context manager for safe session access with proper connection pooling. + + Yields: + aiohttp.ClientSession: Session instance for making HTTP requests + + Note: + This method creates a new session for each request to avoid resource competition + while still maintaining proper connection pooling through the shared connector. + """ + # Create a new session for each request to avoid resource competition + connector = aiohttp.TCPConnector( + limit=self.max_connections, + limit_per_host=self.max_connections // 4, + ttl_dns_cache=300, + use_dns_cache=True, + ) + timeout = aiohttp.ClientTimeout(total=self.timeout) + session = aiohttp.ClientSession(connector=connector, timeout=timeout) + + try: + yield session + finally: + # Always close the session to free up resources + if not session.closed: + await session.close() + + async def _make_async_request( + self, + endpoint: str, + payload: Optional[dict[str, Any]] = None, + method: str = "POST", + timeout: float = DEFAULT_TIMEOUT, + only_master: bool = True, + ) -> dict[str, Any]: + """Make an async HTTP request with retry logic and consistent error handling. + + Args: + endpoint (str): The API endpoint to call (without leading slash) + payload (Optional[Dict[str, Any]], optional): The JSON payload to send. + Defaults to empty dict if None. + method (str, optional): HTTP method to use. Defaults to "POST". + + Returns: + Dict[str, Any]: The JSON response from the server + + Raises: + aiohttp.ClientResponseError: If the HTTP request fails with a client/server error + RuntimeError: If all retry attempts are exhausted + + Note: + - For non-master nodes (node_rank != 0), returns empty dict immediately + - Uses exponential backoff for retries + - Logs warnings for timeout and connection errors, errors for HTTP errors + """ + if only_master and self.node_rank != 0: + return {} + + url = f"http://{self.server_args.host}:{self.server_args.port}/{endpoint}" + + for attempt in range(self.max_attempts): + try: + async with self._get_session() as session: + if method.upper() == "GET": + async with session.get(url, timeout=timeout) as response: + response.raise_for_status() + return await _read_async_response(response) + else: + async with session.post(url, json=payload or {}, timeout=timeout) as response: + response.raise_for_status() + return await _read_async_response(response) + + except asyncio.TimeoutError: + logger.warning(f"Async request to {endpoint} timed out (attempt {attempt + 1})") + except aiohttp.ClientConnectorError: + logger.warning(f"Connection error for {endpoint} (attempt {attempt + 1})") + except aiohttp.ClientResponseError as e: + logger.error(f"HTTP error for {endpoint}: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error for {endpoint}: {e}") + if attempt == self.max_attempts - 1: + raise + + if attempt < self.max_attempts - 1: + await asyncio.sleep(self.retry_delay * (2**attempt)) + + raise RuntimeError(f"Failed to complete async request to {endpoint} after {self.max_attempts} attempts") + + async def release_memory_occupation(self, tags: Optional[list[str]] = None) -> dict[str, Any]: + """Release GPU memory occupation temporarily (async version). + + Args: + tags (Optional[List[str]], optional): List of tags to specify which memory to release. + If None, releases all memory. Defaults to None. ["weights", "kv_cache"] + + Returns: + Dict[str, Any]: Server response indicating memory release status + """ + return await self._make_async_request("release_memory_occupation", {"tags": tags}) + + async def resume_memory_occupation(self, tags: Optional[list[str]] = None) -> dict[str, Any]: + """Resume GPU memory occupation (async version). + + Similar to AsyncEngine, this method handles first-time weight reloading + by calling release_memory_occupation if needed. + + Args: + tags (Optional[List[str]], optional): List of tags to specify which memory to resume. + If None, resumes all memory. Defaults to None. ["weights", "kv_cache"] + + Returns: + Dict[str, Any]: Server response indicating memory resume status + """ + return await self._make_async_request("resume_memory_occupation", {"tags": tags}) + + async def update_weights_from_tensor( + self, + req: UpdateWeightsFromTensorReqInput, + ) -> dict[str, Any]: + """Update model weights from tensor data asynchronously. + + Args: + serialized_named_tensors (List[str]): List of serialized tensor data + load_format (Optional[str], optional): Format specification for loading weights. + Defaults to None. + flush_cache (bool, optional): Whether to flush cache after updating weights. + Defaults to True. + + Returns: + Dict[str, Any]: Server response containing update status + """ + import base64 + + named_tensors = req.serialized_named_tensors + load_format = req.load_format + flush_cache = req.flush_cache + + serialized_named_tensors = [base64.b64encode(named_tensor).decode("utf-8") for named_tensor in named_tensors] + return await self._make_async_request( + "update_weights_from_tensor", + { + "serialized_named_tensors": serialized_named_tensors, + "load_format": load_format, + "flush_cache": flush_cache, + }, + ) + + async def flush_cache(self) -> dict[str, Any]: + """Flush the cache of the server asynchronously. + + Similar to the sync version, this method retries until the cache + is successfully flushed. It uses async sleep between retries. + + Returns: + Dict[str, Any]: Server response indicating cache flush status. + For non-master nodes, returns empty dict. + + Note: + Uses retry logic with limited attempts (max_attempts * 4) to avoid infinite loops. + Each retry includes an async delay to allow pending requests to complete. + """ + if self.node_rank != 0: + return {} + + # Use retry logic with limited attempts to avoid infinite loops + for attempt in range(self.max_attempts * 4): # Allow more retries for cache flush + try: + async with self._get_session() as session: + url = f"http://{self.server_args.host}:{self.server_args.port}/flush_cache" + async with session.get(url) as response: + if response.status == 200: + return await _read_async_response(response) + except Exception as e: + logger.warning(f"Error flushing cache (attempt {attempt + 1}): {e}") + + await asyncio.sleep(self.retry_delay) + + logger.error("Failed to flush cache after maximum attempts") + return {} + + async def generate( + self, + prompt: Optional[str] = None, + sampling_params: Optional[dict[str, Any]] = None, + input_ids: Optional[list[int]] = None, + image_data: Optional[Any] = None, + return_logprob: bool = False, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + token_ids_logprob: Optional[list[int]] = None, + lora_path: Optional[str] = None, + custom_logit_processor: Optional[Callable] = None, + ) -> dict[str, Any]: + """Generate text using the SGLang server asynchronously.""" + logger.info("generate() started") + + payload = { + "text": prompt, + "sampling_params": sampling_params, + "input_ids": input_ids, + "image_data": image_data, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + "token_ids_logprob": token_ids_logprob, + "lora_path": lora_path, + "custom_logit_processor": custom_logit_processor, + } + + # Filter out None values + payload = {k: v for k, v in payload.items() if v is not None} + + # Send request + response = await self._make_async_request("generate", payload, timeout=self.timeout, only_master=False) + + return response + + async def async_generate( + self, + prompt: Optional[str] = None, + sampling_params: Optional[dict[str, Any]] = None, + input_ids: Optional[list[int]] = None, + image_data: Optional[Any] = None, + return_logprob: bool = False, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + token_ids_logprob: Optional[list[int]] = None, + lora_path: Optional[str] = None, + custom_logit_processor: Optional[Callable] = None, + ) -> dict[str, Any]: + """Async generate method that mirrors AsyncEngine.async_generate interface. + + This method provides compatibility with AsyncEngine's async_generate method + by forwarding the call to the generate method. It ensures API consistency + between direct engine usage and HTTP-based engine usage. + + Args: + prompt (Optional[str], optional): Text prompt for generation. Defaults to None. + sampling_params (Optional[Dict[str, Any]], optional): Parameters controlling + text generation sampling. Defaults to None. + input_ids (Optional[List[int]], optional): Alternative to prompt, direct token IDs input. + Defaults to None. + image_data (Optional[Any], optional): Image data for multimodal generation. + Defaults to None. + return_logprob (bool, optional): Whether to return log probabilities. + Defaults to False. + logprob_start_len (Optional[int], optional): Starting length for log probability calculation. + Defaults to None. + top_logprobs_num (Optional[int], optional): Number of top log probabilities to return. + Defaults to None. + token_ids_logprob (Optional[List[int]], optional): Specific token IDs for + log probability calculation. Defaults to None. + lora_path (Optional[str], optional): Path to LoRA adapter weights. Defaults to None. + custom_logit_processor (Optional[Callable], optional): Custom logit processing function. + Defaults to None. + + Returns: + Dict[str, Any]: Generated text and associated metadata from the server + + Note: + This method is provided for API compatibility with AsyncEngine. + It forwards all calls to the generate method. + """ + return await self.generate( + prompt=prompt, + sampling_params=sampling_params, + input_ids=input_ids, + image_data=image_data, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + token_ids_logprob=token_ids_logprob, + lora_path=lora_path, + custom_logit_processor=custom_logit_processor, + ) + + async def reward_score( + self, + prompt: Optional[str] = None, + input_ids: Optional[list[int]] = None, + image_data: Optional[Any] = None, + lora_path: Optional[str] = None, + ) -> dict[str, Any]: + logger.info("reward_score() started") + payload = { + "text": prompt, + "input_ids": input_ids, + "image_data": image_data, + "lora_path": lora_path, + } + # Filter out None values + payload = {k: v for k, v in payload.items() if v is not None} + + # Send request + response = await self._make_async_request("classify", payload, timeout=self.timeout, only_master=False) + + return response + + async def async_reward_score( + self, + prompt: Optional[str] = None, + input_ids: Optional[list[int]] = None, + image_data: Optional[Any] = None, + lora_path: Optional[str] = None, + ) -> dict[str, Any]: + return await self.reward_score( + prompt=prompt, + input_ids=input_ids, + image_data=image_data, + lora_path=lora_path, + ) + + async def abort_request(self, rid: str = "", abort_all: bool = False) -> dict[str, Any]: + """Abort a request asynchronously. + + Args: + rid (str): The ID of the request to abort + abort_all (bool, optional): Whether to abort all requests. Defaults to False. + + Returns: + Dict[str, Any]: Server response indicating abort status + """ + return await self._make_async_request("abort_request", {"rid": rid, "abort_all": abort_all}) diff --git a/e2c/verl/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/e2c/verl/verl/workers/rollout/sglang_rollout/sglang_rollout.py new file mode 100644 index 0000000000000000000000000000000000000000..1ccafc5fdf35e9f8e5e2b98e4d3664d3480fc50e --- /dev/null +++ b/e2c/verl/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -0,0 +1,1609 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import asyncio +import logging +import multiprocessing as mp +import os +from copy import deepcopy +from json import JSONDecodeError +from typing import Any, Generator, Optional +from uuid import uuid4 + +import numpy as np +import ray +import sglang.srt.entrypoints.engine +import torch +import torch.distributed as dist +from sglang.srt.managers.io_struct import ( + ReleaseMemoryOccupationReqInput, + ResumeMemoryOccupationReqInput, + UpdateWeightsFromTensorReqInput, +) +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import ( + assert_pkg_version, + get_ip, + get_open_port, + is_cuda, + set_prometheus_multiproc_dir, + set_ulimit, +) +from sglang.srt.weight_sync.utils import update_weights as sgl_update_weights +from tensordict import TensorDict +from torch.distributed.device_mesh import DeviceMesh, init_device_mesh +from torch.nn.utils.rnn import pad_sequence +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, ProcessorMixin + +from verl import DataProto +from verl.interactions.base import BaseInteraction +from verl.interactions.utils.interaction_registry import initialize_interactions_from_config +from verl.third_party.sglang import parallel_state as sglang_ps +from verl.tools.base_tool import BaseTool +from verl.tools.schemas import OpenAIFunctionCallSchema, OpenAIFunctionParsedSchema, OpenAIFunctionToolCall +from verl.tools.utils.tool_registry import initialize_tools_from_config +from verl.utils.net_utils import is_ipv6 +from verl.utils.profiler import GPUMemoryLogger +from verl.utils.torch_functional import get_response_mask, pad_sequence_to_length +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.base import BaseRollout +from verl.workers.rollout.schemas import ( + AsyncRolloutRequest, + AsyncRolloutRequestStateEnum, + FinishReasonTypeEnum, +) +from verl.workers.rollout.sglang_rollout.http_server_engine import AsyncHttpServerAdapter +from verl.workers.rollout.sglang_rollout.utils import broadcast_pyobj, get_named_tensor_buckets +from verl.workers.rollout.utils import is_valid_ipv6_address + +try: + from sglang.srt.function_call.function_call_parser import FunctionCallParser +except ImportError: + from sglang.srt.function_call_parser import FunctionCallParser + +try: + from sglang.srt.entrypoints.openai.protocol import Tool +except ImportError: + from sglang.srt.openai_api.protocol import Tool + + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +# patch to avoid issue https://github.com/sgl-project/sglang/issues/6723 +def _set_envs_and_config(server_args: ServerArgs): + # Set global environments + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["NCCL_NVLS_ENABLE"] = str(int(server_args.enable_nccl_nvls)) + os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" + os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "4" + os.environ["CUDA_MODULE_LOADING"] = "AUTO" + + # Set prometheus env vars + if server_args.enable_metrics: + set_prometheus_multiproc_dir() + + # Set ulimit + set_ulimit() + + # Check flashinfer version + if server_args.attention_backend == "flashinfer": + assert_pkg_version( + "flashinfer_python", + "0.2.5", + "Please uninstall the old version and reinstall the latest version by following the instructions at https://docs.flashinfer.ai/installation.html.", + ) + if is_cuda(): + assert_pkg_version( + "sgl-kernel", + "0.1.1", + "Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`", + ) + + # Set mp start method + mp.set_start_method("spawn", force=True) + + +sglang.srt.entrypoints.engine._set_envs_and_config = _set_envs_and_config + + +# because chatCompletion is an async method, it makes the whole ray actor be an async actor +# which can not call loop.run_until_complete. So we need to make the engine to be an async class +class AsyncEngine(sglang.srt.entrypoints.engine.Engine): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + async def release_memory_occupation(self, tags: Optional[list[str]] = None): + """Release GPU occupation temporarily.""" + if tags is None: + obj = ReleaseMemoryOccupationReqInput() + else: + obj = ReleaseMemoryOccupationReqInput(tags=tags) + return await self.tokenizer_manager.release_memory_occupation(obj, None) + + async def resume_memory_occupation(self, tags: Optional[list[str]] = None): + """Resume GPU occupation.""" + if tags is None: + obj = ResumeMemoryOccupationReqInput() + else: + obj = ResumeMemoryOccupationReqInput(tags=tags) + return await self.tokenizer_manager.resume_memory_occupation(obj, None) + + async def update_weights_from_tensor(self, update_weights_request: UpdateWeightsFromTensorReqInput): + return await self.tokenizer_manager.update_weights_from_tensor(update_weights_request, None) + + async def flush_cache(self): + return await self.tokenizer_manager.flush_cache() + + async def abort_request(self, rid: str = "", abort_all: bool = False): + """Abort a specific request or all requests. + + Args: + rid: The request ID to abort. If empty and abort_all is False, no action is taken. + abort_all: If True, abort all running requests regardless of rid. + """ + return self.tokenizer_manager.abort_request(rid=rid, abort_all=abort_all) + + +# NOTE(sgm): add for verl. We can optimize it by making +# the dataloader yield List[int] without padding. +def _pre_process_inputs( + pad_token_id, + prompt_token_ids: torch.Tensor, +) -> torch.Tensor: + # remove the left padding in the prompt token_id + non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] + return prompt_token_ids[non_pad_index:] + + +def _extract_logprob_from_output(output): + """ + extract log_prob from single sglang inference output + """ + + def _map_each_response(resp): + input_token_logprobs = resp["meta_info"]["input_token_logprobs"] + log_probs, output_token_ids = zip( + *[(log_prob, token_ids) for log_prob, token_ids, _ in input_token_logprobs[1:]], strict=False + ) + return torch.tensor(output_token_ids), torch.tensor(log_probs) + + output_token_ids, log_probs = _map_each_response(output) + return output_token_ids, log_probs + + +# NOTE(linjunrong): adhoc +def _post_process_outputs(processing_class, output): + try: + # This is when processing_class is a processor + tokenizer = processing_class.tokenizer + except AttributeError: + try: + # This is when processing_class is a tokenizer + tokenizer = processing_class + except AttributeError as e: + raise ValueError(f"Cannot get tokenizer from processing_class {processing_class}") from e + + def _map_each_response(resp): + output_token_logprobs = resp["meta_info"]["output_token_logprobs"] + log_probs, output_token_ids = zip( + *[(log_prob, token_ids) for log_prob, token_ids, _ in output_token_logprobs], strict=True + ) + return torch.tensor(output_token_ids), torch.tensor(log_probs) + + out_map = map(lambda x: _map_each_response(x), output) + batched_output_token_ids = [] + batched_logprobs = [] + for output_token_ids, log_probs in out_map: + batched_output_token_ids.append(output_token_ids) + batched_logprobs.append(log_probs) + pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id + batched_output_token_ids = pad_sequence(batched_output_token_ids, batch_first=True, padding_value=pad_token_id) + if len(batched_logprobs) > 0: + batched_logprobs = pad_sequence(batched_logprobs, batch_first=True, padding_value=pad_token_id) + return batched_output_token_ids, batched_logprobs + + +def get_tool_call_parser_type( + processing_class: PreTrainedTokenizer | PreTrainedTokenizerFast | ProcessorMixin, +) -> str: + items = FunctionCallParser.ToolCallParserEnum.items() + for parser_type, parser_cls in items: + parser = parser_cls() + try: + # This is when processing_class is a tokenizer + tokenizer_vocab = processing_class.get_vocab() + except AttributeError: + try: + # This is when processing_class is a processor + tokenizer_vocab = processing_class.tokenizer.get_vocab() + except AttributeError as e: + raise ValueError(f"Cannot get vocab from processing_class {processing_class}") from e + + if parser.bot_token.strip() in tokenizer_vocab and ( + parser.eot_token == "" or parser.eot_token.strip() in tokenizer_vocab + ): + return parser_type + else: + raise ValueError(f"No tool call parser found for processing_class {processing_class}") + + +class SGLangRollout(BaseRollout): + def __init__( + self, + config: RolloutConfig, + model_config: HFModelConfig, + device_mesh: DeviceMesh, + ): + super().__init__(config, model_config, device_mesh) + + actor_module = model_config.local_path + processing_class = model_config.get_processor() + model_hf_config = model_config.hf_config + trust_remote_code = model_config.trust_remote_code + port = None + kwargs = {} + + os.environ.setdefault("SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK", "true") + + ( + self._tool_schemas, + self._tool_map, + self._tool_call_parser_type, + self._sgl_tools, + self._function_call_parser, + ) = self._initialize_tools(config, processing_class) + self.interaction_map: dict[str, BaseInteraction] = self._initialize_interactions(config) + + # If turn on `free_cache_engine`, SGLang engine's KV cache + # will be freed after each `generate_sequences` call. + logger.info( + f"tool_schemas: {self._tool_schemas}, tool_map: {self._tool_map}, tool_call_parser_type: " + f"{self._tool_call_parser_type}, sgl_tools: {self._sgl_tools}, function_call_parser: " + f"{self._function_call_parser}" + ) + + self._init_distributed_env(device_mesh_cpu=None, **kwargs) + + self._verify_config(model_hf_config=model_hf_config) + # initialize the inference engine + self._init_inference_engine(trust_remote_code, actor_module, port) + + self._init_sampling_params(**kwargs) + + self.processing_class = processing_class + try: + # This is when processing_class is a tokenizer + self.pad_token_id = self.processing_class.pad_token_id + except AttributeError: + try: + # This is when processing_class is a processor + self.pad_token_id = self.processing_class.tokenizer.pad_token_id + except AttributeError as e: + raise ValueError(f"Cannot get pad_token_id from processing_class {self.processing_class}") from e + + def _init_distributed_env(self, device_mesh_cpu, **kwargs): + self._device_mesh_cpu = device_mesh_cpu + os.environ.setdefault("SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK", "true") + self.tensor_parallel_size = self.config.get("tensor_model_parallel_size", 1) + assert self.tensor_parallel_size <= dist.get_world_size(), ( + "tensor parallel size should be less than or equal to the world size" + ) + self.train_tp = kwargs.get("train_tp", None) + if self.train_tp is not None: + # deployed with megatron + os.environ["CUDA_TIMER_STREAM_KAFKA_ENABLE"] = "0" + os.environ["MEGATRON_IMPORT_TIMERS"] = "0" + train_tp = kwargs.get("train_tp", None) + num_tp_per_train_tp = train_tp // self.tensor_parallel_size + sglang_ps.initialize_parallel_state( + tensor_model_parallel_size=self.tensor_parallel_size, + num_tp_per_train_tp=num_tp_per_train_tp, + ) + + tp_size = self.tensor_parallel_size + world_size = int(os.getenv("WORLD_SIZE", "-1")) + + # init device mesh + if self._device_mesh_cpu is None: + device_mesh_kwargs = dict( + mesh_shape=(world_size // tp_size, tp_size, 1), + mesh_dim_names=["dp", "tp", "pp"], + ) + + self._device_mesh_cpu = init_device_mesh("cpu", **device_mesh_kwargs) + + self._rank = self._device_mesh_cpu.get_rank() + self._tp_rank = self._device_mesh_cpu["tp"].get_local_rank() + self._tp_size = self._device_mesh_cpu["tp"].size() + if self._rank == 0: + logger.info(f"_init_distributed_env: :tp_world: {self._tp_size}, global_world: {world_size}") + # get tp_rank of this process in this tp group + visible_devices = [None] * self._device_mesh_cpu.size(1) + + torch.distributed.all_gather_object( + visible_devices, os.environ["CUDA_VISIBLE_DEVICES"], self._device_mesh_cpu.get_group("tp") + ) + self.visible_devices_set = set(",".join(visible_devices).split(",")) + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(sorted(list(self.visible_devices_set))) + + def _verify_config(self, model_hf_config): + if not self.config.get("max_model_len", None): + self.config.max_model_len = self.config.prompt_length + self.config.response_length + assert ( + self.config.max_model_len >= self.config.prompt_length + self.config.response_length + ), f"""max_model_len should be greater than total sequence length (prompt_length + response_length): + {self.config.max_model_len} >= {self.config.prompt_length} + {self.config.response_length}""" + max_position_embeddings = None + if hasattr(model_hf_config, "max_position_embeddings"): + max_position_embeddings = model_hf_config.max_position_embeddings + elif hasattr(model_hf_config, "llm_config") and hasattr(model_hf_config.llm_config, "max_position_embeddings"): + max_position_embeddings = model_hf_config.llm_config.max_position_embeddings + elif hasattr(model_hf_config, "text_config") and hasattr( + model_hf_config.text_config, "max_position_embeddings" + ): + max_position_embeddings = model_hf_config.text_config.max_position_embeddings + if max_position_embeddings is None: + raise ValueError("max_position_embeddings not found in model_hf_config") + rope_scaling_config = getattr(model_hf_config, "rope_scaling", None) + if not rope_scaling_config: + assert max_position_embeddings >= self.config.prompt_length + self.config.response_length, ( + "model context length should be greater than total sequence length" + ) + else: + # handle type where there's a length extend factor + # see https://qwen.readthedocs.io/en/latest/deployment/vllm.html#extended-context-support + # for using yarn as an example + rope_scaling_factor = rope_scaling_config.get("factor", 1.0) + + assert ( + model_hf_config.max_position_embeddings * rope_scaling_factor + >= self.config.prompt_length + self.config.response_length + ), ( + f"model context length should be greater than total sequence length, " + f"got rope_scaling_factor={rope_scaling_factor} and " + f"max_position_embeddings={model_hf_config.max_position_embeddings}" + ) + + # currently max_assistant_turns stand for max number of tool calls + if self.config.multi_turn.max_assistant_turns is None: + self.config.multi_turn.max_assistant_turns = self.config.max_model_len // 3 + if self.config.multi_turn.max_user_turns is None: + self.config.multi_turn.max_user_turns = self.config.max_model_len // 3 + + def _init_inference_engine(self, trust_remote_code, actor_module, port): + # initialize the inference engine + nnodes = -(-self._tp_size // len(self.visible_devices_set)) + if nnodes > 1: + ip = get_ip() + port = get_open_port() if port is None else port + [ip, port] = broadcast_pyobj( + [ip, port], + rank=self._rank, + dist_group=self._device_mesh_cpu.get_group("tp"), + src=self._device_mesh_cpu["tp"].mesh[0].item(), + force_cpu_device=False, + ) + dist_init_addr = f"[{ip}]:{port}" if is_ipv6(ip) else f"{ip}:{port}" + else: + dist_init_addr = None + + load_format = "dummy" if self.config.load_format.startswith("dummy") else self.config.load_format + tp_size_per_node = self._tp_size // nnodes + node_rank = self._tp_rank // tp_size_per_node + first_rank_in_node = self._tp_rank % tp_size_per_node == 0 + engine_kwargs = self.config.get("engine_kwargs", {}).get("sglang", {}) or {} + engine_kwargs = {key: val for key, val in engine_kwargs.items() if val is not None} + + # attention backend will be changed to fa3 if not specified + attention_backend = engine_kwargs.pop("attention_backend", None) + max_running_requests = self.config.get("max_num_seqs", None) + + try: + is_server_mode = self.config.sglang_rollout_mode == "server" + except Exception: + is_server_mode = False + effective_first = first_rank_in_node or is_server_mode + + if self.config.mode == "async" and not self.config.skip_tokenizer_init: + raise ValueError("async mode requires skip_tokenizer_init to be True") + + if effective_first: + rank = dist.get_rank() + os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" + args = { + "model_path": actor_module, + "dtype": self.config.dtype, + "mem_fraction_static": self.config.gpu_memory_utilization, + "enable_memory_saver": True, + "base_gpu_id": 0, + "gpu_id_step": 1, + "tp_size": self._tp_size, + "node_rank": node_rank, + "load_format": load_format, + "dist_init_addr": dist_init_addr, + "nnodes": nnodes, + "trust_remote_code": trust_remote_code, + "max_running_requests": max_running_requests, + # NOTE(linjunrong): add rank to prevent SGLang generate same port inside PortArgs.init_new + # when random.seed is being set during training + "port": 30000 + rank, + # NOTE(Chenyang): if you want to debug the SGLang engine output + # please set the following parameters + # Otherwise, it will make the engine run too slow + "log_level": "info", + # "log_level": "error", + # log_requests=True, + # log_requests_level=2, + # NOTE(Chenyang): turn on max_running_requests to set the max concurrent running requests + # max_running_requests=1, + "mm_attention_backend": "fa3", + "attention_backend": attention_backend if attention_backend is not None else "fa3", + # In async mode, we want token in token out. + "skip_tokenizer_init": self.config.skip_tokenizer_init, + } + + if is_server_mode: + # add server specific args + args["first_rank_in_node"] = first_rank_in_node + args["timeout"] = self.config.server["timeout"] + args["max_attempts"] = self.config.server["max_attempts"] + args["retry_delay"] = self.config.server["retry_delay"] + args["max_connections"] = self.config.server["max_connections"] + args["max_start_wait_time"] = self.config.server["max_start_wait_time"] + self._engine = AsyncHttpServerAdapter(**args) + else: + self._engine = AsyncEngine(**args) + else: + self._engine = None + + self.sharding_manager = None + self.is_sleep = True + + def _init_sampling_params(self, **kwargs): + kwargs = dict( + n=1, + max_new_tokens=self.config.response_length, + presence_penalty=0.0, + frequency_penalty=0.0, + repetition_penalty=self.config.get("repetition_penalty", 1.0), + ) + # supporting adding any sampling params from the config file + for k in self.config.keys(): + if hasattr(SamplingParams(), str(k)) or "stop" in str(k): + kwargs[k] = self.config.get(k) + kwargs["n"] = 1 # already repeat in ray_trainer + self.sampling_params = kwargs + + def _initialize_tools(self, config, processing_class): + """Initialize tools from configuration. + + Args: + config: Configuration object containing tool-related settings, + specifically `config.multi_turn.tool_config_path`. + tokenizer: The tokenizer instance used for parsing tool calls from + the model's generated text. + + Returns: + tuple: A tuple containing: + - tool_schemas (list[dict]): OpenAI-formatted JSON schemas + defining each tool's capabilities. + - tool_map (dict[str, BaseTool]): A dictionary mapping tool + names to their executable `BaseTool` objects. + - tool_call_parser_type (str): The identifier for the specific + parser type (e.g., 'json_mode', 'tool_code') used to extract + tool calls. + - sgl_tools (list[sglang.srt.openai_api.protocol.Tool]): Tool + definitions optimized for SGLang's internal engine. + - function_call_parser (sglang.srt.function_call_parser.FunctionCallParser): + The active parser instance responsible for extracting + structured tool calls from model outputs. + """ + if config.multi_turn.tool_config_path is None: + return [], {}, None, [], None + + tools_config_file = config.multi_turn.tool_config_path + tool_list = initialize_tools_from_config(tools_config_file) + + logger.info(f"Initialize tools from configuration.: tool_list: {tool_list}") + tool_schemas = [tool.get_openai_tool_schema().model_dump() for tool in tool_list] + tool_map = {tool.name: tool for tool in tool_list} + tool_call_parser_type = get_tool_call_parser_type(processing_class) + sgl_tools = [Tool.model_validate(tool_schema) for tool_schema in tool_schemas] + function_call_parser = FunctionCallParser( + sgl_tools, + tool_call_parser_type, + ) + + return ( + tool_schemas, + tool_map, + tool_call_parser_type, + sgl_tools, + function_call_parser, + ) + + def _initialize_interactions(self, config): + """Initialize interactions from configuration. + + Returns: + dict[str, BaseInteraction]: A dictionary mapping interaction names to interaction instances. + """ + if config.multi_turn.interaction_config_path is None: + return {} + + interaction_config_file = config.multi_turn.interaction_config_path + interaction_map = initialize_interactions_from_config(interaction_config_file) + + logger.info(f"Initialize interactions from configuration: interaction_map: {list(interaction_map.keys())}") + return interaction_map + + @GPUMemoryLogger(role="sglang rollout", logger=logger) + @torch.no_grad() + def generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: + """Generate sequences for a batch of prompts. + + Args: + batch (DataProto): Input batch. + + Returns: + DataProto: Output batch. + - prompts: [bsz, prompt_length], prompt token ids from dataset. + - responses: [bsz, response_length], output token ids include response tokens + from LLM generation and observation tokens from tool_calls. + - response_mask: [bsz, response_length], 1 for LLM generated tokens, 0 for observation/padding tokens. + - input_ids: [bsz, prompt_length + response_length], whole sequence token ids, including prompt tokens + and response tokens. + - attention_mask: [bsz, prompt_length + response_length], 0 for padding tokens, 1 for other tokens. + - position_ids: [bsz, prompt_length + response_length], incremental position ids. + + For multi-turn conversations: + responses: |<- LLM generation ->|<- tool_calls ->|<- LLM generation ->|<- padding ->| + response_mask: | 1, 1, 1, ..., 1, 1 | 0, 0, .., 0, 0 | 1, 1, 1, ..., 1, 1 | 0, 0, ..., 0| + """ + if self.config.multi_turn.enable: + return self._req_level_generate_sequences(prompts, **kwargs) + return self._batch_level_generate_sequences(prompts, **kwargs) + + @GPUMemoryLogger(role="sglang rollout", logger=logger) + @torch.no_grad() + def _batch_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: + """Generates single-turn sequences for a batch of prompts. + For single-turn generation, all prompts are processed in one request. + `_batch_level_generate_sequences` involves: + 1. Extracting and pre-processing prompt token IDs from the input + `prompts`. This includes handling padding and preparing raw + token ID lists. + 2. Preparing inputs for the SGLang engine, including multi-modal + data if present. + 3. Invoking the SGLang engine (`self._engine.async_generate`, + an async coroutine) with the batch of processed inputs and + specified sampling parameters on the master TP rank. + 4. Broadcasting the results from the master TP rank to all + other TP ranks. + 5. Post-processing the engine's output to format the generated + token IDs and (if applicable) log probabilities. + 6. Constructing the final sequences by concatenating original + prompts with the generated responses. + 7. Updating attention masks and position IDs to reflect the full + concatenated sequences. + 8. If `self.config.free_cache_engine` is true, the SGLang engine's + KV cache is flushed after generation on the master TP rank. + Args: + prompts: A `DataProto` object containing the batch of + input prompts, including tensor data (like `input_ids`, + `attention_mask`) and meta-information (like `eos_token_id`, + `do_sample`). + **kwargs: Additional keyword arguments that can override the + default sampling parameters (e.g., `temperature`, `top_p`, + `max_new_tokens`). These are temporarily applied using + `update_sampling_params`. + Returns: + DataProto: A `DataProto` object containing the batch of + generated sequences. This includes tensors for `prompts` + (original input IDs), `responses` (generated token IDs), + `input_ids` (concatenated prompt and response), + `attention_mask`, and `position_ids` for the full + sequences. + Note that in GRPO, if the prompts are validated, we repeat the prompts for rollout.n times in ray_trainer. + Thus we do not need to repeat the prompts here and set the sampling parameter n to 1. + """ + # input ids: (bs, prompt_length), left-padded + idx = prompts.batch["input_ids"] + # attention_mask: (bs, seq_length), left-padded + attention_mask = prompts.batch["attention_mask"] + position_ids = prompts.batch["position_ids"] + + # used to generate attention mask for the + # response based on EOS token position + eos_token_id = prompts.meta_info["eos_token_id"] + + batch_size = idx.size(0) + + # Extract non-tensor data + non_tensor_batch = prompts.non_tensor_batch + if "raw_prompt_ids" not in non_tensor_batch: + non_tensor_batch["raw_prompt_ids"] = np.array( + [_pre_process_inputs(self.pad_token_id, idx[i]).tolist() for i in range(batch_size)], + dtype=object, + ) + + if "multi_modal_data" in non_tensor_batch: + sglang_inputs = [] + for raw_prompt_ids, multi_modal_data in zip( + non_tensor_batch.pop("raw_prompt_ids"), + non_tensor_batch.pop("multi_modal_data"), + strict=True, + ): + sglang_inputs.append( + { + "prompt_token_ids": raw_prompt_ids, + "multi_modal_data": multi_modal_data, + "image_data": ( + multi_modal_data.get("image", None) if isinstance(multi_modal_data, dict) else None + ), + } + ) + else: + sglang_inputs = [ + {"prompt_token_ids": raw_prompt_ids} for raw_prompt_ids in non_tensor_batch.pop("raw_prompt_ids") + ] + + for input_data in sglang_inputs: + # Ensure token IDs are lists or numpy arrays + if not isinstance(input_data["prompt_token_ids"], list | np.ndarray): + raise TypeError( + f"prompt_token_ids must be a list or numpy array, got {type(input_data['prompt_token_ids'])}" + ) + + input_data["prompt_token_ids"] = list(input_data["prompt_token_ids"]) + + # Extract token IDs and image data for SGLang Engine + idx_list = [input_data["prompt_token_ids"] for input_data in sglang_inputs] + image_list = [input_data.get("image_data", None) for input_data in sglang_inputs] + + do_sample = prompts.meta_info.get("do_sample", True) + is_validate = prompts.meta_info.get("validate", False) + + # Create request-level sampling parameters + request_sampling_params = self.sampling_params.copy() + if not do_sample: + request_sampling_params.update( + { + "n": 1, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "repetition_penalty": 1.0, + "temperature": 0, + "top_p": 1, + "top_k": -1, + "ignore_eos": False, + "min_new_tokens": 0, + "max_new_tokens": self.config.response_length, + "skip_special_tokens": True, + "spaces_between_special_tokens": True, + } + ) + elif is_validate: + request_sampling_params.update( + { + "top_k": self.config.val_kwargs.top_k, + "top_p": self.config.val_kwargs.top_p, + "temperature": self.config.val_kwargs.temperature, + "n": 1, # if validate, already repeat in ray_trainer + } + ) + + # Update with any additional kwargs + request_sampling_params.update(kwargs) + + if self._tp_rank == 0: + loop = asyncio.get_event_loop() + output = loop.run_until_complete( + self._engine.async_generate( + prompt=None, # because we have already convert it to prompt token id + sampling_params=request_sampling_params, + return_logprob=True, + input_ids=idx_list, + image_data=image_list, + ) + ) + else: + output = None + + # Most naive implementation, can extract tensor and send via gloo if too slow + dist.barrier() + [output] = broadcast_pyobj( + data=[output], + rank=self._rank, + dist_group=self._device_mesh_cpu["tp"].get_group(), + src=self._device_mesh_cpu["tp"].mesh[0].item(), + force_cpu_device=False, + ) + out = _post_process_outputs(self.processing_class, output) + + response = out[0].to(idx.device) + rollout_log_probs = None + if self.config.calculate_log_probs: + rollout_log_probs = out[1].to(idx.device) + + if response.shape[1] < self.config.response_length: + response = pad_sequence_to_length(response, self.config.response_length, self.pad_token_id) + if self.config.calculate_log_probs: + rollout_log_probs = pad_sequence_to_length( + rollout_log_probs, self.config.response_length, self.pad_token_id + ) + + seq = torch.cat([idx, response], dim=-1) + + response_length = response.size(1) + delta_position_id = torch.arange(1, response_length + 1, device=position_ids.device) + delta_position_id = delta_position_id.unsqueeze(0).repeat(batch_size, 1) + if position_ids.dim() == 3: # qwen2vl mrope (batch size, 4, seq len) + delta_position_id = delta_position_id.view(batch_size, 1, -1).expand(batch_size, position_ids.size(1), -1) + + # TODO(sgm): fix position_ids on right_pad + # prompt: left pad + response: right pad + # attention_mask: [0,0,0,0,1,1,1,1, | 1,1,1,0,0,0,0,0] + # position_ids: [0,0,0,0,0,1,2,3, | 4,5,6,7,8,9,10,11] + response_position_ids = position_ids[..., -1:] + delta_position_id + position_ids = torch.cat([position_ids, response_position_ids], dim=-1) + response_attention_mask = get_response_mask( + response_id=response, eos_token=eos_token_id, dtype=attention_mask.dtype + ) + attention_mask = torch.cat((attention_mask, response_attention_mask), dim=-1) + + # all the tp ranks should contain the same data here. data in all ranks are valid + batch = TensorDict( + { + "prompts": idx, + "responses": response, + "input_ids": seq, # here input_ids become the whole sentences + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=batch_size, + ) + if self.config.calculate_log_probs: + # we will recompute old log prob with actor + batch["rollout_log_probs"] = rollout_log_probs + + # free cache engine + if self._engine is not None and self._tp_rank == 0: + loop = asyncio.get_event_loop() + loop.run_until_complete(self._engine.flush_cache()) + + return DataProto(batch=batch, non_tensor_batch=non_tensor_batch) + + async def _async_rollout_a_request( + self, + req: AsyncRolloutRequest, + do_sample: bool = True, + is_validate: bool = False, + **kwargs, + ) -> AsyncRolloutRequest: + assert self._tp_rank == 0, "only the master process can call this function" + _req = deepcopy(req) + finish_reason_type = None + output = None + + current_turns = 0 + user_turns = 0 + user_turn_rewards = [] + + # Create request-level sampling parameters + request_sampling_params = self.sampling_params.copy() + if not do_sample: + request_sampling_params.update( + { + "n": 1, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "repetition_penalty": 1.0, + "temperature": 0, + "top_p": 1, + "top_k": -1, + "ignore_eos": False, + "min_new_tokens": 0, + "max_new_tokens": self.config.response_length, + "skip_special_tokens": True, + "spaces_between_special_tokens": True, + } + ) + elif is_validate: + request_sampling_params.update( + { + "top_k": self.config.val_kwargs.top_k, + "top_p": self.config.val_kwargs.top_p, + "temperature": self.config.val_kwargs.temperature, + "n": 1, # if validate, already repeat in ray_trainer + } + ) + + # Update with any additional kwargs + request_sampling_params.update(kwargs) + + while current_turns < self.config.multi_turn.max_assistant_turns: + if _req.state == AsyncRolloutRequestStateEnum.PENDING: + await self._handle_pending_state(_req) + _req.state = AsyncRolloutRequestStateEnum.RUNNING + elif _req.state == AsyncRolloutRequestStateEnum.TOOL_CALLING: + if _req.messages[-1].tool_calls is not None: + parsed_tool_calls = _req.messages[-1].tool_calls + if self.config.skip_tokenizer_init: + _req.messages[-1].tool_calls = None + tool_call_results = await asyncio.gather( + *[ + self._tool_map[tool_call.function.name].execute( + _req.request_id, + tool_call.function.arguments, + **_req.tools_kwargs.get(tool_call.function.name, {}).get("execute_kwargs", {}), + ) + for tool_call in parsed_tool_calls + ] + ) + _req.add_tool_response_messages(self.processing_class, [resp for resp, _, _ in tool_call_results]) + for tool_call, (resp, reward, metrics) in zip(parsed_tool_calls, tool_call_results, strict=True): + _req.update_metrics(metrics, tool_call.function.name) + if len(_req.input_ids) >= self.config.max_model_len: + finish_reason_type = FinishReasonTypeEnum.STOP + break + _req.state = AsyncRolloutRequestStateEnum.RUNNING + else: + raise ValueError(f"Unexpected tool calling last message state: {_req.messages[-1]}") + elif _req.state == AsyncRolloutRequestStateEnum.RUNNING: + # Only continue the conversation if the prompt length is not greater than max_model_len - 1, + # since SGLang raises an error when max_new_tokens + 1 is greater to max_model_len (the extra + # token accounts for the EOS token). + prompt_length = len(_req.get_generation_prompt_ids(self.processing_class)) + + if prompt_length + 1 >= self.config.max_model_len: + finish_reason_type = FinishReasonTypeEnum.LENGTH + break + + # Video support is not implemented yet + image_data = ( + _req.multi_modal_data["image"] + if _req.multi_modal_data and "image" in _req.multi_modal_data + else None + ) + video_data = ( + _req.multi_modal_data["video"] + if _req.multi_modal_data and "video" in _req.multi_modal_data + else None + ) + if video_data: + logger.warning( + "video support is not implemented yet, current length of video data is %d", len(video_data) + ) + + output = await self._handle_engine_call(_req, request_sampling_params, image_data=image_data) + if self.config.skip_tokenizer_init: + content_ids = output["output_ids"] + content = self.processing_class.decode(content_ids, skip_special_tokens=True) + content_ids = torch.tensor( + content_ids, dtype=_req.input_ids.dtype, device=_req.input_ids.device + ).unsqueeze(0) + else: + content_ids = None + content = output["text"] + + finish_reason_type = FinishReasonTypeEnum.from_str(output["meta_info"]["finish_reason"]["type"]) + current_turns += 1 + if finish_reason_type == FinishReasonTypeEnum.LENGTH: + _req.add_assistant_message(self.processing_class, content=content, content_ids=content_ids) + break + else: + if self._function_call_parser and self._function_call_parser.has_tool_call(content): + finish_reason_type = FinishReasonTypeEnum.TOOL_CALL + _req.state = AsyncRolloutRequestStateEnum.TOOL_CALLING + try: + normed_content, tool_calls = self._function_call_parser.parse_non_stream(content) + except JSONDecodeError: + normed_content = content + tool_calls = [] + except AttributeError: + normed_content = content + tool_calls = [] + parsed_tool_calls = [] + for tool_call in tool_calls: + function, has_decode_error = OpenAIFunctionCallSchema.from_openai_function_parsed_schema( + OpenAIFunctionParsedSchema( + name=tool_call.name, + arguments=tool_call.parameters, + ) + ) + # Drop the tool call if its arguments has decode error + if has_decode_error: + continue + parsed_tool_calls.append( + OpenAIFunctionToolCall( + id=str(tool_call.tool_index), + function=function, + ) + ) + if len(parsed_tool_calls) > 0: + _req.add_assistant_message( + # since the content is updated, we just pass the content not content_ids + self.processing_class, + content=normed_content, + tool_calls=parsed_tool_calls, + ) + else: + _req.add_assistant_message(self.processing_class, content=content, content_ids=content_ids) + finish_reason_type = FinishReasonTypeEnum.STOP + _req.state = AsyncRolloutRequestStateEnum.COMPLETED + break + else: + _req.add_assistant_message( + self.processing_class, + content=content, + content_ids=content_ids, + ) + if ( + _req.interaction_kwargs + and self.interaction_map + and user_turns < self.config.multi_turn.max_user_turns + and current_turns < self.config.multi_turn.max_assistant_turns + ): + _req.state = AsyncRolloutRequestStateEnum.INTERACTING + else: + # Add ending condition + finish_reason_type = FinishReasonTypeEnum.STOP + _req.state = AsyncRolloutRequestStateEnum.COMPLETED + break + elif _req.state == AsyncRolloutRequestStateEnum.INTERACTING: + user_turns += 1 + messages = [{"role": x.role, "content": x.content} for x in _req.messages] + + # Get interaction by name from interaction_kwargs + interaction_name = _req.interaction_kwargs.get( + "name", "gsm8k" + ) # Default to gsm8k for backward compatibility + if interaction_name not in self.interaction_map: + raise ValueError( + f"Interaction '{interaction_name}' not found in interaction_map. Available interactions: " + f"{list(self.interaction_map.keys())}" + ) + + interaction = self.interaction_map[interaction_name] + + should_terminate_sequence, content, reward, metrics = await interaction.generate_response( + _req.request_id, messages, **_req.interaction_kwargs + ) + user_turn_rewards.append(reward) + # Add turn check + if ( + should_terminate_sequence + or user_turns > self.config.multi_turn.max_user_turns + or current_turns > self.config.multi_turn.max_assistant_turns + ): + finish_reason_type = FinishReasonTypeEnum.STOP + _req.state = AsyncRolloutRequestStateEnum.COMPLETED + break + else: + _req.add_user_message(self.processing_class, content) + if len(_req.input_ids) >= self.config.max_model_len: + finish_reason_type = FinishReasonTypeEnum.STOP + break + else: + _req.state = AsyncRolloutRequestStateEnum.RUNNING + + if current_turns >= self.config.multi_turn.max_assistant_turns: + finish_reason_type = FinishReasonTypeEnum.STOP + + # Calculate the reward for each tool + async def calc_reward_and_release_fn(name: str, tool: BaseTool): + reward = await tool.calc_reward(_req.request_id, **_req.tools_kwargs[name].get("calc_reward_kwargs", {})) + await tool.release(_req.request_id, **_req.tools_kwargs[name].get("release_kwargs", {})) + return name, reward + + tool_reward_tasks = [] + for name in _req.tools_kwargs.keys(): + tool = self._tool_map[name] + tool_reward_tasks.append(calc_reward_and_release_fn(name, tool)) + tool_reward_scores = await asyncio.gather(*tool_reward_tasks) + tool_reward_scores = dict(tool_reward_scores) + all_rewards = {**tool_reward_scores, **{"user_turn_rewards": user_turn_rewards}} + _req.finalize(self.processing_class, all_rewards, finish_reason_type) + + if self.config.calculate_log_probs: + debug_sampling_params = {**self.sampling_params} + debug_sampling_params["max_new_tokens"] = 0 + output = await self._engine.async_generate( + prompt=None, + input_ids=_req.input_ids, + sampling_params=debug_sampling_params, + return_logprob=True, + logprob_start_len=0, + ) + # len(input_token_logprobs) = len(input_tokens)-1,because logprob of 1st token is None + _req.output_token_ids, _req.rollout_log_probs = _extract_logprob_from_output(output) + return _req + + async def _handle_engine_call( + self, _req: AsyncRolloutRequest, sampling_params: dict, image_data: Optional[list[Any]] = None + ) -> dict: + generation_prompt_ids = _req.get_generation_prompt_ids(self.processing_class) + return await self._handle_engine_generate(generation_prompt_ids, sampling_params, image_data) + + async def _handle_engine_generate( + self, generation_prompt_ids: list[int], sampling_params: dict, image_data: Optional[list[Any]] = None + ) -> dict: + max_new_tokens = min(self.config.response_length, self.config.max_model_len - len(generation_prompt_ids) - 1) + + kwargs = sampling_params.copy() + kwargs["max_new_tokens"] = max_new_tokens + kwargs["n"] = 1 # group size is supported in preprocess + return_logprob = kwargs.pop("logprobs", False) + + output = await self._engine.async_generate( + input_ids=generation_prompt_ids, + sampling_params=kwargs, + return_logprob=return_logprob, + image_data=image_data, + ) + return output + + async def _handle_pending_state(self, _req: AsyncRolloutRequest) -> AsyncRolloutRequest: + if _req.tool_schemas is not None: + tool_creation_coroutines = [] + for tool_schema in _req.tool_schemas: + tool = self._tool_map[tool_schema.function.name] + create_kwargs = _req.tools_kwargs[tool.name].get("create_kwargs", {}) + tool_creation_coroutines.append(tool.create(_req.request_id, **create_kwargs)) + tool_creation_results = await asyncio.gather(*tool_creation_coroutines) + _req.add_tool_response_messages( + self.processing_class, [tool_result for _, tool_result in tool_creation_results] + ) + if _req.interaction_kwargs and self.interaction_map: + interaction_kwargs = _req.interaction_kwargs + # Get interaction by name from interaction_kwargs + interaction_name = interaction_kwargs.get("name", "gsm8k") # Default to gsm8k for backward compatibility + if interaction_name not in self.interaction_map: + raise ValueError( + f"Interaction '{interaction_name}' not found in interaction_map. Available interactions: " + f"{list(self.interaction_map.keys())}" + ) + + interaction = self.interaction_map[interaction_name] + await interaction.start_interaction(_req.request_id, **interaction_kwargs) + + @GPUMemoryLogger(role="sglang rollout", logger=logger) + @torch.no_grad() + def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: + """Generates multi-turn sequences for a batch of prompts. + For multi-turn generation, each prompt is processed separately via + `_req_level_generate_sequences` for better tool calling control. + Note that in multi-turn generation, we repeat the prompts for rollout.n times in ray_trainer. + Thus we do not need to repeat the prompts here and set the sampling parameter n to 1. + """ + # Async rollout with tools support + do_sample = prompts.meta_info.get("do_sample", True) + is_validate = prompts.meta_info.get("validate", False) + tgt_device = prompts.batch["input_ids"].device + + if self._tp_rank == 0: + req_list = self._preprocess_prompt_to_async_rollout_requests( + prompts, + ) + + # distinguish training and validation + if is_validate: + # Validation mode: process all requests without abort + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather( + *[self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) for req in req_list], + ) + ) + else: + # add progress monitoring and abort function + total_requests = len(req_list) + target_completion = int(total_requests * (1 - self.config.get("over_sample_rate", 0.0))) + # abort when target_completion of requests are completed + + completed_count = 0 + aborted_requests = [] + all_tasks = [] + + async def rollout_a_request_with_cancellation_handler(req): + try: + result = await self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) + return result + except asyncio.CancelledError: + # request is cancelled, return padding + logger.info(f"Request {req.request_id} was cancelled, creating padding") + aborted_requests.append(req.request_id) + return self._create_padding_request(req) + + async def run_with_cancellation(): + nonlocal all_tasks + nonlocal completed_count + all_tasks = [ + asyncio.create_task(rollout_a_request_with_cancellation_handler(req)) for req in req_list + ] + + # Wait for target_completion tasks to complete + try: + for completed_task in asyncio.as_completed(all_tasks): + await completed_task + completed_count += 1 + if completed_count >= target_completion: + break + finally: + # Cancel remaining tasks + for t in all_tasks: + if not t.done(): + t.cancel() + + # Wait for all tasks to finish (including cancelled ones) + final_results = await asyncio.gather(*all_tasks, return_exceptions=True) + # Abort all requests in SGLang engine + await self._engine.abort_request(abort_all=True) + return final_results + + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete(run_with_cancellation()) + + sorted_output_req_list = sorted(output_req_list, key=lambda x: (x.batch_data_id, x.rollout_offset)) + else: + sorted_output_req_list = None + + dist.barrier() + [sorted_output_req_list] = broadcast_pyobj( + data=[sorted_output_req_list], + rank=self._rank, + dist_group=self._device_mesh_cpu["tp"].get_group(), + src=self._device_mesh_cpu["tp"].mesh[0].item(), + force_cpu_device=False, + ) + # Construct the batch data + prompt_ids, response_ids = [], [] + prompt_attention_mask, response_attention_mask = [], [] + prompt_position_ids, response_position_ids = [], [] + response_loss_mask = [] + messages = [] + reward_scores = [] + multi_modal_inputs = [] + request_ids = [] + if self.config.calculate_log_probs: + output_logprobs = [] + rollout_output_token_ids = [] + + for req in sorted_output_req_list: + assert req.state == AsyncRolloutRequestStateEnum.COMPLETED, f"Request {req.request_id} is not completed" + assert ( + req.input_ids.shape[-1] + == req.attention_mask.shape[-1] + == req.position_ids.shape[-1] + == req.loss_mask.shape[-1] + ), f"""Request {req.request_id} has different length of + {req.input_ids.shape[-1]=}, {req.attention_mask.shape[-1]=}, + {req.position_ids.shape[-1]=}, {req.loss_mask.shape[-1]=}""" + error_message_lines = [ + f"""Request {req.request_id} has input_ids length {req.input_ids.shape[-1]} + greater than max_model_len {self.config.max_model_len}""", + f"Decoded input_ids: {self.processing_class.decode(req.input_ids.squeeze(0))}", + f"Decoded prompt_ids: {self.processing_class.decode(req.prompt_ids.squeeze(0))}", + f"Decoded response_ids: {self.processing_class.decode(req.response_ids.squeeze(0))}", + f"Messages: {req.messages}", + f"Max model length: {req.max_model_len}", + ] + error_message = "\n".join(error_message_lines) + assert req.input_ids.shape[-1] <= self.config.max_model_len, error_message + + prompt_ids.append(req.prompt_ids.to(tgt_device).squeeze(0)) + response_ids.append(req.response_ids.to(tgt_device).squeeze(0)) + if req.response_ids.shape[-1] > self.config.response_length: + logger.warning( + f"""{req.request_id=} has response_ids length {req.response_ids.shape[-1]} + greater than max_response_len {self.config.response_length},\n{req=}""" + ) + prompt_attention_mask.append(req.prompt_attention_mask.to(tgt_device).squeeze(0)) + response_attention_mask.append(req.response_attention_mask.to(tgt_device).squeeze(0)) + prompt_position_ids.append(req.prompt_position_ids.to(tgt_device).squeeze(0)) + response_position_ids.append(req.response_position_ids.to(tgt_device).squeeze(0)) + response_loss_mask.append(req.response_loss_mask.to(tgt_device).squeeze(0)) + messages.append({"messages": req.messages}) + reward_scores.append(req.reward_scores) + multi_modal_inputs.append(req.multi_modal_inputs) + request_ids.append(req.request_id) + if self.config.calculate_log_probs: + # extract output log_probs + output_logprobs.append(req.rollout_log_probs[-len(req.response_ids) :]) + rollout_output_token_ids.append(req.output_token_ids[-len(req.response_ids) :]) + + prompt_ids = pad_sequence( + prompt_ids, + batch_first=True, + padding_value=self.pad_token_id, + padding_side="left", + ) + if prompt_ids.shape[-1] < self.config.prompt_length: + prompt_ids = pad_sequence_to_length(prompt_ids, self.config.prompt_length, self.pad_token_id, left_pad=True) + response_ids = pad_sequence(response_ids, batch_first=True, padding_value=self.pad_token_id) + if response_ids.shape[-1] < self.config.response_length: + response_ids = pad_sequence_to_length(response_ids, self.config.response_length, self.pad_token_id) + prompt_attention_mask = pad_sequence( + prompt_attention_mask, + batch_first=True, + padding_value=0, + padding_side="left", + ) + if prompt_attention_mask.shape[-1] < self.config.prompt_length: + prompt_attention_mask = pad_sequence_to_length( + prompt_attention_mask, self.config.prompt_length, 0, left_pad=True + ) + response_attention_mask = pad_sequence(response_attention_mask, batch_first=True, padding_value=0) + if response_attention_mask.shape[-1] < self.config.response_length: + response_attention_mask = pad_sequence_to_length(response_attention_mask, self.config.response_length, 0) + + # padding prompt_position_ids + if prompt_position_ids[0].dim() == 2: + # if prompt_position_ids is a 2D tensor + # e.g. from qwen2vl, prompt_position_ids.shape = (3, seq_len) + transposed_prompt_position_ids = [p.transpose(0, 1) for p in prompt_position_ids] + prompt_position_ids = pad_sequence( + transposed_prompt_position_ids, batch_first=True, padding_value=0, padding_side="left" + ) + prompt_position_ids = prompt_position_ids.transpose(1, 2) + else: + prompt_position_ids = pad_sequence( + prompt_position_ids, batch_first=True, padding_value=0, padding_side="left" + ) + if prompt_position_ids.shape[-1] < self.config.prompt_length: + prompt_position_ids = pad_sequence_to_length( + prompt_position_ids, self.config.prompt_length, 0, left_pad=True + ) + + # padding response_position_ids + if response_position_ids[0].dim() == 2: + # if response_position_ids is a 2D tensor + # e.g. from qwen2vl, response_position_ids.shape = (3, seq_len) + transposed_response_position_ids = [p.transpose(0, 1) for p in response_position_ids] + response_position_ids = pad_sequence( + transposed_response_position_ids, batch_first=True, padding_value=0, padding_side="left" + ) + response_position_ids = response_position_ids.transpose(1, 2) + else: + response_position_ids = pad_sequence(response_position_ids, batch_first=True, padding_value=0) + if response_position_ids.shape[-1] < self.config.response_length: + response_position_ids = pad_sequence_to_length(response_position_ids, self.config.response_length, 0) + + response_loss_mask = pad_sequence(response_loss_mask, batch_first=True, padding_value=0) + if response_loss_mask.shape[1] < self.config.response_length: + response_loss_mask = pad_sequence_to_length(response_loss_mask, self.config.response_length, 0) + if self.config.calculate_log_probs: + output_logprobs = pad_sequence(output_logprobs, padding_value=0.0, batch_first=True) + output_logprobs = pad_sequence_to_length( + output_logprobs, pad_token_id=0.0, max_seq_len=response_ids.shape[-1] + ).to(tgt_device) + rollout_output_token_ids = pad_sequence( + rollout_output_token_ids, padding_value=self.pad_token_id, batch_first=True + ) + rollout_output_token_ids = pad_sequence_to_length( + rollout_output_token_ids, pad_token_id=self.pad_token_id, max_seq_len=response_ids.shape[-1] + ).to(tgt_device) + + input_ids = torch.cat((prompt_ids, response_ids), dim=-1) + attention_mask = torch.cat((prompt_attention_mask, response_attention_mask), dim=-1) + position_ids = torch.cat((prompt_position_ids, response_position_ids), dim=-1) + + # Construct the batch data + batch = TensorDict( + { + "prompts": prompt_ids, + "responses": response_ids, + "response_mask": response_loss_mask, + "input_ids": input_ids, # here input_ids become the whole sentences + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=len(sorted_output_req_list), + ) + if self.config.calculate_log_probs: + batch["rollout_log_probs"] = output_logprobs + batch["rollout_output_token_ids"] = rollout_output_token_ids + + # free cache engine + if self._engine is not None and self._tp_rank == 0: + loop = asyncio.get_event_loop() + loop.run_until_complete(self._engine.flush_cache()) + + non_tensor_batch = { + "messages": np.array(messages), + "reward_scores": np.array(reward_scores), + "request_id": np.array(request_ids), + } + + is_multimodal = isinstance(self.processing_class, ProcessorMixin) and ( + hasattr(self.processing_class, "image_processor") or hasattr(self.model_hf_config, "vision_config") + ) + + if is_multimodal: + non_tensor_batch["multi_modal_inputs"] = np.array(multi_modal_inputs, dtype=object) + + return DataProto( + batch=batch, + non_tensor_batch=non_tensor_batch, + ) + + def _create_padding_request(self, original_req: AsyncRolloutRequest) -> AsyncRolloutRequest: + # create a padding request to replace the aborted request + # the padding request has the following characteristics: + # 1. state is COMPLETED, but contains empty response + # 2. response_loss_mask is all 0, ensuring it is ignored in loss calculation + # 3. keep the original request structure, but the content is empty + # create padding response_ids (all pad_token_id) + padding_response_length = self.config.response_length + device = original_req.input_ids.device if original_req.input_ids is not None else "cpu" + padding_response_ids = torch.full( + (1, padding_response_length), + self.pad_token_id, + dtype=torch.long, + device=device, + ) + + # create padding attention_mask (all 0) + padding_response_attention_mask = torch.zeros( + (1, padding_response_length), + dtype=torch.long, + device=device, + ) + + # create padding position_ids + if original_req.position_ids is not None: + first_dim = 1 + # if position_ids is a 2D tensor (e.g. qwen2vl) + if original_req.position_ids.dim() == 2: + first_dim = original_req.position_ids.shape[0] + padding_response_position_ids = torch.zeros( + (first_dim, padding_response_length), + dtype=torch.long, + device=device, + ) + else: + padding_response_position_ids = None + + # create padding prompt_attention_mask (all 0) + padding_prompt_attention_mask = torch.zeros( + (1, original_req.prompt_attention_mask.shape[-1]), + dtype=torch.long, + device=device, + ) + + # create padding loss_mask (all 0, ensuring it is ignored) + padding_response_loss_mask = torch.zeros( + (1, padding_response_length), + dtype=torch.long, + device=device, + ) + + padding_req = original_req.model_copy(deep=True) + padding_req.state = AsyncRolloutRequestStateEnum.COMPLETED + padding_req.response_ids = padding_response_ids + padding_req.prompt_attention_mask = padding_prompt_attention_mask + padding_req.response_attention_mask = padding_response_attention_mask + padding_req.response_position_ids = padding_response_position_ids + padding_req.response_loss_mask = padding_response_loss_mask + padding_req.reward_scores = {} + padding_req.metrics = {} + padding_req.output_token_ids = None + padding_req.rollout_log_probs = None + return padding_req + + def _preprocess_prompt_to_async_rollout_requests(self, prompts: DataProto, n: int = 1) -> list[AsyncRolloutRequest]: + assert "raw_prompt" in prompts.non_tensor_batch, ( + "need data.return_raw_chat=True, due to no official way do parse_messages" + ) + logger.info( + "n is deprecated for SGLang rollout since ray ppo trainer will repeat the prompts for rollout.n times" + ) + req_list = [] + multi_modal_data_list = prompts.non_tensor_batch.get( + "multi_modal_data", [None] * len(prompts.non_tensor_batch["raw_prompt"]) + ) + + for data_idx, (raw_prompt, multi_modal_data) in enumerate( + zip(prompts.non_tensor_batch["raw_prompt"], multi_modal_data_list, strict=True) + ): + if self._tool_schemas: + _tools_kwargs = prompts.non_tensor_batch["tools_kwargs"][data_idx] + _tool_schemas = [self._tool_map[k].get_openai_tool_schema() for k in _tools_kwargs.keys()] + _input_ids = None + _attention_mask = None + else: + _input_ids = _pre_process_inputs(self.pad_token_id, prompts.batch["input_ids"][data_idx]) + _attention_mask = _pre_process_inputs(0, prompts.batch["attention_mask"][data_idx]) + _tools_kwargs = {} + _tool_schemas = None + + if self.interaction_map: + _interaction_kwargs = prompts.non_tensor_batch["interaction_kwargs"][data_idx] + else: + _interaction_kwargs = {} + + if not isinstance(raw_prompt, list | np.ndarray): + raise TypeError(f"raw_prompt must be a list or numpy array, got {type(raw_prompt)}") + + req = AsyncRolloutRequest( + batch_data_id=data_idx, + rollout_offset=0, + request_id=str(uuid4()), + state=AsyncRolloutRequestStateEnum.PENDING, + messages=list(raw_prompt), + multi_modal_data=multi_modal_data, + tool_schemas=_tool_schemas, + tools_kwargs=_tools_kwargs, + interaction_kwargs=_interaction_kwargs, + input_ids=_input_ids, + response_ids=None, + attention_mask=_attention_mask, + response_attention_mask=None, + response_position_ids=None, + response_loss_mask=None, + reward_scores={}, + max_prompt_len=self.config.prompt_length, + max_response_len=self.config.response_length, + max_model_len=min(self.config.max_model_len, self.config.prompt_length + self.config.response_length), + use_inference_chat_template=self.config.multi_turn.use_inference_chat_template, + tokenization_sanity_check_mode=self.config.multi_turn.tokenization_sanity_check_mode, + processing_class=self.processing_class, + ) + error_message = f"""Request {req.request_id} has mismatched lengths: + input_ids={req.input_ids.shape[-1]}, + attention_mask={req.attention_mask.shape[-1]}, + position_ids={req.position_ids.shape[-1]}, + loss_mask={req.loss_mask.shape[-1]}""" + assert ( + req.input_ids.shape[-1] + == req.attention_mask.shape[-1] + == req.position_ids.shape[-1] + == req.loss_mask.shape[-1] + ), error_message + req_list.append(req) + + return req_list + + async def resume(self, tags: list[str]): + """Resume rollout weights or kv cache in GPU memory. + + Args: + tag: weights or kv_cache. + """ + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.config.free_cache_engine: + await self._engine.resume_memory_occupation(tags=tags) + + async def release(self): + """Release weights and kv cache in GPU memory.""" + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.config.free_cache_engine: + await self._engine.release_memory_occupation(tags=["kv_cache", "weights"]) + + async def update_weights(self, weights: Generator[tuple[str, torch.Tensor], None, None], **kwargs): + """ + Update model weights using tensor buckets, similar to THUDM/slime's implementation. + + Notes: + - For the best performance of `rebuild_cuda_tensor`, it is recommended to: + 1. Enable `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES`. + 2. Manually set `CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7` + when using Tensor Parallelism (TP >= 8). + - See reference implementations in SLIME: + - Main logic: https://github.com/THUDM/slime/blob/fb7605cc5fb09af0f9369d37f7192f12bddee577/slime/ray/ppo_actor.py#L452 + - runtime envs: https://github.com/THUDM/slime/blob/fb7605cc5fb09af0f9369d37f7192f12bddee577/slime/ray/ppo_actor.py#L39 + """ + update_weights_bucket_bytes = int(self.config.update_weights_bucket_megabytes) << 20 + for params_batch in get_named_tensor_buckets(weights, update_weights_bucket_bytes): + await sgl_update_weights( + engine=self._engine, + params_batch=params_batch, + device_mesh_key="infer_tp", + device_mesh=self.device_mesh, + ) + + if self.device_mesh["infer_tp"].get_local_rank() == 0: + await self._engine.flush_cache() + + +class ServerAdapter(BaseRollout): + """SGLang server adapter used in native http server mode, serve as http client to request SGLang server + to resume/release/update weights and kv_cache. + + - hybrid mode: reside in each hybrid worker to sync weights between training engine and SGLang server. + - standalone/colocated mode: just a dummy placeholder to occupy the GPU to prevent ray scheduling new GPU actor. + """ + + def __init__( + self, + config: RolloutConfig, + model_config: HFModelConfig, + device_mesh: DeviceMesh, + ): + super().__init__(config, model_config, device_mesh) + self._engine: AsyncHttpServerAdapter = None + + rank = int(os.environ["RANK"]) + local_world_size = int(os.environ["RAY_LOCAL_WORLD_SIZE"]) + rollout_world_size = self.config.tensor_model_parallel_size * self.config.data_parallel_size + self.replica_rank = rank // rollout_world_size + self.rollout_rank = rank % rollout_world_size + self.node_rank = self.rollout_rank // local_world_size + self.local_rank = self.rollout_rank % local_world_size + + async def _init_server_adapter(self): + if self._engine is not None: + return + + # Lazy init http server adapter because http server is launched after hybrid engine. + self.server_actor = ray.get_actor(f"sglang_server_{self.replica_rank}_{self.node_rank}") + server_address, server_port = await self.server_actor.get_server_address.remote() + logger.debug( + f"replica_rank={self.replica_rank} node_rank={self.node_rank}, " + f"server address: {server_address}, port: {server_port}" + ) + host = f"[{server_address}]" if is_valid_ipv6_address(server_address) else server_address + self._engine = AsyncHttpServerAdapter( + model_path=self.model_config.local_path, host=host, port=server_port, launch_server=False + ) + + async def resume(self, tags: list[str]): + """Resume rollout weights or kv cache in GPU memory. + + Args: + tag: weights or kv_cache. + """ + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.config.free_cache_engine: + await self._init_server_adapter() + await self._engine.resume_memory_occupation(tags=tags) + + async def release(self): + """Release weights and kv cache in GPU memory.""" + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.config.free_cache_engine: + await self._init_server_adapter() + await self._engine.release_memory_occupation(tags=["kv_cache", "weights"]) + + async def update_weights(self, weights: Generator[tuple[str, torch.Tensor], None, None], **kwargs): + """ + Update model weights using tensor buckets, similar to THUDM/slime's implementation. + + Notes: + - For the best performance of `rebuild_cuda_tensor`, it is recommended to: + 1. Enable `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES`. + 2. Manually set `CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7` + when using Tensor Parallelism (TP >= 8). + - See reference implementations in SLIME: + - Main logic: https://github.com/THUDM/slime/blob/fb7605cc5fb09af0f9369d37f7192f12bddee577/slime/ray/ppo_actor.py#L452 + - runtime envs: https://github.com/THUDM/slime/blob/fb7605cc5fb09af0f9369d37f7192f12bddee577/slime/ray/ppo_actor.py#L39 + """ + if self.device_mesh["infer_tp"].get_local_rank() == 0: + await self._init_server_adapter() + + update_weights_bucket_bytes = int(self.config.update_weights_bucket_megabytes) << 20 + for params_batch in get_named_tensor_buckets(weights, update_weights_bucket_bytes): + await sgl_update_weights( + engine=self._engine, + params_batch=params_batch, + device_mesh_key="infer_tp", + device_mesh=self.device_mesh, + ) + + if self.device_mesh["infer_tp"].get_local_rank() == 0: + await self._engine.flush_cache() diff --git a/e2c/verl/verl/workers/rollout/sglang_rollout/utils.py b/e2c/verl/verl/workers/rollout/sglang_rollout/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f64bf63b81f4999089fb8e7fa0a30b608023b8b3 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/sglang_rollout/utils.py @@ -0,0 +1,108 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pickle +from typing import Any, Iterator, Optional + +import numpy as np +import torch +import torch.distributed as dist + +from verl.utils.device import get_device_name + + +def broadcast_pyobj( + data: list[Any], + rank: int, + dist_group: Optional[torch.distributed.ProcessGroup] = None, + src: int = 0, + force_cpu_device: bool = False, +): + """from https://github.com/sgl-project/sglang/blob/844e2f227ab0cce6ef818a719170ce37b9eb1e1b/python/sglang/srt/utils.py#L905 + + Broadcast inputs from src rank to all other ranks with torch.dist backend. + The `rank` here refer to the source rank on global process group (regardless + of dist_group argument). + """ + device = torch.device(get_device_name() if not force_cpu_device else "cpu") + + if rank == src: + if len(data) == 0: + tensor_size = torch.tensor([0], dtype=torch.long, device=device) + dist.broadcast(tensor_size, src=src, group=dist_group) + else: + serialized_data = pickle.dumps(data) + size = len(serialized_data) + + tensor_data = torch.ByteTensor(np.frombuffer(serialized_data, dtype=np.uint8)).to(device) + tensor_size = torch.tensor([size], dtype=torch.long, device=device) + + dist.broadcast(tensor_size, src=src, group=dist_group) + dist.broadcast(tensor_data, src=src, group=dist_group) + return data + else: + tensor_size = torch.tensor([0], dtype=torch.long, device=device) + dist.broadcast(tensor_size, src=src, group=dist_group) + size = tensor_size.item() + + if size == 0: + return [] + + tensor_data = torch.empty(size, dtype=torch.uint8, device=device) + dist.broadcast(tensor_data, src=src, group=dist_group) + + serialized_data = bytes(tensor_data.cpu().numpy()) + data = pickle.loads(serialized_data) + return data + + +def get_named_tensor_buckets( + iterable: Iterator[tuple[str, torch.Tensor]], bucket_bytes: int +) -> Iterator[list[tuple[str, torch.Tensor]]]: + """ + Group tensors into buckets based on a specified size in megabytes. + + Args: + iterable: An iterator of tuples containing tensor names and tensors. + bucket_bytes: The maximum size of each bucket in bytes. + + Yields: + Lists of tuples, where each tuple contains a tensor name and its corresponding tensor. + + Example: + >>> tensors = [('tensor1', torch.randn(1000, 1000)), ('tensor2', torch.randn(2000, 2000))] + >>> for bucket in get_named_tensor_buckets(tensors, bucket_size_mb=10): + ... print(bucket) + [('tensor1', tensor(...)), ('tensor2', tensor(...))] + + """ + if bucket_bytes <= 0: + raise ValueError(f"bucket_bytes must be greater than 0, got {bucket_bytes}") + + current_bucket = [] + current_size = 0 + for name, tensor in iterable: + tensor_size = tensor.element_size() * tensor.numel() + if current_size + tensor_size > bucket_bytes: + if current_bucket: + yield current_bucket + current_bucket = [(name, tensor)] + current_size = tensor_size + else: + current_bucket.append((name, tensor)) + current_size += tensor_size + + if current_bucket: + yield current_bucket diff --git a/e2c/verl/verl/workers/rollout/tokenizer.py b/e2c/verl/verl/workers/rollout/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..1e1212e50dce4785767cdd52c3dcc6288d08fa02 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/tokenizer.py @@ -0,0 +1,163 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The base tokenizer class, required for any hybrid engine based rollout or inference with vLLM. +""" + +from abc import ABC, abstractmethod + +import numpy as np +import torch + +__all__ = ["HybridEngineBaseTokenizer"] + + +class HybridEngineBaseTokenizer(ABC): + """the tokenizer property and function name should align with HF's to meet vllm requirement""" + + @property + @abstractmethod + def vocab_size(self): + """ + `int`: Size of the base vocabulary (without the added tokens). + """ + pass + + @property + @abstractmethod + def pad_token_id(self): + """ + `Optional[int]`: Id of the padding token in the vocabulary. Returns `None` if the token has not been set. + """ + pass + + @property + @abstractmethod + def eos_token_id(self): + """ + `Optional[int]`: Id of the end of sentence token in the vocabulary. Returns `None` if the token has not been + set. + """ + pass + + @property + @abstractmethod + def all_special_ids(self) -> list[int]: + """ + `List[int]`: List the ids of the special tokens(`''`, `''`, etc.) mapped to class attributes. + """ + pass + + @property + @abstractmethod + def all_special_tokens(self) -> list[str]: + """ + `List[str]`: A list of the unique special tokens (`''`, `''`, ..., etc.). + + Convert tokens of `tokenizers.AddedToken` type to string. + """ + pass + + @abstractmethod + def encode(self, text): + """ + Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary. + + Args: + text (`str`, `List[str]` or `List[int]`): + The first sequence to be encoded. This can be a string, a list of strings (tokenized string using the + `tokenize` method) or a list of integers. + + text_pair (`str`, `List[str]` or `List[int]`, *optional*): + Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using + the `tokenize` method) or a list of integers. + """ + pass + + @abstractmethod + def decode( + self, + token_ids: int | list[int] | np.ndarray | torch.Tensor, + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool = None, + **kwargs, + ) -> str: + """ + Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special + tokens and clean up tokenization spaces. + + Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`. + + Args: + token_ids (`Union[int, List[int], np.ndarray, torch.Tensor]`): + List of tokenized input ids. Can be obtained using the `__call__` method. + skip_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not to remove special tokens in the decoding. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces`. + kwargs (additional keyword arguments, *optional*): + Will be passed to the underlying model specific decode method. + + Returns: + `str`: The decoded sentence. + """ + pass + + @abstractmethod + def convert_ids_to_tokens(self, ids: int | list[int], skip_special_tokens: bool = False) -> str | list[str]: + """ + Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and + added tokens. + + Args: + ids (`int` or `List[int]`): + The token id (or token ids) to convert to tokens. + skip_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not to remove special tokens in the decoding. + + Returns: + `str` or `List[str]`: The decoded token(s). + """ + pass + + @abstractmethod + def get_added_vocab(self) -> dict[str, int]: + """ + Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from + the fast call because for now we always add the tokens even if they are already in the vocabulary. This is + something we should change. + + Returns: + `Dict[str, int]`: The added tokens. + """ + pass + + @abstractmethod + def convert_tokens_to_string(self, tokens: list[str]) -> str: + """ + Converts a sequence of tokens in a single string. The most simple way to do it is `" ".join(tokens)` but we + often want to remove sub-word tokenization artifacts at the same time. + + Args: + tokens (`List[str]`): The token to join in a string. + + Returns: + `str`: The joined tokens. + """ + pass + + @property + def is_fast(self): + return False diff --git a/e2c/verl/verl/workers/rollout/utils.py b/e2c/verl/verl/workers/rollout/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..575c98d6874e72dd8972433d7dd1ff7f91759d6a --- /dev/null +++ b/e2c/verl/verl/workers/rollout/utils.py @@ -0,0 +1,68 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import ipaddress +import logging +import os +import socket + +import uvicorn +from fastapi import FastAPI + +logger = logging.getLogger(__file__) + + +def is_valid_ipv6_address(address: str) -> bool: + try: + ipaddress.IPv6Address(address) + return True + except ValueError: + return False + + +def get_free_port(address: str) -> tuple[int, socket.socket]: + family = socket.AF_INET + if is_valid_ipv6_address(address): + family = socket.AF_INET6 + + sock = socket.socket(family=family, type=socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind((address, 0)) + + port = sock.getsockname()[1] + return port, sock + + +async def run_unvicorn(app: FastAPI, server_args, server_address, max_retries=5) -> tuple[int, asyncio.Task]: + server_port, server_task = None, None + + for i in range(max_retries): + try: + server_port, sock = get_free_port(server_address) + app.server_args = server_args + config = uvicorn.Config(app, host=server_address, port=server_port, log_level="warning") + server = uvicorn.Server(config) + server.should_exit = True + await server.serve() + server_task = asyncio.create_task(server.main_loop()) + break + except (OSError, SystemExit) as e: + logger.error(f"Failed to start HTTP server on port {server_port} at try {i}, error: {e}") + else: + logger.error(f"Failed to start HTTP server after {max_retries} retries, exiting...") + os._exit(-1) + + logger.info(f"HTTP server started on port {server_port}") + return server_port, server_task diff --git a/e2c/verl/verl/workers/rollout/vllm_rollout/__init__.py b/e2c/verl/verl/workers/rollout/vllm_rollout/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..767858fe35d4ba3132edf54ba79024bdcaf0b0f6 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/vllm_rollout/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from importlib.metadata import PackageNotFoundError, version + +from .vllm_rollout_spmd import vLLMAsyncRollout, vLLMRollout # noqa: F401 + + +def get_version(pkg): + try: + return version(pkg) + except PackageNotFoundError: + return None + + +vllm_package_name = "vllm" +vllm_package_version = get_version(vllm_package_name) +if vllm_package_version is None: + raise PackageNotFoundError( + "To use vllm rollout, please ensure the 'vllm' package is properly installed. See " + "https://verl.readthedocs.io/en/latest/start/install.html for more details" + ) + +if "ROCM_PATH" in os.environ: + import re + + match = re.match(r"(\d+\.\d+\.?\d*)", vllm_package_version) + if match: + vllm_package_version = match.group(1) + else: + raise ValueError(f"Warning: Could not parse version format: {vllm_package_version}") diff --git a/e2c/verl/verl/workers/rollout/vllm_rollout/vllm_async_server.py b/e2c/verl/verl/workers/rollout/vllm_rollout/vllm_async_server.py new file mode 100644 index 0000000000000000000000000000000000000000..5d4f2f59ce67ae7989b07479662597b83c783e28 --- /dev/null +++ b/e2c/verl/verl/workers/rollout/vllm_rollout/vllm_async_server.py @@ -0,0 +1,500 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import asyncio +import json +import logging +import os +import pickle +from pprint import pprint +from typing import Any, Callable, Optional + +import numpy as np +import ray +import vllm.entrypoints.cli.serve +import zmq +from ray.actor import ActorHandle +from vllm import SamplingParams +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.entrypoints.openai.api_server import ( + build_app, + init_app_state, +) +from vllm.inputs import TokensPrompt +from vllm.outputs import RequestOutput +from vllm.usage.usage_lib import UsageContext +from vllm.utils import FlexibleArgumentParser, get_tcp_uri +from vllm.v1.engine.async_llm import AsyncLLM +from vllm.v1.engine.core import EngineCoreProc +from vllm.v1.engine.utils import CoreEngineProcManager +from vllm.v1.executor.abstract import Executor + +from verl.single_controller.ray import RayClassWithInitArgs +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.config import HFModelConfig, RewardModelConfig, RolloutConfig +from verl.workers.rollout.replica import RolloutMode, RolloutReplica, TokenOutput +from verl.workers.rollout.utils import get_free_port, run_unvicorn +from verl.workers.rollout.vllm_rollout import vLLMAsyncRollout + +logger = logging.getLogger(__file__) +logger.setLevel(logging.INFO) + + +class ExternalZeroMQDistributedExecutor(Executor): + """An executor that engines are launched by external ray actors.""" + + uses_ray: bool = False + + def _init_executor(self) -> None: + dp_rank_local = self.vllm_config.parallel_config.data_parallel_rank_local + tp_size = self.vllm_config.parallel_config.tensor_parallel_size + + addresses = os.environ["VERL_VLLM_ZMQ_ADDRESSES"].split(",") + addresses = addresses[dp_rank_local * tp_size : (dp_rank_local + 1) * tp_size] + self.context = zmq.Context() + self.sockets = [] + for address in addresses: + socket = self.context.socket(zmq.REQ) + socket.connect(address) + self.sockets.append(socket) + + kwargs = dict( + vllm_config=self.vllm_config, + local_rank=None, + rank=None, + distributed_init_method="env://", + is_driver_worker=True, + ) + self.collective_rpc("init_worker", args=([kwargs],)) + self.collective_rpc("init_device") + self.collective_rpc("load_model") + + def collective_rpc( + self, + method: str | Callable, + timeout: Optional[float] = None, + args: tuple = (), + kwargs: Optional[dict[str, Any]] = None, + ) -> list[Any]: + if isinstance(method, str): + sent_method = method + else: + sent_method = pickle.dumps(method) + del method + + message = pickle.dumps((sent_method, args, kwargs or {})) + for socket in self.sockets: + socket.send(message, zmq.DONTWAIT) + + outputs = [] + for socket in self.sockets: + outputs.append(pickle.loads(socket.recv())) + return outputs + + def check_health(self): + return + + +@ray.remote(num_cpus=1) +class vLLMHttpServer: + """vLLM http server in single node, this is equivalent to launch server with command line: + ``` + vllm serve --tensor-parallel-size=8 ... + ``` + """ + + def __init__( + self, + config: RolloutConfig | RewardModelConfig, + model_config: HFModelConfig, + rollout_mode: RolloutMode, + workers: list[ActorHandle], + replica_rank: int, + node_rank: int, + gpus_per_node: int, + nnodes: int, + ): + """ + Args: + config (RolloutConfig | RewardModelConfig): full config. + model_config (HFModelConfig): model config. + rollout_mode (RolloutMode): rollout mode. + replica_rank (int): replica rank, a replica may contain multiple nodes. + node_rank (int): node rank. + gpus_per_node (int): number of gpus per node. + nnodes (int): number of nodes. + """ + super().__init__() + + self.config: RolloutConfig | RewardModelConfig = omega_conf_to_dataclass(config) + self.model_config: HFModelConfig = omega_conf_to_dataclass(model_config, dataclass_type=HFModelConfig) + self.config.max_model_len = self.config.prompt_length + self.config.response_length + self.rollout_mode = rollout_mode + self.workers = workers + + self.replica_rank = replica_rank + self.node_rank = node_rank + self.gpus_per_node = gpus_per_node + self.nnodes = nnodes + + if self.rollout_mode != RolloutMode.HYBRID and self.config.load_format == "dummy": + logger.warning(f"rollout mode is {self.rollout_mode}, load_format is dummy, set to auto") + self.config.load_format = "auto" + + # used for http server + self._server_address = ray.util.get_node_ip_address().strip("[]") + self._server_port = None + + # used for data parallel: --data-parallel-address, --data-parallel-rpc-port + if self.node_rank == 0: + self._master_address = self._server_address + self._master_port, self._master_sock = get_free_port(self._server_address) + self._dp_master_port, self._dp_master_sock = get_free_port(self._server_address) + logger.info( + f"vLLMHttpServer, replica_rank: {self.replica_rank}, master address: {self._master_address}, " + f"master port: {self._master_port}, data parallel master port: {self._dp_master_port}" + ) + else: + self._master_address = None + self._master_port = None + + def get_master_address(self): + """Get master address and port for data parallel.""" + return self._master_address, self._master_port + + def get_server_address(self): + """Get http server address and port.""" + assert self._server_port is not None, "http server is not launched, port is None" + return self._server_address, self._server_port + + async def launch_server(self, master_address: str = None, master_port: int = None): + if self.node_rank != 0: + assert master_address and master_port, "non-master node should provide master address and port" + self._master_address = master_address + self._master_port = master_port + + # 1. setup vllm serve cli args + engine_kwargs = self.config.get("engine_kwargs", {}).get("vllm", {}) or {} + engine_kwargs = {key: val for key, val in engine_kwargs.items() if val is not None} + if self.config.get("limit_images", None): # support for multi-image data + engine_kwargs["limit_mm_per_prompt"] = {"image": self.config.get("limit_images")} + if self.config.cudagraph_capture_sizes: + engine_kwargs["cuda_graph_sizes"] = self.config.cudagraph_capture_sizes + + # Override default generation config from hugging face model config, + # user can still override them by passing kwargs in each request. + override_generation_config = dict( + temperature=self.config.temperature, + top_k=self.config.top_k, + top_p=self.config.top_p, + repetition_penalty=1.0, + max_new_tokens=self.config.response_length, + ) + logger.info(f"override_generation_config: {override_generation_config}") + + args = { + "dtype": self.config.dtype, + "load_format": self.config.load_format, + "skip_tokenizer_init": False, + "trust_remote_code": True, + "max_model_len": self.config.max_model_len, + "max_num_seqs": self.config.max_num_seqs, + "enable_chunked_prefill": self.config.enable_chunked_prefill, + "max_num_batched_tokens": self.config.max_num_batched_tokens, + "enable_prefix_caching": self.config.enable_prefix_caching, + "enable_sleep_mode": True, + "disable_custom_all_reduce": True, + "enforce_eager": self.config.enforce_eager, + "gpu_memory_utilization": self.config.gpu_memory_utilization, + "disable_log_stats": self.config.disable_log_stats, + "tensor_parallel_size": self.config.tensor_model_parallel_size, + "seed": self.config.get("seed", 0), + "override_generation_config": json.dumps(override_generation_config), + **engine_kwargs, + } + if self.config.expert_parallel_size > 1: + assert self.gpus_per_node % self.config.tensor_model_parallel_size == 0, ( + "gpus_per_node should be divisible by tensor_model_parallel_size" + ) + data_parallel_size_local = self.gpus_per_node // self.config.tensor_model_parallel_size + assert len(self.workers) == data_parallel_size_local * self.config.tensor_model_parallel_size, ( + f"num workers ({len(self.workers)}) should be equal to dp_size_local " + ) + f"({data_parallel_size_local}) * tp_size ({self.config.tensor_model_parallel_size})" + + args.update( + { + "enable_expert_parallel": self.config.expert_parallel_size > 1, + "data_parallel_size": self.config.data_parallel_size, + "data_parallel_size_local": data_parallel_size_local, + "data_parallel_start_rank": self.node_rank * data_parallel_size_local, + "data_parallel_address": self._master_address, + "data_parallel_rpc_port": self._master_port, + } + ) + + server_args = ["serve", self.model_config.local_path] + for k, v in args.items(): + if isinstance(v, bool): + if v: + server_args.append(f"--{k}") + else: + server_args.append(f"--{k}") + server_args.append(str(v)) + + if self.replica_rank == 0: + pprint(server_args) + + CMD_MODULES = [vllm.entrypoints.cli.serve] + parser = FlexibleArgumentParser(description="vLLM CLI") + subparsers = parser.add_subparsers(required=False, dest="subparser") + cmds = {} + for cmd_module in CMD_MODULES: + new_cmds = cmd_module.cmd_init() + for cmd in new_cmds: + cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd) + cmds[cmd.name] = cmd + server_args = parser.parse_args(args=server_args) + server_args.model = server_args.model_tag + if server_args.subparser in cmds: + cmds[server_args.subparser].validate(server_args) + + # 2. setup distributed executor backend + distributed_executor_backend = ExternalZeroMQDistributedExecutor if len(self.workers) > 0 else None + server_args.distributed_executor_backend = distributed_executor_backend + + zmq_addresses = ray.get([worker.get_zeromq_address.remote() for worker in self.workers]) + logger.info( + f"replica_rank={self.replica_rank}, node_rank={self.node_rank}, nnodes={self.nnodes}, " + f"get worker zmq addresses: {zmq_addresses}" + ) + os.environ["VERL_VLLM_ZMQ_ADDRESSES"] = ",".join(zmq_addresses) + + # 3. launch server + if self.node_rank == 0: + await self.run_server(server_args) + else: + await self.run_headless(server_args) + + async def run_server(self, args: argparse.Namespace): + engine_args = AsyncEngineArgs.from_cli_args(args) + usage_context = UsageContext.OPENAI_API_SERVER + vllm_config = engine_args.create_engine_config(usage_context=usage_context) + vllm_config.parallel_config.data_parallel_master_port = self._dp_master_port + + engine_client = AsyncLLM.from_vllm_config( + vllm_config=vllm_config, + usage_context=usage_context, + disable_log_requests=engine_args.disable_log_requests, + disable_log_stats=engine_args.disable_log_stats, + ) + + # Don't keep the dummy data in memory + await engine_client.reset_mm_cache() + + app = build_app(args) + await init_app_state(engine_client, vllm_config, app.state, args) + if self.replica_rank == 0 and self.node_rank == 0: + logger.info(f"Initializing a V1 LLM engine with config: {vllm_config}") + + self.engine = engine_client + self._server_port, self._server_task = await run_unvicorn(app, args, self._server_address) + + async def run_headless(self, args: argparse.Namespace): + # Create the EngineConfig. + engine_args = vllm.AsyncEngineArgs.from_cli_args(args) + usage_context = UsageContext.OPENAI_API_SERVER + vllm_config = engine_args.create_engine_config(usage_context=usage_context, headless=True) + + parallel_config = vllm_config.parallel_config + local_engine_count = parallel_config.data_parallel_size_local + + host = parallel_config.data_parallel_master_ip + port = engine_args.data_parallel_rpc_port # add to config too + handshake_address = get_tcp_uri(host, port) + + # Create the engines. + self.engine_manager = CoreEngineProcManager( + target_fn=EngineCoreProc.run_engine_core, + local_engine_count=local_engine_count, + start_index=vllm_config.parallel_config.data_parallel_rank, + local_start_index=0, + vllm_config=vllm_config, + local_client=False, + handshake_address=handshake_address, + executor_class=Executor.get_class(vllm_config), + log_stats=not engine_args.disable_log_stats, + ) + + async def generate( + self, + prompt_ids: list[int], + sampling_params: dict[str, Any], + request_id: str, + image_data: Optional[list[Any]] = None, + ) -> TokenOutput: + """Generate sequence with token-in-token-out.""" + # TODO(@wuxibin): switch to `/generate` http endpoint once multi-modal support ready. + max_tokens = self.config.max_model_len - len(prompt_ids) + sampling_params["logprobs"] = 0 if sampling_params.pop("logprobs", False) else None + sampling_params.setdefault("repetition_penalty", self.config.get("repetition_penalty", 1.0)) + sampling_params = SamplingParams(max_tokens=max_tokens, **sampling_params) + prompt_ids = _qwen2_5_vl_dedup_image_tokens(prompt_ids, self.model_config.processor) + prompt = TokensPrompt( + prompt_token_ids=prompt_ids, multi_modal_data={"image": image_data} if image_data else None + ) + generator = self.engine.generate(prompt=prompt, sampling_params=sampling_params, request_id=request_id) + + # Get final response + final_res: Optional[RequestOutput] = None + async for output in generator: + final_res = output + assert final_res is not None + + token_ids = final_res.outputs[0].token_ids + log_probs = None + if sampling_params.logprobs is not None: + log_probs = [logprobs[token_ids[i]].logprob for i, logprobs in enumerate(final_res.outputs[0].logprobs)] + return TokenOutput(token_ids=token_ids, log_probs=log_probs) + + async def wake_up(self): + if self.rollout_mode == RolloutMode.HYBRID: + # Call all workers to switch between trainer mode and rollout mode. + await asyncio.gather(*[worker.wake_up.remote() for worker in self.workers]) + elif self.rollout_mode == RolloutMode.COLOCATED: + # Directly call engine to wake up without sync weights. + if self.node_rank == 0: + await self.engine.wake_up(tags=["kv_cache", "weights"]) + elif self.rollout_mode == RolloutMode.STANDALONE: + logger.info("skip wake_up in standalone mode") + + async def sleep(self): + if self.rollout_mode == RolloutMode.HYBRID: + if self.node_rank == 0: + await self.engine.reset_prefix_cache() + await asyncio.gather(*[worker.sleep.remote() for worker in self.workers]) + elif self.rollout_mode == RolloutMode.COLOCATED: + if self.node_rank == 0: + await self.engine.reset_prefix_cache() + await self.engine.sleep(level=1) + elif self.rollout_mode == RolloutMode.STANDALONE: + logger.info("skip sleep in standalone mode") + + async def wait_for_requests_to_drain(self): + await self.engine.wait_for_requests_to_drain() + + +_rollout_worker_actor_cls = ray.remote(vLLMAsyncRollout) + + +class vLLMReplica(RolloutReplica): + def get_ray_class_with_init_args(self) -> RayClassWithInitArgs: + """Get rollout worker actor class for colocated and standalone mode.""" + worker_dict_cls = RayClassWithInitArgs( + cls=_rollout_worker_actor_cls, + config=self.config, + model_config=self.model_config, + device_mesh=None, + ) + return worker_dict_cls + + async def launch_servers(self): + """Launch http server in each node.""" + assert len(self.workers) == self.world_size, ( + f"worker number {len(self.workers)} not equal to world size {self.world_size}" + ) + + # get node_id of all workers + worker_node_ids = await asyncio.gather( + *[ + worker.__ray_call__.remote(lambda self: ray.get_runtime_context().get_node_id()) + for worker in self.workers + ] + ) + + # For non-data parallel case, there's only one server whether it's single or multi nodes. + nnodes, gpus_per_node = self.nnodes, self.gpus_per_node + if self.config.data_parallel_size == 1: + nnodes = 1 + gpus_per_node = self.world_size + + # create server actor in each node with node affinity + for node_rank in range(nnodes): + workers = self.workers[node_rank * gpus_per_node : (node_rank + 1) * gpus_per_node] + node_id = worker_node_ids[node_rank * gpus_per_node] + server = vLLMHttpServer.options( + scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=node_id, + soft=False, + ), + name=f"vllm_server_{self.replica_rank}_{node_rank}", + ).remote( + config=self.config, + model_config=self.model_config, + rollout_mode=self.rollout_mode, + workers=workers, + replica_rank=self.replica_rank, + node_rank=node_rank, + gpus_per_node=gpus_per_node, + nnodes=nnodes, + ) + self.servers.append(server) + + # launch http server in each node + master_address, master_port = await self.servers[0].get_master_address.remote() + await asyncio.gather( + *[ + server.launch_server.remote(master_address=master_address, master_port=master_port) + for server in self.servers + ] + ) + + # get http server address from first server + server_address, server_port = await self.servers[0].get_server_address.remote() + self._server_handle = self.servers[0] + self._server_address = f"{server_address}:{server_port}" + + async def sleep(self): + """Sleep each rollout server.""" + # Drain DP engines for safe sleep. + await self.servers[0].wait_for_requests_to_drain.remote() + await asyncio.gather(*[server.sleep.remote() for server in self.servers]) + + +def _qwen2_5_vl_dedup_image_tokens(prompt_ids: list[int], processor): + """Deduplicate consecutive image tokens in prompt_ids for Qwen2.5-VL, since vLLM will replicate the + <|image_pad|> token by image_data. + + For example, + ``` + <|vision_start|><|image_pad|><|image_pad|>...<|image_pad|><|vision_end|> + => + <|vision_start|><|image_pad|><|vision_end|> + ``` + """ + if processor is not None and "Qwen2VLImageProcessor" in processor.image_processor.__class__.__name__: + prompt_ids = np.array(prompt_ids) + + # Create a mask where True indicates elements to keep + mask = np.ones(len(prompt_ids), dtype=bool) + + # Find where the array equals the value + is_value = prompt_ids == processor.image_token_id + + # Find consecutive duplicates by checking if previous element is also the value + mask[1:] &= ~(is_value[1:] & is_value[:-1]) + + return prompt_ids[mask].tolist() + else: + return prompt_ids diff --git a/e2c/verl/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py b/e2c/verl/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py new file mode 100644 index 0000000000000000000000000000000000000000..1e2425a7d3c82e3d48acfc347faf1aa2e47fbf8b --- /dev/null +++ b/e2c/verl/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py @@ -0,0 +1,614 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The vllm_rollout that can be applied in different backend +When working with FSDP: +- Use DTensor weight loader (recommended) or HF weight loader +- Utilize state_dict from the FSDP to synchronize the weights among tp ranks in vLLM +When working with Megatron: +- Use Megatron weight loader +- During training, only the current pp stage holds the parameters +- Before inference, broadcast the parameters of the current pp rank + to all other pp ranks (all pp ranks holds all the parameters) +- Bind the parameters to the inference engine +- Do inference in tp. pp is treated as additional dp +- After inference, all the parameters that doesn't belong to this pp rank is freed. +""" + +import asyncio +import getpass +import inspect +import logging +import os +import pickle +import socket +import time +from contextlib import contextmanager +from dataclasses import asdict +from types import MethodType +from typing import Any, Generator + +import numpy as np +import ray +import torch +import torch.distributed +import zmq +import zmq.asyncio +from filelock import FileLock +from omegaconf import ListConfig +from tensordict import TensorDict +from torch.distributed.device_mesh import DeviceMesh +from vllm import LLM, SamplingParams +from vllm.config import CompilationConfig, CompilationLevel, LoRAConfig +from vllm.lora.request import LoRARequest +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.worker.worker_base import WorkerWrapperBase + +from verl import DataProto +from verl.third_party.vllm import VLLM_SLEEP_LEVEL +from verl.utils.device import is_npu_available +from verl.utils.distributed import initialize_global_process_group_ray +from verl.utils.profiler import GPUMemoryLogger +from verl.utils.ray_utils import ray_noset_visible_devices +from verl.utils.torch_functional import get_response_mask, pad_2d_list_to_length +from verl.utils.vllm import TensorLoRARequest, VLLMHijack, is_version_ge +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.base import BaseRollout + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + +# TODO +# 1. support pp in vllm +# 2. passing tokenizer is not necessary? no encoding/decoding is happending here +# 3. simplify init logics + + +# NOTE(sgm): add for verl. We can optimize it by making the dataloader yield List[int] without padding. +def _pre_process_inputs(pad_token_id, prompt_token_ids: torch.Tensor) -> list[int]: + # remove the left padding in the prompt token_id + # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id + # is not None else self.llm_engine.tokenizer.eos_token_id + non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] + token_ids = prompt_token_ids[non_pad_index:].tolist() + return token_ids + + +if is_version_ge(pkg="vllm", minver="0.7.3"): + VLLMHijack.hijack() + + +class vLLMRollout(BaseRollout): + def __init__( + self, + config: RolloutConfig, + model_config: HFModelConfig, + device_mesh: DeviceMesh, + ): + super().__init__(config, model_config, device_mesh) + + if config.layered_summon: + self.sleep_level = 1 + else: + self.sleep_level = VLLM_SLEEP_LEVEL + + model_path = model_config.local_path + tokenizer = model_config.tokenizer + model_hf_config = model_config.hf_config + trust_remote_code = model_config.trust_remote_code + self.lora_kwargs = ( + {"enable_lora": True, "max_loras": 1, "max_lora_rank": model_config.lora_rank} + if model_config.lora_rank > 0 + else {} + ) + + tensor_parallel_size = self.config.get("tensor_model_parallel_size", 1) + assert tensor_parallel_size <= torch.distributed.get_world_size(), ( + "tensor parallel size should be less than or equal to the world size" + ) + max_num_batched_tokens = self.config.get("max_num_batched_tokens", 8192) + + rope_scaling_config = getattr(model_hf_config, "rope_scaling", None) + if not rope_scaling_config: + max_position_embeddings = None + if hasattr(model_hf_config, "max_position_embeddings"): + max_position_embeddings = model_hf_config.max_position_embeddings + elif hasattr(model_hf_config, "llm_config") and hasattr( + model_hf_config.llm_config, "max_position_embeddings" + ): + max_position_embeddings = model_hf_config.llm_config.max_position_embeddings + elif hasattr(model_hf_config, "text_config") and hasattr( + model_hf_config.text_config, "max_position_embeddings" + ): + max_position_embeddings = model_hf_config.text_config.max_position_embeddings + if max_position_embeddings is None: + raise ValueError("max_position_embeddings not found in model_hf_config") + assert max_position_embeddings >= config.prompt_length + config.response_length, ( + "model context length should be greater than total sequence length" + ) + else: + # handle type where there's a length extend factor + # see https://qwen.readthedocs.io/en/latest/deployment/vllm.html#extended-context-support + # for using yarn as an example + rope_scaling_factor = rope_scaling_config.get("factor", 1.0) + + assert ( + model_hf_config.max_position_embeddings * rope_scaling_factor + >= config.prompt_length + config.response_length + ), ( + "model context length should be greater than total sequence length, " + + f"got rope_scaling_factor={rope_scaling_factor} and " + + f"max_position_embeddings={model_hf_config.max_position_embeddings}" + ) + + max_model_len = int(config.max_model_len or config.prompt_length + config.response_length) + + if max_num_batched_tokens < max_model_len and self.config.enable_chunked_prefill: + raise ValueError( + "Enable chunked prefill, max_num_batched_tokens is smaller than max_model_len, \ + please increase max_num_batched_tokens or disable chunked prefill" + ) + + load_format = "dummy" if config.load_format.startswith("dummy") else config.load_format + + # copy it to avoid secretly modifying the engine config + engine_kwargs = config.get("engine_kwargs", {}).get("vllm", {}) or {} + + # For each vLLM engine parameter, + # - `None` means not setting it, so we pop it, and leave it to vLLM default value + # (which can vary across different vLLM versions); + # - Otherwise it's the desired value we want to explicitly set. + engine_kwargs = {key: val for key, val in engine_kwargs.items() if val is not None} + if config.get("limit_images", None): # support for multi-image data + engine_kwargs["limit_mm_per_prompt"] = {"image": config.get("limit_images")} + + compilation_config = {} + + cudagraph_capture_sizes = config.get("cudagraph_capture_sizes") + # enforce_eager must be False to use cudagraph + if not config.enforce_eager and cudagraph_capture_sizes: + if isinstance(cudagraph_capture_sizes, ListConfig): + compilation_config["compilation_config"] = CompilationConfig( + level=CompilationLevel.PIECEWISE, cudagraph_capture_sizes=cudagraph_capture_sizes + ) + else: + logger.warning(f"cudagraph_capture_sizes must be a list, but got {cudagraph_capture_sizes}") + + self.inference_engine = LLM( + model=model_path, + enable_sleep_mode=config.free_cache_engine, + tensor_parallel_size=tensor_parallel_size, + distributed_executor_backend="external_launcher", + dtype=config.dtype, + enforce_eager=config.enforce_eager, + gpu_memory_utilization=config.gpu_memory_utilization, + disable_custom_all_reduce=True, + skip_tokenizer_init=False, + max_model_len=max_model_len, + max_num_seqs=config.max_num_seqs, + load_format=load_format, + disable_log_stats=config.disable_log_stats, + max_num_batched_tokens=max_num_batched_tokens, + enable_chunked_prefill=config.enable_chunked_prefill, + enable_prefix_caching=config.enable_prefix_caching, + trust_remote_code=trust_remote_code, + seed=config.get("seed", 0), + **compilation_config, + **self.lora_kwargs, + **engine_kwargs, + ) + + kwargs = dict( + n=1, + logprobs=0, # can be set to 0 and let actor to recompute + max_tokens=config.response_length, + repetition_penalty=config.get("repetition_penalty", 1.0), + ) + + kwargs["detokenize"] = False + + # supporting adding any sampling params from the config file + for k in config.keys(): + if hasattr(SamplingParams(), str(k)) and k != "seed": + kwargs[k] = config.get(k) + kwargs["n"] = 1 # already repeat in ray_trainer + print(f"kwargs: {kwargs}") + self.sampling_params = SamplingParams(**kwargs) + + self.pad_token_id = tokenizer.pad_token_id + + @contextmanager + def update_sampling_params(self, **kwargs): + # update sampling params + old_sampling_params_args = {} + if kwargs: + for key, value in kwargs.items(): + if hasattr(self.sampling_params, key): + old_value = getattr(self.sampling_params, key) + old_sampling_params_args[key] = old_value + setattr(self.sampling_params, key, value) + yield + # roll back to previous sampling params + # if len(old_sampling_params_args): + for key, value in old_sampling_params_args.items(): + setattr(self.sampling_params, key, value) + + @GPUMemoryLogger(role="vllm rollout spmd", logger=logger) + @torch.no_grad() + def generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: + """Generate sequences for a batch of prompts. + + Args: + batch (DataProto): Input batch. + + Returns: + DataProto: Output batch. + - prompts: [bsz, prompt_length], prompt token ids from dataset. + - responses: [bsz, response_length], output token ids include response tokens + from LLM generation and observation tokens from tool_calls. + - response_mask: [bsz, response_length], 1 for LLM generated tokens, 0 for observation/padding tokens. + - input_ids: [bsz, prompt_length + response_length], whole sequence token ids, including prompt tokens + and response tokens. + - attention_mask: [bsz, prompt_length + response_length], 0 for padding tokens, 1 for other tokens. + - position_ids: [bsz, prompt_length + response_length], incremental position ids. + + For multi-turn conversations: + responses: |<- LLM generation ->|<- tool_calls ->|<- LLM generation ->|<- padding ->| + response_mask: | 1, 1, 1, ..., 1, 1 | 0, 0, .., 0, 0 | 1, 1, 1, ..., 1, 1 | 0, 0, ..., 0| + """ + idx = prompts.batch["input_ids"] # (bs, prompt_length) + # left-padded attention_mask + attention_mask = prompts.batch["attention_mask"] + position_ids = prompts.batch["position_ids"] + + # used to construct attention_mask + eos_token_id = prompts.meta_info["eos_token_id"] + + batch_size = idx.size(0) + + non_tensor_batch = prompts.non_tensor_batch + if "raw_prompt_ids" not in non_tensor_batch: + non_tensor_batch["raw_prompt_ids"] = np.array( + [_pre_process_inputs(self.pad_token_id, idx[i]) for i in range(batch_size)], dtype=object + ) + + if batch_size != len(non_tensor_batch["raw_prompt_ids"]): + raise RuntimeError("vllm sharding manager is not work properly.") + + if "multi_modal_data" in non_tensor_batch: + vllm_inputs = [] + for raw_prompt_ids, multi_modal_data in zip( + non_tensor_batch.pop("raw_prompt_ids"), non_tensor_batch.pop("multi_modal_data"), strict=True + ): + vllm_inputs.append({"prompt_token_ids": raw_prompt_ids, "multi_modal_data": multi_modal_data}) + else: + vllm_inputs = [ + {"prompt_token_ids": raw_prompt_ids} for raw_prompt_ids in non_tensor_batch.pop("raw_prompt_ids") + ] + + for input_data in vllm_inputs: + # Ensure token IDs are lists or numpy arrays + if not isinstance(input_data["prompt_token_ids"], list | np.ndarray): + raise TypeError( + f"prompt_token_ids must be a list or numpy array, got {type(input_data['prompt_token_ids'])}" + ) + + input_data["prompt_token_ids"] = list(input_data["prompt_token_ids"]) + + do_sample = prompts.meta_info.get("do_sample", True) + is_validate = prompts.meta_info.get("validate", False) + if not do_sample: + kwargs = { + "best_of": 1, + "top_p": 1.0, + "top_k": -1, + "min_p": 0.0, + "temperature": 0, + "n": 1, # if greedy, only 1 response + } + elif is_validate: + # TODO: try ** + kwargs = { + "top_k": self.config.val_kwargs.top_k, + "top_p": self.config.val_kwargs.top_p, + "temperature": self.config.val_kwargs.temperature, + "n": 1, # if validate, already repeat in ray_trainer + } + + lora_requests = None + if self.lora_kwargs: + lora_int_ids = list(self.inference_engine.llm_engine.list_loras()) + if len(lora_int_ids) > 0: + lora_int_id = lora_int_ids[0] + lora_requests = [ + LoRARequest(lora_name=f"{lora_int_id}", lora_int_id=lora_int_id, lora_path="/simon-stub-path") + ] * batch_size + + # users can customize different sampling_params at different run + with self.update_sampling_params(**kwargs): + outputs = self.inference_engine.generate( + prompts=vllm_inputs, # because we have already convert it to prompt token id + sampling_params=self.sampling_params, + lora_request=lora_requests, + use_tqdm=False, + ) + + # TODO(sgm): disable logprob when recompute_log_prob is enable + # if n = 1: (bs, response_length) ; if n > 1: (bs * n, response_length) + + response = [] + rollout_log_probs = [] + for output in outputs: + for sample_id in range(len(output.outputs)): + response_ids = output.outputs[sample_id].token_ids + response.append(response_ids) + if self.config.calculate_log_probs: + curr_log_prob = [] + for i, logprob in enumerate(output.outputs[sample_id].logprobs): + curr_log_prob.append(logprob[response_ids[i]].logprob) + rollout_log_probs.append(curr_log_prob) + + response = pad_2d_list_to_length(response, self.pad_token_id, max_length=self.config.response_length).to( + idx.device + ) + if self.config.calculate_log_probs: + rollout_log_probs = pad_2d_list_to_length( + rollout_log_probs, -1, max_length=self.config.response_length + ).to(idx.device) + rollout_log_probs = rollout_log_probs.to(torch.float32) + + seq = torch.cat([idx, response], dim=-1) + + response_length = response.size(1) + delta_position_id = torch.arange(1, response_length + 1, device=position_ids.device) + delta_position_id = delta_position_id.unsqueeze(0).expand(batch_size, -1) + if position_ids.dim() == 3: # qwen2vl mrope (batch size, 4, seq len) + delta_position_id = delta_position_id.view(batch_size, 1, -1).expand(batch_size, position_ids.size(1), -1) + + # TODO(sgm): fix position_ids on right_pad + # prompt: left pad + response: right pad + # attention_mask: [0,0,0,0,1,1,1,1, | 1,1,1,0,0,0,0,0] + # position_ids: [0,0,0,0,0,1,2,3, | 4,5,6,7,8,9,10,11] + response_position_ids = position_ids[..., -1:] + delta_position_id + position_ids = torch.cat([position_ids, response_position_ids], dim=-1) + response_attention_mask = get_response_mask( + response_id=response, eos_token=eos_token_id, dtype=attention_mask.dtype + ) + attention_mask = torch.cat((attention_mask, response_attention_mask), dim=-1) + + # all the tp ranks should contain the same data here. data in all ranks are valid + batch = TensorDict( + { + "prompts": idx, + "responses": response, + "input_ids": seq, # here input_ids become the whole sentences + "attention_mask": attention_mask, + "position_ids": position_ids, + }, + batch_size=batch_size, + ) + if self.config.calculate_log_probs: + # we will recompute old log prob with actor + batch["rollout_log_probs"] = rollout_log_probs + + return DataProto(batch=batch, non_tensor_batch=non_tensor_batch) + + async def resume(self, tags: list[str]): + """Resume rollout weights or kv cache in GPU memory. + + Args: + tags: weights or kv_cache. + """ + if not self.config.free_cache_engine: + return + + if "tags" in inspect.signature(self.inference_engine.wake_up).parameters: + self.inference_engine.wake_up(tags=tags) + else: + self.inference_engine.wake_up() + + async def release(self): + """Release weights and kv cache in GPU memory.""" + self.inference_engine.reset_prefix_cache() + + if not self.config.free_cache_engine: + return + + self.inference_engine.sleep(level=self.sleep_level) + + async def update_weights(self, weights: Generator[tuple[str, torch.Tensor], None, None], **kwargs): + """Update the weights of the rollout model. + + Args: + weights: A generator that yields the name of the weight tensor and the tensor itself. + """ + peft_config, base_sync_done = kwargs.get("peft_config", None), kwargs.get("base_sync_done", False) + if peft_config and base_sync_done: + lora_int_id = int(time.time_ns() % 0x7FFFFFFF) + lora_reqest = TensorLoRARequest( + lora_name=f"{lora_int_id}", + lora_int_id=lora_int_id, + lora_path="simon_lora_path", + peft_config=asdict(peft_config), + lora_tensors=weights, + ) + self.inference_engine.llm_engine.add_lora(lora_reqest) + logger.info(f"vLLM load weights, loaded_params: {len(weights)}") + else: + from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader + + model = self.inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner.model + patch_vllm_moe_model_weight_loader(model) + model.load_weights(weights) + + +# https://github.com/vllm-project/vllm/issues/13175 +def _monkey_patch_compute_logits(model, vocab_size: int): + original_compute_logits = model.compute_logits + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> torch.Tensor: + logits = original_compute_logits(hidden_states, sampling_metadata) + logits[..., vocab_size:] = float("-inf") + return logits + + model.compute_logits = MethodType(compute_logits, model) + + +class vLLMAsyncRollout(BaseRollout): + """vLLMAsyncRollout is a thin wrapper of WorkerWrapperBase, which is engine in single worker process.""" + + def __init__( + self, + config: RolloutConfig, + model_config: HFModelConfig, + device_mesh: DeviceMesh, + ): + super().__init__(config, model_config, device_mesh) + self.tokenizer = model_config.tokenizer + self.inference_engine: WorkerWrapperBase = None + self.address = self._init_zeromq() + self.lora_config = ( + {"max_loras": 1, "max_lora_rank": model_config.lora_rank} if model_config.lora_rank > 0 else {} + ) + + # https://github.com/vllm-project/vllm/issues/25171 + if config.layered_summon or config.expert_parallel_size > 1: + self.sleep_level = 1 + else: + self.sleep_level = VLLM_SLEEP_LEVEL + + def _init_zeromq(self) -> str: + tensor_parallel_size = self.config.tensor_model_parallel_size + + # single node: ipc, multi nodes: tcp + local_world_size = int(os.environ["RAY_LOCAL_WORLD_SIZE"]) + socket_type = "ipc" if tensor_parallel_size <= local_world_size else "tcp" + + # File lock to prevent multiple workers listen to same port + with FileLock(f"/tmp/verl_vllm_zmq_{getpass.getuser()}.lock"): + if socket_type == "ipc": + pid = os.getpid() + address = f"ipc:///tmp/verl_vllm_zmq_{pid}_{getpass.getuser()}.ipc" + else: + ip, port = self._get_free_port() + address = f"tcp://{ip}:{port}" + context = zmq.asyncio.Context() + self.socket = context.socket(zmq.REP) + self.socket.bind(address) + + loop = asyncio.get_running_loop() + self.zmq_loop_task = loop.create_task(self._loop_forever()) + + return address + + def _get_free_port(self): + ip = ray.util.get_node_ip_address() + with socket.socket() as sock: + sock.bind(("", 0)) + port = sock.getsockname()[1] + return ip, port + + async def _loop_forever(self): + while True: + try: + message = await self.socket.recv() + method, args, kwargs = pickle.loads(message) + result = await self._execute_method(method, *args, **kwargs) + await self.socket.send(pickle.dumps(result)) + except Exception as e: + logger.exception(f"vLLMAsyncRollout _loop_forever error: {e}") + os._exit(-1) + + def _init_worker(self, all_kwargs: list[dict[str, Any]]): + """Initialize worker engine.""" + if not torch.distributed.is_initialized(): + initialize_global_process_group_ray() + all_kwargs[0]["rank"] = int(os.environ["RANK"]) + device_name = "NPU" if is_npu_available else "GPU" + all_kwargs[0]["local_rank"] = ( + 0 + if not ray_noset_visible_devices() + else int(ray.get_runtime_context().get_accelerator_ids()[device_name][0]) + ) + self.vllm_config = all_kwargs[0]["vllm_config"] + if self.lora_config: + self.vllm_config.lora_config = LoRAConfig(**self.lora_config) + self.inference_engine = WorkerWrapperBase(vllm_config=self.vllm_config) + self.inference_engine.init_worker(all_kwargs) + + def _load_model(self, *args, **kwargs): + self.inference_engine.load_model(*args, **kwargs) + _monkey_patch_compute_logits(self.inference_engine.worker.model_runner.model, len(self.tokenizer)) + + async def _execute_method(self, method: str | bytes, *args, **kwargs): + if method == "init_worker": + return self._init_worker(*args, **kwargs) + elif method == "load_model": + return self._load_model(*args, **kwargs) + elif method == "sleep" or method == "wake_up": + raise ValueError("wake_up and sleep should not be called through ZeroMQ") + else: + return self.inference_engine.execute_method(method, *args, **kwargs) + + async def resume(self, tags: list[str]): + """Resume rollout weights or kv cache in GPU memory. + + Args: + tags: weights or kv_cache. + """ + if self.config.free_cache_engine: + self.inference_engine.wake_up(tags=tags) + + async def release(self): + """Release weights and kv cache in GPU memory.""" + if self.config.free_cache_engine: + self.inference_engine.sleep(level=self.sleep_level) + + async def update_weights(self, weights: Generator[tuple[str, torch.Tensor], None, None], **kwargs): + """Update the weights of the rollout model. + + Args: + weights: A generator that yields the name of the weight tensor and the tensor itself. + """ + peft_config, base_sync_done = kwargs.get("peft_config", None), kwargs.get("base_sync_done", False) + if peft_config and base_sync_done: + lora_int_id = int(time.time_ns() % 0x7FFFFFFF) + lora_reqest = TensorLoRARequest( + lora_name=f"{lora_int_id}", + lora_int_id=lora_int_id, + lora_path="simon_lora_path", + peft_config=asdict(peft_config), + lora_tensors=weights, + ) + self.inference_engine.worker.add_lora(lora_reqest) + logger.info(f"vLLM load weights, loaded_params: {len(weights)}") + else: + from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader + + model = self.inference_engine.worker.model_runner.model + patch_vllm_moe_model_weight_loader(model) + model.load_weights(weights) + + def generate_sequences(self, prompts: DataProto) -> DataProto: + """Batch generate sequences in sync mode.""" + raise NotImplementedError + + # ==================== server mode public methods ==================== + + def get_zeromq_address(self): + return self.address diff --git a/e2c/verl/verl/workers/sharding_manager/__init__.py b/e2c/verl/verl/workers/sharding_manager/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce90c5eb352d85c59105c0dc85b5f1dd576f095 --- /dev/null +++ b/e2c/verl/verl/workers/sharding_manager/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/e2c/verl/verl/workers/sharding_manager/base.py b/e2c/verl/verl/workers/sharding_manager/base.py new file mode 100644 index 0000000000000000000000000000000000000000..59537be64efcb0b580385f12ba820f837130e197 --- /dev/null +++ b/e2c/verl/verl/workers/sharding_manager/base.py @@ -0,0 +1,35 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Sharding manager to implement HybridEngine +""" + +from verl import DataProto + + +class BaseShardingManager: + def __init__(self): + self.timing = {} + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + pass + + def preprocess_data(self, data: DataProto) -> DataProto: + return data + + def postprocess_data(self, data: DataProto) -> DataProto: + return data diff --git a/e2c/verl/verl/workers/sharding_manager/fsdp_sglang.py b/e2c/verl/verl/workers/sharding_manager/fsdp_sglang.py new file mode 100644 index 0000000000000000000000000000000000000000..b6433bf8163c1f1b50e0f1712440fa9fa4e95569 --- /dev/null +++ b/e2c/verl/verl/workers/sharding_manager/fsdp_sglang.py @@ -0,0 +1,223 @@ +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging +import os + +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.weight_sync.utils import update_weights as sgl_update_weights +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.fsdp.api import FullStateDictConfig, ShardedStateDictConfig, StateDictType +from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP + +from verl import DataProto +from verl.protocol import all_gather_data_proto +from verl.utils.device import get_device_id, get_torch_device, set_expandable_segments +from verl.utils.fsdp_utils import fsdp_version, load_fsdp_model_to_gpu, offload_fsdp_model_to_cpu +from verl.utils.import_utils import deprecated +from verl.utils.memory_utils import aggressive_empty_cache +from verl.utils.model import convert_weight_keys +from verl.utils.profiler import GPUMemoryLogger, log_gpu_memory_usage, simple_timer +from verl.utils.torch_functional import check_device_is_available +from verl.workers.rollout.sglang_rollout.utils import get_named_tensor_buckets + +from .base import BaseShardingManager + +# from vllm.distributed import parallel_state as sglang_ps +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +@deprecated() +class FSDPSGLangShardingManager(BaseShardingManager): + @check_device_is_available() + def __init__( + self, + module: FSDP, + inference_engine: Engine, + model_config, + rollout_config, + full_params: bool = False, + device_mesh: DeviceMesh = None, + offload_param: bool = False, + multi_stage_wake_up: bool = False, + ): + self.module = module + self.inference_engine = inference_engine + self.model_config = model_config + self.rollout_config = rollout_config + self.device_mesh = device_mesh + self.offload_param = offload_param + self.multi_stage_wake_up = multi_stage_wake_up + + # Full params + self.full_params = full_params + if full_params and fsdp_version(self.module) == 1: + FSDP.set_state_dict_type( + self.module, state_dict_type=StateDictType.FULL_STATE_DICT, state_dict_config=FullStateDictConfig() + ) + elif fsdp_version(self.module) == 1: + FSDP.set_state_dict_type( + self.module, + state_dict_type=StateDictType.SHARDED_STATE_DICT, + state_dict_config=ShardedStateDictConfig(), + ) + + self.tp_size = self.device_mesh["infer_tp"].size() + self.tp_rank = self.device_mesh["infer_tp"].get_local_rank() + + # Note that torch_random_states may be different on each dp rank + self.torch_random_states = get_torch_device().get_rng_state() + # get a random rng states + if self.device_mesh is not None: + gen_dp_rank = self.device_mesh["dp"].get_local_rank() + get_torch_device().manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + else: + self.gen_random_states = None + + @GPUMemoryLogger(role="FSDPSGLangShardingManager enter", logger=logger) + def __enter__(self): + self.timing = {} + with simple_timer("reshard", self.timing): + loop = asyncio.get_event_loop() + loop.run_until_complete(self.wake_up()) + + @GPUMemoryLogger(role="FSDPSGLangShardingManager exit", logger=logger) + def __exit__(self, exc_type, exc_value, traceback): + loop = asyncio.get_event_loop() + loop.run_until_complete(self.sleep()) + + async def update_weights(self, params): + named_tensors = [(k, v) for k, v in params.items()] + update_weights_bucket_bytes = int(self.rollout_config.update_weights_bucket_megabytes) << 20 + for params_batch in get_named_tensor_buckets(named_tensors, update_weights_bucket_bytes): + await sgl_update_weights( + engine=self.inference_engine, + params_batch=params_batch, + device_mesh_key="infer_tp", + device_mesh=self.device_mesh, + ) + + if self.device_mesh["infer_tp"].get_local_rank() == 0: + await self.inference_engine.flush_cache() + + async def release_memory(self): + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.rollout_config.free_cache_engine: + if self.multi_stage_wake_up: + await self.inference_engine.release_memory_occupation(tags=["kv_cache", "weights"]) + else: + await self.inference_engine.release_memory_occupation() + log_gpu_memory_usage("After release memory occupation in sharding manager", logger=logger) + + @GPUMemoryLogger(role="FSDPSGLangShardingManager enter", logger=logger) + async def wake_up(self): + aggressive_empty_cache(force_sync=True) + + log_gpu_memory_usage("Before state_dict() in sharding manager memory", logger=logger) + if self.offload_param: + load_fsdp_model_to_gpu(self.module) + params = self.module.state_dict() + log_gpu_memory_usage("After state_dict() in sharding manager memory", logger=logger) + device = get_device_id() # used when fsdp2 set cpu_offload_policy + params = { + k: v.to(device, non_blocking=True) if fsdp_version(self.module) == 2 else v for k, v in params.items() + } + + # convert weight keys to match the model config + params = convert_weight_keys(params, getattr(self.module, "_fsdp_wrapped_module", self.module)) + + if self.offload_param: + offload_fsdp_model_to_cpu(self.module) + + log_gpu_memory_usage("After offload_param in sharding manager memory", logger=logger) + + # sglang need to set _set_allocator_settings to False + logger.debug("fsdp sglang sharding_manager _set_allocator_settings to False") + # Note(chenyang): SGLang is using torch memory pool to manage memory + # which is incompatible with expandable segments + set_expandable_segments(False) + + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.rollout_config.free_cache_engine: + if self.multi_stage_wake_up: + await self.inference_engine.resume_memory_occupation(tags=["weights"]) + log_gpu_memory_usage("Before resume SGLang weights in sharding manager", logger=logger) + else: + await self.inference_engine.resume_memory_occupation() + log_gpu_memory_usage("Before resume SGLang weights + kv_cache in sharding manager", logger=logger) + + # Copy, not share memory + await self.update_weights(params) + log_gpu_memory_usage("After sync model weights in sharding manager", logger=logger) + + del params + aggressive_empty_cache(force_sync=True) + log_gpu_memory_usage("After del state_dict and empty_cache in sharding manager", logger=logger) + + if ( + self.multi_stage_wake_up + and self.rollout_config.free_cache_engine + and self.device_mesh["infer_tp"].get_local_rank() == 0 + ): + await self.inference_engine.resume_memory_occupation(tags=["kv_cache"]) + log_gpu_memory_usage("After resume SGLang kv_cache in sharding manager", logger=logger) + + # important: need to manually set the random states of each tp to be identical. + if self.device_mesh is not None: + self.torch_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.gen_random_states) + + @GPUMemoryLogger(role="FSDPSGLangShardingManager exit", logger=logger) + async def sleep(self): + if self.rollout_config.free_cache_engine: + log_gpu_memory_usage("Before SGLang offload in sharding manager", logger=logger) + await self.release_memory() + log_gpu_memory_usage("After SGLang offload in sharding manager", logger=logger) + + self.module.train() + + # add empty cache after each compute + aggressive_empty_cache(force_sync=True) + + # always set _set_allocator_settings to True when using sglang + # it is required by fsdp2 to avoid oom + logger.debug("fsdp sglang sharding_manager _set_allocator_settings to True") + set_expandable_segments(True) + + # restore random states + if self.device_mesh is not None: + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + + def preprocess_data(self, data: DataProto) -> DataProto: + """All gather across tp group to make each rank has identical input.""" + if self.tp_size == 1: + return data + + # TODO: Current impl doesn't consider FSDP with torch micro-dp + group = self.device_mesh["infer_tp"].get_group() + + all_gather_data_proto(data=data, process_group=group) + return data + + def postprocess_data(self, data: DataProto) -> DataProto: + """Get chunk data of this tp rank since we do all gather in preprocess.""" + if self.tp_size == 1: + return data + + return data.chunk(chunks=self.tp_size)[self.tp_rank] diff --git a/e2c/verl/verl/workers/sharding_manager/fsdp_ulysses.py b/e2c/verl/verl/workers/sharding_manager/fsdp_ulysses.py new file mode 100644 index 0000000000000000000000000000000000000000..39ccb77ccdd89cdfe886974702d98d9ae608532a --- /dev/null +++ b/e2c/verl/verl/workers/sharding_manager/fsdp_ulysses.py @@ -0,0 +1,72 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Contains a resharding manager that binds weights from FSDP zero3 to XPerfGPT +""" + +from torch.distributed.device_mesh import DeviceMesh + +from verl import DataProto +from verl.protocol import all_gather_data_proto +from verl.utils.ulysses import get_ulysses_sequence_parallel_group, set_ulysses_sequence_parallel_group + +from .base import BaseShardingManager + + +class FSDPUlyssesShardingManager(BaseShardingManager): + """ + Sharding manager to support data resharding when using FSDP + Ulysses + """ + + def __init__(self, device_mesh: DeviceMesh): + super().__init__() + self.device_mesh = device_mesh + self.seed_offset = 12345 + + def __enter__(self): + if self.device_mesh is not None: + # We have a global SP group + # so we have to change to use model-specific sp group + self.prev_sp_group = get_ulysses_sequence_parallel_group() + set_ulysses_sequence_parallel_group(self.device_mesh["sp"].get_group()) + # TODO: check how to set seed for each model + + def __exit__(self, exc_type, exc_value, traceback): + # restore random states + if self.device_mesh is not None: + # revert to previous sp group + set_ulysses_sequence_parallel_group(self.prev_sp_group) + # TODO: check how to set seed for each model + + def preprocess_data(self, data: DataProto) -> DataProto: + """ + AllGather data from sp region + This is because the data is first sharded along the FSDP dimension as we utilize the DP_COMPUTE + In Ulysses, we need to make sure the same data is used across a SP group + """ + if self.device_mesh is not None: + group = self.device_mesh["sp"].get_group() + + all_gather_data_proto(data=data, process_group=group) + return data + + def postprocess_data(self, data: DataProto) -> DataProto: + """ + Split the data to follow FSDP partition + """ + if self.device_mesh is not None: + sp_size = self.device_mesh["sp"].size() + sp_rank = self.device_mesh["sp"].get_local_rank() + data = data.chunk(chunks=sp_size)[sp_rank] + return data diff --git a/e2c/verl/verl/workers/sharding_manager/fsdp_vllm.py b/e2c/verl/verl/workers/sharding_manager/fsdp_vllm.py new file mode 100644 index 0000000000000000000000000000000000000000..fb6d7469512f801faea84f12af9ad4b2693cf134 --- /dev/null +++ b/e2c/verl/verl/workers/sharding_manager/fsdp_vllm.py @@ -0,0 +1,355 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import logging +import os +import time +from collections import OrderedDict + +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.fsdp.api import FullStateDictConfig, ShardedStateDictConfig, StateDictType +from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP + +try: + # for torch 2.5+ + from torch.distributed.tensor import DTensor +except ImportError: + from torch.distributed._tensor import DTensor + +from dataclasses import asdict + +from verl import DataProto +from verl.protocol import all_gather_data_proto +from verl.third_party.vllm import LLM, VLLM_SLEEP_LEVEL +from verl.third_party.vllm import parallel_state as vllm_ps +from verl.utils.device import get_device_id, get_device_name, get_torch_device, set_expandable_segments +from verl.utils.fsdp_utils import ( + fsdp_version, + layered_summon_lora_params, + load_fsdp_model_to_gpu, + offload_fsdp_model_to_cpu, +) +from verl.utils.import_utils import deprecated +from verl.utils.model import check_exclude_modules, check_target_modules, convert_weight_keys +from verl.utils.profiler import GPUMemoryLogger, log_gpu_memory_usage, simple_timer +from verl.utils.torch_functional import check_device_is_available +from verl.utils.vllm import TensorLoRARequest, VLLMHijack, is_version_ge + +from .base import BaseShardingManager + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +@deprecated() +class FSDPVLLMShardingManager(BaseShardingManager): + """Sharding manager for FSDP models with vLLM inference engine integration. + + Manages parameter synchronization between FSDP training models and vLLM + inference engines, handling both full parameters and LoRA adapters with + efficient memory management and device placement. + """ + + @check_device_is_available() + def __init__( + self, + module: FSDP, + inference_engine: LLM, + model_config, + rollout_config, + full_params: bool = False, + device_mesh: DeviceMesh = None, + offload_param: bool = False, + load_format: str = "dummy_hf", + layered_summon: bool = True, + ): + self.module = module + # For AsyncLLM, inference_engine and model_runner are defer initialized in vLLMAsyncRollout.load_model + self.inference_engine = inference_engine + # self.model_runner = inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner if + # inference_engine else None + + self.model_runner = ( + self.inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner + if self.inference_engine + else None + ) + + self.model_config = model_config + self.rollout_config = rollout_config + self.device_mesh = device_mesh + self.offload_param = offload_param + self.load_format = load_format + self.layered_summon = layered_summon + + # Full params + self.full_params = full_params + if full_params and fsdp_version(self.module) == 1: + FSDP.set_state_dict_type( + self.module, state_dict_type=StateDictType.FULL_STATE_DICT, state_dict_config=FullStateDictConfig() + ) + elif fsdp_version(self.module) == 1: + FSDP.set_state_dict_type( + self.module, + state_dict_type=StateDictType.SHARDED_STATE_DICT, + state_dict_config=ShardedStateDictConfig(), + ) + + self.tp_size = self.device_mesh["infer_tp"].size() + self.tp_rank = self.device_mesh["infer_tp"].get_local_rank() + + # Note that torch_random_states may be different on each dp rank + self.torch_random_states = get_torch_device().get_rng_state() + # get a random rng states + if self.device_mesh is not None: + gen_dp_rank = self.device_mesh["dp"].get_local_rank() + get_torch_device().manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + else: + self.gen_random_states = None + + self.base_sync_done: bool = "dummy" not in load_format + if is_version_ge(pkg="vllm", minver="0.7.3"): + VLLMHijack.hijack() + + @GPUMemoryLogger(role="fsdp vllm sharding_manager", logger=logger) + def __enter__(self): + def __collect_lora_params() -> OrderedDict: + """ + collect lora params or full params if base model is not ready in vllm + work with if isinstance(self.module._fsdp_wrapped_module, PeftModel) + """ + from peft.utils.save_and_load import get_peft_model_state_dict + + lora_params = OrderedDict() + peft_model = getattr(self.module, "_fsdp_wrapped_module", self.module) + if fsdp_version(self.module) > 0: + if self.layered_summon: + if not self.base_sync_done: + raise ValueError( + "To use layered_summon, you must make sure base-model is preloaded in vllm, e.g. let " + "rollout.load_format=safetensors" + ) + lora_params = layered_summon_lora_params(self.module) + else: + with FSDP.summon_full_params(self.module, writeback=False): + if self.base_sync_done: + lora_params = get_peft_model_state_dict(peft_model) + lora_params = { + name: param.full_tensor().detach().cpu() + if hasattr(param, "full_tensor") + else param.detach().cpu() + for name, param in lora_params.items() + } + else: + model = peft_model.base_model.model + orig_dev = "cpu" if "cpu" in str(next(model.parameters()).device) else get_device_name() + model = model.to("cpu") + for name, param in model.state_dict().items(): + if any(x in name for x in ["_flat_param", "lora_"]): + continue + name = name.replace("_fsdp_wrapped_module.", "").replace(".base_layer", "") + lora_params[name] = ( + param.full_tensor().detach().cpu() + if hasattr(param, "full_tensor") + else param.detach().cpu() + ) + model = model.to(orig_dev) + get_torch_device().empty_cache() + else: + if self.base_sync_done: + lora_params = get_peft_model_state_dict(peft_model) + else: + model = peft_model.base_model.model + orig_dev = "cpu" if "cpu" in str(next(model.parameters()).device) else get_device_name() + model = model.to("cpu") + for name, param in model.state_dict().items(): + if any(x in name for x in ["_flat_param", "lora_"]): + continue + name = name.replace("_fsdp_wrapped_module.", "").replace(".base_layer", "") + lora_params[name] = param.detach().cpu() + model = model.to(orig_dev) + return lora_params + + # NOTE: Basically, we only need `get_torch_device().empty_cache()` before vllm wake_up and + # after vllm sleep, since vllm has its own caching memory allocator CuMemAllocator. + # Out of vllm scope, we should avoid empty cache to let pytorch using caching memory + # to speed up memory allocations. + # + # pytorch: https://pytorch.org/docs/stable/notes/cuda.html#memory-management + # vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/device_allocator/cumem.py#L103 + self.timing = {} + with simple_timer("reshard", self.timing): + get_torch_device().empty_cache() + + log_gpu_memory_usage("Before state_dict() in sharding manager memory", logger=logger) + if self.offload_param: + load_fsdp_model_to_gpu(self.module) + + peft_config = None + peft_model = getattr(self.module, "_fsdp_wrapped_module", self.module) + if hasattr(peft_model, "peft_config"): + peft_config = peft_model.peft_config.get("default", None) + params = __collect_lora_params() + else: + params = self.module.state_dict() + params = convert_weight_keys(params, getattr(self.module, "_fsdp_wrapped_module", self.module)) + + if self.offload_param: + offload_fsdp_model_to_cpu(self.module) + log_gpu_memory_usage("After state_dict() in sharding manager memory", logger=logger) + + # vllm need to set _set_allocator_settings to False + logger.debug("fsdp vllm sharding_manager _set_allocator_settings to False") + set_expandable_segments(False) + + if self.rollout_config.free_cache_engine: + if "tags" in inspect.signature(self.inference_engine.wake_up).parameters: + self.inference_engine.wake_up(tags=["weights"]) + else: + self.inference_engine.wake_up() + + # update model params + self.update_params(params, peft_config=peft_config) + log_gpu_memory_usage("After sync model weights in sharding manager", logger=logger) + del params + get_torch_device().empty_cache() + + if ( + self.rollout_config.free_cache_engine + and "tags" in inspect.signature(self.inference_engine.wake_up).parameters + ): + self.inference_engine.wake_up(tags=["kv_cache"]) + + log_gpu_memory_usage("After del state_dict and empty_cache in sharding manager", logger=logger) + + # important: need to manually set the random states of each tp to be identical. + if self.device_mesh is not None: + self.torch_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.gen_random_states) + + @GPUMemoryLogger(role="fsdp vllm sharding_manager", logger=logger) + def __exit__(self, exc_type, exc_value, traceback): + if self.rollout_config.free_cache_engine: + self.inference_engine.sleep(level=VLLM_SLEEP_LEVEL) + + self.module.train() + + # add empty cache after each compute + get_torch_device().empty_cache() + + # _set_allocator_settings to True is required by fsdp2 to avoid oom + logger.debug("fsdp vllm sharding_manager _set_allocator_settings to True") + set_expandable_segments(True) + + # restore random states + if self.device_mesh is not None: + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + + @GPUMemoryLogger(role="fsdp vllm sharding_manager", logger=logger) + def preprocess_data(self, data: DataProto) -> DataProto: + """All gather across tp group to make each rank has identical input.""" + if self.tp_size == 1: + return data + + # TODO: Current impl doesn't consider FSDP with torch micro-dp + group = vllm_ps.get_tensor_model_parallel_group().device_group + + all_gather_data_proto(data=data, process_group=group) + return data + + @GPUMemoryLogger(role="fsdp vllm sharding_manager", logger=logger) + def postprocess_data(self, data: DataProto) -> DataProto: + """Get chunk data of this tp rank since we do all gather in preprocess.""" + if self.tp_size == 1: + return data + + return data.chunk(chunks=self.tp_size)[self.tp_rank] + + def update_params(self, updated_params, peft_config=None): + """Update model parameters in the vLLM inference engine. + + Synchronizes parameters from the FSDP training model to the vLLM inference + engine, handling both full model parameters and LoRA adapters with proper + device placement and memory management. + + Args: + updated_params (dict): Dictionary of parameter names to tensor values. + peft_config (optional): PEFT configuration for LoRA adapters. + """ + model = self.model_runner.model + if peft_config: + if self.base_sync_done: + lora_int_id = int(time.time_ns() % 0x7FFFFFFF) + lora_reqest = TensorLoRARequest( + lora_name=f"{lora_int_id}", + lora_int_id=lora_int_id, + lora_path="simon_lora_path", + peft_config=asdict(peft_config), + lora_tensors=updated_params, + ) + self.inference_engine.llm_engine.add_lora(lora_reqest) + logger.info(f"vLLM load weights, loaded_params: {len(updated_params)}") + return + else: + + def replace_lora_wrapper(k): + """Replace LoRA parameter keys with base layer equivalents. + + Transforms LoRA parameter names to their corresponding base layer + names for proper weight loading in vLLM when base model sync is not done. + + Args: + k (str): Original parameter key name. + + Returns: + str: Transformed parameter key for base layer. + """ + stacked_params = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"] + if k.endswith(".weight"): + module_k = k[: -len(".weight")] + if check_exclude_modules(peft_config, module_k): + return k + elif any([module_k.endswith(s) for s in stacked_params]) or check_target_modules( + peft_config, module_k + ): + return f"{module_k}.base_layer.weight" + if k.endswith(".bias"): + module_k = k[: -len(".bias")] + if check_exclude_modules(peft_config, module_k): + return k + elif any([module_k.endswith(s) for s in stacked_params]) or check_target_modules( + peft_config, module_k + ): + return f"{module_k}.base_layer.bias" + return k + + updated_params = {replace_lora_wrapper(k): v for k, v in updated_params.items()} + + from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader + + patch_vllm_moe_model_weight_loader(model) + device = get_device_id() # used when fsdp2 set cpu_offload_policy + loaded_params = model.load_weights( + ( + (name, param.to(device, non_blocking=True).full_tensor() if isinstance(param, DTensor) else param) + for name, param in updated_params.items() + ) + ) + + self.base_sync_done = True + logger.info(f"vLLM load weights, loaded_params: {len(loaded_params) if loaded_params else -1}") diff --git a/e2c/verl/verl/workers/sharding_manager/megatron_sglang.py b/e2c/verl/verl/workers/sharding_manager/megatron_sglang.py new file mode 100644 index 0000000000000000000000000000000000000000..b945b858c71f5c4fd9e7d096e6ae6fe779fea595 --- /dev/null +++ b/e2c/verl/verl/workers/sharding_manager/megatron_sglang.py @@ -0,0 +1,227 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This file contains a Megatron style Hybrid Engine that shares the weights of the actor with the inference engine. +""" + +import asyncio +import logging +import os + +from omegaconf import DictConfig +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.weight_sync.utils import update_weights as sgl_update_weights +from torch import nn +from torch.distributed.device_mesh import DeviceMesh + +from verl.protocol import DataProto, all_gather_data_proto +from verl.utils.device import get_torch_device, set_expandable_segments +from verl.utils.import_utils import deprecated +from verl.utils.megatron_utils import ( + load_megatron_model_to_gpu, + offload_megatron_model_to_cpu, + per_tensor_generator, +) +from verl.utils.memory_utils import aggressive_empty_cache +from verl.utils.profiler import GPUMemoryLogger, log_gpu_memory_usage, simple_timer +from verl.workers.rollout.sglang_rollout.utils import get_named_tensor_buckets + +from .base import BaseShardingManager + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_PPO_LOGGING_LEVEL", "WARN")) + + +""" +Megatron Hybrid Engine: +- During training, only the current pp stage holds the parameters +- Before inference, broadcast the parameters of the current pp rank to all other pp ranks (all pp ranks holds all + the parameters) +- Bind the parameters to the inference engine +- Do inference in tp. pp is treated as additional dp +- After inference, all the parameters that doesn't belong to this pp rank is freed. +""" + + +@deprecated() +class MegatronSGLangShardingManager(BaseShardingManager): + """A sharding manager for Megatron-style training & inference with SGLang. + + This class manages the sharding of model parameters between training and inference + phases in a Megatron-style parallel setup. It handles: + - Loading/offloading parameters between CPU/GPU + - Updating inference engine weights + - Managing random states for reproducibility + - Data preprocessing for distributed inference + + Args: + actor_module (nn.ModuleList): The actor model modules + inference_engine (Engine): The SGLang inference engine + model_config: Configuration for the actor's model + rollout_config: Configuration for rollout generation + transformer_config: Transformer-specific configuration + layer_name_mapping: Mapping between layer names and parameters + weight_converter: Utility for converting weights between formats + device_mesh (DeviceMesh | None): PyTorch device mesh for distributed training + offload_param (bool): Whether to offload parameters to CPU when not in use + """ + + def __init__( + self, + actor_module: nn.ModuleList, + inference_engine: Engine, + model_config: DictConfig, + rollout_config: DictConfig, + transformer_config, + layer_name_mapping, + weight_converter, + device_mesh: DeviceMesh | None = None, + offload_param: bool = False, + bridge=None, + ): + self.actor_module = actor_module + self.inference_engine = inference_engine + self.model_config = model_config + self.rollout_config = rollout_config + self.transformer_config = transformer_config + self.layer_name_mapping = layer_name_mapping + self.weight_converter = weight_converter + self.device_mesh = device_mesh + self.bridge = bridge + self.offload_param = offload_param + + if self.device_mesh is not None: + self.infer_tp_size = self.device_mesh["infer_tp"].mesh.size()[0] + else: + self.infer_tp_size = self.inference_engine._tp_size + + # Note that torch_random_states may be different on each dp rank + self.torch_random_states = get_torch_device().get_rng_state() + # get a random rng states + if self.device_mesh is not None: + gen_dp_rank = self.device_mesh["dp"].get_local_rank() + get_torch_device().manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + else: + self.gen_random_states = None + + @GPUMemoryLogger(role="MegatronSGLangShardingManager enter", logger=logger) + def __enter__(self): + self.timing = {} + with simple_timer("reshard", self.timing): + loop = asyncio.get_event_loop() + loop.run_until_complete(self.wake_up()) + + @GPUMemoryLogger(role="MegatronSGLangShardingManager exit", logger=logger) + def __exit__(self, exc_type, exc_value, traceback): + loop = asyncio.get_event_loop() + loop.run_until_complete(self.sleep()) + + async def update_weights(self, params): + """ + Update model weights using tensor buckets, similar to THUDM/slime's implementation. + + Notes: + - For the best performance of `rebuild_cuda_tensor`, it is recommended to: + 1. Enable `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES`. + 2. Manually set `CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7` + when using Tensor Parallelism (TP >= 8). + - See reference implementations in SLIME: + - Main logic: https://github.com/THUDM/slime/blob/fb7605cc5fb09af0f9369d37f7192f12bddee577/slime/ray/ppo_actor.py#L452 + - runtime envs: https://github.com/THUDM/slime/blob/fb7605cc5fb09af0f9369d37f7192f12bddee577/slime/ray/ppo_actor.py#L39 + """ + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.rollout_config.free_cache_engine: + await self.inference_engine.resume_memory_occupation() + named_tensors = params + + update_weights_bucket_bytes = int(self.rollout_config.update_weights_bucket_megabytes) << 20 + for params_batch in get_named_tensor_buckets(named_tensors, update_weights_bucket_bytes): + await sgl_update_weights( + engine=self.inference_engine, + params_batch=params_batch, + device_mesh_key="infer_tp", + device_mesh=self.device_mesh, + ) + + if self.device_mesh["infer_tp"].get_local_rank() == 0: + await self.inference_engine.flush_cache() + + async def release_memory(self): + if self.device_mesh["infer_tp"].get_local_rank() == 0 and self.rollout_config.free_cache_engine: + await self.inference_engine.release_memory_occupation() + + @GPUMemoryLogger(role="MegatronSGLangShardingManager enter", logger=logger) + async def wake_up(self): + aggressive_empty_cache(force_sync=True) + + if self.offload_param: + load_megatron_model_to_gpu(self.actor_module, load_grad=False) + if self.bridge is not None: + per_tensor_param = self.bridge.export_weights(self.actor_module) + else: + per_tensor_param = per_tensor_generator( + self.actor_module, + self.model_config, + self.weight_converter, + self.transformer_config, + self.layer_name_mapping, + ) + + set_expandable_segments(False) + + await self.update_weights(per_tensor_param) + if self.offload_param: + offload_megatron_model_to_cpu(self.actor_module) + aggressive_empty_cache(force_sync=True) + # important: need to manually set the random states of each tp to be identical. + if self.device_mesh is not None: + self.torch_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.gen_random_states) + + @GPUMemoryLogger(role="MegatronSGLangShardingManager exit", logger=logger) + async def sleep(self): + if self.rollout_config.free_cache_engine: + log_gpu_memory_usage("Before SGLang offload in sharding manager", logger=logger) + await self.release_memory() + log_gpu_memory_usage("After SGLang offload in sharding manager", logger=logger) + + for model in self.actor_module: + model.train() + # add empty cache after each compute + aggressive_empty_cache(force_sync=True) + + set_expandable_segments(True) + + # restore random states + if self.device_mesh is not None: + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + + @GPUMemoryLogger(role="megatron sglang sharding_manager", logger=logger) + def preprocess_data(self, data: DataProto) -> DataProto: + # DP_COMPUTE_PROTO: all training ranks are dp, the same as fsdp + if self.infer_tp_size == 1: + return data + all_gather_data_proto(data, self.device_mesh["infer_tp"].get_group()) + return data + + @GPUMemoryLogger(role="megatron sglang sharding_manager", logger=logger) + def postprocess_data(self, data: DataProto) -> DataProto: + # DP_COMPUTE_PROTO: all training ranks are dp, the same as fsdp + if self.infer_tp_size == 1: + return data + return data.chunk(chunks=self.infer_tp_size)[self.device_mesh["infer_tp"].get_local_rank()] diff --git a/e2c/verl/verl/workers/sharding_manager/megatron_vllm.py b/e2c/verl/verl/workers/sharding_manager/megatron_vllm.py new file mode 100644 index 0000000000000000000000000000000000000000..6adc89c09854ab92fabb0db809208e1fc02d17fa --- /dev/null +++ b/e2c/verl/verl/workers/sharding_manager/megatron_vllm.py @@ -0,0 +1,227 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This file contains a Megatron style Hybrid Engine that shares the weights of the actor with the inference engine. +""" + +import inspect +import logging +import os + +import torch +import torch.distributed +from megatron.core import parallel_state as mpu +from omegaconf import DictConfig +from torch import nn + +from verl import DataProto +from verl.models.mcore.weight_converter import McoreToHFWeightConverterBase +from verl.protocol import all_gather_data_proto +from verl.third_party.vllm import LLM, VLLM_SLEEP_LEVEL +from verl.third_party.vllm import parallel_state as vllm_ps +from verl.utils.device import get_torch_device, set_expandable_segments +from verl.utils.import_utils import deprecated +from verl.utils.megatron_utils import load_megatron_model_to_gpu, offload_megatron_model_to_cpu, per_tensor_generator +from verl.utils.memory_utils import aggressive_empty_cache +from verl.utils.profiler import GPUMemoryLogger, log_gpu_memory_usage +from verl.utils.profiler.performance import simple_timer +from verl.utils.torch_functional import check_device_is_available + +from .base import BaseShardingManager + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) + + +""" +Megatron Hybrid Engine: +- During training, only the current pp stage holds the parameters +- Before inference, broadcast the parameters of the current pp rank + to all other pp ranks (all pp ranks holds all the parameters) +- Bind the parameters to the inference engine +- Do inference in tp. pp is treated as additional dp +- After inference, all the parameters that doesn't belong to this pp rank is freed. +""" + + +@deprecated() +class MegatronVLLMShardingManager(BaseShardingManager): + """A sharding manager that bridges Megatron-LM training with vLLM inference. + + This class handles the parameter sharding and communication between: + - Megatron-LM's tensor/expert parallel training setup + - vLLM's tensor parallel inference setup + + Key responsibilities: + - Manages parameter broadcasting between training and inference configurations + - Handles weight conversion between Megatron and HuggingFace formats + - Coordinates memory management between training and inference phases + - Maintains random state consistency across different parallel groups + + Args: + actor_module (nn.ModuleList): The Megatron-LM model being trained + inference_engine (LLM): The vLLM inference engine + model_config: Configuration for the actor's model + transformer_config: Transformer-specific configuration for the model + rollout_config: Configuration for rollout + layer_name_mapping: Mapping between Megatron and HF layer names + weight_converter (McoreToHFWeightConverterBase): Converts weights between formats + device_mesh: Device mesh for parallel operations + offload_param (bool): Whether to offload parameters when not in use + """ + + @check_device_is_available() + def __init__( + self, + actor_module: nn.ModuleList, + inference_engine: LLM, + model_config: DictConfig, + transformer_config, + rollout_config: DictConfig, + layer_name_mapping, + weight_converter: McoreToHFWeightConverterBase, + device_mesh, + offload_param: bool = True, + bridge=None, + ): + self.actor_module = actor_module + self.inference_engine = inference_engine + self.offload_param = offload_param + + # For AsyncLLM, inference_engine and model_runner are defer initialized in vLLMAsyncRollout.load_model + self.model_runner = ( + self.inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner + if self.inference_engine + else None + ) + + self.model_config = model_config + self.transformer_config = transformer_config + self.rollout_config = rollout_config + self.layer_name_mapping = layer_name_mapping + self.weight_converter = weight_converter + self.bridge = bridge + # initialize groups for vllm inference + self.rank = torch.distributed.get_rank() + self.world_size = torch.distributed.get_world_size() + + self.device_mesh = device_mesh + self.infer_tp_size = self.device_mesh["infer_tp"].size() + self.infer_tp_rank = self.device_mesh["infer_tp"].get_local_rank() + + self.train_tp_size = mpu.get_tensor_model_parallel_world_size() + self.train_tp_rank = mpu.get_tensor_model_parallel_rank() + self.train_tp_group = mpu.get_tensor_model_parallel_group() + self.train_ep_size = mpu.get_expert_model_parallel_world_size() + self.train_ep_rank = mpu.get_expert_model_parallel_rank() + self.train_ep_group = mpu.get_expert_model_parallel_group() + self.train_etp_size = mpu.get_expert_tensor_parallel_world_size() + self.train_etp_rank = mpu.get_expert_tensor_parallel_rank() + self.train_etp_group = mpu.get_expert_tensor_parallel_group() + self.need_tp_reshard = self.train_tp_size != self.infer_tp_size + self.train_tp_larger = self.train_tp_size > self.infer_tp_size + + self.torch_random_states = get_torch_device().get_rng_state() + if self.device_mesh is not None: + gen_dp_rank = self.device_mesh["dp"].get_local_rank() + get_torch_device().manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + else: + self.gen_random_states = None + + @GPUMemoryLogger(role="megatron vllm sharding_manager", logger=logger) + def __enter__(self): + self.timing = {} + with simple_timer("reshard", self.timing): + aggressive_empty_cache(force_sync=True) + + log_gpu_memory_usage("Before state_dict() in sharding manager memory", logger=logger) + if self.offload_param: + load_megatron_model_to_gpu(self.actor_module, load_grad=False) + + set_expandable_segments(False) + + if self.rollout_config.free_cache_engine: + if "tags" in inspect.signature(self.inference_engine.wake_up).parameters: + self.inference_engine.wake_up(tags=["weights"]) + else: + self.inference_engine.wake_up() + if self.bridge is not None: + per_tensor_param = self.bridge.export_weights(self.actor_module) + else: + per_tensor_param = per_tensor_generator( + self.actor_module, + self.model_config, + self.weight_converter, + self.transformer_config, + self.layer_name_mapping, + ) + model = self.model_runner.model + from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader + + patch_vllm_moe_model_weight_loader(model) + loaded_params = model.load_weights(per_tensor_param) + info = f"vLLM load weights, loaded_params: {len(loaded_params)}" + logger.info(info) + + if self.offload_param: + offload_megatron_model_to_cpu(self.actor_module) + aggressive_empty_cache(force_sync=True) + + if ( + self.rollout_config.free_cache_engine + and "tags" in inspect.signature(self.inference_engine.wake_up).parameters + ): + self.inference_engine.wake_up(tags=["kv_cache"]) + + # important: need to manually set the random states of each tp to be identical. + if self.device_mesh is not None: + self.torch_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.gen_random_states) + + @GPUMemoryLogger(role="megatron vllm sharding_manager", logger=logger) + def __exit__(self, exc_type, exc_value, traceback): + if self.rollout_config.free_cache_engine: + self.inference_engine.sleep(level=VLLM_SLEEP_LEVEL) + for model in self.actor_module: + model.train() + + aggressive_empty_cache(force_sync=True) + + set_expandable_segments(True) + + # restore random states + if self.device_mesh is not None: + self.gen_random_states = get_torch_device().get_rng_state() + get_torch_device().set_rng_state(self.torch_random_states) + + @GPUMemoryLogger(role="megatron vllm sharding_manager", logger=logger) + def preprocess_data(self, data: DataProto) -> DataProto: + # DP_COMPUTE_PROTO: all training ranks are dp, the same as fsdp + if self.infer_tp_size == 1: + return data + + # TODO: Current impl doesn't consider FSDP with torch micro-dp + group = vllm_ps.get_tensor_model_parallel_group().device_group + + all_gather_data_proto(data=data, process_group=group) + return data + + @GPUMemoryLogger(role="megatron vllm sharding_manager", logger=logger) + def postprocess_data(self, data: DataProto) -> DataProto: + # DP_COMPUTE_PROTO: all training ranks are dp, the same as fsdp + if self.infer_tp_size == 1: + return data + return data.chunk(chunks=self.infer_tp_size)[self.infer_tp_rank]